ablation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Ablation Experiment"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from copy import deepcopy\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "\n",
    "from lib.problems import ProblemDataset, Generator, eval_cop, sub_sample\n",
    "from lib.utils import CCPInstance\n",
    "from baselines.utils import eval_method\n",
    "from baselines.CPP import methods_registry\n",
    "from baselines.CPP.methods_registry import cap_kmeans\n",
    "from lib.ltr.utils import load_model\n",
    "from lib.ltr.ccp.method import CKMeans\n",
    "from lib.ltr.ccp.method_no_alt_assign import NCCNA"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Shanghai"
   ],
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "DSET = \"shanghai_telecom\" #\n",
    "CAP = 1.1\n",
    "DATA_PTH = f\"data/CCP/benchmark/{DSET}/data_norm_cap{CAP}_.dat\"\n",
    "COLS = [\"num_users\", \"x_coord\", \"y_coord\", \"normalized_workload\"]\n",
    "CKPT = \"outputs/final/ccp_200/gnn_pool_pointwise/2023-01-09_11-19-49_057495/checkpoints/epoch=198_val_acc=0.9858.ckpt\"\n",
    "\n",
    "SAMPLING_METHOD = \"quadrant_rnd\"\n",
    "TRGT_SIZE = 50\n",
    "SIZE = 250\n",
    "\n",
    "SEED = 999\n",
    "N = 200\n",
    "WF = (1.5, 4.0)\n",
    "FULL_K = 40\n",
    "MAX_W = 0.2\n",
    "INF = float(\"inf\")\n",
    "\n",
    "save_pth = f\"./data/CCP/ablation/{DSET}_{N}\"\n",
    "fname = f\"abl_n{N}_s{TRGT_SIZE}_cap{str(CAP).replace('.', '_')}_seed{SEED}\"\n",
    "\n",
    "os.makedirs(save_pth, exist_ok=True)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# load original data\n",
    "data = torch.load(DATA_PTH)\n",
    "print(len(data))\n",
    "xyw = data[:, 1:]\n",
    "rng = np.random.RandomState(SEED)\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "samples = sub_sample(\n",
    "    deepcopy(xyw),\n",
    "    size=SIZE,\n",
    "    n=N,\n",
    "    rng=rng,\n",
    "    method=SAMPLING_METHOD,\n",
    "    weight_factor=WF,\n",
    "    max_weight=MAX_W\n",
    ")\n",
    "# convert to CCPInstances\n",
    "instances = [\n",
    "    CCPInstance(\n",
    "        coords=s[:, :2],\n",
    "        demands=s[:, -1],\n",
    "        graph_size=N,\n",
    "        constraint_value=1.0,\n",
    "        )\n",
    "    for s in samples\n",
    "]\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "K_NOT_KNOWN = False\n",
    "CUDA = False\n",
    "SEED = 42\n",
    "NSEEDS = 1\n",
    "NUM_INIT = 8\n",
    "ITERS = 50\n",
    "TOL = 0.0   # no early stopping\n",
    "INIT_METHODS = [\n",
    "    \"topk\", # originally proposed by Geetha et al. 2009\n",
    "    \"k-means++\", # Arthur and Vassilvitskii, 2006\n",
    "    \"ckm++\", # ours\n",
    "]\n",
    "SAVE_DIR = os.path.join(save_pth, fname)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "metrics = {}\n",
    "RESULTS = {}\n",
    "seeds = [SEED]\n",
    "ds = ProblemDataset(problem=\"ccp\", seed=SEED)\n",
    "ds.size = len(instances)\n",
    "ds.data = instances\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "## reload\n",
    "\n",
    "def to_instance(d: dict):\n",
    "    d.pop(\"problem\")\n",
    "    d.pop(\"size\")\n",
    "    gs = int(d['graph_size'])\n",
    "    d['graph_size'] = gs\n",
    "    d['constraint_value'] = float(d['constraint_value'])\n",
    "    d['num_components'] = int(d['num_components'])\n",
    "    d['coords'] = d['coords'].reshape(gs, 2)\n",
    "    d['demands'] = d['demands'].reshape(gs)\n",
    "    return CCPInstance(**d)\n",
    "\n",
    "all_res = torch.load(\"data/CCP/ablation/shanghai_telecom_200/abl_n200_s50_cap1_1_seed999/ncc_greedy/eval_results_full_1.pkl\")\n",
    "data = [to_instance(e['instance']) for e in all_res]\n",
    "ds.size = len(data)\n",
    "ds.data = data\n",
    "ds.data[0]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "mthd = \"random_select\"\n",
    "result, smry = eval_method(\n",
    "    method=getattr(methods_registry, mthd),\n",
    "    dataset=ds,\n",
    "    seeds=seeds,\n",
    "    save_dir=SAVE_DIR,\n",
    "    cuda=False,\n",
    "    k_not_known=True,\n",
    "    method_str=mthd,\n",
    "    verbose=False,\n",
    ")\n",
    "rnd_res = deepcopy(result)\n",
    "\n",
    "# retrieve the respective k values and set them for the instances\n",
    "# get the minimum k value found by the random method for all seeds\n",
    "k_vals = np.array([r['nc'] for r in rnd_res]).reshape(NSEEDS, -1).min(axis=0)\n",
    "data = ds.data.copy()\n",
    "assert len(k_vals) == len(data), f\"{len(k_vals)} != {len(data)}\"\n",
    "for i in range(len(data)):\n",
    "    data[i] = data[i].update(num_components=int(k_vals[i]))\n",
    "ds.data = data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "init_methods = INIT_METHODS + INIT_METHODS[1:]\n",
    "num_inits = [1] * 3 + [NUM_INIT] * 2\n",
    "\n",
    "mthd = \"cap_kmeans\"\n",
    "infs = {}\n",
    "\n",
    "for init_mthd, num_init in zip(init_methods, num_inits):\n",
    "    m_id = f\"{mthd}_{init_mthd}-{num_init}{'_cuda' if CUDA else ''}\"\n",
    "    print(f\"running for {m_id}...\")\n",
    "\n",
    "    result, smry = eval_method(\n",
    "        method=getattr(methods_registry, mthd),\n",
    "        dataset=ds,\n",
    "        seeds=seeds,\n",
    "        save_dir=SAVE_DIR,\n",
    "        cuda=CUDA,\n",
    "        k_not_known=K_NOT_KNOWN,\n",
    "        verbose=False,\n",
    "        num_init=num_init,\n",
    "        max_iter=ITERS,\n",
    "        tol=TOL,\n",
    "        init_method=init_mthd,\n",
    "    )\n",
    "\n",
    "    RESULTS[m_id] = result\n",
    "    # check if there are any infeasible instances and replace cost with base cost\n",
    "    res = deepcopy(result)\n",
    "    costs = np.array([r['tot_center_dist'] for r in res])\n",
    "    costs_ = costs.reshape(NSEEDS, -1)\n",
    "    inf_idx = []\n",
    "    for i in range(len(ds)):\n",
    "        inst_cost = costs_[:, i]\n",
    "        inf_msk = inst_cost == INF\n",
    "        if np.any(inf_msk):\n",
    "            inf_idx.append(i)\n",
    "\n",
    "    infs[m_id] = {\"cost\": costs, \"summary\": smry, \"inf_idx\": inf_idx}"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# find TRGT_SIZE instances which were feasibly solved by all init methods of cap kmeans\n",
    "unq_inf_idx = set()\n",
    "for v in infs.values():\n",
    "    for e in v[\"inf_idx\"]:\n",
    "        unq_inf_idx.add(e)\n",
    "print(len(unq_inf_idx))\n",
    "valid_idx = [i for i in range(SIZE) if i not in unq_inf_idx]\n",
    "print(valid_idx)\n",
    "assert len(valid_idx) >= TRGT_SIZE\n",
    "valid_idx = valid_idx[:TRGT_SIZE]\n",
    "print(len(valid_idx), valid_idx)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for k, v in infs.items():\n",
    "    # read out results only for valid idx\n",
    "    smry = v['summary']\n",
    "    costs = v['cost']\n",
    "    #print(costs)\n",
    "    costs = np.array([c for i, c in enumerate(costs) if i in valid_idx])\n",
    "    #print(costs)\n",
    "    costs = costs.reshape(NSEEDS, -1)\n",
    "    smry['center_dist_mean'] = np.mean(costs)\n",
    "    smry['center_dist_std'] = np.mean(np.std(costs, axis=0))\n",
    "    rt = [e['run_time'] for e in RESULTS[k]]\n",
    "    rt = [c for i, c in enumerate(rt) if i in valid_idx]\n",
    "    smry['run_time_mean'] = np.mean(rt)\n",
    "    m_id = f\"{mthd}_{k}{'_cuda' if CUDA else ''}\"\n",
    "    print(f\"{m_id} summary: {smry}\")\n",
    "    #metrics[m_id] = smry\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# reduce dataset for all other methods to valid_idx selection\n",
    "ds.size = len(valid_idx)\n",
    "ds.data = [c for i, c in enumerate(ds.data) if i in valid_idx]\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# this setup corresponds to the cap_kmeans method\n",
    "# with an alternative order of assignment,\n",
    "# i.e. instead of sequentially assigning the node with the\n",
    "# highest priority to the closest center,\n",
    "# we cycle through the centers and sequentially add the highest\n",
    "# priority node for that specific center to its cluster\n",
    "# (vanilla_priority=True -> it does not use the NN model at all!)\n",
    "\n",
    "mthd = \"cap_kmeans_alt\"\n",
    "model = load_model(\"ccp\", CKPT)\n",
    "\n",
    "for init_mthd in INIT_METHODS:\n",
    "    m_id = f\"{mthd}_{init_mthd}{'_cuda' if CUDA else ''}\"\n",
    "\n",
    "    ckmeans = CKMeans(\n",
    "        model=model,\n",
    "        seed=SEED,\n",
    "        verbose=False,\n",
    "        ###\n",
    "        num_init=1 if init_mthd == \"topk\" else NUM_INIT,\n",
    "        max_iter=ITERS,\n",
    "        tol=TOL,\n",
    "        init_method=init_mthd,\n",
    "        ###\n",
    "        vanilla_priority=True,\n",
    "        opt_last_frac=0.0,\n",
    "        opt_last_samples=0,\n",
    "        opt_last_prio=False,\n",
    "    )\n",
    "\n",
    "    result, smry = eval_method(\n",
    "        method=ckmeans.inference,\n",
    "        dataset=ds,\n",
    "        seeds=seeds,\n",
    "        save_dir=SAVE_DIR,\n",
    "        cuda=CUDA,\n",
    "        k_not_known=K_NOT_KNOWN,\n",
    "        method_str=mthd,\n",
    "    )\n",
    "    RESULTS[m_id] = result\n",
    "    # check if there are any infeasible instances and replace cost with base cost\n",
    "    res = deepcopy(result)\n",
    "    costs = np.array([r['tot_center_dist'] for r in res])\n",
    "    costs = costs.reshape(NSEEDS, -1)\n",
    "    smry['center_dist_mean'] = np.mean(costs)\n",
    "    smry['center_dist_std'] = np.mean(np.std(costs, axis=0))\n",
    "    print(f\"{m_id} summary: {smry}\")\n",
    "    metrics[m_id] = smry"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# corresponds to Cap-Kmeans using the NN model (neural scoring function)\n",
    "# for priority estimation but without alternative assignment\n",
    "\n",
    "mthd = \"cap_kmeans_nsf\"\n",
    "model = load_model(\"ccp\", CKPT)\n",
    "\n",
    "for init_mthd in INIT_METHODS:\n",
    "    m_id = f\"{mthd}_{init_mthd}{'_cuda' if CUDA else ''}\"\n",
    "\n",
    "    ckmeans = NCCNA(\n",
    "        model=model,\n",
    "        seed=SEED,\n",
    "        verbose=False,\n",
    "        ###\n",
    "        num_init=1 if init_mthd == \"topk\" else NUM_INIT,\n",
    "        max_iter=ITERS,\n",
    "        tol=TOL,\n",
    "        init_method=init_mthd,\n",
    "        ###\n",
    "        vanilla_priority=False,\n",
    "        opt_last_frac=0.0,\n",
    "        opt_last_samples=0,\n",
    "        opt_last_prio=False,\n",
    "    )\n",
    "\n",
    "    result, smry = eval_method(\n",
    "        method=ckmeans.inference,\n",
    "        dataset=ds,\n",
    "        seeds=seeds,\n",
    "        save_dir=SAVE_DIR,\n",
    "        cuda=CUDA,\n",
    "        k_not_known=K_NOT_KNOWN,\n",
    "        method_str=mthd,\n",
    "    )\n",
    "    RESULTS[m_id] = result\n",
    "    # check if there are any infeasible instances and replace cost with base cost\n",
    "    res = deepcopy(result)\n",
    "    costs = np.array([r['tot_center_dist'] for r in res])\n",
    "    costs = costs.reshape(NSEEDS, -1)\n",
    "    smry['center_dist_mean'] = np.mean(costs)\n",
    "    smry['center_dist_std'] = np.mean(np.std(costs, axis=0))\n",
    "    print(f\"{m_id} summary: {smry}\")\n",
    "    metrics[m_id] = smry"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# greedily assigns the last 'opt_last_frac' fraction of total nodes\n",
    "# ordered by their absolute priority to the closest center\n",
    "\n",
    "mthd = \"ncc_greedy\"\n",
    "model = load_model(\"ccp\", CKPT)\n",
    "\n",
    "for init_mthd in INIT_METHODS:\n",
    "    m_id = f\"{mthd}_{init_mthd}{'_cuda' if CUDA else ''}\"\n",
    "\n",
    "    ckmeans = CKMeans(\n",
    "        model=model,\n",
    "        seed=SEED,\n",
    "        verbose=False,\n",
    "        ###\n",
    "        num_init=1 if init_mthd == \"topk\" else NUM_INIT,\n",
    "        max_iter=ITERS,\n",
    "        tol=TOL,\n",
    "        init_method=init_mthd,\n",
    "        ###\n",
    "        vanilla_priority=False,\n",
    "        opt_last_frac=0.7,\n",
    "        opt_last_samples=1, # no multiple samples\n",
    "        opt_last_prio=True\n",
    "    )\n",
    "\n",
    "    result, smry = eval_method(\n",
    "        method=ckmeans.inference,\n",
    "        dataset=ds,\n",
    "        seeds=seeds,\n",
    "        save_dir=SAVE_DIR,\n",
    "        cuda=CUDA,\n",
    "        k_not_known=K_NOT_KNOWN,\n",
    "        method_str=mthd,\n",
    "    )\n",
    "    RESULTS[m_id] = result\n",
    "    # check if there are any infeasible instances and replace cost with base cost\n",
    "    res = deepcopy(result)\n",
    "    costs = np.array([r['tot_center_dist'] for r in res])\n",
    "    costs = costs.reshape(NSEEDS, -1)\n",
    "    smry['center_dist_mean'] = np.mean(costs)\n",
    "    smry['center_dist_std'] = np.mean(np.std(costs, axis=0))\n",
    "    print(f\"{m_id} summary: {smry}\")\n",
    "    metrics[m_id] = smry"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# samples multiple assignments for the last 'opt_last_frac' fraction of total nodes\n",
    "# and selects the best one\n",
    "\n",
    "mthd = \"ncc_samp\"\n",
    "model = load_model(\"ccp\", CKPT)\n",
    "#init_mthd = \"c2km++\"\n",
    "for init_mthd in INIT_METHODS:\n",
    "    m_id = f\"{mthd}_{init_mthd}{'_cuda' if CUDA else ''}\"\n",
    "\n",
    "    ckmeans = CKMeans(\n",
    "        model=model,\n",
    "        seed=SEED,\n",
    "        verbose=False,\n",
    "        ###\n",
    "        num_init=1 if init_mthd == \"topk\" else NUM_INIT,\n",
    "        max_iter=ITERS,\n",
    "        tol=TOL,\n",
    "        init_method=init_mthd,\n",
    "        ###\n",
    "        vanilla_priority=False,\n",
    "        opt_last_frac=0.7,\n",
    "        opt_last_samples=64,\n",
    "        opt_last_prio=True\n",
    "    )\n",
    "\n",
    "    result, smry = eval_method(\n",
    "        method=ckmeans.inference,\n",
    "        dataset=ds,\n",
    "        seeds=seeds,\n",
    "        save_dir=SAVE_DIR,\n",
    "        cuda=CUDA,\n",
    "        k_not_known=K_NOT_KNOWN,\n",
    "        method_str=mthd,\n",
    "    )\n",
    "    RESULTS[m_id] = result\n",
    "    # check if there are any infeasible instances and replace cost with base cost\n",
    "    res = deepcopy(result)\n",
    "    costs = np.array([r['tot_center_dist'] for r in res])\n",
    "    costs = costs.reshape(NSEEDS, -1)\n",
    "    smry['center_dist_mean'] = np.mean(costs)\n",
    "    smry['center_dist_std'] = np.mean(np.std(costs, axis=0))\n",
    "    print(f\"{m_id} summary: {smry}\")\n",
    "    metrics[m_id] = smry"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# convert to dataframe for nice table ;)\n",
    "metric_df = pd.DataFrame(metrics)\n",
    "metric_df\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# !!!\n",
    "# opt_last_frac=0.99, opt_last_samples=16\n",
    "# cap_kmeans_nsf_topk summary: {'center_dist_mean': 0.5042707053562993, 'center_dist_std': 0.0, 'nc_mean': 8.66, 'nc_std': 2.5108564275959706, 'nc_median': 9.0, 'run_time_mean': 1.0674996393000766, 'run_time_total': 53.374981965003826, 'num_infeasible': 0}\n",
    "# cap_kmeans_nsf_k-means++ summary: {'center_dist_mean': 0.46759631351147113, 'center_dist_std': 0.0, 'nc_mean': 8.66, 'nc_std': 2.5108564275959706, 'nc_median': 9.0, 'run_time_mean': 2.732793123839656, 'run_time_total': 136.6396561919828, 'num_infeasible': 0}\n",
    "# cap_kmeans_nsf_ckm++ summary: {'center_dist_mean': 0.46429420882987116, 'center_dist_std': 0.0, 'nc_mean': 8.66, 'nc_std': 2.5108564275959706, 'nc_median': 9.0, 'run_time_mean': 2.7289918054996813, 'run_time_total': 136.44959027498408, 'num_infeasible': 0}\n",
    "\n",
    "# opt_last_frac=0.90, opt_last_samples=16\n",
    "# cap_kmeans_nsf_ckm++ summary: {'center_dist_mean': 0.44793172402515713, 'center_dist_std': 0.0, 'nc_mean': 8.66, 'nc_std': 2.5108564275959706, 'nc_median': 9.0, 'run_time_mean': 2.68136288738031, 'run_time_total': 134.0681443690155, 'num_infeasible': 0}\n",
    "# opt_last_frac=0.80, opt_last_samples=16\n",
    "# cap_kmeans_nsf_ckm++ summary: {'center_dist_mean': 0.44748048922138345, 'center_dist_std': 0.0, 'nc_mean': 8.66, 'nc_std': 2.5108564275959706, 'nc_median': 9.0, 'run_time_mean': 2.618354005799847, 'run_time_total': 130.91770028999235, 'num_infeasible': 0}\n",
    "# opt_last_frac=0.75, opt_last_samples=16\n",
    "# cap_kmeans_nsf_ckm++ summary: {'center_dist_mean': 0.4469523817082363, 'center_dist_std': 0.0, 'nc_mean': 8.66, 'nc_std': 2.5108564275959706, 'nc_median': 9.0, 'run_time_mean': 2.6313813093400675, 'run_time_total': 131.56906546700338, 'num_infeasible': 0}\n",
    "# opt_last_frac=0.70, opt_last_samples=16\n",
    "# cap_kmeans_nsf_ckm++ summary: {'center_dist_mean': 0.44451487442761617, 'center_dist_std': 0.0, 'nc_mean': 8.66, 'nc_std': 2.5108564275959706, 'nc_median': 9.0, 'run_time_mean': 2.6363163709201762, 'run_time_total': 131.8158185460088, 'num_infeasible': 0}\n",
    "# opt_last_frac=0.60, opt_last_samples=16\n",
    "# cap_kmeans_nsf_ckm++ summary: {'center_dist_mean': 0.45103828188218453, 'center_dist_std': 0.0, 'nc_mean': 8.66, 'nc_std': 2.5108564275959706, 'nc_median': 9.0, 'run_time_mean': 2.81216765499998, 'run_time_total': 140.608382749999, 'num_infeasible': 0}\n",
    "\n",
    "# opt_last_frac=0.70, opt_last_samples=16\n",
    "# cap_kmeans_nsf_ckm++ summary: {'center_dist_mean': 0.44451487442761617, 'center_dist_std': 0.0, 'nc_mean': 8.66, 'nc_std': 2.5108564275959706, 'nc_median': 9.0, 'run_time_mean': 2.6363163709201762, 'run_time_total': 131.8158185460088, 'num_infeasible': 0}\n",
    "# opt_last_frac=0.70, opt_last_samples=32\n",
    "# cap_kmeans_nsf_ckm++ summary: {'center_dist_mean': 0.4442066230780878, 'center_dist_std': 0.0, 'nc_mean': 8.66, 'nc_std': 2.5108564275959706, 'nc_median': 9.0, 'run_time_mean': 3.4557080210199955, 'run_time_total': 172.78540105099978, 'num_infeasible': 0}\n",
    "\n",
    "\n",
    "mthd = \"cap_kmeans_nsf\"\n",
    "model = load_model(\"ccp\", CKPT)\n",
    "\n",
    "for init_mthd in INIT_METHODS[-1:]:\n",
    "    m_id = f\"{mthd}_{init_mthd}{'_cuda' if CUDA else ''}\"\n",
    "\n",
    "    ckmeans = CKMeans(\n",
    "        model=model,\n",
    "        seed=SEED,\n",
    "        verbose=False,\n",
    "        ###\n",
    "        num_init=1 if init_mthd == \"topk\" else NUM_INIT,\n",
    "        max_iter=ITERS,\n",
    "        tol=TOL,\n",
    "        init_method=init_mthd,\n",
    "        ###\n",
    "        vanilla_priority=False,\n",
    "        opt_last_frac=0.7,\n",
    "        opt_last_samples=32,\n",
    "        opt_last_prio=True,\n",
    "    )\n",
    "\n",
    "    result, smry = eval_method(\n",
    "        method=ckmeans.inference,\n",
    "        dataset=ds,\n",
    "        seeds=seeds,\n",
    "        save_dir=SAVE_DIR,\n",
    "        cuda=CUDA,\n",
    "        k_not_known=K_NOT_KNOWN,\n",
    "        method_str=mthd,\n",
    "    )\n",
    "    RESULTS[m_id] = result\n",
    "    # check if there are any infeasible instances and replace cost with base cost\n",
    "    res = deepcopy(result)\n",
    "    costs = np.array([r['tot_center_dist'] for r in res])\n",
    "    costs = costs.reshape(NSEEDS, -1)\n",
    "    smry['center_dist_mean'] = np.mean(costs)\n",
    "    smry['center_dist_std'] = np.mean(np.std(costs, axis=0))\n",
    "    print(f\"{m_id} summary: {smry}\")\n",
    "    metrics[m_id] = smry"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "mthd = \"ncc_v2_greedy\"\n",
    "model = load_model(\"ccp\", CKPT)\n",
    "\n",
    "for init_mthd in INIT_METHODS[-1:]:\n",
    "    m_id = f\"{mthd}_{init_mthd}{'_cuda' if CUDA else ''}\"\n",
    "\n",
    "    ckmeans = NCC(\n",
    "        model=model,\n",
    "        seed=SEED,\n",
    "        verbose=False,\n",
    "        ###\n",
    "        num_init=1 if init_mthd == \"topk\" else NUM_INIT,\n",
    "        max_iter=ITERS,\n",
    "        tol=TOL,\n",
    "        init_method=init_mthd,\n",
    "        ###\n",
    "        vanilla_priority=False,\n",
    "        num_samples=1,\n",
    "    )\n",
    "\n",
    "    result, smry = eval_method(\n",
    "        method=ckmeans.inference,\n",
    "        dataset=ds,\n",
    "        seeds=seeds,\n",
    "        save_dir=SAVE_DIR,\n",
    "        cuda=CUDA,\n",
    "        k_not_known=K_NOT_KNOWN,\n",
    "        method_str=mthd,\n",
    "    )\n",
    "    RESULTS[m_id] = result\n",
    "    # check if there are any infeasible instances and replace cost with base cost\n",
    "    res = deepcopy(result)\n",
    "    costs = np.array([r['tot_center_dist'] for r in res])\n",
    "    costs = costs.reshape(NSEEDS, -1)\n",
    "    smry['center_dist_mean'] = np.mean(costs)\n",
    "    smry['center_dist_std'] = np.mean(np.std(costs, axis=0))\n",
    "    print(f\"{m_id} summary: {smry}\")\n",
    "    metrics[m_id] = smry\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "mthd = \"ncc_v2_samp\"\n",
    "model = load_model(\"ccp\", CKPT)\n",
    "\n",
    "for init_mthd in INIT_METHODS[-1:]:\n",
    "    m_id = f\"{mthd}_{init_mthd}{'_cuda' if CUDA else ''}\"\n",
    "\n",
    "    ckmeans = NCC(\n",
    "        model=model,\n",
    "        seed=SEED,\n",
    "        verbose=False,\n",
    "        ###\n",
    "        num_init=1 if init_mthd == \"topk\" else NUM_INIT,\n",
    "        max_iter=ITERS,\n",
    "        tol=TOL,\n",
    "        init_method=init_mthd,\n",
    "        ###\n",
    "        vanilla_priority=False,\n",
    "        num_samples=64,\n",
    "    )\n",
    "\n",
    "    result, smry = eval_method(\n",
    "        method=ckmeans.inference,\n",
    "        dataset=ds,\n",
    "        seeds=seeds,\n",
    "        save_dir=SAVE_DIR,\n",
    "        cuda=CUDA,\n",
    "        k_not_known=K_NOT_KNOWN,\n",
    "        method_str=mthd,\n",
    "    )\n",
    "    RESULTS[m_id] = result\n",
    "    # check if there are any infeasible instances and replace cost with base cost\n",
    "    res = deepcopy(result)\n",
    "    costs = np.array([r['tot_center_dist'] for r in res])\n",
    "    costs = costs.reshape(NSEEDS, -1)\n",
    "    smry['center_dist_mean'] = np.mean(costs)\n",
    "    smry['center_dist_std'] = np.mean(np.std(costs, axis=0))\n",
    "    print(f\"{m_id} summary: {smry}\")\n",
    "    metrics[m_id] = smry"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}