From 6e03a2746df350010555b83829c3b78cf28471c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Aum=C3=BCller?= Date: Tue, 18 Jul 2023 10:31:46 +0200 Subject: [PATCH] archival of neurips21 work, updated documentation --- README.md | 68 ++---------------- neurips21/README.md | 9 +++ {t1_t2 => neurips21/t1_t2}/README.md | 0 .../t1_t2}/results/T1/bigann-1B.png | Bin .../t1_t2}/results/T1/deep-1B.png | Bin .../t1_t2}/results/T1/msspacev-1B.png | Bin .../t1_t2}/results/T1/msturing-1B.png | Bin .../t1_t2}/results/T1/neurips21/bigann-1B.png | Bin .../t1_t2}/results/T1/neurips21/deep-1B.png | Bin .../results/T1/neurips21/msspacev-1B.png | Bin .../results/T1/neurips21/msturing-1B.png | Bin .../t1_t2}/results/T1/neurips21/ssnpp-1B.png | Bin .../t1_t2}/results/T1/neurips21/t1.csv | 0 .../results/T1/neurips21/text2image-1B.png | Bin .../t1_t2}/results/T1/ssnpp-1B.png | Bin .../t1_t2}/results/T1/text2image-1B.png | Bin .../t1_t2}/results/T2/bigann-1B-IO.png | Bin .../t1_t2}/results/T2/bigann-1B.png | Bin .../t1_t2}/results/T2/deep-1B-IO.png | Bin .../t1_t2}/results/T2/deep-1B.png | Bin .../t1_t2}/results/T2/msspacev-1B-IO.png | Bin .../t1_t2}/results/T2/msspacev-1B.png | Bin .../t1_t2}/results/T2/msturing-1B-IO.png | Bin .../t1_t2}/results/T2/msturing-1B.png | Bin .../t1_t2}/results/T2/neurips21/bigann-1B.png | Bin .../t1_t2}/results/T2/neurips21/deep-1B.png | Bin .../results/T2/neurips21/msspacev-1B.png | Bin .../results/T2/neurips21/msturing-1B.png | Bin .../t1_t2}/results/T2/neurips21/ssnpp-1B.png | Bin .../t1_t2}/results/T2/neurips21/t2.csv | 0 .../results/T2/neurips21/text2image-1B.png | Bin .../t1_t2}/results/T2/ssnpp-1B-IO.png | Bin .../t1_t2}/results/T2/ssnpp-1B.png | Bin .../t1_t2}/results/T2/text2image-1B-IO.png | Bin .../t1_t2}/results/T2/text2image-1B.png | Bin .../t3}/LB_history/Dec.2.2021/LEADERBOARDS.md | 0 .../LEADERBOARDS_REJECT_ANOMALIES.md | 0 .../LB_history/Nov.29.2021/LEADERBOARDS.md | 0 .../Nov.29.2021/TASKS_ISSUES_RESOLUTIONS.md | 0 {t3 => neurips21/t3}/LEADERBOARDS.md | 0 {t3 => neurips21/t3}/LEADERBOARDS_PRIVATE.md | 0 .../LEADERBOARDS_PRIVATE_REJECT_ANOMALIES.md | 0 {t3 => neurips21/t3}/LEADERBOARDS_PUBLIC.md | 0 .../LEADERBOARDS_PUBLIC_REJECT_ANOMALIES.md | 0 .../t3}/LEADERBOARDS_REJECT_ANOMALIES.md | 0 {t3 => neurips21/t3}/RANKING.md | 0 {t3 => neurips21/t3}/README.md | 0 .../t3}/TASKS_ISSUES_RESOLUTIONS.md | 0 {t3 => neurips21/t3}/cuanns_ivfpq/Dockerfile | 0 {t3 => neurips21/t3}/cuanns_ivfpq/README.md | 0 {t3 => neurips21/t3}/cuanns_ivfpq/algos.yaml | 0 .../t3}/cuanns_multigpu/Dockerfile | 0 .../t3}/cuanns_multigpu/README.md | 0 .../t3}/cuanns_multigpu/algos.yaml | 0 .../t3}/eval_2021/faiss_t3/prun.sh | 0 {t3 => neurips21/t3}/faiss_t3/Dockerfile | 0 {t3 => neurips21/t3}/faiss_t3/README.md | 0 {t3 => neurips21/t3}/faiss_t3/algos.yaml | 0 .../baseline_plots/bigann-1B-r-vs-p.png | Bin .../baseline_plots/bigann-1B-r-vs-t.png | Bin .../baseline_plots/deep-1B-r-vs-p.png | Bin .../baseline_plots/deep-1B-r-vs-t.png | Bin .../baseline_plots/msspacev-1B-r-vs-p.png | Bin .../baseline_plots/msspacev-1B-r-vs-t.png | Bin .../baseline_plots/msturing-1B-r-vs-p.png | Bin .../baseline_plots/msturing-1B-r-vs-t.png | Bin .../baseline_plots/text2image-1B-r-vs-p.png | Bin .../baseline_plots/text2image-1B-r-vs-t.png | Bin .../t3}/faiss_t3/cost/AdvantechSky6200.pdf | Bin {t3 => neurips21/t3}/faiss_t3/cost/GPU.pdf | Bin {t3 => neurips21/t3}/faiss_t3/cost/RAM.pdf | Bin {t3 => neurips21/t3}/faiss_t3/cost/SSD.pdf | Bin .../t3}/faiss_t3/faiss-gpu_requirements.txt | 0 {t3 => neurips21/t3}/gemini/.gitignore | 0 {t3 => neurips21/t3}/gemini/README.md | 0 {t3 => neurips21/t3}/gemini/algos.yaml | 0 .../t3}/gemini/buildidx/build_index.py | 0 {t3 => neurips21/t3}/gemini/buildidx/htest.py | 0 .../gemini/buildidx/run_bin_build_index.sh | 0 {t3 => neurips21/t3}/gemini/buildidx/test.py | 0 .../t3}/gemini/cost/AdvantechSky6200.pdf | Bin {t3 => neurips21/t3}/gemini/cost/GPU.pdf | Bin {t3 => neurips21/t3}/gemini/cost/RAM.pdf | Bin {t3 => neurips21/t3}/gemini/cost/SSD.pdf | Bin {t3 => neurips21/t3}/gemini/requirements.txt | 0 {t3 => neurips21/t3}/gemini/run_bin_python.sh | 0 .../t3}/gemini/run_conda_python.sh | 0 .../track1_baseline_faiss}/README.md | 0 .../track1_baseline_faiss}/__init__.py | 0 .../track1_baseline_faiss}/baseline_faiss.py | 0 .../baseline_faiss_filtered.py | 0 .../bow_id_selector.swig | 0 .../track1_baseline_faiss}/parse_results.py | 0 .../plots/bigann-1B.png | Bin .../track1_baseline_faiss}/plots/deep-1B.png | Bin .../plots/msspacev-1B.png | Bin .../plots/msturing-1B.png | Bin .../track1_baseline_faiss}/plots/ssnpp-1B.png | Bin .../plots/text2image-1B.png | Bin .../track1_baseline_faiss}/run_baselines.bash | 0 .../test_bow_id_selector.py | 0 .../track3_baseline_faiss}/README.md | 0 .../gpu_baseline_faiss.py | 0 .../plots/T3_deep-1B.png | Bin .../notebooks}/check_1B_groundtruth.ipynb | 0 .../compare_track1_1B_vs_2x500M.ipynb | 0 .../notebooks}/eval_t2i_results.ipynb | 0 .../notebooks}/find_suitable_nq.ipynb | 0 .../notebooks}/large_coarse_quantizer.ipynb | 0 .../neurips23}/parse_filtered_results.ipynb | 0 .../sparse_algorithms}/basic_sparse_index.py | 0 .../sparse_algorithms}/eval_sparse.py | 0 112 files changed, 14 insertions(+), 63 deletions(-) create mode 100644 neurips21/README.md rename {t1_t2 => neurips21/t1_t2}/README.md (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/bigann-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/deep-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/msspacev-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/msturing-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/neurips21/bigann-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/neurips21/deep-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/neurips21/msspacev-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/neurips21/msturing-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/neurips21/ssnpp-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/neurips21/t1.csv (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/neurips21/text2image-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/ssnpp-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T1/text2image-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/bigann-1B-IO.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/bigann-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/deep-1B-IO.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/deep-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/msspacev-1B-IO.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/msspacev-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/msturing-1B-IO.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/msturing-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/neurips21/bigann-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/neurips21/deep-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/neurips21/msspacev-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/neurips21/msturing-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/neurips21/ssnpp-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/neurips21/t2.csv (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/neurips21/text2image-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/ssnpp-1B-IO.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/ssnpp-1B.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/text2image-1B-IO.png (100%) rename {t1_t2 => neurips21/t1_t2}/results/T2/text2image-1B.png (100%) rename {t3 => neurips21/t3}/LB_history/Dec.2.2021/LEADERBOARDS.md (100%) rename {t3 => neurips21/t3}/LB_history/Dec.2.2021/LEADERBOARDS_REJECT_ANOMALIES.md (100%) rename {t3 => neurips21/t3}/LB_history/Nov.29.2021/LEADERBOARDS.md (100%) rename {t3 => neurips21/t3}/LB_history/Nov.29.2021/TASKS_ISSUES_RESOLUTIONS.md (100%) rename {t3 => neurips21/t3}/LEADERBOARDS.md (100%) rename {t3 => neurips21/t3}/LEADERBOARDS_PRIVATE.md (100%) rename {t3 => neurips21/t3}/LEADERBOARDS_PRIVATE_REJECT_ANOMALIES.md (100%) rename {t3 => neurips21/t3}/LEADERBOARDS_PUBLIC.md (100%) rename {t3 => neurips21/t3}/LEADERBOARDS_PUBLIC_REJECT_ANOMALIES.md (100%) rename {t3 => neurips21/t3}/LEADERBOARDS_REJECT_ANOMALIES.md (100%) rename {t3 => neurips21/t3}/RANKING.md (100%) rename {t3 => neurips21/t3}/README.md (100%) rename {t3 => neurips21/t3}/TASKS_ISSUES_RESOLUTIONS.md (100%) rename {t3 => neurips21/t3}/cuanns_ivfpq/Dockerfile (100%) rename {t3 => neurips21/t3}/cuanns_ivfpq/README.md (100%) rename {t3 => neurips21/t3}/cuanns_ivfpq/algos.yaml (100%) rename {t3 => neurips21/t3}/cuanns_multigpu/Dockerfile (100%) rename {t3 => neurips21/t3}/cuanns_multigpu/README.md (100%) rename {t3 => neurips21/t3}/cuanns_multigpu/algos.yaml (100%) rename {t3 => neurips21/t3}/eval_2021/faiss_t3/prun.sh (100%) rename {t3 => neurips21/t3}/faiss_t3/Dockerfile (100%) rename {t3 => neurips21/t3}/faiss_t3/README.md (100%) rename {t3 => neurips21/t3}/faiss_t3/algos.yaml (100%) rename {t3 => neurips21/t3}/faiss_t3/baseline_plots/bigann-1B-r-vs-p.png (100%) rename {t3 => neurips21/t3}/faiss_t3/baseline_plots/bigann-1B-r-vs-t.png (100%) rename {t3 => neurips21/t3}/faiss_t3/baseline_plots/deep-1B-r-vs-p.png (100%) rename {t3 => neurips21/t3}/faiss_t3/baseline_plots/deep-1B-r-vs-t.png (100%) rename {t3 => neurips21/t3}/faiss_t3/baseline_plots/msspacev-1B-r-vs-p.png (100%) rename {t3 => neurips21/t3}/faiss_t3/baseline_plots/msspacev-1B-r-vs-t.png (100%) rename {t3 => neurips21/t3}/faiss_t3/baseline_plots/msturing-1B-r-vs-p.png (100%) rename {t3 => neurips21/t3}/faiss_t3/baseline_plots/msturing-1B-r-vs-t.png (100%) rename {t3 => neurips21/t3}/faiss_t3/baseline_plots/text2image-1B-r-vs-p.png (100%) rename {t3 => neurips21/t3}/faiss_t3/baseline_plots/text2image-1B-r-vs-t.png (100%) rename {t3 => neurips21/t3}/faiss_t3/cost/AdvantechSky6200.pdf (100%) rename {t3 => neurips21/t3}/faiss_t3/cost/GPU.pdf (100%) rename {t3 => neurips21/t3}/faiss_t3/cost/RAM.pdf (100%) rename {t3 => neurips21/t3}/faiss_t3/cost/SSD.pdf (100%) rename {t3 => neurips21/t3}/faiss_t3/faiss-gpu_requirements.txt (100%) rename {t3 => neurips21/t3}/gemini/.gitignore (100%) rename {t3 => neurips21/t3}/gemini/README.md (100%) rename {t3 => neurips21/t3}/gemini/algos.yaml (100%) rename {t3 => neurips21/t3}/gemini/buildidx/build_index.py (100%) rename {t3 => neurips21/t3}/gemini/buildidx/htest.py (100%) rename {t3 => neurips21/t3}/gemini/buildidx/run_bin_build_index.sh (100%) rename {t3 => neurips21/t3}/gemini/buildidx/test.py (100%) rename {t3 => neurips21/t3}/gemini/cost/AdvantechSky6200.pdf (100%) rename {t3 => neurips21/t3}/gemini/cost/GPU.pdf (100%) rename {t3 => neurips21/t3}/gemini/cost/RAM.pdf (100%) rename {t3 => neurips21/t3}/gemini/cost/SSD.pdf (100%) rename {t3 => neurips21/t3}/gemini/requirements.txt (100%) rename {t3 => neurips21/t3}/gemini/run_bin_python.sh (100%) rename {t3 => neurips21/t3}/gemini/run_conda_python.sh (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/README.md (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/__init__.py (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/baseline_faiss.py (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/baseline_faiss_filtered.py (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/bow_id_selector.swig (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/parse_results.py (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/plots/bigann-1B.png (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/plots/deep-1B.png (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/plots/msspacev-1B.png (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/plots/msturing-1B.png (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/plots/ssnpp-1B.png (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/plots/text2image-1B.png (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/run_baselines.bash (100%) rename {track1_baseline_faiss => neurips21/track1_baseline_faiss}/test_bow_id_selector.py (100%) rename {track3_baseline_faiss => neurips21/track3_baseline_faiss}/README.md (100%) rename {track3_baseline_faiss => neurips21/track3_baseline_faiss}/gpu_baseline_faiss.py (100%) rename {track3_baseline_faiss => neurips21/track3_baseline_faiss}/plots/T3_deep-1B.png (100%) rename {notebooks => preparation/neurips21/notebooks}/check_1B_groundtruth.ipynb (100%) rename {notebooks => preparation/neurips21/notebooks}/compare_track1_1B_vs_2x500M.ipynb (100%) rename {notebooks => preparation/neurips21/notebooks}/eval_t2i_results.ipynb (100%) rename {notebooks => preparation/neurips21/notebooks}/find_suitable_nq.ipynb (100%) rename {notebooks => preparation/neurips21/notebooks}/large_coarse_quantizer.ipynb (100%) rename {notebooks => preparation/neurips23}/parse_filtered_results.ipynb (100%) rename {sparse_algorithms => preparation/neurips23/sparse_algorithms}/basic_sparse_index.py (100%) rename {sparse_algorithms => preparation/neurips23/sparse_algorithms}/eval_sparse.py (100%) diff --git a/README.md b/README.md index ae458911..bfd7566c 100644 --- a/README.md +++ b/README.md @@ -2,75 +2,17 @@ -## Install - -The only prerequisite is Python and Docker. - -1. Clone the repo. -2. Run `pip install -r requirements_py3.10.txt` (Use `requirements_py38.txt` if you have Python 3.8.) -3. Install docker by following instructions [here](https://docs.docker.com/engine/install/ubuntu/). -You might also want to follow the post-install steps for running docker in non-root user mode. -3. Run `python install.py` to build all the libraries inside Docker containers. - -## Storing Data - -The framework assumes that all data is stored in `data/`. -Please use a symlink if your datasets and indices are supposed to be stored somewhere else. -The location of the linked folder matters a great deal for SSD-based search performance in T2. -A local SSD such as the one found on Azure Ls-series VMs is better than remote disks, even premium ones. -See [T1/T2](t1_t2/README.md) for more details. - -## Data sets +## Datasets See for details on the different datasets. -### Dataset Preparation - -Before running experiments, datasets have to be downloaded. All preparation can be carried out by calling - -```python -python create_dataset.py --dataset [bigann-1B | deep-1B | text2image-1B | ssnpp-1B | msturing-1B | msspacev-1B] -``` - -Note that downloading the datasets can potentially take many hours. - -For local testing, there exist smaller random datasets `random-xs` and `random-range-xs`. -Furthermore, most datasets have 1M, 10M and 100M versions, run `python create_dataset -h` to get an overview. - - -## Running the benchmark - -Run `python run.py --dataset $DS --algorithm $ALGO` where `DS` is the dataset you are running on, -and `ALGO` is the name of the algorithm. (Use `python run.py --list-algorithms`) to get an overview. -`python run.py -h` provides you with further options. - -The parameters used by the implementation to build and query the index can be found in `algos-2021.yaml`. - -## Running the track 1 baseline -After running the installation, we can evaluate the baseline as follows. - -```bash - -for DS in bigann-1B deep-1B text2image-1B ssnpp-1B msturing-1B msspacev-1B; -do - python run.py --dataset $DS --algorithm faiss-t1; -done -``` - -On a 28-core Xeon E5-2690 v4 that provided 100MB/s downloads, carrying out the baseline experiments took roughly 7 days. - -To evaluate the results, run -```bash -sudo chmod -R 777 results/ -python data_export.py --output res.csv -python3.8 eval/show_operating_points.py --algorithm faiss-t1 --threshold 10000 -``` +## NeurIPS 2023 competition: Practical Vector Search -## Including your algorithm and Evaluating the Results +Please see <./neurips23/README.md> for a guide to the NeurIPS 23 competition. -See [Track T1/T2](t1_t2/README.md) for more details on evaluation for Tracks T1 and T2. +## NeurIPS 2021 competition: Billion-Scale ANN -See [Track T3](t3/README.md) for more details on evaluation for Track T3. +Please see <./neurips21/README.md> for a guide of running billion-scale benchmarks and a summary of the results. # Credits diff --git a/neurips21/README.md b/neurips21/README.md new file mode 100644 index 00000000..5ae2358f --- /dev/null +++ b/neurips21/README.md @@ -0,0 +1,9 @@ +# NeurIPS 2021 + +This summarizes the evaluation protocol and the results of the NeurIPS 2021 Challenge. + +## NeurIPS 2021: Algorithms and Result Evaluation + +See [Track T1/T2](t1_t2/README.md) for more details on evaluation for Tracks T1 and T2. + +See [Track T3](t3/README.md) for more details on evaluation for Track T3. diff --git a/t1_t2/README.md b/neurips21/t1_t2/README.md similarity index 100% rename from t1_t2/README.md rename to neurips21/t1_t2/README.md diff --git a/t1_t2/results/T1/bigann-1B.png b/neurips21/t1_t2/results/T1/bigann-1B.png similarity index 100% rename from t1_t2/results/T1/bigann-1B.png rename to neurips21/t1_t2/results/T1/bigann-1B.png diff --git a/t1_t2/results/T1/deep-1B.png b/neurips21/t1_t2/results/T1/deep-1B.png similarity index 100% rename from t1_t2/results/T1/deep-1B.png rename to neurips21/t1_t2/results/T1/deep-1B.png diff --git a/t1_t2/results/T1/msspacev-1B.png b/neurips21/t1_t2/results/T1/msspacev-1B.png similarity index 100% rename from t1_t2/results/T1/msspacev-1B.png rename to neurips21/t1_t2/results/T1/msspacev-1B.png diff --git a/t1_t2/results/T1/msturing-1B.png b/neurips21/t1_t2/results/T1/msturing-1B.png similarity index 100% rename from t1_t2/results/T1/msturing-1B.png rename to neurips21/t1_t2/results/T1/msturing-1B.png diff --git a/t1_t2/results/T1/neurips21/bigann-1B.png b/neurips21/t1_t2/results/T1/neurips21/bigann-1B.png similarity index 100% rename from t1_t2/results/T1/neurips21/bigann-1B.png rename to neurips21/t1_t2/results/T1/neurips21/bigann-1B.png diff --git a/t1_t2/results/T1/neurips21/deep-1B.png b/neurips21/t1_t2/results/T1/neurips21/deep-1B.png similarity index 100% rename from t1_t2/results/T1/neurips21/deep-1B.png rename to neurips21/t1_t2/results/T1/neurips21/deep-1B.png diff --git a/t1_t2/results/T1/neurips21/msspacev-1B.png b/neurips21/t1_t2/results/T1/neurips21/msspacev-1B.png similarity index 100% rename from t1_t2/results/T1/neurips21/msspacev-1B.png rename to neurips21/t1_t2/results/T1/neurips21/msspacev-1B.png diff --git a/t1_t2/results/T1/neurips21/msturing-1B.png b/neurips21/t1_t2/results/T1/neurips21/msturing-1B.png similarity index 100% rename from t1_t2/results/T1/neurips21/msturing-1B.png rename to neurips21/t1_t2/results/T1/neurips21/msturing-1B.png diff --git a/t1_t2/results/T1/neurips21/ssnpp-1B.png b/neurips21/t1_t2/results/T1/neurips21/ssnpp-1B.png similarity index 100% rename from t1_t2/results/T1/neurips21/ssnpp-1B.png rename to neurips21/t1_t2/results/T1/neurips21/ssnpp-1B.png diff --git a/t1_t2/results/T1/neurips21/t1.csv b/neurips21/t1_t2/results/T1/neurips21/t1.csv similarity index 100% rename from t1_t2/results/T1/neurips21/t1.csv rename to neurips21/t1_t2/results/T1/neurips21/t1.csv diff --git a/t1_t2/results/T1/neurips21/text2image-1B.png b/neurips21/t1_t2/results/T1/neurips21/text2image-1B.png similarity index 100% rename from t1_t2/results/T1/neurips21/text2image-1B.png rename to neurips21/t1_t2/results/T1/neurips21/text2image-1B.png diff --git a/t1_t2/results/T1/ssnpp-1B.png b/neurips21/t1_t2/results/T1/ssnpp-1B.png similarity index 100% rename from t1_t2/results/T1/ssnpp-1B.png rename to neurips21/t1_t2/results/T1/ssnpp-1B.png diff --git a/t1_t2/results/T1/text2image-1B.png b/neurips21/t1_t2/results/T1/text2image-1B.png similarity index 100% rename from t1_t2/results/T1/text2image-1B.png rename to neurips21/t1_t2/results/T1/text2image-1B.png diff --git a/t1_t2/results/T2/bigann-1B-IO.png b/neurips21/t1_t2/results/T2/bigann-1B-IO.png similarity index 100% rename from t1_t2/results/T2/bigann-1B-IO.png rename to neurips21/t1_t2/results/T2/bigann-1B-IO.png diff --git a/t1_t2/results/T2/bigann-1B.png b/neurips21/t1_t2/results/T2/bigann-1B.png similarity index 100% rename from t1_t2/results/T2/bigann-1B.png rename to neurips21/t1_t2/results/T2/bigann-1B.png diff --git a/t1_t2/results/T2/deep-1B-IO.png b/neurips21/t1_t2/results/T2/deep-1B-IO.png similarity index 100% rename from t1_t2/results/T2/deep-1B-IO.png rename to neurips21/t1_t2/results/T2/deep-1B-IO.png diff --git a/t1_t2/results/T2/deep-1B.png b/neurips21/t1_t2/results/T2/deep-1B.png similarity index 100% rename from t1_t2/results/T2/deep-1B.png rename to neurips21/t1_t2/results/T2/deep-1B.png diff --git a/t1_t2/results/T2/msspacev-1B-IO.png b/neurips21/t1_t2/results/T2/msspacev-1B-IO.png similarity index 100% rename from t1_t2/results/T2/msspacev-1B-IO.png rename to neurips21/t1_t2/results/T2/msspacev-1B-IO.png diff --git a/t1_t2/results/T2/msspacev-1B.png b/neurips21/t1_t2/results/T2/msspacev-1B.png similarity index 100% rename from t1_t2/results/T2/msspacev-1B.png rename to neurips21/t1_t2/results/T2/msspacev-1B.png diff --git a/t1_t2/results/T2/msturing-1B-IO.png b/neurips21/t1_t2/results/T2/msturing-1B-IO.png similarity index 100% rename from t1_t2/results/T2/msturing-1B-IO.png rename to neurips21/t1_t2/results/T2/msturing-1B-IO.png diff --git a/t1_t2/results/T2/msturing-1B.png b/neurips21/t1_t2/results/T2/msturing-1B.png similarity index 100% rename from t1_t2/results/T2/msturing-1B.png rename to neurips21/t1_t2/results/T2/msturing-1B.png diff --git a/t1_t2/results/T2/neurips21/bigann-1B.png b/neurips21/t1_t2/results/T2/neurips21/bigann-1B.png similarity index 100% rename from t1_t2/results/T2/neurips21/bigann-1B.png rename to neurips21/t1_t2/results/T2/neurips21/bigann-1B.png diff --git a/t1_t2/results/T2/neurips21/deep-1B.png b/neurips21/t1_t2/results/T2/neurips21/deep-1B.png similarity index 100% rename from t1_t2/results/T2/neurips21/deep-1B.png rename to neurips21/t1_t2/results/T2/neurips21/deep-1B.png diff --git a/t1_t2/results/T2/neurips21/msspacev-1B.png b/neurips21/t1_t2/results/T2/neurips21/msspacev-1B.png similarity index 100% rename from t1_t2/results/T2/neurips21/msspacev-1B.png rename to neurips21/t1_t2/results/T2/neurips21/msspacev-1B.png diff --git a/t1_t2/results/T2/neurips21/msturing-1B.png b/neurips21/t1_t2/results/T2/neurips21/msturing-1B.png similarity index 100% rename from t1_t2/results/T2/neurips21/msturing-1B.png rename to neurips21/t1_t2/results/T2/neurips21/msturing-1B.png diff --git a/t1_t2/results/T2/neurips21/ssnpp-1B.png b/neurips21/t1_t2/results/T2/neurips21/ssnpp-1B.png similarity index 100% rename from t1_t2/results/T2/neurips21/ssnpp-1B.png rename to neurips21/t1_t2/results/T2/neurips21/ssnpp-1B.png diff --git a/t1_t2/results/T2/neurips21/t2.csv b/neurips21/t1_t2/results/T2/neurips21/t2.csv similarity index 100% rename from t1_t2/results/T2/neurips21/t2.csv rename to neurips21/t1_t2/results/T2/neurips21/t2.csv diff --git a/t1_t2/results/T2/neurips21/text2image-1B.png b/neurips21/t1_t2/results/T2/neurips21/text2image-1B.png similarity index 100% rename from t1_t2/results/T2/neurips21/text2image-1B.png rename to neurips21/t1_t2/results/T2/neurips21/text2image-1B.png diff --git a/t1_t2/results/T2/ssnpp-1B-IO.png b/neurips21/t1_t2/results/T2/ssnpp-1B-IO.png similarity index 100% rename from t1_t2/results/T2/ssnpp-1B-IO.png rename to neurips21/t1_t2/results/T2/ssnpp-1B-IO.png diff --git a/t1_t2/results/T2/ssnpp-1B.png b/neurips21/t1_t2/results/T2/ssnpp-1B.png similarity index 100% rename from t1_t2/results/T2/ssnpp-1B.png rename to neurips21/t1_t2/results/T2/ssnpp-1B.png diff --git a/t1_t2/results/T2/text2image-1B-IO.png b/neurips21/t1_t2/results/T2/text2image-1B-IO.png similarity index 100% rename from t1_t2/results/T2/text2image-1B-IO.png rename to neurips21/t1_t2/results/T2/text2image-1B-IO.png diff --git a/t1_t2/results/T2/text2image-1B.png b/neurips21/t1_t2/results/T2/text2image-1B.png similarity index 100% rename from t1_t2/results/T2/text2image-1B.png rename to neurips21/t1_t2/results/T2/text2image-1B.png diff --git a/t3/LB_history/Dec.2.2021/LEADERBOARDS.md b/neurips21/t3/LB_history/Dec.2.2021/LEADERBOARDS.md similarity index 100% rename from t3/LB_history/Dec.2.2021/LEADERBOARDS.md rename to neurips21/t3/LB_history/Dec.2.2021/LEADERBOARDS.md diff --git a/t3/LB_history/Dec.2.2021/LEADERBOARDS_REJECT_ANOMALIES.md b/neurips21/t3/LB_history/Dec.2.2021/LEADERBOARDS_REJECT_ANOMALIES.md similarity index 100% rename from t3/LB_history/Dec.2.2021/LEADERBOARDS_REJECT_ANOMALIES.md rename to neurips21/t3/LB_history/Dec.2.2021/LEADERBOARDS_REJECT_ANOMALIES.md diff --git a/t3/LB_history/Nov.29.2021/LEADERBOARDS.md b/neurips21/t3/LB_history/Nov.29.2021/LEADERBOARDS.md similarity index 100% rename from t3/LB_history/Nov.29.2021/LEADERBOARDS.md rename to neurips21/t3/LB_history/Nov.29.2021/LEADERBOARDS.md diff --git a/t3/LB_history/Nov.29.2021/TASKS_ISSUES_RESOLUTIONS.md b/neurips21/t3/LB_history/Nov.29.2021/TASKS_ISSUES_RESOLUTIONS.md similarity index 100% rename from t3/LB_history/Nov.29.2021/TASKS_ISSUES_RESOLUTIONS.md rename to neurips21/t3/LB_history/Nov.29.2021/TASKS_ISSUES_RESOLUTIONS.md diff --git a/t3/LEADERBOARDS.md b/neurips21/t3/LEADERBOARDS.md similarity index 100% rename from t3/LEADERBOARDS.md rename to neurips21/t3/LEADERBOARDS.md diff --git a/t3/LEADERBOARDS_PRIVATE.md b/neurips21/t3/LEADERBOARDS_PRIVATE.md similarity index 100% rename from t3/LEADERBOARDS_PRIVATE.md rename to neurips21/t3/LEADERBOARDS_PRIVATE.md diff --git a/t3/LEADERBOARDS_PRIVATE_REJECT_ANOMALIES.md b/neurips21/t3/LEADERBOARDS_PRIVATE_REJECT_ANOMALIES.md similarity index 100% rename from t3/LEADERBOARDS_PRIVATE_REJECT_ANOMALIES.md rename to neurips21/t3/LEADERBOARDS_PRIVATE_REJECT_ANOMALIES.md diff --git a/t3/LEADERBOARDS_PUBLIC.md b/neurips21/t3/LEADERBOARDS_PUBLIC.md similarity index 100% rename from t3/LEADERBOARDS_PUBLIC.md rename to neurips21/t3/LEADERBOARDS_PUBLIC.md diff --git a/t3/LEADERBOARDS_PUBLIC_REJECT_ANOMALIES.md b/neurips21/t3/LEADERBOARDS_PUBLIC_REJECT_ANOMALIES.md similarity index 100% rename from t3/LEADERBOARDS_PUBLIC_REJECT_ANOMALIES.md rename to neurips21/t3/LEADERBOARDS_PUBLIC_REJECT_ANOMALIES.md diff --git a/t3/LEADERBOARDS_REJECT_ANOMALIES.md b/neurips21/t3/LEADERBOARDS_REJECT_ANOMALIES.md similarity index 100% rename from t3/LEADERBOARDS_REJECT_ANOMALIES.md rename to neurips21/t3/LEADERBOARDS_REJECT_ANOMALIES.md diff --git a/t3/RANKING.md b/neurips21/t3/RANKING.md similarity index 100% rename from t3/RANKING.md rename to neurips21/t3/RANKING.md diff --git a/t3/README.md b/neurips21/t3/README.md similarity index 100% rename from t3/README.md rename to neurips21/t3/README.md diff --git a/t3/TASKS_ISSUES_RESOLUTIONS.md b/neurips21/t3/TASKS_ISSUES_RESOLUTIONS.md similarity index 100% rename from t3/TASKS_ISSUES_RESOLUTIONS.md rename to neurips21/t3/TASKS_ISSUES_RESOLUTIONS.md diff --git a/t3/cuanns_ivfpq/Dockerfile b/neurips21/t3/cuanns_ivfpq/Dockerfile similarity index 100% rename from t3/cuanns_ivfpq/Dockerfile rename to neurips21/t3/cuanns_ivfpq/Dockerfile diff --git a/t3/cuanns_ivfpq/README.md b/neurips21/t3/cuanns_ivfpq/README.md similarity index 100% rename from t3/cuanns_ivfpq/README.md rename to neurips21/t3/cuanns_ivfpq/README.md diff --git a/t3/cuanns_ivfpq/algos.yaml b/neurips21/t3/cuanns_ivfpq/algos.yaml similarity index 100% rename from t3/cuanns_ivfpq/algos.yaml rename to neurips21/t3/cuanns_ivfpq/algos.yaml diff --git a/t3/cuanns_multigpu/Dockerfile b/neurips21/t3/cuanns_multigpu/Dockerfile similarity index 100% rename from t3/cuanns_multigpu/Dockerfile rename to neurips21/t3/cuanns_multigpu/Dockerfile diff --git a/t3/cuanns_multigpu/README.md b/neurips21/t3/cuanns_multigpu/README.md similarity index 100% rename from t3/cuanns_multigpu/README.md rename to neurips21/t3/cuanns_multigpu/README.md diff --git a/t3/cuanns_multigpu/algos.yaml b/neurips21/t3/cuanns_multigpu/algos.yaml similarity index 100% rename from t3/cuanns_multigpu/algos.yaml rename to neurips21/t3/cuanns_multigpu/algos.yaml diff --git a/t3/eval_2021/faiss_t3/prun.sh b/neurips21/t3/eval_2021/faiss_t3/prun.sh similarity index 100% rename from t3/eval_2021/faiss_t3/prun.sh rename to neurips21/t3/eval_2021/faiss_t3/prun.sh diff --git a/t3/faiss_t3/Dockerfile b/neurips21/t3/faiss_t3/Dockerfile similarity index 100% rename from t3/faiss_t3/Dockerfile rename to neurips21/t3/faiss_t3/Dockerfile diff --git a/t3/faiss_t3/README.md b/neurips21/t3/faiss_t3/README.md similarity index 100% rename from t3/faiss_t3/README.md rename to neurips21/t3/faiss_t3/README.md diff --git a/t3/faiss_t3/algos.yaml b/neurips21/t3/faiss_t3/algos.yaml similarity index 100% rename from t3/faiss_t3/algos.yaml rename to neurips21/t3/faiss_t3/algos.yaml diff --git a/t3/faiss_t3/baseline_plots/bigann-1B-r-vs-p.png b/neurips21/t3/faiss_t3/baseline_plots/bigann-1B-r-vs-p.png similarity index 100% rename from t3/faiss_t3/baseline_plots/bigann-1B-r-vs-p.png rename to neurips21/t3/faiss_t3/baseline_plots/bigann-1B-r-vs-p.png diff --git a/t3/faiss_t3/baseline_plots/bigann-1B-r-vs-t.png b/neurips21/t3/faiss_t3/baseline_plots/bigann-1B-r-vs-t.png similarity index 100% rename from t3/faiss_t3/baseline_plots/bigann-1B-r-vs-t.png rename to neurips21/t3/faiss_t3/baseline_plots/bigann-1B-r-vs-t.png diff --git a/t3/faiss_t3/baseline_plots/deep-1B-r-vs-p.png b/neurips21/t3/faiss_t3/baseline_plots/deep-1B-r-vs-p.png similarity index 100% rename from t3/faiss_t3/baseline_plots/deep-1B-r-vs-p.png rename to neurips21/t3/faiss_t3/baseline_plots/deep-1B-r-vs-p.png diff --git a/t3/faiss_t3/baseline_plots/deep-1B-r-vs-t.png b/neurips21/t3/faiss_t3/baseline_plots/deep-1B-r-vs-t.png similarity index 100% rename from t3/faiss_t3/baseline_plots/deep-1B-r-vs-t.png rename to neurips21/t3/faiss_t3/baseline_plots/deep-1B-r-vs-t.png diff --git a/t3/faiss_t3/baseline_plots/msspacev-1B-r-vs-p.png b/neurips21/t3/faiss_t3/baseline_plots/msspacev-1B-r-vs-p.png similarity index 100% rename from t3/faiss_t3/baseline_plots/msspacev-1B-r-vs-p.png rename to neurips21/t3/faiss_t3/baseline_plots/msspacev-1B-r-vs-p.png diff --git a/t3/faiss_t3/baseline_plots/msspacev-1B-r-vs-t.png b/neurips21/t3/faiss_t3/baseline_plots/msspacev-1B-r-vs-t.png similarity index 100% rename from t3/faiss_t3/baseline_plots/msspacev-1B-r-vs-t.png rename to neurips21/t3/faiss_t3/baseline_plots/msspacev-1B-r-vs-t.png diff --git a/t3/faiss_t3/baseline_plots/msturing-1B-r-vs-p.png b/neurips21/t3/faiss_t3/baseline_plots/msturing-1B-r-vs-p.png similarity index 100% rename from t3/faiss_t3/baseline_plots/msturing-1B-r-vs-p.png rename to neurips21/t3/faiss_t3/baseline_plots/msturing-1B-r-vs-p.png diff --git a/t3/faiss_t3/baseline_plots/msturing-1B-r-vs-t.png b/neurips21/t3/faiss_t3/baseline_plots/msturing-1B-r-vs-t.png similarity index 100% rename from t3/faiss_t3/baseline_plots/msturing-1B-r-vs-t.png rename to neurips21/t3/faiss_t3/baseline_plots/msturing-1B-r-vs-t.png diff --git a/t3/faiss_t3/baseline_plots/text2image-1B-r-vs-p.png b/neurips21/t3/faiss_t3/baseline_plots/text2image-1B-r-vs-p.png similarity index 100% rename from t3/faiss_t3/baseline_plots/text2image-1B-r-vs-p.png rename to neurips21/t3/faiss_t3/baseline_plots/text2image-1B-r-vs-p.png diff --git a/t3/faiss_t3/baseline_plots/text2image-1B-r-vs-t.png b/neurips21/t3/faiss_t3/baseline_plots/text2image-1B-r-vs-t.png similarity index 100% rename from t3/faiss_t3/baseline_plots/text2image-1B-r-vs-t.png rename to neurips21/t3/faiss_t3/baseline_plots/text2image-1B-r-vs-t.png diff --git a/t3/faiss_t3/cost/AdvantechSky6200.pdf b/neurips21/t3/faiss_t3/cost/AdvantechSky6200.pdf similarity index 100% rename from t3/faiss_t3/cost/AdvantechSky6200.pdf rename to neurips21/t3/faiss_t3/cost/AdvantechSky6200.pdf diff --git a/t3/faiss_t3/cost/GPU.pdf b/neurips21/t3/faiss_t3/cost/GPU.pdf similarity index 100% rename from t3/faiss_t3/cost/GPU.pdf rename to neurips21/t3/faiss_t3/cost/GPU.pdf diff --git a/t3/faiss_t3/cost/RAM.pdf b/neurips21/t3/faiss_t3/cost/RAM.pdf similarity index 100% rename from t3/faiss_t3/cost/RAM.pdf rename to neurips21/t3/faiss_t3/cost/RAM.pdf diff --git a/t3/faiss_t3/cost/SSD.pdf b/neurips21/t3/faiss_t3/cost/SSD.pdf similarity index 100% rename from t3/faiss_t3/cost/SSD.pdf rename to neurips21/t3/faiss_t3/cost/SSD.pdf diff --git a/t3/faiss_t3/faiss-gpu_requirements.txt b/neurips21/t3/faiss_t3/faiss-gpu_requirements.txt similarity index 100% rename from t3/faiss_t3/faiss-gpu_requirements.txt rename to neurips21/t3/faiss_t3/faiss-gpu_requirements.txt diff --git a/t3/gemini/.gitignore b/neurips21/t3/gemini/.gitignore similarity index 100% rename from t3/gemini/.gitignore rename to neurips21/t3/gemini/.gitignore diff --git a/t3/gemini/README.md b/neurips21/t3/gemini/README.md similarity index 100% rename from t3/gemini/README.md rename to neurips21/t3/gemini/README.md diff --git a/t3/gemini/algos.yaml b/neurips21/t3/gemini/algos.yaml similarity index 100% rename from t3/gemini/algos.yaml rename to neurips21/t3/gemini/algos.yaml diff --git a/t3/gemini/buildidx/build_index.py b/neurips21/t3/gemini/buildidx/build_index.py similarity index 100% rename from t3/gemini/buildidx/build_index.py rename to neurips21/t3/gemini/buildidx/build_index.py diff --git a/t3/gemini/buildidx/htest.py b/neurips21/t3/gemini/buildidx/htest.py similarity index 100% rename from t3/gemini/buildidx/htest.py rename to neurips21/t3/gemini/buildidx/htest.py diff --git a/t3/gemini/buildidx/run_bin_build_index.sh b/neurips21/t3/gemini/buildidx/run_bin_build_index.sh similarity index 100% rename from t3/gemini/buildidx/run_bin_build_index.sh rename to neurips21/t3/gemini/buildidx/run_bin_build_index.sh diff --git a/t3/gemini/buildidx/test.py b/neurips21/t3/gemini/buildidx/test.py similarity index 100% rename from t3/gemini/buildidx/test.py rename to neurips21/t3/gemini/buildidx/test.py diff --git a/t3/gemini/cost/AdvantechSky6200.pdf b/neurips21/t3/gemini/cost/AdvantechSky6200.pdf similarity index 100% rename from t3/gemini/cost/AdvantechSky6200.pdf rename to neurips21/t3/gemini/cost/AdvantechSky6200.pdf diff --git a/t3/gemini/cost/GPU.pdf b/neurips21/t3/gemini/cost/GPU.pdf similarity index 100% rename from t3/gemini/cost/GPU.pdf rename to neurips21/t3/gemini/cost/GPU.pdf diff --git a/t3/gemini/cost/RAM.pdf b/neurips21/t3/gemini/cost/RAM.pdf similarity index 100% rename from t3/gemini/cost/RAM.pdf rename to neurips21/t3/gemini/cost/RAM.pdf diff --git a/t3/gemini/cost/SSD.pdf b/neurips21/t3/gemini/cost/SSD.pdf similarity index 100% rename from t3/gemini/cost/SSD.pdf rename to neurips21/t3/gemini/cost/SSD.pdf diff --git a/t3/gemini/requirements.txt b/neurips21/t3/gemini/requirements.txt similarity index 100% rename from t3/gemini/requirements.txt rename to neurips21/t3/gemini/requirements.txt diff --git a/t3/gemini/run_bin_python.sh b/neurips21/t3/gemini/run_bin_python.sh similarity index 100% rename from t3/gemini/run_bin_python.sh rename to neurips21/t3/gemini/run_bin_python.sh diff --git a/t3/gemini/run_conda_python.sh b/neurips21/t3/gemini/run_conda_python.sh similarity index 100% rename from t3/gemini/run_conda_python.sh rename to neurips21/t3/gemini/run_conda_python.sh diff --git a/track1_baseline_faiss/README.md b/neurips21/track1_baseline_faiss/README.md similarity index 100% rename from track1_baseline_faiss/README.md rename to neurips21/track1_baseline_faiss/README.md diff --git a/track1_baseline_faiss/__init__.py b/neurips21/track1_baseline_faiss/__init__.py similarity index 100% rename from track1_baseline_faiss/__init__.py rename to neurips21/track1_baseline_faiss/__init__.py diff --git a/track1_baseline_faiss/baseline_faiss.py b/neurips21/track1_baseline_faiss/baseline_faiss.py similarity index 100% rename from track1_baseline_faiss/baseline_faiss.py rename to neurips21/track1_baseline_faiss/baseline_faiss.py diff --git a/track1_baseline_faiss/baseline_faiss_filtered.py b/neurips21/track1_baseline_faiss/baseline_faiss_filtered.py similarity index 100% rename from track1_baseline_faiss/baseline_faiss_filtered.py rename to neurips21/track1_baseline_faiss/baseline_faiss_filtered.py diff --git a/track1_baseline_faiss/bow_id_selector.swig b/neurips21/track1_baseline_faiss/bow_id_selector.swig similarity index 100% rename from track1_baseline_faiss/bow_id_selector.swig rename to neurips21/track1_baseline_faiss/bow_id_selector.swig diff --git a/track1_baseline_faiss/parse_results.py b/neurips21/track1_baseline_faiss/parse_results.py similarity index 100% rename from track1_baseline_faiss/parse_results.py rename to neurips21/track1_baseline_faiss/parse_results.py diff --git a/track1_baseline_faiss/plots/bigann-1B.png b/neurips21/track1_baseline_faiss/plots/bigann-1B.png similarity index 100% rename from track1_baseline_faiss/plots/bigann-1B.png rename to neurips21/track1_baseline_faiss/plots/bigann-1B.png diff --git a/track1_baseline_faiss/plots/deep-1B.png b/neurips21/track1_baseline_faiss/plots/deep-1B.png similarity index 100% rename from track1_baseline_faiss/plots/deep-1B.png rename to neurips21/track1_baseline_faiss/plots/deep-1B.png diff --git a/track1_baseline_faiss/plots/msspacev-1B.png b/neurips21/track1_baseline_faiss/plots/msspacev-1B.png similarity index 100% rename from track1_baseline_faiss/plots/msspacev-1B.png rename to neurips21/track1_baseline_faiss/plots/msspacev-1B.png diff --git a/track1_baseline_faiss/plots/msturing-1B.png b/neurips21/track1_baseline_faiss/plots/msturing-1B.png similarity index 100% rename from track1_baseline_faiss/plots/msturing-1B.png rename to neurips21/track1_baseline_faiss/plots/msturing-1B.png diff --git a/track1_baseline_faiss/plots/ssnpp-1B.png b/neurips21/track1_baseline_faiss/plots/ssnpp-1B.png similarity index 100% rename from track1_baseline_faiss/plots/ssnpp-1B.png rename to neurips21/track1_baseline_faiss/plots/ssnpp-1B.png diff --git a/track1_baseline_faiss/plots/text2image-1B.png b/neurips21/track1_baseline_faiss/plots/text2image-1B.png similarity index 100% rename from track1_baseline_faiss/plots/text2image-1B.png rename to neurips21/track1_baseline_faiss/plots/text2image-1B.png diff --git a/track1_baseline_faiss/run_baselines.bash b/neurips21/track1_baseline_faiss/run_baselines.bash similarity index 100% rename from track1_baseline_faiss/run_baselines.bash rename to neurips21/track1_baseline_faiss/run_baselines.bash diff --git a/track1_baseline_faiss/test_bow_id_selector.py b/neurips21/track1_baseline_faiss/test_bow_id_selector.py similarity index 100% rename from track1_baseline_faiss/test_bow_id_selector.py rename to neurips21/track1_baseline_faiss/test_bow_id_selector.py diff --git a/track3_baseline_faiss/README.md b/neurips21/track3_baseline_faiss/README.md similarity index 100% rename from track3_baseline_faiss/README.md rename to neurips21/track3_baseline_faiss/README.md diff --git a/track3_baseline_faiss/gpu_baseline_faiss.py b/neurips21/track3_baseline_faiss/gpu_baseline_faiss.py similarity index 100% rename from track3_baseline_faiss/gpu_baseline_faiss.py rename to neurips21/track3_baseline_faiss/gpu_baseline_faiss.py diff --git a/track3_baseline_faiss/plots/T3_deep-1B.png b/neurips21/track3_baseline_faiss/plots/T3_deep-1B.png similarity index 100% rename from track3_baseline_faiss/plots/T3_deep-1B.png rename to neurips21/track3_baseline_faiss/plots/T3_deep-1B.png diff --git a/notebooks/check_1B_groundtruth.ipynb b/preparation/neurips21/notebooks/check_1B_groundtruth.ipynb similarity index 100% rename from notebooks/check_1B_groundtruth.ipynb rename to preparation/neurips21/notebooks/check_1B_groundtruth.ipynb diff --git a/notebooks/compare_track1_1B_vs_2x500M.ipynb b/preparation/neurips21/notebooks/compare_track1_1B_vs_2x500M.ipynb similarity index 100% rename from notebooks/compare_track1_1B_vs_2x500M.ipynb rename to preparation/neurips21/notebooks/compare_track1_1B_vs_2x500M.ipynb diff --git a/notebooks/eval_t2i_results.ipynb b/preparation/neurips21/notebooks/eval_t2i_results.ipynb similarity index 100% rename from notebooks/eval_t2i_results.ipynb rename to preparation/neurips21/notebooks/eval_t2i_results.ipynb diff --git a/notebooks/find_suitable_nq.ipynb b/preparation/neurips21/notebooks/find_suitable_nq.ipynb similarity index 100% rename from notebooks/find_suitable_nq.ipynb rename to preparation/neurips21/notebooks/find_suitable_nq.ipynb diff --git a/notebooks/large_coarse_quantizer.ipynb b/preparation/neurips21/notebooks/large_coarse_quantizer.ipynb similarity index 100% rename from notebooks/large_coarse_quantizer.ipynb rename to preparation/neurips21/notebooks/large_coarse_quantizer.ipynb diff --git a/notebooks/parse_filtered_results.ipynb b/preparation/neurips23/parse_filtered_results.ipynb similarity index 100% rename from notebooks/parse_filtered_results.ipynb rename to preparation/neurips23/parse_filtered_results.ipynb diff --git a/sparse_algorithms/basic_sparse_index.py b/preparation/neurips23/sparse_algorithms/basic_sparse_index.py similarity index 100% rename from sparse_algorithms/basic_sparse_index.py rename to preparation/neurips23/sparse_algorithms/basic_sparse_index.py diff --git a/sparse_algorithms/eval_sparse.py b/preparation/neurips23/sparse_algorithms/eval_sparse.py similarity index 100% rename from sparse_algorithms/eval_sparse.py rename to preparation/neurips23/sparse_algorithms/eval_sparse.py