Neurips23 Filter track

WM_filter submission
harsha-simhadri · Oct 28, 2023 · dace55d · dace55d
1 parent 9b3a10a
commit dace55d
Show file tree

Hide file tree

Showing 5 changed files with 660 additions and 0 deletions.
diff --git a/.github/workflows/neurips23.yml b/.github/workflows/neurips23.yml
@@ -12,6 +12,9 @@ jobs:
           - algorithm: faiss
             dataset: random-filter-s
             track: filter
+          - algorithm: wm_filter
+            dataset: random-filter-s
+            track: filter
           - algorithm: linscan
             dataset: sparse-small
             track: sparse

diff --git a/neurips23/filter/wm_filter/Dockerfile b/neurips23/filter/wm_filter/Dockerfile
@@ -0,0 +1,28 @@
+FROM neurips23
+
+RUN apt-get update; DEBIAN_FRONTEND=noninteractive apt install intel-mkl python3-setuptools  wget python3-matplotlib build-essential checkinstall libssl-dev swig4.0 python3-dev  python3-numpy  python3-numpy-dev   -y
+COPY install/requirements_conda.txt ./
+# conda doesn't like some of our packages, use pip
+RUN python3 -m pip install -r requirements_conda.txt
+
+
+# CMAKE with good enough version
+RUN  mkdir /build && wget  https://github.com/Kitware/CMake/archive/refs/tags/v3.27.1.tar.gz  && mv v3.27.1.tar.gz  /build
+RUN cd /build; tar -zxvf v3.27.1.tar.gz
+RUN cd /build/CMake-3.27.1 &&   ./bootstrap && make   && make install
+
+
+RUN cd / &&  git clone https://github.com/alemagnani/faiss.git && cd /faiss && git pull && git checkout wm_filter
+
+RUN cd /faiss && rm -rf ./build
+RUN cd /faiss/; cmake -B build /faiss/  -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DFAISS_OPT_LEVEL=avx2  -DBLA_VENDOR=Intel10_64_dyn -DBUILD_TESTING=ON  -DPython_EXECUTABLE=/usr/bin/python3 -DMKL_LIBRARIES=/usr/lib/x86_64-linux-gnu/libmkl_rt.so
+RUN cd /faiss/; make -C build -j faiss faiss_avx2 swigfaiss  swigfaiss_avx2
+RUN  (cd /faiss/build/faiss/python && python3 setup.py install)
+
+#RUN pip install tritonclient[all]
+ENV PYTHONPATH=/faiss/build/faiss/python/build/lib/
+
+RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)'
+
+
+
diff --git a/neurips23/filter/wm_filter/README.md b/neurips23/filter/wm_filter/README.md
@@ -0,0 +1,7 @@
+
+### Submission for Neurips23 Filter track of WM_filter team
+This submission leverages the IVF index to run the filter in a fast way.
+
+More info to come...
+
+
diff --git a/neurips23/filter/wm_filter/config.yaml b/neurips23/filter/wm_filter/config.yaml
@@ -0,0 +1,50 @@
+random-filter-s:
+  wm_filter:
+    docker-tag: neurips23-filter-wm_filter
+    module: neurips23.filter.wm_filter.wm_filter
+    constructor: FAISS
+    base-args: [ "@metric" ]
+    run-groups:
+      base:
+        args: |
+          [{"indexkey": "IVF1024,SQ8", 
+            "threads": 8,
+            "train_size": 2000000,
+            "type": "direct"
+          }]
+        query-args: |
+          [
+          {"nprobe": 80,  "max_codes": 100, "selector_probe_limit": 80},
+          {"nprobe": 100,  "max_codes": 500, "selector_probe_limit": 100},
+          {"nprobe": 120,  "max_codes": 1000, "selector_probe_limit": 120},
+          {"nprobe": 140,  "max_codes": 1800, "selector_probe_limit": 140},
+          {"nprobe": 160,  "max_codes": 500, "selector_probe_limit": 160}, 
+           {"nprobe": 70,  "max_codes": 1000, "selector_probe_limit": 70}
+          ]
+yfcc-10M:
+    wm_filter:
+      docker-tag: neurips23-filter-wm_filter
+      module: neurips23.filter.wm_filter.wm_filter
+      constructor: FAISS
+      base-args: [ "@metric" ]
+      run-groups:
+        base:
+          args: |
+            [{"indexkey": "IVF1024,SQ8", 
+              "threads": 8,
+              "train_size": 2000000,
+              "type": "direct"
+            }]
+          query-args: |
+            [
+            {"nprobe": 80,  "max_codes": 1800, "selector_probe_limit": 80},
+            {"nprobe": 100,  "max_codes": 1800, "selector_probe_limit": 100},
+            {"nprobe": 120,  "max_codes": 1800, "selector_probe_limit": 120},
+            {"nprobe": 140,  "max_codes": 1800, "selector_probe_limit": 140},
+            {"nprobe": 160,  "max_codes": 1800, "selector_probe_limit": 160}, 
+            {"nprobe": 70,  "max_codes": 2100, "selector_probe_limit": 70},
+            {"nprobe": 100,  "max_codes": 2100, "selector_probe_limit": 100},
+            {"nprobe": 130,  "max_codes": 2100, "selector_probe_limit": 130},
+            {"nprobe": 160,  "max_codes": 2100, "selector_probe_limit": 160},
+            {"nprobe": 200,  "max_codes": 2100, "selector_probe_limit": 200}
+            ]