From cd4a9e5b43f81b183e17f6337b080080947bdbd3 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 29 Oct 2023 15:34:34 +0000 Subject: [PATCH 01/18] using some better pragma to optimize faiss, just 4 test --- .github/workflows/neurips23.yml | 4 + neurips23/filter/faissplus/Dockerfile | 25 ++ neurips23/filter/faissplus/README.md | 102 +++++++ .../filter/faissplus/bow_id_selector.swig | 183 +++++++++++ neurips23/filter/faissplus/config.yaml | 66 ++++ neurips23/filter/faissplus/faiss.py | 287 ++++++++++++++++++ res.csv | 41 +++ results/random-filter-s.png | Bin 0 -> 39572 bytes results/random-s.png | Bin 0 -> 45942 bytes results/yfcc-10M.png | Bin 0 -> 44402 bytes 10 files changed, 708 insertions(+) create mode 100644 neurips23/filter/faissplus/Dockerfile create mode 100644 neurips23/filter/faissplus/README.md create mode 100644 neurips23/filter/faissplus/bow_id_selector.swig create mode 100644 neurips23/filter/faissplus/config.yaml create mode 100644 neurips23/filter/faissplus/faiss.py create mode 100644 res.csv create mode 100644 results/random-filter-s.png create mode 100644 results/random-s.png create mode 100644 results/yfcc-10M.png diff --git a/.github/workflows/neurips23.yml b/.github/workflows/neurips23.yml index 4a8aaaed..64b611ff 100644 --- a/.github/workflows/neurips23.yml +++ b/.github/workflows/neurips23.yml @@ -30,6 +30,10 @@ jobs: - algorithm: vamana dataset: random-xs track: ood + # Test fassplus entry + - algorithm: faissplus + dataset: random-filter-s + track: filter fail-fast: false steps: diff --git a/neurips23/filter/faissplus/Dockerfile b/neurips23/filter/faissplus/Dockerfile new file mode 100644 index 00000000..163391a8 --- /dev/null +++ b/neurips23/filter/faissplus/Dockerfile @@ -0,0 +1,25 @@ +FROM neurips23 + +RUN apt update && apt install -y wget swig +RUN wget https://repo.anaconda.com/archive/Anaconda3-2023.03-0-Linux-x86_64.sh +RUN bash Anaconda3-2023.03-0-Linux-x86_64.sh -b + +ENV PATH /root/anaconda3/bin:$PATH +ENV CONDA_PREFIX /root/anaconda3/ + +RUN conda install -c pytorch faiss-cpu +COPY install/requirements_conda.txt ./ +# conda doesn't like some of our packages, use pip +RUN python3 -m pip install -r requirements_conda.txt + +COPY neurips23/filter/faiss/bow_id_selector.swig ./ + +RUN swig -c++ -python -I$CONDA_PREFIX/include -Ifaiss bow_id_selector.swig +RUN g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ + -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ + -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so -Ifaiss + +RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' + + + diff --git a/neurips23/filter/faissplus/README.md b/neurips23/filter/faissplus/README.md new file mode 100644 index 00000000..c834af51 --- /dev/null +++ b/neurips23/filter/faissplus/README.md @@ -0,0 +1,102 @@ + +# Faiss baseline for the Filtered search track + +The database of size $N=10^7$ can be seen as the combination of: + +- a matrix $M$ of size $N \times d$ of embedding vectors (called `xb` in the code). $d=192$. +- a sparse matrix $M_\mathrm{meta}$ of size $N \times v$, entry $i,j$ is set to 1 iff word $j$ is applicable to vector $i$. $v=200386$, called `meta_b` in the code (a [CSR matrix](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html)) + +The Faiss basleline for the filtered search track is based on two distinct data structures, a word-based inverted file and a Faiss `IndexIVFFlat`. +Both data structured allow to peform filtered searches in two different ways. + +The search is based on a query vector $q\in \mathbb{R}^d$ and associated query words $w_1, w_2$ (there are one or two query words). +The search results are the database vectors that include /all/ query words and that are nearest to $q$ in $L_2$ distance. + +## Word-based inverted file + +This is term-based inverted file that maps each word to the vectors (docs) that contain that term. +In the code it is a CSR matrix called `docs_per_word` (it's just the transposed version of `meta_b`). + +At search time, the subset (`subset`) of vectors eligible for results depends on the number of query words: + +- if there is a single word $w_1$ then it's just the set of non-0 entries in row $w_1$ of the `docs_per_word` matrix. +This can be extracted at no cost + +- if there are two words $w_1$ and $w_2$ then the sets of non-0 entries of rows $w_1$ and $w_2$ are intersected. +This is done with `np.intersect1d` or the C++ function `intersect_sorted`, that is faster (linear in nb of non-0 entries of the two rows). + +When this subset is selected, the result is found by searching the top-k vectors in this subset of rows of $M$. +The result is exact and the search is most efficient when the subset is small (ie. the words are discriminative enough to filter the results well). + +## IndexIVFFlat structure + +This is a Faiss [`IndexIVFFlat`](https://github.com/facebookresearch/faiss/wiki/The-index-factory#encodings) called `index`. + +By default the index performs unfiltered search, ie. the nearest vectors to $q$ can be retrieved. +The accuracy of this search depends on the number of visited centroids of the `IndexIVFFlat` (parameter `nprobe`, the larger the more accurate and the slower). + +One solution would be to over-fetch vectors and perform filtering post-hoc using the words in the result list. +However, it is unclear /how much/ we should overfetch. + +Therefore, another solution is to use the Faiss [filtering functionality](https://github.com/facebookresearch/faiss/wiki/Setting-search-parameters-for-one-query#searching-in-a-subset-of-elements), ie. provide a callback function that is called for each vector id to decide if it should be considered as a result or not. + +The callback function is implemented in C++ in the class `IDSelectorBOW`. +For vector id $i$ it looks up the row $i$ of $M_\mathrm{meta}$ and peforms a binary search on $w_1$ to check of that word belongs to the words associated to vector $i$. +If $w_2$ is also provided, it does the same for $w_2$. +The callback returns true only if all terms are present. + +### Binary filtering + +The issue is that this callback is relatively slow because (1) it requires to access the $M_\mathrm{meta}$ matrix which causes cache misses and (2) it performs an iterative binary search. +Since the callback is called in the tightest inner loop of the search function, and since the IVF search tends to perform many vector comparisons, this has non negligible performance impact. + +To speed up this test, we can use a nifty piece of bit manipulation. +The idea is that the vector ids are 63 bits long (64 bits integers but negative values are reserved, so we cannot use the sign bit). +However, since $N=10^7$ we use only $\lceil \log_2 N \rceil = 24$ bits of these, leaving 63-24 = 39 bits that are always 0. + +Now, we associate to each word $j$ a 39-bit signature $S[j]$, and the to each set of words the binary `or` of these signatures. +The query is represented by $s_\mathrm{q} = S[w_1] \vee S[w_2]$. +Database entry $i$ with words $W_i$ is represented by $s_i = \vee_{w\in W_i} S[w]$. + +Then we have the following implication: if $\\{w_1, w_2\\} \subset W_i$ then all 1 bits of $s_\mathrm{q}$ are also set to 1 in $s_i$. + +$$\\{w_1, w_2\\} \subset W_i \Rightarrow \neg s_i \wedge s_\mathrm{q} = 0$$ + +Which is equivalent to: + +$$\neg s_i \wedge s_\mathrm{q} \neq 0 \Rightarrow \\{w_1, w_2\\} \not\subset W_i $$ + +Of course, this is an implication, not an equivalence. +Therefore, it can only rule out database vectors. +However, the binary test is very cheap to perform (uses a few machine instructions on data that is already in machine registers), so it can be used as a pre-filter to apply the full membership test on candidates. +This is implemented in the `IDSelectorBOWBin` object. + +The remaining degree of freedom is how to choose the binary signatures, because this rule is always valid, but its filtering ability depends on the choice of the signatures $S$. +After a few tests (see [this notebook](https://gist.github.com/mdouze/75103e4cef436510ac9b834f9a77496f#file-eval_binary_signatures-ipynb) ) it seems that a random signature with 0.1 probability for 1s filters our 80% of negative tests. +Asjuting this to the frequency of the words did not seem to yield better results. + +## Choosing between the two implementations + +The two implementations are complementary: the word-first implementation gives exact results, and has a strong filtering ability for rare words. +The `IndexIVFFlat` implementation gives approximate results and is more relevant for words that are more common, where a significant subset of vectors are indeed relevant. + +Therefore, there should be a rule to choose between the two, and the relevant metric is the size of the subset of vectors to consider. +We can use statistics on the words, ie. $\mathrm{nocc}[j]$ is the number of times word $j$ appears in the dataset (this is just the column-wise sum of the $M_\mathrm{meta}$). + +For a single query word $w_1$, the fraction of relevant indices is just $f = \mathrm{nocc}[w_1] / N$. +For two query words, it is more complicated to compute but an estimate is given by $f = \mathrm{nocc}[w_1] \times \mathrm{nocc}[w_2] / N^2$ (this estimate assumes words are independent, which is incorrect). + +Therefore, the rule that we use is based on a threshold $\tau$ (called `metadata_threshold` in the code) : + +- if $f < \tau$ then use the word-first search + +- otherwise use the IVFFlat based index + +Note that the optimal threshold also depends on the target accuracy (since the IVFFlat is not exact, when a higher accuracy is desired), see https://github.com/harsha-simhadri/big-ann-benchmarks/pull/105#issuecomment-1539842223 . + + +## Code layout + +The code is in faiss.py, with performance critical parts implemented in C++ and wrapped with SWIG in `bow_id_selector.swig`. +SWIG directly exposes the C++ classes and functions in Python. + diff --git a/neurips23/filter/faissplus/bow_id_selector.swig b/neurips23/filter/faissplus/bow_id_selector.swig new file mode 100644 index 00000000..6712aa25 --- /dev/null +++ b/neurips23/filter/faissplus/bow_id_selector.swig @@ -0,0 +1,183 @@ + +%module bow_id_selector + +/* +To compile when Faiss is installed via conda: + +swig -c++ -python -I$CONDA_PREFIX/include bow_id_selector.swig && \ +g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ + -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ + -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so + +*/ + + +// Put C++ includes here +%{ + +#include +#include + +%} + +// to get uint32_t and friends +%include + +// This means: assume what's declared in these .h files is provided +// by the Faiss module. +%import(module="faiss") "faiss/MetricType.h" +%import(module="faiss") "faiss/impl/IDSelector.h" + +// functions to be parsed here + +// This is important to release GIL and do Faiss exception handing +%exception { + Py_BEGIN_ALLOW_THREADS + try { + $action + } catch(faiss::FaissException & e) { + PyEval_RestoreThread(_save); + + if (PyErr_Occurred()) { + // some previous code already set the error type. + } else { + PyErr_SetString(PyExc_RuntimeError, e.what()); + } + SWIG_fail; + } catch(std::bad_alloc & ba) { + PyEval_RestoreThread(_save); + PyErr_SetString(PyExc_MemoryError, "std::bad_alloc"); + SWIG_fail; + } + Py_END_ALLOW_THREADS +} + + +// any class or function declared below will be made available +// in the module. +%inline %{ + +struct IDSelectorBOW : faiss::IDSelector { + size_t nb; + using TL = int32_t; + const TL *lims; + const int32_t *indices; + int32_t w1 = -1, w2 = -1; + + IDSelectorBOW( + size_t nb, const TL *lims, const int32_t *indices): + nb(nb), lims(lims), indices(indices) {} + + void set_query_words(int32_t w1, int32_t w2) { + this->w1 = w1; + this->w2 = w2; + } + + // binary search in the indices array + bool find_sorted(TL l0, TL l1, int32_t w) const { + while (l1 > l0 + 1) { + TL lmed = (l0 + l1) / 2; + if (indices[lmed] > w) { + l1 = lmed; + } else { + l0 = lmed; + } + } + return indices[l0] == w; + } + + bool is_member(faiss::idx_t id) const { + TL l0 = lims[id], l1 = lims[id + 1]; + if (l1 <= l0) { + return false; + } + if(!find_sorted(l0, l1, w1)) { + return false; + } + if(w2 >= 0 && !find_sorted(l0, l1, w2)) { + return false; + } + return true; + } + + ~IDSelectorBOW() override {} +}; + + +struct IDSelectorBOWBin : IDSelectorBOW { + /** with additional binary filtering */ + faiss::idx_t id_mask; + + IDSelectorBOWBin( + size_t nb, const TL *lims, const int32_t *indices, faiss::idx_t id_mask): + IDSelectorBOW(nb, lims, indices), id_mask(id_mask) {} + + faiss::idx_t q_mask = 0; + + void set_query_words_mask(int32_t w1, int32_t w2, faiss::idx_t q_mask) { + set_query_words(w1, w2); + this->q_mask = q_mask; + } + + bool is_member(faiss::idx_t id) const { + if (q_mask & ~id) { + return false; + } + return IDSelectorBOW::is_member(id & id_mask); + } + + ~IDSelectorBOWBin() override {} +}; + + +size_t intersect_sorted_c( + size_t n1, const int32_t *a1, + size_t n2, const int32_t *a2, + int32_t *res) +{ + if (n1 == 0 || n2 == 0) { + return 0; + } + size_t i1 = 0, i2 = 0, i = 0; + for(;;) { + if (a1[i1] < a2[i2]) { + i1++; + if (i1 >= n1) { + return i; + } + } else if (a1[i1] > a2[i2]) { + i2++; + if (i2 >= n2) { + return i; + } + } else { // equal + res[i++] = a1[i1++]; + i2++; + if (i1 >= n1 || i2 >= n2) { + return i; + } + } + } +} + +%} + + +%pythoncode %{ + +import numpy as np + +# example additional function that converts the passed-in numpy arrays to +# C++ pointers +def intersect_sorted(a1, a2): + n1, = a1.shape + n2, = a2.shape + res = np.empty(n1 + n2, dtype=a1.dtype) + nres = intersect_sorted_c( + n1, faiss.swig_ptr(a1), + n2, faiss.swig_ptr(a2), + faiss.swig_ptr(res) + ) + return res[:nres] + +%} \ No newline at end of file diff --git a/neurips23/filter/faissplus/config.yaml b/neurips23/filter/faissplus/config.yaml new file mode 100644 index 00000000..ddf2eb93 --- /dev/null +++ b/neurips23/filter/faissplus/config.yaml @@ -0,0 +1,66 @@ +random-filter-s: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF1024,SQ8"}] + query-args: | + [{"nprobe": 1}, + {"nprobe":2}, + {"nprobe":4}] +random-s: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF1024,SQ8"}] + query-args: | + [{"nprobe": 1}, + {"nprobe":2}, + {"nprobe":4}] +yfcc-10M-unfiltered: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF16384,SQ8", "binarysig": true, "threads": 16}] + query-args: | + [{"nprobe": 1}, {"nprobe": 4}, {"nprobe": 16}, {"nprobe": 64}] +yfcc-10M: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF11264,SQ8", + "binarysig": true, + "threads": 16 + }] + query-args: | + [ + {"nprobe": 34, "mt_threshold": 0.00031}, + {"nprobe": 32, "mt_threshold": 0.0003}, + {"nprobe": 32, "mt_threshold": 0.00031}, + {"nprobe": 34, "mt_threshold": 0.0003}, + {"nprobe": 34, "mt_threshold": 0.00035}, + {"nprobe": 32, "mt_threshold": 0.00033}, + {"nprobe": 30, "mt_threshold": 0.00033}, + {"nprobe": 32, "mt_threshold": 0.00035}, + {"nprobe": 34, "mt_threshold": 0.00033}, + {"nprobe": 40, "mt_threshold": 0.0003} + ] diff --git a/neurips23/filter/faissplus/faiss.py b/neurips23/filter/faissplus/faiss.py new file mode 100644 index 00000000..02980d12 --- /dev/null +++ b/neurips23/filter/faissplus/faiss.py @@ -0,0 +1,287 @@ +import pdb +import pickle +import numpy as np +import os + +from multiprocessing.pool import ThreadPool + +import faiss + +from neurips23.filter.base import BaseFilterANN +from benchmark.datasets import DATASETS +from benchmark.dataset_io import download_accelerated + +import bow_id_selector + +def csr_get_row_indices(m, i): + """ get the non-0 column indices for row i in matrix m """ + return m.indices[m.indptr[i] : m.indptr[i + 1]] + +def make_bow_id_selector(mat, id_mask=0): + sp = faiss.swig_ptr + if id_mask == 0: + return bow_id_selector.IDSelectorBOW(mat.shape[0], sp(mat.indptr), sp(mat.indices)) + else: + return bow_id_selector.IDSelectorBOWBin( + mat.shape[0], sp(mat.indptr), sp(mat.indices), id_mask + ) + +def set_invlist_ids(invlists, l, ids): + n, = ids.shape + ids = np.ascontiguousarray(ids, dtype='int64') + assert invlists.list_size(l) == n + faiss.memcpy( + invlists.get_ids(l), + faiss.swig_ptr(ids), n * 8 + ) + + + +def csr_to_bitcodes(matrix, bitsig): + """ Compute binary codes for the rows of the matrix: each binary code is + the OR of bitsig for non-0 entries of the row. + """ + indptr = matrix.indptr + indices = matrix.indices + n = matrix.shape[0] + bit_codes = np.zeros(n, dtype='int64') + for i in range(n): + # print(bitsig[indices[indptr[i]:indptr[i + 1]]]) + bit_codes[i] = np.bitwise_or.reduce(bitsig[indices[indptr[i]:indptr[i + 1]]]) + return bit_codes + + +class BinarySignatures: + """ binary signatures that encode vectors """ + + def __init__(self, meta_b, proba_1): + nvec, nword = meta_b.shape + # number of bits reserved for the vector ids + self.id_bits = int(np.ceil(np.log2(nvec))) + # number of bits for the binary signature + self.sig_bits = nbits = 63 - self.id_bits + + # select binary signatures for the vocabulary + rs = np.random.RandomState(123) # we rely on this to be reproducible! + bitsig = np.packbits(rs.rand(nword, nbits) < proba_1, axis=1) + bitsig = np.pad(bitsig, ((0, 0), (0, 8 - bitsig.shape[1]))).view("int64").ravel() + self.bitsig = bitsig + + # signatures for all the metadata matrix + self.db_sig = csr_to_bitcodes(meta_b, bitsig) << self.id_bits + + # mask to keep only the ids + self.id_mask = (1 << self.id_bits) - 1 + + def query_signature(self, w1, w2): + """ compute the query signature for 1 or 2 words """ + sig = self.bitsig[w1] + if w2 != -1: + sig |= self.bitsig[w2] + return int(sig << self.id_bits) + +class FAISS(BaseFilterANN): + + def __init__(self, metric, index_params): + self._index_params = index_params + self._metric = metric + print(index_params) + self.indexkey = index_params.get("indexkey", "IVF32768,SQ8") + self.binarysig = index_params.get("binarysig", True) + self.binarysig_proba1 = index_params.get("binarysig_proba1", 0.1) + self.metadata_threshold = 1e-3 + self.nt = index_params.get("threads", 1) + + + def fit(self, dataset): + ds = DATASETS[dataset]() + if ds.search_type() == "knn_filtered" and self.binarysig: + print("preparing binary signatures") + meta_b = ds.get_dataset_metadata() + self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) + print("writing to", self.binarysig_name(dataset)) + pickle.dump(self.binsig, open(self.binarysig_name(dataset), "wb"), -1) + else: + self.binsig = None + + if ds.search_type() == "knn_filtered": + self.meta_b = ds.get_dataset_metadata() + self.meta_b.sort_indices() + + index = faiss.index_factory(ds.d, self.indexkey) + xb = ds.get_dataset() + print("train") + index.train(xb) + print("populate") + if self.binsig is None: + index.add(xb) + else: + ids = np.arange(ds.nb) | self.binsig.db_sig + index.add_with_ids(xb, ids) + + self.index = index + self.nb = ds.nb + self.xb = xb + self.ps = faiss.ParameterSpace() + self.ps.initialize(self.index) + print("store", self.index_name(dataset)) + faiss.write_index(index, self.index_name(dataset)) + + + def index_name(self, name): + return f"data/{name}.{self.indexkey}.faissindex" + + def binarysig_name(self, name): + return f"data/{name}.{self.indexkey}.binarysig" + + + def load_index(self, dataset): + """ + Load the index for dataset. Returns False if index + is not available, True otherwise. + + Checking the index usually involves the dataset name + and the index build paramters passed during construction. + """ + if not os.path.exists(self.index_name(dataset)): + if 'url' not in self._index_params: + return False + + print('Downloading index in background. This can take a while.') + download_accelerated(self._index_params['url'], self.index_name(dataset), quiet=True) + + print("Loading index") + + self.index = faiss.read_index(self.index_name(dataset)) + + self.ps = faiss.ParameterSpace() + self.ps.initialize(self.index) + + ds = DATASETS[dataset]() + + if ds.search_type() == "knn_filtered" and self.binarysig: + if not os.path.exists(self.binarysig_name(dataset)): + print("preparing binary signatures") + meta_b = ds.get_dataset_metadata() + self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) + else: + print("loading binary signatures") + self.binsig = pickle.load(open(self.binarysig_name(dataset), "rb")) + else: + self.binsig = None + + if ds.search_type() == "knn_filtered": + self.meta_b = ds.get_dataset_metadata() + self.meta_b.sort_indices() + + self.nb = ds.nb + self.xb = ds.get_dataset() + + return True + + def index_files_to_store(self, dataset): + """ + Specify a triplet with the local directory path of index files, + the common prefix name of index component(s) and a list of + index components that need to be uploaded to (after build) + or downloaded from (for search) cloud storage. + + For local directory path under docker environment, please use + a directory under + data/indices/track(T1 or T2)/algo.__str__()/DATASETS[dataset]().short_name() + """ + raise NotImplementedError() + + def query(self, X, k): + nq = X.shape[0] + self.I = -np.ones((nq, k), dtype='int32') + bs = 1024 + for i0 in range(0, nq, bs): + _, self.I[i0:i0+bs] = self.index.search(X[i0:i0+bs], k) + + + def filtered_query(self, X, filter, k): + print('running filtered query') + nq = X.shape[0] + self.I = -np.ones((nq, k), dtype='int32') + meta_b = self.meta_b + meta_q = filter + docs_per_word = meta_b.T.tocsr() + ndoc_per_word = docs_per_word.indptr[1:] - docs_per_word.indptr[:-1] + freq_per_word = ndoc_per_word / self.nb + + def process_one_row(q): + faiss.omp_set_num_threads(1) + qwords = csr_get_row_indices(meta_q, q) + assert qwords.size in (1, 2) + w1 = qwords[0] + freq = freq_per_word[w1] + if qwords.size == 2: + w2 = qwords[1] + freq *= freq_per_word[w2] + else: + w2 = -1 + if freq < self.metadata_threshold: + # metadata first + docs = csr_get_row_indices(docs_per_word, w1) + if w2 != -1: + docs = bow_id_selector.intersect_sorted( + docs, csr_get_row_indices(docs_per_word, w2)) + + assert len(docs) >= k, pdb.set_trace() + xb_subset = self.xb[docs] + _, Ii = faiss.knn(X[q : q + 1], xb_subset, k=k) + + self.I[q, :] = docs[Ii.ravel()] + else: + # IVF first, filtered search + sel = make_bow_id_selector(meta_b, self.binsig.id_mask if self.binsig else 0) + if self.binsig is None: + sel.set_query_words(int(w1), int(w2)) + else: + sel.set_query_words_mask( + int(w1), int(w2), self.binsig.query_signature(w1, w2)) + + params = faiss.SearchParametersIVF(sel=sel, nprobe=self.nprobe) + + _, Ii = self.index.search( + X[q:q+1], k, params=params + ) + Ii = Ii.ravel() + if self.binsig is None: + self.I[q] = Ii + else: + # we'll just assume there are enough results + # valid = Ii != -1 + # I[q, valid] = Ii[valid] & binsig.id_mask + self.I[q] = Ii & self.binsig.id_mask + + + if self.nt <= 1: + for q in range(nq): + process_one_row(q) + else: + faiss.omp_set_num_threads(self.nt) + pool = ThreadPool(self.nt) + list(pool.map(process_one_row, range(nq))) + + def get_results(self): + return self.I + + def set_query_arguments(self, query_args): + faiss.cvar.indexIVF_stats.reset() + if "nprobe" in query_args: + self.nprobe = query_args['nprobe'] + self.ps.set_index_parameters(self.index, f"nprobe={query_args['nprobe']}") + self.qas = query_args + else: + self.nprobe = 1 + if "mt_threshold" in query_args: + self.metadata_threshold = query_args['mt_threshold'] + else: + self.metadata_threshold = 1e-3 + + def __str__(self): + return f'Faiss({self.indexkey, self.qas})' + + \ No newline at end of file diff --git a/res.csv b/res.csv new file mode 100644 index 00000000..14e0ff22 --- /dev/null +++ b/res.csv @@ -0,0 +1,41 @@ +algorithm,parameters,dataset,count,qps,distcomps,build,indexsize,mean_ssd_ios,mean_latency,track,recall/ap +faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 1}))",random-filter-s,10,17210.013417421313,0.0,4.049878358840942,53980.0,0,0,filter,0.9147000000000001 +faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 2}))",random-filter-s,10,13434.971315820661,0.0,4.049878358840942,53980.0,0,0,filter,0.9701000000000001 +faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 4}))",random-filter-s,10,8898.944679478747,0.0,4.049878358840942,53980.0,0,0,filter,0.9759 +faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 1}))",random-s,10,107284.9213454406,0.0,-1.0,6400.0,0,0,filter,0.9137000000000001 +faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 2}))",random-s,10,52454.37150610923,0.0,-1.0,6400.0,0,0,filter,0.9635999999999999 +faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 4}))",random-s,10,34813.56916973082,0.0,-1.0,6400.0,0,0,filter,0.9692000000000001 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 1, 'mt_threshold': 0.0001}))",yfcc-10M,10,6795.64131225567,0.0,1072.5426700115204,4882552.0,0,0,filter,0.482366 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 1, 'mt_threshold': 0.0003}))",yfcc-10M,10,6077.0430330389545,0.0,1072.5426700115204,4882552.0,0,0,filter,0.5379240000000001 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 4, 'mt_threshold': 0.0001}))",yfcc-10M,10,6428.107866805404,0.0,1072.5426700115204,4882552.0,0,0,filter,0.6253599999999999 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 4, 'mt_threshold': 0.0003}))",yfcc-10M,10,5765.3369217869695,0.0,1072.5426700115204,4882552.0,0,0,filter,0.673125 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 16, 'mt_threshold': 0.0001}))",yfcc-10M,10,5349.823779832417,0.0,1072.5426700115204,4882552.0,0,0,filter,0.78061 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 1, 'mt_threshold': 0.01}))",yfcc-10M,10,1166.2876613086733,0.0,1072.5426700115204,4882552.0,0,0,filter,0.7835880000000001 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 16, 'mt_threshold': 0.0003}))",yfcc-10M,10,4911.40567052702,0.0,1072.5426700115204,4882552.0,0,0,filter,0.8174239999999999 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 32, 'mt_threshold': 0.0001}))",yfcc-10M,10,4384.772235694261,0.0,1072.5426700115204,4882552.0,0,0,filter,0.847014 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 4, 'mt_threshold': 0.01}))",yfcc-10M,10,1161.1679057350602,0.0,1072.5426700115204,4882552.0,0,0,filter,0.862773 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 32, 'mt_threshold': 0.0003}))",yfcc-10M,10,4159.390450900283,0.0,1072.5426700115204,4882552.0,0,0,filter,0.877257 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 64, 'mt_threshold': 0.0001}))",yfcc-10M,10,3262.4235190588574,0.0,1072.5426700115204,4882552.0,0,0,filter,0.901073 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 64, 'mt_threshold': 0.0003}))",yfcc-10M,10,3189.0423626576003,0.0,1072.5426700115204,4882552.0,0,0,filter,0.9240959999999999 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 96, 'mt_threshold': 0.0001}))",yfcc-10M,10,2593.7880792005567,0.0,1072.5426700115204,4882552.0,0,0,filter,0.926014 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 16, 'mt_threshold': 0.01}))",yfcc-10M,10,1148.366996200013,0.0,1072.5426700115204,4882552.0,0,0,filter,0.9367110000000001 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 96, 'mt_threshold': 0.0003}))",yfcc-10M,10,2584.583659327858,0.0,1072.5426700115204,4882552.0,0,0,filter,0.9448719999999999 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 32, 'mt_threshold': 0.01}))",yfcc-10M,10,1122.0103427196223,0.0,1072.5426700115204,4882552.0,0,0,filter,0.9623430000000001 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 64, 'mt_threshold': 0.01}))",yfcc-10M,10,1072.044241086604,0.0,1072.5426700115204,4882552.0,0,0,filter,0.979552 +faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 96, 'mt_threshold': 0.01}))",yfcc-10M,10,1040.7842700388208,0.0,1072.5426700115204,4882552.0,0,0,filter,0.9861280000000001 +faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 1}))",random-filter-s,10,17180.616884446812,0.0,-1.0,9592.0,0,0,filter,0.9147000000000001 +faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 2}))",random-filter-s,10,13201.38362127302,0.0,-1.0,9592.0,0,0,filter,0.9701000000000001 +faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 4}))",random-filter-s,10,8399.779707010324,0.0,-1.0,9592.0,0,0,filter,0.9759 +faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 1}))",random-s,10,90163.24512564759,0.0,5.0532073974609375,49152.0,0,0,filter,0.9137000000000001 +faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 2}))",random-s,10,69982.04691827677,0.0,5.0532073974609375,49152.0,0,0,filter,0.9635999999999999 +faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 4}))",random-s,10,27337.10054813627,0.0,5.0532073974609375,49152.0,0,0,filter,0.9692000000000001 +faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 30, 'mt_threshold': 0.00033}))",yfcc-10M,10,3902.783262534991,0.0,619.2082076072693,5439348.0,0,0,filter,0.895063 +faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 32, 'mt_threshold': 0.0003}))",yfcc-10M,10,3822.097448584031,0.0,619.2082076072693,5439348.0,0,0,filter,0.8968999999999999 +faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 32, 'mt_threshold': 0.00031}))",yfcc-10M,10,3826.360975537842,0.0,619.2082076072693,5439348.0,0,0,filter,0.897869 +faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 32, 'mt_threshold': 0.00033}))",yfcc-10M,10,3797.5221741475025,0.0,619.2082076072693,5439348.0,0,0,filter,0.8996660000000001 +faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 34, 'mt_threshold': 0.0003}))",yfcc-10M,10,3709.367384945006,0.0,619.2082076072693,5439348.0,0,0,filter,0.901073 +faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 32, 'mt_threshold': 0.00035}))",yfcc-10M,10,3775.3003297468717,0.0,619.2082076072693,5439348.0,0,0,filter,0.901529 +faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 34, 'mt_threshold': 0.00031}))",yfcc-10M,10,3753.3730144863166,0.0,619.2082076072693,5439348.0,0,0,filter,0.902008 +faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 34, 'mt_threshold': 0.00033}))",yfcc-10M,10,3709.4452984119116,0.0,619.2082076072693,5439348.0,0,0,filter,0.903754 +faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 34, 'mt_threshold': 0.00035}))",yfcc-10M,10,3685.290238185696,0.0,619.2082076072693,5439348.0,0,0,filter,0.9055630000000001 +faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 40, 'mt_threshold': 0.0003}))",yfcc-10M,10,3471.4920981563846,0.0,619.2082076072693,5439348.0,0,0,filter,0.9119970000000001 diff --git a/results/random-filter-s.png b/results/random-filter-s.png new file mode 100644 index 0000000000000000000000000000000000000000..28288c94570456966698728fa38246b2eb83f65f GIT binary patch literal 39572 zcmce8by$_p7A|&z2`VXqA|R-wfPjj$bhm(XN;lX+cZYN%-Jl{O-5_jIx+S-gcg+UQ zIrsnjT%RA$;cWId-^`j>^{#i#2T5^3{FB5d@$m5Qg@qnV9vwElE%W@vx+^5UPNl5wtXC*8Hu-0||kSJih;^T9me-<`nD*@L^k zJ`g&D`{gXrahMPK_YIE{ANR|H%U9zM;C?%P=rs%O7d%ZjA0!t%ysM|rw(WlO`oJS% z+;47oV6dyPinoY7iCOC$@9WfERJuiE(pS`)7xu{D&(PPpx;m!P)31MC(r}`5ua3Q4 z@%LKH+Q<)28UGt%ql6!P>wijB5uhG+s2CF-yOtD7jQ)H&E z3wjb?cdtJ0X9V-SpgL>Jo*)~bfIZUC1c8yo(t1~<|(^g%1qXwO^YHE&i`J7uT z?K;_t9r;Gx+UhkUi#J(Wuo!mS%uvj} z7T=vw=gcfAH^q8So;Q@uB_pYF*`q6R6NO6Te}%1TPc(8?qx%Or_AOeF+cVN41` zH0f^zrT1Nd|!%s zxxY3g>vM;H`8KezQV|HIUYkt4E7;Skc$f(Mgd@lcy{-Ey6Z}NY@X44y63mJ=7{l(aCp)TQ-F?GFf}o z-=IBlDp)f=HMaVB7PI4;F{|+}$;NQ@ zGk>$g*-TQ27?kzv0%&XS4PI(qeoH{c8UO3oFCVpBCX>Mmwc6U+u_&{y^q!R4$ImTH zF1(nW)X%M_tuigFH=P)9vwC(>OITR=`wr8|2cz007u-gs`-*d^r?7*rSdJaonN-)E z?Nqt6AlTB50uvSIl~zHW`AXXb^?t{N@_{M-)tT<@)ozWfNG@9uKM@g;m0#N4DPrMl z@T;_wQyHi2eEOSHm)@?}m#t5y(cQe+A|t^)cA-T`+SoXKz+r|#SWw`6V5)wF3%}8@ z)B02rdj0Q_kw71x7+AGIXQ~{J$BOvtlQ1a-5HI@Mi$lj8%$9fNl4?d;|1OSn zmf2fYfB$}yn_I20&%%9ewr8Y${vCyjq`0`3w6yd^5|T7Hs{M8&o;A*DBHf%JT(+6Y z`GyH0A(vT=x}H$D{M8ICaKz4ca%21A8XBJP3ks$^ANfJdWs}}K8SP2JUZ<5kIh7RM zh2E7cMFoF10+lB0x??tQ-+H=37{r_^75BF3-pFb0%|BHAqBgsW8e%FE4pr z-9&P`p#x+*PDKr?+gl5+GkuoLa;mBkhF$3?%gwwRmJ>~@2FpwJx6|yG$I%o0CDyRb zcjo#_yOYG=NO#8apUQv}eo58*#qZz0WwX@FEt1ZUO-!UoMDu*~_Eu0<4g=*LurQ); zev40%XE7#Xy)aPTV3IHscNpHH#1p4~aPHD2H7j49?Z4Vr-bxMUCxOn=XO&6&S$#cn z$u3DCD6?3mH<)K*mO0-9BcGj}?XcRZaGmVLC&scrj}j6RXgN8R8^c)J(KF+BPgk}0 zOYo$uug=gjGK#~z9)>P_l76uwI3;v^y|8kt5i{?>+|<%8!8_-inc!?Q{Ot|MJg|8= zNAXtgXU$i-jbUTl6salHgc*asD{X|QHX393-L2{w3X^T-dPSZ*>41Z!G4335pI#xe z1FWZYuR#*+jT-`2u3RZu{6SLh(Lm>1sZ!}ujtlu3NDSP;tNT+*9o5clhsVnTfU zWUo;c_j~|mWVKOR!*wgai5sg5r)x5=UA!G*R;f0w{9-L_iFC#>ri|45@xp~THj`fc)>wX>UgLo> zCH4WkxVH9oLNc<9@y2kYws_$IjCZF*iOpQ9Qm!tTAF+XD!JWR-D|-A>#GDId%U)Q$ zuMr%U2VJ7@B$eJb9UxVha=mEOl|IE?XxK>?X**z_s*n{$?l4JNT~jjyM?wbN*~-^r z6vR<;!*+1l4mx^Qx4)KzVp=0wBn!c++d}K zS;fCZ3k!>tE$rYs<9nN6m(xQ+LQV=E($UjP18W|}>skSZ=i-W4J8~xvUmc~Tp%DZp zCI@?=Za+qdbfiA`Huj%HeEpa-XeFohbbL~@i}Lg5&k5DBK`N%L0kkq!Rru_anWgh> zaDZ7fza3`Ks8AhrTJ|I*cvoOHm`TpGi8O{X3!>9bol;b!s8(Xt3evSkhYMz>TxORo;Bom=hDu?O62;S{ z(OOQcNguGCsbC(#=A`x2sKZfZEMJk1v>y+3*j%hZ`vV(16K+e!f4aZ4uwr}N z2%n5o7VKSTGw)9NM3kd=<pq3r8;`$5~~y}D?`3sGr5}6Vs**ykMvfZatc!*`ECdZ zlEHaR`-w98Qj^`clLQ+fV``c?UoykknW3Tp8`1gl$f<)4TcX{3hVaJ+bM(^Sa>wd} z9bf%;y3P;YVg>9DvZ{n$zc_X_m#w?iY^UB_Qg5j`!<(Kc7B1q4HJkbw;C0~8(Mhx^ z;KI_Go9^+o`5LHz$58-_qEm=jU96#)S;LqGgZ2HaSyhI0oz1RU8TFS))oYGI9v^0z z2)mUWAOAfm=)p#u=4B)u`pv8G>wf-hfn8hyMMjHVhq>1GVN=sGrEP2qVDLDd`k*OK_5n#Snm@eA zW=t&G#n`6?ceV>Ni^qwrXMahZBD!^4|Dzgc-|F8H@0six^7fWbeC3sgCx2zfQQHsB zG;vIjdR}1`z!aFYg8|P16%l|Fe%m(Q|DxicDp;j;>_QeSS09~8h{BMO0>!rzj5@Nx zqY(A6;@BpvzK0~ypLq z4zAZB<#GBoQ2@dA<*BKuA0Y5aOLf1y7Nb9zSj~QY9xiIW&B4)gZzet0t7jb6$V7bw zOg#%cjAFF_to+nY(FS&b9kksO^sXmK?cu{0z5xNFmM$RFx6~t_-jUN;_JcxS!6#u` zA$J9L1)qWmTr^V$fapp}Rn_>i zfcciz;_qr0qThCqTeozoJxSqjgvdr4IFPzNdFA$2FzX)e;#`f6s_XevooT_?8J){- z7yne9fx`fTwQoMsOQF{gqAA1_<+LOmZL_t3Wxqw_p6#;buuu$+?=qN$+gN*uhSD1T z`H$_m*sP3@ZK;Y5nt?;r4hRUaYxX8#2YEF_a4d&-%M2Lpxb9$G-%@LX2OO|WSg_q* zD0SXgV>Rr~oa0u{)_Ox(TeP)?aqd;KY-AAP+jzm9b19wlnnVj8&JCp1oifX~s%nAzOH3sr(19z4QI0V;03IE(fi=^{m+l zMi3Dax~Ci7hQTfjQgq}`G=?XYJ7NvLxE&biMsv^UF$O`a70VPJREn`%Uzq|M(8hmB zZPc(=&C_*a*~om83_=qLTr*K!2k;j z#>_D?$CqnVx*YJlcQeCzydjiOoF)VUsxZh+Zit2by>BzMPMeTb<#B}>vT&3j1cqLd z?HwF%j?VJ)JKt~F%*|thB8}@eZr}n{Y#M~pndNKB3(p}PLebY2IGh2?T)k%bNKWex zkl{jcQPDkcy(>k{8k{-!1^uSjZMMIU8>6^YP=IWU9|G=fN~pNa;5`i;7nceL2Zv4* zMLDCGrsgLkUpTmv-n`qA@PFfXoK5D-mc^r6$;KE6i1l3~xE%|O*}{=mjo#RSQ{=CC zd3~i_LSp(~lnCEDg4HWy){wc@nif*nZmh&Lz8))?&8dHO`E9dNguj2>w{PG4*NZ78n?FiJB&1nRi>LW+N*u4w4zlo- z;L1(@H*ek`9|fu8DD$QRoJ^+EdW4BHbX>2B(K;Ex$t$ef8M&4E?X!oN3LtSH$M$dW}H!G@r{^jIudqMRo%KOjqRmN z!h(sVI?tZzLu{j5Y@ze!tEd4(!ojWHt<@~oP7<@SPY47-@nMPeOd@?|(Ic={*ta)9 zjoQOljl4E~Rqkw;rpwRV-Dxj3Xuly$$9#m6j!q;g(k2YMCJn2*z1-W8t|S9qXr)-! zK1-9oos{lX`&37n{c@R%b^Or!o5hLsTQy;97v6uYm`AZGc&e1^qCo*lgYlm~OF}L^ z(7g?Z1zXI`%G#+{k`V8^=Y@umC(*cvIi)VMTu(iL+9SfWD(jnb|oaN2z`gWEm zgo?6B;*lenzPXNAQ%`bk<(+@cu60h)K}Vkn8nwi{XnsCrR0&}r(h>+u01(jWTI1~5 zR&T{7QnMPgJ%&_eRak9s$6i=43e!5Xei(%r4+b|tuj9dE2XG7)Js+{a9FyLuJvqAd zS8;PA*=kfz7$S6=*y%s+I!NUM?b1KXzv$Z$hNG?tQf5EJK9)x?+RYkGYm8pJt!F5OF)v+`3kb0mv50b}P z-CTo)kh?lW^gL0e3l*4Byw6sne1=o8A5cn}qf>A7_s{UjSMGuI3Ush1;86T!7mvvo zV&*JW*TbGfa>_y8r_;N!5Cztqti{!io7@|avlReU0vsKuTpu$KxevJRI6H656|O_f zhSXu|2Z>n+bMyT;GYHG$1@!~EEL!<5O}#iqeiOUaCYdA~!?phX#}7bO_?3&yKS4&C z0QxowzOoa7;^bn>i8|ff+L6U%h!-8!di3hc1$1?F%_VLeXA95hTmEsE=oh~HTyH*D z;e*6;a=r)v19e$8y?gse_iYj0NysK6IxJ}yW*}m#Oj9W|HEfIj5aqn4q@$~gRtYAp z3+$imqsr_v1-wxw6;z+Usn}c+U095H_{xdsJAS~J`;o=UF4KXLnA+CiC$f(p^3|@L~tQ7i@s#BNz?u12LBS5JG}JQ zbS60PI!{sJyJ^$mi!v4s9TW?>^AKr^IM*DDuwT{iONjRkAWZ+WXg z#xmbOllwGb$81p4H7+73$fmEG`Z65zw#|Rb)&uqvM^>V1LHpCdzpmHtZhwMQ*I?+& zi}hyL?KsfmsSqqZ8JYFU%nsz-!hW`t)!>{`Ad8A`&O#V1C{#y=3Oo14Z0`C5?@o#! zx&7l)ml$Q?wFsgCtW;vUD+6;^&;cM#A#!iQG=;28^RH8oTZ(Yz6zLgg1bNLe6=+ty zz2M*iAzsJr%1!BUR@G7)lLuHc*rs@@()*E<#{wV=Pl2B?MyD%6mA?SEh#$Gl#BW&q}_O^ zp3^jBt@C#=-RVlX4PIJc1bR!Xjf&^GGgOjJF&2GsRF##Dhvanqu;n&@S#gkDssgmx z1yT06WyXb0h-2;VlW)PH;)zo~IJZE!`YEoGeM|6qrdsJ*kw%(7*^wJ^^BWPBuvkN| z<~mKmwv3XW?C`y?E>5?`R?`5n6@%1!K!c8&n)>IQ4Fy)Y#(|NrpJ=lAB;Ikobm2oo zMF@d_v#Bsrr*sa*vcJ-_RN(+*M!PY zuJytf09i){-BLPll>y2v16xw=>J>%e3hDPvDk?s}W^OaT+ku2?z=X$p8{g32uf+=I z;bR1(f39Ru34^mn=nsI@%@*C;_tsfY`~YG7)khYwn z;^h_s)@_Hi*%XMoCLuM)MgattX@0M=vW@l(@8%!A7H>5}*f<@-Gl?3TV2mKjTAH_z z@ddnBnC=22si^(n2=cjrRY;Vg`3##+R$tOsF;E%s;(W1I6UFUVH)mt??ez&BlPJe| z4k#=v5U71>NfaZA#zv21ShoVQ%w!Z%w$^Jz=oN22m*4F)pn;HIoIQ?zWah|}L|D~>`~Oi}N5UDT;7ptiogy&;sjGfwbKk1-qk2R>7R!s*Y$;>s@^ z>w06^QKC*xPQ@~-2e*_C;0=5Dvfv$L0{gFKKNc9qr1R0Vd{w@DrA>6YBgJ32z&Hi6 zIwjaTHRsjN0X7KO z+UzGH6~_6ru9jF$#j2PV^K1X`X#Ge?-vPqmASD3x6>m?H=tkhc_SOcVQv+@Fw=2^t zoHqa}7U->XtyFj2vIcifzFat)u)P#qssGc55&=K}2ZBSVh7D_+;9L1KT(t^CiK#Zx z+pDG<4ntoKY;7(#JM3&>5d?N2jBXhYm=U-zNWp@M*^HAR%bWo#EDws`nIc0Y+y*F5 z5_Z5*0@X|q2m?^0glVBl<@S0SKmpv&;Moi@1sTNbrs>FvO59(3jevU;K()5Eq8WiD zO}nf81Ho@p0v;`st#Q_i)t69-Lb=o?1K|>pmQ5UEUOAjmXiG2Lc+!HZ0u-*BQjlb~ zcBII77w50HB1e<2-}-2IqPZJ~24Dd3Eu$E|hjON7vAMF!?%CDL)^kdWCK;K6|4 zS3$AA)d+zp`!^At1$$LHWrb=vVNiXr9qsr4NMqp9)0bnQyrcxiU>X1g9>C<>1l^Fw zf|3F{(Ewgwq@c)xe3cgR7*rR;XYY%F{6!z-bPPqgIXPAfZPm30o$^v#FP z+>b%Qd&y+1yVXRbtwZ6EDFl$9Le7t<9*VEA9!KGya-RODhzQbXhN9~RE0e7QY&+}S zm3ijFkDwMN&OYdri*yMRJfof*9fa?)-{h3%1t0A>0Od%CkeoRIMwf-q4{geUope06 z&o+ka{s+%Ci4@v1n3bv*VoYEiKlMA3eTOZI5h60^y3=2@y*}gV;c>cZ*!(iYc+3c6 z9YH3vcmceN2iJU2RWf87=LiXPIk zPD_th?;UZ|Q}fc0C9nX1YsjMC^5rvTZoq_ZqB&Z_bOVlHtA2N8BpML?%?=Jby;C0) zrHH}L2uMr&ox5;B8%^}`eY?-iLGiDS;xj=7s@^$$DJoa<@(+lHb@L3`f4ukfWQEP} z09QkN=KdeIY@K={K)JL*JU`qH9P&7dPkb4)MF33fb4aV!@0;?A6kgf}pwj~)(DwIU zUTlzJX+4SNDSv{2yo3Z4=?{p5%o!MuXa4Az}z`K z0?NS7Sh;Bt2k@H!B*apMrfS--`8ow(7-(tl0)A=40_YEV0a!Q#G}p`A4tY|Fij(a& zP)_&(f$*G3cY^38?SV3T9+MkSLY}f1w0$?fM=IdpQ1Tpd*(;7~vx^2Ytw9ZA&XEZB zSefs~ymxnJhI)VpE?AdyxQZD@2OWYi);i;0)e?SFdsmUlOK3Fe>({S{C$&#hyjXLG zdK&4nsfPSSkV0|%jY=DWG zfSIU4Vur}W5F*gqIt91cJie@J8%1k5iDqy!D%wij$}Y9z~LwxxIj+xiSy?_7y2|#x433VeX zTH4wou<)MX%>(Rt8Rm?(F}H)UF18R7j38JN5EGN{H8mu`YBr(1!NHeWi2B}YUS`&7 zVgVf)cRs`85IzP9(Y>*yk|xi9ULxS)vi&^E;PO{&BdBUXPZv*%i(HiUYT(lPAAnIj znPeu&41{ngwp@Rv5v_qiIj;p_YBsc;$^eca zgAIYbIN#gTC#EX^$d!fdUje5P*`hrN!66tCUzg2A!LH}W`Jj^4ifmJYXfV^s7~&#; zNmn2vNKH$Q4VyI{a$bY z9QsnO687X91rOMb;B)^Q^*_w=h1LdkBzgn^D-b}ye-{qmhtq-U)rY7-DVeeVWQsvHu~U;HW5Pk1G1#czmDSyTG_7_Vw$-HZ{19WjU+v?EMT`abEv$ojvvJjf`E&&Hv4+*;~~A8;Z{V-@#rd zkeL>WoUWQ>a+7(xOCCBaZ&TDCrl^>#Qef8o(7R^V!cFD}BCWtbx9OU4*UExH*UDy& znWE?GS@Ffy)fS;LL{I5-0A`flK%djz}FeUNZ1{lG!KU6$3o>zh@??wTQL=g-RT?PZdzfug0DQ&~>}p5~if z@jU$)l$~8cb6M$?q1XN=Z*3|&4Q>(TW6DBVjU-LZ?Q*Z4oHNU(>zrEQ=>FZUq4e}} zC(*7rzOdkDR8WXoYHQ2Yk1a9YAvGG)rcXWNz=h`ig(ayodDt={I*L)jr6Qi{)M-Jb zTe<;9LSF8@ENpgCKYFBbYRY1KEL0Z%DdV2t4RS~cqDP!r4fQ7qOm4^>3US|m;B>f{ z6uEbEv1fJl|9(y!ULAYXWx{54N?U-|ZEA1DiJS~P0~2-^+>gYC?TI&}&Q&eUQ#?sH z5N~|{Bf5H1EJu@N@hHn_|I`BwdlI}8xFbXy9N)+o%u$VZWBjrY+R8d%k8# ze!Ksk`ZW@Q6mc%Bk-=7==&UEF3-?uAQqAYW&E~K7IpeXUeeyFxN3Y4Fgve-vD!PMb+f2?bvr^`kEAOwFLB1-Y{Q>OX;|e3tFwee&|w%t*Ef>!5H z=YRBKD?L3FK;?xS9q)e$i-4oUKex}L9Oe(2%F3u*Ei4JUKfQ_#XJszE71OpqOj{I` zn`@^#2eef(lwrATd&fw#HY_wH15-%P%dN&`)Fm6e>69yakdAO~XJ?JAi>(AV)+}r$ zhq${}6M|*yzlXWi{rL8E_pSbhxwW4aotnm~t83L1^`-eAj8W~{l-FdX)v-@X7e>7| zSFAXem!6~^Pj=cDU%;68Us5x1iC}v6?s!ADI>Fs@cU-h*o63NRY4q9RQ(4XZFU#su ziz$Z@Txg3qxj%Notx8`O@ppVj7=!O0I};{*PO^A^J+wW#GGge6wzrRIvEZlP+%r_a zmZF!w6G$Z2?SC=tiIBhVL!no6dwPDhV6!`+r$LDLzi%D`ol&sJSzGA6UNy#)axG?Q zU!3yJrD`^nmZgl95k5!d&hJUfnBx{vCpLU3Z!5y`ndgVQ|NBJPPC$~UN9;pF{=Kyt zZMz>cGv^dbJ)Eq%zto^Pc0XPJ`s?1BA+q?A_WM(9bfs?8tde|zF_8v0OSvcD{AX!$ z!t#UOStJqSHg=B!_iV^>N2K_L(Ts@l#^8h+{1F zR}-}(6t-&+Yo8v_l%?FC&O~KZds7&zbxi!@sBe$_=?*^f-nAQN*O=yPdN!L(#6|s? zZ#M@xxYg~Sa7O}n{#E*rrNgp>d&9h!EMfGlz7tSj6nyI~~p=KTLQc zR0S61+-_Wk_x`eYm86S&kbIG1p{>UVVwt$zJN=<)9MzrQR+gP_W=%GHzKP7X0$N(j)1HU!gg5zlL3t8gD}4xVo|3|-|%n*SJ47t z&cMKc80dhijKLD&h?E9+kvwob5C{VtS=k>lI3jdJyraMWD!6^}OgYhRYU<>3ar%<{ zA;fK@fK}>&c+>&PurPxdb`W7bxH^#2;v2+Ds|51H#}6MC3yJ~zL6jXMsN1YVT9pr( ztJ?Ew!U0UbX`usDA)yKjlgN3i+UvoSz)VW{vxYDvNEJGu+*CFJ#LdwyTSz4aoR-BPCKm&! z<`cC6P_7Jn^HM6d)>QJ%2GyaW+zD@Eb=`JKNJ>hB^0~wB4>T)K_%`fFmiqYdW83!+ z7a;J>(2KGc@axDkka6Bx?MB?*0DpfO;bwBQ)p$d$-)zK)y=&qCu17^4ME?u|hzGcz zo_ zZID1`06O_sH7>6a3Y0RM&hO4pbPZR%E3W~_*`JytQk)~Q>1^4-2Y+ejFe}exbyh58 zkKulnr>o65$9i-4nND-$Y)oQ0R!p|nC#wJV2U3*LoRDbg~3T^;#n`RKK;0M<4#9!R$O?}SnrtT${}`|^Yrj}iR5W?%F35fLg2>+XhWRVlVZfvC~i24!m^QRQFqeQ4yS`6sP zkm&VXzxw?fTRfRgz#NdMgHo%g~n^{F{ zT{%lUcf2GK7YaDa#c}%7h@}d-wN4WkDS^xJn7X*=IL?^={3xiYs4_bD#=PiKbIbtS zLMS60OQY^2RdDFovoxjl*(LE4>o%R*)AQmr=Rx>@Y6=3oew71Np;Dlj;};M3$WMUl z0hYlIQ~+&nYli6QM}-z;e8y6)`c9@$dtpO+1jzCM!1PWuQa|8DTQ;U1WfrF9}<8U6|WpMG{16 z)ogPWHll+FZ-Dh9X#T?m8bc`hmOz7n z)sHKBlR#mVgerF#)*e*%S-x^-mIf~>p#c=<@NX=*ybxo|=Dbu(gD8F@3Ny#2cnB;|3D(?eY~rWIIn{Sb$9rbyyx#2H@}U+Ux~ZzGLPACk#|wjQkGz zQX6;gy8I!sMoRqcqcWjztZ3gUao>s76P-tj&1 zr3DefkilZ~r1`&m$fc;nc}KkAlu!MP?$jgA+Es>~pJ<|GpU4J;>KveX4V4?T?e7V- z^HaSV$1P3$>vbTrsND;G9_QVvv^HSljJzRtFoJY{oy!`*x>QsKXFu&vDx=Jo<*lY{ z3-NJ{{a|DqV!BFA{oopH81BPfK>0Y2C0s0<$i2Ck73|Dz)MT!;KT4xvcEA%G8fF=tDl z%~({}x7{=tBB%I|ZbSRBRX+qlEqR)pij2$&gR@fmZ@=ASDEAg5zz7jKXqMm-wV1lkzl<$9(k3jutPRf6?!8;?BR3>pAwEZa_mXGHgnPs+ ze>ij(pFTdhC-lJIe@KK!=x+K*NozrLI`p4M%hkEB5V_mXT&HEH;sf`?ZpK!6Yp7^1 z&63nfr=xA{h`nZFH%Uq60SyNx7?!0#h0%saIa!;6p0NWpdy!Hy(=9*w-~gf4R=;=? zTH2g^v)hh_sl{&&9N5d`WUnaM40;!vhKuoctEaKNs{)~Z$-aTp%9y(Mxypg2x%S>8 z3`BF!OU2TUA19^SZ3UC7i+O#RU+Gp|4ki%N7%hk;&YwNIzpOS|wyxRpZWq2;(V&xv zhim^rza$euEur-AP{>}cm+S|MIN7o^26#ekXfO>_E=5R~-?AE>E!dZZ-$V-~p?Ku9 zWfQ3FEG_K~xx=I+yMby<-QClCpqDG6?nHGG;_?l{Rnt)LQnj~#&2n62@0`(1%xoz?N^`de;SW8aB1hXLCDf;^Sz`d*>6)2QqHS=*+rF< zn%A@fyU6z^R|FP79nMkg#P^Di`H^cf_DLKbv)f4O6oRsQvc zol$Hi0h6}m$B#X+#o=SJN{fs;15kZxI zV{N!s1NyjZ!=x`rlJ7sOo2%H-Vpb>P;!r{jXG%{%_`!aHE2qv^Nh^^+=?ST`hlQKP z{&w8oF)3nah6%mjo6wlfhG%?_U(k)hP7$G3pC9e!9mz|uLq)STCTVIakkjG7Bf2k; zl;O6kPBc$h4&yLNW&Z8OA;a~J1q+O^AN8d;gR?hgYs_#`dNAskC(qECqdev^P0PV< z{3ptGe2lil-tq$dA+M*qpmmJ0{-}9!V&e^fSa>EPXaBsuvxh7t%aewN^3>cS_b~s@ zC$lb;o_l8?g?Pwy?+kQ)j8o^Xo!)rzx#$SL@W=8HXfnB3yDlk{I(2qtC^qMj3eA`5>X?J4c< z`-ZnKLto=}RTFli{Usvy>joU>&jV)hmjbE==nb+YU}dsTio5e;w6w)ikN&9^!ol(n z@_nD>f)~rAPLoq2c$c%Y$$pk#e+_l+DJi-N8#&Qu*6rR2o@LAtzZ$o)Y;3)xi6GTE#FV{*S0TMj4_vYRujo=%%srbC!;uCfz`+{Llp_5fu z8)3(7d5j^lUaZVA`2rq3omv3r$m&6{2aPVP0zehPmA3`O{Ws)ZP45BOk|v-Rf%>ak z`ek3ISW<{(ZTIC-uazATx|G)nDANCY+UQ!>E+EkMg?j&M4B@^k$1lw8f{4jopsE*( z`g_-#xN^+;l8J*=V}EsJh^3zH#NXd#<-~sV@v6lI*G)dW{(t|pYuN8^VC!>jvrZ6x;@m@0 zWgWM1mNFArY1em&f=}gL2Nc z3|-u>=4}H8YPVw&9H5&!<)#^;a5j~2&o}-_x|k-1rJ+d!qvBvwI_nYhm3GhFeTjzN zhbC`N*3#3mGJ8t;Uo`me7pN`uy?Fz*JvxtFTo_MwXvlVOYN#y%MJKSCza3X3wJKe7 zgWu5Yc`mn;MFpmv!_ls5*Y>047#+3R&L!; zP>(uYYf#tn3%wHAb8>DJ?$y#ZYGJ&5#Mn%=CDMCO1(cG1KfyNu zf_Z#BW9btpv<6mdwL+nj&e##CQP5wj53Mdd6Tm3SfaYGhyLY9407DF27Xbb^^`@&9 zOGBHY{F5ghP)lVs?JHvH25NB$Fe+l9i8tT4M;cN%f@&EYKL0qGX{L>|+|Jx(JB9IB zr&|hTBaDe!D1exu&Z%3vvkfgA&=iJ9q)dhQr1vBd%mXCv72uSh)=Xkft7PmNFjG?m zy~z!Keg1@s+`!0?0XQo_-Eof9EF1wMz`{UNL>Lo~^LjFL;;sYn>Jj!CPWjGN^TyLg z&QoVsU7&ln^;e9p`8rq-QO4rY5EC}DfpnChM?0^D0(dfJi2*7B3-l~Ow~H(gzXGYt z|NaJiGB+v8W!(T3lu_p=l*xg)K@GT33bY9;L$eIx5}SoALvefNxBKlb;EG)rM*J-V zKb*L5LmwcUz&{PR=?piymdHGu>!r)a5RO@7!~VVj!9Xh`xteXiHNqmp*v-_(*ANd6 z+OkNXNgG0GoqXk0VEd#%?&#biOd?u&*;M1O#E84!id9ZGX=Zf@`G`&X>AJfz*8lX|j%D{eNy{iI`juBBFm@c{Gmh>gK$JdUgxCHq zoKw1Y`L5RM=E-n4E40kaGUX1d(m)6oOFx}_Upz?$I3gWzA{gHdZmPzl{uhO!lHLAQ zC@~lqVCOA*=}dvpwDW*{PIs1>v`!n)Q=T{)%X(cnX#o9a`Ov07c;b$O1a>|4XHCce`65S5P((dsW$$-D-qJ5BTCJxl|Kv=^FFzR(8*GuOrl0~#3YE*ItxP2XXB~~?Lwv(_KYo^`KjjY`_fzS^`nO4+!in=xdsKbV) zT0zsVQBwqm1Mt3AfE9%R?R8+(qlXVKE92IZdKrR&HIlbB@AS>ILb2eO1^rlIMd2jc|K3j+xqn|T38p?M(ZH&@8m_#^Hh$<=_D1O|v3R}LM6e(})2 zk_zM?Bx%H-L9AWM+q;23GgResTc-jnmvsqUH!=u-wka(4iS8t4yi$R2D6m~`2_qs9 zK>wLgq?kgTm`bN`ajaYF7bw~}CCRA3r748sQR# zNBT@3=t=;v6;w9S=hAWHYu!SL*FEDG6*GFn{H_T@1aKt3i_ZMk!*_^O)dk)!AUHUw znP=k(z_UKo=)R5*A3pqN%GF94b$>98w7gV&e$Osmss~ehMd#B=n6o|fTU)_&>4C8Y zS|bD6L``-@&Jt=f@5&yaH{&&oS2Bu$EyE5r_4Uc95u_%^@Pz>nnH~-}Fw|c`pWX=f zAZnn1$_^z10eZ&Gdn17PI!nq1A;btsE$DLP zaj$;jtC?${Wtci3Cef|w^0A$2WC>OMdJ4TblqS0`BwsFAZI1~ZMSO~`z~ z4KpnO)Mr98lFaW;1fEwuc?K6bd|`pJXgz16#6Z-;%tT$uY6w_|{!1sPT?T7o)>+6EL ziI5{Wcmuw8l7`za5w)NOL`IUO*5g#+Y{4wb&H*-j{ovnQ?>=lL8FFfY_J8!V!|J~f&%olEpod0E!-`j8GT}N$^F@7i}4Bi>Ady^ z#&hRx4f*H-2X82nJ+(-@~z-?`up=?BxkQ|VP=zB;r!1P z8tnsDg0t_PaGHQfpne#%+o>Q*H6As8AL(mtvxURtvd{= zMStQ?;SA$dRUIx;&i-M^p#IER-W|5nd_?Zma7ogU!=$@7mSlftK4$fa$j2U?n2#s* zpU9h2%|= zXtya;8ZUkx7g1acQm@x4q_=Omnp^+q^~sy@(9m0eLbWZp;GhMXepcZ1M$pB_2~Etv zv1KtIdH@#$p~D!cb{jVt&A4ZC%5%`rv5UkzkvIJc|i`DW4cm|~gkfB^jt za9hF9qc|`QMTcyr$4q-i?Pb7>g+Y0HiXX4{a(jDuA;$>=^Ot)m#&PMOe%nZU3#sAghD$$3AJ!3P*Yt^G@Mrhy5If0C@BQ} z6co|g?$y_-?l^gg7WdD$$l`4DgOD7hTdBF)?<|0E5QTA)6*?60-Yt;}nxqz@!8(JM z$)gtDdbodWg zLVXIDmFNnHd}tE%A;MYr2lrA7O~RWOTe1$1e`Kv)=EVJBU9O%jk+A ze1qq>)9^ZNHl8|KJGej7B;SaF<8#k5Q6V1&%CE2M&ZdA1gf?Q-E`}~%G3+bKMjQeN z2K1N58xftAK288P$T#uW&eUEYtaJ`o8{8+qK=&yAltYOOv^wNNkOtwU1nLN(P67Xg z%Hz$961c}?t$PUG*^u$fau4*ze~1+k8BNw0{GN(?aXgiZDR#1z5B%40Yi``vAA7vD zDRsF=7vD3G=xL~7<@g z7+fs?F_7YweB3I|N*NQm19N1(e{3lVGh_ibW?|nlXKMFR+*58*F&~6>BIs{d*6G0Q z@PqG*Q2ACZMaR!_mzMgG)?Z-J6uD00WU zZjMk;-ae0ek~#Yzz4#l5Eubq#)U0grtpj#`j@uSOFx2W&0xssnUbRPjm&4joDS%(z z3}hd_m?W;bfpQffVbEGu2>ajYEqAmP`nanyq=9c)vl0?g^i=z58CFjIJ;52zf%Qv%b}|5l%k%l3?qHCrJ|O2;1KLH{J5-6}3UF-}O3Z zTbu34^J#Y#9naGSo`tA(`t0z5y~9h$w@4 zoFGJ0y~2u{+}SQel?>uLjs;Op!$%N2)uWcK_-ZIpvP=Q4Tp_~U?1(oc0Re%ZtmPmN z018R^- ze~BDEF^f;N7KiQzY%CC(;F_mkjzQ=;1+K$UM-JSat_{$zg(q@ve(sY`ga5*^G_qXD ztsOsVyqD>L&}>D$wft&54>F-u35S&uuE zv|o_8{|UlyxZlDwXQvoj3KwdCSI%kONtI^G|Q8` z37`ody02;#vaFkM#|QYdtj}}M9heH~vnLrB+o<(xcM!X@l9CeK!T|jiOJ2#ia~{K) zYLJ^UPODB@L}v$ktdrXVIrPBD`~%kKzrV}K2HL-jx~ib6nhr#eo4_ywi;kV2L>*w$ z8awL>KTqwFS3eC8R91fDZOTv?>noAVeEE_IYe{(VCsankKLK#(Dhk}-~xw&3@*unmd^O}^jpGh5ZDGrM5Hf{ z)XV^$g{UBK>mUU*&(Q(E;!h*dg`6b<2+ZJ8D3rGs72QK>>M$0lAUS9o(k1)gen8YMy`Sd@JuT|zguDBEO$z%>Yv<%Q`7hNHQ<&KpXd;dr&)0e|BG$lA znSObS5$<&O@b>hTSk%`D4wc?=5Lr5O0J`AF!=9l=Pq-`3FCM(F0qibn6tzNKnjo;G z^_0lZu$i#Fn+yyY4ga9a@pC>d;)rorvdi14+*#!y#AH)Zik;2L$tjD|1rif#KuiV` zv}AW*Cy@`jK+qLzaKW2Ysm&a^t)qIP0Q#GZpkGWxOiT{`M`En~9_>Skb=1U(yH$@7 zUl+B_Sc3<$3f#4klmsdny@3JoWUNN1F*YVp77$&sYha|0MduOq3IVPf2&^IS&>D!{ zwr$Z>TnsB=vHSBB-Bl_ASOIKIzU73ta2s?ZX>0+L5M5|z1xe=&Tu{W-DKYl?2)EyO z^#7^t%fqSM+qN~^Drtvyk|I$wB0`3wSjvzoltvU8%8Gm~)nu&+Qr90cF6%u~fIVPGY89BTwh2w1*i@B1>b%R{Do1(F@?kdcWgxhgP1IppX#L8-SFnvYD3gn9d%h$b2Fq7X0_ zLIy#9|0^(kA9C@ADK zW;VoqdRq1OyKJj)I5qlXDig&LLK==NLoll(Lf|xY7)bWli+Kdlh3{EkQ@RpF=Rmxb zda@VE>@Q{))HZ&&|MoDX>7Kcq^!=7P!_4fB?_Yj$zBxV&_6|M-c_r>>Kwj&BCX`v^ zSu~5o%iy4cgTu|+w{?Ku?zC=Krqn_dXbYb;7_up1Yp=nNpr-a|jU7}wikJfC8|~+J zy}@QisRdzhr^BETvBks%XlgIm8&3QBxCC>Tl21}p0k$l6cem_+Wc7)2;&dK@MPf(h zY`Ip-_H2&DBIsFR9_N3#61rNksGav!_Nd#F%xjqD;y?`A2#JkYU&g41r@&2OUHbz1 zsoHPuQWv&cH}MwpQqvRnv1o`%0q`TLt&#c=5xl}sitx&0$39-uXk0|DO8)K8T1wkv zgQnI$_AHi)0+Zvuob)^ROAu)rMH9qC`#fRVC_FaBmlx@{g1bS^;zgqPP;C3Fi@t_M z`}PGNIhs^zYI>7z5pz7Q?7p~|(_<+TpgNtHgyQ@tk@1D?RBSv+oUPosQ zW$!K?7i%$B#6!%|E0x1^iV*);Qta{<&g>9X3gxblQ2d zXsy#Ds*o>WyTuy>1rNU)nYTPghK_k{KlI)=XXYyR; zE(IM|eb2s57NMvMQeni@M0UJ=1u)W#G&6bt_rPDeb<+}iIiaKzVWEbW*|Lae0v9b? z$X)d{@r2Z2O*XGTU;(!WXOx_!yq4ko5h0+hpex3unbG!#b$kJN=f7=A{XCY^T=29- zZuINB=;~|CXZcOvR)uvO6V@Oo;(M1#i>IKe67N+ANRx>o8!~TVPDL20INcqam+|b@ zBjjcNFXS)*|1SWpz1X&i4jr~@0sJKK0j)TO2()w%>~SLy*`4Ji9WF_K|UM${=^v!WuFb9(x4pM3sU7P32Dqj#xd7Mg~mjZA@*% zlOMt6M(}+qbj<6B7Yw}d8li8FB=*f+1;p!)Y=*=gfEZj7RMxhS_)+pW4ed;AM>NrG9k3+j2HWB39EZ=zM-Mv=JMsi{S$=4%hlF*V9BP>$>Gi%Mw z#BMhaFh=FQ9eWQO+6BOC;`7r17)s?oJcja+4zU65jmPvyy#2HUq3lQw&DR8mDbes> z>LIlbV*h|K9*;V9ZH41b!>xhDc@0o}eaKl7ljDDaI`o{;x2$KS0QXS{*Hnb=8l?wJ z3Q7y9*l!T*&CC!Rx#hJO+WQCf2K||8Z-8ooE*^_9!rC|hZ<>@xeeki%!2HWOx)rot zVQ_3=e2!!#7=SRmP}aj?3NNQ9DTHalUF8X|pF{|*&mv*r+I9X!j`u$>*ZBEi=4Uub zG3BPm_7aNjXcnq5W*8WPlp)C>h6CXxyAC15!{Mw6w!R&B!{-pKlKI9@i@#vfY z#T8!F_}WJ?uD7`y3*(e+yu7ZxyDzPwv}iTaH3MiGu$e@5S8gGF;u+J@zqUuToP!Zh zY>86^%$d_#rXGN_{tlKmG{cEjJE8ihsPgQHlcvn})nj+)eh2^FuE+jNnP zR*xqoC9N*Zrp;n}@?7GvyHN@hWoRR80|L ztMwPL;nKDP4p9g5liO?M-d{Fur)M6EM zb?+{D@olRB_noV(h4&pgR}mBx>J4JYORQK)Yz=ROiNcx5!m_yKR&-zob zOb)v{+4*3KMl=v*)zytiIaCwpDK!j&{9j-`=iqV#G~>ki5t&{h!r7j0T{X$@G1JAO zd4nW%EjGt(V>fAs*g(my>-?^{EN7!uFq;x;KZ%>k z%Bvp7sPsS|Gt1vqcUk+gGPGYCW@ihy*`7J=83@b-G++orIpeD1uo5HgRI|uTtsYoW z?$GVMCYv;>)tFobng9`9za2X&>(c zB=0boI6|t(gjYeNH?_VyO-Voq1q(6995h`_##-L)(N2N=Ihke--d^AYBrWva2$+>P z5A%~_pGXGEpQ=iwIX(`FG^9-qv**74{A)6mt-fAg8LYRI?@$fz?UMHTczVvEB<@w* z_15Aq0=aS6(g~SNWj$ICJ#wG40zytZP0vv3l<_{WX&d#s9RGnmP( zhQTlg?T0}CS~nk@A8X6E>t3|^wpi+OvleHVfk1d;9X2U!P<}5yY2ZH_D1;2qUg}q$ zq%TS`XSmZaKs2}WUC`rPKITw-**wuO#0L;L@QE$iEe}WtOdSU2fRvea|C^wBdD6_} z4GU%Kyw=Y8t2fv&BDAP^`rzDHJWZ0!p~_GPJC}zTB?DAZmtFXqJSZmjLJBPPhm?4H zo6Q1BOzN(e2ln!$dnqUaX+s>1ay(TGYAoWELydwZeqT$DXhuElF|zUQ=JX0>^&Gu- zL5x|qrY;Qa5E6-oi5ND+a)WeepwJ5H!^6X~5iV^> z_1?Q5VDcZJ68AoMFYlO52>zJb%O%0P^eX$XZ7eKQwB$eo2pWPhiodz|kzdmh9AH7= zZeKAf;Sdk*uK1Y4*N zY;lI|YguJwJ$6UKRd%%KhpizAG{Yp_RXpCC8&+_wy<)bp`@S{I&4_t?4NNBC=8h^V zk`4G~Y>X9+@tkc*t z&`*>^1Y7}Cp#cv{2b(yGUBst6;LuvyWiKy!rOXu?g^SjN9jhkvJPP`><;t|Yy#Kti zC#!?EMQoVBIZY~{|JwQNwfWzkFD8sCf^EQ#j61y#j3W-}bK0lf;Kw6I+PAqO;_#|H zJ2_D)ARgr(pIt%q?H^)5cT(D`c$PhV7g?y~i|x^@iYaai!N}SCC#`57bKZUCc~R2- zYbO@$>@S1Q66{_KaKg`)$P~dOeNmvstgFF$`^Bf4$cLPJr1Wuy*BS>J#$X@`0Sy83 z`(~)uh?xvwl!1dmB2RLdBTg#r;v#0fh}#KFf=Gtgg&}s5wIE_k7`l*%2Z1Oa@X;Ux zvX`MMvk|`|`!xZdf9$4B51r#dzG4UM6@0E9D>{p~g*2Hsy0wuaH})GxO~~d5d^t(l zNmQH!>43HNUII;H0{k$QK)nz_hzjtiIxIYxXmOf$*F?T=9_?$6CC2LrXD6We1Yu3x z6nqm$_`~H}aBe%6gxs9wGFIS}LtH$;TIHsghLT1eww%ylDrT*$M@U25Xm;U!t zrO$tmB#pR=A@7{$B1=Zxt*2Zr>H;T6sT~4CHMv%QM>YrSEU0ZjREA)o;|a^+#WKgv$>73jA=!#(otc5rf5``KeuHp6$BLK=)g+gW{Nz@Sua6BnHgEEcrcUtw%+hC-A^E!q}I&@w!fj=ht+$RDSd& zGR0!pW0(kN0GayNK?hz@Z zw$Tmj{2NbT%6Ac<6(rsvHmUsa^@kR3qtv2`S&y-&)Pe*1kRWTgKtEKn-~vrfh)c}^ z8#b8sx8#6ic#kw@@%Lw!`SJGj`@ZZA@lZ;m^cqY+4U&D4u2`h706d2;j`4g8dfxTr zo_W7JXK-l7e(=@mN@3>ZnC~$l^Kt>sv-kfS>Pnac67yzQZ7#G+MjTuAEhODO<8-TT z0-h(n;oXkrP&EG_$rIWWz=rS*%5x4aCA1-o;NwgjNZ|8;ig5$&U?z_ZwSTxri(+yp zlV^=%GZq(4a@ym@Z|IZ(qetV(2VVr!PG?o1BqOA#A~!ScO%4&9)Z|b>jbK<`nI@xh zGb?fBUy2xr%~p9m1-nDzvC9^5bl%*$2m08pL7dvd3`Q8>DQ|~K6tG;M7t;ieS-nmG}Z{Oho9PZB{T2Y}@qnMAuGrigdIF`JMr1B8mf zh!G0{?Ej>T2+%6+qeqYGGb+E!Q6hs0{<^NtO2X%-sn&U~f|F$msGXFX39lRHcmiN? zYE*sV;a;3##66rw_~|GMAgpuJbVbL2H2y;v{_sIlx=#LyE2F!zv;;RCw+|z939y@X z11iU&u7$T00EA(YY=T4KjbQ|-1D4C%ew`ko_HUlflgw%DWJmaaURe|?;VGu9@mwwn zxqf&syuqPMh@*fRSxkND&;IHKv>H->xPMSD8Gmw3e>#SpzIf=yUvoUgfFbFSI#>Qv zL=|uu=aRHGQf4G{we7_v=hQ2M-?v?JojQ4H=MAwr+qgr4rvho!y)=o(2bW;Zo58d% zxuGon;+y1;u{HBb!0aP_3WQYvP{i1h^thM?2cK}(Ss4@;#Xo0!Ua2ZyTQ@)R{CRBT zc1U9hu0z0Ow3-6GIEw;nyQ?#E|T)Qv{QwdI#1p zLvSa5<5AuUwS^26pZ9a&`R(Uw*Zsv%&$!=kRc;f(Co=rVm=s>}tR1l7ZcPcJ1g!g$ zArWHZyO`HHs;EhoyuOa_*YCb!!TqME|`k zy`TH5o@&Lr2~KM;+%b1R7J2Dr`M8Y!gcGm z|9&G`1{>gv<2FkO^A#c4RB0bHM2cLq?|o@9^7CHbV*_L8R^>0I(=&;P1m#Bt0coc_ z&3U|&c)Y2Q4eI$Vob&6MI-*0{&&AuuLPfXczK$|uGi8W@@knG6E zy)`#h!;Q{mXk_t{vOi4mX;)@HO`GhL4F4SYIR6#3kMQc^ej4eB$^G^Izyrr^`!{D7 z^Lf`NME#t!TTmCqwr=+F+n2>^&Ga=?UT!%*WHMsRzx3A^-xyQBm=+aXdMqz%Q|7wC zM~o^B@2PA(r#rXb>gl(&LC5IpL6_Pyxhq?QsCMmFDid%Q6-e@o64mS3Y+`QX8u zB;768&fHJY72{05_s^U~B3?omwEO;g*Cbtiy^w{o0_}C~7a<8bIzIlstv$p_7ake{ zy6(zF3}Lm9U&-l<3vRG6ly&PAvvQ9H*JdUj1)b3%v#j%bx0PjflQFNtuZ!8o<9%1m z@~h0nU16G*DP+T9SOK}QJtkScWunf)-yhLS*JXm*nks4=tsn)Y(k{jhDCyPuB2 z$lvfEk21QDFAbzI4s?ONXjwSuei zH`=+RskdG*k=0W$nu-6*eF!vTtUPs;N<*0z4(uL91PTRox2CWfSK~C2TjK7<}UA|<>pNPtt}?D3i1VRY|!PW zwpPmuzv#QWo2RtxP}xK8GOm?pE}Ami%#H{3}SQfo^PR2F`M3my>iMwf%REo4c$xZP86!g5|+9dpVdMFO<3d9eg2s5>3vX-Rqc} zrtZk-Cceb{U^L!!W8;e|`g?X`bWb{H|NM_%KVfa*NMV{CZH11${+FCB%=2dl465N* zmH4{X46{up?G4ksw|#xq9kI>84w&}qjnj2u+7CUrO-(!5uicn6cud~13@Bp~$W!}yk4EZmUyGGFUYstE$l~ID+YyZ8e*3w4?9Rg8%n127(Kc#o` z{yOS>;2KF81QPl0Xq3T}WAM_!xcR=b)}weDrMihc$Mf}4IE0NxF5wS53VhgE=l*&y z@NUyDE|z-YoHrVJ9V^r=zAo%PHIq)35=-`a%lx}wWFnF(;J|@pPo2SecZ3i_?80Y7(#?=G9tCKEie0;f=>KC-Dn4Iu8Ce8zxk&|U zvzIDx&42dhiyG0Ifn7wkwnEv3w5A|7Mx-c59Y)+Ij1n@Teu1u#{QnJrQZEvw?G|w= zo8g>**n!EtFIWtI^%}r!a_tBDWdebr0u~|EQzEy>gufWT=F9WAD1=xIMHb3`(j$G% zhe0y>hg7Z6*2F3VZtURy(Xa=|-pHGX&AP02I4sUgChGqoJ!MhezU&Q@Pmls=27d%6 zhp#-+ewY2FEv~G)D~^@^&-RcKD^Ef1a*n}i<(U*mxN2?MxY36ndCg>~x;lvex(vVx zy_oXM&{wKXKw3JlVyB8#A_829D({T%>VU!BEx-kaol_7jpxC?ewbp#ThW{`Wv$?my zw1l-Du3e;%U>2}pn|kjDSGg!}G@#?I?JaVG*_nIZ^7Z@2ZWT0v5(SPh47inm?8+Yxv&8)P>A2A4fpA~n$Qi4F_YaeV2J)`BRuPxRwjaS(>1(sBJ{m{xR!7+ zziY0}`S3#zh2DEPULMk_iMd93#y zWGJ!PLdH-No30r`hqX-oLlB8fZW}IXTtxSP#Ek~SvOITvUqAYu$&+fV-@?E04)4RtH$blVt6gi)IU>-|6&~r zb9^Zf#!mEp&;4iwVRH^}32Ws1WP2mXD&X?F*pwttXFCWW$>4(x;4dSz#vVs+jZoMN zq0%Hm0F>Td*`e1wQI!myyKd7Pb|Kl-2}1yd!1Swg8dPGl8@06*UX&@&rkQCvQbR|Kh+8 zVz$IDy5nV?zCSMDgURI&s%Y%)i03K;1tIllGT_+K&N%j98Rsu%G`Q0SCE)!xUu9pfBU#uYNETLEwws0 zz|c%@nas7@*YH;n&$-^M2{hAOAak2$<{^~itX!w*C)909J7+acF1@Z0a+-H(2)!)B z(J*b&Y%M5GH<%VOf7+qeXVUPD?vxzye6#%nVaEGRYHK(cm-a){^eg2>4DINXzs}L@ zDRy0i8w1hadIBRR>(1Hu?w~nh9t@NUJ^<8V)MgNW7q)&|Y=9tYWDi5V_b%YI~G`lbr{ z+?+H&eL7{82>((F7s~>P59WV5liTOW+V}Y0p?zM5ySdkp971CeJ@B(*+~U>xp{`hm ze;)WBh>nWh;B4QrPaBL6mC$F2)ChPBR9di{-j_i8yP_din&o`lCDy z6M(*=bRNAw$c2Gt%-SMHb^8Pg$1vU9Fz%{{W_;!MW>J$Uq+3JXyuhy;MMtb{YmfXz5i~wxUGR-(=c`v= zYu@VhigfSUY)=+U9DN^gqfefHe>~B4g8c0R&79D?{2lzxi?165?p?Ejwn90kJ_HC4 zUF#^)$ggMZ?QspRsyQ32@#B0PiE+QECsoY45Qn|8dKZoQ+3Z|hnj2dg>P#O z3`EfLc<^@gfuTL6cb`xBDxD|Gjl=wG+ef^oeI|d#g~uRM=-fF?j*)}3xr13DP$?a) zS@764{=A1bKW!HgIxFXA67cj)w;O#*JfE(qrOq79Ou&e~xJMy72*;`FRkEzK?{_-* ze3cL#KQ-hJjii31y}(er`x3Lc2<;WN|M+-#IN8)hwB)y5l(hBQvib0*)3jCftb{)) zl5VH3!TX$x@rl~^9(knD7i9qRx$Y(O+YEX%d-7QFC7nDyOfJJV-R@*%V^)jEPp?k( zZ%Xtj;>){qY39}|QhX`oCI7dfq~jU?Pyg0#O>G7qGphNO2taEKU1z>Ndcm55nylv5 zz#}CHb3(zKTqWyt2~Ez!kxzDH7yg747IBWfb<2ekW3V???Y?xLw6_IUv9R*q%=Exa z75d%=f~jH*>MxN4<#o`cAXKSkrYTQ8-z5I5IUro8qr3G{$a%<1Y!Gv0-l6?JP#8?W z&JT0uj+F}o zr=>&s!?+rWGh3X7Dq9_&;-Sh9)W%PG{gKf9GyLGI8G*$cXO^P=r!1@fYy-s3Y;P``Bf@uN^< z$SN~ja}>?MR_!>Y*#D&M4zef#xv5Xf>awjqIQ83vmm|)$2DvZ$4;x$vQ$lnK&FiKU z<=ZM~q5Rk2^eP;$v>Jp*N*9WdYE;tgPe1?C1+b+WC+vjQs`*D%qj@mlAmVusM5jfvv;iIV zg_DNliVueF+5vw50qSP`G=8MzR?Z*LBQXtKj|3bhfET3W$42)!F+E||fUfiOLD9ZNMD@$l1;#yX*evp(G8lgqCp{K^sQ zRwrFPQ57ybC+L1MCJRVw(R${SObH%{&s3uMOMg!mBP%dGqW1kQQ>Z^XQ;m=_!s$dG zzEFWgtVIz)c^7X&dotPN)5J_hOhO_%<|{m+%h2>GqbOQ!b=uY<^FnhLaV5bpTg~JH zZVA6lgM0{kSD~VoTplr(JSNN@8#utXYlTl}%Z}z$8$*7Z1L(sBHW>mGmg@>?k{$j> z(Y1D!jH4}jhy@T00y?0*9{b+r$B)N(fEB5v6LgW)kX6)TeBfkbg#MRg^-$Z9(pJJ> z+J_{PN$FK(cy}Oa5B=e1zr93G1JsjJ6y&UYGrQiJCm(4TMc-c|>_M4Z7bPf!W766< z&U03?`Ub9l2f4!DpBrV4V-}NTO3Haoy+=Njfu$#U>g5fmw$&eAVV$r$0u(sH z`QMBN>!55~!Pz_rTNw7FwmZ@y#%2CaF{``QVW5kN!-Cv`{s@Rm%7+&htDg_Eq|NDG*rRLWi||;1>a0>=rNo; zJyQJrN89J<-Q`p5)G7La$t;1zwhx=#7Fzzz`X25yiaue1oIcvyjMI$gXs0LeA>lL~P^Mgz zDd@rjFNG6ZKuk=G(~tDaNpQ-YyT0pr69 zv1f{sEhSu5hMXLjyupA5q|wvxqQqg=5KF)rO|WIMez8=-ymkzL#TK%6v4X$9S?|$0 zvd^hWeTmwQeOxESU*$V$olxa`qk<=Z<9NKvzQB*r5pLhU-Dpd#qnT^&!-HdbxwYsMJzfob_+F^fU@1AF2&Qus&^DTJd`U@*)WSV^x9*Pgym+9pW=)Id4<~0cLwWUY4M(iNsoenEe+wHIFfSeOUE*W+anM^>pnLjcxdJ(pMOYqy~&+#A+dT?^I#uX`JmQU>`^# zNEm!JZ&^DeiB;JJBgxp2&tL|(9ul`zIXp2%_CVZT0D-tVtR;biR!S6(tCokoO>%nU z+>Pyi2&}Fsc*E>WxTR_=uvB9?sGSbb@TN>508#}cW@gB${Sw+EYj|sPp+RaXDg@+r zs`1)o7n>~tp>d|cVRvLFx`}1Z-LvWX4rDC={O^R6#RU?Lps5nqDJ9bE{U{uAGPf>` z2nZ4jIRS}<>DjZdaRj~#J)UMnj=-#<(IuBaj6<)v-gfE+Cm#BxNAR{t?iKh`e?76mxo$U+2|0cps|l1?^wEYX>u1j zjs|W%mcI)T9|b&~L<3P9ER9gV}^w4M|_*G(F zF`nPT&aN2lYnCARUx?}}EE+Onar&5C8^FH~hJ2@p;0wIDmt{5t$f{EztPxphXnTkp z0q9Xa)OtwBRv*(HF>e)4(GWnW8Pns5g%gPlf&$pX(xl5hRciwVx{i-MUpWZwO16g& zWe{C_jnU|SE?1A{;Rz=+E9_mcqxUMMTJxmMCs|uj4HLf>@Pj(As|}JJZ905rJRHqZ z{K^sAZQXV(pEQgqhakYo0mYx>ws1hM)`+UgAP&=UK;w`OoCSfaSFbK7aBcF|IZ|($ zWJx>satO+|kd}=dAK-qO?5dljcPR0I8>?Mbb#utB3#Z0!tiRJ8V-|T%s<{iiXrX1$ z6z#_OHVDPPUIF%vGV}>G90y@1rGA6FCtd~m)i&~~DZlWks3}b1yK5c0BL`WfziLc} z6pn`&7|yjlTYQtFV|=J9kThde(=5P>Do69*Z8QD$g~EYo9*6CTb~n9a5Ts@1Blh#J z-qm(%5Cp+;iI($wb0Q-ncO!nqhlLsC)CmBsd^qGOH^qymevAZ~IK4#y_?ox+o5Hu# zf;tYyXeY>H70@_a;}|;wsWfSTlMaz&L(iWPdvVyTw#wguku7a3_>n0dE{eV|p^(S5 z>PwlYRsE10R>VulhL6=$fdn8JfwBy?c;|wjeO6tFJ`!dhK#b~vNR8L{qCv#$CU`!T zn8NpA@EN5W$h5{01ST77A6Y!%=z~h3A~5a5`B^>y^*nkB#f^-BquznIyqbIoiDts5 z+69NMt({s2W$yXxMzu!vGGm)bk{p;-rm`USX6m7rX(Jd0*Z=z;-FOlKOCz38rRF2Y z+QiPH|LDXX)pK*1x`mLH)@*PcwRa6vi1fjqSFc{(;@T+$O{@uyV8dR30*+BM5T`qb zL}^pyg;NI*r;1v=&nHnZ&iKPeFYHS`FpsJG0)g*xlThhEY+U}@cfF{~)EL5otq3et zs0cWZV(qTlX{JpQM;z?yApXP?;4p-wrG@aqsX+G{KYp-mUxPVI8fteHPWKjyI1YDnV^Zrs zKDDnhWjRwsFbSt}zoL9sj_osuME9V&9z^_y4J;hG+3kVH98F!mdmSeMH-#!UnJ!lj zlJI@>jo!3~`@Fp6lTRK?xWqO>oO+mR^=dzUkY7*to^kzh;JDdPCd?%}m+Oj2KxF_5ss;#De=OxNiy*Zcfr zd`V=|;1=@~tT-dkXvtQd1_w0c_D0xwV6}OuK6Nou1Sc>t`9SB-E5cu&k(+GdFJ0z3 z_5xWnPY!Z5vTy__WWP6ume}Gw3EcR7UHG&uG8z$_uE=y_ zSPsu+(zpy9BZBW{G5RD-ekI+29dMA4fECoj4!3!%Z$FNa#g+ZRFGhmKE!&ncMI^9vSmzw=g{P%yq grvK*`Kbx7yaoAxK&)%bZ$wyQ6sVSx?oVfab0Cz^ThX4Qo literal 0 HcmV?d00001 diff --git a/results/random-s.png b/results/random-s.png new file mode 100644 index 0000000000000000000000000000000000000000..3badd1f9a9804ebe9a5ce25069453daf4321a34e GIT binary patch literal 45942 zcmc$`1yq&W*FK7gqT&$&1&O25El3MkD4Xt*5G16#(?AX_8xWC}ZUF&l6#;1_ML>{n z)7^RJBF^uBzyCM>_q$`XsfDenk?!C2`qnl^7Uo=R zTx{1_{(fw0YiT3E&TjVa1#A}9hU|YSHkQCcPFjko+2G-w)j>apQbkgX@bL16Fn4b$ zIz~~>R+Jf79~BWCwI#??r`2|KLX#s379xVF3RDnQIN_0 zMNA@hNHpTsw?o8tc}Tp{Sq|s@6UV|7T5u}KR$_YQtZ6dl0{P>}mp<*X% zmp9my7XDhji!Uz2VBk-vv4I8t-d{Z%uKFC={p-LHvBACHt>R(%_Wma9Nkh2%H_x+w z|Bo+=le`uo`G(Kx?SkX)&t9Ank^>1??u;RIiR_h| zd9qt6GO^j~OA`&hEr~52@<$1|C||yOS-srZ&PNCrma17=_~JB~d}d2)q@<@5?Z1n5%?99}zU}+_6 zXUbQ`f12ck)#xl%D8KK{*xIiK-FPeNM>;@bJcTPmxRroz(8^{7J9jG4D-}Ju;bzX8c(t7CH1P3I_)#;f*+r@w84^yLsI9{W2bCa`W$fLp=|2!#b6Rbx=odbY693@qB}BF zGL*B_joVY@;$us1Jv(mP`yrQ_ zMmu9EvF#}`)JmDEa!;N-K`*VGtr0OX=i}q!JYPp2-4>p%l9RnLQ>ks(92HEXuc!C; z^YfF&-`{$DuBsv-B6-{adEnIM!ZDPCzQ2v z>G1pmKU{u`u)@aj6uR--H(6L`O_!BvyD=`kuYW0}%HFEnocAlSn@EPw&X8aK>eRed zltSoz2h37F;a&jo#fzz_sSHA!vsL0^VwPh?W<5OBH8tthBlQFX1oF`Wc8deg&T1`J z=;ai=v!7DuHtCc^FZJVHyzutwkcQj&^XIvp7aw<|%H2`UP+A)AXnSwerjhO)NWp&v z{7JZ$GX{NCUx}T5fqH>Oe}(HK)gtp~*uwOYP00i#Ors?e*==nyB2%sOLPDA?LMu_@ z9U1f)9qHuU#*ceHSc%KXwAhue^m}OY{&Gxs`s^9WxpVP^lmd^zbFsN;d2HEn8@G$W zcVD}9jhx>)k?8z+&GlaE23EAdFq0{0qkQ`GDIHcnMj_07XUkz}vReUc$k*S$Jw!%0 z!qC)--*xq!{mied=4e5jKzO;Zgie-vL46WI%DbV#%V-}KS!iQg zvx$87|B+c%UEf@2>B!TUICkvVSQBr5s4XMLuE(s15gZhZ>&zvd_DdFJ!4v$a ztG+F?h~&WgG4t`M7X5NI$)1Tf^&$&uO`;j=-(Kc5tenU$m1Zwr#4Z#V1a-rz-$rZc zq)8Wo@jgr4nXZ%(H6_ZeL|eJ^sJBbH&b_kxy-^62`>Kvox*|qLN2g?Gd&8#u?6CgI zbRRjV-apyJf5^eDFbN2#pA}wxm|dSexl($^n-Fmt{7f1Ha3RL zQz=QD#OuY2aWJ#anaUmaZqmDY>!Edul;x{K6bZ41&de=or!9Zx#to}m!m9&EG#&cv;6aLhq^w|9$r^58 zbZ3QD2cG%*`eHkbmB56NtH!N8t*HYC|5hnOd2IB1Li2Kz2=A$l{3|YJV&TOTuW@1 z;7nP21w?WcZ@c~iSvuE`iEE58CqxaNOzs;XfxaAv+aAamCM=Us5o%16N(QfooZxVgZ)2EyD6u!CPly~^>;l)0Oek1VT zOMRMm#pQ!rVmEK+wU6-PG=Mb-Nl?!_BJKfT^-!%OB!5%YZBwtBgapF zt2!%^3DJ5C{8@;tk-93FCJIIkEiE#!uJb-rZi{mK6odg3{H<%NC(qN3uHj-;ip=}k zvmWqH{m8weSz;@9@7@au7B9-67dpGMbwS%TKK_n(x>~*gaxvWYld4-V0Oz%laI{NV zA;#@&;zYu**tcm4DFWJEU`dO^Au{RCOXKT3{o8aHSo``gSEuZssd5Q=`Q6uH@o=w& zoEKBMxMu*U6ufu{Bbs|bl*};k*1W27EltjpZwNi zQJMAis~9YAQ#UT&o2nH;SPm=$S}*_#nPoIQEl23(c4@ zV6WhPjmF#41f1p*4O(Kdnj(28z*N8H%ZoGh{&IpA1&a z?TH(e$EMXCY_7|Sc@@X$A;4%{^Uln69={fFEthyAn=bO0zLuz5U3I*A<3?|i8_eko znxV9mn0mXuPtWi#fj76N^4MDHg5QEmgRpPVL>|7pv%OOJg60wIsAClsW$o!o4KMXv z!3#DQCMMD(04%II!`nMx8p)~edya@mcN1)qEyx2v-)X;UT?#?_MtBcCh@_Y6PMkW$ zB@NrH8|MMWdInPiczaF@5$eRTg0cJ7zU41Z5+TU!8_Zi2(i zujWG7CS+KD@K@iu?SFqhmY`Mc4DW}4)a))ig@0_WEgV}mX-^IQ4t~J3rE;NJ@E(Mr z;5ToKFTOAw(rtLf3KLL=r9HO%pv2awB8Sn1B}*-zuB&+RwY6_R05=M#IIHEZWQfbn zIpI1NYXG{KiE)A#oit^hbkwMDExugmxV?tcTwfY%tL2E0tO2WzGJ}A39S}_@ZekoZ zSoU(p$hP~dg*EGn&dHTXF!YOLWMs|DFin@R{@_lDS8q7=ZJ=BR#?zK8)m%8aBDk{B z2k%4{#-wCTHPc(%_EPW0`e*{%)^S3L*E1&dMzC;j^{1!D*rwoV_w@7UJ=^xkf6zOQrOy!KZqROqNZsi|VVX;pu@Ovw>gM^k}>u^T+ zQ@gKz>b75=R7=pU_APeLCgItBI(@WbOi7EuHAlCBukOgv7Ks+|&;sE6J%Y>7 z7DfkQb9~^_v*y;7AmMc}*d68hob)M^{oCp+q&Rg(fVyz60)$ZYLen%@1>)`r0gvsn z+js7qnhbH<28*E5bOCrOE-BgWdFV*g$`IIEo+^tm{4PeJiXh)}zBNgLgo-K~JWk(8 z33y#uF9MRNuBPp&MF<1wjg68)y5WhhUtbK*ky};_fO)$$H#gU;`m@L)=bwL0gsTcT zOtX~s#boGJfSr1!t9}i1tM(;XlU7kJu}vet{x}Y@pwb%-%j>M&D>`fQBaqR&;n2CA z%_!DAWVW@wRR8tsHvl!7>!$%+0LlYb71l7sq?j7#GE?5=MDf>Oaex@)D!11&^37nn z($x#n!JQ77o+2ddoc&$JzOkMp(c`+Yw#M%f>6)aRqm>1qxTE-Z0R7OF(pEN?H$!14 zCk2pUQ-IUBK+njCff<&o4WwL}$f;z4f41&a@F2sGuoKhr3n#U|k*!s(PU&@9mi?6Hl#LfEAenM_Ah%1) zMapBXK|3@wG;wVKX9Am%ZKThekP>cPuNB!2o+#>S6 zapXr!OG^ZCX|G)f7CRq+gZaR+ZQXwlkA8FA2#X*|A0#-4aDlC_np%B*0{o_lc(wNS z`~^&Sa+~P=`xCGyewMp9RQNnfP!a%~RBCd^C z{*`1Ja2?wHt_w{{@F+V>xYmPaJFi`6q0_$+NF|icWA<~{I7cN%D`xA|`JE0Xl!5ft zrrM4fVgL<8kY~;B&iwM_F1(Noz@&x+aLYmKhk?4HDxIsj6kV!- zGEB%%+de)yVh0=v|rDf-J-=3wY-@OzZZ8&aiGEt`R^RyT-F>xY5zA=cS-ylnR4@iMkE$?no zpM7L!wkES*^%d``c!1fD|1?H8X#waZk14-+R+Z(|jh=wRq+O~ze$wZ7R2ul@pXt=Z!C zoFknNEIOk2Z5G=VWX9f@WCzo%ZEWC*Bc-JQH+R6Q5lrU8Y@&U!4meH|ONY3N+PT0> zyv_RK63zX;`t7uaWYNh{|;m$n) zPR{wfl`rSfv%%rn!9-mc7m4KUvkfsX8Kc`;h}~(2I1c#w?=76j&i6ZKg%k&d@-i~4 zf9=G2Y=Z+mCDfVqK!uT!QFMJhFQUo7ZoD0X5*~*eKF#!#y(v6; z9%hhuj?NeGyRF-18$bITSPPNN2m)L&k)FubVn~AxsTUcS!G|wo=g%uV-1+C9e>4Q& z0zEeR#pk@!vIux6^YSHSoRzq^uXGgOZ|nz~@Ru*+Ame`Anuvjj`Lv-R8^A}wuq7XI z#UB)#i0#X8yf3WXElO0lsrW1h^0YHDvaW>quDCU8!iBb@-{LiH$Gc)sZ14FVqUXy)UR#6KP@BPD1 zhr``rW?ofRR%*E|DTP-!O`TBuG4}%=-92UB1i3&`3oI~8NZoL+PA0?q$KOAXY~id; z%uq7jMb3=;a}kjtr1Wev>=tEnFJNW}T+cZ5xiHSSsH=9jtkg5-Xnf~hxO4|-4MhOT zf10BwIE6(ZE1c@D+{x>YRN>_5Bk9`E&;n3Ia^XS>3`d{`_D;CPh`x@-ln?`o4}*h) z8lCUgzO^J6dmNvWylyfBt-!pX42&3%t1N2}Jle zgD(o&XRa?#Wx&cBfBEN}^Hf1sI}rRpvEPN=)P{&Nm}@3BHtEWp?NT6vMqGO#!E6We z5z$|WEr!vgfcrF_?kz6aUZ1eTMLUc`r37ZUyd5GxrLe0i1W_e;954j(dO4S`T)7Ra zqJT(;GAA1$s4pNMcW`K^!1DL)t@VkVH(Z8+>P6;Lu9A>+DdrnUp%fW>Y8XT&0LqAT zfH2h9&CbFiiKZ1O9bs;ZUm9B*D-O$QfJ)7Q{L`Smaw-9Ca$f!I%?%!As<&7H$cO26Z8Sv)g&4M$w<|ZzX>}`Py27n&{w6nFSf)$-iOeO@aG)OogX1xOVl?0iA zCKLfwU@mDez{|u6Z_eHcxOnZhw(Fc1jN1S#YiqGh2GM^@5E?V|8(tx_6XW!UOe%s? zbgHMQ3)rQrz)WX7C}INs2oh8!$i=R^Z#qi7xsE~f6Yy18NAL+p5%mC$aXz!PBnmho zqvj}nM9mhsZP+VisVhOOX0n`EjY8QZw|TEB&AJQi}g-O^51Frv+XMYHYF zbQnYa1^nhO=LuHuRwXn~GoT8F07)A_4rDs%4~op!;i|=M!`Kn>2f#wX!^4ro6Jw^d%k zu2$iRltRY*_;Kp(+qc_G>`Xe+6z;?|Pb=ku(b1ok&D zpqUg@K0wHZ%4PBzj&umB812P2xa?Hyu<^_-onJwxS|L$}OtQrE8XE@6Vb;rQ0GE>x z0S}Aa`A(;*{hiJo@Bz|@2m=Qi?UVSl27_2>y*i=*GOl|NP2?dhL~~8DZg>nzy0oQa zqZ@ic4?%H^Ll{g~qsQ!0O^4aF zn5nP9S8E^=2LL7}hniKhnOs+ncC6tJ*zhHIA099YfAkrbVRPe(Q|0R5sjECZohQzo zxnaGxFH+S^#{De z=f3$t=i&?CR0tozI^2VV>`X?7@!fm($Q^z;r8F%3p?BFBZV}n(ef;x%nofaw08HT} z;HH~8R=}^+K>aiT-fx%g8L1Bo0A7I{RY7bfh9+5SV1rgS2p0j@^5V=Te<~s8i%{yh z2iWl3`SXuXU*!0^i?kB>jvC1J`~iNF0sjWjfkXZMV;#vi*XzW@#jVE#p-6xx%5r() zM{Yu{d7nVIu+5*wYvWxx7ONW@m*nK+YM`pCOG88RVX@AlVXUk|8W9`Q6&vPP0S%;T z6z9!-IWNOk^pw5Pz7i(xI^^FkO$0|LSv?olL$=B}44Wws+xE4w(bmaSr?z@~Yva@= zmE{WHNU|;sn*g0$RShleTMw*qG>Ti%Je<3D@iF-Pp4U8PZ?1W4Z))`4w=eB`4H=$X zfw2;r;}0XR9{58Uclc*aHwOVPRsII5L*J8>^#46GW1-Zo}wHp;8-BW{~rGlzAD z;FkbfwRvSAsz;1nE$_?P)`MJK(MR9nZq@A8kW9#5;2z~eP`C%KTl42$F-Wz}$gBEYj>Ov{LlU2WN*}Iuqr(ivQ;LjbQs1jh3 zQ{K6@a3cpN0+jMK>tQjJdc1%jbB>bI%yzuJ4jHRN{r z3y;A}^2P6=jBkf-Uh8?M^9+?pQOO0WsW#>gT3WC$4+1Xnb^^mQJ|$aU01POC`fr5Y zN`T`U|Nf{HZb0Qrp#96<;=n53Lk5`$J1jr&F5`O$s6w+JkNhmLV^%sk@)6wS5-R1= z(vGBGAUck>O4a%0kY~dvq~J(h;5Ju%!Fk{bzN1y)qtR;N2G&rBPwWZuB(+`g} z#>N6wG7c`b6WpuJVSEOi3+!69{K78FnZO<>^LDF?n&0QM>%&p4=NI11jt^Stox-8GPu$Pz&6AqDWDt;}gbk23-x z1avu#C0-l_AeLR#$w&NmbpN~lMD={iQDC!+3h?U^BHt zZ3va)Ap3F=q=-}HXqKZo_=y6C40(=~JC&COdVXGtER-&pfI?~bj#qt^i%S6tzXYdF z4T;=g$iIkVID7EJ$;CzMLn~B&g6_-iXz~7AiOsArks*W_DCm?DpFe*U%Go0g;iXYK zfHgXjBrc$uD->o09i|)Y0syf?X-6LL015-`0RE$IIJG9VfKNnV>=-^iGvwkZKLsjM zA{KNu^1vj7LkSZ!EMP95HCmQHsFDT?iRzsY-BH~_dx{m!6ig>^4xJU7#)mlIP>pvuqSw9-#1aH2k>zHQO#+LJQ(jHl+{Opmr^!0&7?5`rhyy^ z98``2hcK;BMt~5>)D$bCEhDs~DAz}{J3h}FYm1~qKrtee0m;gwX@+)X1uA4h>eFZ2 zCgtnrrvQToP`gw?YP+nuZZ@3_PX_VpKG)$sivk zJ|8}Oa2=A6kx_AWE?M9M{<1ZR^;cez(~YQtjYWRXeLPDTG zeg;WGz%J)?^={79!G`iBVQx>jfA+uKP(m)9J(uH7tl1LLgJuAtD$wG#X*K-dV?ojt z?bhoMT>q<_$&h-5G7DAI0L_8wp-wmVt8VUaH~j{=Q?7beo_bdC&#kSkwaZU9gRBDQ zJ^%BQ=bx)t=xFDVp3wm~0)}7$=}B`T!puK%A0ZZ(CJlf)Dh_Mf2SY9r{rU4}guM)7 zqq@FMhqwj692P*nmlRa>`y1eaD7QUx^bJvAwt^HCQu@odIjAug*x;m*1ZN-X*-rkr zhCEbj($N6hG;2C|yWhWmlMC3U*7%bpz?6X3kVFhOl;9JfPKd#_!xra;6dmOX$smY> zYMKy;oRG#OU)%t4Hk2(PaEVXtDtn37ydCP9sjyJ|kC1W+mDNF+HV!4~_Dr>WTgXvx z=1O~Gt**b>ZKFC73!xn>GoaxubF812A7!7{$!6pJuRD;YL z5VkUy8*IZR$kYU>ALXhpf|ZOzEnF^=$BY={Kfq_1fV`|(7&hv6fRm^ejB>4Xt@4sz z?i*&f^vpMIXaH%iqw^?K zEg~WUC0I~bsM>rGs^)JKihuB9*b<`zuq_PuZLkLm?`IjqZ6^EL#n1~hpy5P+T=9S) z-Tgy-^WS*#a^L^X9{L|%=KgCh0O2_-FW~IPceY!SSm%?%^-i8ylE3S@BKNN<5@I0w zvQTokjusOrveV3Hc!`9cA!uJ!DSl>!QIS77;(FcMOu9vd8qsgOBmyWy?f&_1$GRW; z;@bal9izp_E4!98Z>`Okl7RPh??`vL$`YrzFUn-#jw+QVyO(YAqOUnyk)i9g^fB~U z1~>f}i+-=hqa z2UpYjHT|(}v}Fo)*q^W@Y&WaVP#h8d2*uv~-b@QM5g-ZUW56BUlcdp>Mnt4HoV&y) ztF=Glq2kkFeD(EMtKUx@o*oiEw5Krjl&_TRb#xT(EK`zmuY;a=Wd(K@1Ym;IXPcAS^WD2WER55}pi+7k1N z@DKy{{%qh4{Eq~#vt%!HzFP8?Q63*4ncSbp4(u-vZ8du7uQ9I~Ty)U+p%E4&gR2$gv7b^HXv-Ulh1sj<`zSJ+b_(k^@D7`qxrwUf$-+R=v#iNGx$m~mD zM$7*B_crYxsxL0zm&g9!e#_qh^Lq@b-l2=C}j;f_s$#xrjmsxlI7gF zc|YHId$F57`x;}0rA&`CKlD>ng;(Y~rnt9||1*_w?4ye>{QXZ`cpnO7uunORZe4xe z&iWv<#3`S`$0v`Tjww}%x%qr_8&%cRx_{oF^(HJ~=?SKOw@O29u&3d8o4Z*C5lTET`CVB#z9bj3tW^7#VIsvagK66&*mE;CV;kh6r6h(4zV!~j0B-G&~iX!KNv*PX;8Ws2oZvWMj7-uNY;dD4P{h#4_P^I zyY~V7u`05l5qS<8+Gl*x5r72)_)B0792nrH47i}V5VE8K^FDPeE35JT%1U5jLEjX0 zk^(c{Tb$o$kYBMeEeNG>3H155_-DX)M7crise1DJjfCLVZqAQBf?I`o=~@1A{+d{S^TXHbz=H2^*FDTbh+pl}*dQPJY}_eDkdTVTL|{98xLVq;^Olru+Lp~fXpoE_}O z24eu7K)72>Lp3%W1YE{Y#VG!%lBJ$BFfiasjURgKe}{S-q?L`51vVxdiy}L6&JW{M z1{tU16%@)T5BtidNezY`8>$U*NSnn#kqrzFWgd{F)cFM;s-}l*2%aY!L?tvlGcz+~ z>8bna0HN}&NWIWupk(#lE^ z5gOS)*Z!9jj-%O4rW&7m(f)++vwdjz6drj0`}glZAuZ2m1$5o-wh|4n>?#J~L(umk z8r3isa^(8omZ^}oHVXp)vW~$200x98QQuXYTbZyYKHQM8kDbIZO70pH9^NkQ3JoPj z`3_`lp|(bcegmgB4#*R_B0vZBKE`9)DNCbR9)ynxh?|3wbSj`5Y~t}VKoYO@Z)_Hv_ z#SSVC#vnF9GUIB@> zeNK^n4c-mkG>p#86cLT>jB%#>acP#3&cj20d)`kX@+z$;Y4WLf|Ep3b(XF$LV4`(d z<27@JqJ*-q_M`nOaY#3djpGN2NTJK0B);R(dhZ3_k2qCGw;l;X{KP+S@`1EhK99VH zP>+8QQEDfZeX8E@F`PZPL^^v+QE@Pi;vly07bh!#i?-fxzWIn7ej_bPOIW`Tn($(W zv>+sjWd1((cJFeuOj~IZw0Y`BrXXm#%CV9ET)*0H<<`u ze;>bj%G4J$@xhRXHp?i{JLBQ0TzThYv!;~82Cz!v?(@xbl@*BMxmWgMD&`Y!|K!wP zotRYzuKBq18{ZulOIqme*4fNR^MkAT+unx{}p%iAQZFY17?+k3FADt zU*Ae*P}b)e=RU&xXjm+h@)o85wg12$&lc-g=@Bt;>~Bj)`ipU|50;r_MVx}4!a7Q1 zCip5GWBkL_McV1@9n{O_gv7I6t;dgj4I86mXwbl)~oa+~<-|9zuBTDw?I z>0qUr+;)-Fe{L=A{1Oi%42>*A;kLxulo7Hq8Oq@Da475;^`kdm|(6RQ?}X~uTz|yoMq=v%Ss;DevAuk zAPpEr*vjT*y2AC8r zqb%L<&v;-YL$B@6s{YdrLM1{bE7||t0{iS>BQTs-J&&(Xxi`X_5d&I2LttiL0*vHw zqX)SD@m;#EJ{2icMwDye^~vFw69wZVgggn@uAUoBq9`77~7hW(xM zHldu7R95fc#afL@HGdoAAC{yueBkv%mHpmPV)kT(U03K@51-kDU>9f86iU8+u4cHu zpHZ$pZ?i8I7RP-5*{-PXBnS|8|7qgfSl*Ac`>&crRCnnk$^N^f$rIOIszReyTmEu) ztt*{YA+S2=koG|)!9*-NUlS9W$fb}{9~?@vRXl3d-Wb8@^_*BU&NDo8y)UgfRwM`1 zSg3^wNup3~%x*#%=TyaUUFY*rR8zcl>lT7ONMT`(#8XBkWvRfdff0jCE56h2TpX)X4(PC9Q~duJrIg zHCL0)%G{0dCXkO{|+_{=6bJxY{;wSc|@U4HbgOUX!oagW758q zf}hDC`>uiVE6%=b9fGsr+nbxTmTO_{9 zmjS*tSt>FaM0KKf?=rwAk81p_e4tvW+z`&*!n^%nVCzE}3;Y2E$8gUjpN*pgA z{pIADGfDvNuItwmBL*2VkM)(xoh%?|A%Btq_>4^iGBvyw@b>>R)IKIGqXp#X40iRu z;Ny@7hgSVT$D5^^dm9-6RFR6R4*4GYcL~ZC`Av!@hWsuR2(xB=Q^=SunPBrOrFEB| zv`=!|Dp*)8G45#3lZn0J-|hhIvb!8=*f>of4eHT=2BbL1$msm4ku94*f%IpETWQgq zQkIMkri_j^8GZfzvD8Ei3kTJSz5gfOe_vTib>_69NU3Iny!?AQZl&-@(~%h*y@y|Q zL6)v)(uk>;I8%}K(ENHyL%MwBJ1f{xkbh~TCfv%c^(!|tAZI9hTN`;5sHpB} z0)5dmnL)i%Cr`>jA{++dD<~Iez>LWPSx-z!nFU%bz71M>`Uxa*f_0<8nEiY&r>px` zVx$?UQjnRU(!F6UP`t$~t0>{y!|Mx?mGCRJmXJRB4JY*dc%m6kI?~6@m3X{#a=ZWY z-L4>7LU>B90UxNOJ-BJOR9gI2H-5L7xDU z7oBxziY*=P&-fb!a8D)YpGl$tGv%5xkxB9@<2-UVe;`3=rU8Gj|pjX_bZb|pk``b0M1&y z)IJNnK9v1NEj%_N=_Fqd^lE8fq0=U#+V6r1T!-X91t3mj)vnNhG>NS^ayO)nNJmi9Evk#SUPoSWui+42HCJu2JO;dE8I@=!>${OK%#?e znQhkT4UbuRj!mXkOn8OcMmp#PkKymRUd}T|4ioEfj=k04>mfAMH@fTiVI9=zaW@KS zc=3i%n(cYLvZur2t1RCxB#(yp8B;4#K zj`+*}#MKo>CBe3~u&@M9diH6BbP6CRIrqVMoFdS2iR3SqH4Wl*fsRyD9N*79LM_gb z`i-w`OvoA}=bO1Wm%J&Vb+9Hd?o;2k8~>`Ck7!DXByrkvOtP5=5xksKIYWX74Tg## zuCn0P5w%O#8MZ%q68*~_;)IjNi!xcN4fUNE$v+DId!xzL7RhvB^y;#RL)zg3Sm`O> zV#=B3B>W)Y$jAZmlCg}HUwY(R(#oFS!6*Cb03$!Yl&KtkHkS-*uboU@Yssf4BPLt( zyULV{Ft&DjkC%&;{bZ`V<%u`d@YS-=OhQH_n)jWcfb04JYbv9prynObL!&8lJLiEOym zmNqbP0FUlTiHInHY~g3&pM17J7oyr*w1ln)X-W(9wiwHO&8lfScaE)Aaqwf;!83{B znahTH479aBdAqHpQp6=2J+>&Ic)XYYeN4GtpgzK0*RR)lMma-WJZqNC#n0epPHmdHLPD+-tb@BRA2 zteTaP(K?Ti=7pW!d#+$k4NmAAWcL#`HP$R;GmL}{)y~V9Y{{yZJFz`M8`WPJOuzjN zq`zKnx3$tAyo&wNel#q2ryjk-gGv95OF3Gq8SEY|Jgt8F4oPeLv+%b^d`S-=z3)5K zc0roE+?$*2y~bfpx^pN;Jm6i8o>hG=qdY8oMfJx;G+THd@%K#+Z}t#0I-p;LI=$#) zyKkL0dGaJyhat{f6A~K7gCaoVwbOU7APfjypDGep{~)`b-&AQJrR4sHjQt+I2H8Od za=D7*Jv~%R+gjgq3u#tC@s5c@_Rq_|4551+^2uMPh&~V;WCNc?TB6&|Lf_L+*4UUa z2?FVVT3f8P%TAYS(iL9y4uZlH8rRK;Z|%l*{J&nk0&+qRl9{Q;7p-Y%!yfA0{C-ez zIj>$!Yt_t0A@C}U7BcK%oWQy2n3uBH_(P)m316IKDoh}gxk6%LQc8m!=SX^ujwZ$G zm4lIg(&J1^dcfV>xoC^2FJL1Kn|1$c6HKc{5idZV62 z@YQPa<>DP@0q|Pq4{YjmMqMJ^bg5!eNxcSr%^1 zcU6uTHcG0Q@*oo=I60^L-1DG_Fh&Wo9`U&*{@asDWei}kgS2BDz^0ZCz-*<0Kz1j>fnZvijLJ~ z3X@^H|BT*RW^mKIXd)=zFl?@Yz2t*i>FH#ZPu>S9r`poy7Kyq2*67B_{D^>{TL}@a zNujelU(5IO>_I&&LZ2t1{WHXvmVz#UohdOfhOw8P%=_$tY;v(ont&`E$g!g_`y=M{ zsMkN9%m?{vGIzeZrtqf08r0(j>}W9dOjZMT4%YWQfk8g`@uPe}SFza8Z9}`jXW|sA ze~IiLN=>KN_rG>MH3wC>9|Id3&b=S@iPSS@`3ir3ZKF`_MDik z`D-&^tk7=`Uh&c^=plPlySBi|$yH8#X5e6}Syw?N&-VN?=!6Rw;ZPm(y=e&zO$I0raq`NnrBuz%H=H2%-j%`OY-IP z*iyQs<$sbPQ(h>#9_F5UXVn{k2s)etsC^ZMWXWKii_O9>bkMOCsMicmr2sjlA>|Fn zZzBSzH3!NQ#H6HjaKHofK7#0}I;jPkRyr!&T;MQ*bkJ!*dgz?(i1aF;0b%CgkOcwl zcaQ=5;k)76hiErc zqF2#!5{1;a17$KGQv*RK6Dw;|w;`m$KcUqSlAjc4$WjDgidxj7aUhUDLLR`f5=i0! zTDnvaprBUpX5j~grr)6-5K+2YAW}okR?f?lX`sdnZA1qV;XxY!1B6EjDLbQ$%QRoW z!ebaafX9V03#h?TK}&*71S)o-e`mW0A#qH2y5;{LnI@*69F6uPi7^>*B6wC-NxFj5 zC*|7gr=UD5%K-D*R%rGBwKa+CZ02vzhgUv^sth`m=DN)vNlYsgmPkM-0F^F5>hu3E zQ^>%)5;z0_puH3*TYUfdM;4M!&|3@%XSKWt-#L#d*^q5QEsbe@xZC?OA$-azX$^{oE?UX4j$k+6#tINqNE0pYezcVAU%7!C#Fs5mf(zB4Ih@FZ}>& zkzYy9^e{lw)&w++Qkpz1`@Q4DvHz#6lVapkcfR3XZAW_0pHfg66i%V*JH=ZMK)GZK zEpwn~PPKR7^%CJj|1 zx-eY3CI`F8VYwR)8G@@(L&0P|kUZhVPmyRHx}s8`Efxel|4X`u=c|w=J=hDMtwVcL z8pvqIK*YR0b7EUUH zBbt^#`^wki?d@#>y+cT91ngQoIxh)5YN|R(LQo$JOtnRLBQwT*-AEH8z^D{U&TFp0 z$-C=1W%Y>q-5-d=@F(opPiJapsD$hAWc*xPD4}Ke+7o|hjj00@1&vKmw3PrGsV@rp zZ*#rk-CVD7yX2!V)UlP|WUNz@yN_qsYUGLB92#1ZXnNCJC^Bgu{k2|RsZxGRj%-8~ z$hKXf%T23yKTOk|2A_YHm)H1Nl$7Gacx7PzAaGJb38V2v{5^a)-@rmbVnP1y$B_e= zE@%)bg>C=B>a+0Z-iy}l-g9IOX8-GbG-EF*AV#D}u zjg6%{Oula5@2tLMbil7pzuw%^&z8L=VcaIVufL@JytqUpRV{Pf7F9)mxf&C6R__Cx)J1fb0p(Nj|8aeWV(M$XhVYW)_ zRY)h)GZ??|ARr-ht`v(8Ei;l(J4bs69X$p{S!Y~8YG=A@TEvl);uM&Y=~3q{Fyl zuq`DT4y<@VXkd`;e$(?p!qM#%Sxgj=ilpmLVWX4}KHQxo2F7Ghykk7(lQ#mfift`2 zwb1Bh28Taj2IJai9p~NvJ;B7y-Qna4E)f|vbVND@AKw+U-NcxT6&kfDqfrl95v9OF zIK!cEG#EII3mE7ERG0;NXdDhsW0=R=#l;Ma)0n6#H8lZ~A{R#Tk72c#C|JT&*az4| zXbNIvViJd^bfAVh031nQj~qpVj7w}kdqVb~y5ZJBkPrYG$i5*xn3$Z*bp5&__B|Y( z)o0Tp1TD!WW66==y1!pJ`xm;+6FX?gId;6rI0^|>7=ObMfrCYR8uazNs7DWV@z9{2 zcwqCH4~EG0qAxoWFtOF>OE&YKWWRooSQCKY{-^cxI{3t&+@6vSYeR>yfYU=Q11Q9y z3i_D5QZoo#Wrq*#CVa9Qtt*pqpFvI-1WCN{ueV;3*t|9#qZjezWMsyFUiBUle*?fxoe2HVp7S@yk!32mchafmHSWtZ#s}%)kHLCCh)31QJnkzbHMUza8wk$ zj6LjNM%huCW;oSGuRtA@w9#2h9$ORIh#rG5lY6mh*XrX|Uo$JdWFZab0Wp|59C!6q zuT@Gp|`V8lvszuU+iFy zqWT1MYRiJd?|^n9C}X6j4(-{7>gzr%AC!I~r0CWnC4Zef@zuNb)TL5Ch@sG1bR9w# zaNgsf><I7MH7v zp*Aq9lW=|@P^c{q_3Z$i7vU8ys6E^rU+B@5zO>K?#;e>)$uIv>lcL+gMvO@gh_yn3 zjpdv5y-jZgVJj0rIXcg&ZebG;<`VF^B>%a8OtCe#$KU%oP`hbKfYQ^_nr(VoTibJP zdlC8n^&#<_$zU(^lna2BSK#j4xzhpZ#+8%1Y>+2bcYYy+|9X28D{im`_!eB{1aN`J zGs5hw21vZJzd&)u{s|5MNd`1p45lC1-4Jn!5oSF6&9UJ$5r+@!4Xdsc_;jS#ey-~H zY^)^eSN-7fVK14%-M5of?w+pHhOdCqFC$b(L+YH^KI*kjefp&3Pxv8Gwk-{s(*ArG zMa=@t1bf%bkkxAITJ(8!jtpJYo@c1$HhMFv`ep1zo=<^;|>1o}X5Z_dbowX0bMiT zty6BFpoFpE`)rRXd0y~42s4Sow6WN&pLdhfRDD?$@Q~&*Bmg&tGNz;`KrVxD?slI zE*6>~chC6(pv#R;?#<#)H$K;ZbO7|J&@C!a+N~#lDnn@k zx`=>GmQVGtGXy!v#G)Y?>vhP!t!X+%tBqN4edh5kq{H0Mm@g zE~c*GS_Fey_$dUU!jeojxBRZ%4mDH>{n8C$HPluGO~G})3oYBA^~sNhV{hW$fa+ll znLq~%sptb1-=dyqKrgLggK<9`%jlZnAQRMUVD{tPEp+M*9EI9Iy?djpQZr?`q_W8q zg#<}Dg>21qATBScJodze@gbH2lwo#ITPL}{Td~7)l=>FHTmSQ2Jm4$NW5>1Z-6Da0 z5|)y(quPmzNYnGaqhkEC$kmX~_m2cHfm zlR$Suj9_Zbt!s09dzJpV!Rmv{g=xXlEtzr$5)IDTrbA61kq6MjtOCtaBQu2$+rZQK zvFsgLEMEFbIisRQ&TSVvR=xCs1^)sIjt+r@#SWo?b0um=G<3xGI28s7^`pnd>*|_^ zh^xLmX_Vx)g$}*!R{%m#I~?Dh@BK&*MG3`*7Mc6`yEqWd!O%x1R$0Jn!ct_i(7@m~e7Z$2j&&H@KjUg)x_(<|-e$Gw}@aKi;8Y zH=VTPyY4iv=jH8<)D}oe3uSVs-orU3aunD0aFedQa#m|MC;$cpXGS;gQ!|>?1~v#M z>pzTZnEMS;I=4sjCsbA5M204M%S5;SU)6niIFA>|LkB-i6@WC@KW=e zg(Rskap~gs_b)}k%W={&U$rS)xnCSiN!If}Fo!83V9TbU7&J@DZowVRgua?wD4if! z*;GsU8wA3w@Hr9~sXCam7}htg9;tq&qr7bM!K9TnCcCX=WZ=JS+B7wP{w(fGzT$xI z)KLR;NP%M52tNoh!&r0i3r`_8za4+ni%|9GyO~?;zGA_V77ZfW{0eDkUk%_VCqF|& z1shpF{{Dh)(+|(DFvI^{7S}WnGEjA_Rc>D1SXc^P1hk>68(*&oqnC*58*N1(vZvJy zbL2D=7Z4RSGMudmIz-8rYPa6~-K^)3S?Dh5f91NMf@|-Ag@Fz)f-3kAsZNo?;STyq zI=rI*2gR^(2@nQwZrUQrak?`8A*eIe4#MM7V7gp~FuM(sSN>L5bi(p3CfTu74bL7X zal6x7ZX8WUsK{W}*=W;fQQ?MP1lf^LWtluS$q|}K+I$jOE!}>hlnBqLf`5cRq8gFK z=g${n=-uCcj#NhA$I=yGLnl8^G6sSapQopt6&)No)8{D2u;*V^z{~A#>M3Zes)h?` zg<6@DS3Hb(8NgMLw9ud&@mBKEeLgxCerY7k@`M0fozPiiDjF~WckU~`f28gV{V*9@1PR77AFTK%c6A>)PTpCtmm-*=+Aqf$Tfy%?Sun-C`*k8hN z~}m3Nz)b7Zd_KdxW%$wk;kxoIkL2!C-Vj9J7I zgpGr*T)9bVe4G?lfIJ|o0pJ%!;9_4nW*jzs4Pb<%63olXi-H#ae8H797cX9{AkS^E zhT0(FCKRzy?0z4#;59v&|0>SgN#Z;oW(IV13^gbK0W2o88FEBu{&;=x!foMyXy+C+ ze>qf*Ly*rjc(YJf=)R^4b@4-nnRmO*&9#Fc<`)&H9h_*j&hYA2>(tsF=`Tw>Jw0`W z1LV59#m^R_oQHGbHRU^_`ml9ggYNn*HYEKHWb)t}A4yzhM{|g5yMeH+ ze9*E&5!CXkt84ZJKZ9vz^zPfIX@M#F3%XD_nGC>SzSj2g_L=^Hfkz8CRfYe-W_jL| zygp(mP5BJJSnTet8-KJmmbJvz#a~Lex175yK5I(!J?u0i{1XmgE8$$sa%&MNk*=#S9fT|R>JPK7$6Tp@{h+p^ zcnqifxVPNjTY8nq7jFOl{m0Z}q*VH_)-AA!q}3emsL_ly`h_J)_FRyP9DYDt?ZLr8 z7UR7GreQGOz3R|H9QKcJ;&D8f!cR;)008QcONO}H8!pY*^+OD^svIiHKxvp9BJn)* zgLUTlE%>}iLt@fZdyQ(7muJ5mdf|ORW8*?y$)!i9@&@#-Sa3g^GjLDP^G3yBs6oik zRvV@oEM8o1TTH@tm>k^o92(T-YHLDZx=QDsERuwP(E1@cPH@Y?juAUpG>3$hWJL5S z@mcwuo%2ag%VY&kZ~;h1+ziEwBZLp_@Diy2jvJ!6mi9)tX|1<;BToJ`WI#^_EJ%_I zdh{u51X4}}J93Vu4U)9Z@HP(9Fe04JnE;UTc2TH62uws9%91 zS(3^(1s6=}bNV8FZXNb9xEyZ-qH4cP1L!lc_#b_FJo$!UylQ2FQ357UE6{@?r_YcR zgY`zBt3ToZlqtw~q__)nf+RI$UI=F5c6bxb0~;TPGM_$ST`s_Y6?aMZuuN=Guxf`Q zpG3(?B}PNN+9A^JdZZG<8pk!G9}0S@Iu&R{46fVOPhYxy1@ZvIc0%98 z6-OgRsJ5tzU-d*(Bh|(raHR@89*$pCNA#EMM#J*(q};@Dbi zbWkAy2Vj7z#@qLHKLo_YG)Zks?gxxAu*hO9#D5i97oFiQ$l+AK^)l(ZLMU zT}{f(WBhY|G-yBe7<4<uNuoT0hA9%1vvL~3 zq6-UU4B`1x1{rM>?yO`jlpnExDwxBtNEnB|lCkAip(bhhf|;iFvA%n{Uu-le4)ORb zVcPI)Kz|FFG(q+c!ayszFCeMGAJm&6G%6@4AR;bXw$BV-y!G_+gGq!Wk?$EDSkPYx zLU)5FQi|RL2rq<4vgO%VTTd|qOaa;N8v|;b9HfWP-%jkiVt*u0K!B~ z1y+{5W8G!ho)knUfw@KSl5<3A)kgF;m`1~qBART5ut%ivAq?!q12VvpzKn(f@Nn5O zc0fYRhK!ZrG(w(D)I_$H=O6I9vN(f3 zWAEKvrzW+{;{DWduizvwVi?Gwe9=w8?42MR=9Bl&#p(*`>y60w90_LypyPq2)t+_| zhatwMH31FMZrmxQZwK_(Tw3l2YML5iYkNM^w&&bFuMJ)JE0?gsA{jdvRm)9`YlAUi&*rIot{tSa;h` zTtV;p+Di+3hd~T5`X)VUk1qs0EpNV&;`?n`-_?$_IEYxe701~&MTQ2XndC_=)ILzg z-}3mW_C5fEbadfdh65ruNscgztv9NEbahd?gE(>!r!1Lj%;>A}2%9eo&uP-VAD7a; z3+5h z^qlJ95q+p7B+x)Hck{V@tB-$>XQaI;-d)|&Rg+hspJdim3*cmqn?TX*=gck@5P~t=8R>rCS9Zj&cnq3i8_c@| z+jLliSRsZDJANwKbA*TMOTT;{@F8)wlX25)#5H7gk2l{%giiuIDg&F=!|cog83V{g z^|blnD#tEeSN*IQxYs_=&VBMU5@Y^ z{+OyCwhZTLWkq}UGV}SH?uOg(FEf-r*XXRz<0~Fes(R_cr0jW+TsjU?KM=|*42q{M zl$I*VB0G0Hmv5QPL<eg=8PVbyv|$+8BQndg2c`f=>JtpddYksZzlF6RiU@pY65(v z+T<1cSJ=La+VR$p_tFU0Gc5B^0?;{`2tn-`krz}VC9P8}ZQxEO?W7pvz2=xh?|1V$ z=UQtDpjf5MP3VI&0At40cXm+ABee~wxrPh~kmy3a+^!;8pTrq(<$9Y{(sKUit0g7t zpM9taiGF>n%ZC{cG3|?u2NHDv)2&EKd#C3|r<&}xw~f1Jkj4^^sETZ%fvV2ZR%YEf zzj-`CJ9Q;|#RH>q?RKsERrdHP<43=`ifM^}~$ z^wEe56_u2P;z!n%)BQ+|YokUtX{SUh2QuV-*$WiAN2)aSnSQTMs(H5sG`sCLBz)T? zZ;+7_Owu<~SQD*!^El#(aFpZ>Xfxin?IcC-U?q$9O`E%R8&+5VKW(W9jWl58Kz7$4 zCtCs@XNDtXjlvEMa~@3)C@3&ul;HpZ592$|A9Acg4UkFw4FD^Bm}`VxD#fkGzU3<< z6q-ofTfPDhf-UP#c=JT#uL-2s>^ksS>V^sxb%-58zQq0gS1%6RB-B1WGv&(4MPWI4 z7aR&w@{YVbqhUkpX_%5-ft-cxG*E7#NHHA+1Ak^aFQ1oankz`Kq-h+J_7ki?dl?fnxE0OBDK*_BHuaEw#IHXEWgZ(a~F(I2dvTqJ_fZ1Oe0x>dMOtG%5Y)Yg? zSuYNfkOv+#bn0TkCh7>Uow5}iBS?Qz;9(PiE-+O*LA_wnRbp*9Q9%A504k#m| zpOqJcNp&@~YDCoGv&B#_p^iNW;Yl0~7kvz)N8h~@vB6YK2L$V?m<5!+qTsZ>EJ85| zyeC$t9vSD2JE_4h&{Vi1)%wmL_I8nu~v7%Myni^Qb!=Et;n@98>&3z53hwPf=n4<=-OK3 zOujKN*45^nz8K;UuyOG)yU+IpO~!4J>%epmh|F^9sho! z)}%Q^LZ(A*Ejc-|a7U-=JGN;xpuSM8;{fJQi|yM4{1IG{HtH*e35l)5uf<_KYCF`S z!9O724mv1kQYOK_f!G76XnL12${{#Y|%)hD|dEwCJ;nTZZ;H?dgTzPG+&hGP}C z4h?n=#S5~M5)LUAzO@#?2;@IIrca-q-qRf9^lZW5549UJ zdY>LlqO2P&gAT`ACP7F{7(M*HChWKuRPk#$u!(N3x`a+9CL(r~$ig zGemKOorSbOc=-L2Xa816dT9Hg#$#Z~7*@3Z{ADNu)im|Z!e-I*ZPd~3wg^PG2Aacg zy5DwKd*plkhy_iVOeP-gQ|Qr-yKY+l?fyMG=e3cDRrLaxw^E{__^f3NbtGmk1T&}< zm{X^{$FDE%U4AUU)EPHr&gB*{_`N2L3QYo%=F%)~lh(w8n=W|?a<8a_?7^Cdl81JU z{0s`T;2pW=UDuI-RRm`9vqqD%H&995#IyYbOX8Q;C z>=*(LhsM5&LMHm@$D6HRFbt&IW>>}xpLymcJq~q#RzFUjbn82PRXN%s@$U7!Th|e` zx%#)~{F-E|k|elVDl3u}6t+D~0VxJEf4>?`CTN$!vWpv`ALP^)*)Vt&d{&BmMg=_j zfU)t{aT@2FVv8mUJ`Rael*2~V^`B`f6#x8jJk-5qsisBdN$B%XJ0V@{^e;r5%WzfTY|VDbVD^45#U%1nu0;J$VnG zD%JPdoZ9Qpnf`Sbs0bX+{p-})Y8(rk)e8fzm6~uRxx+<44NC5FDy6M8&cH~ z14M(t3v9v=FZAjAHz-L|iXaSazLpC?nPp?1_x<^&KQW7#<4&Fq(7(qANd-FF1(N=3 z>z2hw^nX&pmRN3SH6X1RhCq6>pF7i%*FtWXJS3$BZoZz-{~Q#>CK9{%yR zpC5C=^d>LF$K+Fuy}~K}2Z;euN2Een{k}j|J#1Z{ zWk9y$;YyYbbm&rZKqOWMy~EKo4;1wWn#G+zsGr+sx9MQYTvpWptvBxSw}L)Vg_^*P z7KzrfOi7;YtatcvH_K@?oV@xX{iTI}-hQTh;GLW~r~l@8`_KXpf|qmB->m1?bcu>& z0(Q|l!mbZY@*1X1X!1ea%lcix?N4HM%3HRer2AYgiW#_6&Siw5?IkzCs#1A(l4&9D zoR1xO0PGinxD^PPOw> zKC+?_q4b>7i_bKcec6-k@raT$h8lQ0gdWY9i9_Ms{KeZ&$(q$u?w|_sh+#$)*bFv8)u1QpFvNhpIEyM-$~we#^zI z$HtTl=T{;!Qg{PYY&nCh!XQZ7^XJ`UX0^bDAN06|55U`6fjJkA^N5_@98zFG>wW)T z*#^4Yuqo?EV*wJzm1`?~v$K^l^7gA~`K;97M6xmgRkWqV-J&2;4ABC4@-pFFFOBsc zz2n3%A85TDvH||1YHFW7KD(b^$>+BkVF* zmg@DZ%FC|RMcaMTd-K|4ZQ2i**PN%7i1@AzJdffEL)1Y4jm~DM(#3N4X{9;eRowBs z?pX&+DQ1nuP7AjbAna*1o`XH6;0Z^tSA4Qy5vS(og%5OJ&yoKd_npk90dNZ;=_rd( zyHCg___7(UKR+9=Z~OI!ovQw(0zQ3?(G623G@ZskV6eXe`cNY59r=2Ty1XF|UkQ>L zZGG&`U-2R=!@lXw#Gv>&&mVu5p-v?+z)P%NrRX{p5gvZmrK=f=a|gTJVa8EZgLFtV zs3yvha?^3j_`>DUnQ*AxPg(nbY*dSM2&vnlgU|2J&KI#-_HnxVjXhPS1pa0cICwzLgtf@YnP6RwWR2VUB^8DN4vjopMAO?^cVi6ldM?Zhx!qq=VUZg6He$0JAN9UW7<=sitEwB0p*8$0EM6;D>4{%tx33{#OB1M z5W)o|!wsPT_@om#jx40G1t;SnHMqQFO_|UD%bmOl+m79Tim~vM>nN~U^?+Kp?1syb zTp@vByU{IG7;^+aZy)NN!XI=HWqd%;iP<_Et z@#iN~cZ};5RV5C8ezWB0Fax3f+2f}xIB_%~7N@1eoA}{AVP zw8{{EHeyNj46Ux{E|Rk-83WCpjTWcPV7#ee00%Y%u~_n8veWO8+4tl z*4LvW9MbYlWaTMfptonmx^-RApQozo9k^uuRJ>sAQ;CAn!~dW6&5!%G?3K;W0-b%3 zL}2{(T6wclDN!KHcq%^=;)oQt0zCray}UT4@=59ZuLShVO;7DAwtNZ;t6{S$QqbQ~ z35h0z(836Y|LDg$*>)KApzmeQVq z&vgd|yJr7TVET3+-Tri_Sj_6Kbex|5{xUW7;9zXyi6WmDi%+OJI=ljYqbwy0j9$gA$ZeXvh%2F$2<$^8dmZ* zoyo5^YA#g`+@K(LysLg!ja9F$W3tvZ1xpt+`TbBi`{4{c_N>8POR}aC@#aucL$cD1 z?O~R#F_Sz@+W=ctOkSNC2KNth@Q`Q=+74P8w2uE%*Devw&yHB_xX@R~yAl8s5UYdi z;vmrAXW+6C>UmKXLv|$aJid;zpitxP@*79w$n%35BVk7*8&=xYv6X0sk8m937eOmc z){;H8kVf+OGM1$pM0>@6zWwa|q31Rwk0fefd>&uAq*(X?cyWX>b>nbmS&GBzZoOrt zFb=Qh)u!lhOc^xoxY%eF0WAkWTHFJVe3g; zZ??%-2qT<`If&_Ak%tLVkR|RTTbnU#mp;J8=KP6gmJN>B+o1rlU$zCrOE~9}<7^b3 zn>^n;P?Vz3$#q?Q7yPXOdQ}2&>FQ92r8^?jhdC|gaIeP$YaiA3y`xqe#Q0U>#&k5v zrWyA*^jx&^;q@1nwfirpnu;NaHsBkyrc3;sIv6^5{7OS0LnRLf8YJ1FL#j~j8SDJ> zJ233WWzuXnLxI`f-%nlMd7Ok)Q&(!jY=A%ug^|P;ud)hmIfaYIc6Td%I8lnYA+SX52l$uIs{O_5 z&?jRRf9{^WH!pkxR|RKS;JzoKloJyJ7?oLvD`QzzVp((3#A+yhuYk&o_XnKkezagl zfhZzTfPRh=8?Gr+S4Q*{*??H$_TAwMLRk6ag;F{j9InJ_jM4``kN3<8v6Qbv9d$?D z{F*|3e(fyUe~`%~clq)*0W(2C)iGnZpIUIwQxpUm! zaDlV(vufP+XWbAKy2+a>=RGD**7=HW!QRGWC$~a5n6G@Wx##u^Vc(Nm*E*DZxZ(CT zVfN~@&wT|SHh$Iq==Lanm_oIKd5UBF^zv4lP3Tr)VQKk9cbMMDp>licVwIgcFMjLl zB9aac)$Y%Adr-?FbZy?ec^9Hti55p#?WUFv4N@31*{4ynu6Al#3^gl1xxAO#dUVYv z?{YGG_8j`@X^B2Vy@8%Rv!9%2-N-PZP4B*by!~{$i$D*5gic)VU%O)D&K67Lx-?%i zMN=wbXya%|O*{N*=SJF9XsLVP+s-^1P6mn^WF3+QJI~ELW~ZgkyO>kXxp;9g;i>>W zQg(VRWnjp3?l~5&O1=B=p^l4-%idKizjLqnXuH}CVhg2K*zUZTDJItT^-*VjWwvK2 zudScORx61Xt`-n1deNuRy{3R#Q9v^zHR|x~l@-Sx%1~pV%I}6~39g z)LPI!qt5!Xn3!1W?Cf8svatD`NfBpF>U7CHoo~17>rC8tKjnI(x4y*Ly$d8?|B8-@ zFN2>_bbr4ao%kNvMpUs!p#`+1@cMzzhqqUa!DJkwTRx~JV=+$vITPO!t- zR{(PT{ry=idZ=WYhK`O#>rd*SK-{z;t)qjiuRiN!t<_Loes9M3)8Z{$3QVA|-_gF2 znq0o3wvY@$l+jQ~YU}yoBr?X*VOeqFKI>wU_sYB{x1Ny;=D{&hiOx(jKkKnRQgnf3 zjKBD|Bm1qbrljXO3!Z!;vCwLGQ>nQRBds4M1_q^nPpeZ?dse(A+< zo$(j{^LlyjM9qMjny@vd4U2du3>#)o*xkNdBPxXJS&0Sj$+KL;nFv>khn&C9Lfds` z9PGFR*&kajnVZPfki&6u_%5Hk+r97UfS8zd^Pz`p>41?81B&#R64 z{jHCSDvQN9H~e{f(hoUat%gM!>`~}u)JEM>9d9yuF$;p=*jxGbjI|u^$@7uJ4r^TF zZWJhrfotOiLRZ?I?cgDX^+`WaU%ZX|3CE5Od4%X>Nr^tF^T^1o95V2IqKNl*4Ev#+ zJQjDjGc2)UcE1$v#L4sUMW+8w9$O_KU?z3o?@`_EZ7cJtt?q*tm-4en@ZFo~!@D-u zxJoS{yIxIgp}3njUrbsMmh^f=w>U?~ml^)>UCNBN`<~s>m})YeJ#^zlof@mCZp-u|L^_v13C2V3kbIxoCCuh@}er3^Z|a>{$8pn9q_?thKgCIrfq3 zSP$E-p5vcgmKsbk;S?H~Ay-o4Rhat|Ye!eb#>w%PGAP#jjH|5DT(#n>piCDCqYxwpUdsUh?aNKM#-p9J1 z95aoFy<|8RD<1tS&tN9Rt0){Snvfo-r!Q<-a^vNS$!kutr)PCBSDDVyA5x9(@AMOg zEe$ww#^?+)kY%sFcKuP_laM9IsA+U=KU)=3BH%KAo?~KNyl}?m^s$f2M`ogE&Zh3} zn)26k=G6Q9Z99A7^2o@zwl7b$x^lGB&}DVDe!=qz`aicPF3TNPJbXEeE?jr+)Or#+ z$Uk&*>z1lV9-e-JaGomb{hqonTeTYQ zg-p7+g#Bhc*3^E(UMaU&1cs#ir(FL9l$y&COZNx;Jll*PDU` zGH;E^$zYX8)GWf}y?Iii0KRO4wR0jmw;^?2;f}iF=}c34SY9r6_tsn}Yud`opuIj0 z)V*;Y)^8rJIh1hko+`qm8^l*2elk#kmkSQRnf|b|+toiPNGP>74yiar*W~912!iiw zlIM^IMr2WRbxri$yQPQ$$-c@G8{07n+eh62(A0yg8;eAoYOe+P`7cR+)wPv1Yp#wO zKmW;!I}_~S%nUGp`Y#;3f(%OJctrH6L}ZSprzCKGd!%tHOu`tzJp=g3-j+_R#J^i7)Q{{j~;mdE6j(O)Elr`4U_f0zM5~n*0TYgeJRQ43I#u9KfGe zXX76jXbI8fu|9x#)z#Jb^(O1OojZ40SXfNDU}49Sp7jgUO*@$HK+fz-Ow2TN*63Ed) zQ&SU7qX8yO?7fcY20t>G(b3VzM+FD;t@>HKKJvi>Eg~DiiSR?WYi$S+r_Zk{(E&@D zLb?~-Wo2r}?rzjh;}DGk&Z3YpCg;$1iS*IEymf!M`y5h=O{fyHnBU;${v|9RPIu-@ zJp@fldIyYSMHaFYc*p9e0SN-@u1tQmK{!>*5Vh@&{q%{zf?Kz4z(4}|Y z%g*upoO-w-Q()t4;9Py67pN3*N^#38%FD;ZJ&lTwC$12+ZMpqIJzVdGp;KFM<*sC` zF)bvtGtdVisT%)kLSB8F*I_3aVlNqD9gp^GgG||?+pOFFk@KU2YK4azH zSMWB;wA3jyZ^l={+o}7!S8{HUt^k3MqPY-Yv^0rf&2jl)Sft4MB*&5$g zZc^gEp>nDe7uy1cPz9_Oi1kk^W$4$Xp%tKSXM9dq0#T>T{bya|ty@b5T6cM>xwxcJ zPvkMARc%Ze9 z!Vm-LT;2Qo^XGOFK8O7Sn{8x>!%O+u=Abs1x`j~7bkLhNH3lC)J_@G(!FI!8irS{d zIAFPMiDZwQU+eBas8PwIz+?E7;~1R`Yg)_0JcmA~_d5%+w!5M`@1IO-`r#U(w0f%k zzD?Y6^6ZE}tiNfseQn1U)`to{7zuq2@|@Ya!RV0HlWPdE*vI)g?(*x1U70AEJ6UZlznvBM4l9v_LjfVd{C@s zV7KsuL@##0Mx!3Dt}fR2{!CNqp!>i!P5-wqo?osoIcH-hx=L##x}v|*${z9@9h}{@ zJ&G@P9D5-6+QMe*`PIK}sZKT-&kleN+KKP)PPu568SH(6>(EGlvAuY;9PK-HhHl)Y z6}rnVKVPx0r-Zx2MSQ|vf8Hp`9e6oR`H}+N%nSqfhF=qWN=vt}9>~dSUH1$9)>vvK z7s_5h0MB}nW9G;ObBO*mGANFj??!$(<(sBMQl>|CW^Zj{USDm8{kM~Xjx$ESY+SyK zo6ey6(`=QU6?x7_%iE5nvBR6u<-jWNytY(j?v5Utc#J(s%2yc(#%vUS@U>?k^4iqk z1tXud9abXy1UMg$MD!e^b$B@udc?9rAFsUkW*IZiAZuV>=s3h*k1w)6HhFf#p?hoO zwhYfNufK$OtuBZzWe0S~i5`wbxyBp_&j0-2T~~}L%+0=@cgFip&S(E%c6gJfpT!od zk-L+_fzLej`!hREQ>r!L&j4*~lR^p(Ur)z+7aJ8ags;wFf9XAO-gys9FP-kFp82ry@SBG#w8kW1 zWwR#IIQW||#eVHet?%$r6FWP0Bw?A8+i+;i#R0xueu7(se_t$e9_&YBe*XL!z-LVtk1k#}aJwT=~k2gdYybmj1zrG7xpu6>z0)xSrC*zSB9TRgREB^j{wY0P} zv?n9!B=vO_+qiN33tiN|K?PqN`6`NfouHs?o#n}3oD<3VWrC_J;lCskx_BgrM(d^X zvXpE6_YGq5QCzvBUY!&t(=Rf3Mj$)R09}sD$>f7N76U`3(x)TXP-8wHePoEZxS+$? z5`{N9AgKc*Xm#V8KY_`IQ4y_KP*9L$q(t5z)uC}Qnma_`m_}KMXq8kv!f8hB${OpX zXfQd!daKr&*;rjrTYPrxo!2TX>Icqty?e&TJbDU^#>FoL4-I_fa>VXYEKg~Qov8+j zW401}uN1f#5z!pTa~3##V!)JH-5y|Y^qo?W_lL-)8nT5l)XMl7gsBqk3N}bW%a}5N4aSHpU66%V4nZI$a+=ngEuy*1%-v^DP5l9 z4u$BaPgcNvLLPjELzD#yT-Nb{0RhOL5}(M7&rt@goUdaB$CH;W?0D{i;}Q`SR4H;^ zTPB)S-#;+DZ*Xed1BbugAVo_;8(-_XbWKbEbBBcJXhle>fZlGYdiClRqG|4{4i;Xo z?6aHRUTZF|tZYLrL=vq`L)#1o=gzn8H8qfW{Y#)*{H^#e2iz(0b0Gi~?c~h@N6$Gf zt#DYD741%+80s;Q@^;gKPf2=C=)%1HQ{TePuIo+eCqHd3TU<<{mC%7trevLhZ11*# z${C3e6^W4I4OJb^n>-ndn*UrLMeEz`lY5mvCh2jTrWBYY$$#j3Rht;0w9rc5&@i5g zC+J#7)ko+jNvjUYQWgD(wsN?&NKKAs#iyrJ*FR?Xg$oxp?256N8)~gG95(wQe+zn1 z`8hbCie@WESjt1K#pkOm{6ifqN*M-{xjk=xW<`Qx6oX0rrJW!>5EK^tdIT>vXahzW zAq4!tiH*G~t?d4uZ|tTO({FRwFZx)uQ)`C?0P6C~i?<$+@Ba95D>SMT^|Qh5)zHug zN9)pE$kALsTv0%1xgPcijIx~@>sTaUAg_d?B5R5O17)R*jLf}?RtR>fmt;3~hjtVR zDoDsniA+R9#Iepj_{7XiDFdep&GhE$mfvBJ5R;W9{Uv!g`qN9Q?kLOmJ(gHv;hzx2 zK*Q5hI3-)h@Nd72A2Boyk5h49INzf zBbodO&z7o0Z!x_pl8cSdy;NLU+7Mx{;L4Q}9aZ{{FXdsJ)->1XCp9I6g>|ubWLTIi zcndB1Laax6G)cGihDk%CGOwdrp!Ce)oMfG)ZLP*S(}pl#_K|w_g^9UC+%zlIPe*I&jwaBdD0ymYTlO7mB=&JnW;@+;)@yKwHdG!|^7g|M}YLNluG4 zuu-VX&eJyd9N|98%|3&|A5E*1&X#HT$KP;y1C%_mQOK@Tm;G0Fy90m*{`oG81r6&x zx-~W0yn5Jy(IQI+hth|Gw=dn&m{{b)4y;lJ-_?C;T-^9E_@UWs);*OK&WO%3T%Ogh zh7fc^=&sG}ZMO_&DF03aHamX0>a%(e&zxu7oeT3Uj1T@k@OUyd|2qC`W{8}zD|=ke zS)l=@uHdoi-vdGX!bV5OSv(5CCx@_4<6|{@&Ncq*yR9*`=)y?+Z5%cgrFDDTGl;*p+ViJ+2LwUleRZ zD+FaO%~e>=zR`<<@@B06p%=$%b3tWq2M_Dg$IT16dN;PI@IaR<;swWiZKLT8Ve0G& zpz3^yPx7I;uVVoZ-Rp7eXQ<@fZ?{rJ1>rbPhCyX`xLM$snIi~Uj_liX$pt+X3CH6j z9{svywfX4ws3ljq$IoXaZ5&rQMv@>=FTylG^!3T?a%vOP`tVMleY3o_UXpgLIZDuD z|6sqi(Zy{NXa8c~-H&yN`%;{~827|#;h%xFVf-FAS(*io{qy}P3`Vrd&Z!r6@h{ts zz-jKtR~s)1Dw)0hhi=abpW?kEKFKUAuQ?lZAZ}~YemIhUF^_Y*19F%geiC({_yTof6=I<4pi-3DJdpZBd5Br;tD@s|IGg` zN2JF$eH(5Wux=zrqVV4qnnMr8tG}nlX`YLFp#4|N@1_$z);` z?KSY&R6%4l1cwCY;B*0M*xR^pXwJj>e+^O%Z~vErR0{+fFY{&_#X_J@L`tG1F79Oc zDR6DF>d^1%9hjaly1A-pYcNP*tsstS1L~nBCghz#E)S4zEoPvJ?Z1YsS#R6B1vhTg z28K%Jx3C<5;(WKTm_W<8oPJj1fj4UO1LVpI7d6Yb!}e-`5{^+>kx4W zR%L#^zPAxb$bUY=%KJnY?Aat_lezmY4J}o@&b2*mOQX?1o3?9{zcyKC5ODMVC}b5l zF2kzodR-G2<}sMPaCOWe&qp1B>T1PPDj&ZsE{@7HOkg<>_qqZu17KU&_;tmmO`Akb z-{l-5k_ZB2iW{i5cwQ)8)z#J2SVcv}hg2uf9e#vUr-rVsZmDH*(KRWT`~3MG&M}#p z`VfgveUqveS5RCGPm^l%c54d@bm&tz*8dYYFCGv^!8Rhb)h>7KFvR-VOIjgZM}T{o zz*wX2u1#O(W%HdG^jQV`92lZVU}415r`vIow~~1b3b~rQcQ1z8#m~<#tOo|VfWesj z2vf(76<_sp&*uDo&duC9R^4q%~Y@*ITk#R@C2fn^$-}az(1U1NKb|A|uqv8xA-`3y zAcz}+;_4(;CA{Q|e6=$2RdsWEolVQ?iMdk$_)gi-r2#Jy3KzA}Jz8eMMOsra#q7u$Tf#ca62wn%PyXj)6ZGucG+D2jeHlCJlcX#)( zNX=nbW=>S$8F3Pn?egqZr=kJj#nj=%wcCO!0>5#X>Xai4SWm zX4cL-FV1t5b`fw9sH;kf>`khexS$;S8B{+vzIwniK~Yf=);0Du60;2)uV}^VyF9H* zgY79qa1YiMLu3LK-`{gvSZ70NO~BLbAz%7NH_nV(D8U&#TK&dwiP7uYYvF*f#Gih$ zy8{RR{LDY7D$foE;wBOIl#%_PX6a zXqA@uK9_6B~np>Ue3qA!};mWzj+B~xgP(9?z|=} zE!tvQx}0^%+Zak$9Zc1jGh$7(x0jgX@ zNBEB@&`}36kS8N9p{?qLu0T|vp;{C~ZmO+scM1!A?oGvG#qB#nS3|%LdEFz>uH(?r zR0y6>u|pXg>BoHrUEZ32pLt(LnCvevuHu?-!smy}Fr&)}>cYZS$F}>0B8@{CVTJ8x zq1pZ43e)jFb*O^DI$Fb;8$m6Q zSU`aD_8$qEFikF46ah^t!3hlwCIk-w22@7~0A+Fc=YG`7Tj5^Bs=r~6YK6B}6*lI0 z6lB*w_8;96f7?CFW5AVaV!f|I2g6Z5rfyuY%qYWEE_h#X%%SR~w5=V=dPFCurcFyV zR+hRf=x{(ao_fE+35bL=L~w>%(8tGY~Uh{fDYllE2~h zEq{VGXK+SuuPdtj$_N=+*5f?-A4GUl<7d6kS^D&e8DP=$-nJMqoHGn&NsBtpb2^|1 z_7h-+8fq$V_s*T>;6bH1T&oszbm8so^@APt#P9?rwj*$|qoP7zCeR2Xei2+$ zs^hPMV!)A!jaWk0WkWbqDv_#NW6Jn&Pc|a#b zc7^1tmblHhtn_!sC)f1n{pH9V(%OX++5#4Mh_cSgo|65Dzaf_r0;Vv&Jwn>?#aDuVlzRpBUL6e)d(oc@ukv&-HAF#@UsTeY^m0V?a{? z7ksgz|9gD^HL|%9MOpbkKkudhI^>z*0$W;UM{Ffr*MK54x;O=<9zJ|Oy zNa1qBwQc1Cucdcljj|0(HvbHUk8mVOou+yIIstJhaW}SStmn9#s~CKbv2&+CwCPSx zPNm($fn9Q+7{JB_*QagUW7?aUNBuX*wu10$u(B=oUz6Q|2?nl01vU&vEX8s0J#Nwr zHQ<-DCX__da$ny%#@SXB+Jg6Gp$c!cVb8l7VuomH3Z&MqI`d?&*$Y_(r#mH{U%idY z0Q`~oSb#wrPg{F?_q!9*+M*(FFe?f?N1y2l`Rrh3Z@)I?kg}J|smHm1%W;)k&V=DR zaZM^BfiZOnc4_)sx&+KYQH~5@4O^c{AsgZBn1ufE>21+F{3bc-DfGD^ji5$U-eq@k8H}P_!)LhsPZPJu}gsk3`dpL@t zZ@a(ML$o&pVurVBifoHxa*2?T&;X#;@3Uvk`t|cub)yly=w#4YWvI|-aIK#0U*a*? zjqxR*Kin83dcv2^+SHP4lp}@&0`(h)US9H;#rk3cgAh;D+y}8;_6YQtzPR304`Th0 z=U|H`j?{V?xOdTYjPCl`QQNXhS^2xnkJtMKAtdWjQdI2U5cDD&m435FV;wnaqaIWH zng1RUoy0p&Q&)Gc-zOsgr>=;{NsvJv=;>%l-;I@R0gIrQu9I_gosvt8(4rL7kvf-_ zkSz&-#bZgZ&Wsq{5_{X7^XhJeJ}HRnz?sn6yZ0vP1Rz3CmFcT4A(ymm?W;IUD_%;6 z!c+;xDr%Dn)wmwm@D=dEiKZ-=^)>F_@|U9^q->1#Bm4|PdSPhpNQ!GMt#O}CQN&M9 z)_@}HI=b4R8%`v=nRiM^NK8*yu)y?dpYH5o(}B{8uc6cb<=`F;Hvi)+|F6CztDFR} zE!;u{7~?s4FXx%;Mi)&D%TXL0AE!{fX?&HQO90xbusW(}YiR}WdG_0hZP-9dnuFuj zIac*|(F6CPzWOTADHem$gBlNZ-+YtAk48h?Mz;;-p~@D8J%)p0PC=`qAlwc>1ENFR zKO`g$N0%bgO*-^oac~s!3ITq8Gjj2GV5M7H)%iMctIV&zr}+g3$C5!mS`AboC=g&! zjSiv))FYxHV`+sSWGT2eO9%OIcb2fK@7s#VfCa~M{`*5anDpVqA-*o-nMEEqj~~ko zd{{2?5{w#t2J}%WS$%b#7+c$kY0|31pMlUCwL~Q*HbB3?z)0GKLG-5S<3zmc_;?YE zP9O_8D_Ft%X=*FWWHQUK%ig)O3}`>uXClFL1rdiVHVcUil+_()YYs;x>Q&_QWa`Gd zAf!+(#uQLOiY&kgG;r2^mjW*G{O^xwe?`+Id06_DPV@M3;`t*6ZsG)1J^3`17OAmr zMOBq0_CTEf$L0#ZI|QAQ;PBt<#4}x_Cr&Nvww2>d*;;M2 zft<2j(au*>N9Qi}yM)&XnpMPjbOqI$fxgHL2c4$^H9C;H6A2)r9tyqC-y~oM+rboh z^j&QLHU}&(0}!iPDKBqtqekmVCItVU1+iWnbfAx?W*e50n^~%@(F-`K;^dnD4Bq@bUJg?Oh$X zpgB()e)bK#6v1tx|4g5baEShuoif5A_$OiI|M;a}L&{F6PxVCPuhRE2BW!<5%l?kl1A8Y!TO3Bw(B9g@%HG2C{&`27t(~cr!vzF1e!(b>3(J#qIi6m1DCaq8Q<~23v$f+JD zXUgF}Rr7r_51d*~>5TDzv=3m^3ph=B>z|>+A7c-{|0pf`{{85!uTQXtrB%M}`z^$- zB}tO}id5~uxXiJ0*A*|~f_hwbzf9HOuM04B5@tt>Y?&A9FXyZ(L@VIkV(#-Dt@sB9 z0YBt#Iy*@AJ`x<6?(Kadry-*ze)7I9wGYi4gNY3Y^xnG;YyAKIGD=#OAlW9d)krtD z-VztSFa;6kx!9)r>@pR;6q->Y4({g;l2=nmY@H9PNnr3?yKsBiz-5?<|KU$buFzEL z{dGfAojK&>sZ`A;WHQlj_IB^ALGTa!1?~s(WHsCRP(jDei6s1cbOF&JiZhR?PI^<)svLYbxu3p zx_R@X*Y1u^zGYPAjTw&(nwQGCe7sslr9PwWy&r0a~SXU{HD!JZRvM4+1S`NmOBjit$Ibbw-%fJ zlz9|yZEt&=>!n&B4mD^kcCwOf_E_ynl6rG?e5fWUT+k|7#9<;qu+RMth3K4j(}3G* zkJIbVtNUO1!&tFmVtP*xQLJFSHmCD*Z3d-sOBV$1%Xf;})3oC)dy0m2oM6`BLN@P<9H(_AT+P$9bEMi*H9MBZTXG2b<{e5+;+tvy zj2wDsjxSHKroc9u{d#}vaa4XmenU^OlN*!fk*E2qfBSZK@}pfRlCQsEE4MXgT^w!d zSefn`e{(Qajge$Ofmg7{J_z>e()fM+`}>uz!ufEoPIK$0U3+?PU8rd5s@G1*!Q*rv zy}dEp3w7MD{QT@^-KMgtD5~^z zfry5VILf<%hxrA`(rN9GHZ3ch1cuxBqqfLT&wgvdnqWAF;C}SlzS@GusH^h$mD#>om7NX)FVnwY945`dPbtI$?6VFj`$Ybz8ojbwpN*?)EwnQsB_(BJ zV!8>N{Oj$xtD+M}8N_wJe*OBV+^gJmbtXL^;GbjXL|RK{o1HGff<35z7x&{0ZCziV zK`@7A`}S&ICTy5n>*k=p1X}V}R2=N*&r3Q?wzrk8_PTX$;(7DGz&~5x-#W6sxtXdG zFSP;Nl+*g=&6~D(fAN%Xe)C$({8Z2F4YaY+nNc8&wZ?p=6RpYNyhc8Z7cb)SyK-TF z=!~=qZ4;3s&&^qr9y&A`Zl3Y0<+jAM3M)UqIuf>tj!f?2o(y>6#7ov_WivCgkVTKJ z`KqFsqDe|>>W{X=wc{I;nH?}3(TS4f_Pb`?1zA&$#VP8kA9#&^U8dE%%vi>cQ$8pjjE3OPlEA!_4aK$o>{(on3TWd zV$!-QR}=@+pvxlc)&yLp33iO~?Aes!*|Mv3cIU6&&9`iu>#qR$7ekvR7tCG+o0zlo zvDUeFx-0)Svi#1@1>e4Xvj@k3Wn-&}w=^?L`SJD~x|S$Ln=5U<;-$H1>%cR_3OxEF zg9pP%zbY05W~jLF3F(p66m<>|$B&@OMT?DBxvw5wv3+#8qDf*mZ8D=+X=i6AMKd#6 zd~2?PbF|jHxK_zhVt;1?Nshy$hQoOCEm*4*m3SY0=YG*Z})JRm(|F!)2$zo{4wqeFE^dOI`w&kj!qu%{&ro54m-%VbC^>zBeK|{ z>g~CS-&yf6(2S&LS2?gM8(@jsG7Tg=<}1%k^p%$~baw)j zR;R={>n*)l3Q`ksasAoS)#!s1^l5WylJ&#~M`T>^?R;(Bf?Q@_AiPG?-T8MrC7 z+HE)SGwvyiVmMPo!-qc4O-1weG?h&KQYO8vB%^`vFDHJ*`_eoAC7JFmy=zszohM-N z`)^qiO=FBiCaqP8N`8-H#KB`{6P_PAGXWq#t>uHFLyMxIEFK>e)K==|bWYq|%U^Wi z1+o*!dBf9dTDF_sd40syg}9`|1O>6)o&fo^XFXABdL~^bw;6mOXy98qw?DdG+sm#) zfznH0cqYe_Vl-eD%r)EEMMCxWVPT7QcQ*NLhh+5Z8u;;C#}ko7f;n?YNJw}rHVC@T zeLXa6TX^L`J@xARDHa8y9q27}OLdy<{qxy0$aQ&w*|(zq1elw6L+XjH{LHf59k(~s zyw@-3eEzvGTnCaabndF7LQB$Z%|yB2rC-tv~)(J1Gx3DmcTY60*)(2d}qz*7!BXeb0YRXVb2$? zOr;xE_=I0E#j_~9iv{mccxns23t(E!$xw_C2p4t9zJLGzQq1m_boti8>m5)t3pcE+ zi+RN6%5)20EdKl744GJB#v24%KO9kJdfc6)Jve$L;QjhcCn+$tr|JCc)Enx3@DoCF`eah^zT711osALXQumi0hsQKx-ICD(x z`Tde%JWeyZH}F|zZR~>}A3s3n{XpW*0^;R^H`@$VpK$7#=_#g#Y4fY9s&?bJLhHsr zu5H@XH4E!tdG5*Zm2EB5ab}L%79OqO@mpH5zsS$uX4JR7IQphGn1dFkmnmJyG}vhp z({R$x-hK&}$t^0{{Se@Lh6BCJ&Fgu!De&`==aO^oM8O}slmPk207DqfiW z%1n>>yM_nxfaHBz#+N=u{OSdSiTiCK5U-i3Pw()cG163qZ=^nKkZZCz@%4q8wYdt4 z9$a#AvJ@=2AIbiM1$BUC3}HUITXs&Juor$C%agQtKQP1M9d2{8Ub|fZ#!O61LpqrN z{-nUR*>;1z&eDPpl2TR08&db?z6PX2gZBdw;_Prea+aH$wfHa@{r+S)lUmi3*@y2GSd731 z&SZ1x6;Z#T6>MB7f?IQKxD+xqa=dtC$8Vz*sD z(;WfETWf>{JKLMIRP1WjR#v?wodL$8Yk!~8FZ&#&!4LA|@(e2u-oW#M zWcu6sRv6yAd6ODYFE1Fro?$Sg=@)9=@fcpu7;TX6s&_Ai{XwES4~C*nFQmgUIH=q6 zAlG&oa@IZ&l7V*KcIxcRF?j=~$7H=Uq+`vl_)urzl9R4}bd_)*$j;8%Aca#WINPIM zUS4SLX0k!2sF9>JMhXwwY@KuJHhlK{d105i()4$pqwydO^yM6vhrlgMl|0Dl1W)eU z(Ls)R{_=v1Buz`aGzIci{O0YgU<{}zDHAGRpGgJ$jZXl~Xxb3YfA-45fRiUr+K<%J zpkM|<8+GuDAU%^aTsvS3Ke{ZA%+T|O;YudWvb4JGo)+kGMO}4L8Xq4w>nX|w2%~nd zBRC)+HXtBCchY$3wv5c&L^l{hH4BS0aNbj83?_|{ftB}Fy7C_7oKp%8t?@j=sTKR_ z6VLKgXUkv-I0RK|>ra~{G_&L=$dZ$nUu1WbP0>gXuUf?#Iah##O9mHGWHjDgm;-t| zdO{w8ts#4n9Q(@*JohA{UCE8Vm%$Fx6|Zq?eUb!or}p&VF;V@iMrC-v%{d4L>xOMj zzda+<_L(;RoDYjx({DWnwto5JWl-zqK0anOcxL)4gqj<#&e~gbSqSJs2r6Rz*Spwa zyg4yNCpQH|Z;b26(W9*&RpPnawwo&xW4E`qx@QMQIv|2~s5mBAFseKMLC<)}x{mtn zI|C4WSV*;yh5E%W-zzJ}0Z2@Mo*kzXiY+J*lHJ|x6$P=NFXhXQcz8K;+6G>et;LCa zq3GM)j`AJm3ZBmvo!J~kz3;D>wPO8ywNZ4swY9}n{LFd8Ee4N&f7zt5V6?DyhUTj4 z(nJ4gh?FLQQ+Va?AIh{kspon-)Bbv^#Ln6ckjrL>ZpMD?&jN8V`x$#JBu3GS->kK| zC+K?7@1SuA73SBpS5|~Wqas0KmwTN0l07z8=u1WGLb+$@@!>IRnQ9=Sg{FOFr8>E0 z!@U`RUMx$OH7*&H=_)1N@i80&tM>C!em7$UkLZjINYKE*K*rdrr@oh_@mO=>k;SVd zSV!t72MD|X+dgEJZJ2=tGzFR7n6F|GE4n!g@EM+h2m@Q)qLtR4ai=(FHZL+Vmbmtmx~TDl(PG{sKL#yxcbNB?#ihULW5kSC<_Qu44B5Kbl z7TOMf2EXHP_rQt5?yRdab#rs`XWL=9)j67)Vn@R*F$Ouy8`=wuNwb^W+mV%QldvLU_U}@8X8Kh&B zR}BT9hK7cujyv+R@}PB~Wv-pVrtoDFti%Mwiwz5Q;elJVX%HOHchRiwY^^VnA3N3t zCdpVJBBwslA#qwPDMnEi#OSf-&iW{a$Zlc>vtCJF{oqh89E8o`yv6_Uft=jIaH+dDYwghwS><{N9F$B2$*|dcv@xzCacZABc4%GVix-DIAcE?gXUd0IYiSc)sC z%{BGEPnqd6}w}H9(~&@7TAxy}6p|wmO5hN#AXn zACZXy5sLDmTy21^7MGQZ^QW`0-Hu(4bc!9*-Q2+2(g)Rzv?Vr$asWqpKK*XK(%_Fb zcV3^q)ZH`>VE}KJn*rsc-ZKlfdZfQH$j#d_->;Po4ZFBA^Zn_Ha_UV6oQk`fq@D9X z)85m{QM{pF@H z2thT$2~pC}BwxCl_YoGlaOLL z9naT6-F0<B;>lNqosxn6Ls6-A zZ4O6W>XU*)p>Fh}EX9twCQbCD2M-qY@f;>23v3xLnJTm`4vAbDUUnkIP@2?(dfUK; zv^jL>B>|UtR!}J`z#O{5;JnGE8W#F-_9^y(f`v+GNAF{4j zg5p@Pf~z9!U__Ip{pn}?>y3VaR(Yx?tKI_9t>b$j4rD-SD{jO|{PIbTu&Ql?0_(5w zp>LP~cCuZ$qQ+giq?%{<$heHPml{w|q*79}*UmcUW^+qR&u&SSgqL_-aV!nlixSS~ z?ptd%{a1Y|+T8PF(VxiCGk9 zugE}r)dB(*k!Wdao6V(LaDM#0KQep}%i~;3BnEeYEw}-CiJ0RmF}L0ARjM;*LX(1Z zx!QoJJ}2R+d*lq4eJC?C^YHJ=9aWl&e-J{ZF=4SX z_-v-*wn05j&fpq=f|yjvgrxya*LCoXpy2ECq}kI)QVk9GFAv2D7AYtm${f}IOwBHWat%hZRUi<-=ZAqPYJXkH2<97HKf)E&D-B`wE7d=#hBfPK!WvNEX$v~M-ivFbUy;cJYv39 zn;;vs1Vm^GV8#@^;yj2R6Z3mrbSprv;3Nc5~bY7P+P}UGx!SX8tu-5=Ld&P6hSus*bMX=X79+6s* z|CmhA?^TTuc$7VT&O!L1^xIZI&-q8~j|6b^jAYh02}@?JeEen4~&)E*(tH_;h~XicoumV-}#S9^1N0p&lItw_cwH z0FKx~fUYn?$R*|Ljw*st3#SvY(e?S-6zLMteADgXh0CQHN_K`3bsbPh_vA)Vb` zAC1w;v(OI73+`BPuDg^!=0j`w^Kz4DJS=Mp1XDU$_n)7>q>~`c;Blo+l;zeqx0 z`^FE34kDsEck?WML40VpC2T81H_=;K4BXq0We#K%Rl&&Gxxe}OQ&|tuX2ZdngpiOA zFhKLlS5!0fx7Q(kbmSPlsFqdfatKVKHSWftbp2A1CNV*9vsCo-8Q`4}3k`&Nwu`HOvkFR(&FTKV0!^3h(J7|AS+&#$>i=7Rkam2&Q8w~yPO1+#>l zdSG<4CAE*?6nLfrK;LiJ)l;Z7QrkX4z+eOFOA97*NeD7LIX{48G6uNO^QqGUyac}= zV3fkpSo=cVAM)I##;qB8U7xz9f8=)Po4bQa&<-&c@Z-4aBU178>7h3;TpIAHd=PSV z%bR5A!#uMzWN;D^fvZw5^%JL0KUmw`q=jMr0Ip3eWS9=9 zp%avpglTh}Q-hBo^=!%F+h0IUTs(jN3phnyuv2ED^WRVX02cTdP*xD((14f-Am|R6 zF^FWo6@Xlc=?k*b$WL(@lyyIBPYW~5?|K75L5=tnRly4vzEVqgmZ+3_dQ?2fp`qf? zxFTXdc26Nnq^(rc@&{jQX4%FB&Y+9us1e|+AD~rcYN;9xXh~ptelS%N?St+8*3@@C zDDa*Yw6uU1$Lki_N`Z0ogP`E7jg1Yx!)Se&k*?cx?u@Hmkv*?QhVHnE3E(*?2(*u; zwwbAA=yp2)eSFA|nC+@D)gG8CyA5FgA1Jf=frAInqG1tHc<7O)I{c^1(v|QNbaaup zn!kRy`#GD{YufLG?LMFtOh`<0p^NBn%rfuD(Dj076in<{82M$O#HyBu^K=&nDqp_= zSWE-tzZ9s2AILB@@S8OYc0dZeK|==l;@`J#pC34(vt9XCP1(;f4TsXj9<P70_z1R~}RJf_GeC{gTN631a z`<>;l#X@=mhHY^aQhq`}*5M3BL35?RSG)l(_#!7~eZlrh*PFpAe-oGJ2cL80L(ezP zxP(SVT8KhEK?>4Ie!#O1hTOH}eM={dPr`kKT?b=e3?Y8?_R+?Y);WmoMt0UAt}fxV zA5$KL$A^j~89Mg~6x)yEai*&!d8t!F?HbwU6@XQeenEz^!ZM#0j4}@}))pmT+NIA5!a2fc|jo)>uXL9plo8ATK9tFwz!`VuTGKd3K1@Kecopoo?P zd=oFkv6dy^^XZpu%~$~;{s_2m!w+V617Ec$lv&QfH$dt=@WYsz*}a_~Q(Xz>2bn#v z8nWQ_?YLj+X+ZEU4c6dSexK*$q;lbT3^& zX-)>OEfr8M_tSeUsWe23-G!lOijH5x#10uAr^Y1e04zrZ z1PF?E01JO-KyyQ~hVu0353JG7*Fok}I#TyJO#fNVuDEP#yE3JX*lpMT=VxTfFt<2` zg*%Nl!9su!@rU8E;<+)(Qor^e#K`fwe!*1IirtgTgnju4GHCYuQyk*hf!Ul~n;+CE zb{nt@23t2@33?P~0fu8Dvo{D~A19%S|C4Q2; zaFJv`@so1(yZtN_kjOzr0FX-72qgZSQlx^10F)Y-ff|bsvo}p8_pwl*OCTk%AwqnJ zB#j*~;%y&QDmn~fF#+B+PvLf1RmitO;%Ij8q4&2y@C^Vfbx{3Co7u91kYrqJR6OT@ zeucmkg9Dh{@uFmAAbTfTP3=f|^5lDPE)3H9Dc^qndE>bKJ0aX}iaCz_DZm$~HWsdva0Hw&Hcbno3Q($@2U7%uWYKk&- z#Y$jv!Gd2(Hz?PKxUU7!Z5!|f>R^KP5X?qAA*A2qz&RnFXjMpBDhh#(DlOGJcY?#i zCwkukMQCqxdcrU49M|iMVds;FG z%SRwS#Qlz9MPVUOoPhX8?Q{(T-~E*2e}rrI{rmeQ$E0aPs6d{h+W}Sa_}k&q@Pxv~L=M)<@87Xku3YKr<;DXBgxZQ6 z!rgQY`Z?_K!%^&!|N3os*p=viTk~2i%3Gn5DS*i{+3+$1(ZOBKWFrg0z@++VNHQ5tb=sKoyRuWW6vLoPU$=dEnvYP$3xD$p~B@fM=XQ1j3}ilCgtSAx!C3 zUU?r`_0&W3&XT}mXhN|E#1$wrT{zhawO5GH0_?bq@{!<@5I_p388ahSxrWu*Ppb`lIXiZ91;!TpC>+Xk7?tL_s{{L03M|{Zt zWAWemx(Ix!yd`w%r2WKvl~^s~2c=+Urti`xpeT@F0bwTbUaT?klV#(x1G4zZ*`_cY zS3E1&Vs9szFK|J=yze=j=pfU4yf>u`JmW6 zxT&MDeEoxQZf)XRoK?F3Tk)S=SwdADPP#qy+l=y=b7sGEkY#Fny* z3YcAjC~?Xz_g}tHwl+?9690ZkPC>19Wm>MMNIm(gW3`NIjcQ+})Ehdta-pV}(0=Rc zOxxja$DxumRtxA`3?}>2vu8s^t)GR3v>s3YVf&!#e+{T_LYpCX6#=UpcIa_ki~S7~i$r?NtpByUuvCs&>fcvkw9ASuQ1(NKBQL&veWNsUCveM~mDv)#?ZPR|)8o)b^|ZdDdos4Kk6 zRJLNC5m;k(>02sHcr;uJ`-AN*{-niLD6pN7D$V-7I6e8FDAA5!-j+|PIc1xUsk;nyf7Lgx4wA0#u%Tg zU%GqAMlYBBMlg&xjtHdWM9bOL)jX%PR#W5}FPqqASWwG#l^d9Du5$FJW*R4Jsa(kK zP4S-0;3xuJmJGXrhDdN`d3|f;;4x#XpOP2y`!beJv2w8(#bDFQnb{I;958K-L~eiiJ+nz)y!=3A8)qyXc~r+P<>;**xK6%e>~g)ri! zmfL4E)nC29J{f}tm=aZ*U>5RInbdJC_RH~e#tDgR4y%?p0>yX&%Tq(6-L{8dz*EE#5)tux%0@Nz5SiptpEuW}N$ zH+@>djJ63192|);LUAw^d&5fdu%}OPR?Rn8MjCKdtuhb1cHbXnfO+5Bdj^yd8>g#t zRBijz1>;O7k7j(o$-KLfiuc-9Usw#%a`P|dA(ScNn{saPX?|4*saoNeyx9r7oXn!|{c^THu5AJwcSJ zfZHS97}C#XiQXFN#zIlHQ+-}G1Nx!zEUElmMyh#Mp=chWA3r}m6kOn(1$tK;#G99z zQ9G{%1F@W~dm zz(NvJd1HPM3@%u7(?b+;K7oT-V9BWTVqrj%d&_wusD{aWc_LdQGvhBX$A12r6M2v1 zfRlf#S8OaqN58y{46v@v;%Hr_El&F^Kl{`t5PIRnpSxOXEkr1C z%U8{}RIhDM=WA{@PS@n_8jKum;xL`oS#^x~BB}RDhst?6H@iKZBdAlCXtHf3krc&J z^L1J*Sp8AkQH)@>ul`54DXNY)5N}&kD|by+R!+ZAwT45pmhL1YQE9+?_#DHnta87a zBf)%OCi+QAJJ4TGD)ylQiQ>fc%LOM-5=tzi@Ez$vTU%fS{tle%Xb7)S&CoS`b_6^Z z(E`UlJ2KUL3(S3@WfzO)_3MpPEiJ)C9>t*enp<1z3#sbzI}PD%1-BV|*kD(xi7z5p z?|7{~X~({zOS^r$Ac!duwv`vuC<0 zQ=pwx#0CU8GBEt+Kwf8P-cCPwoWLfqf((CNb(T}+kS$L5M{k<;+fh3@QLOy>LiRJC z(+_^by<~g$vX#pP4|S9lKPhqwqs4yJ(`|w`JCbPxsvB z+`LTT^d2FZLZaL&AAGleKh@6=UfyCdDMpmdM4`SilY&75 z=Au0j87A~=+o7#9=LjnR9yv0ifZ_aV?gws%n;Vypgwsvx7JcRi*UhKkL;+TS)2~%6 zYzk_=K|qp;ON`Ne)@}Qm{dUC2B-j@$KNJZdYY~>FaZ4eJUDmd;YO+2-R(s+%@Qcv1 zaEp!xKokW6QJ8>>@-iXnYr{PBEG=OFA(?d3kS*cpBjzwGHLCBS5$Hm zF3kuEDsK@V>a7Xz>$>mp;QHOoJAAQvSHaABP*Z`yqD@6$?~6IO2+G!YUsb0m<=+7p zv|Yc$TW=G$CV=~b$0mD%fkYVRX#?MV0QL4T?fU&}9glP1f|(E6!7$JdB_DAqvDS@i z&NN)1nH8V`zlo3vuuFk%5d_L2Vunw^l|p0>m(xNz4VG|KWB>l4s))#RJiR)NK4Cd9Ly^->Ob9=rtIjS<^_wv{C zU=EK{|G=UgAa>hOe*DrHg-h4#isv*OS4CDl9$q>pws)s87%{^ezH%%ryW}m+l;B37 zhsWtL3Fhw&;<73KP0Kas_#}VEKfCo&7oo@{$7vm18kBg(n*~ECT zXv)3vXM`{JKzXb(l4aM;X?k47;%IPz@yq}I^5BYNG57Oh#Xr!w`v8MQ|!x&88 z-YQ=Qgh(mi{K+_LEA!{D>Xu2&Iu{n!kO)LEeB3(tUsOLu=|cy)oX|&xs;P9#!+;&hw;be2wvf13Yu>fRz~R zORP+QI|;E<2MGYMN_(A3YlOfM7HRv>L=jNl(=`-uyRiQx@v0PlX2&txO#irDsmGE>Ez-aQcAJzO!z zX5J7^UetfkpRcMhO0Kiu7jaB;dt(}Xvbhp0EZQBy)kCqrHWp`w8_isee?y z1|^GTe?BLp0YwEM%CqWk_d!bVC`Eh#A_4Vz{pFEVSHLL)A`bZCABuVB>_l-bd z4JwjP(?G@VAE2ZT24KpL>gO9dGE3e={k9(DB%7fi`vY*sC}j=VY&{bg0V*p720j_S z(t!FdsAf4k+{*xMT964#hFJO!q=Zpt3*wwmY0ax8S1?QgSIb`Mm)6#?{|we? zPUR-7;nDIsO*T>B{#BqDkRP}VCv7N2=`o=7iG!s47}QJVBcesN8M=i&Lh^HRGPArL zQk8Kij|Q2r79apxp%q8A8Hz;Wpf15260oRv50$|J-GUO1kl=Hg~m82aN-pI7K&D_dAy@SWchI~Ux7Scp=}G0<|L9%z0lXqiDp^uX+?1E;bKsZO(@ z>Og$L*x17;Q5RJ)*ClI)Q^5J=J?b(8qO)l5?bXko+pf=jeT##*@vTt)7}oFrD0wT? zbkLq@kO`yLho&YXQ3jL#1yr7xYqyLlv}pi4*3&gVKaWoU@-s0pk*c;7<9_LX*)c+h zBfb6Rv|Lx|-N%AGfFm=Y!q~v~*?qX)?!4gV+2*PwvUiAU0cNrtWwerFJauRVEPh_} z7i9$Q446XHafVNTWpty{1cq?64Dz4skmWHdhYl#j+^VqJHHFuYin=Tqha9MftDz#} zJLf!?cJ|H3j~{DVS4z(3Km5gL;ESOY0mAEvYD=RwDOn+wf10iu72RgukR(|&-S=bI zHW_NJPg8i~rO@*drFiUKD2mok25zZK><2lM~^@nM+(5~z_gdLe{ zS4b{9PHCYEF0^`|t@^|pVk8Vu<_?JLi?TrPyF#D9^0y=059bFe5vey*wrPoS*@$Vk z&#J0_1A+|+&xzGOuMSkO=S%M@UshW>2N-GHFdssd*GBTxv9s02IqN)t zNfv%2-3Wk7yi0fvk+K`zJGrsK$jD zkzknLdO^*Ap!tYO2oy@#(us~mcmvWsy7$t+>$~$5HD=oej}5UqLAQ+ybjyyAW4!79 zed~~B*8A%lOY>NH;Dlt&J2nuOvfXP*X4X({ie5<7X^ZDv&+pR;Ig3%A`u8^6!r~o> z`OgTZX#m(tBKMcz3aT0Eg2@&P&zE?KOwyNAlz8L3q+jC$2yXnMEK;K z6*)hT^iEld=Kzuv+*_OeCnrNWRR$J@$u0WyuZg)y92lLLK7Prbd3z}c@42H5Yyf3 ziFNFC;B)r&De)~c6H`^u!Ey)=nYK(p4R~h8>#*k7 zPabYz7I}|uM>Zvn){Pj0PqE(6IUx3zMYnadypcrY%d-?S(C zpS~c$R30N@S8q|iLkF-0(iFE!*4}@U_6sm*2=CNXkiP?rA&PiZ^DNl@K%iB5l6X0W z8)7L^U8kt6uFrvkndON10zN7%S6|ZGqC0$I_;wqkxfyiy@7;SCur?}>~^lcEM z1aEE5;_H}Y$xC)z!=Oo=ju{gxv-i~$Ib|rO~3^CeX16rlV zQd2pU3o$o_2^fsQzqg!6b3Vrdd}1$qkim+V<<-Ffd&6ZjHZi@Ibs=n@jH+IrTG2>R z`~GVm<`p?{P_h3E5!?e45@yRon_IXoffXG0WgvWZOy7d+-8Crq(#W?waSNoAoru4^ z8G$$)|K>08Ag4n4Uvda|R;t$2&?&`;TH^=(su=7sqM~4KRU7A~eEXeM^vNu^>RD-+ z@XB`=sv$8F;+6e;5#eprKES9PTVIc#VBSv9%+odFAIT+U9NS8yFIdrXLZsD=WO) z+M@iO5w-76UNA81%o$o2sxtIuwt>qW)=Ydwaq$3V=*-^wy@T{G|LzeQfse~{E+oY78M`GoWlOf}iwYYu#R_I$X28#gEg*M%|x`yX8JFp!J$?g#I; z=cpju-(uI`OyMEH2=B=P;hv8n)~51R+VF8-eA%u(B!V`|!eN5a!yAK<-P_TV z+&~DRq{Z9vo!X)n9zwQFysCgDVR|B!xUG=9FcvuD5euaRdrB+E1AN>ri{cPaz|F@Q zGzP#%tq)Sv4A-h6W|Yotr8?70opgCB87OZfG;zj+QV0ONRlfn(S^I`MP7d0oV*w9j zcjcsqdJa4r5Z_WxiuTm0ftvL1D*GfmZ&FYrfgsGMDqo9xdJyqj$jox3X9e$rw?&1*axuCj6pSO-%sq99N&vqu@^yiw;}|B zy;?rVUA33-R8Jkz5;Xk9h>|Y`xn|@Ozn+r0k^Cj+CF6#XD<9N`zC@dy(9aj8hk#SZ z6Fj%pz^ul&NI}fXs;R1~;zNc>B&Omp7-t#XEJpb-4lX4!1CPY-iSksxlY{mZJB1oR z!uiE6hO76Iwk_QM{rxefkM_e5u{e2#n@53hBCb8A5b{G8yQ%}_ zX8R0aNE!=G5_Q%>*{$+WK0MBP`vIHN|LNIf;ws_hV???NQ+=xZ6&2NrzZVwiu1dRaGH;N0 zU$6WJs1tZg?a7k^S*`X{B`$LM<-Mg*d0yowL%GzeFXgW>iej)Q_n?q+!u8pL8zHZ# z#A*Sk1RCvnA~gBjpJ=(f8G&@xT=Q6!9Z}eQOYL)iv_6nSxD&`M$qR5=2fQV=8qyi+ za}9!S1mUJgm93;0g#56&d&{+Ub52pPMh8)k3z_BY4x>uMhx-yAj)~y`3Z;BiKWuB8 z`?MA(dEvA`=h-WBkM3cV4{MPk{N&xVrv5SZQ{0WOzaae*iV+|KgHX9H$=>K+G22OcBjBL$VV9~%SMHtQ?T{sV#JjBT!FI_k+-pBI zYiOLrWn4B<^Xd~vYrMW^Xe6^vcNgaah@{7<-;P$B_@Icwfs(yQLo(|@wic!5Y$@OR zrW=8EXSEzMI-@Y`FT}(gU}-l`IJv-n?w&v2MPcnMPQ_!f&-VO5?9Ia#9nfjp>b5%Z z&8ZJDr`7*1x18Un3pb3p40_`^`=QmtbHopjw|5B?+P(c$CY0(A3)?O3f!K_6<%?diHf>kNf+6y&8&3FJw&<3X^#FHL5{SYdnM8E-A7EcST zdO}$`6cAPSTR()j85PW5fdfZSpNk%=h=>k4+vS*GcPzYi61qa$(CI+Kb)jd*9e|Pt z=948tktF>t=qp0JqcNMXcXLCF7i2s=JyTQ@(-Y?%puI=6AnuE#FZE_O6(7J=DM$lDj0A?ruOBjXg;A4vH$FEZ%HkbyX93 z@A!d0LcIr2RH>$;6Yjb+)}PT8du>1Z9~6aY!oeGeD1&A-H5ig($IDl*n$hV?h?4-0 zt__OH#2N!tMqxb~ct5nZszDi0QU2iIAhRr#UqO#9M=ZyoSD}7+inP+EQdgz5CE+*V z?o|rlLMqF)vi%ginyR2XU4h@M8d7S3m4|iNq6eX3>J$(3Mzo<;2_Rv{;N%N*MiM%i z2gsXA;Jctmj+}yGRpM3A3hlY&dfE44GsOlbWz8|u{ zEFS`uggkv6M+gUCaTgp8OUC_&NUBbb(u8`{V~(?qRn^ml_%p=CafUXv zx4)kpicYaNNEfEzZ`pAY()!UG0enX&C_3OwtaK<6ONIt)sQZ5f{cK~Tr%s(3jK2H< z+D|vcj6xO_HZQN>+{^5T6X4*5=g{BrZYZm3I=gFHyEa{e!>c;>|6tFm)1yL*7nP#k zz73h&a&gjb7VN`*vM~Z+#Fg^TWA{NC{f#wus)TxIYsvM3t|_3qK|rA|uMJ9idp4oJ zB^}PQNrf@t6X3upb7Hv1@WcO_P_ys zS#UJYr~k+5HpGhqVLE|GX&|l!XTI`5(+}#mxh*>XY^HQoADyyAY(C-C9;lN)1jACU zo`RK)rL}5(j;rg&L4i;uoMjmGBUP_IShdpC@pk+5mer{Lw7#(k1?ZiW@uzd zh1LQl*>G`>qWT9p2ccaU6aiFn9E#{|8LYCjmVoJELIItkj3-^OBir=fYmf(w^#jIs1ohrJl*3Q2`=TN$ zjuUQ&e#JaEUn_jl1r8BQk=Wh3Y`{yj1%(QyUxSPLIX)T_O83@c5^~wy1}&!!N*c0h z1kIsKB(E*Leo@W4&M0fbqGa!-=dd#M{xF#&jau4X$ujKMn>}ZW7@+_4`@;9&W(>jg z#GNF$BB6fE5n%(S~mZ`L1H?AA6tcIe(pBEScKOEKtH!6Rq^q70x zShAIUG_n=+L5Ab~6&2RUh6S`S&^_WV?T91?XtAqcR_CQh20|0)?`IhuJHd+Df_L3=p zir6dvV!rw#6YYvD#oJz_K5<9tLN~e2V-k<=fl*wvFjnTVUXe3_Io*JJbbS%2m?)kIswf0fDlhCy9}x+FPmfH78Gh`11CgSS`LiGjuOYM_};`%|zRP!QD{K z)R%(}*)RyCowt`EKDq(Xeup6mmUM5$zVt!FsPHa5OUCCKE*KHHlnrqEiiAoiJ6*{|gJK}4;f zXOdMim7r$^JUs$k&(EG8!6kf+Q4<>@DoR_USm31tlafp$0if z7lrg#a*zE+w|+^2{AMBKa}XI(JJOIwIPk^XMd=HjsIxLUuSk4*>CQROUmwGQ)lLfm z2RK&>a$?QiB>Rn7Xi?5Tmhuxsl;vmnk;D~q0(M%_LkTbPtCnbBSbG4mUj=)HR6;!0KK+|g zlS>g@_9laa9pNA#wYclgIXB@vG$>9K1e>4;o!)8C?{)Q@u&J z-+~PaA0b&=PzR&}3xxOX)KuPhmfK|SpqzSi{TXz|8I?UnSEKF=0S+N{IU5GgF)q8Q zE>@`i$%bYYG#k9Ln6^Chnu25N?t%uKW^hnvL@~2kOC|rBLiPZvBAl0eC+ILjV9IL8))?7gT8q6xZeTb+!UcZb-pTcLh7HeFBWC z-0U3X-3iASf&1o^)hV!!gY;$E4d3JA&?MH1GHriJh_=Y)9|)-HFI=FFM$Wl-HkZN!J@(XgQ@Wq;!AM5wdt z2Y@F$gI6Tbjz6b+{{_izC$T**`Az=zH~R$^5LrrMdXnO>TM%E(7l^^(vXv))jpTEl z-*-HBaiHV;r7`|;vp>i?y09SkcoHl!h?FtkZR#m%#E2BVi)LZrmRvahz4suZfEE#u zZgJ20-Uhz$*if^aUVcE`QNg9Q3pyc-sYVG@3{iBrteZZZ%!q*%snq@W@#7#nO5Avy zL9Ow;tm)?kKoSLvF2CeK*VbbYXnlnGJKbT>h*&*5ugYFq!hmf<{V1#8ViM9_mBjZIROd=03c zK&67O4Z`2m_qZf9q0iJt^%0s+p`2!;-&4tmVjE%84f8Dajfg;=RY5#OS?v|czw>Ia zzwt#-0)@aB2oT;O3zQ1s1F8TOtba|9V7wN6*4#FEWuVxYs|V<`c=hI#Ki3-h@(pX& zF_q-v z0i<5?rOQi~FHW6zfz6R5dP10==^Z6vl`zpgA~qT=WLaUB;{lP0GeoLO1?kWQFP}9^ zUHWl8)yp?Q#f>d(uKt<1NT}P|b)KiUSE^lwH-VKy?A?Kwg9i`V$(VAH^Z6h6*Qzr8 z3S_xaq`uyv(qo=*v6%Iv&3^cB%ng!5T`mx8@bN>y*V;h~2V>TOehyRwPy(Ufn!NFT z@)-~j)AUep$F_ClM--RGOb7Nfj-5`5gE4p3>Qpoguhdz?tm8 zolnj%K8E>p1m(;E;$&qSPVyZHFNx|lfBgHM1xx7{qR8g=?r~H+FTKhpd3kPz{MhW^ z9_JNd$`6c_FoEG;!dhcj4ai9S5$7OTi6IIncx4jz0XzYi!Y$9RZqy`7F_>!_e!5KF zF!@xDPx*N(L3T)fwV#Ab>x#8j1fyKt4dG@D2gKmI!jcr$!a=61p3VX<{4~MPDgf_J{Z2OO@Qaat5BzkwUnpheQ6@M zu8(gMWRji#7Mm|m(84|_0bc6bery7jMw%}Z$75+86Rgf%spmoAU3atS%A&X$Jz>;lF;q(|mMkg0oN%`2 zuWR0JgEGa!;AavjhHazBp#u%Z#z1#@`JEnKZ-8x{+;P-WA#yu|7X|r-4*Z&knJd^R zz2J2S^f{6vM!n5J`GEaa8wp*;)nqlMb&ZQJWw~MkA8Z@t5{A`VJaVQO2wSpBVa`a5 zM4@Xr3&kYshAlV%lfO9hLjo12H2`Qi`x}oBNP)Vkh3Ki&EMt;alIg}@H1l|vsh zt5P5gdtvBDteaRjp!SG!AIGai$luSU2K~z-<{nDG8L?5<=)rK4#OA|`qoSO`Jwrq- zeN@BtKf-iOfL8q}_7}{`7+?uKaJk!kcO88gYT5dl^g7<(eNcR1j8{$W@ zR!m`Xvli1NOo(0$igBxsk^oQ#3b-l6h6)xT(#I9(5n1~^55FBxe~h%4CpS2QElYho+5;!%r8CaUs3S$X0=+_ zr6A6P1j)J1P&Q6UZLg3u3e4&++69=?uu?MhCEkm3u6V-YtwynmH#eh z+;EyjajIqI+?7`b-c#eIdbIk03}^cm*59_YV-9f5&h84RT6s1j@$vApW zi!NqOnBIQ>LxBqC{UP7k8A?7APjG{b33>JioQU{{|E?qXY*d*j;%DT6JpQ{=5bO#( zKc~yOhjpR)V+{6XNZP9&DV%ysACvLWV(a7G=U*0Isndayq3rQa`6XSWzei(jQG<|p z_uUiZfJ{L@K>OWcTQQfte)-Wqj<9Sg$Q@wzg8`PqSEr*;DQki!WtO2Y^Q!L`x}sb?Mt_uhN)TD`eH17H@W} zy@R4OP-m1xS~ZO-nf)5!dI-;nk5|3v5-Z=usNf>?{sj$CZmT`v=a58^7!v~^B33J) zEYSe^1eJF1Go~C3IQgkN;G9ANy_e+L3N_@yS}4YJ^ignS#n82#H{ zR~#T71+|b`66A|c&LNt`qdr_85mGOemM79XnPJ1E;Xm)E8ziSI9po(cO>^ zd7N@42;E@UdIGEm`aWvEj&~uuzE_BKB2aFLh=uUc?p%S0N&7{r+|d!S8nX@ik0h?1_jryV1@VMq`=y|9$;E?7@vZ+Hmg{%`!Vi)Mq{4qr5&FWKg}E0uXkfH4C03A#__r7E;$lnIl2(J9TLJ7L7YQW-MDa7 zw+F%TtHI&l?JF9%@vG~KT-8EpU~j}vA7XT&f`SIBLzY1{R)x`OrQJ^R-*08%Hu5HK z1K3ouQ+=W{5aCX?`>Bn<9M=kp{D-2Js$LEr^o53rRbW{CNz9F4R|&VtbYQfxRaI4j z0({0+ctxFI=(Ut@CF<_(>W+?%kHY14llA2hwlooT1(-PuM9_md&+kQ-B+aAaO08c` zNpg&6IC#LvWFp3Gu*=lW?=t^OMUR>TNTE2gjo7Rzz;6j)g0pfP4X3ZdLMv; ze?1ZUlcgPLTcvIinFH!8x#wncP-F`*9lNkTAps(rL7K1n2LNdNU+y2RjHv5vH%oEO zyDMS4RvVP;$!>#Jz9~G<3IC92yWvtr%o&uK((RSQ?{5h3!h~SBv`XjIiSJH7IJJ&&w}}fJ?!l#f z<4`GWp)zdkGT{0o1M>sw?90wkVk(K?#Kk@te!|*tK(cY*Dd}UBM)WNBk`1z6K#bp$ zY}=#q;Zwdr`1rUZTrdT;Y}rzJPAx)O4<>JAn)x^)3>-RV7I>|MHR@s18hGu}ZT(s4 zKf_?XX1r`PtK7TLbfznkhEkC%JDXF0zkfUJceYwgq*0`rT5{lgz(K;-1gxMT@SYmVkJ11ieG+>PeK@ORI~w( zdBltX;bkrO=}$mc?*V0AgOqI?etHZeD#04x6B~Mg_~->->mvKc=PcVeV&i}Vbxuf= z077nVLhYGBR<)^6$aj*lca8)YN(`97sIz_GIw^~it^!{i zv8u^X4N%kk@ktJ22ZO+4#l^*BIYpNSA-rk*-UD6@OM)>&&l0SGM1U}VAkGum8SGXp zE=dG-3JSg!@zKaB>#^&F#Xs@R%78DxzuO;NzCS}SDZ2l`u*HAyKd7{D5>KovVc8V3 zsh`W7`JbQZp}tX?3%5zw;uS_eWu*G;_{*#VcWu^E-!eEf?8cUM!bZ^JtwjwyNp_XB ziPS!_u=VLGy^onnOhu%n4N&DOzfw!N31i}TppyhixO(*}7|Xk1?T)K-ga0FO)Y3^d zkZ?#YtQ_=6xQF$-hpdO#TVZvR1V>VfsSU(9%>#^IvKNthFDagk<N)m9gs0K z+FmdePyv55x_cLFshVY$B=18}nK*1gJrIlHZ;l-Ls64}QvkNlypAn;%so=YzzI$r_ zac?kxc-slP2a$jPVQ#V)iQN9g=jEFivgSrbr^3`VH5_~@g9)zBHlKTHlV0s__UQGe z8YW-;ZN=fF$L}70Jax)>=eC1+EG&704=%IrPz{SGML*ar5iQ7t?4DRIn#G9ok`*g- zv7)YCv!*km$Q&N5BrC;cGXmMyIsCe5yABa$5QCAw;U@@=EvGi@&i!!}(FiGF5gU29 zn!I$0#MV4F(=wNUit1DKM^{OQ8l-H7KV^PQx?lX9PJwq;_$56!YLK1<2v2*mx+46_ zElO*gaLQVA`?&uC3KW#`|1y+?oxIGiPo|^?%RvQ&QCttH5Mo6O=T3mHIPu9N6E$zB zM1WLpXJ-aI&Xoo48Q3KFdHg z24|Pwd;}YVM%a4~OQ=ePPj0)Pw?kdTjQjMWMT@(#jw-vGsp{{w{v=%bTOi;D^OFwf z7O+9u@ls}&0nFEuQ=AaOTjpeI0*8Qw35{-8OdLb+OOjF{cLIQefx4ijRAd-XIl1=} z=TBnZM{Ir7u^p}vJkh#5GfElV6u>B;*Zv5gjoQRV;!yf^zW7&iI+Z*?(y4>)P51A` zKVspF7|j6v5^Bhr9v>gS;2r~X<@BrLM?(h+PjRY;M}$v#Q=2(oagb(lkj&|lZ7G9; z5IHo7&Ius_Az&`j@KIP2cKb*TQ~|vm+E|7ds5P}8iX&WGjj3RL4B1j*l!7{)zrym0 zG+QWhFd&msU9+*bPD?*F`c7M+mjSJ&EvBzH`6xSEM8lGGZ-m#GHt8u5@AWD4-F-ia)PyfbT! z9KjkMA%Wp2#5PJ)gqY1rGW0yrd+|xL=%UOA)W;)jMdt8aSb2M`nNBK`uH5T6l5(Mo z4T(%hY*i^ee3x{oq1O_yo2oPF2TVhiE7$B$dgpCpu_3h;;YX@YBGr-E+ZXE~@Q1TN zU-e3eKd9aZDmkTms_8S`%I)ZhG6+Q!+5`AIN-$QaaRHXYtgTe z&DSH%LS$m1c+L8RrS`pNl+=Cwsno&Ya;0NcpWDsK@BQg}Z{K$>=~DrGWMp_dP;1nX z4;80I%Tu24EVGkr7WnG;WtD}!jm?2uzxg}!oYOZYAC&j{aN=iY_@BRxE@v+r%Na(h zBjG*KD>}ZJNPn9=3W{!tZhfR~L__=lQ=m}1P$cv0(s%efE8+3^ewD7-1o_k4$l3AK z>BoUPd5S}dOxJr~Zz_W;)|xwqA$Aclw$P)2L$X>!JHJh5!H!+KRIqbY5vvwdtJWM> zCYR&6E%3pgJ>LgYes)z92i!7U=dGRFUhnrVAh`RdMB0eoPM%u|FB;7NApSbhHfW^U z*k4IO1P^m2*T!}wy>B`T-dknxcd+fSqUTf`YPg@V7qw25d@~2>#BCGx2}PSL8!mUn zs@l`*&4N>Ea#KlCm;IV&`t=G+D&M7*VngLL%~R&CrbJh~X7_5E08KV(M7YV82!@rAFQK z{oRlLd|Lhy6^cK}t*ckBJA2lq?@Qd<)dHlLgcarC>a}rGYT?z@uF4+-47t4R<`T3N zuXOCRzgB(budA#CLf+*1;{Y135hos^ma>E47AjQ7kVJ2gurbRla!30Sk~$$+i-Y~E z8}?$dqjD3G6|uA;w)$97ni2jUcGA;*3zQJ}c@(_Rh+!kq8)DCs{^f!1&7)RPtzobY zi?MFvmDpV|^g?KtN}7IVnQk>t+TBOTl;7V8!@geB{N)`WbMmg>b=w&P`-VTk+R%aT zCxWRM>}9=(#D0Bx3FQ(g9FgP*F+QvflK;eNhn6+O)dwtw1R$OOpoYiE2r+S}p*|cA zD*Djk6>GXDw2l>SSquZ-dbq!7!yOKElU_VOUFhUiz6g2QNz8`upv1tx&_P@Trbq)| zAZ`lD35eE=#DGM1Mo2V|^Zi|ivlo{&W!Wy>PRHBix2Dx4#_dX z$v?c~{#3TjGTiczoBb@tJ=+n#Uv; zJzUy4H~n8ioI22(Y?E><$~Oe;HZ~$VH4Kx`x~!wCYbV^8k!35B%>}stGKL>_eol1= zWN5VKZn|oRelmob1;5oqk|Dt%OoO>VXgu zWV{%jHLdPlpIx-YuA>V@vbb@ zSbH(h*H?Nsk6R2xo?Wxqgu({cROZ)xxIui$Cd?Yt64PY5&{d6X@B7E?_6s`kyif(9 zndO*SpFqxy#z?6PmeQ-#>OYc#xJ{~;bNOgHvz+;rbMz9l;3Wp+v@)7Sw)0_KvE}HA+XUU(9EynI*R*wD>zXo?#>G*c?aCgsY z#rgNytZvowqLyO`K#24RQENAnggWFjXrdF379VGbvKW2x<^fGs0u5H*=em4RYB`ts z;X8+)ym{T^_cQsJV#(O0$92WUtgCo!%&Xpf=*peg)P6428ewxY{F)~uILK|Znj-(_ zd1grastR4t%~K!DM9e8~4SJp)<8?fqsDq*iUTxyLYj*k_UnIKp;nzU8>(p80SQCX68kT5s^eicqZ2Za9) zzt?nS^U4ODG;sQL0`0lP#r1D`*?Gv&&~IaF~vwS6MeV;nJ2 zFw(VdNGB^a-JzTA1t)KpZxp#ArW*q39v-loHcwFh%XIl4ou15Ww;D>-9nc;PS+Z8( z5Ha#2i1go|FNJIkOx@tNg~_mH;1Eo4DIDqlqN^r55lu>ySgI-6`xlqN&z`vD!h!nJ z6CrYG=MWhNsikTCociUdk_$} z0t9aIhaq$B(_i7&HCQM=*z#`i77Psiy^!yL0#Ss0`xiIv)1CrvD(NqpzXs+;>bW2; zm^_`IbC-DhMCVk~e!WE9%Af0Q?qHthL^GA}qKE0L`*MNs%5c#8JUf%M!$idQPv+b= z4CXxV8wlf^tT9!8^o|c%q+LLpl>Wc|vNC@;EF|a45Md&MX-wY2oVJaR;t(krbF?Dt zqsOXM=EutZU}-PJs}-vfntuL^_}H(qvJ3(iRH|oZoFp@Qc#2}y-TW0BgGX}oZn+tM@D?vka#4uGiGrC@jmKmM z2$1Anq(dC3dSCS+1Z5`mKf%RIvhQ{D@33E|8T{_?XZN*>Idw8m)NbD*&&+M;Y3(E< zSSie&H9K%R6V0ufp*E#P6Q=;=++$arI#d!?fS&TIg8hB zAxJusF9ifl)v!X2wv2E(0xT_vdl#B5Ae8xMJZRMczW#x+P|LWkVpmsm3Xio0Kw89q zY(4Q2(1Q!x61d0VX6?Wfd!d~y2>X~E!Tv(#MxhYS*ezjKBv)IAiyqir66X4B$r@Nj z0uWUruOf^LxhcdF6Wu%5_Uqw1+ngwNcTtp{>ys++!ue*gYfetWA2E(k8m?}~~n z`C2en!7|mJubX)X7F!2;(CRgvv++(9V;7iys#`hyb981}dZjS8F_RinEUcNnhG=RV=s;VA` z>9;oM_M}V$`M%)%-t$hmQ|Dgb4xZv3ny2UK3fd-Nq7_GTQM7xpJP)5J0LTPf0;gPK z-A_i2_Xqrzhl;Sh-UKj3I<>H_rUxRRf=<+D8nAPc&Xa(^WE!&3RPLD9 zLOSD+*PUNqaKpwmv0i`urGHFq-ImHBf6ox`Zjl;7)!+fXl}wTI4!_$BA&mcmBhJ~? zK3p1~A1ScGplOGnWy{)?d%z_0_q^@nQ}OE@eWvw%2R~?HUyFZR)z+Rt^O$~U=~6Uf ztZ9CKG7r5^)bT~59n!)B*z)S*(&Q^ln?+a`^2+RZk4Tssi$jo%(DpxuRS#WvUc9K& zkw2(}h#cKb||6?1opVuMw)?!+u7nw$=xe_5h0k(m%+@B$SvtF!pG77JArjm1E^kx}; z*xHoM&KgAu^gm#{!k!lmu>Ui?K&7h;aG4!?OY|ZcP*hRTL08w+;I+Y$nzsSi0dv-}e}rZq~J&}tK&fUL3lgbDr5T2lQ5cG_my*o4as3K3N^?3h5M zRJXzRP<1cvv9sWla~_EYw8h2^^BugjgNE5$8P}XlJY4}l6N}RF;5~2hz^?H}Y(}~* zL0Td7;S>GvvUWYfWMasAFXWv-q`nt57?MlYsUonXsmli8c2-Vs4Zn%k1Y+0vvnK_CVYIzaY(s%K78W1^k3wCp1I&u+RD~!_^}>@E4m#l zx-tmN$3FS?uu-*baXO0ay=Y!SEhpgvnsEk%Fr)w5%)8@Ijb8L?j{pBj`x zaZHnW5E%)lYa{hIdOYsW)Yamw)am;>NREk~uuXhs1p5N!cT}vwq#=$qY-$j*k=8;0 zP))s{^1^?->kXDHA+l$oBz+$6%m{yML2`~y$5St5EWIPNhDy^f2u*?B6Wj%%$8q(` z^jYf4d%9Pfver)^qzemxo}lLPH$K%cJDdLY#@qOnI`Mjoa&u`6-TMIEkbS;hr;!!h zC6o>N*2eVi7>MyY&I1xLv!g+9mebO=v-=egrW!Ln5vMsjE&64vV#W3Af~#hZD2JCk zRpiYWl39!}z|t(O5q*aBeSgxQiMNE7yyw;A0wXZCQZVfU(%tijFJtnjPu4U`b`1h# zc()a;J^x&O4^WvMzvjN=jPa(>Bt2Ry2MyNA2d}SUT4v3+Yw+PiiGY`DVz_npIh2Z+ zYc*&+$|Qb>pwryXDJuD+x_Pq z)c!9W_6A)dYTc;9t^98qHOrnHiJ90uw+9i(UV+Yva;je1i@u*le$X9pN_DerKGyK~ zFlBa4$;jwu7mL^G=j$asozQu$roKKK_10r%hsoNMd%4T4bj}+ zpBsDV^rULB_~COK3&X%6T2Gn?qT+CuOTs3@a`8qxbsx#z_@>K68z<^t=4br16ciql z8ufjSKdhAeO9YNbUq~zW7Fx!ZR?l1pBb2bjqA3PLfIsvHh>_?IH5|TnHsEHS< zwqag<+WSoZ@xqMc3+ES;$gvSeHXfT$mFoA;&dyd25i!5ULFTRcu6pN1Wou4aKLDP( z6?nN(+c0_$%R(OW$>au(qwaRZz=lFKo2XHXYjbI#jauK~)6%pYHhfyEU+KAC&fT>F zOo*ftsi!k6Ns)bR!T3AZ_U)S+qu!Bk8}{B>-ak>N+w8kFWr|(go|e935WPP_&cAFH zhlfiEN+>}#Sz(CTI;C1BR7qc>%`@Fc1Cjt*d-7{oop^KqNptgz@oyu38Ou#zEFBKW zv16(6_`wS4;WD(o;-(~Mm!K#cj2u}SNsIkEHVj>qb8SX?(v_6GmrR%gg$!caU+zb7|W)gpT!8-glX_$~N z%NO9>EZ}fujrKC_2qp}*=jo9UR?{Ub>y(+W%*by&F13Whq_BEbA-+I}I&&cQqDwsB}97a22a})>~kpYOBeemT1Gs_@Z3=ix~{UW_~*3+__%w_n20{8m&LuWRzxS7$Hm@l{yfV?R@u;x49ZaQQHKIqz1(EkY{`WfXMe{Lq*A<$?r&Lt8iqbtxH7Y9i9{ zh4@PZ`+PZiV~%ncldr=vUM|SU`79XgErWyP(bq=Yvse+FoSfVQZ-2d9$IG8Te-=TX z4!bK?EC4ynnBu@W)_J4#A!6r_{{j$Q_E1bfYB1Lp?1x5C3)GP(P;HLf8R;rA=r-sg ziq=m?k*QW?l%3%tU_~3}(;H%D8NIr6iW;4kCK@h&a4tDK&5wl~Yw%7Ok&2p^vs^oU+>+6l$P{|Gf0JoFUSsB?P;Ho$3 zT9>wOL)xWpuLTXW_d-ak|11`I6k+nJ=;p|QXT!cp+A*(XwKywjlk<^B{#}o)m6vSX znr_>tjiBdy3wbR|`oYvGGw7s79+QucPZRnzhd?Gtp7z0WlEY=9qE18Y5i}S^p}+5* z5HzqAZ*d)7AaJdur?0OJJv^yorM`FX-r*pY)sHF@SSQ2uqbd`Rjv3EA9IB4tOECyZ zG6<>m-I}mjJgNJII2-8NiXVBNd#&D7$i7Z^%NOf36{;ER>LIF;rj`oj2u4n~?@YtA zN`Lus>>+EK$zNB|Oog8!g7}9tLCurBSjR_6*;q5y9*s;rIxEZRUYapAIw65i<=iK? zoYd5h`y8J}(C?CQv%@OJQ!4w+dlXiT8eF_mb>J+zvMmt!?AYDktJTeS>|Awob2t>d zmbi#xrlzK-NG3;E0vomiIvJFinaNE#dGaKAK)EB;N;0PCg&cx6l{_99Q=_T%iSLsp z$#i0?u5wY8N5%Iln^%c|-J-G(%ZQNs?juhc(RVP7^b|%%9284kY*4mA*u+z`$;BGA z=`b*!qqu16{S5pnI zH1eR_g&>w%g}VTV-#0cTSl%@@{PCPY^K+;Y8T-b9{S?Umil=m zH#55qV;SCqCH!~#5`N>v`6W~p#{|X_&ZhBV$!@XvMOg5_c@qzind|PK_ zb(^uUW944M{%4)|H+!~~I!WKkZ{hdCn+^M&?QVuUp2l+RF(qaq|B>gwJI0|7 zHCu$5x2j65He58aRLK{gX5yz`6O;NW`h|M3(RTwT?O+u`i|0L*WuWRbF=W{Rr2Is!g>c3RT$t~M{ghTsxZ66-O%jGa#CTdtB zpu0x>xB2bppnq@g@fD{Q3TyK*?w=2G(e+Q78uiq9)b;3B;9axNa}Uc+ejdWjg-hT2 z_h0w@>+^aTFQ&i$xPQSqtLya9c>U>mkp5Q2H{4j(xfH5DTS&N!9Pc*?*Rs)1g$0BS zU$vS)Zr~OTf%%W@$o=~xsf*dg4$}9)Q;YgDbW>>WL@lDveSCcNsaMHa$KkCwqWb!K zrbYQ3mEHR3yt~6~6jNNb3uSSW2(Y8_=CZ@|FBhB?D~)kkmS$+=qW0&(jNc3Ky{f^< ziW5%Hut9c2czo@Px~92dFRwkz;@SSO1J|70cRM)5U1%)?iBjP0DbwxbPE4L;jQdE{ z(`ZJV->seMbU2@Kowz7Y@52)>Our@h8+yb;SiXSsaxo8OHdD9sT+mXU;?~P;)PVv2 zpj}GY+0}X0dL>Fhu`WH;Mg?&>Ie%84%A-$(ZikEMdj@>(ySA(BtJoD(+T%W2XSe%g zV6v`Jv2mHBevza<}=pH zr5ZI|!Rg7jfs2=R??dFYIc|w#v z7NNPCrl!3&*?a6$a`N(mPn#^GpFsOYeyMMn=PA=|^vS_r`!anbTb$+QJ;s5aKL5;) zV2bk(`Z07)Wvmwt+>-hSWz?=l@A@bToleGQA;BXsYn}gEPF@xIv~*>bC_=|k`1ldc>VJo!lBj?ldgJL&i2Tw52@CO0SAGcb@C%pXk*;snt?F^s?osk6<>pRT$$|YNl_iMNh1X z{&MTbu1q}-u5E%`f%#j#z2%Rvm1i{ zQ~bUMM7&Z&&LWOKTJNsgvn?*d*$V!=jpxe1aP$M%BBP?B(r6KSo`lni1)FeMR+`k( z_w4JuHP0~F4(5$nAT4kLn?Mw8>=3JVMqOK80_~^E`!Ls`;)!>=-5-EMy%k}^#aYCs zUEnA=Rt_V~B@wy6@uUwdE2Mo@8u}@gLm9z#eNF8$;0Ib@Xl{ zL?18UNNWl5S|d^h5}Ehmr0YS%s0_w&y3h^P6^*uvbI0-q%fZBVc6J79L z2!R3%EG+vF^6sEWN=f0cSqHwZ78g%+K6ziOd_*EcK2mGxhy?BNUb{0Xw1AZSY|2PQ ztonAE#gio_**Q73)s&qb*EUa^T={%9j=LCyEK!p}&-7m*BH~C_$|B_oEX&Tl3DT+5 zm2r8p2mL3@AaD^qQx#=YDU|!;wJXwGqi63ytF;78jW$PpzT{O$;7M;GW`0%riE;ik z@gs9T?>BaeH7t#A+VQf*(e%`4(la=la2PJ&{B&ehq4LAMd<8T87YXvQ)=-h8+{zKs zSzXTPK@P5rGD#qTM&uTjMA_gTZP9wP9eJM$;@^)Q@`t$N*VG!;VII~-KFC{tsh@XoMH2SGBdZvu&sM1qwwglGri%Grw-2gR0J zfF!eC%u2g57$uok4PDP9d3rKTCnLhr5#I0Jw=YF}iid*4wYs)8{qxNG_eWY<6P)Rp z#C$@b$9d1CKVYj%zmDB{n({^6$b)JJ_=8F%zFl8sL&yuH42x;wBGUqEkm^sLg3%{8 zvFj_!=HXy3#yoqri&BjU7BTiV(#W|o7@1Wpf^Mn4=L(g<4eogSiLwMljH0rIGm9e1 zrbp`-cM;GlK%LUp*B8MfN>5L3|73c<90Y4o%sX6q7buF#NDxSZ$X$G3@kalRS3Ye$ zy3t-+2=EP_wj9EC)g3AiYenyL*hWqn2o+2Zg<)z)bOxK+K>gblCpU98RxQ%#Qx3U+ zj3-s=g-JRriA^7}ElWI_z6pEO6UXjmApsdqz3(Q7QFQ!;wSr_ua#M|!&Jhg_4PEH# ztK#P7h9qJrq|$W<>VI}Xd~+lAkW3h~F|uIvA-kEFY$%8r1QFIJiXcgohaZWWlynS* z`mF!I#^?qlMn^!=%}&-sPyOc_VI+ED3V1Q?P*CdKG9zTTabwSgi2m*fg_pa)M9CQ-)H>% z@WEO`ZM*fg!Wu*^`TuT*ToUdvYa1v#eJT6Pmzuhd53MuF+)|D_Y5o!&@D6Q&LKy4fyyVYFzsCranC?j@pqV-WG$%;PK%bwFmnd zfzm1r7NysDZe3IJS0m*;wOsTe6MLks8YI&`VQiyfPcVph%y>S}7+)2g9AE)J zHp7VA%Y|8PK4wH4>n#e^IT*)zgWZD>KU&m=GAkUQKcw2bb%9%U>HFL0>h5``t~LK} z`AWXj=r1DNlV0>ed#Lf*IxYIqqXzPDH*fgr`WtRc`<7;c|NDZeoah{>(Pz(qs8O0* z|2iCWco`#5)XLmHTKiZjY7srU|DTx|IWkk!Yx)_PR(EpH2TO76`@0F+Mu+H}Jem1< zyavcMg5r-c8t)ia7pTd1$Fi*3J~8ph_GKrnHPjE(E?`6>e3;r)*)+p!DM>brG$WIW z^+#CxGV@nvN0P1LprwJF0ip>|ei*gfnXyW$$f<51AIb*a7-WQjWNBM=Ja65=OLR+q-DF^4Zox zqcMatRK~qT=kMHVztkjt3*({^@&}LPBBwhrKbH@m;S$}^5g>q2kD6Uyzt?BImK{ZV z&-{lcBImNYSH^D*+=4s{x$b`*QTlZ<{gY!!5W_*whZ$!kN9?fvW`F)t)BN#4;P(4v zp66fg$`*X;%XrioeIJje4OMy!WEoAA2rO8~|C%w4k7N6xPvfink3KWvH9b7``QYzb zFC!Ya2*G$Ures=~=OaYpxUzUXi^3{n#tkQlm3#69l_I{=VGL^;U|PK_kq&rOB;4p_ z46a3;*uIo2ST(e-GA>HT>ddj`sz-SPjf-z8GNxC}7YlE*{r%Yr&(wvt73Y%`Ud=awCu1*%p4Y^%PGVT|+cq8!I<%Vcm}*GQOnyF^ zt9LLiIz9RDouO*JxRLCA->FKPEnSXQjsksj7XkHC7KOM)b-(O2yA-K*bEuZn>mtv$ zGOmQ;7PK+19p1>c}Es>`tEF0{RgOImaW@DplJ)rLKR~6|p|O zXcHZ;qkdXQAM92YJJ~r`Nt74vT*%$S@^v(s-L+*(W2E-t0;$vI z-!wN2Obp#D=^!yX`4P+=$Y%d^A!l!Yx)y6Gin=I>b*0vth#8ElGD%c zVk`KA(oPRFB?U`4WE@r1c)XiF-A4A9AR_M08&nyNlMNu7N#m3I7L9H9<8rMV_1$JT z%i~j}6T}%a<19fE1~h77CMVgO&c=}+|NGJ+rIF73B-1GM7!LJE_k=FRMt}MjOm_Ki z;1s>oU`T$nj8eA^t4UwZlPwyG{>)9Ducy5>JPGp(er1&U8C~pLR(F)VU`(`AdAL3c z#vx=*$iB^v7Z_SmFql7O^I_{Pv&3hail}fP%4(uHWf@c;)D8t+DgvOk1XhMax8$ZD zH3S136Fa)e$?RchXwB!(4-vhHdw{F#a&MyiqPrjJOgqsd0?4Z=K>>9q>u_EsXAVgg z3$znx1XtL$Y)?@;Li};7O7K>{yV6d!l!!GL_@$Z_bppCSP0&1~82mVjGL_4z{QGTEQ$q#Vn+^85>Qro|@%wAb8JKkzwQ_Rz7mH2GPFLIijWGgG&e_SI#F|r3! z0#aXeZR4bG*Zkk_1@5d{w$W-rbJ^!JW1dd46J4Snv(tAu_a%23DJdy^g|d*O021H- z+%AF?AU!kl*Y({Lkqb?)YU}Dc(C45;ZnjO%f@BE@TSZ&Nq_ppEmXkG}`rI%CjEM)G zDo*xS8qpJiPU*3KO9=FCH{5vr-=KVJ#q5TvH~}fC2FJOK5r@&nGc3V$xSo^?;KTA@ zyGsP=cYrjpdo=gaqphf44T=#MH4)i|MI z`sIJZVU_o@Q)Bu0Gr#uD0K^Z}j$Ol+_HmK$;s4)geFE#>p#1rC<}$EusNAGmJQ|NT z+Ik>zZ$!<{`8`3u^YSl%GE54tF8S`Y@f!4s)Skp--FrQW1b|mjNo>AoZ9TFq;%VI9 zwEz;qTgARiRC`pUUhn|((gI)1g13?L0GMUM(4(p&Vx@AQKYt#{LM0M8m-kI@)dZwc zfzGtE5ql6M8>fugLT*y#@^v^uOh}H<9X?e7*`7rx3qW3i6iTQC=X0ejE5gb`_w5GG zhICFde<&cO9hQ1d1imK-IGI{DpHAK4p=6>52q||K7tGQMI>G3!NWqI2cIovJGGAWQ z9Y4uKkp-e|S?I|u0Y2C;)Z(3BpjM(-SAXvIAGiW}9#X_2rkBi%xvFgQ_&G@F1O$P= z8da%Xch>H%_=%$5uV24hyA?KzIg?iDET(>MGU!TRrB+JR;|jcTzmA8_q>p z0L?#+B3(s*)GGhiuNv92_nb;~O3nX-ESqARSUARTiPK`)}->{J+Bjhc< zINsIwqEu)CHMNYdbVt(M)tvndRY)BjmHoekLjP^!rmt@Q10bVsp8vN!|9}7LIo@nf zl>fCjbw{GcVX#d5{p%Y`MX}OM|Ge*^Y-?-# zUy@s^14oY@&HU<2i-jvg-pJG z3wO>v!pui^Ac~fj*2ft6cmL$h^VO%`HhOUB$tFs-$NMMd{2v27+W_L@u!F~#1Ife4 z>eYp5>Hg*?DZi3hBsnN2pOd&R^$ng`uq&N-ZV@-o?N z7CVmUpi`v_Npvw=cc8Lq11V~~THWhxKs^6_TbqtGhJ1O<46mme?mZKrm2#-~v+wpV zzWrtbckh`|I9b>bZUZV^f=I~Ek?bV7bNiEtG3eBnXt&LYsEJrpz z{p)2xe^<$={%7YRTX6V^TI~08lPPm0fN4bVB)ROW4ouWU&em22y!m{xLDskfT-OhL zabZ+!mT{5%H%4(Q`KTXq;S^3#iQ9m1)(N3)1sSSbU?PL_hd0LiBGGnNRK&c*zbss8 zAHD7jAI~uRdc7F~iKIMmhAbNl9m?@o8m!>KD;K$B)sA*gmgOJ9By4R6%7apLDU?J; z-xM3wxfH_N!Q%1&8aUPgD>XH$or8lVwr&X!%WdDUw@ebB#inKkeNq^zY8hQ$2`C0B z>MmU)C-Z{3Wwg_VQ@JSs>exBO1)&>s@{yhFRt#E7Gk2TJO-vMk9l%8&pa#>ux&?|Z zE>rq5%Q!g94X36FX67+nFlD%blXEJ^;KXh`Y68lD#uv)KLguDC=5~|S781r; z_e9QmT|_4;`Y0$^>?vUGKI zgD}a=v1^{2eh5y-=!Qscuy;M0f2yaC_6+9-3hGfSDk|_M=4zV1KKUZw&HbxWmpP#- zU>TiyqU@+t9|ytsaAX`{8A2t&)|iGBv*YSgp<_6_N5C?gncL(v%jR);i^_=RG-aYO zEyX4jcx{F3D`Tshv#niKD#ie$Ir_m5Q^j}w*<*)uT0rSACV_yMPo$iO`@7!jFHd6t zQ-B4@oIIaVXVo%t$~0Bh4O-^0E{LD7d)Q7sI_OBlRYf8gA$t#@5!pf})`1#s@!I?! zHwY)7#Gh|QLp;e3H>u_^S>uHI>+qu$ZoC~f5(2u(WnHpk5~x-3r;OQlW7SEs>#1hn zus1%cu zj5lmMi6K4w*=Xktr(Cf(mabBhejs7nIWj$Y%VYdjgdZz(Q8kerZ}jyd2fH>;w=M>; zp>ky5UCIz54B3+xvSNPRxK}+nGJ$~V_)@*&H+(9%9_f!uq_g?+8DkCYh>eXMf}W>@ z;MWu5ZTEM-%*+gg5uLFRJeZ#NNIrGN3qEpp4zRW1g)?V(jS=meMS7 zN3clg0CP4_ZU49HLzu>f`2e>|5l8ZRbO;#p^{#GJ!D^`%A`&AFuyV*_w#y@s%;X_- z-qUbz$oZ->`*x?SMy)_$s6J{$6NpN7<0-UtaCeYsx&!NP?sN-O-|%@ekj08vLg35F zac~uqrBS9BN{EZlk3h)o1dV*h`uc@ zWsdQd;p8IL?7EL z&yB};vN~uo&F;iGEM?F6^c!S11Gpv>B2wodoT(w~6*6_$qLgK$ur_VVbzpnL`U#hF?nWtbjF>*Q!qCb7iyoo)%odhK?CAA@6HFjO^e4nDs1M8t zZN#-xG_n{(qflY!3;UcXs!Ci)ZS`QeLp6gV37wOKeM=fXJM`&;`j;jC z{YKt)gy96^OZo`$)5EO_HvtR>=pkq8?d>HL8Ow>G4zgg{AXttib%3D?hkQ}kQ+gz4 z_de|d4JJg=-U{%(4Wzd*si}J?Ljw*4IbCU_el<3kmOs@z7@LtHM(Sy@zf$jywkR!N zq9dWtkC0iE{OY^6pi%CA64z|lu)${h+pR8m^_O8Kv1#>M{A)u2c281gQlWy0x)Z~v zrZ!&P0F96hW@x3+lb#MsYB4+=ZEmgJ5_`A5jJjBvePr5b8-=NC=gT zzJ7gR{ZOc6ieijiQc})ZSt(Mn*K{wFbAc(J6OwluydNDK2OW=+q6!Khs|Aj0x2nLq zjsRs}wG9moDU za!6|X^;N1Lushnm{o`mm$(%^7aC*E*u7Z?mxXGg(2AZ)S%I$Ix%lttK8&a_>XJWE0 zDZ`c5D=H``sA3s3aQ*p?B>cmfFY|VR0DB7Kn1ux20>Nmdo@ICdHz7{ilJZKemRdOk zZ7l1D*OGTzy`M+m1|6dZKzB!hX{re;4zK@rXRfKX_^-%IbldJy+F~s0i literal 0 HcmV?d00001 From 2213f18d35d5bbc8f0e52452fa9e054ed064f8b9 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 30 Oct 2023 15:40:35 +0000 Subject: [PATCH 02/18] remove results files --- results/random-filter-s.png | Bin 39572 -> 0 bytes results/random-s.png | Bin 45942 -> 0 bytes results/yfcc-10M.png | Bin 44402 -> 0 bytes 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 results/random-filter-s.png delete mode 100644 results/random-s.png delete mode 100644 results/yfcc-10M.png diff --git a/results/random-filter-s.png b/results/random-filter-s.png deleted file mode 100644 index 28288c94570456966698728fa38246b2eb83f65f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 39572 zcmce8by$_p7A|&z2`VXqA|R-wfPjj$bhm(XN;lX+cZYN%-Jl{O-5_jIx+S-gcg+UQ zIrsnjT%RA$;cWId-^`j>^{#i#2T5^3{FB5d@$m5Qg@qnV9vwElE%W@vx+^5UPNl5wtXC*8Hu-0||kSJih;^T9me-<`nD*@L^k zJ`g&D`{gXrahMPK_YIE{ANR|H%U9zM;C?%P=rs%O7d%ZjA0!t%ysM|rw(WlO`oJS% z+;47oV6dyPinoY7iCOC$@9WfERJuiE(pS`)7xu{D&(PPpx;m!P)31MC(r}`5ua3Q4 z@%LKH+Q<)28UGt%ql6!P>wijB5uhG+s2CF-yOtD7jQ)H&E z3wjb?cdtJ0X9V-SpgL>Jo*)~bfIZUC1c8yo(t1~<|(^g%1qXwO^YHE&i`J7uT z?K;_t9r;Gx+UhkUi#J(Wuo!mS%uvj} z7T=vw=gcfAH^q8So;Q@uB_pYF*`q6R6NO6Te}%1TPc(8?qx%Or_AOeF+cVN41` zH0f^zrT1Nd|!%s zxxY3g>vM;H`8KezQV|HIUYkt4E7;Skc$f(Mgd@lcy{-Ey6Z}NY@X44y63mJ=7{l(aCp)TQ-F?GFf}o z-=IBlDp)f=HMaVB7PI4;F{|+}$;NQ@ zGk>$g*-TQ27?kzv0%&XS4PI(qeoH{c8UO3oFCVpBCX>Mmwc6U+u_&{y^q!R4$ImTH zF1(nW)X%M_tuigFH=P)9vwC(>OITR=`wr8|2cz007u-gs`-*d^r?7*rSdJaonN-)E z?Nqt6AlTB50uvSIl~zHW`AXXb^?t{N@_{M-)tT<@)ozWfNG@9uKM@g;m0#N4DPrMl z@T;_wQyHi2eEOSHm)@?}m#t5y(cQe+A|t^)cA-T`+SoXKz+r|#SWw`6V5)wF3%}8@ z)B02rdj0Q_kw71x7+AGIXQ~{J$BOvtlQ1a-5HI@Mi$lj8%$9fNl4?d;|1OSn zmf2fYfB$}yn_I20&%%9ewr8Y${vCyjq`0`3w6yd^5|T7Hs{M8&o;A*DBHf%JT(+6Y z`GyH0A(vT=x}H$D{M8ICaKz4ca%21A8XBJP3ks$^ANfJdWs}}K8SP2JUZ<5kIh7RM zh2E7cMFoF10+lB0x??tQ-+H=37{r_^75BF3-pFb0%|BHAqBgsW8e%FE4pr z-9&P`p#x+*PDKr?+gl5+GkuoLa;mBkhF$3?%gwwRmJ>~@2FpwJx6|yG$I%o0CDyRb zcjo#_yOYG=NO#8apUQv}eo58*#qZz0WwX@FEt1ZUO-!UoMDu*~_Eu0<4g=*LurQ); zev40%XE7#Xy)aPTV3IHscNpHH#1p4~aPHD2H7j49?Z4Vr-bxMUCxOn=XO&6&S$#cn z$u3DCD6?3mH<)K*mO0-9BcGj}?XcRZaGmVLC&scrj}j6RXgN8R8^c)J(KF+BPgk}0 zOYo$uug=gjGK#~z9)>P_l76uwI3;v^y|8kt5i{?>+|<%8!8_-inc!?Q{Ot|MJg|8= zNAXtgXU$i-jbUTl6salHgc*asD{X|QHX393-L2{w3X^T-dPSZ*>41Z!G4335pI#xe z1FWZYuR#*+jT-`2u3RZu{6SLh(Lm>1sZ!}ujtlu3NDSP;tNT+*9o5clhsVnTfU zWUo;c_j~|mWVKOR!*wgai5sg5r)x5=UA!G*R;f0w{9-L_iFC#>ri|45@xp~THj`fc)>wX>UgLo> zCH4WkxVH9oLNc<9@y2kYws_$IjCZF*iOpQ9Qm!tTAF+XD!JWR-D|-A>#GDId%U)Q$ zuMr%U2VJ7@B$eJb9UxVha=mEOl|IE?XxK>?X**z_s*n{$?l4JNT~jjyM?wbN*~-^r z6vR<;!*+1l4mx^Qx4)KzVp=0wBn!c++d}K zS;fCZ3k!>tE$rYs<9nN6m(xQ+LQV=E($UjP18W|}>skSZ=i-W4J8~xvUmc~Tp%DZp zCI@?=Za+qdbfiA`Huj%HeEpa-XeFohbbL~@i}Lg5&k5DBK`N%L0kkq!Rru_anWgh> zaDZ7fza3`Ks8AhrTJ|I*cvoOHm`TpGi8O{X3!>9bol;b!s8(Xt3evSkhYMz>TxORo;Bom=hDu?O62;S{ z(OOQcNguGCsbC(#=A`x2sKZfZEMJk1v>y+3*j%hZ`vV(16K+e!f4aZ4uwr}N z2%n5o7VKSTGw)9NM3kd=<pq3r8;`$5~~y}D?`3sGr5}6Vs**ykMvfZatc!*`ECdZ zlEHaR`-w98Qj^`clLQ+fV``c?UoykknW3Tp8`1gl$f<)4TcX{3hVaJ+bM(^Sa>wd} z9bf%;y3P;YVg>9DvZ{n$zc_X_m#w?iY^UB_Qg5j`!<(Kc7B1q4HJkbw;C0~8(Mhx^ z;KI_Go9^+o`5LHz$58-_qEm=jU96#)S;LqGgZ2HaSyhI0oz1RU8TFS))oYGI9v^0z z2)mUWAOAfm=)p#u=4B)u`pv8G>wf-hfn8hyMMjHVhq>1GVN=sGrEP2qVDLDd`k*OK_5n#Snm@eA zW=t&G#n`6?ceV>Ni^qwrXMahZBD!^4|Dzgc-|F8H@0six^7fWbeC3sgCx2zfQQHsB zG;vIjdR}1`z!aFYg8|P16%l|Fe%m(Q|DxicDp;j;>_QeSS09~8h{BMO0>!rzj5@Nx zqY(A6;@BpvzK0~ypLq z4zAZB<#GBoQ2@dA<*BKuA0Y5aOLf1y7Nb9zSj~QY9xiIW&B4)gZzet0t7jb6$V7bw zOg#%cjAFF_to+nY(FS&b9kksO^sXmK?cu{0z5xNFmM$RFx6~t_-jUN;_JcxS!6#u` zA$J9L1)qWmTr^V$fapp}Rn_>i zfcciz;_qr0qThCqTeozoJxSqjgvdr4IFPzNdFA$2FzX)e;#`f6s_XevooT_?8J){- z7yne9fx`fTwQoMsOQF{gqAA1_<+LOmZL_t3Wxqw_p6#;buuu$+?=qN$+gN*uhSD1T z`H$_m*sP3@ZK;Y5nt?;r4hRUaYxX8#2YEF_a4d&-%M2Lpxb9$G-%@LX2OO|WSg_q* zD0SXgV>Rr~oa0u{)_Ox(TeP)?aqd;KY-AAP+jzm9b19wlnnVj8&JCp1oifX~s%nAzOH3sr(19z4QI0V;03IE(fi=^{m+l zMi3Dax~Ci7hQTfjQgq}`G=?XYJ7NvLxE&biMsv^UF$O`a70VPJREn`%Uzq|M(8hmB zZPc(=&C_*a*~om83_=qLTr*K!2k;j z#>_D?$CqnVx*YJlcQeCzydjiOoF)VUsxZh+Zit2by>BzMPMeTb<#B}>vT&3j1cqLd z?HwF%j?VJ)JKt~F%*|thB8}@eZr}n{Y#M~pndNKB3(p}PLebY2IGh2?T)k%bNKWex zkl{jcQPDkcy(>k{8k{-!1^uSjZMMIU8>6^YP=IWU9|G=fN~pNa;5`i;7nceL2Zv4* zMLDCGrsgLkUpTmv-n`qA@PFfXoK5D-mc^r6$;KE6i1l3~xE%|O*}{=mjo#RSQ{=CC zd3~i_LSp(~lnCEDg4HWy){wc@nif*nZmh&Lz8))?&8dHO`E9dNguj2>w{PG4*NZ78n?FiJB&1nRi>LW+N*u4w4zlo- z;L1(@H*ek`9|fu8DD$QRoJ^+EdW4BHbX>2B(K;Ex$t$ef8M&4E?X!oN3LtSH$M$dW}H!G@r{^jIudqMRo%KOjqRmN z!h(sVI?tZzLu{j5Y@ze!tEd4(!ojWHt<@~oP7<@SPY47-@nMPeOd@?|(Ic={*ta)9 zjoQOljl4E~Rqkw;rpwRV-Dxj3Xuly$$9#m6j!q;g(k2YMCJn2*z1-W8t|S9qXr)-! zK1-9oos{lX`&37n{c@R%b^Or!o5hLsTQy;97v6uYm`AZGc&e1^qCo*lgYlm~OF}L^ z(7g?Z1zXI`%G#+{k`V8^=Y@umC(*cvIi)VMTu(iL+9SfWD(jnb|oaN2z`gWEm zgo?6B;*lenzPXNAQ%`bk<(+@cu60h)K}Vkn8nwi{XnsCrR0&}r(h>+u01(jWTI1~5 zR&T{7QnMPgJ%&_eRak9s$6i=43e!5Xei(%r4+b|tuj9dE2XG7)Js+{a9FyLuJvqAd zS8;PA*=kfz7$S6=*y%s+I!NUM?b1KXzv$Z$hNG?tQf5EJK9)x?+RYkGYm8pJt!F5OF)v+`3kb0mv50b}P z-CTo)kh?lW^gL0e3l*4Byw6sne1=o8A5cn}qf>A7_s{UjSMGuI3Ush1;86T!7mvvo zV&*JW*TbGfa>_y8r_;N!5Cztqti{!io7@|avlReU0vsKuTpu$KxevJRI6H656|O_f zhSXu|2Z>n+bMyT;GYHG$1@!~EEL!<5O}#iqeiOUaCYdA~!?phX#}7bO_?3&yKS4&C z0QxowzOoa7;^bn>i8|ff+L6U%h!-8!di3hc1$1?F%_VLeXA95hTmEsE=oh~HTyH*D z;e*6;a=r)v19e$8y?gse_iYj0NysK6IxJ}yW*}m#Oj9W|HEfIj5aqn4q@$~gRtYAp z3+$imqsr_v1-wxw6;z+Usn}c+U095H_{xdsJAS~J`;o=UF4KXLnA+CiC$f(p^3|@L~tQ7i@s#BNz?u12LBS5JG}JQ zbS60PI!{sJyJ^$mi!v4s9TW?>^AKr^IM*DDuwT{iONjRkAWZ+WXg z#xmbOllwGb$81p4H7+73$fmEG`Z65zw#|Rb)&uqvM^>V1LHpCdzpmHtZhwMQ*I?+& zi}hyL?KsfmsSqqZ8JYFU%nsz-!hW`t)!>{`Ad8A`&O#V1C{#y=3Oo14Z0`C5?@o#! zx&7l)ml$Q?wFsgCtW;vUD+6;^&;cM#A#!iQG=;28^RH8oTZ(Yz6zLgg1bNLe6=+ty zz2M*iAzsJr%1!BUR@G7)lLuHc*rs@@()*E<#{wV=Pl2B?MyD%6mA?SEh#$Gl#BW&q}_O^ zp3^jBt@C#=-RVlX4PIJc1bR!Xjf&^GGgOjJF&2GsRF##Dhvanqu;n&@S#gkDssgmx z1yT06WyXb0h-2;VlW)PH;)zo~IJZE!`YEoGeM|6qrdsJ*kw%(7*^wJ^^BWPBuvkN| z<~mKmwv3XW?C`y?E>5?`R?`5n6@%1!K!c8&n)>IQ4Fy)Y#(|NrpJ=lAB;Ikobm2oo zMF@d_v#Bsrr*sa*vcJ-_RN(+*M!PY zuJytf09i){-BLPll>y2v16xw=>J>%e3hDPvDk?s}W^OaT+ku2?z=X$p8{g32uf+=I z;bR1(f39Ru34^mn=nsI@%@*C;_tsfY`~YG7)khYwn z;^h_s)@_Hi*%XMoCLuM)MgattX@0M=vW@l(@8%!A7H>5}*f<@-Gl?3TV2mKjTAH_z z@ddnBnC=22si^(n2=cjrRY;Vg`3##+R$tOsF;E%s;(W1I6UFUVH)mt??ez&BlPJe| z4k#=v5U71>NfaZA#zv21ShoVQ%w!Z%w$^Jz=oN22m*4F)pn;HIoIQ?zWah|}L|D~>`~Oi}N5UDT;7ptiogy&;sjGfwbKk1-qk2R>7R!s*Y$;>s@^ z>w06^QKC*xPQ@~-2e*_C;0=5Dvfv$L0{gFKKNc9qr1R0Vd{w@DrA>6YBgJ32z&Hi6 zIwjaTHRsjN0X7KO z+UzGH6~_6ru9jF$#j2PV^K1X`X#Ge?-vPqmASD3x6>m?H=tkhc_SOcVQv+@Fw=2^t zoHqa}7U->XtyFj2vIcifzFat)u)P#qssGc55&=K}2ZBSVh7D_+;9L1KT(t^CiK#Zx z+pDG<4ntoKY;7(#JM3&>5d?N2jBXhYm=U-zNWp@M*^HAR%bWo#EDws`nIc0Y+y*F5 z5_Z5*0@X|q2m?^0glVBl<@S0SKmpv&;Moi@1sTNbrs>FvO59(3jevU;K()5Eq8WiD zO}nf81Ho@p0v;`st#Q_i)t69-Lb=o?1K|>pmQ5UEUOAjmXiG2Lc+!HZ0u-*BQjlb~ zcBII77w50HB1e<2-}-2IqPZJ~24Dd3Eu$E|hjON7vAMF!?%CDL)^kdWCK;K6|4 zS3$AA)d+zp`!^At1$$LHWrb=vVNiXr9qsr4NMqp9)0bnQyrcxiU>X1g9>C<>1l^Fw zf|3F{(Ewgwq@c)xe3cgR7*rR;XYY%F{6!z-bPPqgIXPAfZPm30o$^v#FP z+>b%Qd&y+1yVXRbtwZ6EDFl$9Le7t<9*VEA9!KGya-RODhzQbXhN9~RE0e7QY&+}S zm3ijFkDwMN&OYdri*yMRJfof*9fa?)-{h3%1t0A>0Od%CkeoRIMwf-q4{geUope06 z&o+ka{s+%Ci4@v1n3bv*VoYEiKlMA3eTOZI5h60^y3=2@y*}gV;c>cZ*!(iYc+3c6 z9YH3vcmceN2iJU2RWf87=LiXPIk zPD_th?;UZ|Q}fc0C9nX1YsjMC^5rvTZoq_ZqB&Z_bOVlHtA2N8BpML?%?=Jby;C0) zrHH}L2uMr&ox5;B8%^}`eY?-iLGiDS;xj=7s@^$$DJoa<@(+lHb@L3`f4ukfWQEP} z09QkN=KdeIY@K={K)JL*JU`qH9P&7dPkb4)MF33fb4aV!@0;?A6kgf}pwj~)(DwIU zUTlzJX+4SNDSv{2yo3Z4=?{p5%o!MuXa4Az}z`K z0?NS7Sh;Bt2k@H!B*apMrfS--`8ow(7-(tl0)A=40_YEV0a!Q#G}p`A4tY|Fij(a& zP)_&(f$*G3cY^38?SV3T9+MkSLY}f1w0$?fM=IdpQ1Tpd*(;7~vx^2Ytw9ZA&XEZB zSefs~ymxnJhI)VpE?AdyxQZD@2OWYi);i;0)e?SFdsmUlOK3Fe>({S{C$&#hyjXLG zdK&4nsfPSSkV0|%jY=DWG zfSIU4Vur}W5F*gqIt91cJie@J8%1k5iDqy!D%wij$}Y9z~LwxxIj+xiSy?_7y2|#x433VeX zTH4wou<)MX%>(Rt8Rm?(F}H)UF18R7j38JN5EGN{H8mu`YBr(1!NHeWi2B}YUS`&7 zVgVf)cRs`85IzP9(Y>*yk|xi9ULxS)vi&^E;PO{&BdBUXPZv*%i(HiUYT(lPAAnIj znPeu&41{ngwp@Rv5v_qiIj;p_YBsc;$^eca zgAIYbIN#gTC#EX^$d!fdUje5P*`hrN!66tCUzg2A!LH}W`Jj^4ifmJYXfV^s7~&#; zNmn2vNKH$Q4VyI{a$bY z9QsnO687X91rOMb;B)^Q^*_w=h1LdkBzgn^D-b}ye-{qmhtq-U)rY7-DVeeVWQsvHu~U;HW5Pk1G1#czmDSyTG_7_Vw$-HZ{19WjU+v?EMT`abEv$ojvvJjf`E&&Hv4+*;~~A8;Z{V-@#rd zkeL>WoUWQ>a+7(xOCCBaZ&TDCrl^>#Qef8o(7R^V!cFD}BCWtbx9OU4*UExH*UDy& znWE?GS@Ffy)fS;LL{I5-0A`flK%djz}FeUNZ1{lG!KU6$3o>zh@??wTQL=g-RT?PZdzfug0DQ&~>}p5~if z@jU$)l$~8cb6M$?q1XN=Z*3|&4Q>(TW6DBVjU-LZ?Q*Z4oHNU(>zrEQ=>FZUq4e}} zC(*7rzOdkDR8WXoYHQ2Yk1a9YAvGG)rcXWNz=h`ig(ayodDt={I*L)jr6Qi{)M-Jb zTe<;9LSF8@ENpgCKYFBbYRY1KEL0Z%DdV2t4RS~cqDP!r4fQ7qOm4^>3US|m;B>f{ z6uEbEv1fJl|9(y!ULAYXWx{54N?U-|ZEA1DiJS~P0~2-^+>gYC?TI&}&Q&eUQ#?sH z5N~|{Bf5H1EJu@N@hHn_|I`BwdlI}8xFbXy9N)+o%u$VZWBjrY+R8d%k8# ze!Ksk`ZW@Q6mc%Bk-=7==&UEF3-?uAQqAYW&E~K7IpeXUeeyFxN3Y4Fgve-vD!PMb+f2?bvr^`kEAOwFLB1-Y{Q>OX;|e3tFwee&|w%t*Ef>!5H z=YRBKD?L3FK;?xS9q)e$i-4oUKex}L9Oe(2%F3u*Ei4JUKfQ_#XJszE71OpqOj{I` zn`@^#2eef(lwrATd&fw#HY_wH15-%P%dN&`)Fm6e>69yakdAO~XJ?JAi>(AV)+}r$ zhq${}6M|*yzlXWi{rL8E_pSbhxwW4aotnm~t83L1^`-eAj8W~{l-FdX)v-@X7e>7| zSFAXem!6~^Pj=cDU%;68Us5x1iC}v6?s!ADI>Fs@cU-h*o63NRY4q9RQ(4XZFU#su ziz$Z@Txg3qxj%Notx8`O@ppVj7=!O0I};{*PO^A^J+wW#GGge6wzrRIvEZlP+%r_a zmZF!w6G$Z2?SC=tiIBhVL!no6dwPDhV6!`+r$LDLzi%D`ol&sJSzGA6UNy#)axG?Q zU!3yJrD`^nmZgl95k5!d&hJUfnBx{vCpLU3Z!5y`ndgVQ|NBJPPC$~UN9;pF{=Kyt zZMz>cGv^dbJ)Eq%zto^Pc0XPJ`s?1BA+q?A_WM(9bfs?8tde|zF_8v0OSvcD{AX!$ z!t#UOStJqSHg=B!_iV^>N2K_L(Ts@l#^8h+{1F zR}-}(6t-&+Yo8v_l%?FC&O~KZds7&zbxi!@sBe$_=?*^f-nAQN*O=yPdN!L(#6|s? zZ#M@xxYg~Sa7O}n{#E*rrNgp>d&9h!EMfGlz7tSj6nyI~~p=KTLQc zR0S61+-_Wk_x`eYm86S&kbIG1p{>UVVwt$zJN=<)9MzrQR+gP_W=%GHzKP7X0$N(j)1HU!gg5zlL3t8gD}4xVo|3|-|%n*SJ47t z&cMKc80dhijKLD&h?E9+kvwob5C{VtS=k>lI3jdJyraMWD!6^}OgYhRYU<>3ar%<{ zA;fK@fK}>&c+>&PurPxdb`W7bxH^#2;v2+Ds|51H#}6MC3yJ~zL6jXMsN1YVT9pr( ztJ?Ew!U0UbX`usDA)yKjlgN3i+UvoSz)VW{vxYDvNEJGu+*CFJ#LdwyTSz4aoR-BPCKm&! z<`cC6P_7Jn^HM6d)>QJ%2GyaW+zD@Eb=`JKNJ>hB^0~wB4>T)K_%`fFmiqYdW83!+ z7a;J>(2KGc@axDkka6Bx?MB?*0DpfO;bwBQ)p$d$-)zK)y=&qCu17^4ME?u|hzGcz zo_ zZID1`06O_sH7>6a3Y0RM&hO4pbPZR%E3W~_*`JytQk)~Q>1^4-2Y+ejFe}exbyh58 zkKulnr>o65$9i-4nND-$Y)oQ0R!p|nC#wJV2U3*LoRDbg~3T^;#n`RKK;0M<4#9!R$O?}SnrtT${}`|^Yrj}iR5W?%F35fLg2>+XhWRVlVZfvC~i24!m^QRQFqeQ4yS`6sP zkm&VXzxw?fTRfRgz#NdMgHo%g~n^{F{ zT{%lUcf2GK7YaDa#c}%7h@}d-wN4WkDS^xJn7X*=IL?^={3xiYs4_bD#=PiKbIbtS zLMS60OQY^2RdDFovoxjl*(LE4>o%R*)AQmr=Rx>@Y6=3oew71Np;Dlj;};M3$WMUl z0hYlIQ~+&nYli6QM}-z;e8y6)`c9@$dtpO+1jzCM!1PWuQa|8DTQ;U1WfrF9}<8U6|WpMG{16 z)ogPWHll+FZ-Dh9X#T?m8bc`hmOz7n z)sHKBlR#mVgerF#)*e*%S-x^-mIf~>p#c=<@NX=*ybxo|=Dbu(gD8F@3Ny#2cnB;|3D(?eY~rWIIn{Sb$9rbyyx#2H@}U+Ux~ZzGLPACk#|wjQkGz zQX6;gy8I!sMoRqcqcWjztZ3gUao>s76P-tj&1 zr3DefkilZ~r1`&m$fc;nc}KkAlu!MP?$jgA+Es>~pJ<|GpU4J;>KveX4V4?T?e7V- z^HaSV$1P3$>vbTrsND;G9_QVvv^HSljJzRtFoJY{oy!`*x>QsKXFu&vDx=Jo<*lY{ z3-NJ{{a|DqV!BFA{oopH81BPfK>0Y2C0s0<$i2Ck73|Dz)MT!;KT4xvcEA%G8fF=tDl z%~({}x7{=tBB%I|ZbSRBRX+qlEqR)pij2$&gR@fmZ@=ASDEAg5zz7jKXqMm-wV1lkzl<$9(k3jutPRf6?!8;?BR3>pAwEZa_mXGHgnPs+ ze>ij(pFTdhC-lJIe@KK!=x+K*NozrLI`p4M%hkEB5V_mXT&HEH;sf`?ZpK!6Yp7^1 z&63nfr=xA{h`nZFH%Uq60SyNx7?!0#h0%saIa!;6p0NWpdy!Hy(=9*w-~gf4R=;=? zTH2g^v)hh_sl{&&9N5d`WUnaM40;!vhKuoctEaKNs{)~Z$-aTp%9y(Mxypg2x%S>8 z3`BF!OU2TUA19^SZ3UC7i+O#RU+Gp|4ki%N7%hk;&YwNIzpOS|wyxRpZWq2;(V&xv zhim^rza$euEur-AP{>}cm+S|MIN7o^26#ekXfO>_E=5R~-?AE>E!dZZ-$V-~p?Ku9 zWfQ3FEG_K~xx=I+yMby<-QClCpqDG6?nHGG;_?l{Rnt)LQnj~#&2n62@0`(1%xoz?N^`de;SW8aB1hXLCDf;^Sz`d*>6)2QqHS=*+rF< zn%A@fyU6z^R|FP79nMkg#P^Di`H^cf_DLKbv)f4O6oRsQvc zol$Hi0h6}m$B#X+#o=SJN{fs;15kZxI zV{N!s1NyjZ!=x`rlJ7sOo2%H-Vpb>P;!r{jXG%{%_`!aHE2qv^Nh^^+=?ST`hlQKP z{&w8oF)3nah6%mjo6wlfhG%?_U(k)hP7$G3pC9e!9mz|uLq)STCTVIakkjG7Bf2k; zl;O6kPBc$h4&yLNW&Z8OA;a~J1q+O^AN8d;gR?hgYs_#`dNAskC(qECqdev^P0PV< z{3ptGe2lil-tq$dA+M*qpmmJ0{-}9!V&e^fSa>EPXaBsuvxh7t%aewN^3>cS_b~s@ zC$lb;o_l8?g?Pwy?+kQ)j8o^Xo!)rzx#$SL@W=8HXfnB3yDlk{I(2qtC^qMj3eA`5>X?J4c< z`-ZnKLto=}RTFli{Usvy>joU>&jV)hmjbE==nb+YU}dsTio5e;w6w)ikN&9^!ol(n z@_nD>f)~rAPLoq2c$c%Y$$pk#e+_l+DJi-N8#&Qu*6rR2o@LAtzZ$o)Y;3)xi6GTE#FV{*S0TMj4_vYRujo=%%srbC!;uCfz`+{Llp_5fu z8)3(7d5j^lUaZVA`2rq3omv3r$m&6{2aPVP0zehPmA3`O{Ws)ZP45BOk|v-Rf%>ak z`ek3ISW<{(ZTIC-uazATx|G)nDANCY+UQ!>E+EkMg?j&M4B@^k$1lw8f{4jopsE*( z`g_-#xN^+;l8J*=V}EsJh^3zH#NXd#<-~sV@v6lI*G)dW{(t|pYuN8^VC!>jvrZ6x;@m@0 zWgWM1mNFArY1em&f=}gL2Nc z3|-u>=4}H8YPVw&9H5&!<)#^;a5j~2&o}-_x|k-1rJ+d!qvBvwI_nYhm3GhFeTjzN zhbC`N*3#3mGJ8t;Uo`me7pN`uy?Fz*JvxtFTo_MwXvlVOYN#y%MJKSCza3X3wJKe7 zgWu5Yc`mn;MFpmv!_ls5*Y>047#+3R&L!; zP>(uYYf#tn3%wHAb8>DJ?$y#ZYGJ&5#Mn%=CDMCO1(cG1KfyNu zf_Z#BW9btpv<6mdwL+nj&e##CQP5wj53Mdd6Tm3SfaYGhyLY9407DF27Xbb^^`@&9 zOGBHY{F5ghP)lVs?JHvH25NB$Fe+l9i8tT4M;cN%f@&EYKL0qGX{L>|+|Jx(JB9IB zr&|hTBaDe!D1exu&Z%3vvkfgA&=iJ9q)dhQr1vBd%mXCv72uSh)=Xkft7PmNFjG?m zy~z!Keg1@s+`!0?0XQo_-Eof9EF1wMz`{UNL>Lo~^LjFL;;sYn>Jj!CPWjGN^TyLg z&QoVsU7&ln^;e9p`8rq-QO4rY5EC}DfpnChM?0^D0(dfJi2*7B3-l~Ow~H(gzXGYt z|NaJiGB+v8W!(T3lu_p=l*xg)K@GT33bY9;L$eIx5}SoALvefNxBKlb;EG)rM*J-V zKb*L5LmwcUz&{PR=?piymdHGu>!r)a5RO@7!~VVj!9Xh`xteXiHNqmp*v-_(*ANd6 z+OkNXNgG0GoqXk0VEd#%?&#biOd?u&*;M1O#E84!id9ZGX=Zf@`G`&X>AJfz*8lX|j%D{eNy{iI`juBBFm@c{Gmh>gK$JdUgxCHq zoKw1Y`L5RM=E-n4E40kaGUX1d(m)6oOFx}_Upz?$I3gWzA{gHdZmPzl{uhO!lHLAQ zC@~lqVCOA*=}dvpwDW*{PIs1>v`!n)Q=T{)%X(cnX#o9a`Ov07c;b$O1a>|4XHCce`65S5P((dsW$$-D-qJ5BTCJxl|Kv=^FFzR(8*GuOrl0~#3YE*ItxP2XXB~~?Lwv(_KYo^`KjjY`_fzS^`nO4+!in=xdsKbV) zT0zsVQBwqm1Mt3AfE9%R?R8+(qlXVKE92IZdKrR&HIlbB@AS>ILb2eO1^rlIMd2jc|K3j+xqn|T38p?M(ZH&@8m_#^Hh$<=_D1O|v3R}LM6e(})2 zk_zM?Bx%H-L9AWM+q;23GgResTc-jnmvsqUH!=u-wka(4iS8t4yi$R2D6m~`2_qs9 zK>wLgq?kgTm`bN`ajaYF7bw~}CCRA3r748sQR# zNBT@3=t=;v6;w9S=hAWHYu!SL*FEDG6*GFn{H_T@1aKt3i_ZMk!*_^O)dk)!AUHUw znP=k(z_UKo=)R5*A3pqN%GF94b$>98w7gV&e$Osmss~ehMd#B=n6o|fTU)_&>4C8Y zS|bD6L``-@&Jt=f@5&yaH{&&oS2Bu$EyE5r_4Uc95u_%^@Pz>nnH~-}Fw|c`pWX=f zAZnn1$_^z10eZ&Gdn17PI!nq1A;btsE$DLP zaj$;jtC?${Wtci3Cef|w^0A$2WC>OMdJ4TblqS0`BwsFAZI1~ZMSO~`z~ z4KpnO)Mr98lFaW;1fEwuc?K6bd|`pJXgz16#6Z-;%tT$uY6w_|{!1sPT?T7o)>+6EL ziI5{Wcmuw8l7`za5w)NOL`IUO*5g#+Y{4wb&H*-j{ovnQ?>=lL8FFfY_J8!V!|J~f&%olEpod0E!-`j8GT}N$^F@7i}4Bi>Ady^ z#&hRx4f*H-2X82nJ+(-@~z-?`up=?BxkQ|VP=zB;r!1P z8tnsDg0t_PaGHQfpne#%+o>Q*H6As8AL(mtvxURtvd{= zMStQ?;SA$dRUIx;&i-M^p#IER-W|5nd_?Zma7ogU!=$@7mSlftK4$fa$j2U?n2#s* zpU9h2%|= zXtya;8ZUkx7g1acQm@x4q_=Omnp^+q^~sy@(9m0eLbWZp;GhMXepcZ1M$pB_2~Etv zv1KtIdH@#$p~D!cb{jVt&A4ZC%5%`rv5UkzkvIJc|i`DW4cm|~gkfB^jt za9hF9qc|`QMTcyr$4q-i?Pb7>g+Y0HiXX4{a(jDuA;$>=^Ot)m#&PMOe%nZU3#sAghD$$3AJ!3P*Yt^G@Mrhy5If0C@BQ} z6co|g?$y_-?l^gg7WdD$$l`4DgOD7hTdBF)?<|0E5QTA)6*?60-Yt;}nxqz@!8(JM z$)gtDdbodWg zLVXIDmFNnHd}tE%A;MYr2lrA7O~RWOTe1$1e`Kv)=EVJBU9O%jk+A ze1qq>)9^ZNHl8|KJGej7B;SaF<8#k5Q6V1&%CE2M&ZdA1gf?Q-E`}~%G3+bKMjQeN z2K1N58xftAK288P$T#uW&eUEYtaJ`o8{8+qK=&yAltYOOv^wNNkOtwU1nLN(P67Xg z%Hz$961c}?t$PUG*^u$fau4*ze~1+k8BNw0{GN(?aXgiZDR#1z5B%40Yi``vAA7vD zDRsF=7vD3G=xL~7<@g z7+fs?F_7YweB3I|N*NQm19N1(e{3lVGh_ibW?|nlXKMFR+*58*F&~6>BIs{d*6G0Q z@PqG*Q2ACZMaR!_mzMgG)?Z-J6uD00WU zZjMk;-ae0ek~#Yzz4#l5Eubq#)U0grtpj#`j@uSOFx2W&0xssnUbRPjm&4joDS%(z z3}hd_m?W;bfpQffVbEGu2>ajYEqAmP`nanyq=9c)vl0?g^i=z58CFjIJ;52zf%Qv%b}|5l%k%l3?qHCrJ|O2;1KLH{J5-6}3UF-}O3Z zTbu34^J#Y#9naGSo`tA(`t0z5y~9h$w@4 zoFGJ0y~2u{+}SQel?>uLjs;Op!$%N2)uWcK_-ZIpvP=Q4Tp_~U?1(oc0Re%ZtmPmN z018R^- ze~BDEF^f;N7KiQzY%CC(;F_mkjzQ=;1+K$UM-JSat_{$zg(q@ve(sY`ga5*^G_qXD ztsOsVyqD>L&}>D$wft&54>F-u35S&uuE zv|o_8{|UlyxZlDwXQvoj3KwdCSI%kONtI^G|Q8` z37`ody02;#vaFkM#|QYdtj}}M9heH~vnLrB+o<(xcM!X@l9CeK!T|jiOJ2#ia~{K) zYLJ^UPODB@L}v$ktdrXVIrPBD`~%kKzrV}K2HL-jx~ib6nhr#eo4_ywi;kV2L>*w$ z8awL>KTqwFS3eC8R91fDZOTv?>noAVeEE_IYe{(VCsankKLK#(Dhk}-~xw&3@*unmd^O}^jpGh5ZDGrM5Hf{ z)XV^$g{UBK>mUU*&(Q(E;!h*dg`6b<2+ZJ8D3rGs72QK>>M$0lAUS9o(k1)gen8YMy`Sd@JuT|zguDBEO$z%>Yv<%Q`7hNHQ<&KpXd;dr&)0e|BG$lA znSObS5$<&O@b>hTSk%`D4wc?=5Lr5O0J`AF!=9l=Pq-`3FCM(F0qibn6tzNKnjo;G z^_0lZu$i#Fn+yyY4ga9a@pC>d;)rorvdi14+*#!y#AH)Zik;2L$tjD|1rif#KuiV` zv}AW*Cy@`jK+qLzaKW2Ysm&a^t)qIP0Q#GZpkGWxOiT{`M`En~9_>Skb=1U(yH$@7 zUl+B_Sc3<$3f#4klmsdny@3JoWUNN1F*YVp77$&sYha|0MduOq3IVPf2&^IS&>D!{ zwr$Z>TnsB=vHSBB-Bl_ASOIKIzU73ta2s?ZX>0+L5M5|z1xe=&Tu{W-DKYl?2)EyO z^#7^t%fqSM+qN~^Drtvyk|I$wB0`3wSjvzoltvU8%8Gm~)nu&+Qr90cF6%u~fIVPGY89BTwh2w1*i@B1>b%R{Do1(F@?kdcWgxhgP1IppX#L8-SFnvYD3gn9d%h$b2Fq7X0_ zLIy#9|0^(kA9C@ADK zW;VoqdRq1OyKJj)I5qlXDig&LLK==NLoll(Lf|xY7)bWli+Kdlh3{EkQ@RpF=Rmxb zda@VE>@Q{))HZ&&|MoDX>7Kcq^!=7P!_4fB?_Yj$zBxV&_6|M-c_r>>Kwj&BCX`v^ zSu~5o%iy4cgTu|+w{?Ku?zC=Krqn_dXbYb;7_up1Yp=nNpr-a|jU7}wikJfC8|~+J zy}@QisRdzhr^BETvBks%XlgIm8&3QBxCC>Tl21}p0k$l6cem_+Wc7)2;&dK@MPf(h zY`Ip-_H2&DBIsFR9_N3#61rNksGav!_Nd#F%xjqD;y?`A2#JkYU&g41r@&2OUHbz1 zsoHPuQWv&cH}MwpQqvRnv1o`%0q`TLt&#c=5xl}sitx&0$39-uXk0|DO8)K8T1wkv zgQnI$_AHi)0+Zvuob)^ROAu)rMH9qC`#fRVC_FaBmlx@{g1bS^;zgqPP;C3Fi@t_M z`}PGNIhs^zYI>7z5pz7Q?7p~|(_<+TpgNtHgyQ@tk@1D?RBSv+oUPosQ zW$!K?7i%$B#6!%|E0x1^iV*);Qta{<&g>9X3gxblQ2d zXsy#Ds*o>WyTuy>1rNU)nYTPghK_k{KlI)=XXYyR; zE(IM|eb2s57NMvMQeni@M0UJ=1u)W#G&6bt_rPDeb<+}iIiaKzVWEbW*|Lae0v9b? z$X)d{@r2Z2O*XGTU;(!WXOx_!yq4ko5h0+hpex3unbG!#b$kJN=f7=A{XCY^T=29- zZuINB=;~|CXZcOvR)uvO6V@Oo;(M1#i>IKe67N+ANRx>o8!~TVPDL20INcqam+|b@ zBjjcNFXS)*|1SWpz1X&i4jr~@0sJKK0j)TO2()w%>~SLy*`4Ji9WF_K|UM${=^v!WuFb9(x4pM3sU7P32Dqj#xd7Mg~mjZA@*% zlOMt6M(}+qbj<6B7Yw}d8li8FB=*f+1;p!)Y=*=gfEZj7RMxhS_)+pW4ed;AM>NrG9k3+j2HWB39EZ=zM-Mv=JMsi{S$=4%hlF*V9BP>$>Gi%Mw z#BMhaFh=FQ9eWQO+6BOC;`7r17)s?oJcja+4zU65jmPvyy#2HUq3lQw&DR8mDbes> z>LIlbV*h|K9*;V9ZH41b!>xhDc@0o}eaKl7ljDDaI`o{;x2$KS0QXS{*Hnb=8l?wJ z3Q7y9*l!T*&CC!Rx#hJO+WQCf2K||8Z-8ooE*^_9!rC|hZ<>@xeeki%!2HWOx)rot zVQ_3=e2!!#7=SRmP}aj?3NNQ9DTHalUF8X|pF{|*&mv*r+I9X!j`u$>*ZBEi=4Uub zG3BPm_7aNjXcnq5W*8WPlp)C>h6CXxyAC15!{Mw6w!R&B!{-pKlKI9@i@#vfY z#T8!F_}WJ?uD7`y3*(e+yu7ZxyDzPwv}iTaH3MiGu$e@5S8gGF;u+J@zqUuToP!Zh zY>86^%$d_#rXGN_{tlKmG{cEjJE8ihsPgQHlcvn})nj+)eh2^FuE+jNnP zR*xqoC9N*Zrp;n}@?7GvyHN@hWoRR80|L ztMwPL;nKDP4p9g5liO?M-d{Fur)M6EM zb?+{D@olRB_noV(h4&pgR}mBx>J4JYORQK)Yz=ROiNcx5!m_yKR&-zob zOb)v{+4*3KMl=v*)zytiIaCwpDK!j&{9j-`=iqV#G~>ki5t&{h!r7j0T{X$@G1JAO zd4nW%EjGt(V>fAs*g(my>-?^{EN7!uFq;x;KZ%>k z%Bvp7sPsS|Gt1vqcUk+gGPGYCW@ihy*`7J=83@b-G++orIpeD1uo5HgRI|uTtsYoW z?$GVMCYv;>)tFobng9`9za2X&>(c zB=0boI6|t(gjYeNH?_VyO-Voq1q(6995h`_##-L)(N2N=Ihke--d^AYBrWva2$+>P z5A%~_pGXGEpQ=iwIX(`FG^9-qv**74{A)6mt-fAg8LYRI?@$fz?UMHTczVvEB<@w* z_15Aq0=aS6(g~SNWj$ICJ#wG40zytZP0vv3l<_{WX&d#s9RGnmP( zhQTlg?T0}CS~nk@A8X6E>t3|^wpi+OvleHVfk1d;9X2U!P<}5yY2ZH_D1;2qUg}q$ zq%TS`XSmZaKs2}WUC`rPKITw-**wuO#0L;L@QE$iEe}WtOdSU2fRvea|C^wBdD6_} z4GU%Kyw=Y8t2fv&BDAP^`rzDHJWZ0!p~_GPJC}zTB?DAZmtFXqJSZmjLJBPPhm?4H zo6Q1BOzN(e2ln!$dnqUaX+s>1ay(TGYAoWELydwZeqT$DXhuElF|zUQ=JX0>^&Gu- zL5x|qrY;Qa5E6-oi5ND+a)WeepwJ5H!^6X~5iV^> z_1?Q5VDcZJ68AoMFYlO52>zJb%O%0P^eX$XZ7eKQwB$eo2pWPhiodz|kzdmh9AH7= zZeKAf;Sdk*uK1Y4*N zY;lI|YguJwJ$6UKRd%%KhpizAG{Yp_RXpCC8&+_wy<)bp`@S{I&4_t?4NNBC=8h^V zk`4G~Y>X9+@tkc*t z&`*>^1Y7}Cp#cv{2b(yGUBst6;LuvyWiKy!rOXu?g^SjN9jhkvJPP`><;t|Yy#Kti zC#!?EMQoVBIZY~{|JwQNwfWzkFD8sCf^EQ#j61y#j3W-}bK0lf;Kw6I+PAqO;_#|H zJ2_D)ARgr(pIt%q?H^)5cT(D`c$PhV7g?y~i|x^@iYaai!N}SCC#`57bKZUCc~R2- zYbO@$>@S1Q66{_KaKg`)$P~dOeNmvstgFF$`^Bf4$cLPJr1Wuy*BS>J#$X@`0Sy83 z`(~)uh?xvwl!1dmB2RLdBTg#r;v#0fh}#KFf=Gtgg&}s5wIE_k7`l*%2Z1Oa@X;Ux zvX`MMvk|`|`!xZdf9$4B51r#dzG4UM6@0E9D>{p~g*2Hsy0wuaH})GxO~~d5d^t(l zNmQH!>43HNUII;H0{k$QK)nz_hzjtiIxIYxXmOf$*F?T=9_?$6CC2LrXD6We1Yu3x z6nqm$_`~H}aBe%6gxs9wGFIS}LtH$;TIHsghLT1eww%ylDrT*$M@U25Xm;U!t zrO$tmB#pR=A@7{$B1=Zxt*2Zr>H;T6sT~4CHMv%QM>YrSEU0ZjREA)o;|a^+#WKgv$>73jA=!#(otc5rf5``KeuHp6$BLK=)g+gW{Nz@Sua6BnHgEEcrcUtw%+hC-A^E!q}I&@w!fj=ht+$RDSd& zGR0!pW0(kN0GayNK?hz@Z zw$Tmj{2NbT%6Ac<6(rsvHmUsa^@kR3qtv2`S&y-&)Pe*1kRWTgKtEKn-~vrfh)c}^ z8#b8sx8#6ic#kw@@%Lw!`SJGj`@ZZA@lZ;m^cqY+4U&D4u2`h706d2;j`4g8dfxTr zo_W7JXK-l7e(=@mN@3>ZnC~$l^Kt>sv-kfS>Pnac67yzQZ7#G+MjTuAEhODO<8-TT z0-h(n;oXkrP&EG_$rIWWz=rS*%5x4aCA1-o;NwgjNZ|8;ig5$&U?z_ZwSTxri(+yp zlV^=%GZq(4a@ym@Z|IZ(qetV(2VVr!PG?o1BqOA#A~!ScO%4&9)Z|b>jbK<`nI@xh zGb?fBUy2xr%~p9m1-nDzvC9^5bl%*$2m08pL7dvd3`Q8>DQ|~K6tG;M7t;ieS-nmG}Z{Oho9PZB{T2Y}@qnMAuGrigdIF`JMr1B8mf zh!G0{?Ej>T2+%6+qeqYGGb+E!Q6hs0{<^NtO2X%-sn&U~f|F$msGXFX39lRHcmiN? zYE*sV;a;3##66rw_~|GMAgpuJbVbL2H2y;v{_sIlx=#LyE2F!zv;;RCw+|z939y@X z11iU&u7$T00EA(YY=T4KjbQ|-1D4C%ew`ko_HUlflgw%DWJmaaURe|?;VGu9@mwwn zxqf&syuqPMh@*fRSxkND&;IHKv>H->xPMSD8Gmw3e>#SpzIf=yUvoUgfFbFSI#>Qv zL=|uu=aRHGQf4G{we7_v=hQ2M-?v?JojQ4H=MAwr+qgr4rvho!y)=o(2bW;Zo58d% zxuGon;+y1;u{HBb!0aP_3WQYvP{i1h^thM?2cK}(Ss4@;#Xo0!Ua2ZyTQ@)R{CRBT zc1U9hu0z0Ow3-6GIEw;nyQ?#E|T)Qv{QwdI#1p zLvSa5<5AuUwS^26pZ9a&`R(Uw*Zsv%&$!=kRc;f(Co=rVm=s>}tR1l7ZcPcJ1g!g$ zArWHZyO`HHs;EhoyuOa_*YCb!!TqME|`k zy`TH5o@&Lr2~KM;+%b1R7J2Dr`M8Y!gcGm z|9&G`1{>gv<2FkO^A#c4RB0bHM2cLq?|o@9^7CHbV*_L8R^>0I(=&;P1m#Bt0coc_ z&3U|&c)Y2Q4eI$Vob&6MI-*0{&&AuuLPfXczK$|uGi8W@@knG6E zy)`#h!;Q{mXk_t{vOi4mX;)@HO`GhL4F4SYIR6#3kMQc^ej4eB$^G^Izyrr^`!{D7 z^Lf`NME#t!TTmCqwr=+F+n2>^&Ga=?UT!%*WHMsRzx3A^-xyQBm=+aXdMqz%Q|7wC zM~o^B@2PA(r#rXb>gl(&LC5IpL6_Pyxhq?QsCMmFDid%Q6-e@o64mS3Y+`QX8u zB;768&fHJY72{05_s^U~B3?omwEO;g*Cbtiy^w{o0_}C~7a<8bIzIlstv$p_7ake{ zy6(zF3}Lm9U&-l<3vRG6ly&PAvvQ9H*JdUj1)b3%v#j%bx0PjflQFNtuZ!8o<9%1m z@~h0nU16G*DP+T9SOK}QJtkScWunf)-yhLS*JXm*nks4=tsn)Y(k{jhDCyPuB2 z$lvfEk21QDFAbzI4s?ONXjwSuei zH`=+RskdG*k=0W$nu-6*eF!vTtUPs;N<*0z4(uL91PTRox2CWfSK~C2TjK7<}UA|<>pNPtt}?D3i1VRY|!PW zwpPmuzv#QWo2RtxP}xK8GOm?pE}Ami%#H{3}SQfo^PR2F`M3my>iMwf%REo4c$xZP86!g5|+9dpVdMFO<3d9eg2s5>3vX-Rqc} zrtZk-Cceb{U^L!!W8;e|`g?X`bWb{H|NM_%KVfa*NMV{CZH11${+FCB%=2dl465N* zmH4{X46{up?G4ksw|#xq9kI>84w&}qjnj2u+7CUrO-(!5uicn6cud~13@Bp~$W!}yk4EZmUyGGFUYstE$l~ID+YyZ8e*3w4?9Rg8%n127(Kc#o` z{yOS>;2KF81QPl0Xq3T}WAM_!xcR=b)}weDrMihc$Mf}4IE0NxF5wS53VhgE=l*&y z@NUyDE|z-YoHrVJ9V^r=zAo%PHIq)35=-`a%lx}wWFnF(;J|@pPo2SecZ3i_?80Y7(#?=G9tCKEie0;f=>KC-Dn4Iu8Ce8zxk&|U zvzIDx&42dhiyG0Ifn7wkwnEv3w5A|7Mx-c59Y)+Ij1n@Teu1u#{QnJrQZEvw?G|w= zo8g>**n!EtFIWtI^%}r!a_tBDWdebr0u~|EQzEy>gufWT=F9WAD1=xIMHb3`(j$G% zhe0y>hg7Z6*2F3VZtURy(Xa=|-pHGX&AP02I4sUgChGqoJ!MhezU&Q@Pmls=27d%6 zhp#-+ewY2FEv~G)D~^@^&-RcKD^Ef1a*n}i<(U*mxN2?MxY36ndCg>~x;lvex(vVx zy_oXM&{wKXKw3JlVyB8#A_829D({T%>VU!BEx-kaol_7jpxC?ewbp#ThW{`Wv$?my zw1l-Du3e;%U>2}pn|kjDSGg!}G@#?I?JaVG*_nIZ^7Z@2ZWT0v5(SPh47inm?8+Yxv&8)P>A2A4fpA~n$Qi4F_YaeV2J)`BRuPxRwjaS(>1(sBJ{m{xR!7+ zziY0}`S3#zh2DEPULMk_iMd93#y zWGJ!PLdH-No30r`hqX-oLlB8fZW}IXTtxSP#Ek~SvOITvUqAYu$&+fV-@?E04)4RtH$blVt6gi)IU>-|6&~r zb9^Zf#!mEp&;4iwVRH^}32Ws1WP2mXD&X?F*pwttXFCWW$>4(x;4dSz#vVs+jZoMN zq0%Hm0F>Td*`e1wQI!myyKd7Pb|Kl-2}1yd!1Swg8dPGl8@06*UX&@&rkQCvQbR|Kh+8 zVz$IDy5nV?zCSMDgURI&s%Y%)i03K;1tIllGT_+K&N%j98Rsu%G`Q0SCE)!xUu9pfBU#uYNETLEwws0 zz|c%@nas7@*YH;n&$-^M2{hAOAak2$<{^~itX!w*C)909J7+acF1@Z0a+-H(2)!)B z(J*b&Y%M5GH<%VOf7+qeXVUPD?vxzye6#%nVaEGRYHK(cm-a){^eg2>4DINXzs}L@ zDRy0i8w1hadIBRR>(1Hu?w~nh9t@NUJ^<8V)MgNW7q)&|Y=9tYWDi5V_b%YI~G`lbr{ z+?+H&eL7{82>((F7s~>P59WV5liTOW+V}Y0p?zM5ySdkp971CeJ@B(*+~U>xp{`hm ze;)WBh>nWh;B4QrPaBL6mC$F2)ChPBR9di{-j_i8yP_din&o`lCDy z6M(*=bRNAw$c2Gt%-SMHb^8Pg$1vU9Fz%{{W_;!MW>J$Uq+3JXyuhy;MMtb{YmfXz5i~wxUGR-(=c`v= zYu@VhigfSUY)=+U9DN^gqfefHe>~B4g8c0R&79D?{2lzxi?165?p?Ejwn90kJ_HC4 zUF#^)$ggMZ?QspRsyQ32@#B0PiE+QECsoY45Qn|8dKZoQ+3Z|hnj2dg>P#O z3`EfLc<^@gfuTL6cb`xBDxD|Gjl=wG+ef^oeI|d#g~uRM=-fF?j*)}3xr13DP$?a) zS@764{=A1bKW!HgIxFXA67cj)w;O#*JfE(qrOq79Ou&e~xJMy72*;`FRkEzK?{_-* ze3cL#KQ-hJjii31y}(er`x3Lc2<;WN|M+-#IN8)hwB)y5l(hBQvib0*)3jCftb{)) zl5VH3!TX$x@rl~^9(knD7i9qRx$Y(O+YEX%d-7QFC7nDyOfJJV-R@*%V^)jEPp?k( zZ%Xtj;>){qY39}|QhX`oCI7dfq~jU?Pyg0#O>G7qGphNO2taEKU1z>Ndcm55nylv5 zz#}CHb3(zKTqWyt2~Ez!kxzDH7yg747IBWfb<2ekW3V???Y?xLw6_IUv9R*q%=Exa z75d%=f~jH*>MxN4<#o`cAXKSkrYTQ8-z5I5IUro8qr3G{$a%<1Y!Gv0-l6?JP#8?W z&JT0uj+F}o zr=>&s!?+rWGh3X7Dq9_&;-Sh9)W%PG{gKf9GyLGI8G*$cXO^P=r!1@fYy-s3Y;P``Bf@uN^< z$SN~ja}>?MR_!>Y*#D&M4zef#xv5Xf>awjqIQ83vmm|)$2DvZ$4;x$vQ$lnK&FiKU z<=ZM~q5Rk2^eP;$v>Jp*N*9WdYE;tgPe1?C1+b+WC+vjQs`*D%qj@mlAmVusM5jfvv;iIV zg_DNliVueF+5vw50qSP`G=8MzR?Z*LBQXtKj|3bhfET3W$42)!F+E||fUfiOLD9ZNMD@$l1;#yX*evp(G8lgqCp{K^sQ zRwrFPQ57ybC+L1MCJRVw(R${SObH%{&s3uMOMg!mBP%dGqW1kQQ>Z^XQ;m=_!s$dG zzEFWgtVIz)c^7X&dotPN)5J_hOhO_%<|{m+%h2>GqbOQ!b=uY<^FnhLaV5bpTg~JH zZVA6lgM0{kSD~VoTplr(JSNN@8#utXYlTl}%Z}z$8$*7Z1L(sBHW>mGmg@>?k{$j> z(Y1D!jH4}jhy@T00y?0*9{b+r$B)N(fEB5v6LgW)kX6)TeBfkbg#MRg^-$Z9(pJJ> z+J_{PN$FK(cy}Oa5B=e1zr93G1JsjJ6y&UYGrQiJCm(4TMc-c|>_M4Z7bPf!W766< z&U03?`Ub9l2f4!DpBrV4V-}NTO3Haoy+=Njfu$#U>g5fmw$&eAVV$r$0u(sH z`QMBN>!55~!Pz_rTNw7FwmZ@y#%2CaF{``QVW5kN!-Cv`{s@Rm%7+&htDg_Eq|NDG*rRLWi||;1>a0>=rNo; zJyQJrN89J<-Q`p5)G7La$t;1zwhx=#7Fzzz`X25yiaue1oIcvyjMI$gXs0LeA>lL~P^Mgz zDd@rjFNG6ZKuk=G(~tDaNpQ-YyT0pr69 zv1f{sEhSu5hMXLjyupA5q|wvxqQqg=5KF)rO|WIMez8=-ymkzL#TK%6v4X$9S?|$0 zvd^hWeTmwQeOxESU*$V$olxa`qk<=Z<9NKvzQB*r5pLhU-Dpd#qnT^&!-HdbxwYsMJzfob_+F^fU@1AF2&Qus&^DTJd`U@*)WSV^x9*Pgym+9pW=)Id4<~0cLwWUY4M(iNsoenEe+wHIFfSeOUE*W+anM^>pnLjcxdJ(pMOYqy~&+#A+dT?^I#uX`JmQU>`^# zNEm!JZ&^DeiB;JJBgxp2&tL|(9ul`zIXp2%_CVZT0D-tVtR;biR!S6(tCokoO>%nU z+>Pyi2&}Fsc*E>WxTR_=uvB9?sGSbb@TN>508#}cW@gB${Sw+EYj|sPp+RaXDg@+r zs`1)o7n>~tp>d|cVRvLFx`}1Z-LvWX4rDC={O^R6#RU?Lps5nqDJ9bE{U{uAGPf>` z2nZ4jIRS}<>DjZdaRj~#J)UMnj=-#<(IuBaj6<)v-gfE+Cm#BxNAR{t?iKh`e?76mxo$U+2|0cps|l1?^wEYX>u1j zjs|W%mcI)T9|b&~L<3P9ER9gV}^w4M|_*G(F zF`nPT&aN2lYnCARUx?}}EE+Onar&5C8^FH~hJ2@p;0wIDmt{5t$f{EztPxphXnTkp z0q9Xa)OtwBRv*(HF>e)4(GWnW8Pns5g%gPlf&$pX(xl5hRciwVx{i-MUpWZwO16g& zWe{C_jnU|SE?1A{;Rz=+E9_mcqxUMMTJxmMCs|uj4HLf>@Pj(As|}JJZ905rJRHqZ z{K^sAZQXV(pEQgqhakYo0mYx>ws1hM)`+UgAP&=UK;w`OoCSfaSFbK7aBcF|IZ|($ zWJx>satO+|kd}=dAK-qO?5dljcPR0I8>?Mbb#utB3#Z0!tiRJ8V-|T%s<{iiXrX1$ z6z#_OHVDPPUIF%vGV}>G90y@1rGA6FCtd~m)i&~~DZlWks3}b1yK5c0BL`WfziLc} z6pn`&7|yjlTYQtFV|=J9kThde(=5P>Do69*Z8QD$g~EYo9*6CTb~n9a5Ts@1Blh#J z-qm(%5Cp+;iI($wb0Q-ncO!nqhlLsC)CmBsd^qGOH^qymevAZ~IK4#y_?ox+o5Hu# zf;tYyXeY>H70@_a;}|;wsWfSTlMaz&L(iWPdvVyTw#wguku7a3_>n0dE{eV|p^(S5 z>PwlYRsE10R>VulhL6=$fdn8JfwBy?c;|wjeO6tFJ`!dhK#b~vNR8L{qCv#$CU`!T zn8NpA@EN5W$h5{01ST77A6Y!%=z~h3A~5a5`B^>y^*nkB#f^-BquznIyqbIoiDts5 z+69NMt({s2W$yXxMzu!vGGm)bk{p;-rm`USX6m7rX(Jd0*Z=z;-FOlKOCz38rRF2Y z+QiPH|LDXX)pK*1x`mLH)@*PcwRa6vi1fjqSFc{(;@T+$O{@uyV8dR30*+BM5T`qb zL}^pyg;NI*r;1v=&nHnZ&iKPeFYHS`FpsJG0)g*xlThhEY+U}@cfF{~)EL5otq3et zs0cWZV(qTlX{JpQM;z?yApXP?;4p-wrG@aqsX+G{KYp-mUxPVI8fteHPWKjyI1YDnV^Zrs zKDDnhWjRwsFbSt}zoL9sj_osuME9V&9z^_y4J;hG+3kVH98F!mdmSeMH-#!UnJ!lj zlJI@>jo!3~`@Fp6lTRK?xWqO>oO+mR^=dzUkY7*to^kzh;JDdPCd?%}m+Oj2KxF_5ss;#De=OxNiy*Zcfr zd`V=|;1=@~tT-dkXvtQd1_w0c_D0xwV6}OuK6Nou1Sc>t`9SB-E5cu&k(+GdFJ0z3 z_5xWnPY!Z5vTy__WWP6ume}Gw3EcR7UHG&uG8z$_uE=y_ zSPsu+(zpy9BZBW{G5RD-ekI+29dMA4fECoj4!3!%Z$FNa#g+ZRFGhmKE!&ncMI^9vSmzw=g{P%yq grvK*`Kbx7yaoAxK&)%bZ$wyQ6sVSx?oVfab0Cz^ThX4Qo diff --git a/results/random-s.png b/results/random-s.png deleted file mode 100644 index 3badd1f9a9804ebe9a5ce25069453daf4321a34e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 45942 zcmc$`1yq&W*FK7gqT&$&1&O25El3MkD4Xt*5G16#(?AX_8xWC}ZUF&l6#;1_ML>{n z)7^RJBF^uBzyCM>_q$`XsfDenk?!C2`qnl^7Uo=R zTx{1_{(fw0YiT3E&TjVa1#A}9hU|YSHkQCcPFjko+2G-w)j>apQbkgX@bL16Fn4b$ zIz~~>R+Jf79~BWCwI#??r`2|KLX#s379xVF3RDnQIN_0 zMNA@hNHpTsw?o8tc}Tp{Sq|s@6UV|7T5u}KR$_YQtZ6dl0{P>}mp<*X% zmp9my7XDhji!Uz2VBk-vv4I8t-d{Z%uKFC={p-LHvBACHt>R(%_Wma9Nkh2%H_x+w z|Bo+=le`uo`G(Kx?SkX)&t9Ank^>1??u;RIiR_h| zd9qt6GO^j~OA`&hEr~52@<$1|C||yOS-srZ&PNCrma17=_~JB~d}d2)q@<@5?Z1n5%?99}zU}+_6 zXUbQ`f12ck)#xl%D8KK{*xIiK-FPeNM>;@bJcTPmxRroz(8^{7J9jG4D-}Ju;bzX8c(t7CH1P3I_)#;f*+r@w84^yLsI9{W2bCa`W$fLp=|2!#b6Rbx=odbY693@qB}BF zGL*B_joVY@;$us1Jv(mP`yrQ_ zMmu9EvF#}`)JmDEa!;N-K`*VGtr0OX=i}q!JYPp2-4>p%l9RnLQ>ks(92HEXuc!C; z^YfF&-`{$DuBsv-B6-{adEnIM!ZDPCzQ2v z>G1pmKU{u`u)@aj6uR--H(6L`O_!BvyD=`kuYW0}%HFEnocAlSn@EPw&X8aK>eRed zltSoz2h37F;a&jo#fzz_sSHA!vsL0^VwPh?W<5OBH8tthBlQFX1oF`Wc8deg&T1`J z=;ai=v!7DuHtCc^FZJVHyzutwkcQj&^XIvp7aw<|%H2`UP+A)AXnSwerjhO)NWp&v z{7JZ$GX{NCUx}T5fqH>Oe}(HK)gtp~*uwOYP00i#Ors?e*==nyB2%sOLPDA?LMu_@ z9U1f)9qHuU#*ceHSc%KXwAhue^m}OY{&Gxs`s^9WxpVP^lmd^zbFsN;d2HEn8@G$W zcVD}9jhx>)k?8z+&GlaE23EAdFq0{0qkQ`GDIHcnMj_07XUkz}vReUc$k*S$Jw!%0 z!qC)--*xq!{mied=4e5jKzO;Zgie-vL46WI%DbV#%V-}KS!iQg zvx$87|B+c%UEf@2>B!TUICkvVSQBr5s4XMLuE(s15gZhZ>&zvd_DdFJ!4v$a ztG+F?h~&WgG4t`M7X5NI$)1Tf^&$&uO`;j=-(Kc5tenU$m1Zwr#4Z#V1a-rz-$rZc zq)8Wo@jgr4nXZ%(H6_ZeL|eJ^sJBbH&b_kxy-^62`>Kvox*|qLN2g?Gd&8#u?6CgI zbRRjV-apyJf5^eDFbN2#pA}wxm|dSexl($^n-Fmt{7f1Ha3RL zQz=QD#OuY2aWJ#anaUmaZqmDY>!Edul;x{K6bZ41&de=or!9Zx#to}m!m9&EG#&cv;6aLhq^w|9$r^58 zbZ3QD2cG%*`eHkbmB56NtH!N8t*HYC|5hnOd2IB1Li2Kz2=A$l{3|YJV&TOTuW@1 z;7nP21w?WcZ@c~iSvuE`iEE58CqxaNOzs;XfxaAv+aAamCM=Us5o%16N(QfooZxVgZ)2EyD6u!CPly~^>;l)0Oek1VT zOMRMm#pQ!rVmEK+wU6-PG=Mb-Nl?!_BJKfT^-!%OB!5%YZBwtBgapF zt2!%^3DJ5C{8@;tk-93FCJIIkEiE#!uJb-rZi{mK6odg3{H<%NC(qN3uHj-;ip=}k zvmWqH{m8weSz;@9@7@au7B9-67dpGMbwS%TKK_n(x>~*gaxvWYld4-V0Oz%laI{NV zA;#@&;zYu**tcm4DFWJEU`dO^Au{RCOXKT3{o8aHSo``gSEuZssd5Q=`Q6uH@o=w& zoEKBMxMu*U6ufu{Bbs|bl*};k*1W27EltjpZwNi zQJMAis~9YAQ#UT&o2nH;SPm=$S}*_#nPoIQEl23(c4@ zV6WhPjmF#41f1p*4O(Kdnj(28z*N8H%ZoGh{&IpA1&a z?TH(e$EMXCY_7|Sc@@X$A;4%{^Uln69={fFEthyAn=bO0zLuz5U3I*A<3?|i8_eko znxV9mn0mXuPtWi#fj76N^4MDHg5QEmgRpPVL>|7pv%OOJg60wIsAClsW$o!o4KMXv z!3#DQCMMD(04%II!`nMx8p)~edya@mcN1)qEyx2v-)X;UT?#?_MtBcCh@_Y6PMkW$ zB@NrH8|MMWdInPiczaF@5$eRTg0cJ7zU41Z5+TU!8_Zi2(i zujWG7CS+KD@K@iu?SFqhmY`Mc4DW}4)a))ig@0_WEgV}mX-^IQ4t~J3rE;NJ@E(Mr z;5ToKFTOAw(rtLf3KLL=r9HO%pv2awB8Sn1B}*-zuB&+RwY6_R05=M#IIHEZWQfbn zIpI1NYXG{KiE)A#oit^hbkwMDExugmxV?tcTwfY%tL2E0tO2WzGJ}A39S}_@ZekoZ zSoU(p$hP~dg*EGn&dHTXF!YOLWMs|DFin@R{@_lDS8q7=ZJ=BR#?zK8)m%8aBDk{B z2k%4{#-wCTHPc(%_EPW0`e*{%)^S3L*E1&dMzC;j^{1!D*rwoV_w@7UJ=^xkf6zOQrOy!KZqROqNZsi|VVX;pu@Ovw>gM^k}>u^T+ zQ@gKz>b75=R7=pU_APeLCgItBI(@WbOi7EuHAlCBukOgv7Ks+|&;sE6J%Y>7 z7DfkQb9~^_v*y;7AmMc}*d68hob)M^{oCp+q&Rg(fVyz60)$ZYLen%@1>)`r0gvsn z+js7qnhbH<28*E5bOCrOE-BgWdFV*g$`IIEo+^tm{4PeJiXh)}zBNgLgo-K~JWk(8 z33y#uF9MRNuBPp&MF<1wjg68)y5WhhUtbK*ky};_fO)$$H#gU;`m@L)=bwL0gsTcT zOtX~s#boGJfSr1!t9}i1tM(;XlU7kJu}vet{x}Y@pwb%-%j>M&D>`fQBaqR&;n2CA z%_!DAWVW@wRR8tsHvl!7>!$%+0LlYb71l7sq?j7#GE?5=MDf>Oaex@)D!11&^37nn z($x#n!JQ77o+2ddoc&$JzOkMp(c`+Yw#M%f>6)aRqm>1qxTE-Z0R7OF(pEN?H$!14 zCk2pUQ-IUBK+njCff<&o4WwL}$f;z4f41&a@F2sGuoKhr3n#U|k*!s(PU&@9mi?6Hl#LfEAenM_Ah%1) zMapBXK|3@wG;wVKX9Am%ZKThekP>cPuNB!2o+#>S6 zapXr!OG^ZCX|G)f7CRq+gZaR+ZQXwlkA8FA2#X*|A0#-4aDlC_np%B*0{o_lc(wNS z`~^&Sa+~P=`xCGyewMp9RQNnfP!a%~RBCd^C z{*`1Ja2?wHt_w{{@F+V>xYmPaJFi`6q0_$+NF|icWA<~{I7cN%D`xA|`JE0Xl!5ft zrrM4fVgL<8kY~;B&iwM_F1(Noz@&x+aLYmKhk?4HDxIsj6kV!- zGEB%%+de)yVh0=v|rDf-J-=3wY-@OzZZ8&aiGEt`R^RyT-F>xY5zA=cS-ylnR4@iMkE$?no zpM7L!wkES*^%d``c!1fD|1?H8X#waZk14-+R+Z(|jh=wRq+O~ze$wZ7R2ul@pXt=Z!C zoFknNEIOk2Z5G=VWX9f@WCzo%ZEWC*Bc-JQH+R6Q5lrU8Y@&U!4meH|ONY3N+PT0> zyv_RK63zX;`t7uaWYNh{|;m$n) zPR{wfl`rSfv%%rn!9-mc7m4KUvkfsX8Kc`;h}~(2I1c#w?=76j&i6ZKg%k&d@-i~4 zf9=G2Y=Z+mCDfVqK!uT!QFMJhFQUo7ZoD0X5*~*eKF#!#y(v6; z9%hhuj?NeGyRF-18$bITSPPNN2m)L&k)FubVn~AxsTUcS!G|wo=g%uV-1+C9e>4Q& z0zEeR#pk@!vIux6^YSHSoRzq^uXGgOZ|nz~@Ru*+Ame`Anuvjj`Lv-R8^A}wuq7XI z#UB)#i0#X8yf3WXElO0lsrW1h^0YHDvaW>quDCU8!iBb@-{LiH$Gc)sZ14FVqUXy)UR#6KP@BPD1 zhr``rW?ofRR%*E|DTP-!O`TBuG4}%=-92UB1i3&`3oI~8NZoL+PA0?q$KOAXY~id; z%uq7jMb3=;a}kjtr1Wev>=tEnFJNW}T+cZ5xiHSSsH=9jtkg5-Xnf~hxO4|-4MhOT zf10BwIE6(ZE1c@D+{x>YRN>_5Bk9`E&;n3Ia^XS>3`d{`_D;CPh`x@-ln?`o4}*h) z8lCUgzO^J6dmNvWylyfBt-!pX42&3%t1N2}Jle zgD(o&XRa?#Wx&cBfBEN}^Hf1sI}rRpvEPN=)P{&Nm}@3BHtEWp?NT6vMqGO#!E6We z5z$|WEr!vgfcrF_?kz6aUZ1eTMLUc`r37ZUyd5GxrLe0i1W_e;954j(dO4S`T)7Ra zqJT(;GAA1$s4pNMcW`K^!1DL)t@VkVH(Z8+>P6;Lu9A>+DdrnUp%fW>Y8XT&0LqAT zfH2h9&CbFiiKZ1O9bs;ZUm9B*D-O$QfJ)7Q{L`Smaw-9Ca$f!I%?%!As<&7H$cO26Z8Sv)g&4M$w<|ZzX>}`Py27n&{w6nFSf)$-iOeO@aG)OogX1xOVl?0iA zCKLfwU@mDez{|u6Z_eHcxOnZhw(Fc1jN1S#YiqGh2GM^@5E?V|8(tx_6XW!UOe%s? zbgHMQ3)rQrz)WX7C}INs2oh8!$i=R^Z#qi7xsE~f6Yy18NAL+p5%mC$aXz!PBnmho zqvj}nM9mhsZP+VisVhOOX0n`EjY8QZw|TEB&AJQi}g-O^51Frv+XMYHYF zbQnYa1^nhO=LuHuRwXn~GoT8F07)A_4rDs%4~op!;i|=M!`Kn>2f#wX!^4ro6Jw^d%k zu2$iRltRY*_;Kp(+qc_G>`Xe+6z;?|Pb=ku(b1ok&D zpqUg@K0wHZ%4PBzj&umB812P2xa?Hyu<^_-onJwxS|L$}OtQrE8XE@6Vb;rQ0GE>x z0S}Aa`A(;*{hiJo@Bz|@2m=Qi?UVSl27_2>y*i=*GOl|NP2?dhL~~8DZg>nzy0oQa zqZ@ic4?%H^Ll{g~qsQ!0O^4aF zn5nP9S8E^=2LL7}hniKhnOs+ncC6tJ*zhHIA099YfAkrbVRPe(Q|0R5sjECZohQzo zxnaGxFH+S^#{De z=f3$t=i&?CR0tozI^2VV>`X?7@!fm($Q^z;r8F%3p?BFBZV}n(ef;x%nofaw08HT} z;HH~8R=}^+K>aiT-fx%g8L1Bo0A7I{RY7bfh9+5SV1rgS2p0j@^5V=Te<~s8i%{yh z2iWl3`SXuXU*!0^i?kB>jvC1J`~iNF0sjWjfkXZMV;#vi*XzW@#jVE#p-6xx%5r() zM{Yu{d7nVIu+5*wYvWxx7ONW@m*nK+YM`pCOG88RVX@AlVXUk|8W9`Q6&vPP0S%;T z6z9!-IWNOk^pw5Pz7i(xI^^FkO$0|LSv?olL$=B}44Wws+xE4w(bmaSr?z@~Yva@= zmE{WHNU|;sn*g0$RShleTMw*qG>Ti%Je<3D@iF-Pp4U8PZ?1W4Z))`4w=eB`4H=$X zfw2;r;}0XR9{58Uclc*aHwOVPRsII5L*J8>^#46GW1-Zo}wHp;8-BW{~rGlzAD z;FkbfwRvSAsz;1nE$_?P)`MJK(MR9nZq@A8kW9#5;2z~eP`C%KTl42$F-Wz}$gBEYj>Ov{LlU2WN*}Iuqr(ivQ;LjbQs1jh3 zQ{K6@a3cpN0+jMK>tQjJdc1%jbB>bI%yzuJ4jHRN{r z3y;A}^2P6=jBkf-Uh8?M^9+?pQOO0WsW#>gT3WC$4+1Xnb^^mQJ|$aU01POC`fr5Y zN`T`U|Nf{HZb0Qrp#96<;=n53Lk5`$J1jr&F5`O$s6w+JkNhmLV^%sk@)6wS5-R1= z(vGBGAUck>O4a%0kY~dvq~J(h;5Ju%!Fk{bzN1y)qtR;N2G&rBPwWZuB(+`g} z#>N6wG7c`b6WpuJVSEOi3+!69{K78FnZO<>^LDF?n&0QM>%&p4=NI11jt^Stox-8GPu$Pz&6AqDWDt;}gbk23-x z1avu#C0-l_AeLR#$w&NmbpN~lMD={iQDC!+3h?U^BHt zZ3va)Ap3F=q=-}HXqKZo_=y6C40(=~JC&COdVXGtER-&pfI?~bj#qt^i%S6tzXYdF z4T;=g$iIkVID7EJ$;CzMLn~B&g6_-iXz~7AiOsArks*W_DCm?DpFe*U%Go0g;iXYK zfHgXjBrc$uD->o09i|)Y0syf?X-6LL015-`0RE$IIJG9VfKNnV>=-^iGvwkZKLsjM zA{KNu^1vj7LkSZ!EMP95HCmQHsFDT?iRzsY-BH~_dx{m!6ig>^4xJU7#)mlIP>pvuqSw9-#1aH2k>zHQO#+LJQ(jHl+{Opmr^!0&7?5`rhyy^ z98``2hcK;BMt~5>)D$bCEhDs~DAz}{J3h}FYm1~qKrtee0m;gwX@+)X1uA4h>eFZ2 zCgtnrrvQToP`gw?YP+nuZZ@3_PX_VpKG)$sivk zJ|8}Oa2=A6kx_AWE?M9M{<1ZR^;cez(~YQtjYWRXeLPDTG zeg;WGz%J)?^={79!G`iBVQx>jfA+uKP(m)9J(uH7tl1LLgJuAtD$wG#X*K-dV?ojt z?bhoMT>q<_$&h-5G7DAI0L_8wp-wmVt8VUaH~j{=Q?7beo_bdC&#kSkwaZU9gRBDQ zJ^%BQ=bx)t=xFDVp3wm~0)}7$=}B`T!puK%A0ZZ(CJlf)Dh_Mf2SY9r{rU4}guM)7 zqq@FMhqwj692P*nmlRa>`y1eaD7QUx^bJvAwt^HCQu@odIjAug*x;m*1ZN-X*-rkr zhCEbj($N6hG;2C|yWhWmlMC3U*7%bpz?6X3kVFhOl;9JfPKd#_!xra;6dmOX$smY> zYMKy;oRG#OU)%t4Hk2(PaEVXtDtn37ydCP9sjyJ|kC1W+mDNF+HV!4~_Dr>WTgXvx z=1O~Gt**b>ZKFC73!xn>GoaxubF812A7!7{$!6pJuRD;YL z5VkUy8*IZR$kYU>ALXhpf|ZOzEnF^=$BY={Kfq_1fV`|(7&hv6fRm^ejB>4Xt@4sz z?i*&f^vpMIXaH%iqw^?K zEg~WUC0I~bsM>rGs^)JKihuB9*b<`zuq_PuZLkLm?`IjqZ6^EL#n1~hpy5P+T=9S) z-Tgy-^WS*#a^L^X9{L|%=KgCh0O2_-FW~IPceY!SSm%?%^-i8ylE3S@BKNN<5@I0w zvQTokjusOrveV3Hc!`9cA!uJ!DSl>!QIS77;(FcMOu9vd8qsgOBmyWy?f&_1$GRW; z;@bal9izp_E4!98Z>`Okl7RPh??`vL$`YrzFUn-#jw+QVyO(YAqOUnyk)i9g^fB~U z1~>f}i+-=hqa z2UpYjHT|(}v}Fo)*q^W@Y&WaVP#h8d2*uv~-b@QM5g-ZUW56BUlcdp>Mnt4HoV&y) ztF=Glq2kkFeD(EMtKUx@o*oiEw5Krjl&_TRb#xT(EK`zmuY;a=Wd(K@1Ym;IXPcAS^WD2WER55}pi+7k1N z@DKy{{%qh4{Eq~#vt%!HzFP8?Q63*4ncSbp4(u-vZ8du7uQ9I~Ty)U+p%E4&gR2$gv7b^HXv-Ulh1sj<`zSJ+b_(k^@D7`qxrwUf$-+R=v#iNGx$m~mD zM$7*B_crYxsxL0zm&g9!e#_qh^Lq@b-l2=C}j;f_s$#xrjmsxlI7gF zc|YHId$F57`x;}0rA&`CKlD>ng;(Y~rnt9||1*_w?4ye>{QXZ`cpnO7uunORZe4xe z&iWv<#3`S`$0v`Tjww}%x%qr_8&%cRx_{oF^(HJ~=?SKOw@O29u&3d8o4Z*C5lTET`CVB#z9bj3tW^7#VIsvagK66&*mE;CV;kh6r6h(4zV!~j0B-G&~iX!KNv*PX;8Ws2oZvWMj7-uNY;dD4P{h#4_P^I zyY~V7u`05l5qS<8+Gl*x5r72)_)B0792nrH47i}V5VE8K^FDPeE35JT%1U5jLEjX0 zk^(c{Tb$o$kYBMeEeNG>3H155_-DX)M7crise1DJjfCLVZqAQBf?I`o=~@1A{+d{S^TXHbz=H2^*FDTbh+pl}*dQPJY}_eDkdTVTL|{98xLVq;^Olru+Lp~fXpoE_}O z24eu7K)72>Lp3%W1YE{Y#VG!%lBJ$BFfiasjURgKe}{S-q?L`51vVxdiy}L6&JW{M z1{tU16%@)T5BtidNezY`8>$U*NSnn#kqrzFWgd{F)cFM;s-}l*2%aY!L?tvlGcz+~ z>8bna0HN}&NWIWupk(#lE^ z5gOS)*Z!9jj-%O4rW&7m(f)++vwdjz6drj0`}glZAuZ2m1$5o-wh|4n>?#J~L(umk z8r3isa^(8omZ^}oHVXp)vW~$200x98QQuXYTbZyYKHQM8kDbIZO70pH9^NkQ3JoPj z`3_`lp|(bcegmgB4#*R_B0vZBKE`9)DNCbR9)ynxh?|3wbSj`5Y~t}VKoYO@Z)_Hv_ z#SSVC#vnF9GUIB@> zeNK^n4c-mkG>p#86cLT>jB%#>acP#3&cj20d)`kX@+z$;Y4WLf|Ep3b(XF$LV4`(d z<27@JqJ*-q_M`nOaY#3djpGN2NTJK0B);R(dhZ3_k2qCGw;l;X{KP+S@`1EhK99VH zP>+8QQEDfZeX8E@F`PZPL^^v+QE@Pi;vly07bh!#i?-fxzWIn7ej_bPOIW`Tn($(W zv>+sjWd1((cJFeuOj~IZw0Y`BrXXm#%CV9ET)*0H<<`u ze;>bj%G4J$@xhRXHp?i{JLBQ0TzThYv!;~82Cz!v?(@xbl@*BMxmWgMD&`Y!|K!wP zotRYzuKBq18{ZulOIqme*4fNR^MkAT+unx{}p%iAQZFY17?+k3FADt zU*Ae*P}b)e=RU&xXjm+h@)o85wg12$&lc-g=@Bt;>~Bj)`ipU|50;r_MVx}4!a7Q1 zCip5GWBkL_McV1@9n{O_gv7I6t;dgj4I86mXwbl)~oa+~<-|9zuBTDw?I z>0qUr+;)-Fe{L=A{1Oi%42>*A;kLxulo7Hq8Oq@Da475;^`kdm|(6RQ?}X~uTz|yoMq=v%Ss;DevAuk zAPpEr*vjT*y2AC8r zqb%L<&v;-YL$B@6s{YdrLM1{bE7||t0{iS>BQTs-J&&(Xxi`X_5d&I2LttiL0*vHw zqX)SD@m;#EJ{2icMwDye^~vFw69wZVgggn@uAUoBq9`77~7hW(xM zHldu7R95fc#afL@HGdoAAC{yueBkv%mHpmPV)kT(U03K@51-kDU>9f86iU8+u4cHu zpHZ$pZ?i8I7RP-5*{-PXBnS|8|7qgfSl*Ac`>&crRCnnk$^N^f$rIOIszReyTmEu) ztt*{YA+S2=koG|)!9*-NUlS9W$fb}{9~?@vRXl3d-Wb8@^_*BU&NDo8y)UgfRwM`1 zSg3^wNup3~%x*#%=TyaUUFY*rR8zcl>lT7ONMT`(#8XBkWvRfdff0jCE56h2TpX)X4(PC9Q~duJrIg zHCL0)%G{0dCXkO{|+_{=6bJxY{;wSc|@U4HbgOUX!oagW758q zf}hDC`>uiVE6%=b9fGsr+nbxTmTO_{9 zmjS*tSt>FaM0KKf?=rwAk81p_e4tvW+z`&*!n^%nVCzE}3;Y2E$8gUjpN*pgA z{pIADGfDvNuItwmBL*2VkM)(xoh%?|A%Btq_>4^iGBvyw@b>>R)IKIGqXp#X40iRu z;Ny@7hgSVT$D5^^dm9-6RFR6R4*4GYcL~ZC`Av!@hWsuR2(xB=Q^=SunPBrOrFEB| zv`=!|Dp*)8G45#3lZn0J-|hhIvb!8=*f>of4eHT=2BbL1$msm4ku94*f%IpETWQgq zQkIMkri_j^8GZfzvD8Ei3kTJSz5gfOe_vTib>_69NU3Iny!?AQZl&-@(~%h*y@y|Q zL6)v)(uk>;I8%}K(ENHyL%MwBJ1f{xkbh~TCfv%c^(!|tAZI9hTN`;5sHpB} z0)5dmnL)i%Cr`>jA{++dD<~Iez>LWPSx-z!nFU%bz71M>`Uxa*f_0<8nEiY&r>px` zVx$?UQjnRU(!F6UP`t$~t0>{y!|Mx?mGCRJmXJRB4JY*dc%m6kI?~6@m3X{#a=ZWY z-L4>7LU>B90UxNOJ-BJOR9gI2H-5L7xDU z7oBxziY*=P&-fb!a8D)YpGl$tGv%5xkxB9@<2-UVe;`3=rU8Gj|pjX_bZb|pk``b0M1&y z)IJNnK9v1NEj%_N=_Fqd^lE8fq0=U#+V6r1T!-X91t3mj)vnNhG>NS^ayO)nNJmi9Evk#SUPoSWui+42HCJu2JO;dE8I@=!>${OK%#?e znQhkT4UbuRj!mXkOn8OcMmp#PkKymRUd}T|4ioEfj=k04>mfAMH@fTiVI9=zaW@KS zc=3i%n(cYLvZur2t1RCxB#(yp8B;4#K zj`+*}#MKo>CBe3~u&@M9diH6BbP6CRIrqVMoFdS2iR3SqH4Wl*fsRyD9N*79LM_gb z`i-w`OvoA}=bO1Wm%J&Vb+9Hd?o;2k8~>`Ck7!DXByrkvOtP5=5xksKIYWX74Tg## zuCn0P5w%O#8MZ%q68*~_;)IjNi!xcN4fUNE$v+DId!xzL7RhvB^y;#RL)zg3Sm`O> zV#=B3B>W)Y$jAZmlCg}HUwY(R(#oFS!6*Cb03$!Yl&KtkHkS-*uboU@Yssf4BPLt( zyULV{Ft&DjkC%&;{bZ`V<%u`d@YS-=OhQH_n)jWcfb04JYbv9prynObL!&8lJLiEOym zmNqbP0FUlTiHInHY~g3&pM17J7oyr*w1ln)X-W(9wiwHO&8lfScaE)Aaqwf;!83{B znahTH479aBdAqHpQp6=2J+>&Ic)XYYeN4GtpgzK0*RR)lMma-WJZqNC#n0epPHmdHLPD+-tb@BRA2 zteTaP(K?Ti=7pW!d#+$k4NmAAWcL#`HP$R;GmL}{)y~V9Y{{yZJFz`M8`WPJOuzjN zq`zKnx3$tAyo&wNel#q2ryjk-gGv95OF3Gq8SEY|Jgt8F4oPeLv+%b^d`S-=z3)5K zc0roE+?$*2y~bfpx^pN;Jm6i8o>hG=qdY8oMfJx;G+THd@%K#+Z}t#0I-p;LI=$#) zyKkL0dGaJyhat{f6A~K7gCaoVwbOU7APfjypDGep{~)`b-&AQJrR4sHjQt+I2H8Od za=D7*Jv~%R+gjgq3u#tC@s5c@_Rq_|4551+^2uMPh&~V;WCNc?TB6&|Lf_L+*4UUa z2?FVVT3f8P%TAYS(iL9y4uZlH8rRK;Z|%l*{J&nk0&+qRl9{Q;7p-Y%!yfA0{C-ez zIj>$!Yt_t0A@C}U7BcK%oWQy2n3uBH_(P)m316IKDoh}gxk6%LQc8m!=SX^ujwZ$G zm4lIg(&J1^dcfV>xoC^2FJL1Kn|1$c6HKc{5idZV62 z@YQPa<>DP@0q|Pq4{YjmMqMJ^bg5!eNxcSr%^1 zcU6uTHcG0Q@*oo=I60^L-1DG_Fh&Wo9`U&*{@asDWei}kgS2BDz^0ZCz-*<0Kz1j>fnZvijLJ~ z3X@^H|BT*RW^mKIXd)=zFl?@Yz2t*i>FH#ZPu>S9r`poy7Kyq2*67B_{D^>{TL}@a zNujelU(5IO>_I&&LZ2t1{WHXvmVz#UohdOfhOw8P%=_$tY;v(ont&`E$g!g_`y=M{ zsMkN9%m?{vGIzeZrtqf08r0(j>}W9dOjZMT4%YWQfk8g`@uPe}SFza8Z9}`jXW|sA ze~IiLN=>KN_rG>MH3wC>9|Id3&b=S@iPSS@`3ir3ZKF`_MDik z`D-&^tk7=`Uh&c^=plPlySBi|$yH8#X5e6}Syw?N&-VN?=!6Rw;ZPm(y=e&zO$I0raq`NnrBuz%H=H2%-j%`OY-IP z*iyQs<$sbPQ(h>#9_F5UXVn{k2s)etsC^ZMWXWKii_O9>bkMOCsMicmr2sjlA>|Fn zZzBSzH3!NQ#H6HjaKHofK7#0}I;jPkRyr!&T;MQ*bkJ!*dgz?(i1aF;0b%CgkOcwl zcaQ=5;k)76hiErc zqF2#!5{1;a17$KGQv*RK6Dw;|w;`m$KcUqSlAjc4$WjDgidxj7aUhUDLLR`f5=i0! zTDnvaprBUpX5j~grr)6-5K+2YAW}okR?f?lX`sdnZA1qV;XxY!1B6EjDLbQ$%QRoW z!ebaafX9V03#h?TK}&*71S)o-e`mW0A#qH2y5;{LnI@*69F6uPi7^>*B6wC-NxFj5 zC*|7gr=UD5%K-D*R%rGBwKa+CZ02vzhgUv^sth`m=DN)vNlYsgmPkM-0F^F5>hu3E zQ^>%)5;z0_puH3*TYUfdM;4M!&|3@%XSKWt-#L#d*^q5QEsbe@xZC?OA$-azX$^{oE?UX4j$k+6#tINqNE0pYezcVAU%7!C#Fs5mf(zB4Ih@FZ}>& zkzYy9^e{lw)&w++Qkpz1`@Q4DvHz#6lVapkcfR3XZAW_0pHfg66i%V*JH=ZMK)GZK zEpwn~PPKR7^%CJj|1 zx-eY3CI`F8VYwR)8G@@(L&0P|kUZhVPmyRHx}s8`Efxel|4X`u=c|w=J=hDMtwVcL z8pvqIK*YR0b7EUUH zBbt^#`^wki?d@#>y+cT91ngQoIxh)5YN|R(LQo$JOtnRLBQwT*-AEH8z^D{U&TFp0 z$-C=1W%Y>q-5-d=@F(opPiJapsD$hAWc*xPD4}Ke+7o|hjj00@1&vKmw3PrGsV@rp zZ*#rk-CVD7yX2!V)UlP|WUNz@yN_qsYUGLB92#1ZXnNCJC^Bgu{k2|RsZxGRj%-8~ z$hKXf%T23yKTOk|2A_YHm)H1Nl$7Gacx7PzAaGJb38V2v{5^a)-@rmbVnP1y$B_e= zE@%)bg>C=B>a+0Z-iy}l-g9IOX8-GbG-EF*AV#D}u zjg6%{Oula5@2tLMbil7pzuw%^&z8L=VcaIVufL@JytqUpRV{Pf7F9)mxf&C6R__Cx)J1fb0p(Nj|8aeWV(M$XhVYW)_ zRY)h)GZ??|ARr-ht`v(8Ei;l(J4bs69X$p{S!Y~8YG=A@TEvl);uM&Y=~3q{Fyl zuq`DT4y<@VXkd`;e$(?p!qM#%Sxgj=ilpmLVWX4}KHQxo2F7Ghykk7(lQ#mfift`2 zwb1Bh28Taj2IJai9p~NvJ;B7y-Qna4E)f|vbVND@AKw+U-NcxT6&kfDqfrl95v9OF zIK!cEG#EII3mE7ERG0;NXdDhsW0=R=#l;Ma)0n6#H8lZ~A{R#Tk72c#C|JT&*az4| zXbNIvViJd^bfAVh031nQj~qpVj7w}kdqVb~y5ZJBkPrYG$i5*xn3$Z*bp5&__B|Y( z)o0Tp1TD!WW66==y1!pJ`xm;+6FX?gId;6rI0^|>7=ObMfrCYR8uazNs7DWV@z9{2 zcwqCH4~EG0qAxoWFtOF>OE&YKWWRooSQCKY{-^cxI{3t&+@6vSYeR>yfYU=Q11Q9y z3i_D5QZoo#Wrq*#CVa9Qtt*pqpFvI-1WCN{ueV;3*t|9#qZjezWMsyFUiBUle*?fxoe2HVp7S@yk!32mchafmHSWtZ#s}%)kHLCCh)31QJnkzbHMUza8wk$ zj6LjNM%huCW;oSGuRtA@w9#2h9$ORIh#rG5lY6mh*XrX|Uo$JdWFZab0Wp|59C!6q zuT@Gp|`V8lvszuU+iFy zqWT1MYRiJd?|^n9C}X6j4(-{7>gzr%AC!I~r0CWnC4Zef@zuNb)TL5Ch@sG1bR9w# zaNgsf><I7MH7v zp*Aq9lW=|@P^c{q_3Z$i7vU8ys6E^rU+B@5zO>K?#;e>)$uIv>lcL+gMvO@gh_yn3 zjpdv5y-jZgVJj0rIXcg&ZebG;<`VF^B>%a8OtCe#$KU%oP`hbKfYQ^_nr(VoTibJP zdlC8n^&#<_$zU(^lna2BSK#j4xzhpZ#+8%1Y>+2bcYYy+|9X28D{im`_!eB{1aN`J zGs5hw21vZJzd&)u{s|5MNd`1p45lC1-4Jn!5oSF6&9UJ$5r+@!4Xdsc_;jS#ey-~H zY^)^eSN-7fVK14%-M5of?w+pHhOdCqFC$b(L+YH^KI*kjefp&3Pxv8Gwk-{s(*ArG zMa=@t1bf%bkkxAITJ(8!jtpJYo@c1$HhMFv`ep1zo=<^;|>1o}X5Z_dbowX0bMiT zty6BFpoFpE`)rRXd0y~42s4Sow6WN&pLdhfRDD?$@Q~&*Bmg&tGNz;`KrVxD?slI zE*6>~chC6(pv#R;?#<#)H$K;ZbO7|J&@C!a+N~#lDnn@k zx`=>GmQVGtGXy!v#G)Y?>vhP!t!X+%tBqN4edh5kq{H0Mm@g zE~c*GS_Fey_$dUU!jeojxBRZ%4mDH>{n8C$HPluGO~G})3oYBA^~sNhV{hW$fa+ll znLq~%sptb1-=dyqKrgLggK<9`%jlZnAQRMUVD{tPEp+M*9EI9Iy?djpQZr?`q_W8q zg#<}Dg>21qATBScJodze@gbH2lwo#ITPL}{Td~7)l=>FHTmSQ2Jm4$NW5>1Z-6Da0 z5|)y(quPmzNYnGaqhkEC$kmX~_m2cHfm zlR$Suj9_Zbt!s09dzJpV!Rmv{g=xXlEtzr$5)IDTrbA61kq6MjtOCtaBQu2$+rZQK zvFsgLEMEFbIisRQ&TSVvR=xCs1^)sIjt+r@#SWo?b0um=G<3xGI28s7^`pnd>*|_^ zh^xLmX_Vx)g$}*!R{%m#I~?Dh@BK&*MG3`*7Mc6`yEqWd!O%x1R$0Jn!ct_i(7@m~e7Z$2j&&H@KjUg)x_(<|-e$Gw}@aKi;8Y zH=VTPyY4iv=jH8<)D}oe3uSVs-orU3aunD0aFedQa#m|MC;$cpXGS;gQ!|>?1~v#M z>pzTZnEMS;I=4sjCsbA5M204M%S5;SU)6niIFA>|LkB-i6@WC@KW=e zg(Rskap~gs_b)}k%W={&U$rS)xnCSiN!If}Fo!83V9TbU7&J@DZowVRgua?wD4if! z*;GsU8wA3w@Hr9~sXCam7}htg9;tq&qr7bM!K9TnCcCX=WZ=JS+B7wP{w(fGzT$xI z)KLR;NP%M52tNoh!&r0i3r`_8za4+ni%|9GyO~?;zGA_V77ZfW{0eDkUk%_VCqF|& z1shpF{{Dh)(+|(DFvI^{7S}WnGEjA_Rc>D1SXc^P1hk>68(*&oqnC*58*N1(vZvJy zbL2D=7Z4RSGMudmIz-8rYPa6~-K^)3S?Dh5f91NMf@|-Ag@Fz)f-3kAsZNo?;STyq zI=rI*2gR^(2@nQwZrUQrak?`8A*eIe4#MM7V7gp~FuM(sSN>L5bi(p3CfTu74bL7X zal6x7ZX8WUsK{W}*=W;fQQ?MP1lf^LWtluS$q|}K+I$jOE!}>hlnBqLf`5cRq8gFK z=g${n=-uCcj#NhA$I=yGLnl8^G6sSapQopt6&)No)8{D2u;*V^z{~A#>M3Zes)h?` zg<6@DS3Hb(8NgMLw9ud&@mBKEeLgxCerY7k@`M0fozPiiDjF~WckU~`f28gV{V*9@1PR77AFTK%c6A>)PTpCtmm-*=+Aqf$Tfy%?Sun-C`*k8hN z~}m3Nz)b7Zd_KdxW%$wk;kxoIkL2!C-Vj9J7I zgpGr*T)9bVe4G?lfIJ|o0pJ%!;9_4nW*jzs4Pb<%63olXi-H#ae8H797cX9{AkS^E zhT0(FCKRzy?0z4#;59v&|0>SgN#Z;oW(IV13^gbK0W2o88FEBu{&;=x!foMyXy+C+ ze>qf*Ly*rjc(YJf=)R^4b@4-nnRmO*&9#Fc<`)&H9h_*j&hYA2>(tsF=`Tw>Jw0`W z1LV59#m^R_oQHGbHRU^_`ml9ggYNn*HYEKHWb)t}A4yzhM{|g5yMeH+ ze9*E&5!CXkt84ZJKZ9vz^zPfIX@M#F3%XD_nGC>SzSj2g_L=^Hfkz8CRfYe-W_jL| zygp(mP5BJJSnTet8-KJmmbJvz#a~Lex175yK5I(!J?u0i{1XmgE8$$sa%&MNk*=#S9fT|R>JPK7$6Tp@{h+p^ zcnqifxVPNjTY8nq7jFOl{m0Z}q*VH_)-AA!q}3emsL_ly`h_J)_FRyP9DYDt?ZLr8 z7UR7GreQGOz3R|H9QKcJ;&D8f!cR;)008QcONO}H8!pY*^+OD^svIiHKxvp9BJn)* zgLUTlE%>}iLt@fZdyQ(7muJ5mdf|ORW8*?y$)!i9@&@#-Sa3g^GjLDP^G3yBs6oik zRvV@oEM8o1TTH@tm>k^o92(T-YHLDZx=QDsERuwP(E1@cPH@Y?juAUpG>3$hWJL5S z@mcwuo%2ag%VY&kZ~;h1+ziEwBZLp_@Diy2jvJ!6mi9)tX|1<;BToJ`WI#^_EJ%_I zdh{u51X4}}J93Vu4U)9Z@HP(9Fe04JnE;UTc2TH62uws9%91 zS(3^(1s6=}bNV8FZXNb9xEyZ-qH4cP1L!lc_#b_FJo$!UylQ2FQ357UE6{@?r_YcR zgY`zBt3ToZlqtw~q__)nf+RI$UI=F5c6bxb0~;TPGM_$ST`s_Y6?aMZuuN=Guxf`Q zpG3(?B}PNN+9A^JdZZG<8pk!G9}0S@Iu&R{46fVOPhYxy1@ZvIc0%98 z6-OgRsJ5tzU-d*(Bh|(raHR@89*$pCNA#EMM#J*(q};@Dbi zbWkAy2Vj7z#@qLHKLo_YG)Zks?gxxAu*hO9#D5i97oFiQ$l+AK^)l(ZLMU zT}{f(WBhY|G-yBe7<4<uNuoT0hA9%1vvL~3 zq6-UU4B`1x1{rM>?yO`jlpnExDwxBtNEnB|lCkAip(bhhf|;iFvA%n{Uu-le4)ORb zVcPI)Kz|FFG(q+c!ayszFCeMGAJm&6G%6@4AR;bXw$BV-y!G_+gGq!Wk?$EDSkPYx zLU)5FQi|RL2rq<4vgO%VTTd|qOaa;N8v|;b9HfWP-%jkiVt*u0K!B~ z1y+{5W8G!ho)knUfw@KSl5<3A)kgF;m`1~qBART5ut%ivAq?!q12VvpzKn(f@Nn5O zc0fYRhK!ZrG(w(D)I_$H=O6I9vN(f3 zWAEKvrzW+{;{DWduizvwVi?Gwe9=w8?42MR=9Bl&#p(*`>y60w90_LypyPq2)t+_| zhatwMH31FMZrmxQZwK_(Tw3l2YML5iYkNM^w&&bFuMJ)JE0?gsA{jdvRm)9`YlAUi&*rIot{tSa;h` zTtV;p+Di+3hd~T5`X)VUk1qs0EpNV&;`?n`-_?$_IEYxe701~&MTQ2XndC_=)ILzg z-}3mW_C5fEbadfdh65ruNscgztv9NEbahd?gE(>!r!1Lj%;>A}2%9eo&uP-VAD7a; z3+5h z^qlJ95q+p7B+x)Hck{V@tB-$>XQaI;-d)|&Rg+hspJdim3*cmqn?TX*=gck@5P~t=8R>rCS9Zj&cnq3i8_c@| z+jLliSRsZDJANwKbA*TMOTT;{@F8)wlX25)#5H7gk2l{%giiuIDg&F=!|cog83V{g z^|blnD#tEeSN*IQxYs_=&VBMU5@Y^ z{+OyCwhZTLWkq}UGV}SH?uOg(FEf-r*XXRz<0~Fes(R_cr0jW+TsjU?KM=|*42q{M zl$I*VB0G0Hmv5QPL<eg=8PVbyv|$+8BQndg2c`f=>JtpddYksZzlF6RiU@pY65(v z+T<1cSJ=La+VR$p_tFU0Gc5B^0?;{`2tn-`krz}VC9P8}ZQxEO?W7pvz2=xh?|1V$ z=UQtDpjf5MP3VI&0At40cXm+ABee~wxrPh~kmy3a+^!;8pTrq(<$9Y{(sKUit0g7t zpM9taiGF>n%ZC{cG3|?u2NHDv)2&EKd#C3|r<&}xw~f1Jkj4^^sETZ%fvV2ZR%YEf zzj-`CJ9Q;|#RH>q?RKsERrdHP<43=`ifM^}~$ z^wEe56_u2P;z!n%)BQ+|YokUtX{SUh2QuV-*$WiAN2)aSnSQTMs(H5sG`sCLBz)T? zZ;+7_Owu<~SQD*!^El#(aFpZ>Xfxin?IcC-U?q$9O`E%R8&+5VKW(W9jWl58Kz7$4 zCtCs@XNDtXjlvEMa~@3)C@3&ul;HpZ592$|A9Acg4UkFw4FD^Bm}`VxD#fkGzU3<< z6q-ofTfPDhf-UP#c=JT#uL-2s>^ksS>V^sxb%-58zQq0gS1%6RB-B1WGv&(4MPWI4 z7aR&w@{YVbqhUkpX_%5-ft-cxG*E7#NHHA+1Ak^aFQ1oankz`Kq-h+J_7ki?dl?fnxE0OBDK*_BHuaEw#IHXEWgZ(a~F(I2dvTqJ_fZ1Oe0x>dMOtG%5Y)Yg? zSuYNfkOv+#bn0TkCh7>Uow5}iBS?Qz;9(PiE-+O*LA_wnRbp*9Q9%A504k#m| zpOqJcNp&@~YDCoGv&B#_p^iNW;Yl0~7kvz)N8h~@vB6YK2L$V?m<5!+qTsZ>EJ85| zyeC$t9vSD2JE_4h&{Vi1)%wmL_I8nu~v7%Myni^Qb!=Et;n@98>&3z53hwPf=n4<=-OK3 zOujKN*45^nz8K;UuyOG)yU+IpO~!4J>%epmh|F^9sho! z)}%Q^LZ(A*Ejc-|a7U-=JGN;xpuSM8;{fJQi|yM4{1IG{HtH*e35l)5uf<_KYCF`S z!9O724mv1kQYOK_f!G76XnL12${{#Y|%)hD|dEwCJ;nTZZ;H?dgTzPG+&hGP}C z4h?n=#S5~M5)LUAzO@#?2;@IIrca-q-qRf9^lZW5549UJ zdY>LlqO2P&gAT`ACP7F{7(M*HChWKuRPk#$u!(N3x`a+9CL(r~$ig zGemKOorSbOc=-L2Xa816dT9Hg#$#Z~7*@3Z{ADNu)im|Z!e-I*ZPd~3wg^PG2Aacg zy5DwKd*plkhy_iVOeP-gQ|Qr-yKY+l?fyMG=e3cDRrLaxw^E{__^f3NbtGmk1T&}< zm{X^{$FDE%U4AUU)EPHr&gB*{_`N2L3QYo%=F%)~lh(w8n=W|?a<8a_?7^Cdl81JU z{0s`T;2pW=UDuI-RRm`9vqqD%H&995#IyYbOX8Q;C z>=*(LhsM5&LMHm@$D6HRFbt&IW>>}xpLymcJq~q#RzFUjbn82PRXN%s@$U7!Th|e` zx%#)~{F-E|k|elVDl3u}6t+D~0VxJEf4>?`CTN$!vWpv`ALP^)*)Vt&d{&BmMg=_j zfU)t{aT@2FVv8mUJ`Rael*2~V^`B`f6#x8jJk-5qsisBdN$B%XJ0V@{^e;r5%WzfTY|VDbVD^45#U%1nu0;J$VnG zD%JPdoZ9Qpnf`Sbs0bX+{p-})Y8(rk)e8fzm6~uRxx+<44NC5FDy6M8&cH~ z14M(t3v9v=FZAjAHz-L|iXaSazLpC?nPp?1_x<^&KQW7#<4&Fq(7(qANd-FF1(N=3 z>z2hw^nX&pmRN3SH6X1RhCq6>pF7i%*FtWXJS3$BZoZz-{~Q#>CK9{%yR zpC5C=^d>LF$K+Fuy}~K}2Z;euN2Een{k}j|J#1Z{ zWk9y$;YyYbbm&rZKqOWMy~EKo4;1wWn#G+zsGr+sx9MQYTvpWptvBxSw}L)Vg_^*P z7KzrfOi7;YtatcvH_K@?oV@xX{iTI}-hQTh;GLW~r~l@8`_KXpf|qmB->m1?bcu>& z0(Q|l!mbZY@*1X1X!1ea%lcix?N4HM%3HRer2AYgiW#_6&Siw5?IkzCs#1A(l4&9D zoR1xO0PGinxD^PPOw> zKC+?_q4b>7i_bKcec6-k@raT$h8lQ0gdWY9i9_Ms{KeZ&$(q$u?w|_sh+#$)*bFv8)u1QpFvNhpIEyM-$~we#^zI z$HtTl=T{;!Qg{PYY&nCh!XQZ7^XJ`UX0^bDAN06|55U`6fjJkA^N5_@98zFG>wW)T z*#^4Yuqo?EV*wJzm1`?~v$K^l^7gA~`K;97M6xmgRkWqV-J&2;4ABC4@-pFFFOBsc zz2n3%A85TDvH||1YHFW7KD(b^$>+BkVF* zmg@DZ%FC|RMcaMTd-K|4ZQ2i**PN%7i1@AzJdffEL)1Y4jm~DM(#3N4X{9;eRowBs z?pX&+DQ1nuP7AjbAna*1o`XH6;0Z^tSA4Qy5vS(og%5OJ&yoKd_npk90dNZ;=_rd( zyHCg___7(UKR+9=Z~OI!ovQw(0zQ3?(G623G@ZskV6eXe`cNY59r=2Ty1XF|UkQ>L zZGG&`U-2R=!@lXw#Gv>&&mVu5p-v?+z)P%NrRX{p5gvZmrK=f=a|gTJVa8EZgLFtV zs3yvha?^3j_`>DUnQ*AxPg(nbY*dSM2&vnlgU|2J&KI#-_HnxVjXhPS1pa0cICwzLgtf@YnP6RwWR2VUB^8DN4vjopMAO?^cVi6ldM?Zhx!qq=VUZg6He$0JAN9UW7<=sitEwB0p*8$0EM6;D>4{%tx33{#OB1M z5W)o|!wsPT_@om#jx40G1t;SnHMqQFO_|UD%bmOl+m79Tim~vM>nN~U^?+Kp?1syb zTp@vByU{IG7;^+aZy)NN!XI=HWqd%;iP<_Et z@#iN~cZ};5RV5C8ezWB0Fax3f+2f}xIB_%~7N@1eoA}{AVP zw8{{EHeyNj46Ux{E|Rk-83WCpjTWcPV7#ee00%Y%u~_n8veWO8+4tl z*4LvW9MbYlWaTMfptonmx^-RApQozo9k^uuRJ>sAQ;CAn!~dW6&5!%G?3K;W0-b%3 zL}2{(T6wclDN!KHcq%^=;)oQt0zCray}UT4@=59ZuLShVO;7DAwtNZ;t6{S$QqbQ~ z35h0z(836Y|LDg$*>)KApzmeQVq z&vgd|yJr7TVET3+-Tri_Sj_6Kbex|5{xUW7;9zXyi6WmDi%+OJI=ljYqbwy0j9$gA$ZeXvh%2F$2<$^8dmZ* zoyo5^YA#g`+@K(LysLg!ja9F$W3tvZ1xpt+`TbBi`{4{c_N>8POR}aC@#aucL$cD1 z?O~R#F_Sz@+W=ctOkSNC2KNth@Q`Q=+74P8w2uE%*Devw&yHB_xX@R~yAl8s5UYdi z;vmrAXW+6C>UmKXLv|$aJid;zpitxP@*79w$n%35BVk7*8&=xYv6X0sk8m937eOmc z){;H8kVf+OGM1$pM0>@6zWwa|q31Rwk0fefd>&uAq*(X?cyWX>b>nbmS&GBzZoOrt zFb=Qh)u!lhOc^xoxY%eF0WAkWTHFJVe3g; zZ??%-2qT<`If&_Ak%tLVkR|RTTbnU#mp;J8=KP6gmJN>B+o1rlU$zCrOE~9}<7^b3 zn>^n;P?Vz3$#q?Q7yPXOdQ}2&>FQ92r8^?jhdC|gaIeP$YaiA3y`xqe#Q0U>#&k5v zrWyA*^jx&^;q@1nwfirpnu;NaHsBkyrc3;sIv6^5{7OS0LnRLf8YJ1FL#j~j8SDJ> zJ233WWzuXnLxI`f-%nlMd7Ok)Q&(!jY=A%ug^|P;ud)hmIfaYIc6Td%I8lnYA+SX52l$uIs{O_5 z&?jRRf9{^WH!pkxR|RKS;JzoKloJyJ7?oLvD`QzzVp((3#A+yhuYk&o_XnKkezagl zfhZzTfPRh=8?Gr+S4Q*{*??H$_TAwMLRk6ag;F{j9InJ_jM4``kN3<8v6Qbv9d$?D z{F*|3e(fyUe~`%~clq)*0W(2C)iGnZpIUIwQxpUm! zaDlV(vufP+XWbAKy2+a>=RGD**7=HW!QRGWC$~a5n6G@Wx##u^Vc(Nm*E*DZxZ(CT zVfN~@&wT|SHh$Iq==Lanm_oIKd5UBF^zv4lP3Tr)VQKk9cbMMDp>licVwIgcFMjLl zB9aac)$Y%Adr-?FbZy?ec^9Hti55p#?WUFv4N@31*{4ynu6Al#3^gl1xxAO#dUVYv z?{YGG_8j`@X^B2Vy@8%Rv!9%2-N-PZP4B*by!~{$i$D*5gic)VU%O)D&K67Lx-?%i zMN=wbXya%|O*{N*=SJF9XsLVP+s-^1P6mn^WF3+QJI~ELW~ZgkyO>kXxp;9g;i>>W zQg(VRWnjp3?l~5&O1=B=p^l4-%idKizjLqnXuH}CVhg2K*zUZTDJItT^-*VjWwvK2 zudScORx61Xt`-n1deNuRy{3R#Q9v^zHR|x~l@-Sx%1~pV%I}6~39g z)LPI!qt5!Xn3!1W?Cf8svatD`NfBpF>U7CHoo~17>rC8tKjnI(x4y*Ly$d8?|B8-@ zFN2>_bbr4ao%kNvMpUs!p#`+1@cMzzhqqUa!DJkwTRx~JV=+$vITPO!t- zR{(PT{ry=idZ=WYhK`O#>rd*SK-{z;t)qjiuRiN!t<_Loes9M3)8Z{$3QVA|-_gF2 znq0o3wvY@$l+jQ~YU}yoBr?X*VOeqFKI>wU_sYB{x1Ny;=D{&hiOx(jKkKnRQgnf3 zjKBD|Bm1qbrljXO3!Z!;vCwLGQ>nQRBds4M1_q^nPpeZ?dse(A+< zo$(j{^LlyjM9qMjny@vd4U2du3>#)o*xkNdBPxXJS&0Sj$+KL;nFv>khn&C9Lfds` z9PGFR*&kajnVZPfki&6u_%5Hk+r97UfS8zd^Pz`p>41?81B&#R64 z{jHCSDvQN9H~e{f(hoUat%gM!>`~}u)JEM>9d9yuF$;p=*jxGbjI|u^$@7uJ4r^TF zZWJhrfotOiLRZ?I?cgDX^+`WaU%ZX|3CE5Od4%X>Nr^tF^T^1o95V2IqKNl*4Ev#+ zJQjDjGc2)UcE1$v#L4sUMW+8w9$O_KU?z3o?@`_EZ7cJtt?q*tm-4en@ZFo~!@D-u zxJoS{yIxIgp}3njUrbsMmh^f=w>U?~ml^)>UCNBN`<~s>m})YeJ#^zlof@mCZp-u|L^_v13C2V3kbIxoCCuh@}er3^Z|a>{$8pn9q_?thKgCIrfq3 zSP$E-p5vcgmKsbk;S?H~Ay-o4Rhat|Ye!eb#>w%PGAP#jjH|5DT(#n>piCDCqYxwpUdsUh?aNKM#-p9J1 z95aoFy<|8RD<1tS&tN9Rt0){Snvfo-r!Q<-a^vNS$!kutr)PCBSDDVyA5x9(@AMOg zEe$ww#^?+)kY%sFcKuP_laM9IsA+U=KU)=3BH%KAo?~KNyl}?m^s$f2M`ogE&Zh3} zn)26k=G6Q9Z99A7^2o@zwl7b$x^lGB&}DVDe!=qz`aicPF3TNPJbXEeE?jr+)Or#+ z$Uk&*>z1lV9-e-JaGomb{hqonTeTYQ zg-p7+g#Bhc*3^E(UMaU&1cs#ir(FL9l$y&COZNx;Jll*PDU` zGH;E^$zYX8)GWf}y?Iii0KRO4wR0jmw;^?2;f}iF=}c34SY9r6_tsn}Yud`opuIj0 z)V*;Y)^8rJIh1hko+`qm8^l*2elk#kmkSQRnf|b|+toiPNGP>74yiar*W~912!iiw zlIM^IMr2WRbxri$yQPQ$$-c@G8{07n+eh62(A0yg8;eAoYOe+P`7cR+)wPv1Yp#wO zKmW;!I}_~S%nUGp`Y#;3f(%OJctrH6L}ZSprzCKGd!%tHOu`tzJp=g3-j+_R#J^i7)Q{{j~;mdE6j(O)Elr`4U_f0zM5~n*0TYgeJRQ43I#u9KfGe zXX76jXbI8fu|9x#)z#Jb^(O1OojZ40SXfNDU}49Sp7jgUO*@$HK+fz-Ow2TN*63Ed) zQ&SU7qX8yO?7fcY20t>G(b3VzM+FD;t@>HKKJvi>Eg~DiiSR?WYi$S+r_Zk{(E&@D zLb?~-Wo2r}?rzjh;}DGk&Z3YpCg;$1iS*IEymf!M`y5h=O{fyHnBU;${v|9RPIu-@ zJp@fldIyYSMHaFYc*p9e0SN-@u1tQmK{!>*5Vh@&{q%{zf?Kz4z(4}|Y z%g*upoO-w-Q()t4;9Py67pN3*N^#38%FD;ZJ&lTwC$12+ZMpqIJzVdGp;KFM<*sC` zF)bvtGtdVisT%)kLSB8F*I_3aVlNqD9gp^GgG||?+pOFFk@KU2YK4azH zSMWB;wA3jyZ^l={+o}7!S8{HUt^k3MqPY-Yv^0rf&2jl)Sft4MB*&5$g zZc^gEp>nDe7uy1cPz9_Oi1kk^W$4$Xp%tKSXM9dq0#T>T{bya|ty@b5T6cM>xwxcJ zPvkMARc%Ze9 z!Vm-LT;2Qo^XGOFK8O7Sn{8x>!%O+u=Abs1x`j~7bkLhNH3lC)J_@G(!FI!8irS{d zIAFPMiDZwQU+eBas8PwIz+?E7;~1R`Yg)_0JcmA~_d5%+w!5M`@1IO-`r#U(w0f%k zzD?Y6^6ZE}tiNfseQn1U)`to{7zuq2@|@Ya!RV0HlWPdE*vI)g?(*x1U70AEJ6UZlznvBM4l9v_LjfVd{C@s zV7KsuL@##0Mx!3Dt}fR2{!CNqp!>i!P5-wqo?osoIcH-hx=L##x}v|*${z9@9h}{@ zJ&G@P9D5-6+QMe*`PIK}sZKT-&kleN+KKP)PPu568SH(6>(EGlvAuY;9PK-HhHl)Y z6}rnVKVPx0r-Zx2MSQ|vf8Hp`9e6oR`H}+N%nSqfhF=qWN=vt}9>~dSUH1$9)>vvK z7s_5h0MB}nW9G;ObBO*mGANFj??!$(<(sBMQl>|CW^Zj{USDm8{kM~Xjx$ESY+SyK zo6ey6(`=QU6?x7_%iE5nvBR6u<-jWNytY(j?v5Utc#J(s%2yc(#%vUS@U>?k^4iqk z1tXud9abXy1UMg$MD!e^b$B@udc?9rAFsUkW*IZiAZuV>=s3h*k1w)6HhFf#p?hoO zwhYfNufK$OtuBZzWe0S~i5`wbxyBp_&j0-2T~~}L%+0=@cgFip&S(E%c6gJfpT!od zk-L+_fzLej`!hREQ>r!L&j4*~lR^p(Ur)z+7aJ8ags;wFf9XAO-gys9FP-kFp82ry@SBG#w8kW1 zWwR#IIQW||#eVHet?%$r6FWP0Bw?A8+i+;i#R0xueu7(se_t$e9_&YBe*XL!z-LVtk1k#}aJwT=~k2gdYybmj1zrG7xpu6>z0)xSrC*zSB9TRgREB^j{wY0P} zv?n9!B=vO_+qiN33tiN|K?PqN`6`NfouHs?o#n}3oD<3VWrC_J;lCskx_BgrM(d^X zvXpE6_YGq5QCzvBUY!&t(=Rf3Mj$)R09}sD$>f7N76U`3(x)TXP-8wHePoEZxS+$? z5`{N9AgKc*Xm#V8KY_`IQ4y_KP*9L$q(t5z)uC}Qnma_`m_}KMXq8kv!f8hB${OpX zXfQd!daKr&*;rjrTYPrxo!2TX>Icqty?e&TJbDU^#>FoL4-I_fa>VXYEKg~Qov8+j zW401}uN1f#5z!pTa~3##V!)JH-5y|Y^qo?W_lL-)8nT5l)XMl7gsBqk3N}bW%a}5N4aSHpU66%V4nZI$a+=ngEuy*1%-v^DP5l9 z4u$BaPgcNvLLPjELzD#yT-Nb{0RhOL5}(M7&rt@goUdaB$CH;W?0D{i;}Q`SR4H;^ zTPB)S-#;+DZ*Xed1BbugAVo_;8(-_XbWKbEbBBcJXhle>fZlGYdiClRqG|4{4i;Xo z?6aHRUTZF|tZYLrL=vq`L)#1o=gzn8H8qfW{Y#)*{H^#e2iz(0b0Gi~?c~h@N6$Gf zt#DYD741%+80s;Q@^;gKPf2=C=)%1HQ{TePuIo+eCqHd3TU<<{mC%7trevLhZ11*# z${C3e6^W4I4OJb^n>-ndn*UrLMeEz`lY5mvCh2jTrWBYY$$#j3Rht;0w9rc5&@i5g zC+J#7)ko+jNvjUYQWgD(wsN?&NKKAs#iyrJ*FR?Xg$oxp?256N8)~gG95(wQe+zn1 z`8hbCie@WESjt1K#pkOm{6ifqN*M-{xjk=xW<`Qx6oX0rrJW!>5EK^tdIT>vXahzW zAq4!tiH*G~t?d4uZ|tTO({FRwFZx)uQ)`C?0P6C~i?<$+@Ba95D>SMT^|Qh5)zHug zN9)pE$kALsTv0%1xgPcijIx~@>sTaUAg_d?B5R5O17)R*jLf}?RtR>fmt;3~hjtVR zDoDsniA+R9#Iepj_{7XiDFdep&GhE$mfvBJ5R;W9{Uv!g`qN9Q?kLOmJ(gHv;hzx2 zK*Q5hI3-)h@Nd72A2Boyk5h49INzf zBbodO&z7o0Z!x_pl8cSdy;NLU+7Mx{;L4Q}9aZ{{FXdsJ)->1XCp9I6g>|ubWLTIi zcndB1Laax6G)cGihDk%CGOwdrp!Ce)oMfG)ZLP*S(}pl#_K|w_g^9UC+%zlIPe*I&jwaBdD0ymYTlO7mB=&JnW;@+;)@yKwHdG!|^7g|M}YLNluG4 zuu-VX&eJyd9N|98%|3&|A5E*1&X#HT$KP;y1C%_mQOK@Tm;G0Fy90m*{`oG81r6&x zx-~W0yn5Jy(IQI+hth|Gw=dn&m{{b)4y;lJ-_?C;T-^9E_@UWs);*OK&WO%3T%Ogh zh7fc^=&sG}ZMO_&DF03aHamX0>a%(e&zxu7oeT3Uj1T@k@OUyd|2qC`W{8}zD|=ke zS)l=@uHdoi-vdGX!bV5OSv(5CCx@_4<6|{@&Ncq*yR9*`=)y?+Z5%cgrFDDTGl;*p+ViJ+2LwUleRZ zD+FaO%~e>=zR`<<@@B06p%=$%b3tWq2M_Dg$IT16dN;PI@IaR<;swWiZKLT8Ve0G& zpz3^yPx7I;uVVoZ-Rp7eXQ<@fZ?{rJ1>rbPhCyX`xLM$snIi~Uj_liX$pt+X3CH6j z9{svywfX4ws3ljq$IoXaZ5&rQMv@>=FTylG^!3T?a%vOP`tVMleY3o_UXpgLIZDuD z|6sqi(Zy{NXa8c~-H&yN`%;{~827|#;h%xFVf-FAS(*io{qy}P3`Vrd&Z!r6@h{ts zz-jKtR~s)1Dw)0hhi=abpW?kEKFKUAuQ?lZAZ}~YemIhUF^_Y*19F%geiC({_yTof6=I<4pi-3DJdpZBd5Br;tD@s|IGg` zN2JF$eH(5Wux=zrqVV4qnnMr8tG}nlX`YLFp#4|N@1_$z);` z?KSY&R6%4l1cwCY;B*0M*xR^pXwJj>e+^O%Z~vErR0{+fFY{&_#X_J@L`tG1F79Oc zDR6DF>d^1%9hjaly1A-pYcNP*tsstS1L~nBCghz#E)S4zEoPvJ?Z1YsS#R6B1vhTg z28K%Jx3C<5;(WKTm_W<8oPJj1fj4UO1LVpI7d6Yb!}e-`5{^+>kx4W zR%L#^zPAxb$bUY=%KJnY?Aat_lezmY4J}o@&b2*mOQX?1o3?9{zcyKC5ODMVC}b5l zF2kzodR-G2<}sMPaCOWe&qp1B>T1PPDj&ZsE{@7HOkg<>_qqZu17KU&_;tmmO`Akb z-{l-5k_ZB2iW{i5cwQ)8)z#J2SVcv}hg2uf9e#vUr-rVsZmDH*(KRWT`~3MG&M}#p z`VfgveUqveS5RCGPm^l%c54d@bm&tz*8dYYFCGv^!8Rhb)h>7KFvR-VOIjgZM}T{o zz*wX2u1#O(W%HdG^jQV`92lZVU}415r`vIow~~1b3b~rQcQ1z8#m~<#tOo|VfWesj z2vf(76<_sp&*uDo&duC9R^4q%~Y@*ITk#R@C2fn^$-}az(1U1NKb|A|uqv8xA-`3y zAcz}+;_4(;CA{Q|e6=$2RdsWEolVQ?iMdk$_)gi-r2#Jy3KzA}Jz8eMMOsra#q7u$Tf#ca62wn%PyXj)6ZGucG+D2jeHlCJlcX#)( zNX=nbW=>S$8F3Pn?egqZr=kJj#nj=%wcCO!0>5#X>Xai4SWm zX4cL-FV1t5b`fw9sH;kf>`khexS$;S8B{+vzIwniK~Yf=);0Du60;2)uV}^VyF9H* zgY79qa1YiMLu3LK-`{gvSZ70NO~BLbAz%7NH_nV(D8U&#TK&dwiP7uYYvF*f#Gih$ zy8{RR{LDY7D$foE;wBOIl#%_PX6a zXqA@uK9_6B~np>Ue3qA!};mWzj+B~xgP(9?z|=} zE!tvQx}0^%+Zak$9Zc1jGh$7(x0jgX@ zNBEB@&`}36kS8N9p{?qLu0T|vp;{C~ZmO+scM1!A?oGvG#qB#nS3|%LdEFz>uH(?r zR0y6>u|pXg>BoHrUEZ32pLt(LnCvevuHu?-!smy}Fr&)}>cYZS$F}>0B8@{CVTJ8x zq1pZ43e)jFb*O^DI$Fb;8$m6Q zSU`aD_8$qEFikF46ah^t!3hlwCIk-w22@7~0A+Fc=YG`7Tj5^Bs=r~6YK6B}6*lI0 z6lB*w_8;96f7?CFW5AVaV!f|I2g6Z5rfyuY%qYWEE_h#X%%SR~w5=V=dPFCurcFyV zR+hRf=x{(ao_fE+35bL=L~w>%(8tGY~Uh{fDYllE2~h zEq{VGXK+SuuPdtj$_N=+*5f?-A4GUl<7d6kS^D&e8DP=$-nJMqoHGn&NsBtpb2^|1 z_7h-+8fq$V_s*T>;6bH1T&oszbm8so^@APt#P9?rwj*$|qoP7zCeR2Xei2+$ zs^hPMV!)A!jaWk0WkWbqDv_#NW6Jn&Pc|a#b zc7^1tmblHhtn_!sC)f1n{pH9V(%OX++5#4Mh_cSgo|65Dzaf_r0;Vv&Jwn>?#aDuVlzRpBUL6e)d(oc@ukv&-HAF#@UsTeY^m0V?a{? z7ksgz|9gD^HL|%9MOpbkKkudhI^>z*0$W;UM{Ffr*MK54x;O=<9zJ|Oy zNa1qBwQc1Cucdcljj|0(HvbHUk8mVOou+yIIstJhaW}SStmn9#s~CKbv2&+CwCPSx zPNm($fn9Q+7{JB_*QagUW7?aUNBuX*wu10$u(B=oUz6Q|2?nl01vU&vEX8s0J#Nwr zHQ<-DCX__da$ny%#@SXB+Jg6Gp$c!cVb8l7VuomH3Z&MqI`d?&*$Y_(r#mH{U%idY z0Q`~oSb#wrPg{F?_q!9*+M*(FFe?f?N1y2l`Rrh3Z@)I?kg}J|smHm1%W;)k&V=DR zaZM^BfiZOnc4_)sx&+KYQH~5@4O^c{AsgZBn1ufE>21+F{3bc-DfGD^ji5$U-eq@k8H}P_!)LhsPZPJu}gsk3`dpL@t zZ@a(ML$o&pVurVBifoHxa*2?T&;X#;@3Uvk`t|cub)yly=w#4YWvI|-aIK#0U*a*? zjqxR*Kin83dcv2^+SHP4lp}@&0`(h)US9H;#rk3cgAh;D+y}8;_6YQtzPR304`Th0 z=U|H`j?{V?xOdTYjPCl`QQNXhS^2xnkJtMKAtdWjQdI2U5cDD&m435FV;wnaqaIWH zng1RUoy0p&Q&)Gc-zOsgr>=;{NsvJv=;>%l-;I@R0gIrQu9I_gosvt8(4rL7kvf-_ zkSz&-#bZgZ&Wsq{5_{X7^XhJeJ}HRnz?sn6yZ0vP1Rz3CmFcT4A(ymm?W;IUD_%;6 z!c+;xDr%Dn)wmwm@D=dEiKZ-=^)>F_@|U9^q->1#Bm4|PdSPhpNQ!GMt#O}CQN&M9 z)_@}HI=b4R8%`v=nRiM^NK8*yu)y?dpYH5o(}B{8uc6cb<=`F;Hvi)+|F6CztDFR} zE!;u{7~?s4FXx%;Mi)&D%TXL0AE!{fX?&HQO90xbusW(}YiR}WdG_0hZP-9dnuFuj zIac*|(F6CPzWOTADHem$gBlNZ-+YtAk48h?Mz;;-p~@D8J%)p0PC=`qAlwc>1ENFR zKO`g$N0%bgO*-^oac~s!3ITq8Gjj2GV5M7H)%iMctIV&zr}+g3$C5!mS`AboC=g&! zjSiv))FYxHV`+sSWGT2eO9%OIcb2fK@7s#VfCa~M{`*5anDpVqA-*o-nMEEqj~~ko zd{{2?5{w#t2J}%WS$%b#7+c$kY0|31pMlUCwL~Q*HbB3?z)0GKLG-5S<3zmc_;?YE zP9O_8D_Ft%X=*FWWHQUK%ig)O3}`>uXClFL1rdiVHVcUil+_()YYs;x>Q&_QWa`Gd zAf!+(#uQLOiY&kgG;r2^mjW*G{O^xwe?`+Id06_DPV@M3;`t*6ZsG)1J^3`17OAmr zMOBq0_CTEf$L0#ZI|QAQ;PBt<#4}x_Cr&Nvww2>d*;;M2 zft<2j(au*>N9Qi}yM)&XnpMPjbOqI$fxgHL2c4$^H9C;H6A2)r9tyqC-y~oM+rboh z^j&QLHU}&(0}!iPDKBqtqekmVCItVU1+iWnbfAx?W*e50n^~%@(F-`K;^dnD4Bq@bUJg?Oh$X zpgB()e)bK#6v1tx|4g5baEShuoif5A_$OiI|M;a}L&{F6PxVCPuhRE2BW!<5%l?kl1A8Y!TO3Bw(B9g@%HG2C{&`27t(~cr!vzF1e!(b>3(J#qIi6m1DCaq8Q<~23v$f+JD zXUgF}Rr7r_51d*~>5TDzv=3m^3ph=B>z|>+A7c-{|0pf`{{85!uTQXtrB%M}`z^$- zB}tO}id5~uxXiJ0*A*|~f_hwbzf9HOuM04B5@tt>Y?&A9FXyZ(L@VIkV(#-Dt@sB9 z0YBt#Iy*@AJ`x<6?(Kadry-*ze)7I9wGYi4gNY3Y^xnG;YyAKIGD=#OAlW9d)krtD z-VztSFa;6kx!9)r>@pR;6q->Y4({g;l2=nmY@H9PNnr3?yKsBiz-5?<|KU$buFzEL z{dGfAojK&>sZ`A;WHQlj_IB^ALGTa!1?~s(WHsCRP(jDei6s1cbOF&JiZhR?PI^<)svLYbxu3p zx_R@X*Y1u^zGYPAjTw&(nwQGCe7sslr9PwWy&r0a~SXU{HD!JZRvM4+1S`NmOBjit$Ibbw-%fJ zlz9|yZEt&=>!n&B4mD^kcCwOf_E_ynl6rG?e5fWUT+k|7#9<;qu+RMth3K4j(}3G* zkJIbVtNUO1!&tFmVtP*xQLJFSHmCD*Z3d-sOBV$1%Xf;})3oC)dy0m2oM6`BLN@P<9H(_AT+P$9bEMi*H9MBZTXG2b<{e5+;+tvy zj2wDsjxSHKroc9u{d#}vaa4XmenU^OlN*!fk*E2qfBSZK@}pfRlCQsEE4MXgT^w!d zSefn`e{(Qajge$Ofmg7{J_z>e()fM+`}>uz!ufEoPIK$0U3+?PU8rd5s@G1*!Q*rv zy}dEp3w7MD{QT@^-KMgtD5~^z zfry5VILf<%hxrA`(rN9GHZ3ch1cuxBqqfLT&wgvdnqWAF;C}SlzS@GusH^h$mD#>om7NX)FVnwY945`dPbtI$?6VFj`$Ybz8ojbwpN*?)EwnQsB_(BJ zV!8>N{Oj$xtD+M}8N_wJe*OBV+^gJmbtXL^;GbjXL|RK{o1HGff<35z7x&{0ZCziV zK`@7A`}S&ICTy5n>*k=p1X}V}R2=N*&r3Q?wzrk8_PTX$;(7DGz&~5x-#W6sxtXdG zFSP;Nl+*g=&6~D(fAN%Xe)C$({8Z2F4YaY+nNc8&wZ?p=6RpYNyhc8Z7cb)SyK-TF z=!~=qZ4;3s&&^qr9y&A`Zl3Y0<+jAM3M)UqIuf>tj!f?2o(y>6#7ov_WivCgkVTKJ z`KqFsqDe|>>W{X=wc{I;nH?}3(TS4f_Pb`?1zA&$#VP8kA9#&^U8dE%%vi>cQ$8pjjE3OPlEA!_4aK$o>{(on3TWd zV$!-QR}=@+pvxlc)&yLp33iO~?Aes!*|Mv3cIU6&&9`iu>#qR$7ekvR7tCG+o0zlo zvDUeFx-0)Svi#1@1>e4Xvj@k3Wn-&}w=^?L`SJD~x|S$Ln=5U<;-$H1>%cR_3OxEF zg9pP%zbY05W~jLF3F(p66m<>|$B&@OMT?DBxvw5wv3+#8qDf*mZ8D=+X=i6AMKd#6 zd~2?PbF|jHxK_zhVt;1?Nshy$hQoOCEm*4*m3SY0=YG*Z})JRm(|F!)2$zo{4wqeFE^dOI`w&kj!qu%{&ro54m-%VbC^>zBeK|{ z>g~CS-&yf6(2S&LS2?gM8(@jsG7Tg=<}1%k^p%$~baw)j zR;R={>n*)l3Q`ksasAoS)#!s1^l5WylJ&#~M`T>^?R;(Bf?Q@_AiPG?-T8MrC7 z+HE)SGwvyiVmMPo!-qc4O-1weG?h&KQYO8vB%^`vFDHJ*`_eoAC7JFmy=zszohM-N z`)^qiO=FBiCaqP8N`8-H#KB`{6P_PAGXWq#t>uHFLyMxIEFK>e)K==|bWYq|%U^Wi z1+o*!dBf9dTDF_sd40syg}9`|1O>6)o&fo^XFXABdL~^bw;6mOXy98qw?DdG+sm#) zfznH0cqYe_Vl-eD%r)EEMMCxWVPT7QcQ*NLhh+5Z8u;;C#}ko7f;n?YNJw}rHVC@T zeLXa6TX^L`J@xARDHa8y9q27}OLdy<{qxy0$aQ&w*|(zq1elw6L+XjH{LHf59k(~s zyw@-3eEzvGTnCaabndF7LQB$Z%|yB2rC-tv~)(J1Gx3DmcTY60*)(2d}qz*7!BXeb0YRXVb2$? zOr;xE_=I0E#j_~9iv{mccxns23t(E!$xw_C2p4t9zJLGzQq1m_boti8>m5)t3pcE+ zi+RN6%5)20EdKl744GJB#v24%KO9kJdfc6)Jve$L;QjhcCn+$tr|JCc)Enx3@DoCF`eah^zT711osALXQumi0hsQKx-ICD(x z`Tde%JWeyZH}F|zZR~>}A3s3n{XpW*0^;R^H`@$VpK$7#=_#g#Y4fY9s&?bJLhHsr zu5H@XH4E!tdG5*Zm2EB5ab}L%79OqO@mpH5zsS$uX4JR7IQphGn1dFkmnmJyG}vhp z({R$x-hK&}$t^0{{Se@Lh6BCJ&Fgu!De&`==aO^oM8O}slmPk207DqfiW z%1n>>yM_nxfaHBz#+N=u{OSdSiTiCK5U-i3Pw()cG163qZ=^nKkZZCz@%4q8wYdt4 z9$a#AvJ@=2AIbiM1$BUC3}HUITXs&Juor$C%agQtKQP1M9d2{8Ub|fZ#!O61LpqrN z{-nUR*>;1z&eDPpl2TR08&db?z6PX2gZBdw;_Prea+aH$wfHa@{r+S)lUmi3*@y2GSd731 z&SZ1x6;Z#T6>MB7f?IQKxD+xqa=dtC$8Vz*sD z(;WfETWf>{JKLMIRP1WjR#v?wodL$8Yk!~8FZ&#&!4LA|@(e2u-oW#M zWcu6sRv6yAd6ODYFE1Fro?$Sg=@)9=@fcpu7;TX6s&_Ai{XwES4~C*nFQmgUIH=q6 zAlG&oa@IZ&l7V*KcIxcRF?j=~$7H=Uq+`vl_)urzl9R4}bd_)*$j;8%Aca#WINPIM zUS4SLX0k!2sF9>JMhXwwY@KuJHhlK{d105i()4$pqwydO^yM6vhrlgMl|0Dl1W)eU z(Ls)R{_=v1Buz`aGzIci{O0YgU<{}zDHAGRpGgJ$jZXl~Xxb3YfA-45fRiUr+K<%J zpkM|<8+GuDAU%^aTsvS3Ke{ZA%+T|O;YudWvb4JGo)+kGMO}4L8Xq4w>nX|w2%~nd zBRC)+HXtBCchY$3wv5c&L^l{hH4BS0aNbj83?_|{ftB}Fy7C_7oKp%8t?@j=sTKR_ z6VLKgXUkv-I0RK|>ra~{G_&L=$dZ$nUu1WbP0>gXuUf?#Iah##O9mHGWHjDgm;-t| zdO{w8ts#4n9Q(@*JohA{UCE8Vm%$Fx6|Zq?eUb!or}p&VF;V@iMrC-v%{d4L>xOMj zzda+<_L(;RoDYjx({DWnwto5JWl-zqK0anOcxL)4gqj<#&e~gbSqSJs2r6Rz*Spwa zyg4yNCpQH|Z;b26(W9*&RpPnawwo&xW4E`qx@QMQIv|2~s5mBAFseKMLC<)}x{mtn zI|C4WSV*;yh5E%W-zzJ}0Z2@Mo*kzXiY+J*lHJ|x6$P=NFXhXQcz8K;+6G>et;LCa zq3GM)j`AJm3ZBmvo!J~kz3;D>wPO8ywNZ4swY9}n{LFd8Ee4N&f7zt5V6?DyhUTj4 z(nJ4gh?FLQQ+Va?AIh{kspon-)Bbv^#Ln6ckjrL>ZpMD?&jN8V`x$#JBu3GS->kK| zC+K?7@1SuA73SBpS5|~Wqas0KmwTN0l07z8=u1WGLb+$@@!>IRnQ9=Sg{FOFr8>E0 z!@U`RUMx$OH7*&H=_)1N@i80&tM>C!em7$UkLZjINYKE*K*rdrr@oh_@mO=>k;SVd zSV!t72MD|X+dgEJZJ2=tGzFR7n6F|GE4n!g@EM+h2m@Q)qLtR4ai=(FHZL+Vmbmtmx~TDl(PG{sKL#yxcbNB?#ihULW5kSC<_Qu44B5Kbl z7TOMf2EXHP_rQt5?yRdab#rs`XWL=9)j67)Vn@R*F$Ouy8`=wuNwb^W+mV%QldvLU_U}@8X8Kh&B zR}BT9hK7cujyv+R@}PB~Wv-pVrtoDFti%Mwiwz5Q;elJVX%HOHchRiwY^^VnA3N3t zCdpVJBBwslA#qwPDMnEi#OSf-&iW{a$Zlc>vtCJF{oqh89E8o`yv6_Uft=jIaH+dDYwghwS><{N9F$B2$*|dcv@xzCacZABc4%GVix-DIAcE?gXUd0IYiSc)sC z%{BGEPnqd6}w}H9(~&@7TAxy}6p|wmO5hN#AXn zACZXy5sLDmTy21^7MGQZ^QW`0-Hu(4bc!9*-Q2+2(g)Rzv?Vr$asWqpKK*XK(%_Fb zcV3^q)ZH`>VE}KJn*rsc-ZKlfdZfQH$j#d_->;Po4ZFBA^Zn_Ha_UV6oQk`fq@D9X z)85m{QM{pF@H z2thT$2~pC}BwxCl_YoGlaOLL z9naT6-F0<B;>lNqosxn6Ls6-A zZ4O6W>XU*)p>Fh}EX9twCQbCD2M-qY@f;>23v3xLnJTm`4vAbDUUnkIP@2?(dfUK; zv^jL>B>|UtR!}J`z#O{5;JnGE8W#F-_9^y(f`v+GNAF{4j zg5p@Pf~z9!U__Ip{pn}?>y3VaR(Yx?tKI_9t>b$j4rD-SD{jO|{PIbTu&Ql?0_(5w zp>LP~cCuZ$qQ+giq?%{<$heHPml{w|q*79}*UmcUW^+qR&u&SSgqL_-aV!nlixSS~ z?ptd%{a1Y|+T8PF(VxiCGk9 zugE}r)dB(*k!Wdao6V(LaDM#0KQep}%i~;3BnEeYEw}-CiJ0RmF}L0ARjM;*LX(1Z zx!QoJJ}2R+d*lq4eJC?C^YHJ=9aWl&e-J{ZF=4SX z_-v-*wn05j&fpq=f|yjvgrxya*LCoXpy2ECq}kI)QVk9GFAv2D7AYtm${f}IOwBHWat%hZRUi<-=ZAqPYJXkH2<97HKf)E&D-B`wE7d=#hBfPK!WvNEX$v~M-ivFbUy;cJYv39 zn;;vs1Vm^GV8#@^;yj2R6Z3mrbSprv;3Nc5~bY7P+P}UGx!SX8tu-5=Ld&P6hSus*bMX=X79+6s* z|CmhA?^TTuc$7VT&O!L1^xIZI&-q8~j|6b^jAYh02}@?JeEen4~&)E*(tH_;h~XicoumV-}#S9^1N0p&lItw_cwH z0FKx~fUYn?$R*|Ljw*st3#SvY(e?S-6zLMteADgXh0CQHN_K`3bsbPh_vA)Vb` zAC1w;v(OI73+`BPuDg^!=0j`w^Kz4DJS=Mp1XDU$_n)7>q>~`c;Blo+l;zeqx0 z`^FE34kDsEck?WML40VpC2T81H_=;K4BXq0We#K%Rl&&Gxxe}OQ&|tuX2ZdngpiOA zFhKLlS5!0fx7Q(kbmSPlsFqdfatKVKHSWftbp2A1CNV*9vsCo-8Q`4}3k`&Nwu`HOvkFR(&FTKV0!^3h(J7|AS+&#$>i=7Rkam2&Q8w~yPO1+#>l zdSG<4CAE*?6nLfrK;LiJ)l;Z7QrkX4z+eOFOA97*NeD7LIX{48G6uNO^QqGUyac}= zV3fkpSo=cVAM)I##;qB8U7xz9f8=)Po4bQa&<-&c@Z-4aBU178>7h3;TpIAHd=PSV z%bR5A!#uMzWN;D^fvZw5^%JL0KUmw`q=jMr0Ip3eWS9=9 zp%avpglTh}Q-hBo^=!%F+h0IUTs(jN3phnyuv2ED^WRVX02cTdP*xD((14f-Am|R6 zF^FWo6@Xlc=?k*b$WL(@lyyIBPYW~5?|K75L5=tnRly4vzEVqgmZ+3_dQ?2fp`qf? zxFTXdc26Nnq^(rc@&{jQX4%FB&Y+9us1e|+AD~rcYN;9xXh~ptelS%N?St+8*3@@C zDDa*Yw6uU1$Lki_N`Z0ogP`E7jg1Yx!)Se&k*?cx?u@Hmkv*?QhVHnE3E(*?2(*u; zwwbAA=yp2)eSFA|nC+@D)gG8CyA5FgA1Jf=frAInqG1tHc<7O)I{c^1(v|QNbaaup zn!kRy`#GD{YufLG?LMFtOh`<0p^NBn%rfuD(Dj076in<{82M$O#HyBu^K=&nDqp_= zSWE-tzZ9s2AILB@@S8OYc0dZeK|==l;@`J#pC34(vt9XCP1(;f4TsXj9<P70_z1R~}RJf_GeC{gTN631a z`<>;l#X@=mhHY^aQhq`}*5M3BL35?RSG)l(_#!7~eZlrh*PFpAe-oGJ2cL80L(ezP zxP(SVT8KhEK?>4Ie!#O1hTOH}eM={dPr`kKT?b=e3?Y8?_R+?Y);WmoMt0UAt}fxV zA5$KL$A^j~89Mg~6x)yEai*&!d8t!F?HbwU6@XQeenEz^!ZM#0j4}@}))pmT+NIA5!a2fc|jo)>uXL9plo8ATK9tFwz!`VuTGKd3K1@Kecopoo?P zd=oFkv6dy^^XZpu%~$~;{s_2m!w+V617Ec$lv&QfH$dt=@WYsz*}a_~Q(Xz>2bn#v z8nWQ_?YLj+X+ZEU4c6dSexK*$q;lbT3^& zX-)>OEfr8M_tSeUsWe23-G!lOijH5x#10uAr^Y1e04zrZ z1PF?E01JO-KyyQ~hVu0353JG7*Fok}I#TyJO#fNVuDEP#yE3JX*lpMT=VxTfFt<2` zg*%Nl!9su!@rU8E;<+)(Qor^e#K`fwe!*1IirtgTgnju4GHCYuQyk*hf!Ul~n;+CE zb{nt@23t2@33?P~0fu8Dvo{D~A19%S|C4Q2; zaFJv`@so1(yZtN_kjOzr0FX-72qgZSQlx^10F)Y-ff|bsvo}p8_pwl*OCTk%AwqnJ zB#j*~;%y&QDmn~fF#+B+PvLf1RmitO;%Ij8q4&2y@C^Vfbx{3Co7u91kYrqJR6OT@ zeucmkg9Dh{@uFmAAbTfTP3=f|^5lDPE)3H9Dc^qndE>bKJ0aX}iaCz_DZm$~HWsdva0Hw&Hcbno3Q($@2U7%uWYKk&- z#Y$jv!Gd2(Hz?PKxUU7!Z5!|f>R^KP5X?qAA*A2qz&RnFXjMpBDhh#(DlOGJcY?#i zCwkukMQCqxdcrU49M|iMVds;FG z%SRwS#Qlz9MPVUOoPhX8?Q{(T-~E*2e}rrI{rmeQ$E0aPs6d{h+W}Sa_}k&q@Pxv~L=M)<@87Xku3YKr<;DXBgxZQ6 z!rgQY`Z?_K!%^&!|N3os*p=viTk~2i%3Gn5DS*i{+3+$1(ZOBKWFrg0z@++VNHQ5tb=sKoyRuWW6vLoPU$=dEnvYP$3xD$p~B@fM=XQ1j3}ilCgtSAx!C3 zUU?r`_0&W3&XT}mXhN|E#1$wrT{zhawO5GH0_?bq@{!<@5I_p388ahSxrWu*Ppb`lIXiZ91;!TpC>+Xk7?tL_s{{L03M|{Zt zWAWemx(Ix!yd`w%r2WKvl~^s~2c=+Urti`xpeT@F0bwTbUaT?klV#(x1G4zZ*`_cY zS3E1&Vs9szFK|J=yze=j=pfU4yf>u`JmW6 zxT&MDeEoxQZf)XRoK?F3Tk)S=SwdADPP#qy+l=y=b7sGEkY#Fny* z3YcAjC~?Xz_g}tHwl+?9690ZkPC>19Wm>MMNIm(gW3`NIjcQ+})Ehdta-pV}(0=Rc zOxxja$DxumRtxA`3?}>2vu8s^t)GR3v>s3YVf&!#e+{T_LYpCX6#=UpcIa_ki~S7~i$r?NtpByUuvCs&>fcvkw9ASuQ1(NKBQL&veWNsUCveM~mDv)#?ZPR|)8o)b^|ZdDdos4Kk6 zRJLNC5m;k(>02sHcr;uJ`-AN*{-niLD6pN7D$V-7I6e8FDAA5!-j+|PIc1xUsk;nyf7Lgx4wA0#u%Tg zU%GqAMlYBBMlg&xjtHdWM9bOL)jX%PR#W5}FPqqASWwG#l^d9Du5$FJW*R4Jsa(kK zP4S-0;3xuJmJGXrhDdN`d3|f;;4x#XpOP2y`!beJv2w8(#bDFQnb{I;958K-L~eiiJ+nz)y!=3A8)qyXc~r+P<>;**xK6%e>~g)ri! zmfL4E)nC29J{f}tm=aZ*U>5RInbdJC_RH~e#tDgR4y%?p0>yX&%Tq(6-L{8dz*EE#5)tux%0@Nz5SiptpEuW}N$ zH+@>djJ63192|);LUAw^d&5fdu%}OPR?Rn8MjCKdtuhb1cHbXnfO+5Bdj^yd8>g#t zRBijz1>;O7k7j(o$-KLfiuc-9Usw#%a`P|dA(ScNn{saPX?|4*saoNeyx9r7oXn!|{c^THu5AJwcSJ zfZHS97}C#XiQXFN#zIlHQ+-}G1Nx!zEUElmMyh#Mp=chWA3r}m6kOn(1$tK;#G99z zQ9G{%1F@W~dm zz(NvJd1HPM3@%u7(?b+;K7oT-V9BWTVqrj%d&_wusD{aWc_LdQGvhBX$A12r6M2v1 zfRlf#S8OaqN58y{46v@v;%Hr_El&F^Kl{`t5PIRnpSxOXEkr1C z%U8{}RIhDM=WA{@PS@n_8jKum;xL`oS#^x~BB}RDhst?6H@iKZBdAlCXtHf3krc&J z^L1J*Sp8AkQH)@>ul`54DXNY)5N}&kD|by+R!+ZAwT45pmhL1YQE9+?_#DHnta87a zBf)%OCi+QAJJ4TGD)ylQiQ>fc%LOM-5=tzi@Ez$vTU%fS{tle%Xb7)S&CoS`b_6^Z z(E`UlJ2KUL3(S3@WfzO)_3MpPEiJ)C9>t*enp<1z3#sbzI}PD%1-BV|*kD(xi7z5p z?|7{~X~({zOS^r$Ac!duwv`vuC<0 zQ=pwx#0CU8GBEt+Kwf8P-cCPwoWLfqf((CNb(T}+kS$L5M{k<;+fh3@QLOy>LiRJC z(+_^by<~g$vX#pP4|S9lKPhqwqs4yJ(`|w`JCbPxsvB z+`LTT^d2FZLZaL&AAGleKh@6=UfyCdDMpmdM4`SilY&75 z=Au0j87A~=+o7#9=LjnR9yv0ifZ_aV?gws%n;Vypgwsvx7JcRi*UhKkL;+TS)2~%6 zYzk_=K|qp;ON`Ne)@}Qm{dUC2B-j@$KNJZdYY~>FaZ4eJUDmd;YO+2-R(s+%@Qcv1 zaEp!xKokW6QJ8>>@-iXnYr{PBEG=OFA(?d3kS*cpBjzwGHLCBS5$Hm zF3kuEDsK@V>a7Xz>$>mp;QHOoJAAQvSHaABP*Z`yqD@6$?~6IO2+G!YUsb0m<=+7p zv|Yc$TW=G$CV=~b$0mD%fkYVRX#?MV0QL4T?fU&}9glP1f|(E6!7$JdB_DAqvDS@i z&NN)1nH8V`zlo3vuuFk%5d_L2Vunw^l|p0>m(xNz4VG|KWB>l4s))#RJiR)NK4Cd9Ly^->Ob9=rtIjS<^_wv{C zU=EK{|G=UgAa>hOe*DrHg-h4#isv*OS4CDl9$q>pws)s87%{^ezH%%ryW}m+l;B37 zhsWtL3Fhw&;<73KP0Kas_#}VEKfCo&7oo@{$7vm18kBg(n*~ECT zXv)3vXM`{JKzXb(l4aM;X?k47;%IPz@yq}I^5BYNG57Oh#Xr!w`v8MQ|!x&88 z-YQ=Qgh(mi{K+_LEA!{D>Xu2&Iu{n!kO)LEeB3(tUsOLu=|cy)oX|&xs;P9#!+;&hw;be2wvf13Yu>fRz~R zORP+QI|;E<2MGYMN_(A3YlOfM7HRv>L=jNl(=`-uyRiQx@v0PlX2&txO#irDsmGE>Ez-aQcAJzO!z zX5J7^UetfkpRcMhO0Kiu7jaB;dt(}Xvbhp0EZQBy)kCqrHWp`w8_isee?y z1|^GTe?BLp0YwEM%CqWk_d!bVC`Eh#A_4Vz{pFEVSHLL)A`bZCABuVB>_l-bd z4JwjP(?G@VAE2ZT24KpL>gO9dGE3e={k9(DB%7fi`vY*sC}j=VY&{bg0V*p720j_S z(t!FdsAf4k+{*xMT964#hFJO!q=Zpt3*wwmY0ax8S1?QgSIb`Mm)6#?{|we? zPUR-7;nDIsO*T>B{#BqDkRP}VCv7N2=`o=7iG!s47}QJVBcesN8M=i&Lh^HRGPArL zQk8Kij|Q2r79apxp%q8A8Hz;Wpf15260oRv50$|J-GUO1kl=Hg~m82aN-pI7K&D_dAy@SWchI~Ux7Scp=}G0<|L9%z0lXqiDp^uX+?1E;bKsZO(@ z>Og$L*x17;Q5RJ)*ClI)Q^5J=J?b(8qO)l5?bXko+pf=jeT##*@vTt)7}oFrD0wT? zbkLq@kO`yLho&YXQ3jL#1yr7xYqyLlv}pi4*3&gVKaWoU@-s0pk*c;7<9_LX*)c+h zBfb6Rv|Lx|-N%AGfFm=Y!q~v~*?qX)?!4gV+2*PwvUiAU0cNrtWwerFJauRVEPh_} z7i9$Q446XHafVNTWpty{1cq?64Dz4skmWHdhYl#j+^VqJHHFuYin=Tqha9MftDz#} zJLf!?cJ|H3j~{DVS4z(3Km5gL;ESOY0mAEvYD=RwDOn+wf10iu72RgukR(|&-S=bI zHW_NJPg8i~rO@*drFiUKD2mok25zZK><2lM~^@nM+(5~z_gdLe{ zS4b{9PHCYEF0^`|t@^|pVk8Vu<_?JLi?TrPyF#D9^0y=059bFe5vey*wrPoS*@$Vk z&#J0_1A+|+&xzGOuMSkO=S%M@UshW>2N-GHFdssd*GBTxv9s02IqN)t zNfv%2-3Wk7yi0fvk+K`zJGrsK$jD zkzknLdO^*Ap!tYO2oy@#(us~mcmvWsy7$t+>$~$5HD=oej}5UqLAQ+ybjyyAW4!79 zed~~B*8A%lOY>NH;Dlt&J2nuOvfXP*X4X({ie5<7X^ZDv&+pR;Ig3%A`u8^6!r~o> z`OgTZX#m(tBKMcz3aT0Eg2@&P&zE?KOwyNAlz8L3q+jC$2yXnMEK;K z6*)hT^iEld=Kzuv+*_OeCnrNWRR$J@$u0WyuZg)y92lLLK7Prbd3z}c@42H5Yyf3 ziFNFC;B)r&De)~c6H`^u!Ey)=nYK(p4R~h8>#*k7 zPabYz7I}|uM>Zvn){Pj0PqE(6IUx3zMYnadypcrY%d-?S(C zpS~c$R30N@S8q|iLkF-0(iFE!*4}@U_6sm*2=CNXkiP?rA&PiZ^DNl@K%iB5l6X0W z8)7L^U8kt6uFrvkndON10zN7%S6|ZGqC0$I_;wqkxfyiy@7;SCur?}>~^lcEM z1aEE5;_H}Y$xC)z!=Oo=ju{gxv-i~$Ib|rO~3^CeX16rlV zQd2pU3o$o_2^fsQzqg!6b3Vrdd}1$qkim+V<<-Ffd&6ZjHZi@Ibs=n@jH+IrTG2>R z`~GVm<`p?{P_h3E5!?e45@yRon_IXoffXG0WgvWZOy7d+-8Crq(#W?waSNoAoru4^ z8G$$)|K>08Ag4n4Uvda|R;t$2&?&`;TH^=(su=7sqM~4KRU7A~eEXeM^vNu^>RD-+ z@XB`=sv$8F;+6e;5#eprKES9PTVIc#VBSv9%+odFAIT+U9NS8yFIdrXLZsD=WO) z+M@iO5w-76UNA81%o$o2sxtIuwt>qW)=Ydwaq$3V=*-^wy@T{G|LzeQfse~{E+oY78M`GoWlOf}iwYYu#R_I$X28#gEg*M%|x`yX8JFp!J$?g#I; z=cpju-(uI`OyMEH2=B=P;hv8n)~51R+VF8-eA%u(B!V`|!eN5a!yAK<-P_TV z+&~DRq{Z9vo!X)n9zwQFysCgDVR|B!xUG=9FcvuD5euaRdrB+E1AN>ri{cPaz|F@Q zGzP#%tq)Sv4A-h6W|Yotr8?70opgCB87OZfG;zj+QV0ONRlfn(S^I`MP7d0oV*w9j zcjcsqdJa4r5Z_WxiuTm0ftvL1D*GfmZ&FYrfgsGMDqo9xdJyqj$jox3X9e$rw?&1*axuCj6pSO-%sq99N&vqu@^yiw;}|B zy;?rVUA33-R8Jkz5;Xk9h>|Y`xn|@Ozn+r0k^Cj+CF6#XD<9N`zC@dy(9aj8hk#SZ z6Fj%pz^ul&NI}fXs;R1~;zNc>B&Omp7-t#XEJpb-4lX4!1CPY-iSksxlY{mZJB1oR z!uiE6hO76Iwk_QM{rxefkM_e5u{e2#n@53hBCb8A5b{G8yQ%}_ zX8R0aNE!=G5_Q%>*{$+WK0MBP`vIHN|LNIf;ws_hV???NQ+=xZ6&2NrzZVwiu1dRaGH;N0 zU$6WJs1tZg?a7k^S*`X{B`$LM<-Mg*d0yowL%GzeFXgW>iej)Q_n?q+!u8pL8zHZ# z#A*Sk1RCvnA~gBjpJ=(f8G&@xT=Q6!9Z}eQOYL)iv_6nSxD&`M$qR5=2fQV=8qyi+ za}9!S1mUJgm93;0g#56&d&{+Ub52pPMh8)k3z_BY4x>uMhx-yAj)~y`3Z;BiKWuB8 z`?MA(dEvA`=h-WBkM3cV4{MPk{N&xVrv5SZQ{0WOzaae*iV+|KgHX9H$=>K+G22OcBjBL$VV9~%SMHtQ?T{sV#JjBT!FI_k+-pBI zYiOLrWn4B<^Xd~vYrMW^Xe6^vcNgaah@{7<-;P$B_@Icwfs(yQLo(|@wic!5Y$@OR zrW=8EXSEzMI-@Y`FT}(gU}-l`IJv-n?w&v2MPcnMPQ_!f&-VO5?9Ia#9nfjp>b5%Z z&8ZJDr`7*1x18Un3pb3p40_`^`=QmtbHopjw|5B?+P(c$CY0(A3)?O3f!K_6<%?diHf>kNf+6y&8&3FJw&<3X^#FHL5{SYdnM8E-A7EcST zdO}$`6cAPSTR()j85PW5fdfZSpNk%=h=>k4+vS*GcPzYi61qa$(CI+Kb)jd*9e|Pt z=948tktF>t=qp0JqcNMXcXLCF7i2s=JyTQ@(-Y?%puI=6AnuE#FZE_O6(7J=DM$lDj0A?ruOBjXg;A4vH$FEZ%HkbyX93 z@A!d0LcIr2RH>$;6Yjb+)}PT8du>1Z9~6aY!oeGeD1&A-H5ig($IDl*n$hV?h?4-0 zt__OH#2N!tMqxb~ct5nZszDi0QU2iIAhRr#UqO#9M=ZyoSD}7+inP+EQdgz5CE+*V z?o|rlLMqF)vi%ginyR2XU4h@M8d7S3m4|iNq6eX3>J$(3Mzo<;2_Rv{;N%N*MiM%i z2gsXA;Jctmj+}yGRpM3A3hlY&dfE44GsOlbWz8|u{ zEFS`uggkv6M+gUCaTgp8OUC_&NUBbb(u8`{V~(?qRn^ml_%p=CafUXv zx4)kpicYaNNEfEzZ`pAY()!UG0enX&C_3OwtaK<6ONIt)sQZ5f{cK~Tr%s(3jK2H< z+D|vcj6xO_HZQN>+{^5T6X4*5=g{BrZYZm3I=gFHyEa{e!>c;>|6tFm)1yL*7nP#k zz73h&a&gjb7VN`*vM~Z+#Fg^TWA{NC{f#wus)TxIYsvM3t|_3qK|rA|uMJ9idp4oJ zB^}PQNrf@t6X3upb7Hv1@WcO_P_ys zS#UJYr~k+5HpGhqVLE|GX&|l!XTI`5(+}#mxh*>XY^HQoADyyAY(C-C9;lN)1jACU zo`RK)rL}5(j;rg&L4i;uoMjmGBUP_IShdpC@pk+5mer{Lw7#(k1?ZiW@uzd zh1LQl*>G`>qWT9p2ccaU6aiFn9E#{|8LYCjmVoJELIItkj3-^OBir=fYmf(w^#jIs1ohrJl*3Q2`=TN$ zjuUQ&e#JaEUn_jl1r8BQk=Wh3Y`{yj1%(QyUxSPLIX)T_O83@c5^~wy1}&!!N*c0h z1kIsKB(E*Leo@W4&M0fbqGa!-=dd#M{xF#&jau4X$ujKMn>}ZW7@+_4`@;9&W(>jg z#GNF$BB6fE5n%(S~mZ`L1H?AA6tcIe(pBEScKOEKtH!6Rq^q70x zShAIUG_n=+L5Ab~6&2RUh6S`S&^_WV?T91?XtAqcR_CQh20|0)?`IhuJHd+Df_L3=p zir6dvV!rw#6YYvD#oJz_K5<9tLN~e2V-k<=fl*wvFjnTVUXe3_Io*JJbbS%2m?)kIswf0fDlhCy9}x+FPmfH78Gh`11CgSS`LiGjuOYM_};`%|zRP!QD{K z)R%(}*)RyCowt`EKDq(Xeup6mmUM5$zVt!FsPHa5OUCCKE*KHHlnrqEiiAoiJ6*{|gJK}4;f zXOdMim7r$^JUs$k&(EG8!6kf+Q4<>@DoR_USm31tlafp$0if z7lrg#a*zE+w|+^2{AMBKa}XI(JJOIwIPk^XMd=HjsIxLUuSk4*>CQROUmwGQ)lLfm z2RK&>a$?QiB>Rn7Xi?5Tmhuxsl;vmnk;D~q0(M%_LkTbPtCnbBSbG4mUj=)HR6;!0KK+|g zlS>g@_9laa9pNA#wYclgIXB@vG$>9K1e>4;o!)8C?{)Q@u&J z-+~PaA0b&=PzR&}3xxOX)KuPhmfK|SpqzSi{TXz|8I?UnSEKF=0S+N{IU5GgF)q8Q zE>@`i$%bYYG#k9Ln6^Chnu25N?t%uKW^hnvL@~2kOC|rBLiPZvBAl0eC+ILjV9IL8))?7gT8q6xZeTb+!UcZb-pTcLh7HeFBWC z-0U3X-3iASf&1o^)hV!!gY;$E4d3JA&?MH1GHriJh_=Y)9|)-HFI=FFM$Wl-HkZN!J@(XgQ@Wq;!AM5wdt z2Y@F$gI6Tbjz6b+{{_izC$T**`Az=zH~R$^5LrrMdXnO>TM%E(7l^^(vXv))jpTEl z-*-HBaiHV;r7`|;vp>i?y09SkcoHl!h?FtkZR#m%#E2BVi)LZrmRvahz4suZfEE#u zZgJ20-Uhz$*if^aUVcE`QNg9Q3pyc-sYVG@3{iBrteZZZ%!q*%snq@W@#7#nO5Avy zL9Ow;tm)?kKoSLvF2CeK*VbbYXnlnGJKbT>h*&*5ugYFq!hmf<{V1#8ViM9_mBjZIROd=03c zK&67O4Z`2m_qZf9q0iJt^%0s+p`2!;-&4tmVjE%84f8Dajfg;=RY5#OS?v|czw>Ia zzwt#-0)@aB2oT;O3zQ1s1F8TOtba|9V7wN6*4#FEWuVxYs|V<`c=hI#Ki3-h@(pX& zF_q-v z0i<5?rOQi~FHW6zfz6R5dP10==^Z6vl`zpgA~qT=WLaUB;{lP0GeoLO1?kWQFP}9^ zUHWl8)yp?Q#f>d(uKt<1NT}P|b)KiUSE^lwH-VKy?A?Kwg9i`V$(VAH^Z6h6*Qzr8 z3S_xaq`uyv(qo=*v6%Iv&3^cB%ng!5T`mx8@bN>y*V;h~2V>TOehyRwPy(Ufn!NFT z@)-~j)AUep$F_ClM--RGOb7Nfj-5`5gE4p3>Qpoguhdz?tm8 zolnj%K8E>p1m(;E;$&qSPVyZHFNx|lfBgHM1xx7{qR8g=?r~H+FTKhpd3kPz{MhW^ z9_JNd$`6c_FoEG;!dhcj4ai9S5$7OTi6IIncx4jz0XzYi!Y$9RZqy`7F_>!_e!5KF zF!@xDPx*N(L3T)fwV#Ab>x#8j1fyKt4dG@D2gKmI!jcr$!a=61p3VX<{4~MPDgf_J{Z2OO@Qaat5BzkwUnpheQ6@M zu8(gMWRji#7Mm|m(84|_0bc6bery7jMw%}Z$75+86Rgf%spmoAU3atS%A&X$Jz>;lF;q(|mMkg0oN%`2 zuWR0JgEGa!;AavjhHazBp#u%Z#z1#@`JEnKZ-8x{+;P-WA#yu|7X|r-4*Z&knJd^R zz2J2S^f{6vM!n5J`GEaa8wp*;)nqlMb&ZQJWw~MkA8Z@t5{A`VJaVQO2wSpBVa`a5 zM4@Xr3&kYshAlV%lfO9hLjo12H2`Qi`x}oBNP)Vkh3Ki&EMt;alIg}@H1l|vsh zt5P5gdtvBDteaRjp!SG!AIGai$luSU2K~z-<{nDG8L?5<=)rK4#OA|`qoSO`Jwrq- zeN@BtKf-iOfL8q}_7}{`7+?uKaJk!kcO88gYT5dl^g7<(eNcR1j8{$W@ zR!m`Xvli1NOo(0$igBxsk^oQ#3b-l6h6)xT(#I9(5n1~^55FBxe~h%4CpS2QElYho+5;!%r8CaUs3S$X0=+_ zr6A6P1j)J1P&Q6UZLg3u3e4&++69=?uu?MhCEkm3u6V-YtwynmH#eh z+;EyjajIqI+?7`b-c#eIdbIk03}^cm*59_YV-9f5&h84RT6s1j@$vApW zi!NqOnBIQ>LxBqC{UP7k8A?7APjG{b33>JioQU{{|E?qXY*d*j;%DT6JpQ{=5bO#( zKc~yOhjpR)V+{6XNZP9&DV%ysACvLWV(a7G=U*0Isndayq3rQa`6XSWzei(jQG<|p z_uUiZfJ{L@K>OWcTQQfte)-Wqj<9Sg$Q@wzg8`PqSEr*;DQki!WtO2Y^Q!L`x}sb?Mt_uhN)TD`eH17H@W} zy@R4OP-m1xS~ZO-nf)5!dI-;nk5|3v5-Z=usNf>?{sj$CZmT`v=a58^7!v~^B33J) zEYSe^1eJF1Go~C3IQgkN;G9ANy_e+L3N_@yS}4YJ^ignS#n82#H{ zR~#T71+|b`66A|c&LNt`qdr_85mGOemM79XnPJ1E;Xm)E8ziSI9po(cO>^ zd7N@42;E@UdIGEm`aWvEj&~uuzE_BKB2aFLh=uUc?p%S0N&7{r+|d!S8nX@ik0h?1_jryV1@VMq`=y|9$;E?7@vZ+Hmg{%`!Vi)Mq{4qr5&FWKg}E0uXkfH4C03A#__r7E;$lnIl2(J9TLJ7L7YQW-MDa7 zw+F%TtHI&l?JF9%@vG~KT-8EpU~j}vA7XT&f`SIBLzY1{R)x`OrQJ^R-*08%Hu5HK z1K3ouQ+=W{5aCX?`>Bn<9M=kp{D-2Js$LEr^o53rRbW{CNz9F4R|&VtbYQfxRaI4j z0({0+ctxFI=(Ut@CF<_(>W+?%kHY14llA2hwlooT1(-PuM9_md&+kQ-B+aAaO08c` zNpg&6IC#LvWFp3Gu*=lW?=t^OMUR>TNTE2gjo7Rzz;6j)g0pfP4X3ZdLMv; ze?1ZUlcgPLTcvIinFH!8x#wncP-F`*9lNkTAps(rL7K1n2LNdNU+y2RjHv5vH%oEO zyDMS4RvVP;$!>#Jz9~G<3IC92yWvtr%o&uK((RSQ?{5h3!h~SBv`XjIiSJH7IJJ&&w}}fJ?!l#f z<4`GWp)zdkGT{0o1M>sw?90wkVk(K?#Kk@te!|*tK(cY*Dd}UBM)WNBk`1z6K#bp$ zY}=#q;Zwdr`1rUZTrdT;Y}rzJPAx)O4<>JAn)x^)3>-RV7I>|MHR@s18hGu}ZT(s4 zKf_?XX1r`PtK7TLbfznkhEkC%JDXF0zkfUJceYwgq*0`rT5{lgz(K;-1gxMT@SYmVkJ11ieG+>PeK@ORI~w( zdBltX;bkrO=}$mc?*V0AgOqI?etHZeD#04x6B~Mg_~->->mvKc=PcVeV&i}Vbxuf= z077nVLhYGBR<)^6$aj*lca8)YN(`97sIz_GIw^~it^!{i zv8u^X4N%kk@ktJ22ZO+4#l^*BIYpNSA-rk*-UD6@OM)>&&l0SGM1U}VAkGum8SGXp zE=dG-3JSg!@zKaB>#^&F#Xs@R%78DxzuO;NzCS}SDZ2l`u*HAyKd7{D5>KovVc8V3 zsh`W7`JbQZp}tX?3%5zw;uS_eWu*G;_{*#VcWu^E-!eEf?8cUM!bZ^JtwjwyNp_XB ziPS!_u=VLGy^onnOhu%n4N&DOzfw!N31i}TppyhixO(*}7|Xk1?T)K-ga0FO)Y3^d zkZ?#YtQ_=6xQF$-hpdO#TVZvR1V>VfsSU(9%>#^IvKNthFDagk<N)m9gs0K z+FmdePyv55x_cLFshVY$B=18}nK*1gJrIlHZ;l-Ls64}QvkNlypAn;%so=YzzI$r_ zac?kxc-slP2a$jPVQ#V)iQN9g=jEFivgSrbr^3`VH5_~@g9)zBHlKTHlV0s__UQGe z8YW-;ZN=fF$L}70Jax)>=eC1+EG&704=%IrPz{SGML*ar5iQ7t?4DRIn#G9ok`*g- zv7)YCv!*km$Q&N5BrC;cGXmMyIsCe5yABa$5QCAw;U@@=EvGi@&i!!}(FiGF5gU29 zn!I$0#MV4F(=wNUit1DKM^{OQ8l-H7KV^PQx?lX9PJwq;_$56!YLK1<2v2*mx+46_ zElO*gaLQVA`?&uC3KW#`|1y+?oxIGiPo|^?%RvQ&QCttH5Mo6O=T3mHIPu9N6E$zB zM1WLpXJ-aI&Xoo48Q3KFdHg z24|Pwd;}YVM%a4~OQ=ePPj0)Pw?kdTjQjMWMT@(#jw-vGsp{{w{v=%bTOi;D^OFwf z7O+9u@ls}&0nFEuQ=AaOTjpeI0*8Qw35{-8OdLb+OOjF{cLIQefx4ijRAd-XIl1=} z=TBnZM{Ir7u^p}vJkh#5GfElV6u>B;*Zv5gjoQRV;!yf^zW7&iI+Z*?(y4>)P51A` zKVspF7|j6v5^Bhr9v>gS;2r~X<@BrLM?(h+PjRY;M}$v#Q=2(oagb(lkj&|lZ7G9; z5IHo7&Ius_Az&`j@KIP2cKb*TQ~|vm+E|7ds5P}8iX&WGjj3RL4B1j*l!7{)zrym0 zG+QWhFd&msU9+*bPD?*F`c7M+mjSJ&EvBzH`6xSEM8lGGZ-m#GHt8u5@AWD4-F-ia)PyfbT! z9KjkMA%Wp2#5PJ)gqY1rGW0yrd+|xL=%UOA)W;)jMdt8aSb2M`nNBK`uH5T6l5(Mo z4T(%hY*i^ee3x{oq1O_yo2oPF2TVhiE7$B$dgpCpu_3h;;YX@YBGr-E+ZXE~@Q1TN zU-e3eKd9aZDmkTms_8S`%I)ZhG6+Q!+5`AIN-$QaaRHXYtgTe z&DSH%LS$m1c+L8RrS`pNl+=Cwsno&Ya;0NcpWDsK@BQg}Z{K$>=~DrGWMp_dP;1nX z4;80I%Tu24EVGkr7WnG;WtD}!jm?2uzxg}!oYOZYAC&j{aN=iY_@BRxE@v+r%Na(h zBjG*KD>}ZJNPn9=3W{!tZhfR~L__=lQ=m}1P$cv0(s%efE8+3^ewD7-1o_k4$l3AK z>BoUPd5S}dOxJr~Zz_W;)|xwqA$Aclw$P)2L$X>!JHJh5!H!+KRIqbY5vvwdtJWM> zCYR&6E%3pgJ>LgYes)z92i!7U=dGRFUhnrVAh`RdMB0eoPM%u|FB;7NApSbhHfW^U z*k4IO1P^m2*T!}wy>B`T-dknxcd+fSqUTf`YPg@V7qw25d@~2>#BCGx2}PSL8!mUn zs@l`*&4N>Ea#KlCm;IV&`t=G+D&M7*VngLL%~R&CrbJh~X7_5E08KV(M7YV82!@rAFQK z{oRlLd|Lhy6^cK}t*ckBJA2lq?@Qd<)dHlLgcarC>a}rGYT?z@uF4+-47t4R<`T3N zuXOCRzgB(budA#CLf+*1;{Y135hos^ma>E47AjQ7kVJ2gurbRla!30Sk~$$+i-Y~E z8}?$dqjD3G6|uA;w)$97ni2jUcGA;*3zQJ}c@(_Rh+!kq8)DCs{^f!1&7)RPtzobY zi?MFvmDpV|^g?KtN}7IVnQk>t+TBOTl;7V8!@geB{N)`WbMmg>b=w&P`-VTk+R%aT zCxWRM>}9=(#D0Bx3FQ(g9FgP*F+QvflK;eNhn6+O)dwtw1R$OOpoYiE2r+S}p*|cA zD*Djk6>GXDw2l>SSquZ-dbq!7!yOKElU_VOUFhUiz6g2QNz8`upv1tx&_P@Trbq)| zAZ`lD35eE=#DGM1Mo2V|^Zi|ivlo{&W!Wy>PRHBix2Dx4#_dX z$v?c~{#3TjGTiczoBb@tJ=+n#Uv; zJzUy4H~n8ioI22(Y?E><$~Oe;HZ~$VH4Kx`x~!wCYbV^8k!35B%>}stGKL>_eol1= zWN5VKZn|oRelmob1;5oqk|Dt%OoO>VXgu zWV{%jHLdPlpIx-YuA>V@vbb@ zSbH(h*H?Nsk6R2xo?Wxqgu({cROZ)xxIui$Cd?Yt64PY5&{d6X@B7E?_6s`kyif(9 zndO*SpFqxy#z?6PmeQ-#>OYc#xJ{~;bNOgHvz+;rbMz9l;3Wp+v@)7Sw)0_KvE}HA+XUU(9EynI*R*wD>zXo?#>G*c?aCgsY z#rgNytZvowqLyO`K#24RQENAnggWFjXrdF379VGbvKW2x<^fGs0u5H*=em4RYB`ts z;X8+)ym{T^_cQsJV#(O0$92WUtgCo!%&Xpf=*peg)P6428ewxY{F)~uILK|Znj-(_ zd1grastR4t%~K!DM9e8~4SJp)<8?fqsDq*iUTxyLYj*k_UnIKp;nzU8>(p80SQCX68kT5s^eicqZ2Za9) zzt?nS^U4ODG;sQL0`0lP#r1D`*?Gv&&~IaF~vwS6MeV;nJ2 zFw(VdNGB^a-JzTA1t)KpZxp#ArW*q39v-loHcwFh%XIl4ou15Ww;D>-9nc;PS+Z8( z5Ha#2i1go|FNJIkOx@tNg~_mH;1Eo4DIDqlqN^r55lu>ySgI-6`xlqN&z`vD!h!nJ z6CrYG=MWhNsikTCociUdk_$} z0t9aIhaq$B(_i7&HCQM=*z#`i77Psiy^!yL0#Ss0`xiIv)1CrvD(NqpzXs+;>bW2; zm^_`IbC-DhMCVk~e!WE9%Af0Q?qHthL^GA}qKE0L`*MNs%5c#8JUf%M!$idQPv+b= z4CXxV8wlf^tT9!8^o|c%q+LLpl>Wc|vNC@;EF|a45Md&MX-wY2oVJaR;t(krbF?Dt zqsOXM=EutZU}-PJs}-vfntuL^_}H(qvJ3(iRH|oZoFp@Qc#2}y-TW0BgGX}oZn+tM@D?vka#4uGiGrC@jmKmM z2$1Anq(dC3dSCS+1Z5`mKf%RIvhQ{D@33E|8T{_?XZN*>Idw8m)NbD*&&+M;Y3(E< zSSie&H9K%R6V0ufp*E#P6Q=;=++$arI#d!?fS&TIg8hB zAxJusF9ifl)v!X2wv2E(0xT_vdl#B5Ae8xMJZRMczW#x+P|LWkVpmsm3Xio0Kw89q zY(4Q2(1Q!x61d0VX6?Wfd!d~y2>X~E!Tv(#MxhYS*ezjKBv)IAiyqir66X4B$r@Nj z0uWUruOf^LxhcdF6Wu%5_Uqw1+ngwNcTtp{>ys++!ue*gYfetWA2E(k8m?}~~n z`C2en!7|mJubX)X7F!2;(CRgvv++(9V;7iys#`hyb981}dZjS8F_RinEUcNnhG=RV=s;VA` z>9;oM_M}V$`M%)%-t$hmQ|Dgb4xZv3ny2UK3fd-Nq7_GTQM7xpJP)5J0LTPf0;gPK z-A_i2_Xqrzhl;Sh-UKj3I<>H_rUxRRf=<+D8nAPc&Xa(^WE!&3RPLD9 zLOSD+*PUNqaKpwmv0i`urGHFq-ImHBf6ox`Zjl;7)!+fXl}wTI4!_$BA&mcmBhJ~? zK3p1~A1ScGplOGnWy{)?d%z_0_q^@nQ}OE@eWvw%2R~?HUyFZR)z+Rt^O$~U=~6Uf ztZ9CKG7r5^)bT~59n!)B*z)S*(&Q^ln?+a`^2+RZk4Tssi$jo%(DpxuRS#WvUc9K& zkw2(}h#cKb||6?1opVuMw)?!+u7nw$=xe_5h0k(m%+@B$SvtF!pG77JArjm1E^kx}; z*xHoM&KgAu^gm#{!k!lmu>Ui?K&7h;aG4!?OY|ZcP*hRTL08w+;I+Y$nzsSi0dv-}e}rZq~J&}tK&fUL3lgbDr5T2lQ5cG_my*o4as3K3N^?3h5M zRJXzRP<1cvv9sWla~_EYw8h2^^BugjgNE5$8P}XlJY4}l6N}RF;5~2hz^?H}Y(}~* zL0Td7;S>GvvUWYfWMasAFXWv-q`nt57?MlYsUonXsmli8c2-Vs4Zn%k1Y+0vvnK_CVYIzaY(s%K78W1^k3wCp1I&u+RD~!_^}>@E4m#l zx-tmN$3FS?uu-*baXO0ay=Y!SEhpgvnsEk%Fr)w5%)8@Ijb8L?j{pBj`x zaZHnW5E%)lYa{hIdOYsW)Yamw)am;>NREk~uuXhs1p5N!cT}vwq#=$qY-$j*k=8;0 zP))s{^1^?->kXDHA+l$oBz+$6%m{yML2`~y$5St5EWIPNhDy^f2u*?B6Wj%%$8q(` z^jYf4d%9Pfver)^qzemxo}lLPH$K%cJDdLY#@qOnI`Mjoa&u`6-TMIEkbS;hr;!!h zC6o>N*2eVi7>MyY&I1xLv!g+9mebO=v-=egrW!Ln5vMsjE&64vV#W3Af~#hZD2JCk zRpiYWl39!}z|t(O5q*aBeSgxQiMNE7yyw;A0wXZCQZVfU(%tijFJtnjPu4U`b`1h# zc()a;J^x&O4^WvMzvjN=jPa(>Bt2Ry2MyNA2d}SUT4v3+Yw+PiiGY`DVz_npIh2Z+ zYc*&+$|Qb>pwryXDJuD+x_Pq z)c!9W_6A)dYTc;9t^98qHOrnHiJ90uw+9i(UV+Yva;je1i@u*le$X9pN_DerKGyK~ zFlBa4$;jwu7mL^G=j$asozQu$roKKK_10r%hsoNMd%4T4bj}+ zpBsDV^rULB_~COK3&X%6T2Gn?qT+CuOTs3@a`8qxbsx#z_@>K68z<^t=4br16ciql z8ufjSKdhAeO9YNbUq~zW7Fx!ZR?l1pBb2bjqA3PLfIsvHh>_?IH5|TnHsEHS< zwqag<+WSoZ@xqMc3+ES;$gvSeHXfT$mFoA;&dyd25i!5ULFTRcu6pN1Wou4aKLDP( z6?nN(+c0_$%R(OW$>au(qwaRZz=lFKo2XHXYjbI#jauK~)6%pYHhfyEU+KAC&fT>F zOo*ftsi!k6Ns)bR!T3AZ_U)S+qu!Bk8}{B>-ak>N+w8kFWr|(go|e935WPP_&cAFH zhlfiEN+>}#Sz(CTI;C1BR7qc>%`@Fc1Cjt*d-7{oop^KqNptgz@oyu38Ou#zEFBKW zv16(6_`wS4;WD(o;-(~Mm!K#cj2u}SNsIkEHVj>qb8SX?(v_6GmrR%gg$!caU+zb7|W)gpT!8-glX_$~N z%NO9>EZ}fujrKC_2qp}*=jo9UR?{Ub>y(+W%*by&F13Whq_BEbA-+I}I&&cQqDwsB}97a22a})>~kpYOBeemT1Gs_@Z3=ix~{UW_~*3+__%w_n20{8m&LuWRzxS7$Hm@l{yfV?R@u;x49ZaQQHKIqz1(EkY{`WfXMe{Lq*A<$?r&Lt8iqbtxH7Y9i9{ zh4@PZ`+PZiV~%ncldr=vUM|SU`79XgErWyP(bq=Yvse+FoSfVQZ-2d9$IG8Te-=TX z4!bK?EC4ynnBu@W)_J4#A!6r_{{j$Q_E1bfYB1Lp?1x5C3)GP(P;HLf8R;rA=r-sg ziq=m?k*QW?l%3%tU_~3}(;H%D8NIr6iW;4kCK@h&a4tDK&5wl~Yw%7Ok&2p^vs^oU+>+6l$P{|Gf0JoFUSsB?P;Ho$3 zT9>wOL)xWpuLTXW_d-ak|11`I6k+nJ=;p|QXT!cp+A*(XwKywjlk<^B{#}o)m6vSX znr_>tjiBdy3wbR|`oYvGGw7s79+QucPZRnzhd?Gtp7z0WlEY=9qE18Y5i}S^p}+5* z5HzqAZ*d)7AaJdur?0OJJv^yorM`FX-r*pY)sHF@SSQ2uqbd`Rjv3EA9IB4tOECyZ zG6<>m-I}mjJgNJII2-8NiXVBNd#&D7$i7Z^%NOf36{;ER>LIF;rj`oj2u4n~?@YtA zN`Lus>>+EK$zNB|Oog8!g7}9tLCurBSjR_6*;q5y9*s;rIxEZRUYapAIw65i<=iK? zoYd5h`y8J}(C?CQv%@OJQ!4w+dlXiT8eF_mb>J+zvMmt!?AYDktJTeS>|Awob2t>d zmbi#xrlzK-NG3;E0vomiIvJFinaNE#dGaKAK)EB;N;0PCg&cx6l{_99Q=_T%iSLsp z$#i0?u5wY8N5%Iln^%c|-J-G(%ZQNs?juhc(RVP7^b|%%9284kY*4mA*u+z`$;BGA z=`b*!qqu16{S5pnI zH1eR_g&>w%g}VTV-#0cTSl%@@{PCPY^K+;Y8T-b9{S?Umil=m zH#55qV;SCqCH!~#5`N>v`6W~p#{|X_&ZhBV$!@XvMOg5_c@qzind|PK_ zb(^uUW944M{%4)|H+!~~I!WKkZ{hdCn+^M&?QVuUp2l+RF(qaq|B>gwJI0|7 zHCu$5x2j65He58aRLK{gX5yz`6O;NW`h|M3(RTwT?O+u`i|0L*WuWRbF=W{Rr2Is!g>c3RT$t~M{ghTsxZ66-O%jGa#CTdtB zpu0x>xB2bppnq@g@fD{Q3TyK*?w=2G(e+Q78uiq9)b;3B;9axNa}Uc+ejdWjg-hT2 z_h0w@>+^aTFQ&i$xPQSqtLya9c>U>mkp5Q2H{4j(xfH5DTS&N!9Pc*?*Rs)1g$0BS zU$vS)Zr~OTf%%W@$o=~xsf*dg4$}9)Q;YgDbW>>WL@lDveSCcNsaMHa$KkCwqWb!K zrbYQ3mEHR3yt~6~6jNNb3uSSW2(Y8_=CZ@|FBhB?D~)kkmS$+=qW0&(jNc3Ky{f^< ziW5%Hut9c2czo@Px~92dFRwkz;@SSO1J|70cRM)5U1%)?iBjP0DbwxbPE4L;jQdE{ z(`ZJV->seMbU2@Kowz7Y@52)>Our@h8+yb;SiXSsaxo8OHdD9sT+mXU;?~P;)PVv2 zpj}GY+0}X0dL>Fhu`WH;Mg?&>Ie%84%A-$(ZikEMdj@>(ySA(BtJoD(+T%W2XSe%g zV6v`Jv2mHBevza<}=pH zr5ZI|!Rg7jfs2=R??dFYIc|w#v z7NNPCrl!3&*?a6$a`N(mPn#^GpFsOYeyMMn=PA=|^vS_r`!anbTb$+QJ;s5aKL5;) zV2bk(`Z07)Wvmwt+>-hSWz?=l@A@bToleGQA;BXsYn}gEPF@xIv~*>bC_=|k`1ldc>VJo!lBj?ldgJL&i2Tw52@CO0SAGcb@C%pXk*;snt?F^s?osk6<>pRT$$|YNl_iMNh1X z{&MTbu1q}-u5E%`f%#j#z2%Rvm1i{ zQ~bUMM7&Z&&LWOKTJNsgvn?*d*$V!=jpxe1aP$M%BBP?B(r6KSo`lni1)FeMR+`k( z_w4JuHP0~F4(5$nAT4kLn?Mw8>=3JVMqOK80_~^E`!Ls`;)!>=-5-EMy%k}^#aYCs zUEnA=Rt_V~B@wy6@uUwdE2Mo@8u}@gLm9z#eNF8$;0Ib@Xl{ zL?18UNNWl5S|d^h5}Ehmr0YS%s0_w&y3h^P6^*uvbI0-q%fZBVc6J79L z2!R3%EG+vF^6sEWN=f0cSqHwZ78g%+K6ziOd_*EcK2mGxhy?BNUb{0Xw1AZSY|2PQ ztonAE#gio_**Q73)s&qb*EUa^T={%9j=LCyEK!p}&-7m*BH~C_$|B_oEX&Tl3DT+5 zm2r8p2mL3@AaD^qQx#=YDU|!;wJXwGqi63ytF;78jW$PpzT{O$;7M;GW`0%riE;ik z@gs9T?>BaeH7t#A+VQf*(e%`4(la=la2PJ&{B&ehq4LAMd<8T87YXvQ)=-h8+{zKs zSzXTPK@P5rGD#qTM&uTjMA_gTZP9wP9eJM$;@^)Q@`t$N*VG!;VII~-KFC{tsh@XoMH2SGBdZvu&sM1qwwglGri%Grw-2gR0J zfF!eC%u2g57$uok4PDP9d3rKTCnLhr5#I0Jw=YF}iid*4wYs)8{qxNG_eWY<6P)Rp z#C$@b$9d1CKVYj%zmDB{n({^6$b)JJ_=8F%zFl8sL&yuH42x;wBGUqEkm^sLg3%{8 zvFj_!=HXy3#yoqri&BjU7BTiV(#W|o7@1Wpf^Mn4=L(g<4eogSiLwMljH0rIGm9e1 zrbp`-cM;GlK%LUp*B8MfN>5L3|73c<90Y4o%sX6q7buF#NDxSZ$X$G3@kalRS3Ye$ zy3t-+2=EP_wj9EC)g3AiYenyL*hWqn2o+2Zg<)z)bOxK+K>gblCpU98RxQ%#Qx3U+ zj3-s=g-JRriA^7}ElWI_z6pEO6UXjmApsdqz3(Q7QFQ!;wSr_ua#M|!&Jhg_4PEH# ztK#P7h9qJrq|$W<>VI}Xd~+lAkW3h~F|uIvA-kEFY$%8r1QFIJiXcgohaZWWlynS* z`mF!I#^?qlMn^!=%}&-sPyOc_VI+ED3V1Q?P*CdKG9zTTabwSgi2m*fg_pa)M9CQ-)H>% z@WEO`ZM*fg!Wu*^`TuT*ToUdvYa1v#eJT6Pmzuhd53MuF+)|D_Y5o!&@D6Q&LKy4fyyVYFzsCranC?j@pqV-WG$%;PK%bwFmnd zfzm1r7NysDZe3IJS0m*;wOsTe6MLks8YI&`VQiyfPcVph%y>S}7+)2g9AE)J zHp7VA%Y|8PK4wH4>n#e^IT*)zgWZD>KU&m=GAkUQKcw2bb%9%U>HFL0>h5``t~LK} z`AWXj=r1DNlV0>ed#Lf*IxYIqqXzPDH*fgr`WtRc`<7;c|NDZeoah{>(Pz(qs8O0* z|2iCWco`#5)XLmHTKiZjY7srU|DTx|IWkk!Yx)_PR(EpH2TO76`@0F+Mu+H}Jem1< zyavcMg5r-c8t)ia7pTd1$Fi*3J~8ph_GKrnHPjE(E?`6>e3;r)*)+p!DM>brG$WIW z^+#CxGV@nvN0P1LprwJF0ip>|ei*gfnXyW$$f<51AIb*a7-WQjWNBM=Ja65=OLR+q-DF^4Zox zqcMatRK~qT=kMHVztkjt3*({^@&}LPBBwhrKbH@m;S$}^5g>q2kD6Uyzt?BImK{ZV z&-{lcBImNYSH^D*+=4s{x$b`*QTlZ<{gY!!5W_*whZ$!kN9?fvW`F)t)BN#4;P(4v zp66fg$`*X;%XrioeIJje4OMy!WEoAA2rO8~|C%w4k7N6xPvfink3KWvH9b7``QYzb zFC!Ya2*G$Ures=~=OaYpxUzUXi^3{n#tkQlm3#69l_I{=VGL^;U|PK_kq&rOB;4p_ z46a3;*uIo2ST(e-GA>HT>ddj`sz-SPjf-z8GNxC}7YlE*{r%Yr&(wvt73Y%`Ud=awCu1*%p4Y^%PGVT|+cq8!I<%Vcm}*GQOnyF^ zt9LLiIz9RDouO*JxRLCA->FKPEnSXQjsksj7XkHC7KOM)b-(O2yA-K*bEuZn>mtv$ zGOmQ;7PK+19p1>c}Es>`tEF0{RgOImaW@DplJ)rLKR~6|p|O zXcHZ;qkdXQAM92YJJ~r`Nt74vT*%$S@^v(s-L+*(W2E-t0;$vI z-!wN2Obp#D=^!yX`4P+=$Y%d^A!l!Yx)y6Gin=I>b*0vth#8ElGD%c zVk`KA(oPRFB?U`4WE@r1c)XiF-A4A9AR_M08&nyNlMNu7N#m3I7L9H9<8rMV_1$JT z%i~j}6T}%a<19fE1~h77CMVgO&c=}+|NGJ+rIF73B-1GM7!LJE_k=FRMt}MjOm_Ki z;1s>oU`T$nj8eA^t4UwZlPwyG{>)9Ducy5>JPGp(er1&U8C~pLR(F)VU`(`AdAL3c z#vx=*$iB^v7Z_SmFql7O^I_{Pv&3hail}fP%4(uHWf@c;)D8t+DgvOk1XhMax8$ZD zH3S136Fa)e$?RchXwB!(4-vhHdw{F#a&MyiqPrjJOgqsd0?4Z=K>>9q>u_EsXAVgg z3$znx1XtL$Y)?@;Li};7O7K>{yV6d!l!!GL_@$Z_bppCSP0&1~82mVjGL_4z{QGTEQ$q#Vn+^85>Qro|@%wAb8JKkzwQ_Rz7mH2GPFLIijWGgG&e_SI#F|r3! z0#aXeZR4bG*Zkk_1@5d{w$W-rbJ^!JW1dd46J4Snv(tAu_a%23DJdy^g|d*O021H- z+%AF?AU!kl*Y({Lkqb?)YU}Dc(C45;ZnjO%f@BE@TSZ&Nq_ppEmXkG}`rI%CjEM)G zDo*xS8qpJiPU*3KO9=FCH{5vr-=KVJ#q5TvH~}fC2FJOK5r@&nGc3V$xSo^?;KTA@ zyGsP=cYrjpdo=gaqphf44T=#MH4)i|MI z`sIJZVU_o@Q)Bu0Gr#uD0K^Z}j$Ol+_HmK$;s4)geFE#>p#1rC<}$EusNAGmJQ|NT z+Ik>zZ$!<{`8`3u^YSl%GE54tF8S`Y@f!4s)Skp--FrQW1b|mjNo>AoZ9TFq;%VI9 zwEz;qTgARiRC`pUUhn|((gI)1g13?L0GMUM(4(p&Vx@AQKYt#{LM0M8m-kI@)dZwc zfzGtE5ql6M8>fugLT*y#@^v^uOh}H<9X?e7*`7rx3qW3i6iTQC=X0ejE5gb`_w5GG zhICFde<&cO9hQ1d1imK-IGI{DpHAK4p=6>52q||K7tGQMI>G3!NWqI2cIovJGGAWQ z9Y4uKkp-e|S?I|u0Y2C;)Z(3BpjM(-SAXvIAGiW}9#X_2rkBi%xvFgQ_&G@F1O$P= z8da%Xch>H%_=%$5uV24hyA?KzIg?iDET(>MGU!TRrB+JR;|jcTzmA8_q>p z0L?#+B3(s*)GGhiuNv92_nb;~O3nX-ESqARSUARTiPK`)}->{J+Bjhc< zINsIwqEu)CHMNYdbVt(M)tvndRY)BjmHoekLjP^!rmt@Q10bVsp8vN!|9}7LIo@nf zl>fCjbw{GcVX#d5{p%Y`MX}OM|Ge*^Y-?-# zUy@s^14oY@&HU<2i-jvg-pJG z3wO>v!pui^Ac~fj*2ft6cmL$h^VO%`HhOUB$tFs-$NMMd{2v27+W_L@u!F~#1Ife4 z>eYp5>Hg*?DZi3hBsnN2pOd&R^$ng`uq&N-ZV@-o?N z7CVmUpi`v_Npvw=cc8Lq11V~~THWhxKs^6_TbqtGhJ1O<46mme?mZKrm2#-~v+wpV zzWrtbckh`|I9b>bZUZV^f=I~Ek?bV7bNiEtG3eBnXt&LYsEJrpz z{p)2xe^<$={%7YRTX6V^TI~08lPPm0fN4bVB)ROW4ouWU&em22y!m{xLDskfT-OhL zabZ+!mT{5%H%4(Q`KTXq;S^3#iQ9m1)(N3)1sSSbU?PL_hd0LiBGGnNRK&c*zbss8 zAHD7jAI~uRdc7F~iKIMmhAbNl9m?@o8m!>KD;K$B)sA*gmgOJ9By4R6%7apLDU?J; z-xM3wxfH_N!Q%1&8aUPgD>XH$or8lVwr&X!%WdDUw@ebB#inKkeNq^zY8hQ$2`C0B z>MmU)C-Z{3Wwg_VQ@JSs>exBO1)&>s@{yhFRt#E7Gk2TJO-vMk9l%8&pa#>ux&?|Z zE>rq5%Q!g94X36FX67+nFlD%blXEJ^;KXh`Y68lD#uv)KLguDC=5~|S781r; z_e9QmT|_4;`Y0$^>?vUGKI zgD}a=v1^{2eh5y-=!Qscuy;M0f2yaC_6+9-3hGfSDk|_M=4zV1KKUZw&HbxWmpP#- zU>TiyqU@+t9|ytsaAX`{8A2t&)|iGBv*YSgp<_6_N5C?gncL(v%jR);i^_=RG-aYO zEyX4jcx{F3D`Tshv#niKD#ie$Ir_m5Q^j}w*<*)uT0rSACV_yMPo$iO`@7!jFHd6t zQ-B4@oIIaVXVo%t$~0Bh4O-^0E{LD7d)Q7sI_OBlRYf8gA$t#@5!pf})`1#s@!I?! zHwY)7#Gh|QLp;e3H>u_^S>uHI>+qu$ZoC~f5(2u(WnHpk5~x-3r;OQlW7SEs>#1hn zus1%cu zj5lmMi6K4w*=Xktr(Cf(mabBhejs7nIWj$Y%VYdjgdZz(Q8kerZ}jyd2fH>;w=M>; zp>ky5UCIz54B3+xvSNPRxK}+nGJ$~V_)@*&H+(9%9_f!uq_g?+8DkCYh>eXMf}W>@ z;MWu5ZTEM-%*+gg5uLFRJeZ#NNIrGN3qEpp4zRW1g)?V(jS=meMS7 zN3clg0CP4_ZU49HLzu>f`2e>|5l8ZRbO;#p^{#GJ!D^`%A`&AFuyV*_w#y@s%;X_- z-qUbz$oZ->`*x?SMy)_$s6J{$6NpN7<0-UtaCeYsx&!NP?sN-O-|%@ekj08vLg35F zac~uqrBS9BN{EZlk3h)o1dV*h`uc@ zWsdQd;p8IL?7EL z&yB};vN~uo&F;iGEM?F6^c!S11Gpv>B2wodoT(w~6*6_$qLgK$ur_VVbzpnL`U#hF?nWtbjF>*Q!qCb7iyoo)%odhK?CAA@6HFjO^e4nDs1M8t zZN#-xG_n{(qflY!3;UcXs!Ci)ZS`QeLp6gV37wOKeM=fXJM`&;`j;jC z{YKt)gy96^OZo`$)5EO_HvtR>=pkq8?d>HL8Ow>G4zgg{AXttib%3D?hkQ}kQ+gz4 z_de|d4JJg=-U{%(4Wzd*si}J?Ljw*4IbCU_el<3kmOs@z7@LtHM(Sy@zf$jywkR!N zq9dWtkC0iE{OY^6pi%CA64z|lu)${h+pR8m^_O8Kv1#>M{A)u2c281gQlWy0x)Z~v zrZ!&P0F96hW@x3+lb#MsYB4+=ZEmgJ5_`A5jJjBvePr5b8-=NC=gT zzJ7gR{ZOc6ieijiQc})ZSt(Mn*K{wFbAc(J6OwluydNDK2OW=+q6!Khs|Aj0x2nLq zjsRs}wG9moDU za!6|X^;N1Lushnm{o`mm$(%^7aC*E*u7Z?mxXGg(2AZ)S%I$Ix%lttK8&a_>XJWE0 zDZ`c5D=H``sA3s3aQ*p?B>cmfFY|VR0DB7Kn1ux20>Nmdo@ICdHz7{ilJZKemRdOk zZ7l1D*OGTzy`M+m1|6dZKzB!hX{re;4zK@rXRfKX_^-%IbldJy+F~s0i From 7f064fdebbe5146090607f5a7c90e7b0eae19208 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 30 Oct 2023 15:54:57 +0000 Subject: [PATCH 03/18] remove results files --- neurips23/filter/faissplus/README.md | 102 --------------------------- 1 file changed, 102 deletions(-) delete mode 100644 neurips23/filter/faissplus/README.md diff --git a/neurips23/filter/faissplus/README.md b/neurips23/filter/faissplus/README.md deleted file mode 100644 index c834af51..00000000 --- a/neurips23/filter/faissplus/README.md +++ /dev/null @@ -1,102 +0,0 @@ - -# Faiss baseline for the Filtered search track - -The database of size $N=10^7$ can be seen as the combination of: - -- a matrix $M$ of size $N \times d$ of embedding vectors (called `xb` in the code). $d=192$. -- a sparse matrix $M_\mathrm{meta}$ of size $N \times v$, entry $i,j$ is set to 1 iff word $j$ is applicable to vector $i$. $v=200386$, called `meta_b` in the code (a [CSR matrix](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html)) - -The Faiss basleline for the filtered search track is based on two distinct data structures, a word-based inverted file and a Faiss `IndexIVFFlat`. -Both data structured allow to peform filtered searches in two different ways. - -The search is based on a query vector $q\in \mathbb{R}^d$ and associated query words $w_1, w_2$ (there are one or two query words). -The search results are the database vectors that include /all/ query words and that are nearest to $q$ in $L_2$ distance. - -## Word-based inverted file - -This is term-based inverted file that maps each word to the vectors (docs) that contain that term. -In the code it is a CSR matrix called `docs_per_word` (it's just the transposed version of `meta_b`). - -At search time, the subset (`subset`) of vectors eligible for results depends on the number of query words: - -- if there is a single word $w_1$ then it's just the set of non-0 entries in row $w_1$ of the `docs_per_word` matrix. -This can be extracted at no cost - -- if there are two words $w_1$ and $w_2$ then the sets of non-0 entries of rows $w_1$ and $w_2$ are intersected. -This is done with `np.intersect1d` or the C++ function `intersect_sorted`, that is faster (linear in nb of non-0 entries of the two rows). - -When this subset is selected, the result is found by searching the top-k vectors in this subset of rows of $M$. -The result is exact and the search is most efficient when the subset is small (ie. the words are discriminative enough to filter the results well). - -## IndexIVFFlat structure - -This is a Faiss [`IndexIVFFlat`](https://github.com/facebookresearch/faiss/wiki/The-index-factory#encodings) called `index`. - -By default the index performs unfiltered search, ie. the nearest vectors to $q$ can be retrieved. -The accuracy of this search depends on the number of visited centroids of the `IndexIVFFlat` (parameter `nprobe`, the larger the more accurate and the slower). - -One solution would be to over-fetch vectors and perform filtering post-hoc using the words in the result list. -However, it is unclear /how much/ we should overfetch. - -Therefore, another solution is to use the Faiss [filtering functionality](https://github.com/facebookresearch/faiss/wiki/Setting-search-parameters-for-one-query#searching-in-a-subset-of-elements), ie. provide a callback function that is called for each vector id to decide if it should be considered as a result or not. - -The callback function is implemented in C++ in the class `IDSelectorBOW`. -For vector id $i$ it looks up the row $i$ of $M_\mathrm{meta}$ and peforms a binary search on $w_1$ to check of that word belongs to the words associated to vector $i$. -If $w_2$ is also provided, it does the same for $w_2$. -The callback returns true only if all terms are present. - -### Binary filtering - -The issue is that this callback is relatively slow because (1) it requires to access the $M_\mathrm{meta}$ matrix which causes cache misses and (2) it performs an iterative binary search. -Since the callback is called in the tightest inner loop of the search function, and since the IVF search tends to perform many vector comparisons, this has non negligible performance impact. - -To speed up this test, we can use a nifty piece of bit manipulation. -The idea is that the vector ids are 63 bits long (64 bits integers but negative values are reserved, so we cannot use the sign bit). -However, since $N=10^7$ we use only $\lceil \log_2 N \rceil = 24$ bits of these, leaving 63-24 = 39 bits that are always 0. - -Now, we associate to each word $j$ a 39-bit signature $S[j]$, and the to each set of words the binary `or` of these signatures. -The query is represented by $s_\mathrm{q} = S[w_1] \vee S[w_2]$. -Database entry $i$ with words $W_i$ is represented by $s_i = \vee_{w\in W_i} S[w]$. - -Then we have the following implication: if $\\{w_1, w_2\\} \subset W_i$ then all 1 bits of $s_\mathrm{q}$ are also set to 1 in $s_i$. - -$$\\{w_1, w_2\\} \subset W_i \Rightarrow \neg s_i \wedge s_\mathrm{q} = 0$$ - -Which is equivalent to: - -$$\neg s_i \wedge s_\mathrm{q} \neq 0 \Rightarrow \\{w_1, w_2\\} \not\subset W_i $$ - -Of course, this is an implication, not an equivalence. -Therefore, it can only rule out database vectors. -However, the binary test is very cheap to perform (uses a few machine instructions on data that is already in machine registers), so it can be used as a pre-filter to apply the full membership test on candidates. -This is implemented in the `IDSelectorBOWBin` object. - -The remaining degree of freedom is how to choose the binary signatures, because this rule is always valid, but its filtering ability depends on the choice of the signatures $S$. -After a few tests (see [this notebook](https://gist.github.com/mdouze/75103e4cef436510ac9b834f9a77496f#file-eval_binary_signatures-ipynb) ) it seems that a random signature with 0.1 probability for 1s filters our 80% of negative tests. -Asjuting this to the frequency of the words did not seem to yield better results. - -## Choosing between the two implementations - -The two implementations are complementary: the word-first implementation gives exact results, and has a strong filtering ability for rare words. -The `IndexIVFFlat` implementation gives approximate results and is more relevant for words that are more common, where a significant subset of vectors are indeed relevant. - -Therefore, there should be a rule to choose between the two, and the relevant metric is the size of the subset of vectors to consider. -We can use statistics on the words, ie. $\mathrm{nocc}[j]$ is the number of times word $j$ appears in the dataset (this is just the column-wise sum of the $M_\mathrm{meta}$). - -For a single query word $w_1$, the fraction of relevant indices is just $f = \mathrm{nocc}[w_1] / N$. -For two query words, it is more complicated to compute but an estimate is given by $f = \mathrm{nocc}[w_1] \times \mathrm{nocc}[w_2] / N^2$ (this estimate assumes words are independent, which is incorrect). - -Therefore, the rule that we use is based on a threshold $\tau$ (called `metadata_threshold` in the code) : - -- if $f < \tau$ then use the word-first search - -- otherwise use the IVFFlat based index - -Note that the optimal threshold also depends on the target accuracy (since the IVFFlat is not exact, when a higher accuracy is desired), see https://github.com/harsha-simhadri/big-ann-benchmarks/pull/105#issuecomment-1539842223 . - - -## Code layout - -The code is in faiss.py, with performance critical parts implemented in C++ and wrapped with SWIG in `bow_id_selector.swig`. -SWIG directly exposes the C++ classes and functions in Python. - From 7cc49ea62dbb2cdad90f4943cc2df2cabe14bac4 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 30 Oct 2023 15:56:16 +0000 Subject: [PATCH 04/18] remove results files --- res.csv | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 res.csv diff --git a/res.csv b/res.csv deleted file mode 100644 index 14e0ff22..00000000 --- a/res.csv +++ /dev/null @@ -1,41 +0,0 @@ -algorithm,parameters,dataset,count,qps,distcomps,build,indexsize,mean_ssd_ios,mean_latency,track,recall/ap -faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 1}))",random-filter-s,10,17210.013417421313,0.0,4.049878358840942,53980.0,0,0,filter,0.9147000000000001 -faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 2}))",random-filter-s,10,13434.971315820661,0.0,4.049878358840942,53980.0,0,0,filter,0.9701000000000001 -faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 4}))",random-filter-s,10,8898.944679478747,0.0,4.049878358840942,53980.0,0,0,filter,0.9759 -faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 1}))",random-s,10,107284.9213454406,0.0,-1.0,6400.0,0,0,filter,0.9137000000000001 -faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 2}))",random-s,10,52454.37150610923,0.0,-1.0,6400.0,0,0,filter,0.9635999999999999 -faiss,"Faiss(('IVF1024,SQ8', {'nprobe': 4}))",random-s,10,34813.56916973082,0.0,-1.0,6400.0,0,0,filter,0.9692000000000001 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 1, 'mt_threshold': 0.0001}))",yfcc-10M,10,6795.64131225567,0.0,1072.5426700115204,4882552.0,0,0,filter,0.482366 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 1, 'mt_threshold': 0.0003}))",yfcc-10M,10,6077.0430330389545,0.0,1072.5426700115204,4882552.0,0,0,filter,0.5379240000000001 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 4, 'mt_threshold': 0.0001}))",yfcc-10M,10,6428.107866805404,0.0,1072.5426700115204,4882552.0,0,0,filter,0.6253599999999999 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 4, 'mt_threshold': 0.0003}))",yfcc-10M,10,5765.3369217869695,0.0,1072.5426700115204,4882552.0,0,0,filter,0.673125 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 16, 'mt_threshold': 0.0001}))",yfcc-10M,10,5349.823779832417,0.0,1072.5426700115204,4882552.0,0,0,filter,0.78061 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 1, 'mt_threshold': 0.01}))",yfcc-10M,10,1166.2876613086733,0.0,1072.5426700115204,4882552.0,0,0,filter,0.7835880000000001 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 16, 'mt_threshold': 0.0003}))",yfcc-10M,10,4911.40567052702,0.0,1072.5426700115204,4882552.0,0,0,filter,0.8174239999999999 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 32, 'mt_threshold': 0.0001}))",yfcc-10M,10,4384.772235694261,0.0,1072.5426700115204,4882552.0,0,0,filter,0.847014 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 4, 'mt_threshold': 0.01}))",yfcc-10M,10,1161.1679057350602,0.0,1072.5426700115204,4882552.0,0,0,filter,0.862773 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 32, 'mt_threshold': 0.0003}))",yfcc-10M,10,4159.390450900283,0.0,1072.5426700115204,4882552.0,0,0,filter,0.877257 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 64, 'mt_threshold': 0.0001}))",yfcc-10M,10,3262.4235190588574,0.0,1072.5426700115204,4882552.0,0,0,filter,0.901073 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 64, 'mt_threshold': 0.0003}))",yfcc-10M,10,3189.0423626576003,0.0,1072.5426700115204,4882552.0,0,0,filter,0.9240959999999999 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 96, 'mt_threshold': 0.0001}))",yfcc-10M,10,2593.7880792005567,0.0,1072.5426700115204,4882552.0,0,0,filter,0.926014 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 16, 'mt_threshold': 0.01}))",yfcc-10M,10,1148.366996200013,0.0,1072.5426700115204,4882552.0,0,0,filter,0.9367110000000001 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 96, 'mt_threshold': 0.0003}))",yfcc-10M,10,2584.583659327858,0.0,1072.5426700115204,4882552.0,0,0,filter,0.9448719999999999 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 32, 'mt_threshold': 0.01}))",yfcc-10M,10,1122.0103427196223,0.0,1072.5426700115204,4882552.0,0,0,filter,0.9623430000000001 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 64, 'mt_threshold': 0.01}))",yfcc-10M,10,1072.044241086604,0.0,1072.5426700115204,4882552.0,0,0,filter,0.979552 -faiss,"Faiss(('IVF16384,SQ8', {'nprobe': 96, 'mt_threshold': 0.01}))",yfcc-10M,10,1040.7842700388208,0.0,1072.5426700115204,4882552.0,0,0,filter,0.9861280000000001 -faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 1}))",random-filter-s,10,17180.616884446812,0.0,-1.0,9592.0,0,0,filter,0.9147000000000001 -faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 2}))",random-filter-s,10,13201.38362127302,0.0,-1.0,9592.0,0,0,filter,0.9701000000000001 -faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 4}))",random-filter-s,10,8399.779707010324,0.0,-1.0,9592.0,0,0,filter,0.9759 -faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 1}))",random-s,10,90163.24512564759,0.0,5.0532073974609375,49152.0,0,0,filter,0.9137000000000001 -faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 2}))",random-s,10,69982.04691827677,0.0,5.0532073974609375,49152.0,0,0,filter,0.9635999999999999 -faissplus,"Faiss(('IVF1024,SQ8', {'nprobe': 4}))",random-s,10,27337.10054813627,0.0,5.0532073974609375,49152.0,0,0,filter,0.9692000000000001 -faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 30, 'mt_threshold': 0.00033}))",yfcc-10M,10,3902.783262534991,0.0,619.2082076072693,5439348.0,0,0,filter,0.895063 -faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 32, 'mt_threshold': 0.0003}))",yfcc-10M,10,3822.097448584031,0.0,619.2082076072693,5439348.0,0,0,filter,0.8968999999999999 -faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 32, 'mt_threshold': 0.00031}))",yfcc-10M,10,3826.360975537842,0.0,619.2082076072693,5439348.0,0,0,filter,0.897869 -faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 32, 'mt_threshold': 0.00033}))",yfcc-10M,10,3797.5221741475025,0.0,619.2082076072693,5439348.0,0,0,filter,0.8996660000000001 -faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 34, 'mt_threshold': 0.0003}))",yfcc-10M,10,3709.367384945006,0.0,619.2082076072693,5439348.0,0,0,filter,0.901073 -faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 32, 'mt_threshold': 0.00035}))",yfcc-10M,10,3775.3003297468717,0.0,619.2082076072693,5439348.0,0,0,filter,0.901529 -faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 34, 'mt_threshold': 0.00031}))",yfcc-10M,10,3753.3730144863166,0.0,619.2082076072693,5439348.0,0,0,filter,0.902008 -faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 34, 'mt_threshold': 0.00033}))",yfcc-10M,10,3709.4452984119116,0.0,619.2082076072693,5439348.0,0,0,filter,0.903754 -faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 34, 'mt_threshold': 0.00035}))",yfcc-10M,10,3685.290238185696,0.0,619.2082076072693,5439348.0,0,0,filter,0.9055630000000001 -faissplus,"Faiss(('IVF11264,SQ8', {'nprobe': 40, 'mt_threshold': 0.0003}))",yfcc-10M,10,3471.4920981563846,0.0,619.2082076072693,5439348.0,0,0,filter,0.9119970000000001 From 0826f34a32ee10d2d56c78f0ff57c927ec060cda Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 31 Oct 2023 07:16:51 +0000 Subject: [PATCH 05/18] aaaaa --- .github/workflows/neurips23.yml | 2 +- neurips23/filter/faiss/Dockerfile | 25 - neurips23/filter/faiss/README.md | 102 ---- neurips23/filter/faiss/bow_id_selector.swig | 183 ------ neurips23/filter/faiss/config.yaml | 74 --- neurips23/filter/faiss/faiss.py | 287 --------- neurips23/filter/faissplus/Dockerfile | 25 - .../filter/faissplus/bow_id_selector.swig | 183 ------ neurips23/filter/faissplus/config.yaml | 66 -- neurips23/filter/faissplus/faiss.py | 287 --------- neurips23/filter/fdufilterdiskann/Dockerfile | 17 + neurips23/filter/fdufilterdiskann/config.yaml | 35 ++ .../fdufilterdiskann/fdufilterdiskann.py | 178 ++++++ neurips23/filter/wm_filter/Dockerfile | 28 - neurips23/filter/wm_filter/README.md | 7 - neurips23/filter/wm_filter/config.yaml | 50 -- neurips23/filter/wm_filter/wm_filter.py | 572 ------------------ 17 files changed, 231 insertions(+), 1890 deletions(-) delete mode 100644 neurips23/filter/faiss/Dockerfile delete mode 100644 neurips23/filter/faiss/README.md delete mode 100644 neurips23/filter/faiss/bow_id_selector.swig delete mode 100644 neurips23/filter/faiss/config.yaml delete mode 100644 neurips23/filter/faiss/faiss.py delete mode 100644 neurips23/filter/faissplus/Dockerfile delete mode 100644 neurips23/filter/faissplus/bow_id_selector.swig delete mode 100644 neurips23/filter/faissplus/config.yaml delete mode 100644 neurips23/filter/faissplus/faiss.py create mode 100644 neurips23/filter/fdufilterdiskann/Dockerfile create mode 100644 neurips23/filter/fdufilterdiskann/config.yaml create mode 100644 neurips23/filter/fdufilterdiskann/fdufilterdiskann.py delete mode 100644 neurips23/filter/wm_filter/Dockerfile delete mode 100644 neurips23/filter/wm_filter/README.md delete mode 100644 neurips23/filter/wm_filter/config.yaml delete mode 100644 neurips23/filter/wm_filter/wm_filter.py diff --git a/.github/workflows/neurips23.yml b/.github/workflows/neurips23.yml index 64b611ff..e8630498 100644 --- a/.github/workflows/neurips23.yml +++ b/.github/workflows/neurips23.yml @@ -31,7 +31,7 @@ jobs: dataset: random-xs track: ood # Test fassplus entry - - algorithm: faissplus + - algorithm: FduFilterDiskANN dataset: random-filter-s track: filter fail-fast: false diff --git a/neurips23/filter/faiss/Dockerfile b/neurips23/filter/faiss/Dockerfile deleted file mode 100644 index 163391a8..00000000 --- a/neurips23/filter/faiss/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM neurips23 - -RUN apt update && apt install -y wget swig -RUN wget https://repo.anaconda.com/archive/Anaconda3-2023.03-0-Linux-x86_64.sh -RUN bash Anaconda3-2023.03-0-Linux-x86_64.sh -b - -ENV PATH /root/anaconda3/bin:$PATH -ENV CONDA_PREFIX /root/anaconda3/ - -RUN conda install -c pytorch faiss-cpu -COPY install/requirements_conda.txt ./ -# conda doesn't like some of our packages, use pip -RUN python3 -m pip install -r requirements_conda.txt - -COPY neurips23/filter/faiss/bow_id_selector.swig ./ - -RUN swig -c++ -python -I$CONDA_PREFIX/include -Ifaiss bow_id_selector.swig -RUN g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ - -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ - -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so -Ifaiss - -RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' - - - diff --git a/neurips23/filter/faiss/README.md b/neurips23/filter/faiss/README.md deleted file mode 100644 index c834af51..00000000 --- a/neurips23/filter/faiss/README.md +++ /dev/null @@ -1,102 +0,0 @@ - -# Faiss baseline for the Filtered search track - -The database of size $N=10^7$ can be seen as the combination of: - -- a matrix $M$ of size $N \times d$ of embedding vectors (called `xb` in the code). $d=192$. -- a sparse matrix $M_\mathrm{meta}$ of size $N \times v$, entry $i,j$ is set to 1 iff word $j$ is applicable to vector $i$. $v=200386$, called `meta_b` in the code (a [CSR matrix](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html)) - -The Faiss basleline for the filtered search track is based on two distinct data structures, a word-based inverted file and a Faiss `IndexIVFFlat`. -Both data structured allow to peform filtered searches in two different ways. - -The search is based on a query vector $q\in \mathbb{R}^d$ and associated query words $w_1, w_2$ (there are one or two query words). -The search results are the database vectors that include /all/ query words and that are nearest to $q$ in $L_2$ distance. - -## Word-based inverted file - -This is term-based inverted file that maps each word to the vectors (docs) that contain that term. -In the code it is a CSR matrix called `docs_per_word` (it's just the transposed version of `meta_b`). - -At search time, the subset (`subset`) of vectors eligible for results depends on the number of query words: - -- if there is a single word $w_1$ then it's just the set of non-0 entries in row $w_1$ of the `docs_per_word` matrix. -This can be extracted at no cost - -- if there are two words $w_1$ and $w_2$ then the sets of non-0 entries of rows $w_1$ and $w_2$ are intersected. -This is done with `np.intersect1d` or the C++ function `intersect_sorted`, that is faster (linear in nb of non-0 entries of the two rows). - -When this subset is selected, the result is found by searching the top-k vectors in this subset of rows of $M$. -The result is exact and the search is most efficient when the subset is small (ie. the words are discriminative enough to filter the results well). - -## IndexIVFFlat structure - -This is a Faiss [`IndexIVFFlat`](https://github.com/facebookresearch/faiss/wiki/The-index-factory#encodings) called `index`. - -By default the index performs unfiltered search, ie. the nearest vectors to $q$ can be retrieved. -The accuracy of this search depends on the number of visited centroids of the `IndexIVFFlat` (parameter `nprobe`, the larger the more accurate and the slower). - -One solution would be to over-fetch vectors and perform filtering post-hoc using the words in the result list. -However, it is unclear /how much/ we should overfetch. - -Therefore, another solution is to use the Faiss [filtering functionality](https://github.com/facebookresearch/faiss/wiki/Setting-search-parameters-for-one-query#searching-in-a-subset-of-elements), ie. provide a callback function that is called for each vector id to decide if it should be considered as a result or not. - -The callback function is implemented in C++ in the class `IDSelectorBOW`. -For vector id $i$ it looks up the row $i$ of $M_\mathrm{meta}$ and peforms a binary search on $w_1$ to check of that word belongs to the words associated to vector $i$. -If $w_2$ is also provided, it does the same for $w_2$. -The callback returns true only if all terms are present. - -### Binary filtering - -The issue is that this callback is relatively slow because (1) it requires to access the $M_\mathrm{meta}$ matrix which causes cache misses and (2) it performs an iterative binary search. -Since the callback is called in the tightest inner loop of the search function, and since the IVF search tends to perform many vector comparisons, this has non negligible performance impact. - -To speed up this test, we can use a nifty piece of bit manipulation. -The idea is that the vector ids are 63 bits long (64 bits integers but negative values are reserved, so we cannot use the sign bit). -However, since $N=10^7$ we use only $\lceil \log_2 N \rceil = 24$ bits of these, leaving 63-24 = 39 bits that are always 0. - -Now, we associate to each word $j$ a 39-bit signature $S[j]$, and the to each set of words the binary `or` of these signatures. -The query is represented by $s_\mathrm{q} = S[w_1] \vee S[w_2]$. -Database entry $i$ with words $W_i$ is represented by $s_i = \vee_{w\in W_i} S[w]$. - -Then we have the following implication: if $\\{w_1, w_2\\} \subset W_i$ then all 1 bits of $s_\mathrm{q}$ are also set to 1 in $s_i$. - -$$\\{w_1, w_2\\} \subset W_i \Rightarrow \neg s_i \wedge s_\mathrm{q} = 0$$ - -Which is equivalent to: - -$$\neg s_i \wedge s_\mathrm{q} \neq 0 \Rightarrow \\{w_1, w_2\\} \not\subset W_i $$ - -Of course, this is an implication, not an equivalence. -Therefore, it can only rule out database vectors. -However, the binary test is very cheap to perform (uses a few machine instructions on data that is already in machine registers), so it can be used as a pre-filter to apply the full membership test on candidates. -This is implemented in the `IDSelectorBOWBin` object. - -The remaining degree of freedom is how to choose the binary signatures, because this rule is always valid, but its filtering ability depends on the choice of the signatures $S$. -After a few tests (see [this notebook](https://gist.github.com/mdouze/75103e4cef436510ac9b834f9a77496f#file-eval_binary_signatures-ipynb) ) it seems that a random signature with 0.1 probability for 1s filters our 80% of negative tests. -Asjuting this to the frequency of the words did not seem to yield better results. - -## Choosing between the two implementations - -The two implementations are complementary: the word-first implementation gives exact results, and has a strong filtering ability for rare words. -The `IndexIVFFlat` implementation gives approximate results and is more relevant for words that are more common, where a significant subset of vectors are indeed relevant. - -Therefore, there should be a rule to choose between the two, and the relevant metric is the size of the subset of vectors to consider. -We can use statistics on the words, ie. $\mathrm{nocc}[j]$ is the number of times word $j$ appears in the dataset (this is just the column-wise sum of the $M_\mathrm{meta}$). - -For a single query word $w_1$, the fraction of relevant indices is just $f = \mathrm{nocc}[w_1] / N$. -For two query words, it is more complicated to compute but an estimate is given by $f = \mathrm{nocc}[w_1] \times \mathrm{nocc}[w_2] / N^2$ (this estimate assumes words are independent, which is incorrect). - -Therefore, the rule that we use is based on a threshold $\tau$ (called `metadata_threshold` in the code) : - -- if $f < \tau$ then use the word-first search - -- otherwise use the IVFFlat based index - -Note that the optimal threshold also depends on the target accuracy (since the IVFFlat is not exact, when a higher accuracy is desired), see https://github.com/harsha-simhadri/big-ann-benchmarks/pull/105#issuecomment-1539842223 . - - -## Code layout - -The code is in faiss.py, with performance critical parts implemented in C++ and wrapped with SWIG in `bow_id_selector.swig`. -SWIG directly exposes the C++ classes and functions in Python. - diff --git a/neurips23/filter/faiss/bow_id_selector.swig b/neurips23/filter/faiss/bow_id_selector.swig deleted file mode 100644 index 6712aa25..00000000 --- a/neurips23/filter/faiss/bow_id_selector.swig +++ /dev/null @@ -1,183 +0,0 @@ - -%module bow_id_selector - -/* -To compile when Faiss is installed via conda: - -swig -c++ -python -I$CONDA_PREFIX/include bow_id_selector.swig && \ -g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ - -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ - -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so - -*/ - - -// Put C++ includes here -%{ - -#include -#include - -%} - -// to get uint32_t and friends -%include - -// This means: assume what's declared in these .h files is provided -// by the Faiss module. -%import(module="faiss") "faiss/MetricType.h" -%import(module="faiss") "faiss/impl/IDSelector.h" - -// functions to be parsed here - -// This is important to release GIL and do Faiss exception handing -%exception { - Py_BEGIN_ALLOW_THREADS - try { - $action - } catch(faiss::FaissException & e) { - PyEval_RestoreThread(_save); - - if (PyErr_Occurred()) { - // some previous code already set the error type. - } else { - PyErr_SetString(PyExc_RuntimeError, e.what()); - } - SWIG_fail; - } catch(std::bad_alloc & ba) { - PyEval_RestoreThread(_save); - PyErr_SetString(PyExc_MemoryError, "std::bad_alloc"); - SWIG_fail; - } - Py_END_ALLOW_THREADS -} - - -// any class or function declared below will be made available -// in the module. -%inline %{ - -struct IDSelectorBOW : faiss::IDSelector { - size_t nb; - using TL = int32_t; - const TL *lims; - const int32_t *indices; - int32_t w1 = -1, w2 = -1; - - IDSelectorBOW( - size_t nb, const TL *lims, const int32_t *indices): - nb(nb), lims(lims), indices(indices) {} - - void set_query_words(int32_t w1, int32_t w2) { - this->w1 = w1; - this->w2 = w2; - } - - // binary search in the indices array - bool find_sorted(TL l0, TL l1, int32_t w) const { - while (l1 > l0 + 1) { - TL lmed = (l0 + l1) / 2; - if (indices[lmed] > w) { - l1 = lmed; - } else { - l0 = lmed; - } - } - return indices[l0] == w; - } - - bool is_member(faiss::idx_t id) const { - TL l0 = lims[id], l1 = lims[id + 1]; - if (l1 <= l0) { - return false; - } - if(!find_sorted(l0, l1, w1)) { - return false; - } - if(w2 >= 0 && !find_sorted(l0, l1, w2)) { - return false; - } - return true; - } - - ~IDSelectorBOW() override {} -}; - - -struct IDSelectorBOWBin : IDSelectorBOW { - /** with additional binary filtering */ - faiss::idx_t id_mask; - - IDSelectorBOWBin( - size_t nb, const TL *lims, const int32_t *indices, faiss::idx_t id_mask): - IDSelectorBOW(nb, lims, indices), id_mask(id_mask) {} - - faiss::idx_t q_mask = 0; - - void set_query_words_mask(int32_t w1, int32_t w2, faiss::idx_t q_mask) { - set_query_words(w1, w2); - this->q_mask = q_mask; - } - - bool is_member(faiss::idx_t id) const { - if (q_mask & ~id) { - return false; - } - return IDSelectorBOW::is_member(id & id_mask); - } - - ~IDSelectorBOWBin() override {} -}; - - -size_t intersect_sorted_c( - size_t n1, const int32_t *a1, - size_t n2, const int32_t *a2, - int32_t *res) -{ - if (n1 == 0 || n2 == 0) { - return 0; - } - size_t i1 = 0, i2 = 0, i = 0; - for(;;) { - if (a1[i1] < a2[i2]) { - i1++; - if (i1 >= n1) { - return i; - } - } else if (a1[i1] > a2[i2]) { - i2++; - if (i2 >= n2) { - return i; - } - } else { // equal - res[i++] = a1[i1++]; - i2++; - if (i1 >= n1 || i2 >= n2) { - return i; - } - } - } -} - -%} - - -%pythoncode %{ - -import numpy as np - -# example additional function that converts the passed-in numpy arrays to -# C++ pointers -def intersect_sorted(a1, a2): - n1, = a1.shape - n2, = a2.shape - res = np.empty(n1 + n2, dtype=a1.dtype) - nres = intersect_sorted_c( - n1, faiss.swig_ptr(a1), - n2, faiss.swig_ptr(a2), - faiss.swig_ptr(res) - ) - return res[:nres] - -%} \ No newline at end of file diff --git a/neurips23/filter/faiss/config.yaml b/neurips23/filter/faiss/config.yaml deleted file mode 100644 index 62cc5b24..00000000 --- a/neurips23/filter/faiss/config.yaml +++ /dev/null @@ -1,74 +0,0 @@ -random-filter-s: - faiss: - docker-tag: neurips23-filter-faiss - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF1024,SQ8"}] - query-args: | - [{"nprobe": 1}, - {"nprobe":2}, - {"nprobe":4}] -random-s: - faiss: - docker-tag: neurips23-filter-faiss - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF1024,SQ8"}] - query-args: | - [{"nprobe": 1}, - {"nprobe":2}, - {"nprobe":4}] -yfcc-10M-unfiltered: - faiss: - docker-tag: neurips23-filter-faiss - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF16384,SQ8", "binarysig": true, "threads": 16}] - query-args: | - [{"nprobe": 1}, {"nprobe": 4}, {"nprobe": 16}, {"nprobe": 64}] -yfcc-10M: - faiss: - docker-tag: neurips23-filter-faiss - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF16384,SQ8", - "binarysig": true, - "threads": 16 - }] - query-args: | - [{"nprobe": 1, "mt_threshold":0.0003}, - {"nprobe": 4, "mt_threshold":0.0003}, - {"nprobe": 16, "mt_threshold":0.0003}, - {"nprobe": 32, "mt_threshold":0.0003}, - {"nprobe": 64, "mt_threshold":0.0003}, - {"nprobe": 96, "mt_threshold":0.0003}, - {"nprobe": 1, "mt_threshold":0.0001}, - {"nprobe": 4, "mt_threshold":0.0001}, - {"nprobe": 16, "mt_threshold":0.0001}, - {"nprobe": 32, "mt_threshold":0.0001}, - {"nprobe": 64, "mt_threshold":0.0001}, - {"nprobe": 96, "mt_threshold":0.0001}, - {"nprobe": 1, "mt_threshold":0.01}, - {"nprobe": 4, "mt_threshold":0.01}, - {"nprobe": 16, "mt_threshold":0.01}, - {"nprobe": 32, "mt_threshold":0.01}, - {"nprobe": 64, "mt_threshold":0.01}, - {"nprobe": 96, "mt_threshold":0.01} - ] - diff --git a/neurips23/filter/faiss/faiss.py b/neurips23/filter/faiss/faiss.py deleted file mode 100644 index 02980d12..00000000 --- a/neurips23/filter/faiss/faiss.py +++ /dev/null @@ -1,287 +0,0 @@ -import pdb -import pickle -import numpy as np -import os - -from multiprocessing.pool import ThreadPool - -import faiss - -from neurips23.filter.base import BaseFilterANN -from benchmark.datasets import DATASETS -from benchmark.dataset_io import download_accelerated - -import bow_id_selector - -def csr_get_row_indices(m, i): - """ get the non-0 column indices for row i in matrix m """ - return m.indices[m.indptr[i] : m.indptr[i + 1]] - -def make_bow_id_selector(mat, id_mask=0): - sp = faiss.swig_ptr - if id_mask == 0: - return bow_id_selector.IDSelectorBOW(mat.shape[0], sp(mat.indptr), sp(mat.indices)) - else: - return bow_id_selector.IDSelectorBOWBin( - mat.shape[0], sp(mat.indptr), sp(mat.indices), id_mask - ) - -def set_invlist_ids(invlists, l, ids): - n, = ids.shape - ids = np.ascontiguousarray(ids, dtype='int64') - assert invlists.list_size(l) == n - faiss.memcpy( - invlists.get_ids(l), - faiss.swig_ptr(ids), n * 8 - ) - - - -def csr_to_bitcodes(matrix, bitsig): - """ Compute binary codes for the rows of the matrix: each binary code is - the OR of bitsig for non-0 entries of the row. - """ - indptr = matrix.indptr - indices = matrix.indices - n = matrix.shape[0] - bit_codes = np.zeros(n, dtype='int64') - for i in range(n): - # print(bitsig[indices[indptr[i]:indptr[i + 1]]]) - bit_codes[i] = np.bitwise_or.reduce(bitsig[indices[indptr[i]:indptr[i + 1]]]) - return bit_codes - - -class BinarySignatures: - """ binary signatures that encode vectors """ - - def __init__(self, meta_b, proba_1): - nvec, nword = meta_b.shape - # number of bits reserved for the vector ids - self.id_bits = int(np.ceil(np.log2(nvec))) - # number of bits for the binary signature - self.sig_bits = nbits = 63 - self.id_bits - - # select binary signatures for the vocabulary - rs = np.random.RandomState(123) # we rely on this to be reproducible! - bitsig = np.packbits(rs.rand(nword, nbits) < proba_1, axis=1) - bitsig = np.pad(bitsig, ((0, 0), (0, 8 - bitsig.shape[1]))).view("int64").ravel() - self.bitsig = bitsig - - # signatures for all the metadata matrix - self.db_sig = csr_to_bitcodes(meta_b, bitsig) << self.id_bits - - # mask to keep only the ids - self.id_mask = (1 << self.id_bits) - 1 - - def query_signature(self, w1, w2): - """ compute the query signature for 1 or 2 words """ - sig = self.bitsig[w1] - if w2 != -1: - sig |= self.bitsig[w2] - return int(sig << self.id_bits) - -class FAISS(BaseFilterANN): - - def __init__(self, metric, index_params): - self._index_params = index_params - self._metric = metric - print(index_params) - self.indexkey = index_params.get("indexkey", "IVF32768,SQ8") - self.binarysig = index_params.get("binarysig", True) - self.binarysig_proba1 = index_params.get("binarysig_proba1", 0.1) - self.metadata_threshold = 1e-3 - self.nt = index_params.get("threads", 1) - - - def fit(self, dataset): - ds = DATASETS[dataset]() - if ds.search_type() == "knn_filtered" and self.binarysig: - print("preparing binary signatures") - meta_b = ds.get_dataset_metadata() - self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) - print("writing to", self.binarysig_name(dataset)) - pickle.dump(self.binsig, open(self.binarysig_name(dataset), "wb"), -1) - else: - self.binsig = None - - if ds.search_type() == "knn_filtered": - self.meta_b = ds.get_dataset_metadata() - self.meta_b.sort_indices() - - index = faiss.index_factory(ds.d, self.indexkey) - xb = ds.get_dataset() - print("train") - index.train(xb) - print("populate") - if self.binsig is None: - index.add(xb) - else: - ids = np.arange(ds.nb) | self.binsig.db_sig - index.add_with_ids(xb, ids) - - self.index = index - self.nb = ds.nb - self.xb = xb - self.ps = faiss.ParameterSpace() - self.ps.initialize(self.index) - print("store", self.index_name(dataset)) - faiss.write_index(index, self.index_name(dataset)) - - - def index_name(self, name): - return f"data/{name}.{self.indexkey}.faissindex" - - def binarysig_name(self, name): - return f"data/{name}.{self.indexkey}.binarysig" - - - def load_index(self, dataset): - """ - Load the index for dataset. Returns False if index - is not available, True otherwise. - - Checking the index usually involves the dataset name - and the index build paramters passed during construction. - """ - if not os.path.exists(self.index_name(dataset)): - if 'url' not in self._index_params: - return False - - print('Downloading index in background. This can take a while.') - download_accelerated(self._index_params['url'], self.index_name(dataset), quiet=True) - - print("Loading index") - - self.index = faiss.read_index(self.index_name(dataset)) - - self.ps = faiss.ParameterSpace() - self.ps.initialize(self.index) - - ds = DATASETS[dataset]() - - if ds.search_type() == "knn_filtered" and self.binarysig: - if not os.path.exists(self.binarysig_name(dataset)): - print("preparing binary signatures") - meta_b = ds.get_dataset_metadata() - self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) - else: - print("loading binary signatures") - self.binsig = pickle.load(open(self.binarysig_name(dataset), "rb")) - else: - self.binsig = None - - if ds.search_type() == "knn_filtered": - self.meta_b = ds.get_dataset_metadata() - self.meta_b.sort_indices() - - self.nb = ds.nb - self.xb = ds.get_dataset() - - return True - - def index_files_to_store(self, dataset): - """ - Specify a triplet with the local directory path of index files, - the common prefix name of index component(s) and a list of - index components that need to be uploaded to (after build) - or downloaded from (for search) cloud storage. - - For local directory path under docker environment, please use - a directory under - data/indices/track(T1 or T2)/algo.__str__()/DATASETS[dataset]().short_name() - """ - raise NotImplementedError() - - def query(self, X, k): - nq = X.shape[0] - self.I = -np.ones((nq, k), dtype='int32') - bs = 1024 - for i0 in range(0, nq, bs): - _, self.I[i0:i0+bs] = self.index.search(X[i0:i0+bs], k) - - - def filtered_query(self, X, filter, k): - print('running filtered query') - nq = X.shape[0] - self.I = -np.ones((nq, k), dtype='int32') - meta_b = self.meta_b - meta_q = filter - docs_per_word = meta_b.T.tocsr() - ndoc_per_word = docs_per_word.indptr[1:] - docs_per_word.indptr[:-1] - freq_per_word = ndoc_per_word / self.nb - - def process_one_row(q): - faiss.omp_set_num_threads(1) - qwords = csr_get_row_indices(meta_q, q) - assert qwords.size in (1, 2) - w1 = qwords[0] - freq = freq_per_word[w1] - if qwords.size == 2: - w2 = qwords[1] - freq *= freq_per_word[w2] - else: - w2 = -1 - if freq < self.metadata_threshold: - # metadata first - docs = csr_get_row_indices(docs_per_word, w1) - if w2 != -1: - docs = bow_id_selector.intersect_sorted( - docs, csr_get_row_indices(docs_per_word, w2)) - - assert len(docs) >= k, pdb.set_trace() - xb_subset = self.xb[docs] - _, Ii = faiss.knn(X[q : q + 1], xb_subset, k=k) - - self.I[q, :] = docs[Ii.ravel()] - else: - # IVF first, filtered search - sel = make_bow_id_selector(meta_b, self.binsig.id_mask if self.binsig else 0) - if self.binsig is None: - sel.set_query_words(int(w1), int(w2)) - else: - sel.set_query_words_mask( - int(w1), int(w2), self.binsig.query_signature(w1, w2)) - - params = faiss.SearchParametersIVF(sel=sel, nprobe=self.nprobe) - - _, Ii = self.index.search( - X[q:q+1], k, params=params - ) - Ii = Ii.ravel() - if self.binsig is None: - self.I[q] = Ii - else: - # we'll just assume there are enough results - # valid = Ii != -1 - # I[q, valid] = Ii[valid] & binsig.id_mask - self.I[q] = Ii & self.binsig.id_mask - - - if self.nt <= 1: - for q in range(nq): - process_one_row(q) - else: - faiss.omp_set_num_threads(self.nt) - pool = ThreadPool(self.nt) - list(pool.map(process_one_row, range(nq))) - - def get_results(self): - return self.I - - def set_query_arguments(self, query_args): - faiss.cvar.indexIVF_stats.reset() - if "nprobe" in query_args: - self.nprobe = query_args['nprobe'] - self.ps.set_index_parameters(self.index, f"nprobe={query_args['nprobe']}") - self.qas = query_args - else: - self.nprobe = 1 - if "mt_threshold" in query_args: - self.metadata_threshold = query_args['mt_threshold'] - else: - self.metadata_threshold = 1e-3 - - def __str__(self): - return f'Faiss({self.indexkey, self.qas})' - - \ No newline at end of file diff --git a/neurips23/filter/faissplus/Dockerfile b/neurips23/filter/faissplus/Dockerfile deleted file mode 100644 index 163391a8..00000000 --- a/neurips23/filter/faissplus/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM neurips23 - -RUN apt update && apt install -y wget swig -RUN wget https://repo.anaconda.com/archive/Anaconda3-2023.03-0-Linux-x86_64.sh -RUN bash Anaconda3-2023.03-0-Linux-x86_64.sh -b - -ENV PATH /root/anaconda3/bin:$PATH -ENV CONDA_PREFIX /root/anaconda3/ - -RUN conda install -c pytorch faiss-cpu -COPY install/requirements_conda.txt ./ -# conda doesn't like some of our packages, use pip -RUN python3 -m pip install -r requirements_conda.txt - -COPY neurips23/filter/faiss/bow_id_selector.swig ./ - -RUN swig -c++ -python -I$CONDA_PREFIX/include -Ifaiss bow_id_selector.swig -RUN g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ - -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ - -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so -Ifaiss - -RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' - - - diff --git a/neurips23/filter/faissplus/bow_id_selector.swig b/neurips23/filter/faissplus/bow_id_selector.swig deleted file mode 100644 index 6712aa25..00000000 --- a/neurips23/filter/faissplus/bow_id_selector.swig +++ /dev/null @@ -1,183 +0,0 @@ - -%module bow_id_selector - -/* -To compile when Faiss is installed via conda: - -swig -c++ -python -I$CONDA_PREFIX/include bow_id_selector.swig && \ -g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ - -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ - -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so - -*/ - - -// Put C++ includes here -%{ - -#include -#include - -%} - -// to get uint32_t and friends -%include - -// This means: assume what's declared in these .h files is provided -// by the Faiss module. -%import(module="faiss") "faiss/MetricType.h" -%import(module="faiss") "faiss/impl/IDSelector.h" - -// functions to be parsed here - -// This is important to release GIL and do Faiss exception handing -%exception { - Py_BEGIN_ALLOW_THREADS - try { - $action - } catch(faiss::FaissException & e) { - PyEval_RestoreThread(_save); - - if (PyErr_Occurred()) { - // some previous code already set the error type. - } else { - PyErr_SetString(PyExc_RuntimeError, e.what()); - } - SWIG_fail; - } catch(std::bad_alloc & ba) { - PyEval_RestoreThread(_save); - PyErr_SetString(PyExc_MemoryError, "std::bad_alloc"); - SWIG_fail; - } - Py_END_ALLOW_THREADS -} - - -// any class or function declared below will be made available -// in the module. -%inline %{ - -struct IDSelectorBOW : faiss::IDSelector { - size_t nb; - using TL = int32_t; - const TL *lims; - const int32_t *indices; - int32_t w1 = -1, w2 = -1; - - IDSelectorBOW( - size_t nb, const TL *lims, const int32_t *indices): - nb(nb), lims(lims), indices(indices) {} - - void set_query_words(int32_t w1, int32_t w2) { - this->w1 = w1; - this->w2 = w2; - } - - // binary search in the indices array - bool find_sorted(TL l0, TL l1, int32_t w) const { - while (l1 > l0 + 1) { - TL lmed = (l0 + l1) / 2; - if (indices[lmed] > w) { - l1 = lmed; - } else { - l0 = lmed; - } - } - return indices[l0] == w; - } - - bool is_member(faiss::idx_t id) const { - TL l0 = lims[id], l1 = lims[id + 1]; - if (l1 <= l0) { - return false; - } - if(!find_sorted(l0, l1, w1)) { - return false; - } - if(w2 >= 0 && !find_sorted(l0, l1, w2)) { - return false; - } - return true; - } - - ~IDSelectorBOW() override {} -}; - - -struct IDSelectorBOWBin : IDSelectorBOW { - /** with additional binary filtering */ - faiss::idx_t id_mask; - - IDSelectorBOWBin( - size_t nb, const TL *lims, const int32_t *indices, faiss::idx_t id_mask): - IDSelectorBOW(nb, lims, indices), id_mask(id_mask) {} - - faiss::idx_t q_mask = 0; - - void set_query_words_mask(int32_t w1, int32_t w2, faiss::idx_t q_mask) { - set_query_words(w1, w2); - this->q_mask = q_mask; - } - - bool is_member(faiss::idx_t id) const { - if (q_mask & ~id) { - return false; - } - return IDSelectorBOW::is_member(id & id_mask); - } - - ~IDSelectorBOWBin() override {} -}; - - -size_t intersect_sorted_c( - size_t n1, const int32_t *a1, - size_t n2, const int32_t *a2, - int32_t *res) -{ - if (n1 == 0 || n2 == 0) { - return 0; - } - size_t i1 = 0, i2 = 0, i = 0; - for(;;) { - if (a1[i1] < a2[i2]) { - i1++; - if (i1 >= n1) { - return i; - } - } else if (a1[i1] > a2[i2]) { - i2++; - if (i2 >= n2) { - return i; - } - } else { // equal - res[i++] = a1[i1++]; - i2++; - if (i1 >= n1 || i2 >= n2) { - return i; - } - } - } -} - -%} - - -%pythoncode %{ - -import numpy as np - -# example additional function that converts the passed-in numpy arrays to -# C++ pointers -def intersect_sorted(a1, a2): - n1, = a1.shape - n2, = a2.shape - res = np.empty(n1 + n2, dtype=a1.dtype) - nres = intersect_sorted_c( - n1, faiss.swig_ptr(a1), - n2, faiss.swig_ptr(a2), - faiss.swig_ptr(res) - ) - return res[:nres] - -%} \ No newline at end of file diff --git a/neurips23/filter/faissplus/config.yaml b/neurips23/filter/faissplus/config.yaml deleted file mode 100644 index ddf2eb93..00000000 --- a/neurips23/filter/faissplus/config.yaml +++ /dev/null @@ -1,66 +0,0 @@ -random-filter-s: - faissplus: - docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF1024,SQ8"}] - query-args: | - [{"nprobe": 1}, - {"nprobe":2}, - {"nprobe":4}] -random-s: - faissplus: - docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF1024,SQ8"}] - query-args: | - [{"nprobe": 1}, - {"nprobe":2}, - {"nprobe":4}] -yfcc-10M-unfiltered: - faissplus: - docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF16384,SQ8", "binarysig": true, "threads": 16}] - query-args: | - [{"nprobe": 1}, {"nprobe": 4}, {"nprobe": 16}, {"nprobe": 64}] -yfcc-10M: - faissplus: - docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF11264,SQ8", - "binarysig": true, - "threads": 16 - }] - query-args: | - [ - {"nprobe": 34, "mt_threshold": 0.00031}, - {"nprobe": 32, "mt_threshold": 0.0003}, - {"nprobe": 32, "mt_threshold": 0.00031}, - {"nprobe": 34, "mt_threshold": 0.0003}, - {"nprobe": 34, "mt_threshold": 0.00035}, - {"nprobe": 32, "mt_threshold": 0.00033}, - {"nprobe": 30, "mt_threshold": 0.00033}, - {"nprobe": 32, "mt_threshold": 0.00035}, - {"nprobe": 34, "mt_threshold": 0.00033}, - {"nprobe": 40, "mt_threshold": 0.0003} - ] diff --git a/neurips23/filter/faissplus/faiss.py b/neurips23/filter/faissplus/faiss.py deleted file mode 100644 index 02980d12..00000000 --- a/neurips23/filter/faissplus/faiss.py +++ /dev/null @@ -1,287 +0,0 @@ -import pdb -import pickle -import numpy as np -import os - -from multiprocessing.pool import ThreadPool - -import faiss - -from neurips23.filter.base import BaseFilterANN -from benchmark.datasets import DATASETS -from benchmark.dataset_io import download_accelerated - -import bow_id_selector - -def csr_get_row_indices(m, i): - """ get the non-0 column indices for row i in matrix m """ - return m.indices[m.indptr[i] : m.indptr[i + 1]] - -def make_bow_id_selector(mat, id_mask=0): - sp = faiss.swig_ptr - if id_mask == 0: - return bow_id_selector.IDSelectorBOW(mat.shape[0], sp(mat.indptr), sp(mat.indices)) - else: - return bow_id_selector.IDSelectorBOWBin( - mat.shape[0], sp(mat.indptr), sp(mat.indices), id_mask - ) - -def set_invlist_ids(invlists, l, ids): - n, = ids.shape - ids = np.ascontiguousarray(ids, dtype='int64') - assert invlists.list_size(l) == n - faiss.memcpy( - invlists.get_ids(l), - faiss.swig_ptr(ids), n * 8 - ) - - - -def csr_to_bitcodes(matrix, bitsig): - """ Compute binary codes for the rows of the matrix: each binary code is - the OR of bitsig for non-0 entries of the row. - """ - indptr = matrix.indptr - indices = matrix.indices - n = matrix.shape[0] - bit_codes = np.zeros(n, dtype='int64') - for i in range(n): - # print(bitsig[indices[indptr[i]:indptr[i + 1]]]) - bit_codes[i] = np.bitwise_or.reduce(bitsig[indices[indptr[i]:indptr[i + 1]]]) - return bit_codes - - -class BinarySignatures: - """ binary signatures that encode vectors """ - - def __init__(self, meta_b, proba_1): - nvec, nword = meta_b.shape - # number of bits reserved for the vector ids - self.id_bits = int(np.ceil(np.log2(nvec))) - # number of bits for the binary signature - self.sig_bits = nbits = 63 - self.id_bits - - # select binary signatures for the vocabulary - rs = np.random.RandomState(123) # we rely on this to be reproducible! - bitsig = np.packbits(rs.rand(nword, nbits) < proba_1, axis=1) - bitsig = np.pad(bitsig, ((0, 0), (0, 8 - bitsig.shape[1]))).view("int64").ravel() - self.bitsig = bitsig - - # signatures for all the metadata matrix - self.db_sig = csr_to_bitcodes(meta_b, bitsig) << self.id_bits - - # mask to keep only the ids - self.id_mask = (1 << self.id_bits) - 1 - - def query_signature(self, w1, w2): - """ compute the query signature for 1 or 2 words """ - sig = self.bitsig[w1] - if w2 != -1: - sig |= self.bitsig[w2] - return int(sig << self.id_bits) - -class FAISS(BaseFilterANN): - - def __init__(self, metric, index_params): - self._index_params = index_params - self._metric = metric - print(index_params) - self.indexkey = index_params.get("indexkey", "IVF32768,SQ8") - self.binarysig = index_params.get("binarysig", True) - self.binarysig_proba1 = index_params.get("binarysig_proba1", 0.1) - self.metadata_threshold = 1e-3 - self.nt = index_params.get("threads", 1) - - - def fit(self, dataset): - ds = DATASETS[dataset]() - if ds.search_type() == "knn_filtered" and self.binarysig: - print("preparing binary signatures") - meta_b = ds.get_dataset_metadata() - self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) - print("writing to", self.binarysig_name(dataset)) - pickle.dump(self.binsig, open(self.binarysig_name(dataset), "wb"), -1) - else: - self.binsig = None - - if ds.search_type() == "knn_filtered": - self.meta_b = ds.get_dataset_metadata() - self.meta_b.sort_indices() - - index = faiss.index_factory(ds.d, self.indexkey) - xb = ds.get_dataset() - print("train") - index.train(xb) - print("populate") - if self.binsig is None: - index.add(xb) - else: - ids = np.arange(ds.nb) | self.binsig.db_sig - index.add_with_ids(xb, ids) - - self.index = index - self.nb = ds.nb - self.xb = xb - self.ps = faiss.ParameterSpace() - self.ps.initialize(self.index) - print("store", self.index_name(dataset)) - faiss.write_index(index, self.index_name(dataset)) - - - def index_name(self, name): - return f"data/{name}.{self.indexkey}.faissindex" - - def binarysig_name(self, name): - return f"data/{name}.{self.indexkey}.binarysig" - - - def load_index(self, dataset): - """ - Load the index for dataset. Returns False if index - is not available, True otherwise. - - Checking the index usually involves the dataset name - and the index build paramters passed during construction. - """ - if not os.path.exists(self.index_name(dataset)): - if 'url' not in self._index_params: - return False - - print('Downloading index in background. This can take a while.') - download_accelerated(self._index_params['url'], self.index_name(dataset), quiet=True) - - print("Loading index") - - self.index = faiss.read_index(self.index_name(dataset)) - - self.ps = faiss.ParameterSpace() - self.ps.initialize(self.index) - - ds = DATASETS[dataset]() - - if ds.search_type() == "knn_filtered" and self.binarysig: - if not os.path.exists(self.binarysig_name(dataset)): - print("preparing binary signatures") - meta_b = ds.get_dataset_metadata() - self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) - else: - print("loading binary signatures") - self.binsig = pickle.load(open(self.binarysig_name(dataset), "rb")) - else: - self.binsig = None - - if ds.search_type() == "knn_filtered": - self.meta_b = ds.get_dataset_metadata() - self.meta_b.sort_indices() - - self.nb = ds.nb - self.xb = ds.get_dataset() - - return True - - def index_files_to_store(self, dataset): - """ - Specify a triplet with the local directory path of index files, - the common prefix name of index component(s) and a list of - index components that need to be uploaded to (after build) - or downloaded from (for search) cloud storage. - - For local directory path under docker environment, please use - a directory under - data/indices/track(T1 or T2)/algo.__str__()/DATASETS[dataset]().short_name() - """ - raise NotImplementedError() - - def query(self, X, k): - nq = X.shape[0] - self.I = -np.ones((nq, k), dtype='int32') - bs = 1024 - for i0 in range(0, nq, bs): - _, self.I[i0:i0+bs] = self.index.search(X[i0:i0+bs], k) - - - def filtered_query(self, X, filter, k): - print('running filtered query') - nq = X.shape[0] - self.I = -np.ones((nq, k), dtype='int32') - meta_b = self.meta_b - meta_q = filter - docs_per_word = meta_b.T.tocsr() - ndoc_per_word = docs_per_word.indptr[1:] - docs_per_word.indptr[:-1] - freq_per_word = ndoc_per_word / self.nb - - def process_one_row(q): - faiss.omp_set_num_threads(1) - qwords = csr_get_row_indices(meta_q, q) - assert qwords.size in (1, 2) - w1 = qwords[0] - freq = freq_per_word[w1] - if qwords.size == 2: - w2 = qwords[1] - freq *= freq_per_word[w2] - else: - w2 = -1 - if freq < self.metadata_threshold: - # metadata first - docs = csr_get_row_indices(docs_per_word, w1) - if w2 != -1: - docs = bow_id_selector.intersect_sorted( - docs, csr_get_row_indices(docs_per_word, w2)) - - assert len(docs) >= k, pdb.set_trace() - xb_subset = self.xb[docs] - _, Ii = faiss.knn(X[q : q + 1], xb_subset, k=k) - - self.I[q, :] = docs[Ii.ravel()] - else: - # IVF first, filtered search - sel = make_bow_id_selector(meta_b, self.binsig.id_mask if self.binsig else 0) - if self.binsig is None: - sel.set_query_words(int(w1), int(w2)) - else: - sel.set_query_words_mask( - int(w1), int(w2), self.binsig.query_signature(w1, w2)) - - params = faiss.SearchParametersIVF(sel=sel, nprobe=self.nprobe) - - _, Ii = self.index.search( - X[q:q+1], k, params=params - ) - Ii = Ii.ravel() - if self.binsig is None: - self.I[q] = Ii - else: - # we'll just assume there are enough results - # valid = Ii != -1 - # I[q, valid] = Ii[valid] & binsig.id_mask - self.I[q] = Ii & self.binsig.id_mask - - - if self.nt <= 1: - for q in range(nq): - process_one_row(q) - else: - faiss.omp_set_num_threads(self.nt) - pool = ThreadPool(self.nt) - list(pool.map(process_one_row, range(nq))) - - def get_results(self): - return self.I - - def set_query_arguments(self, query_args): - faiss.cvar.indexIVF_stats.reset() - if "nprobe" in query_args: - self.nprobe = query_args['nprobe'] - self.ps.set_index_parameters(self.index, f"nprobe={query_args['nprobe']}") - self.qas = query_args - else: - self.nprobe = 1 - if "mt_threshold" in query_args: - self.metadata_threshold = query_args['mt_threshold'] - else: - self.metadata_threshold = 1e-3 - - def __str__(self): - return f'Faiss({self.indexkey, self.qas})' - - \ No newline at end of file diff --git a/neurips23/filter/fdufilterdiskann/Dockerfile b/neurips23/filter/fdufilterdiskann/Dockerfile new file mode 100644 index 00000000..0b0ee1c6 --- /dev/null +++ b/neurips23/filter/fdufilterdiskann/Dockerfile @@ -0,0 +1,17 @@ +FROM neurips23 + +RUN apt update +RUN apt install -y software-properties-common +RUN add-apt-repository -y ppa:git-core/ppa +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive apt install -y git make cmake g++ libaio-dev libgoogle-perftools-dev libunwind-dev clang-format libboost-dev libboost-program-options-dev libmkl-full-dev libcpprest-dev python3.10 + +# COPY FilterDiskann /home/app/FilterDiskann +WORKDIR /home/app +RUN git clone --recursive --branch main https://github.com/PUITAR/FduFilterDiskANN.git +WORKDIR /home/app/FduFilterDiskANN/pybindings + +RUN pip3 install virtualenv build +RUN pip3 install pybind11[global] +RUN pip3 install . +WORKDIR /home/app diff --git a/neurips23/filter/fdufilterdiskann/config.yaml b/neurips23/filter/fdufilterdiskann/config.yaml new file mode 100644 index 00000000..77b559a3 --- /dev/null +++ b/neurips23/filter/fdufilterdiskann/config.yaml @@ -0,0 +1,35 @@ +random-filter-s: + fdufilterdiskann: + docker-tag: neurips23-filter-fdufilterdiskann + module: neurips23.filter.fdufilterdiskann.fdufilterdiskann + constructor: fdufilterdiskann + base-args: ["@metric"] + run-groups: + base: + args: | + [{"R":2, "L":10, "buildthreads":16, "alpha":1.2}] + query-args: | + [{"Ls":10, "T":1, "threshold_1":20000, "threshold_2":40000}] +yfcc-10M: + fdufilterdiskann: + docker-tag: neurips23-filter-fdufilterdiskann + module: neurips23.filter.fdufilterdiskann.fdufilterdiskann + constructor: fdufilterdiskann + base-args: ["@metric"] + run-groups: + base: + args: | + [{"R":60, "L":80, "buildthreads":16, "alpha":1.0}] + query-args: | + [ + {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":30000}, + {"Ls":10, "T":16, "threshold_1":30000, "threshold_2":40000}, + {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, + {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, + {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, + {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, + {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, + {"Ls":20, "T":16, "threshold_1":20000, "threshold_2":40000}, + {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, + {"Ls":15, "T":16, "threshold_1":20000, "threshold_2":40000} + ] diff --git a/neurips23/filter/fdufilterdiskann/fdufilterdiskann.py b/neurips23/filter/fdufilterdiskann/fdufilterdiskann.py new file mode 100644 index 00000000..6cb657fa --- /dev/null +++ b/neurips23/filter/fdufilterdiskann/fdufilterdiskann.py @@ -0,0 +1,178 @@ +from __future__ import absolute_import +import os +import time +import numpy as np + +# import diskannpy +import filterdiskann + +from neurips23.filter.base import BaseFilterANN +from benchmark.datasets import DATASETS, download_accelerated +from multiprocessing.pool import ThreadPool + + +def csr_get_row_indices(m, i): + """ get the non-0 column indices for row i in matrix m """ + return m.indices[m.indptr[i] : m.indptr[i + 1]] + + +class fdufilterdiskann(BaseFilterANN): + def __init__(self, metric, index_params): + self.name = "fdufilterdiskann" + if index_params.get("R") == None: + print("Error: missing parameter R") + return + self._index_params = index_params + self._metric = metric + + self.R = index_params.get("R") + self.L = index_params.get("L") + self.nt = index_params.get("buildthreads", 1) + + + def index_name(self): + return f"R{self.R}_L{self.L}" + + def create_index_dir(self, dataset): + index_dir = os.path.join(os.getcwd(), "data", "indices", "filter") + os.makedirs(index_dir, mode=0o777, exist_ok=True) + index_dir = os.path.join(index_dir, "fdufilterdiskann") + os.makedirs(index_dir, mode=0o777, exist_ok=True) + index_dir = os.path.join(index_dir, dataset.short_name()) + os.makedirs(index_dir, mode=0o777, exist_ok=True) + index_dir = os.path.join(index_dir, self.index_name()) + os.makedirs(index_dir, mode=0o777, exist_ok=True) + return index_dir + + def translate_dist_fn(self, metric): + if metric == 'euclidean': + return 'l2' + elif metric == 'ip': + return 'mips' + else: + raise Exception('Invalid metric') + + def translate_dtype(self, dtype:str): + if dtype == 'uint8': + return np.uint8 + elif dtype == 'int8': + return np.int8 + elif dtype == 'float32': + return np.float32 + else: + raise Exception('Invalid data type') + + def fit(self, dataset): + """ + Build the index for the data points given in dataset name. + """ + + ds = DATASETS[dataset]() + d = ds.d + + buildthreads = self._index_params.get("buildthreads", 1) + self.nt = buildthreads + self.meta_b = ds.get_dataset_metadata() + self.meta_b.sort_indices() # 对每行的列排序 + index_dir = self.create_index_dir(ds) + + if hasattr(self, "index"): + print("Index object exists already") + return + + print(ds.get_dataset_fn()) + + start = time.time() + + print(ds.ds_metadata_fn) + + print("building index") + alpha = self._index_params.get("alpha", 1.2) + filterdiskann.build(ds.get_dataset_fn(), index_dir + '/' + self.index_name(), os.path.join(ds.basedir, ds.ds_metadata_fn), + buildthreads, self.R, self.L, alpha) + + end = time.time() + print("DiskANN index built in %.3f s" % (end - start)) + + + print('Loading index..') + search_threads, search_L = self._index_params.get("T", 16), self._index_params.get("Ls", 20) + print('search threads:', search_threads, 'search L:', search_L) + self.index = filterdiskann.FilterDiskANN(filterdiskann.Metric.L2, index_dir + '/' + self.index_name(), ds.nb, + ds.d, search_threads, search_L) + print('Index ready for search') + + def get_index_components(self, dataset): + index_components = ['', '.data'] + ds = DATASETS[dataset]() + if ds.distance() == "ip": + index_components = index_components + [] + return index_components + + def index_files_to_store(self, dataset): + return [self.create_index_dir(DATASETS[dataset]()), self.index_name(), self.get_index_components(dataset)] + + def load_index(self, dataset): + """ + Load the index for dataset. Returns False if index + is not available, True otherwise. + + Checking the index usually involves the dataset name + and the index build paramters passed during construction. + """ + ds = DATASETS[dataset]() + index_dir = self.create_index_dir(ds) + if not (os.path.exists(index_dir)) and 'url' not in self._index_params: + return False + + index_path = os.path.join(index_dir, self.index_name()) + index_components = self.get_index_components(dataset) + + for component in index_components: + index_file = index_path + component + if not (os.path.exists(index_file)): + if 'url' in self._index_params: + index_file_source = self._index_params['url'] + '/' + self.index_name() + component + print(f"Downloading index in background. This can take a while.") + download_accelerated(index_file_source, index_file, quiet=True) + else: + return False + + print("Loading index") + + search_threads, search_L = self._index_params.get("T", 16), self._index_params.get("Ls", 20) + + print('search threads:', search_threads, 'search L:', search_L) + self.index = filterdiskann.FilterDiskANN(filterdiskann.Metric.L2, index_dir + '/' + self.index_name(), ds.nb, + ds.d, search_threads, search_L) + print ("Load index success.") + return True + + + def set_query_arguments(self, query_args): + self._query_args = query_args + self.Ls = 0 if query_args.get("Ls") == None else query_args.get("Ls") + self.search_threads = self._query_args.get("T") + + def filtered_query(self, X, filter, k): + print('running filtered query diskann') + nq = X.shape[0] + self.I = np.zeros((nq, k), dtype='uint32', order='C') # result IDs + # # meta_b = self.meta_b # data_metadata + meta_q = filter # query_metadata + threshold_1 = self._query_args.get("threshold_1") + threshold_2 = self._query_args.get("threshold_2") + + print("runing in ", self.search_threads, ' nq:', nq) + + self.index.search(X, meta_q.indptr, meta_q.indices, nq, k, self.Ls, self.search_threads, threshold_1, threshold_2, self.I) + + + def get_results(self): + return self.I + + def __str__(self): + return f"diskann({self.index_name(), self._query_args})" + +if __name__ == "__main__": + print(1) \ No newline at end of file diff --git a/neurips23/filter/wm_filter/Dockerfile b/neurips23/filter/wm_filter/Dockerfile deleted file mode 100644 index cf63f4a4..00000000 --- a/neurips23/filter/wm_filter/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM neurips23 - -RUN apt-get update; DEBIAN_FRONTEND=noninteractive apt install intel-mkl python3-setuptools wget python3-matplotlib build-essential checkinstall libssl-dev swig4.0 python3-dev python3-numpy python3-numpy-dev -y -COPY install/requirements_conda.txt ./ -# conda doesn't like some of our packages, use pip -RUN python3 -m pip install -r requirements_conda.txt - - -# CMAKE with good enough version -RUN mkdir /build && wget https://github.com/Kitware/CMake/archive/refs/tags/v3.27.1.tar.gz && mv v3.27.1.tar.gz /build -RUN cd /build; tar -zxvf v3.27.1.tar.gz -RUN cd /build/CMake-3.27.1 && ./bootstrap && make && make install - - -RUN cd / && git clone https://github.com/alemagnani/faiss.git && cd /faiss && git pull && git checkout wm_filter - -RUN cd /faiss && rm -rf ./build -RUN cd /faiss/; cmake -B build /faiss/ -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DFAISS_OPT_LEVEL=avx2 -DBLA_VENDOR=Intel10_64_dyn -DBUILD_TESTING=ON -DPython_EXECUTABLE=/usr/bin/python3 -DMKL_LIBRARIES=/usr/lib/x86_64-linux-gnu/libmkl_rt.so -RUN cd /faiss/; make -C build -j faiss faiss_avx2 swigfaiss swigfaiss_avx2 -RUN (cd /faiss/build/faiss/python && python3 setup.py install) - -#RUN pip install tritonclient[all] -ENV PYTHONPATH=/faiss/build/faiss/python/build/lib/ - -RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' - - - diff --git a/neurips23/filter/wm_filter/README.md b/neurips23/filter/wm_filter/README.md deleted file mode 100644 index 0d451065..00000000 --- a/neurips23/filter/wm_filter/README.md +++ /dev/null @@ -1,7 +0,0 @@ - -### Submission for Neurips23 Filter track of WM_filter team -This submission leverages the IVF index to run the filter in a fast way. - -More info to come... - - diff --git a/neurips23/filter/wm_filter/config.yaml b/neurips23/filter/wm_filter/config.yaml deleted file mode 100644 index 4397152e..00000000 --- a/neurips23/filter/wm_filter/config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -random-filter-s: - wm_filter: - docker-tag: neurips23-filter-wm_filter - module: neurips23.filter.wm_filter.wm_filter - constructor: FAISS - base-args: [ "@metric" ] - run-groups: - base: - args: | - [{"indexkey": "IVF1024,SQ8", - "threads": 8, - "train_size": 2000000, - "type": "direct" - }] - query-args: | - [ - {"nprobe": 80, "max_codes": 100, "selector_probe_limit": 80}, - {"nprobe": 100, "max_codes": 500, "selector_probe_limit": 100}, - {"nprobe": 120, "max_codes": 1000, "selector_probe_limit": 120}, - {"nprobe": 140, "max_codes": 1800, "selector_probe_limit": 140}, - {"nprobe": 160, "max_codes": 500, "selector_probe_limit": 160}, - {"nprobe": 70, "max_codes": 1000, "selector_probe_limit": 70} - ] -yfcc-10M: - wm_filter: - docker-tag: neurips23-filter-wm_filter - module: neurips23.filter.wm_filter.wm_filter - constructor: FAISS - base-args: [ "@metric" ] - run-groups: - base: - args: | - [{"indexkey": "IVF1024,SQ8", - "threads": 8, - "train_size": 2000000, - "type": "direct" - }] - query-args: | - [ - {"nprobe": 80, "max_codes": 1800, "selector_probe_limit": 80}, - {"nprobe": 100, "max_codes": 1800, "selector_probe_limit": 100}, - {"nprobe": 120, "max_codes": 1800, "selector_probe_limit": 120}, - {"nprobe": 140, "max_codes": 1800, "selector_probe_limit": 140}, - {"nprobe": 160, "max_codes": 1800, "selector_probe_limit": 160}, - {"nprobe": 70, "max_codes": 2100, "selector_probe_limit": 70}, - {"nprobe": 100, "max_codes": 2100, "selector_probe_limit": 100}, - {"nprobe": 130, "max_codes": 2100, "selector_probe_limit": 130}, - {"nprobe": 160, "max_codes": 2100, "selector_probe_limit": 160}, - {"nprobe": 200, "max_codes": 2100, "selector_probe_limit": 200} - ] diff --git a/neurips23/filter/wm_filter/wm_filter.py b/neurips23/filter/wm_filter/wm_filter.py deleted file mode 100644 index 671b19de..00000000 --- a/neurips23/filter/wm_filter/wm_filter.py +++ /dev/null @@ -1,572 +0,0 @@ -import pdb -import pickle -import numpy as np -import os - -from multiprocessing.pool import ThreadPool -from threading import current_thread - -import faiss - - -from faiss.contrib.inspect_tools import get_invlist -from neurips23.filter.base import BaseFilterANN -from benchmark.datasets import DATASETS -from benchmark.dataset_io import download_accelerated -from math import log10, pow - - -def csr_get_row_indices(m, i): - """ get the non-0 column indices for row i in matrix m """ - return m.indices[m.indptr[i] : m.indptr[i + 1]] - -def make_id_selector_ivf_two(docs_per_word): - sp = faiss.swig_ptr - return faiss.IDSelectorIVFTwo(sp(docs_per_word.indices), sp(docs_per_word.indptr)) - -def make_id_selector_cluster_aware(indices, limits, clusters, cluster_limits): - sp = faiss.swig_ptr - return faiss.IDSelectorIVFClusterAware(sp(indices), sp(limits), sp(clusters), sp(cluster_limits)) - -def make_id_selector_cluster_aware_intersect(indices, limits, clusters, cluster_limits, tmp_size): - sp = faiss.swig_ptr - return faiss.IDSelectorIVFClusterAwareIntersect(sp(indices), sp(limits), sp(clusters), sp(cluster_limits), int(tmp_size)) - -def make_id_selector_cluster_aware_direct(id_position_in_cluster, limits, clusters, cluster_limits, tmp_size): - sp = faiss.swig_ptr - return faiss.IDSelectorIVFClusterAwareIntersectDirect(sp(id_position_in_cluster), sp(limits), sp(clusters), sp(cluster_limits), int(tmp_size)) - -def make_id_selector_cluster_aware_direct_exp(id_position_in_cluster, limits, nprobes, tmp_size): - sp = faiss.swig_ptr - return faiss.IDSelectorIVFClusterAwareIntersectDirectExp(sp(id_position_in_cluster), sp(limits), int(nprobes), int(tmp_size)) - - -def find_invlists(index): - try: - inverted_lists = index.invlists - except: - base_index = faiss.downcast_index(index.base_index) - print('cannot find the inverted list trying one level down') - print('type of index', type(base_index)) - inverted_lists = base_index.invlists - return inverted_lists - -def print_stats(): - m = 1000000. - intersection = faiss.cvar.IDSelectorMy_Stats.intersection/m - find_cluster = faiss.cvar.IDSelectorMy_Stats.find_cluster/m - set_list_time = faiss.cvar.IDSelectorMy_Stats.set_list_time/m - scan_codes = faiss.cvar.IDSelectorMy_Stats.scan_codes/m - one_list = faiss.cvar.IDSelectorMy_Stats.one_list/m - extra = faiss.cvar.IDSelectorMy_Stats.extra / m - inter_plus_find = intersection + find_cluster - print('intersection: {}, find_cluster: {}, intersection+ find cluster: {}, set list time: {}, scan_codes: {}, one list: {}, extra: {}'.format(intersection, find_cluster, inter_plus_find, set_list_time, scan_codes, one_list, extra)) - - -def spot_check_filter(docs_per_word, index, indices, limits, clusters, cluster_limits): - print('running spot check') - - - inverted_lists = find_invlists(index) - - from_id_to_map = dict() - for i in range(inverted_lists.nlist): - list_ids, _ = get_invlist(inverted_lists, i) - for id in list_ids: - from_id_to_map[id] = i - - indptr = docs_per_word.indptr - - ## lets' run some spot check - for word in [0, 5, 7]: - #for word in range(docs_per_word.shape[0]): - #for word in [docs_per_word.shape[0]-1 ]: - c_start = cluster_limits[word] - c_end = cluster_limits[word + 1] - assert c_end >= c_start - - start = indptr[word] - end = indptr[word + 1] - ids_in_word = {id for id in docs_per_word.indices[start:end]} - - cluster_base = -1 - for pos, cluster in enumerate(clusters[c_start: c_end]): - if cluster_base == -1: - cluster_base = cluster - else: - assert cluster != cluster_base - cluster_base = cluster - for id in indices[limits[c_start + pos]: limits[c_start + pos + 1]]: - assert from_id_to_map[id] == cluster - assert id in ids_in_word - ids_in_word.remove(id) - assert len(ids_in_word) == 0 # we should have covered all the ids in the word with the clusters - - -def spot_check_filter_exp(docs_per_word, index, indices, limits): - print('running spot check') - - - inverted_lists = find_invlists(index) - - from_id_to_map = dict() - for i in range(inverted_lists.nlist): - list_ids, _ = get_invlist(inverted_lists, i) - for id in list_ids: - from_id_to_map[id] = i - - indptr = docs_per_word.indptr - - nprobes = inverted_lists.nlist - - ## lets' run some spot check - for word in [0, 5000, 12124, 151123, 198000]: - #for word in range(docs_per_word.shape[0]): - #for word in [docs_per_word.shape[0]-1 ]: - local_ids_to_cluster = dict() - #print(limits[nprobes * word: nprobes * word + nprobes]) - for cluster in range(nprobes): - c_start = limits[word * nprobes + cluster] - c_end = limits[word * nprobes + cluster+1] - - if c_end >=0 and c_start >=0 and c_end > c_start: - for id in indices[c_start: c_end]: - local_ids_to_cluster[id] = cluster - - - - start = indptr[word] - end = indptr[word + 1] - ids_in_word = {id for id in docs_per_word.indices[start:end]} - print(len(ids_in_word), len(local_ids_to_cluster)) - assert len(ids_in_word) == len(local_ids_to_cluster) - for id in ids_in_word: - cluster_found = from_id_to_map[id] - assert cluster_found == local_ids_to_cluster[id] - print('done checking word ', word) - - print('done spot check') - - -def find_max_interval(limits): - - out = -1 - for i in range(len(limits)-1): - delta = limits[i+1] - limits[i] - if delta > out: - out = delta - return out - - -def prepare_filter_by_cluster(docs_per_word, index): - print('creating filter cluster') - inverted_lists = find_invlists(index) - from_id_to_map = dict() - from_id_to_pos = dict() - for i in range(inverted_lists.nlist): - list_ids, _ = get_invlist(inverted_lists, i) - for pos, id in enumerate(list_ids): - #print('list: ', i, "id: ", id, "pos: ",pos) - from_id_to_map[id] = i - from_id_to_pos[id] = pos - print('loaded the mapping with {} entries'.format(len(from_id_to_map))) - - ## reorganize the docs per word - # - cluster_limits = [0] - clusters = list() - limits = list() - id_position_in_cluster = list() - - indices = np.array(docs_per_word.indices) - indptr = docs_per_word.indptr - for word in range(docs_per_word.shape[0]): - start = indptr[word] - end = indptr[word + 1] - if word % 10000 == 0: - print('processed {} words'.format(word)) - array_ind_cluster = [(id, from_id_to_map[id]) for id in indices[start:end]] - array_ind_cluster.sort(key=lambda x: x[1]) - - if len(array_ind_cluster) == 0: - pass - local_clusters = [] - local_limits = [] - current_cluster = -1 - for pos, arr in enumerate(array_ind_cluster): - id, cluster = arr - if current_cluster == -1 or cluster != current_cluster: - current_cluster = cluster - local_clusters.append(cluster) - local_limits.append(start + pos) - indices[start + pos] = id - id_position_in_cluster.append(from_id_to_pos[id]) - - clusters.extend(local_clusters) - limits.extend(local_limits) - new_cluster_limit = len(local_clusters) + cluster_limits[-1] - cluster_limits.append( new_cluster_limit) - limits.append(len(indices)) - - clusters = np.array(clusters, dtype=np.int16) - limits = np.array(limits, dtype=np.int32) - cluster_limits = np.array(cluster_limits, dtype=np.int32) - id_position_in_cluster = np.array(id_position_in_cluster, dtype=np.int32) - - return indices, limits, clusters, cluster_limits, id_position_in_cluster - - -def prepare_filter_by_cluster_exp(docs_per_word, index): - print('creating filter cluster expanded') - inverted_lists = find_invlists(index) - from_id_to_map = dict() - from_id_to_pos = dict() - - nprobes = inverted_lists.nlist - for i in range(inverted_lists.nlist): - list_ids, _ = get_invlist(inverted_lists, i) - for pos, id in enumerate(list_ids): - #print('list: ', i, "id: ", id, "pos: ",pos) - from_id_to_map[id] = i - from_id_to_pos[id] = pos - print('loaded the mapping with {} entries'.format(len(from_id_to_map))) - - ## reorganize the docs per word - # - - limits = -np.ones( (docs_per_word.shape[0] * nprobes + 1,), dtype=np.int32) - id_position_in_cluster = list() - - indices = np.array(docs_per_word.indices) - indptr = docs_per_word.indptr - for word in range(docs_per_word.shape[0]): - start = indptr[word] - end = indptr[word + 1] - if word % 10000 == 0: - print('processed {} words'.format(word)) - array_ind_cluster = [(id, from_id_to_map[id]) for id in indices[start:end]] - array_ind_cluster.sort(key=lambda x: x[1]) - - - - local_limits = [] - current_cluster = -1 - - for pos, arr in enumerate(array_ind_cluster): - id, cluster = arr - if current_cluster == -1 or cluster != current_cluster: - - if current_cluster != -1: - limits[word * nprobes + current_cluster + 1] = start + pos - - - current_cluster = cluster - local_limits.append(start + pos) - - limits[word * nprobes + current_cluster] = start + pos - - indices[start + pos] = id - id_position_in_cluster.append(from_id_to_pos[id]) - - limits[word * nprobes + current_cluster + 1] = start + len(array_ind_cluster) - - - limits = np.array(limits, dtype=np.int32) - - id_position_in_cluster = np.array(id_position_in_cluster, dtype=np.int32) - - return indices, limits, id_position_in_cluster, nprobes - - -class FAISS(BaseFilterANN): - - def __init__(self, metric, index_params): - self._index_params = index_params - self._metric = metric - - self.train_size = index_params.get('train_size', None) - self.indexkey = index_params.get("indexkey", "IVF32768,SQ8") - self.metadata_threshold = 1e-3 - self.nt = index_params.get("threads", 1) - self.type = index_params.get("type", "intersect") - - self.clustet_dist = [] - - - def fit(self, dataset): - faiss.omp_set_num_threads(self.nt) - ds = DATASETS[dataset]() - - print('the size of the index', ds.d) - index = faiss.index_factory(ds.d, self.indexkey) - xb = ds.get_dataset() - - print("train") - print('train_size', self.train_size) - if self.train_size is not None: - x_train = xb[:self.train_size] - else: - x_train = xb - index.train(x_train) - print("populate") - - bs = 1024 - for i0 in range(0, ds.nb, bs): - index.add(xb[i0: i0 + bs]) - - - print('ids added') - self.index = index - self.nb = ds.nb - self.xb = xb - self.ps = faiss.ParameterSpace() - self.ps.initialize(self.index) - print("store", self.index_name(dataset)) - faiss.write_index(index, self.index_name(dataset)) - - if ds.search_type() == "knn_filtered": - words_per_doc = ds.get_dataset_metadata() - words_per_doc.sort_indices() - self.docs_per_word = words_per_doc.T.tocsr() - self.docs_per_word.sort_indices() - self.ndoc_per_word = self.docs_per_word.indptr[1:] - self.docs_per_word.indptr[:-1] - self.freq_per_word = self.ndoc_per_word / self.nb - del words_per_doc - - if self.type == 'exp': - self.indices, self.limits, self.id_position_in_cluster, self.total_clusters = prepare_filter_by_cluster_exp( - self.docs_per_word, self.index) - pickle.dump( - (self.indices, self.limits, self.id_position_in_cluster, self.total_clusters ), - open(self.cluster_sig_name(dataset), "wb"), -1) - #spot_check_filter_exp(self.docs_per_word, self.index, self.indices, self.limits) - else: - self.indices, self.limits, self.clusters, self.cluster_limits, self.id_position_in_cluster = prepare_filter_by_cluster(self.docs_per_word, self.index) - print('dumping cluster map') - pickle.dump((self.indices, self.limits, self.clusters, self.cluster_limits, self.id_position_in_cluster), open(self.cluster_sig_name(dataset), "wb"), -1) - #spot_check_filter(self.docs_per_word, self.index, self.indices, self.limits, self.clusters, - # self.cluster_limits) - - self.max_range = find_max_interval(self.limits) - print('the max range is {}'.format(self.max_range)) - - def index_name(self, name): - - if self.type == 'exp': - return f"data/{name}.{self.indexkey}_exp_wm.faissindex" - else: - return f"data/{name}.{self.indexkey}_wm.faissindex" - - - def cluster_sig_name(self, name): - if self.type == 'exp': - return f"data/{name}.{self.indexkey}_exp_cluster_wm.pickle" - return f"data/{name}.{self.indexkey}_cluster_wm.pickle" - - - def get_probes(self, freq, a, b, min_prob = 4, max_prob=256): - #print("b: ", b) - probes = int( pow(2, - a * log10(freq )+ b)) - probes = max(min_prob, probes) - probes = min(max_prob, probes) - return probes - - def load_index(self, dataset): - """ - Load the index for dataset. Returns False if index - is not available, True otherwise. - - Checking the index usually involves the dataset name - and the index build paramters passed during construction. - """ - if not os.path.exists(self.index_name(dataset)): - if 'url' not in self._index_params: - return False - - print('Downloading index in background. This can take a while.') - download_accelerated(self._index_params['url'], self.index_name(dataset), quiet=True) - - print("Loading index") - ds = DATASETS[dataset]() - self.nb = ds.nb - self.xb = ds.get_dataset() - - if ds.search_type() == "knn_filtered": - words_per_doc = ds.get_dataset_metadata() - words_per_doc.sort_indices() - self.docs_per_word = words_per_doc.T.tocsr() - self.docs_per_word.sort_indices() - self.ndoc_per_word = self.docs_per_word.indptr[1:] - self.docs_per_word.indptr[:-1] - self.freq_per_word = self.ndoc_per_word / self.nb - del words_per_doc - - self.index = faiss.read_index(self.index_name(dataset)) - - if ds.search_type() == "knn_filtered": - if os.path.isfile( self.cluster_sig_name(dataset)): - print('loading cluster file') - if self.type == 'exp': - self.indices, self.limits, self.id_position_in_cluster, self.total_clusters = pickle.load( - open(self.cluster_sig_name(dataset), "rb")) - #spot_check_filter_exp(self.docs_per_word, self.index, self.indices, self.limits) - - else: - self.indices, self.limits, self.clusters, self.cluster_limits, self.id_position_in_cluster = pickle.load(open(self.cluster_sig_name(dataset), "rb")) - else: - print('cluster file not found') - if self.type == 'exp': - self.indices, self.limits, self.id_position_in_cluster, self.total_clusters = prepare_filter_by_cluster_exp( - self.docs_per_word, self.index) - pickle.dump( - (self.indices, self.limits, self.id_position_in_cluster, self.total_clusters ), - open(self.cluster_sig_name(dataset), "wb"), -1) - #spot_check_filter_exp(self.docs_per_word, self.index, self.indices, self.limits) - - else: - self.indices, self.limits, self.clusters, self.cluster_limits, self.id_position_in_cluster = prepare_filter_by_cluster(self.docs_per_word, self.index) - pickle.dump((self.indices, self.limits, self.clusters, self.cluster_limits, self.id_position_in_cluster), open(self.cluster_sig_name(dataset), "wb"), -1) - - #spot_check_filter(self.docs_per_word, self.index, self.indices, self.limits, self.clusters, self.cluster_limits) - - self.max_range = find_max_interval(self.limits) - print('the max range is {}'.format(self.max_range)) - - self.ps = faiss.ParameterSpace() - self.ps.initialize(self.index) - - - # delete not necessary data - del self.xb - del ds - if self.type == "exp" or self.type == 'direct': - print(" deleting indices") - del self.indices - #del self.docs_per_word - return True - - def index_files_to_store(self, dataset): - """ - Specify a triplet with the local directory path of index files, - the common prefix name of index component(s) and a list of - index components that need to be uploaded to (after build) - or downloaded from (for search) cloud storage. - - For local directory path under docker environment, please use - a directory under - data/indices/track(T1 or T2)/algo.__str__()/DATASETS[dataset]().short_name() - """ - raise NotImplementedError() - - def query(self, X, k): - nq = X.shape[0] - self.I = -np.ones((nq, k), dtype='int32') - bs = 1024 - - try: - print('k_factor', self.index.k_factor) - self.index.k_factor = self.k_factor - except Exception as e: - print(e) - pass - for i0 in range(0, nq, bs): - _, self.I[i0:i0+bs] = self.index.search(X[i0:i0+bs], k) - - - - def filtered_query(self, X, filter, k): - - # try: - # self.index.k_factor = self.k_factor - # except Exception as e: - # pass - - nq = X.shape[0] - self.I = -np.ones((nq, k), dtype='int32') - - meta_q = filter - selector_by_thread = dict() - - def process_one_row(q): - faiss.omp_set_num_threads(1) - thread = current_thread() - - qwords = csr_get_row_indices(meta_q, q) - w1 = qwords[0] - if qwords.size == 2: - w2 = qwords[1] - else: - w2 = -1 - - if thread not in selector_by_thread: - - sel = make_id_selector_cluster_aware_direct(self.id_position_in_cluster, self.limits, self.clusters, - self.cluster_limits, self.max_range) - # # IVF first, filtered search - # if self.type == 'simple': - # sel = make_id_selector_ivf_two(self.docs_per_word) - # elif self.type == "aware": - # sel = make_id_selector_cluster_aware(self.indices, self.limits, self.clusters, self.cluster_limits) - # elif self.type == 'intersect': - # sel = make_id_selector_cluster_aware_intersect(self.indices, self.limits, self.clusters, self.cluster_limits, self.max_range) - # elif self.type == 'direct': - # sel = make_id_selector_cluster_aware_direct(self.id_position_in_cluster, self.limits, self.clusters, - # self.cluster_limits, self.max_range) - # elif self.type == 'exp': - # sel = make_id_selector_cluster_aware_direct_exp(self.id_position_in_cluster, self.limits, self.total_clusters, self.max_range) - # else: - # raise Exception('unknown type ', self.type) - selector_by_thread[thread] = sel - else: - sel = selector_by_thread.get(thread) - - sel.set_words(int(w1), int(w2)) - - params = faiss.SearchParametersIVF(sel=sel, nprobe=self.nprobe, max_codes=self.max_codes, selector_probe_limit=self.selector_probe_limit) - _, Ii = self.index.search( X[q:q+1], k, params=params) - Ii = Ii.ravel() - self.I[q] = Ii - - if self.nt <= 1: - for q in range(nq): - process_one_row(q) - else: - faiss.omp_set_num_threads(self.nt) - - pool = ThreadPool(self.nt) - list(pool.map(process_one_row, range(nq))) - - def get_results(self): - return self.I - - def set_query_arguments(self, query_args): - #faiss.cvar.indexIVF_stats.reset() - #faiss.cvar.IDSelectorMy_Stats.reset() - if "nprobe" in query_args: - self.nprobe = query_args['nprobe'] - self.ps.set_index_parameters(self.index, f"nprobe={query_args['nprobe']}") - self.qas = query_args - else: - self.nprobe = 1 - if "max_codes" in query_args: - self.max_codes = query_args["max_codes"] - self.ps.set_index_parameters(self.index, f"max_codes={query_args['max_codes']}") - self.qas = query_args - else: - self.max_codes = -1 - if "selector_probe_limit" in query_args: - self.selector_probe_limit = query_args['selector_probe_limit'] - self.ps.set_index_parameters(self.index, f"selector_probe_limit={query_args['selector_probe_limit']}") - self.qas = query_args - else: - self.selector_probe_limit = 0 - - if "k_factor" in query_args: - self.k_factor = query_args['k_factor'] - self.qas = query_args - - - - def __str__(self): - return f'Faiss({self.indexkey,self.type, self.qas})' - - \ No newline at end of file From 549aaeff04b835a19d0194c0f5432edc7cbf45b5 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 31 Oct 2023 07:20:01 +0000 Subject: [PATCH 06/18] fix bugs --- .github/workflows/neurips23.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/neurips23.yml b/.github/workflows/neurips23.yml index e8630498..c2a17364 100644 --- a/.github/workflows/neurips23.yml +++ b/.github/workflows/neurips23.yml @@ -31,7 +31,7 @@ jobs: dataset: random-xs track: ood # Test fassplus entry - - algorithm: FduFilterDiskANN + - algorithm: fdufilterdiskann dataset: random-filter-s track: filter fail-fast: false From 22ce1591807357b2f56a2f88c5d0caddd029ab37 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 31 Oct 2023 07:50:04 +0000 Subject: [PATCH 07/18] we have 2 versions: faissplus and fdufilterdiskann --- neurips23/filter/faissplus/Dockerfile | 22 ++ .../filter/faissplus/bow_id_selector.swig | 183 +++++++++++ neurips23/filter/faissplus/config.yaml | 66 ++++ neurips23/filter/faissplus/faiss.py | 287 ++++++++++++++++++ 4 files changed, 558 insertions(+) create mode 100644 neurips23/filter/faissplus/Dockerfile create mode 100644 neurips23/filter/faissplus/bow_id_selector.swig create mode 100644 neurips23/filter/faissplus/config.yaml create mode 100644 neurips23/filter/faissplus/faiss.py diff --git a/neurips23/filter/faissplus/Dockerfile b/neurips23/filter/faissplus/Dockerfile new file mode 100644 index 00000000..a43c6c9e --- /dev/null +++ b/neurips23/filter/faissplus/Dockerfile @@ -0,0 +1,22 @@ +FROM neurips23 + +RUN apt update && apt install -y wget swig +RUN wget https://repo.anaconda.com/archive/Anaconda3-2023.03-0-Linux-x86_64.sh +RUN bash Anaconda3-2023.03-0-Linux-x86_64.sh -b + +ENV PATH /root/anaconda3/bin:$PATH +ENV CONDA_PREFIX /root/anaconda3/ + +RUN conda install -c pytorch faiss-cpu +COPY install/requirements_conda.txt ./ +# conda doesn't like some of our packages, use pip +RUN python3 -m pip install -r requirements_conda.txt + +COPY neurips23/filter/faiss/bow_id_selector.swig ./ + +RUN swig -c++ -python -I$CONDA_PREFIX/include -Ifaiss bow_id_selector.swig +RUN g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ + -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ + -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so -Ifaiss + +RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' \ No newline at end of file diff --git a/neurips23/filter/faissplus/bow_id_selector.swig b/neurips23/filter/faissplus/bow_id_selector.swig new file mode 100644 index 00000000..6712aa25 --- /dev/null +++ b/neurips23/filter/faissplus/bow_id_selector.swig @@ -0,0 +1,183 @@ + +%module bow_id_selector + +/* +To compile when Faiss is installed via conda: + +swig -c++ -python -I$CONDA_PREFIX/include bow_id_selector.swig && \ +g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ + -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ + -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so + +*/ + + +// Put C++ includes here +%{ + +#include +#include + +%} + +// to get uint32_t and friends +%include + +// This means: assume what's declared in these .h files is provided +// by the Faiss module. +%import(module="faiss") "faiss/MetricType.h" +%import(module="faiss") "faiss/impl/IDSelector.h" + +// functions to be parsed here + +// This is important to release GIL and do Faiss exception handing +%exception { + Py_BEGIN_ALLOW_THREADS + try { + $action + } catch(faiss::FaissException & e) { + PyEval_RestoreThread(_save); + + if (PyErr_Occurred()) { + // some previous code already set the error type. + } else { + PyErr_SetString(PyExc_RuntimeError, e.what()); + } + SWIG_fail; + } catch(std::bad_alloc & ba) { + PyEval_RestoreThread(_save); + PyErr_SetString(PyExc_MemoryError, "std::bad_alloc"); + SWIG_fail; + } + Py_END_ALLOW_THREADS +} + + +// any class or function declared below will be made available +// in the module. +%inline %{ + +struct IDSelectorBOW : faiss::IDSelector { + size_t nb; + using TL = int32_t; + const TL *lims; + const int32_t *indices; + int32_t w1 = -1, w2 = -1; + + IDSelectorBOW( + size_t nb, const TL *lims, const int32_t *indices): + nb(nb), lims(lims), indices(indices) {} + + void set_query_words(int32_t w1, int32_t w2) { + this->w1 = w1; + this->w2 = w2; + } + + // binary search in the indices array + bool find_sorted(TL l0, TL l1, int32_t w) const { + while (l1 > l0 + 1) { + TL lmed = (l0 + l1) / 2; + if (indices[lmed] > w) { + l1 = lmed; + } else { + l0 = lmed; + } + } + return indices[l0] == w; + } + + bool is_member(faiss::idx_t id) const { + TL l0 = lims[id], l1 = lims[id + 1]; + if (l1 <= l0) { + return false; + } + if(!find_sorted(l0, l1, w1)) { + return false; + } + if(w2 >= 0 && !find_sorted(l0, l1, w2)) { + return false; + } + return true; + } + + ~IDSelectorBOW() override {} +}; + + +struct IDSelectorBOWBin : IDSelectorBOW { + /** with additional binary filtering */ + faiss::idx_t id_mask; + + IDSelectorBOWBin( + size_t nb, const TL *lims, const int32_t *indices, faiss::idx_t id_mask): + IDSelectorBOW(nb, lims, indices), id_mask(id_mask) {} + + faiss::idx_t q_mask = 0; + + void set_query_words_mask(int32_t w1, int32_t w2, faiss::idx_t q_mask) { + set_query_words(w1, w2); + this->q_mask = q_mask; + } + + bool is_member(faiss::idx_t id) const { + if (q_mask & ~id) { + return false; + } + return IDSelectorBOW::is_member(id & id_mask); + } + + ~IDSelectorBOWBin() override {} +}; + + +size_t intersect_sorted_c( + size_t n1, const int32_t *a1, + size_t n2, const int32_t *a2, + int32_t *res) +{ + if (n1 == 0 || n2 == 0) { + return 0; + } + size_t i1 = 0, i2 = 0, i = 0; + for(;;) { + if (a1[i1] < a2[i2]) { + i1++; + if (i1 >= n1) { + return i; + } + } else if (a1[i1] > a2[i2]) { + i2++; + if (i2 >= n2) { + return i; + } + } else { // equal + res[i++] = a1[i1++]; + i2++; + if (i1 >= n1 || i2 >= n2) { + return i; + } + } + } +} + +%} + + +%pythoncode %{ + +import numpy as np + +# example additional function that converts the passed-in numpy arrays to +# C++ pointers +def intersect_sorted(a1, a2): + n1, = a1.shape + n2, = a2.shape + res = np.empty(n1 + n2, dtype=a1.dtype) + nres = intersect_sorted_c( + n1, faiss.swig_ptr(a1), + n2, faiss.swig_ptr(a2), + faiss.swig_ptr(res) + ) + return res[:nres] + +%} \ No newline at end of file diff --git a/neurips23/filter/faissplus/config.yaml b/neurips23/filter/faissplus/config.yaml new file mode 100644 index 00000000..8dbc474f --- /dev/null +++ b/neurips23/filter/faissplus/config.yaml @@ -0,0 +1,66 @@ +random-filter-s: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF1024,SQ8"}] + query-args: | + [{"nprobe": 1}, + {"nprobe":2}, + {"nprobe":4}] +random-s: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF1024,SQ8"}] + query-args: | + [{"nprobe": 1}, + {"nprobe":2}, + {"nprobe":4}] +yfcc-10M-unfiltered: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF16384,SQ8", "binarysig": true, "threads": 16}] + query-args: | + [{"nprobe": 1}, {"nprobe": 4}, {"nprobe": 16}, {"nprobe": 64}] +yfcc-10M: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF11264,SQ8", + "binarysig": true, + "threads": 16 + }] + query-args: | + [ + {"nprobe": 34, "mt_threshold": 0.00031}, + {"nprobe": 32, "mt_threshold": 0.0003}, + {"nprobe": 32, "mt_threshold": 0.00031}, + {"nprobe": 34, "mt_threshold": 0.0003}, + {"nprobe": 34, "mt_threshold": 0.00035}, + {"nprobe": 32, "mt_threshold": 0.00033}, + {"nprobe": 30, "mt_threshold": 0.00033}, + {"nprobe": 32, "mt_threshold": 0.00035}, + {"nprobe": 34, "mt_threshold": 0.00033}, + {"nprobe": 40, "mt_threshold": 0.0003} + ] \ No newline at end of file diff --git a/neurips23/filter/faissplus/faiss.py b/neurips23/filter/faissplus/faiss.py new file mode 100644 index 00000000..02980d12 --- /dev/null +++ b/neurips23/filter/faissplus/faiss.py @@ -0,0 +1,287 @@ +import pdb +import pickle +import numpy as np +import os + +from multiprocessing.pool import ThreadPool + +import faiss + +from neurips23.filter.base import BaseFilterANN +from benchmark.datasets import DATASETS +from benchmark.dataset_io import download_accelerated + +import bow_id_selector + +def csr_get_row_indices(m, i): + """ get the non-0 column indices for row i in matrix m """ + return m.indices[m.indptr[i] : m.indptr[i + 1]] + +def make_bow_id_selector(mat, id_mask=0): + sp = faiss.swig_ptr + if id_mask == 0: + return bow_id_selector.IDSelectorBOW(mat.shape[0], sp(mat.indptr), sp(mat.indices)) + else: + return bow_id_selector.IDSelectorBOWBin( + mat.shape[0], sp(mat.indptr), sp(mat.indices), id_mask + ) + +def set_invlist_ids(invlists, l, ids): + n, = ids.shape + ids = np.ascontiguousarray(ids, dtype='int64') + assert invlists.list_size(l) == n + faiss.memcpy( + invlists.get_ids(l), + faiss.swig_ptr(ids), n * 8 + ) + + + +def csr_to_bitcodes(matrix, bitsig): + """ Compute binary codes for the rows of the matrix: each binary code is + the OR of bitsig for non-0 entries of the row. + """ + indptr = matrix.indptr + indices = matrix.indices + n = matrix.shape[0] + bit_codes = np.zeros(n, dtype='int64') + for i in range(n): + # print(bitsig[indices[indptr[i]:indptr[i + 1]]]) + bit_codes[i] = np.bitwise_or.reduce(bitsig[indices[indptr[i]:indptr[i + 1]]]) + return bit_codes + + +class BinarySignatures: + """ binary signatures that encode vectors """ + + def __init__(self, meta_b, proba_1): + nvec, nword = meta_b.shape + # number of bits reserved for the vector ids + self.id_bits = int(np.ceil(np.log2(nvec))) + # number of bits for the binary signature + self.sig_bits = nbits = 63 - self.id_bits + + # select binary signatures for the vocabulary + rs = np.random.RandomState(123) # we rely on this to be reproducible! + bitsig = np.packbits(rs.rand(nword, nbits) < proba_1, axis=1) + bitsig = np.pad(bitsig, ((0, 0), (0, 8 - bitsig.shape[1]))).view("int64").ravel() + self.bitsig = bitsig + + # signatures for all the metadata matrix + self.db_sig = csr_to_bitcodes(meta_b, bitsig) << self.id_bits + + # mask to keep only the ids + self.id_mask = (1 << self.id_bits) - 1 + + def query_signature(self, w1, w2): + """ compute the query signature for 1 or 2 words """ + sig = self.bitsig[w1] + if w2 != -1: + sig |= self.bitsig[w2] + return int(sig << self.id_bits) + +class FAISS(BaseFilterANN): + + def __init__(self, metric, index_params): + self._index_params = index_params + self._metric = metric + print(index_params) + self.indexkey = index_params.get("indexkey", "IVF32768,SQ8") + self.binarysig = index_params.get("binarysig", True) + self.binarysig_proba1 = index_params.get("binarysig_proba1", 0.1) + self.metadata_threshold = 1e-3 + self.nt = index_params.get("threads", 1) + + + def fit(self, dataset): + ds = DATASETS[dataset]() + if ds.search_type() == "knn_filtered" and self.binarysig: + print("preparing binary signatures") + meta_b = ds.get_dataset_metadata() + self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) + print("writing to", self.binarysig_name(dataset)) + pickle.dump(self.binsig, open(self.binarysig_name(dataset), "wb"), -1) + else: + self.binsig = None + + if ds.search_type() == "knn_filtered": + self.meta_b = ds.get_dataset_metadata() + self.meta_b.sort_indices() + + index = faiss.index_factory(ds.d, self.indexkey) + xb = ds.get_dataset() + print("train") + index.train(xb) + print("populate") + if self.binsig is None: + index.add(xb) + else: + ids = np.arange(ds.nb) | self.binsig.db_sig + index.add_with_ids(xb, ids) + + self.index = index + self.nb = ds.nb + self.xb = xb + self.ps = faiss.ParameterSpace() + self.ps.initialize(self.index) + print("store", self.index_name(dataset)) + faiss.write_index(index, self.index_name(dataset)) + + + def index_name(self, name): + return f"data/{name}.{self.indexkey}.faissindex" + + def binarysig_name(self, name): + return f"data/{name}.{self.indexkey}.binarysig" + + + def load_index(self, dataset): + """ + Load the index for dataset. Returns False if index + is not available, True otherwise. + + Checking the index usually involves the dataset name + and the index build paramters passed during construction. + """ + if not os.path.exists(self.index_name(dataset)): + if 'url' not in self._index_params: + return False + + print('Downloading index in background. This can take a while.') + download_accelerated(self._index_params['url'], self.index_name(dataset), quiet=True) + + print("Loading index") + + self.index = faiss.read_index(self.index_name(dataset)) + + self.ps = faiss.ParameterSpace() + self.ps.initialize(self.index) + + ds = DATASETS[dataset]() + + if ds.search_type() == "knn_filtered" and self.binarysig: + if not os.path.exists(self.binarysig_name(dataset)): + print("preparing binary signatures") + meta_b = ds.get_dataset_metadata() + self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) + else: + print("loading binary signatures") + self.binsig = pickle.load(open(self.binarysig_name(dataset), "rb")) + else: + self.binsig = None + + if ds.search_type() == "knn_filtered": + self.meta_b = ds.get_dataset_metadata() + self.meta_b.sort_indices() + + self.nb = ds.nb + self.xb = ds.get_dataset() + + return True + + def index_files_to_store(self, dataset): + """ + Specify a triplet with the local directory path of index files, + the common prefix name of index component(s) and a list of + index components that need to be uploaded to (after build) + or downloaded from (for search) cloud storage. + + For local directory path under docker environment, please use + a directory under + data/indices/track(T1 or T2)/algo.__str__()/DATASETS[dataset]().short_name() + """ + raise NotImplementedError() + + def query(self, X, k): + nq = X.shape[0] + self.I = -np.ones((nq, k), dtype='int32') + bs = 1024 + for i0 in range(0, nq, bs): + _, self.I[i0:i0+bs] = self.index.search(X[i0:i0+bs], k) + + + def filtered_query(self, X, filter, k): + print('running filtered query') + nq = X.shape[0] + self.I = -np.ones((nq, k), dtype='int32') + meta_b = self.meta_b + meta_q = filter + docs_per_word = meta_b.T.tocsr() + ndoc_per_word = docs_per_word.indptr[1:] - docs_per_word.indptr[:-1] + freq_per_word = ndoc_per_word / self.nb + + def process_one_row(q): + faiss.omp_set_num_threads(1) + qwords = csr_get_row_indices(meta_q, q) + assert qwords.size in (1, 2) + w1 = qwords[0] + freq = freq_per_word[w1] + if qwords.size == 2: + w2 = qwords[1] + freq *= freq_per_word[w2] + else: + w2 = -1 + if freq < self.metadata_threshold: + # metadata first + docs = csr_get_row_indices(docs_per_word, w1) + if w2 != -1: + docs = bow_id_selector.intersect_sorted( + docs, csr_get_row_indices(docs_per_word, w2)) + + assert len(docs) >= k, pdb.set_trace() + xb_subset = self.xb[docs] + _, Ii = faiss.knn(X[q : q + 1], xb_subset, k=k) + + self.I[q, :] = docs[Ii.ravel()] + else: + # IVF first, filtered search + sel = make_bow_id_selector(meta_b, self.binsig.id_mask if self.binsig else 0) + if self.binsig is None: + sel.set_query_words(int(w1), int(w2)) + else: + sel.set_query_words_mask( + int(w1), int(w2), self.binsig.query_signature(w1, w2)) + + params = faiss.SearchParametersIVF(sel=sel, nprobe=self.nprobe) + + _, Ii = self.index.search( + X[q:q+1], k, params=params + ) + Ii = Ii.ravel() + if self.binsig is None: + self.I[q] = Ii + else: + # we'll just assume there are enough results + # valid = Ii != -1 + # I[q, valid] = Ii[valid] & binsig.id_mask + self.I[q] = Ii & self.binsig.id_mask + + + if self.nt <= 1: + for q in range(nq): + process_one_row(q) + else: + faiss.omp_set_num_threads(self.nt) + pool = ThreadPool(self.nt) + list(pool.map(process_one_row, range(nq))) + + def get_results(self): + return self.I + + def set_query_arguments(self, query_args): + faiss.cvar.indexIVF_stats.reset() + if "nprobe" in query_args: + self.nprobe = query_args['nprobe'] + self.ps.set_index_parameters(self.index, f"nprobe={query_args['nprobe']}") + self.qas = query_args + else: + self.nprobe = 1 + if "mt_threshold" in query_args: + self.metadata_threshold = query_args['mt_threshold'] + else: + self.metadata_threshold = 1e-3 + + def __str__(self): + return f'Faiss({self.indexkey, self.qas})' + + \ No newline at end of file From dad194720eab5ea6bb305529e9133a2fa4807752 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 31 Oct 2023 08:00:45 +0000 Subject: [PATCH 08/18] version1: filter/fdufilterdiskann --- neurips23/filter/faissplus/Dockerfile | 22 -- .../filter/faissplus/bow_id_selector.swig | 183 ----------- neurips23/filter/faissplus/config.yaml | 66 ---- neurips23/filter/faissplus/faiss.py | 287 ------------------ 4 files changed, 558 deletions(-) delete mode 100644 neurips23/filter/faissplus/Dockerfile delete mode 100644 neurips23/filter/faissplus/bow_id_selector.swig delete mode 100644 neurips23/filter/faissplus/config.yaml delete mode 100644 neurips23/filter/faissplus/faiss.py diff --git a/neurips23/filter/faissplus/Dockerfile b/neurips23/filter/faissplus/Dockerfile deleted file mode 100644 index a43c6c9e..00000000 --- a/neurips23/filter/faissplus/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM neurips23 - -RUN apt update && apt install -y wget swig -RUN wget https://repo.anaconda.com/archive/Anaconda3-2023.03-0-Linux-x86_64.sh -RUN bash Anaconda3-2023.03-0-Linux-x86_64.sh -b - -ENV PATH /root/anaconda3/bin:$PATH -ENV CONDA_PREFIX /root/anaconda3/ - -RUN conda install -c pytorch faiss-cpu -COPY install/requirements_conda.txt ./ -# conda doesn't like some of our packages, use pip -RUN python3 -m pip install -r requirements_conda.txt - -COPY neurips23/filter/faiss/bow_id_selector.swig ./ - -RUN swig -c++ -python -I$CONDA_PREFIX/include -Ifaiss bow_id_selector.swig -RUN g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ - -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ - -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so -Ifaiss - -RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' \ No newline at end of file diff --git a/neurips23/filter/faissplus/bow_id_selector.swig b/neurips23/filter/faissplus/bow_id_selector.swig deleted file mode 100644 index 6712aa25..00000000 --- a/neurips23/filter/faissplus/bow_id_selector.swig +++ /dev/null @@ -1,183 +0,0 @@ - -%module bow_id_selector - -/* -To compile when Faiss is installed via conda: - -swig -c++ -python -I$CONDA_PREFIX/include bow_id_selector.swig && \ -g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ - -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ - -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so - -*/ - - -// Put C++ includes here -%{ - -#include -#include - -%} - -// to get uint32_t and friends -%include - -// This means: assume what's declared in these .h files is provided -// by the Faiss module. -%import(module="faiss") "faiss/MetricType.h" -%import(module="faiss") "faiss/impl/IDSelector.h" - -// functions to be parsed here - -// This is important to release GIL and do Faiss exception handing -%exception { - Py_BEGIN_ALLOW_THREADS - try { - $action - } catch(faiss::FaissException & e) { - PyEval_RestoreThread(_save); - - if (PyErr_Occurred()) { - // some previous code already set the error type. - } else { - PyErr_SetString(PyExc_RuntimeError, e.what()); - } - SWIG_fail; - } catch(std::bad_alloc & ba) { - PyEval_RestoreThread(_save); - PyErr_SetString(PyExc_MemoryError, "std::bad_alloc"); - SWIG_fail; - } - Py_END_ALLOW_THREADS -} - - -// any class or function declared below will be made available -// in the module. -%inline %{ - -struct IDSelectorBOW : faiss::IDSelector { - size_t nb; - using TL = int32_t; - const TL *lims; - const int32_t *indices; - int32_t w1 = -1, w2 = -1; - - IDSelectorBOW( - size_t nb, const TL *lims, const int32_t *indices): - nb(nb), lims(lims), indices(indices) {} - - void set_query_words(int32_t w1, int32_t w2) { - this->w1 = w1; - this->w2 = w2; - } - - // binary search in the indices array - bool find_sorted(TL l0, TL l1, int32_t w) const { - while (l1 > l0 + 1) { - TL lmed = (l0 + l1) / 2; - if (indices[lmed] > w) { - l1 = lmed; - } else { - l0 = lmed; - } - } - return indices[l0] == w; - } - - bool is_member(faiss::idx_t id) const { - TL l0 = lims[id], l1 = lims[id + 1]; - if (l1 <= l0) { - return false; - } - if(!find_sorted(l0, l1, w1)) { - return false; - } - if(w2 >= 0 && !find_sorted(l0, l1, w2)) { - return false; - } - return true; - } - - ~IDSelectorBOW() override {} -}; - - -struct IDSelectorBOWBin : IDSelectorBOW { - /** with additional binary filtering */ - faiss::idx_t id_mask; - - IDSelectorBOWBin( - size_t nb, const TL *lims, const int32_t *indices, faiss::idx_t id_mask): - IDSelectorBOW(nb, lims, indices), id_mask(id_mask) {} - - faiss::idx_t q_mask = 0; - - void set_query_words_mask(int32_t w1, int32_t w2, faiss::idx_t q_mask) { - set_query_words(w1, w2); - this->q_mask = q_mask; - } - - bool is_member(faiss::idx_t id) const { - if (q_mask & ~id) { - return false; - } - return IDSelectorBOW::is_member(id & id_mask); - } - - ~IDSelectorBOWBin() override {} -}; - - -size_t intersect_sorted_c( - size_t n1, const int32_t *a1, - size_t n2, const int32_t *a2, - int32_t *res) -{ - if (n1 == 0 || n2 == 0) { - return 0; - } - size_t i1 = 0, i2 = 0, i = 0; - for(;;) { - if (a1[i1] < a2[i2]) { - i1++; - if (i1 >= n1) { - return i; - } - } else if (a1[i1] > a2[i2]) { - i2++; - if (i2 >= n2) { - return i; - } - } else { // equal - res[i++] = a1[i1++]; - i2++; - if (i1 >= n1 || i2 >= n2) { - return i; - } - } - } -} - -%} - - -%pythoncode %{ - -import numpy as np - -# example additional function that converts the passed-in numpy arrays to -# C++ pointers -def intersect_sorted(a1, a2): - n1, = a1.shape - n2, = a2.shape - res = np.empty(n1 + n2, dtype=a1.dtype) - nres = intersect_sorted_c( - n1, faiss.swig_ptr(a1), - n2, faiss.swig_ptr(a2), - faiss.swig_ptr(res) - ) - return res[:nres] - -%} \ No newline at end of file diff --git a/neurips23/filter/faissplus/config.yaml b/neurips23/filter/faissplus/config.yaml deleted file mode 100644 index 8dbc474f..00000000 --- a/neurips23/filter/faissplus/config.yaml +++ /dev/null @@ -1,66 +0,0 @@ -random-filter-s: - faissplus: - docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF1024,SQ8"}] - query-args: | - [{"nprobe": 1}, - {"nprobe":2}, - {"nprobe":4}] -random-s: - faissplus: - docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF1024,SQ8"}] - query-args: | - [{"nprobe": 1}, - {"nprobe":2}, - {"nprobe":4}] -yfcc-10M-unfiltered: - faissplus: - docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF16384,SQ8", "binarysig": true, "threads": 16}] - query-args: | - [{"nprobe": 1}, {"nprobe": 4}, {"nprobe": 16}, {"nprobe": 64}] -yfcc-10M: - faissplus: - docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss - constructor: FAISS - base-args: ["@metric"] - run-groups: - base: - args: | - [{"indexkey": "IVF11264,SQ8", - "binarysig": true, - "threads": 16 - }] - query-args: | - [ - {"nprobe": 34, "mt_threshold": 0.00031}, - {"nprobe": 32, "mt_threshold": 0.0003}, - {"nprobe": 32, "mt_threshold": 0.00031}, - {"nprobe": 34, "mt_threshold": 0.0003}, - {"nprobe": 34, "mt_threshold": 0.00035}, - {"nprobe": 32, "mt_threshold": 0.00033}, - {"nprobe": 30, "mt_threshold": 0.00033}, - {"nprobe": 32, "mt_threshold": 0.00035}, - {"nprobe": 34, "mt_threshold": 0.00033}, - {"nprobe": 40, "mt_threshold": 0.0003} - ] \ No newline at end of file diff --git a/neurips23/filter/faissplus/faiss.py b/neurips23/filter/faissplus/faiss.py deleted file mode 100644 index 02980d12..00000000 --- a/neurips23/filter/faissplus/faiss.py +++ /dev/null @@ -1,287 +0,0 @@ -import pdb -import pickle -import numpy as np -import os - -from multiprocessing.pool import ThreadPool - -import faiss - -from neurips23.filter.base import BaseFilterANN -from benchmark.datasets import DATASETS -from benchmark.dataset_io import download_accelerated - -import bow_id_selector - -def csr_get_row_indices(m, i): - """ get the non-0 column indices for row i in matrix m """ - return m.indices[m.indptr[i] : m.indptr[i + 1]] - -def make_bow_id_selector(mat, id_mask=0): - sp = faiss.swig_ptr - if id_mask == 0: - return bow_id_selector.IDSelectorBOW(mat.shape[0], sp(mat.indptr), sp(mat.indices)) - else: - return bow_id_selector.IDSelectorBOWBin( - mat.shape[0], sp(mat.indptr), sp(mat.indices), id_mask - ) - -def set_invlist_ids(invlists, l, ids): - n, = ids.shape - ids = np.ascontiguousarray(ids, dtype='int64') - assert invlists.list_size(l) == n - faiss.memcpy( - invlists.get_ids(l), - faiss.swig_ptr(ids), n * 8 - ) - - - -def csr_to_bitcodes(matrix, bitsig): - """ Compute binary codes for the rows of the matrix: each binary code is - the OR of bitsig for non-0 entries of the row. - """ - indptr = matrix.indptr - indices = matrix.indices - n = matrix.shape[0] - bit_codes = np.zeros(n, dtype='int64') - for i in range(n): - # print(bitsig[indices[indptr[i]:indptr[i + 1]]]) - bit_codes[i] = np.bitwise_or.reduce(bitsig[indices[indptr[i]:indptr[i + 1]]]) - return bit_codes - - -class BinarySignatures: - """ binary signatures that encode vectors """ - - def __init__(self, meta_b, proba_1): - nvec, nword = meta_b.shape - # number of bits reserved for the vector ids - self.id_bits = int(np.ceil(np.log2(nvec))) - # number of bits for the binary signature - self.sig_bits = nbits = 63 - self.id_bits - - # select binary signatures for the vocabulary - rs = np.random.RandomState(123) # we rely on this to be reproducible! - bitsig = np.packbits(rs.rand(nword, nbits) < proba_1, axis=1) - bitsig = np.pad(bitsig, ((0, 0), (0, 8 - bitsig.shape[1]))).view("int64").ravel() - self.bitsig = bitsig - - # signatures for all the metadata matrix - self.db_sig = csr_to_bitcodes(meta_b, bitsig) << self.id_bits - - # mask to keep only the ids - self.id_mask = (1 << self.id_bits) - 1 - - def query_signature(self, w1, w2): - """ compute the query signature for 1 or 2 words """ - sig = self.bitsig[w1] - if w2 != -1: - sig |= self.bitsig[w2] - return int(sig << self.id_bits) - -class FAISS(BaseFilterANN): - - def __init__(self, metric, index_params): - self._index_params = index_params - self._metric = metric - print(index_params) - self.indexkey = index_params.get("indexkey", "IVF32768,SQ8") - self.binarysig = index_params.get("binarysig", True) - self.binarysig_proba1 = index_params.get("binarysig_proba1", 0.1) - self.metadata_threshold = 1e-3 - self.nt = index_params.get("threads", 1) - - - def fit(self, dataset): - ds = DATASETS[dataset]() - if ds.search_type() == "knn_filtered" and self.binarysig: - print("preparing binary signatures") - meta_b = ds.get_dataset_metadata() - self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) - print("writing to", self.binarysig_name(dataset)) - pickle.dump(self.binsig, open(self.binarysig_name(dataset), "wb"), -1) - else: - self.binsig = None - - if ds.search_type() == "knn_filtered": - self.meta_b = ds.get_dataset_metadata() - self.meta_b.sort_indices() - - index = faiss.index_factory(ds.d, self.indexkey) - xb = ds.get_dataset() - print("train") - index.train(xb) - print("populate") - if self.binsig is None: - index.add(xb) - else: - ids = np.arange(ds.nb) | self.binsig.db_sig - index.add_with_ids(xb, ids) - - self.index = index - self.nb = ds.nb - self.xb = xb - self.ps = faiss.ParameterSpace() - self.ps.initialize(self.index) - print("store", self.index_name(dataset)) - faiss.write_index(index, self.index_name(dataset)) - - - def index_name(self, name): - return f"data/{name}.{self.indexkey}.faissindex" - - def binarysig_name(self, name): - return f"data/{name}.{self.indexkey}.binarysig" - - - def load_index(self, dataset): - """ - Load the index for dataset. Returns False if index - is not available, True otherwise. - - Checking the index usually involves the dataset name - and the index build paramters passed during construction. - """ - if not os.path.exists(self.index_name(dataset)): - if 'url' not in self._index_params: - return False - - print('Downloading index in background. This can take a while.') - download_accelerated(self._index_params['url'], self.index_name(dataset), quiet=True) - - print("Loading index") - - self.index = faiss.read_index(self.index_name(dataset)) - - self.ps = faiss.ParameterSpace() - self.ps.initialize(self.index) - - ds = DATASETS[dataset]() - - if ds.search_type() == "knn_filtered" and self.binarysig: - if not os.path.exists(self.binarysig_name(dataset)): - print("preparing binary signatures") - meta_b = ds.get_dataset_metadata() - self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) - else: - print("loading binary signatures") - self.binsig = pickle.load(open(self.binarysig_name(dataset), "rb")) - else: - self.binsig = None - - if ds.search_type() == "knn_filtered": - self.meta_b = ds.get_dataset_metadata() - self.meta_b.sort_indices() - - self.nb = ds.nb - self.xb = ds.get_dataset() - - return True - - def index_files_to_store(self, dataset): - """ - Specify a triplet with the local directory path of index files, - the common prefix name of index component(s) and a list of - index components that need to be uploaded to (after build) - or downloaded from (for search) cloud storage. - - For local directory path under docker environment, please use - a directory under - data/indices/track(T1 or T2)/algo.__str__()/DATASETS[dataset]().short_name() - """ - raise NotImplementedError() - - def query(self, X, k): - nq = X.shape[0] - self.I = -np.ones((nq, k), dtype='int32') - bs = 1024 - for i0 in range(0, nq, bs): - _, self.I[i0:i0+bs] = self.index.search(X[i0:i0+bs], k) - - - def filtered_query(self, X, filter, k): - print('running filtered query') - nq = X.shape[0] - self.I = -np.ones((nq, k), dtype='int32') - meta_b = self.meta_b - meta_q = filter - docs_per_word = meta_b.T.tocsr() - ndoc_per_word = docs_per_word.indptr[1:] - docs_per_word.indptr[:-1] - freq_per_word = ndoc_per_word / self.nb - - def process_one_row(q): - faiss.omp_set_num_threads(1) - qwords = csr_get_row_indices(meta_q, q) - assert qwords.size in (1, 2) - w1 = qwords[0] - freq = freq_per_word[w1] - if qwords.size == 2: - w2 = qwords[1] - freq *= freq_per_word[w2] - else: - w2 = -1 - if freq < self.metadata_threshold: - # metadata first - docs = csr_get_row_indices(docs_per_word, w1) - if w2 != -1: - docs = bow_id_selector.intersect_sorted( - docs, csr_get_row_indices(docs_per_word, w2)) - - assert len(docs) >= k, pdb.set_trace() - xb_subset = self.xb[docs] - _, Ii = faiss.knn(X[q : q + 1], xb_subset, k=k) - - self.I[q, :] = docs[Ii.ravel()] - else: - # IVF first, filtered search - sel = make_bow_id_selector(meta_b, self.binsig.id_mask if self.binsig else 0) - if self.binsig is None: - sel.set_query_words(int(w1), int(w2)) - else: - sel.set_query_words_mask( - int(w1), int(w2), self.binsig.query_signature(w1, w2)) - - params = faiss.SearchParametersIVF(sel=sel, nprobe=self.nprobe) - - _, Ii = self.index.search( - X[q:q+1], k, params=params - ) - Ii = Ii.ravel() - if self.binsig is None: - self.I[q] = Ii - else: - # we'll just assume there are enough results - # valid = Ii != -1 - # I[q, valid] = Ii[valid] & binsig.id_mask - self.I[q] = Ii & self.binsig.id_mask - - - if self.nt <= 1: - for q in range(nq): - process_one_row(q) - else: - faiss.omp_set_num_threads(self.nt) - pool = ThreadPool(self.nt) - list(pool.map(process_one_row, range(nq))) - - def get_results(self): - return self.I - - def set_query_arguments(self, query_args): - faiss.cvar.indexIVF_stats.reset() - if "nprobe" in query_args: - self.nprobe = query_args['nprobe'] - self.ps.set_index_parameters(self.index, f"nprobe={query_args['nprobe']}") - self.qas = query_args - else: - self.nprobe = 1 - if "mt_threshold" in query_args: - self.metadata_threshold = query_args['mt_threshold'] - else: - self.metadata_threshold = 1e-3 - - def __str__(self): - return f'Faiss({self.indexkey, self.qas})' - - \ No newline at end of file From 4e10d14849bdb9fbd9c6dcc48041a5bd2da7c990 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 31 Oct 2023 08:10:46 +0000 Subject: [PATCH 09/18] algorithm 1: faissplus, some parameters to make faiss better --- neurips23/filter/faissplus/Dockerfile | 22 ++ neurips23/filter/faissplus/config.yaml | 66 ++++ neurips23/filter/faissplus/faiss.py | 287 ++++++++++++++++++ neurips23/filter/fdufilterdiskann/Dockerfile | 17 -- neurips23/filter/fdufilterdiskann/config.yaml | 35 --- .../fdufilterdiskann/fdufilterdiskann.py | 178 ----------- 6 files changed, 375 insertions(+), 230 deletions(-) create mode 100644 neurips23/filter/faissplus/Dockerfile create mode 100644 neurips23/filter/faissplus/config.yaml create mode 100644 neurips23/filter/faissplus/faiss.py delete mode 100644 neurips23/filter/fdufilterdiskann/Dockerfile delete mode 100644 neurips23/filter/fdufilterdiskann/config.yaml delete mode 100644 neurips23/filter/fdufilterdiskann/fdufilterdiskann.py diff --git a/neurips23/filter/faissplus/Dockerfile b/neurips23/filter/faissplus/Dockerfile new file mode 100644 index 00000000..a43c6c9e --- /dev/null +++ b/neurips23/filter/faissplus/Dockerfile @@ -0,0 +1,22 @@ +FROM neurips23 + +RUN apt update && apt install -y wget swig +RUN wget https://repo.anaconda.com/archive/Anaconda3-2023.03-0-Linux-x86_64.sh +RUN bash Anaconda3-2023.03-0-Linux-x86_64.sh -b + +ENV PATH /root/anaconda3/bin:$PATH +ENV CONDA_PREFIX /root/anaconda3/ + +RUN conda install -c pytorch faiss-cpu +COPY install/requirements_conda.txt ./ +# conda doesn't like some of our packages, use pip +RUN python3 -m pip install -r requirements_conda.txt + +COPY neurips23/filter/faiss/bow_id_selector.swig ./ + +RUN swig -c++ -python -I$CONDA_PREFIX/include -Ifaiss bow_id_selector.swig +RUN g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ + -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ + -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so -Ifaiss + +RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' \ No newline at end of file diff --git a/neurips23/filter/faissplus/config.yaml b/neurips23/filter/faissplus/config.yaml new file mode 100644 index 00000000..8dbc474f --- /dev/null +++ b/neurips23/filter/faissplus/config.yaml @@ -0,0 +1,66 @@ +random-filter-s: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF1024,SQ8"}] + query-args: | + [{"nprobe": 1}, + {"nprobe":2}, + {"nprobe":4}] +random-s: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF1024,SQ8"}] + query-args: | + [{"nprobe": 1}, + {"nprobe":2}, + {"nprobe":4}] +yfcc-10M-unfiltered: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF16384,SQ8", "binarysig": true, "threads": 16}] + query-args: | + [{"nprobe": 1}, {"nprobe": 4}, {"nprobe": 16}, {"nprobe": 64}] +yfcc-10M: + faissplus: + docker-tag: neurips23-filter-faissplus + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF11264,SQ8", + "binarysig": true, + "threads": 16 + }] + query-args: | + [ + {"nprobe": 34, "mt_threshold": 0.00031}, + {"nprobe": 32, "mt_threshold": 0.0003}, + {"nprobe": 32, "mt_threshold": 0.00031}, + {"nprobe": 34, "mt_threshold": 0.0003}, + {"nprobe": 34, "mt_threshold": 0.00035}, + {"nprobe": 32, "mt_threshold": 0.00033}, + {"nprobe": 30, "mt_threshold": 0.00033}, + {"nprobe": 32, "mt_threshold": 0.00035}, + {"nprobe": 34, "mt_threshold": 0.00033}, + {"nprobe": 40, "mt_threshold": 0.0003} + ] \ No newline at end of file diff --git a/neurips23/filter/faissplus/faiss.py b/neurips23/filter/faissplus/faiss.py new file mode 100644 index 00000000..02980d12 --- /dev/null +++ b/neurips23/filter/faissplus/faiss.py @@ -0,0 +1,287 @@ +import pdb +import pickle +import numpy as np +import os + +from multiprocessing.pool import ThreadPool + +import faiss + +from neurips23.filter.base import BaseFilterANN +from benchmark.datasets import DATASETS +from benchmark.dataset_io import download_accelerated + +import bow_id_selector + +def csr_get_row_indices(m, i): + """ get the non-0 column indices for row i in matrix m """ + return m.indices[m.indptr[i] : m.indptr[i + 1]] + +def make_bow_id_selector(mat, id_mask=0): + sp = faiss.swig_ptr + if id_mask == 0: + return bow_id_selector.IDSelectorBOW(mat.shape[0], sp(mat.indptr), sp(mat.indices)) + else: + return bow_id_selector.IDSelectorBOWBin( + mat.shape[0], sp(mat.indptr), sp(mat.indices), id_mask + ) + +def set_invlist_ids(invlists, l, ids): + n, = ids.shape + ids = np.ascontiguousarray(ids, dtype='int64') + assert invlists.list_size(l) == n + faiss.memcpy( + invlists.get_ids(l), + faiss.swig_ptr(ids), n * 8 + ) + + + +def csr_to_bitcodes(matrix, bitsig): + """ Compute binary codes for the rows of the matrix: each binary code is + the OR of bitsig for non-0 entries of the row. + """ + indptr = matrix.indptr + indices = matrix.indices + n = matrix.shape[0] + bit_codes = np.zeros(n, dtype='int64') + for i in range(n): + # print(bitsig[indices[indptr[i]:indptr[i + 1]]]) + bit_codes[i] = np.bitwise_or.reduce(bitsig[indices[indptr[i]:indptr[i + 1]]]) + return bit_codes + + +class BinarySignatures: + """ binary signatures that encode vectors """ + + def __init__(self, meta_b, proba_1): + nvec, nword = meta_b.shape + # number of bits reserved for the vector ids + self.id_bits = int(np.ceil(np.log2(nvec))) + # number of bits for the binary signature + self.sig_bits = nbits = 63 - self.id_bits + + # select binary signatures for the vocabulary + rs = np.random.RandomState(123) # we rely on this to be reproducible! + bitsig = np.packbits(rs.rand(nword, nbits) < proba_1, axis=1) + bitsig = np.pad(bitsig, ((0, 0), (0, 8 - bitsig.shape[1]))).view("int64").ravel() + self.bitsig = bitsig + + # signatures for all the metadata matrix + self.db_sig = csr_to_bitcodes(meta_b, bitsig) << self.id_bits + + # mask to keep only the ids + self.id_mask = (1 << self.id_bits) - 1 + + def query_signature(self, w1, w2): + """ compute the query signature for 1 or 2 words """ + sig = self.bitsig[w1] + if w2 != -1: + sig |= self.bitsig[w2] + return int(sig << self.id_bits) + +class FAISS(BaseFilterANN): + + def __init__(self, metric, index_params): + self._index_params = index_params + self._metric = metric + print(index_params) + self.indexkey = index_params.get("indexkey", "IVF32768,SQ8") + self.binarysig = index_params.get("binarysig", True) + self.binarysig_proba1 = index_params.get("binarysig_proba1", 0.1) + self.metadata_threshold = 1e-3 + self.nt = index_params.get("threads", 1) + + + def fit(self, dataset): + ds = DATASETS[dataset]() + if ds.search_type() == "knn_filtered" and self.binarysig: + print("preparing binary signatures") + meta_b = ds.get_dataset_metadata() + self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) + print("writing to", self.binarysig_name(dataset)) + pickle.dump(self.binsig, open(self.binarysig_name(dataset), "wb"), -1) + else: + self.binsig = None + + if ds.search_type() == "knn_filtered": + self.meta_b = ds.get_dataset_metadata() + self.meta_b.sort_indices() + + index = faiss.index_factory(ds.d, self.indexkey) + xb = ds.get_dataset() + print("train") + index.train(xb) + print("populate") + if self.binsig is None: + index.add(xb) + else: + ids = np.arange(ds.nb) | self.binsig.db_sig + index.add_with_ids(xb, ids) + + self.index = index + self.nb = ds.nb + self.xb = xb + self.ps = faiss.ParameterSpace() + self.ps.initialize(self.index) + print("store", self.index_name(dataset)) + faiss.write_index(index, self.index_name(dataset)) + + + def index_name(self, name): + return f"data/{name}.{self.indexkey}.faissindex" + + def binarysig_name(self, name): + return f"data/{name}.{self.indexkey}.binarysig" + + + def load_index(self, dataset): + """ + Load the index for dataset. Returns False if index + is not available, True otherwise. + + Checking the index usually involves the dataset name + and the index build paramters passed during construction. + """ + if not os.path.exists(self.index_name(dataset)): + if 'url' not in self._index_params: + return False + + print('Downloading index in background. This can take a while.') + download_accelerated(self._index_params['url'], self.index_name(dataset), quiet=True) + + print("Loading index") + + self.index = faiss.read_index(self.index_name(dataset)) + + self.ps = faiss.ParameterSpace() + self.ps.initialize(self.index) + + ds = DATASETS[dataset]() + + if ds.search_type() == "knn_filtered" and self.binarysig: + if not os.path.exists(self.binarysig_name(dataset)): + print("preparing binary signatures") + meta_b = ds.get_dataset_metadata() + self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) + else: + print("loading binary signatures") + self.binsig = pickle.load(open(self.binarysig_name(dataset), "rb")) + else: + self.binsig = None + + if ds.search_type() == "knn_filtered": + self.meta_b = ds.get_dataset_metadata() + self.meta_b.sort_indices() + + self.nb = ds.nb + self.xb = ds.get_dataset() + + return True + + def index_files_to_store(self, dataset): + """ + Specify a triplet with the local directory path of index files, + the common prefix name of index component(s) and a list of + index components that need to be uploaded to (after build) + or downloaded from (for search) cloud storage. + + For local directory path under docker environment, please use + a directory under + data/indices/track(T1 or T2)/algo.__str__()/DATASETS[dataset]().short_name() + """ + raise NotImplementedError() + + def query(self, X, k): + nq = X.shape[0] + self.I = -np.ones((nq, k), dtype='int32') + bs = 1024 + for i0 in range(0, nq, bs): + _, self.I[i0:i0+bs] = self.index.search(X[i0:i0+bs], k) + + + def filtered_query(self, X, filter, k): + print('running filtered query') + nq = X.shape[0] + self.I = -np.ones((nq, k), dtype='int32') + meta_b = self.meta_b + meta_q = filter + docs_per_word = meta_b.T.tocsr() + ndoc_per_word = docs_per_word.indptr[1:] - docs_per_word.indptr[:-1] + freq_per_word = ndoc_per_word / self.nb + + def process_one_row(q): + faiss.omp_set_num_threads(1) + qwords = csr_get_row_indices(meta_q, q) + assert qwords.size in (1, 2) + w1 = qwords[0] + freq = freq_per_word[w1] + if qwords.size == 2: + w2 = qwords[1] + freq *= freq_per_word[w2] + else: + w2 = -1 + if freq < self.metadata_threshold: + # metadata first + docs = csr_get_row_indices(docs_per_word, w1) + if w2 != -1: + docs = bow_id_selector.intersect_sorted( + docs, csr_get_row_indices(docs_per_word, w2)) + + assert len(docs) >= k, pdb.set_trace() + xb_subset = self.xb[docs] + _, Ii = faiss.knn(X[q : q + 1], xb_subset, k=k) + + self.I[q, :] = docs[Ii.ravel()] + else: + # IVF first, filtered search + sel = make_bow_id_selector(meta_b, self.binsig.id_mask if self.binsig else 0) + if self.binsig is None: + sel.set_query_words(int(w1), int(w2)) + else: + sel.set_query_words_mask( + int(w1), int(w2), self.binsig.query_signature(w1, w2)) + + params = faiss.SearchParametersIVF(sel=sel, nprobe=self.nprobe) + + _, Ii = self.index.search( + X[q:q+1], k, params=params + ) + Ii = Ii.ravel() + if self.binsig is None: + self.I[q] = Ii + else: + # we'll just assume there are enough results + # valid = Ii != -1 + # I[q, valid] = Ii[valid] & binsig.id_mask + self.I[q] = Ii & self.binsig.id_mask + + + if self.nt <= 1: + for q in range(nq): + process_one_row(q) + else: + faiss.omp_set_num_threads(self.nt) + pool = ThreadPool(self.nt) + list(pool.map(process_one_row, range(nq))) + + def get_results(self): + return self.I + + def set_query_arguments(self, query_args): + faiss.cvar.indexIVF_stats.reset() + if "nprobe" in query_args: + self.nprobe = query_args['nprobe'] + self.ps.set_index_parameters(self.index, f"nprobe={query_args['nprobe']}") + self.qas = query_args + else: + self.nprobe = 1 + if "mt_threshold" in query_args: + self.metadata_threshold = query_args['mt_threshold'] + else: + self.metadata_threshold = 1e-3 + + def __str__(self): + return f'Faiss({self.indexkey, self.qas})' + + \ No newline at end of file diff --git a/neurips23/filter/fdufilterdiskann/Dockerfile b/neurips23/filter/fdufilterdiskann/Dockerfile deleted file mode 100644 index 0b0ee1c6..00000000 --- a/neurips23/filter/fdufilterdiskann/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM neurips23 - -RUN apt update -RUN apt install -y software-properties-common -RUN add-apt-repository -y ppa:git-core/ppa -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive apt install -y git make cmake g++ libaio-dev libgoogle-perftools-dev libunwind-dev clang-format libboost-dev libboost-program-options-dev libmkl-full-dev libcpprest-dev python3.10 - -# COPY FilterDiskann /home/app/FilterDiskann -WORKDIR /home/app -RUN git clone --recursive --branch main https://github.com/PUITAR/FduFilterDiskANN.git -WORKDIR /home/app/FduFilterDiskANN/pybindings - -RUN pip3 install virtualenv build -RUN pip3 install pybind11[global] -RUN pip3 install . -WORKDIR /home/app diff --git a/neurips23/filter/fdufilterdiskann/config.yaml b/neurips23/filter/fdufilterdiskann/config.yaml deleted file mode 100644 index 77b559a3..00000000 --- a/neurips23/filter/fdufilterdiskann/config.yaml +++ /dev/null @@ -1,35 +0,0 @@ -random-filter-s: - fdufilterdiskann: - docker-tag: neurips23-filter-fdufilterdiskann - module: neurips23.filter.fdufilterdiskann.fdufilterdiskann - constructor: fdufilterdiskann - base-args: ["@metric"] - run-groups: - base: - args: | - [{"R":2, "L":10, "buildthreads":16, "alpha":1.2}] - query-args: | - [{"Ls":10, "T":1, "threshold_1":20000, "threshold_2":40000}] -yfcc-10M: - fdufilterdiskann: - docker-tag: neurips23-filter-fdufilterdiskann - module: neurips23.filter.fdufilterdiskann.fdufilterdiskann - constructor: fdufilterdiskann - base-args: ["@metric"] - run-groups: - base: - args: | - [{"R":60, "L":80, "buildthreads":16, "alpha":1.0}] - query-args: | - [ - {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":30000}, - {"Ls":10, "T":16, "threshold_1":30000, "threshold_2":40000}, - {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, - {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, - {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, - {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, - {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, - {"Ls":20, "T":16, "threshold_1":20000, "threshold_2":40000}, - {"Ls":10, "T":16, "threshold_1":20000, "threshold_2":40000}, - {"Ls":15, "T":16, "threshold_1":20000, "threshold_2":40000} - ] diff --git a/neurips23/filter/fdufilterdiskann/fdufilterdiskann.py b/neurips23/filter/fdufilterdiskann/fdufilterdiskann.py deleted file mode 100644 index 6cb657fa..00000000 --- a/neurips23/filter/fdufilterdiskann/fdufilterdiskann.py +++ /dev/null @@ -1,178 +0,0 @@ -from __future__ import absolute_import -import os -import time -import numpy as np - -# import diskannpy -import filterdiskann - -from neurips23.filter.base import BaseFilterANN -from benchmark.datasets import DATASETS, download_accelerated -from multiprocessing.pool import ThreadPool - - -def csr_get_row_indices(m, i): - """ get the non-0 column indices for row i in matrix m """ - return m.indices[m.indptr[i] : m.indptr[i + 1]] - - -class fdufilterdiskann(BaseFilterANN): - def __init__(self, metric, index_params): - self.name = "fdufilterdiskann" - if index_params.get("R") == None: - print("Error: missing parameter R") - return - self._index_params = index_params - self._metric = metric - - self.R = index_params.get("R") - self.L = index_params.get("L") - self.nt = index_params.get("buildthreads", 1) - - - def index_name(self): - return f"R{self.R}_L{self.L}" - - def create_index_dir(self, dataset): - index_dir = os.path.join(os.getcwd(), "data", "indices", "filter") - os.makedirs(index_dir, mode=0o777, exist_ok=True) - index_dir = os.path.join(index_dir, "fdufilterdiskann") - os.makedirs(index_dir, mode=0o777, exist_ok=True) - index_dir = os.path.join(index_dir, dataset.short_name()) - os.makedirs(index_dir, mode=0o777, exist_ok=True) - index_dir = os.path.join(index_dir, self.index_name()) - os.makedirs(index_dir, mode=0o777, exist_ok=True) - return index_dir - - def translate_dist_fn(self, metric): - if metric == 'euclidean': - return 'l2' - elif metric == 'ip': - return 'mips' - else: - raise Exception('Invalid metric') - - def translate_dtype(self, dtype:str): - if dtype == 'uint8': - return np.uint8 - elif dtype == 'int8': - return np.int8 - elif dtype == 'float32': - return np.float32 - else: - raise Exception('Invalid data type') - - def fit(self, dataset): - """ - Build the index for the data points given in dataset name. - """ - - ds = DATASETS[dataset]() - d = ds.d - - buildthreads = self._index_params.get("buildthreads", 1) - self.nt = buildthreads - self.meta_b = ds.get_dataset_metadata() - self.meta_b.sort_indices() # 对每行的列排序 - index_dir = self.create_index_dir(ds) - - if hasattr(self, "index"): - print("Index object exists already") - return - - print(ds.get_dataset_fn()) - - start = time.time() - - print(ds.ds_metadata_fn) - - print("building index") - alpha = self._index_params.get("alpha", 1.2) - filterdiskann.build(ds.get_dataset_fn(), index_dir + '/' + self.index_name(), os.path.join(ds.basedir, ds.ds_metadata_fn), - buildthreads, self.R, self.L, alpha) - - end = time.time() - print("DiskANN index built in %.3f s" % (end - start)) - - - print('Loading index..') - search_threads, search_L = self._index_params.get("T", 16), self._index_params.get("Ls", 20) - print('search threads:', search_threads, 'search L:', search_L) - self.index = filterdiskann.FilterDiskANN(filterdiskann.Metric.L2, index_dir + '/' + self.index_name(), ds.nb, - ds.d, search_threads, search_L) - print('Index ready for search') - - def get_index_components(self, dataset): - index_components = ['', '.data'] - ds = DATASETS[dataset]() - if ds.distance() == "ip": - index_components = index_components + [] - return index_components - - def index_files_to_store(self, dataset): - return [self.create_index_dir(DATASETS[dataset]()), self.index_name(), self.get_index_components(dataset)] - - def load_index(self, dataset): - """ - Load the index for dataset. Returns False if index - is not available, True otherwise. - - Checking the index usually involves the dataset name - and the index build paramters passed during construction. - """ - ds = DATASETS[dataset]() - index_dir = self.create_index_dir(ds) - if not (os.path.exists(index_dir)) and 'url' not in self._index_params: - return False - - index_path = os.path.join(index_dir, self.index_name()) - index_components = self.get_index_components(dataset) - - for component in index_components: - index_file = index_path + component - if not (os.path.exists(index_file)): - if 'url' in self._index_params: - index_file_source = self._index_params['url'] + '/' + self.index_name() + component - print(f"Downloading index in background. This can take a while.") - download_accelerated(index_file_source, index_file, quiet=True) - else: - return False - - print("Loading index") - - search_threads, search_L = self._index_params.get("T", 16), self._index_params.get("Ls", 20) - - print('search threads:', search_threads, 'search L:', search_L) - self.index = filterdiskann.FilterDiskANN(filterdiskann.Metric.L2, index_dir + '/' + self.index_name(), ds.nb, - ds.d, search_threads, search_L) - print ("Load index success.") - return True - - - def set_query_arguments(self, query_args): - self._query_args = query_args - self.Ls = 0 if query_args.get("Ls") == None else query_args.get("Ls") - self.search_threads = self._query_args.get("T") - - def filtered_query(self, X, filter, k): - print('running filtered query diskann') - nq = X.shape[0] - self.I = np.zeros((nq, k), dtype='uint32', order='C') # result IDs - # # meta_b = self.meta_b # data_metadata - meta_q = filter # query_metadata - threshold_1 = self._query_args.get("threshold_1") - threshold_2 = self._query_args.get("threshold_2") - - print("runing in ", self.search_threads, ' nq:', nq) - - self.index.search(X, meta_q.indptr, meta_q.indices, nq, k, self.Ls, self.search_threads, threshold_1, threshold_2, self.I) - - - def get_results(self): - return self.I - - def __str__(self): - return f"diskann({self.index_name(), self._query_args})" - -if __name__ == "__main__": - print(1) \ No newline at end of file From 2a2a3eb26bd202f05dcac6a2ac963f0449e5da34 Mon Sep 17 00:00:00 2001 From: yangming <79465126+PUITAR@users.noreply.github.com> Date: Tue, 31 Oct 2023 16:26:20 +0800 Subject: [PATCH 10/18] Update neurips23.yml --- .github/workflows/neurips23.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/neurips23.yml b/.github/workflows/neurips23.yml index c2a17364..64b611ff 100644 --- a/.github/workflows/neurips23.yml +++ b/.github/workflows/neurips23.yml @@ -31,7 +31,7 @@ jobs: dataset: random-xs track: ood # Test fassplus entry - - algorithm: fdufilterdiskann + - algorithm: faissplus dataset: random-filter-s track: filter fail-fast: false From 81d7ea195f89dfa18ab21887ea5520d554010419 Mon Sep 17 00:00:00 2001 From: yangming <79465126+PUITAR@users.noreply.github.com> Date: Tue, 31 Oct 2023 20:36:30 +0800 Subject: [PATCH 11/18] Create bow_id_selector.swig add a file include in need. --- .../filter/faissplus/bow_id_selector.swig | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 neurips23/filter/faissplus/bow_id_selector.swig diff --git a/neurips23/filter/faissplus/bow_id_selector.swig b/neurips23/filter/faissplus/bow_id_selector.swig new file mode 100644 index 00000000..984dec4c --- /dev/null +++ b/neurips23/filter/faissplus/bow_id_selector.swig @@ -0,0 +1,183 @@ + +%module bow_id_selector + +/* +To compile when Faiss is installed via conda: + +swig -c++ -python -I$CONDA_PREFIX/include bow_id_selector.swig && \ +g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ + -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ + -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so + +*/ + + +// Put C++ includes here +%{ + +#include +#include + +%} + +// to get uint32_t and friends +%include + +// This means: assume what's declared in these .h files is provided +// by the Faiss module. +%import(module="faiss") "faiss/MetricType.h" +%import(module="faiss") "faiss/impl/IDSelector.h" + +// functions to be parsed here + +// This is important to release GIL and do Faiss exception handing +%exception { + Py_BEGIN_ALLOW_THREADS + try { + $action + } catch(faiss::FaissException & e) { + PyEval_RestoreThread(_save); + + if (PyErr_Occurred()) { + // some previous code already set the error type. + } else { + PyErr_SetString(PyExc_RuntimeError, e.what()); + } + SWIG_fail; + } catch(std::bad_alloc & ba) { + PyEval_RestoreThread(_save); + PyErr_SetString(PyExc_MemoryError, "std::bad_alloc"); + SWIG_fail; + } + Py_END_ALLOW_THREADS +} + + +// any class or function declared below will be made available +// in the module. +%inline %{ + +struct IDSelectorBOW : faiss::IDSelector { + size_t nb; + using TL = int32_t; + const TL *lims; + const int32_t *indices; + int32_t w1 = -1, w2 = -1; + + IDSelectorBOW( + size_t nb, const TL *lims, const int32_t *indices): + nb(nb), lims(lims), indices(indices) {} + + void set_query_words(int32_t w1, int32_t w2) { + this->w1 = w1; + this->w2 = w2; + } + + // binary search in the indices array + bool find_sorted(TL l0, TL l1, int32_t w) const { + while (l1 > l0 + 1) { + TL lmed = (l0 + l1) / 2; + if (indices[lmed] > w) { + l1 = lmed; + } else { + l0 = lmed; + } + } + return indices[l0] == w; + } + + bool is_member(faiss::idx_t id) const { + TL l0 = lims[id], l1 = lims[id + 1]; + if (l1 <= l0) { + return false; + } + if(!find_sorted(l0, l1, w1)) { + return false; + } + if(w2 >= 0 && !find_sorted(l0, l1, w2)) { + return false; + } + return true; + } + + ~IDSelectorBOW() override {} +}; + + +struct IDSelectorBOWBin : IDSelectorBOW { + /** with additional binary filtering */ + faiss::idx_t id_mask; + + IDSelectorBOWBin( + size_t nb, const TL *lims, const int32_t *indices, faiss::idx_t id_mask): + IDSelectorBOW(nb, lims, indices), id_mask(id_mask) {} + + faiss::idx_t q_mask = 0; + + void set_query_words_mask(int32_t w1, int32_t w2, faiss::idx_t q_mask) { + set_query_words(w1, w2); + this->q_mask = q_mask; + } + + bool is_member(faiss::idx_t id) const { + if (q_mask & ~id) { + return false; + } + return IDSelectorBOW::is_member(id & id_mask); + } + + ~IDSelectorBOWBin() override {} +}; + + +size_t intersect_sorted_c( + size_t n1, const int32_t *a1, + size_t n2, const int32_t *a2, + int32_t *res) +{ + if (n1 == 0 || n2 == 0) { + return 0; + } + size_t i1 = 0, i2 = 0, i = 0; + for(;;) { + if (a1[i1] < a2[i2]) { + i1++; + if (i1 >= n1) { + return i; + } + } else if (a1[i1] > a2[i2]) { + i2++; + if (i2 >= n2) { + return i; + } + } else { // equal + res[i++] = a1[i1++]; + i2++; + if (i1 >= n1 || i2 >= n2) { + return i; + } + } + } +} + +%} + + +%pythoncode %{ + +import numpy as np + +# example additional function that converts the passed-in numpy arrays to +# C++ pointers +def intersect_sorted(a1, a2): + n1, = a1.shape + n2, = a2.shape + res = np.empty(n1 + n2, dtype=a1.dtype) + nres = intersect_sorted_c( + n1, faiss.swig_ptr(a1), + n2, faiss.swig_ptr(a2), + faiss.swig_ptr(res) + ) + return res[:nres] + +%} From a3d1d0daf4187b59a46fa99f09645588fc1344e7 Mon Sep 17 00:00:00 2001 From: yangming <79465126+PUITAR@users.noreply.github.com> Date: Wed, 1 Nov 2023 09:42:49 +0800 Subject: [PATCH 12/18] Update bow_id_selector.swig Only by adding this file can run this algo --- neurips23/filter/faissplus/bow_id_selector.swig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/neurips23/filter/faissplus/bow_id_selector.swig b/neurips23/filter/faissplus/bow_id_selector.swig index 984dec4c..748b9db9 100644 --- a/neurips23/filter/faissplus/bow_id_selector.swig +++ b/neurips23/filter/faissplus/bow_id_selector.swig @@ -181,3 +181,5 @@ def intersect_sorted(a1, a2): return res[:nres] %} + + From 336b48b88888ed30da61275463a6db7d45120bf3 Mon Sep 17 00:00:00 2001 From: yangming <79465126+PUITAR@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:20:31 +0800 Subject: [PATCH 13/18] Update Dockerfile docker file changes.. --- neurips23/filter/faissplus/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurips23/filter/faissplus/Dockerfile b/neurips23/filter/faissplus/Dockerfile index a43c6c9e..4a54e23b 100644 --- a/neurips23/filter/faissplus/Dockerfile +++ b/neurips23/filter/faissplus/Dockerfile @@ -12,11 +12,11 @@ COPY install/requirements_conda.txt ./ # conda doesn't like some of our packages, use pip RUN python3 -m pip install -r requirements_conda.txt -COPY neurips23/filter/faiss/bow_id_selector.swig ./ +COPY neurips23/filter/faissplus/bow_id_selector.swig ./ RUN swig -c++ -python -I$CONDA_PREFIX/include -Ifaiss bow_id_selector.swig RUN g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so -Ifaiss -RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' \ No newline at end of file +RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' From ff60e94e2ea3862b86a5c5dc985bf866de15523c Mon Sep 17 00:00:00 2001 From: yangming <79465126+PUITAR@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:27:21 +0800 Subject: [PATCH 14/18] Update Dockerfile faiss -> faissplus --- neurips23/filter/faissplus/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/neurips23/filter/faissplus/Dockerfile b/neurips23/filter/faissplus/Dockerfile index 4a54e23b..5bfb528e 100644 --- a/neurips23/filter/faissplus/Dockerfile +++ b/neurips23/filter/faissplus/Dockerfile @@ -20,3 +20,4 @@ RUN g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so -Ifaiss RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' + From 981273668d9d52bea628a5e1886b808a681a8e80 Mon Sep 17 00:00:00 2001 From: yangming <79465126+PUITAR@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:42:44 +0800 Subject: [PATCH 15/18] Rename faiss.py to faissplus.py --- neurips23/filter/faissplus/{faiss.py => faissplus.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename neurips23/filter/faissplus/{faiss.py => faissplus.py} (99%) diff --git a/neurips23/filter/faissplus/faiss.py b/neurips23/filter/faissplus/faissplus.py similarity index 99% rename from neurips23/filter/faissplus/faiss.py rename to neurips23/filter/faissplus/faissplus.py index 02980d12..f45d00a7 100644 --- a/neurips23/filter/faissplus/faiss.py +++ b/neurips23/filter/faissplus/faissplus.py @@ -284,4 +284,4 @@ def set_query_arguments(self, query_args): def __str__(self): return f'Faiss({self.indexkey, self.qas})' - \ No newline at end of file + From ba9ed0063f0c170255fbb1d4c26be3d21a452dc9 Mon Sep 17 00:00:00 2001 From: yangming <79465126+PUITAR@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:44:53 +0800 Subject: [PATCH 16/18] Rename faissplus.py to faiss.py --- neurips23/filter/faissplus/{faissplus.py => faiss.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename neurips23/filter/faissplus/{faissplus.py => faiss.py} (100%) diff --git a/neurips23/filter/faissplus/faissplus.py b/neurips23/filter/faissplus/faiss.py similarity index 100% rename from neurips23/filter/faissplus/faissplus.py rename to neurips23/filter/faissplus/faiss.py From 10f66f61fb3bf21cd0b244f93b89279e90920f2b Mon Sep 17 00:00:00 2001 From: yangming <79465126+PUITAR@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:51:04 +0800 Subject: [PATCH 17/18] Update config.yaml --- neurips23/filter/faissplus/config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/neurips23/filter/faissplus/config.yaml b/neurips23/filter/faissplus/config.yaml index 8dbc474f..bb8ce6fd 100644 --- a/neurips23/filter/faissplus/config.yaml +++ b/neurips23/filter/faissplus/config.yaml @@ -1,7 +1,7 @@ random-filter-s: faissplus: docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss + module: neurips23.filter.faissplus.faiss constructor: FAISS base-args: ["@metric"] run-groups: @@ -15,7 +15,7 @@ random-filter-s: random-s: faissplus: docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss + module: neurips23.filter.faissplus.faiss constructor: FAISS base-args: ["@metric"] run-groups: @@ -29,7 +29,7 @@ random-s: yfcc-10M-unfiltered: faissplus: docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss + module: neurips23.filter.faissplus.faiss constructor: FAISS base-args: ["@metric"] run-groups: @@ -41,7 +41,7 @@ yfcc-10M-unfiltered: yfcc-10M: faissplus: docker-tag: neurips23-filter-faissplus - module: neurips23.filter.faiss.faiss + module: neurips23.filter.faissplus.faiss constructor: FAISS base-args: ["@metric"] run-groups: @@ -63,4 +63,4 @@ yfcc-10M: {"nprobe": 32, "mt_threshold": 0.00035}, {"nprobe": 34, "mt_threshold": 0.00033}, {"nprobe": 40, "mt_threshold": 0.0003} - ] \ No newline at end of file + ] From 590a7261af1cc4c0d615e43a3d30ba916d852476 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 1 Nov 2023 03:36:12 +0000 Subject: [PATCH 18/18] add faiss and wm_filter --- neurips23/filter/faiss/Dockerfile | 25 + neurips23/filter/faiss/README.md | 102 ++++ neurips23/filter/faiss/bow_id_selector.swig | 183 +++++++ neurips23/filter/faiss/config.yaml | 74 +++ neurips23/filter/faiss/faiss.py | 287 ++++++++++ neurips23/filter/wm_filter/Dockerfile | 28 + neurips23/filter/wm_filter/README.md | 7 + neurips23/filter/wm_filter/config.yaml | 50 ++ neurips23/filter/wm_filter/wm_filter.py | 572 ++++++++++++++++++++ 9 files changed, 1328 insertions(+) create mode 100644 neurips23/filter/faiss/Dockerfile create mode 100644 neurips23/filter/faiss/README.md create mode 100644 neurips23/filter/faiss/bow_id_selector.swig create mode 100644 neurips23/filter/faiss/config.yaml create mode 100644 neurips23/filter/faiss/faiss.py create mode 100644 neurips23/filter/wm_filter/Dockerfile create mode 100644 neurips23/filter/wm_filter/README.md create mode 100644 neurips23/filter/wm_filter/config.yaml create mode 100644 neurips23/filter/wm_filter/wm_filter.py diff --git a/neurips23/filter/faiss/Dockerfile b/neurips23/filter/faiss/Dockerfile new file mode 100644 index 00000000..163391a8 --- /dev/null +++ b/neurips23/filter/faiss/Dockerfile @@ -0,0 +1,25 @@ +FROM neurips23 + +RUN apt update && apt install -y wget swig +RUN wget https://repo.anaconda.com/archive/Anaconda3-2023.03-0-Linux-x86_64.sh +RUN bash Anaconda3-2023.03-0-Linux-x86_64.sh -b + +ENV PATH /root/anaconda3/bin:$PATH +ENV CONDA_PREFIX /root/anaconda3/ + +RUN conda install -c pytorch faiss-cpu +COPY install/requirements_conda.txt ./ +# conda doesn't like some of our packages, use pip +RUN python3 -m pip install -r requirements_conda.txt + +COPY neurips23/filter/faiss/bow_id_selector.swig ./ + +RUN swig -c++ -python -I$CONDA_PREFIX/include -Ifaiss bow_id_selector.swig +RUN g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ + -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ + -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so -Ifaiss + +RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' + + + diff --git a/neurips23/filter/faiss/README.md b/neurips23/filter/faiss/README.md new file mode 100644 index 00000000..c834af51 --- /dev/null +++ b/neurips23/filter/faiss/README.md @@ -0,0 +1,102 @@ + +# Faiss baseline for the Filtered search track + +The database of size $N=10^7$ can be seen as the combination of: + +- a matrix $M$ of size $N \times d$ of embedding vectors (called `xb` in the code). $d=192$. +- a sparse matrix $M_\mathrm{meta}$ of size $N \times v$, entry $i,j$ is set to 1 iff word $j$ is applicable to vector $i$. $v=200386$, called `meta_b` in the code (a [CSR matrix](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html)) + +The Faiss basleline for the filtered search track is based on two distinct data structures, a word-based inverted file and a Faiss `IndexIVFFlat`. +Both data structured allow to peform filtered searches in two different ways. + +The search is based on a query vector $q\in \mathbb{R}^d$ and associated query words $w_1, w_2$ (there are one or two query words). +The search results are the database vectors that include /all/ query words and that are nearest to $q$ in $L_2$ distance. + +## Word-based inverted file + +This is term-based inverted file that maps each word to the vectors (docs) that contain that term. +In the code it is a CSR matrix called `docs_per_word` (it's just the transposed version of `meta_b`). + +At search time, the subset (`subset`) of vectors eligible for results depends on the number of query words: + +- if there is a single word $w_1$ then it's just the set of non-0 entries in row $w_1$ of the `docs_per_word` matrix. +This can be extracted at no cost + +- if there are two words $w_1$ and $w_2$ then the sets of non-0 entries of rows $w_1$ and $w_2$ are intersected. +This is done with `np.intersect1d` or the C++ function `intersect_sorted`, that is faster (linear in nb of non-0 entries of the two rows). + +When this subset is selected, the result is found by searching the top-k vectors in this subset of rows of $M$. +The result is exact and the search is most efficient when the subset is small (ie. the words are discriminative enough to filter the results well). + +## IndexIVFFlat structure + +This is a Faiss [`IndexIVFFlat`](https://github.com/facebookresearch/faiss/wiki/The-index-factory#encodings) called `index`. + +By default the index performs unfiltered search, ie. the nearest vectors to $q$ can be retrieved. +The accuracy of this search depends on the number of visited centroids of the `IndexIVFFlat` (parameter `nprobe`, the larger the more accurate and the slower). + +One solution would be to over-fetch vectors and perform filtering post-hoc using the words in the result list. +However, it is unclear /how much/ we should overfetch. + +Therefore, another solution is to use the Faiss [filtering functionality](https://github.com/facebookresearch/faiss/wiki/Setting-search-parameters-for-one-query#searching-in-a-subset-of-elements), ie. provide a callback function that is called for each vector id to decide if it should be considered as a result or not. + +The callback function is implemented in C++ in the class `IDSelectorBOW`. +For vector id $i$ it looks up the row $i$ of $M_\mathrm{meta}$ and peforms a binary search on $w_1$ to check of that word belongs to the words associated to vector $i$. +If $w_2$ is also provided, it does the same for $w_2$. +The callback returns true only if all terms are present. + +### Binary filtering + +The issue is that this callback is relatively slow because (1) it requires to access the $M_\mathrm{meta}$ matrix which causes cache misses and (2) it performs an iterative binary search. +Since the callback is called in the tightest inner loop of the search function, and since the IVF search tends to perform many vector comparisons, this has non negligible performance impact. + +To speed up this test, we can use a nifty piece of bit manipulation. +The idea is that the vector ids are 63 bits long (64 bits integers but negative values are reserved, so we cannot use the sign bit). +However, since $N=10^7$ we use only $\lceil \log_2 N \rceil = 24$ bits of these, leaving 63-24 = 39 bits that are always 0. + +Now, we associate to each word $j$ a 39-bit signature $S[j]$, and the to each set of words the binary `or` of these signatures. +The query is represented by $s_\mathrm{q} = S[w_1] \vee S[w_2]$. +Database entry $i$ with words $W_i$ is represented by $s_i = \vee_{w\in W_i} S[w]$. + +Then we have the following implication: if $\\{w_1, w_2\\} \subset W_i$ then all 1 bits of $s_\mathrm{q}$ are also set to 1 in $s_i$. + +$$\\{w_1, w_2\\} \subset W_i \Rightarrow \neg s_i \wedge s_\mathrm{q} = 0$$ + +Which is equivalent to: + +$$\neg s_i \wedge s_\mathrm{q} \neq 0 \Rightarrow \\{w_1, w_2\\} \not\subset W_i $$ + +Of course, this is an implication, not an equivalence. +Therefore, it can only rule out database vectors. +However, the binary test is very cheap to perform (uses a few machine instructions on data that is already in machine registers), so it can be used as a pre-filter to apply the full membership test on candidates. +This is implemented in the `IDSelectorBOWBin` object. + +The remaining degree of freedom is how to choose the binary signatures, because this rule is always valid, but its filtering ability depends on the choice of the signatures $S$. +After a few tests (see [this notebook](https://gist.github.com/mdouze/75103e4cef436510ac9b834f9a77496f#file-eval_binary_signatures-ipynb) ) it seems that a random signature with 0.1 probability for 1s filters our 80% of negative tests. +Asjuting this to the frequency of the words did not seem to yield better results. + +## Choosing between the two implementations + +The two implementations are complementary: the word-first implementation gives exact results, and has a strong filtering ability for rare words. +The `IndexIVFFlat` implementation gives approximate results and is more relevant for words that are more common, where a significant subset of vectors are indeed relevant. + +Therefore, there should be a rule to choose between the two, and the relevant metric is the size of the subset of vectors to consider. +We can use statistics on the words, ie. $\mathrm{nocc}[j]$ is the number of times word $j$ appears in the dataset (this is just the column-wise sum of the $M_\mathrm{meta}$). + +For a single query word $w_1$, the fraction of relevant indices is just $f = \mathrm{nocc}[w_1] / N$. +For two query words, it is more complicated to compute but an estimate is given by $f = \mathrm{nocc}[w_1] \times \mathrm{nocc}[w_2] / N^2$ (this estimate assumes words are independent, which is incorrect). + +Therefore, the rule that we use is based on a threshold $\tau$ (called `metadata_threshold` in the code) : + +- if $f < \tau$ then use the word-first search + +- otherwise use the IVFFlat based index + +Note that the optimal threshold also depends on the target accuracy (since the IVFFlat is not exact, when a higher accuracy is desired), see https://github.com/harsha-simhadri/big-ann-benchmarks/pull/105#issuecomment-1539842223 . + + +## Code layout + +The code is in faiss.py, with performance critical parts implemented in C++ and wrapped with SWIG in `bow_id_selector.swig`. +SWIG directly exposes the C++ classes and functions in Python. + diff --git a/neurips23/filter/faiss/bow_id_selector.swig b/neurips23/filter/faiss/bow_id_selector.swig new file mode 100644 index 00000000..6712aa25 --- /dev/null +++ b/neurips23/filter/faiss/bow_id_selector.swig @@ -0,0 +1,183 @@ + +%module bow_id_selector + +/* +To compile when Faiss is installed via conda: + +swig -c++ -python -I$CONDA_PREFIX/include bow_id_selector.swig && \ +g++ -shared -O3 -g -fPIC bow_id_selector_wrap.cxx -o _bow_id_selector.so \ + -I $( python -c "import distutils.sysconfig ; print(distutils.sysconfig.get_python_inc())" ) \ + -I $CONDA_PREFIX/include $CONDA_PREFIX/lib/libfaiss_avx2.so + +*/ + + +// Put C++ includes here +%{ + +#include +#include + +%} + +// to get uint32_t and friends +%include + +// This means: assume what's declared in these .h files is provided +// by the Faiss module. +%import(module="faiss") "faiss/MetricType.h" +%import(module="faiss") "faiss/impl/IDSelector.h" + +// functions to be parsed here + +// This is important to release GIL and do Faiss exception handing +%exception { + Py_BEGIN_ALLOW_THREADS + try { + $action + } catch(faiss::FaissException & e) { + PyEval_RestoreThread(_save); + + if (PyErr_Occurred()) { + // some previous code already set the error type. + } else { + PyErr_SetString(PyExc_RuntimeError, e.what()); + } + SWIG_fail; + } catch(std::bad_alloc & ba) { + PyEval_RestoreThread(_save); + PyErr_SetString(PyExc_MemoryError, "std::bad_alloc"); + SWIG_fail; + } + Py_END_ALLOW_THREADS +} + + +// any class or function declared below will be made available +// in the module. +%inline %{ + +struct IDSelectorBOW : faiss::IDSelector { + size_t nb; + using TL = int32_t; + const TL *lims; + const int32_t *indices; + int32_t w1 = -1, w2 = -1; + + IDSelectorBOW( + size_t nb, const TL *lims, const int32_t *indices): + nb(nb), lims(lims), indices(indices) {} + + void set_query_words(int32_t w1, int32_t w2) { + this->w1 = w1; + this->w2 = w2; + } + + // binary search in the indices array + bool find_sorted(TL l0, TL l1, int32_t w) const { + while (l1 > l0 + 1) { + TL lmed = (l0 + l1) / 2; + if (indices[lmed] > w) { + l1 = lmed; + } else { + l0 = lmed; + } + } + return indices[l0] == w; + } + + bool is_member(faiss::idx_t id) const { + TL l0 = lims[id], l1 = lims[id + 1]; + if (l1 <= l0) { + return false; + } + if(!find_sorted(l0, l1, w1)) { + return false; + } + if(w2 >= 0 && !find_sorted(l0, l1, w2)) { + return false; + } + return true; + } + + ~IDSelectorBOW() override {} +}; + + +struct IDSelectorBOWBin : IDSelectorBOW { + /** with additional binary filtering */ + faiss::idx_t id_mask; + + IDSelectorBOWBin( + size_t nb, const TL *lims, const int32_t *indices, faiss::idx_t id_mask): + IDSelectorBOW(nb, lims, indices), id_mask(id_mask) {} + + faiss::idx_t q_mask = 0; + + void set_query_words_mask(int32_t w1, int32_t w2, faiss::idx_t q_mask) { + set_query_words(w1, w2); + this->q_mask = q_mask; + } + + bool is_member(faiss::idx_t id) const { + if (q_mask & ~id) { + return false; + } + return IDSelectorBOW::is_member(id & id_mask); + } + + ~IDSelectorBOWBin() override {} +}; + + +size_t intersect_sorted_c( + size_t n1, const int32_t *a1, + size_t n2, const int32_t *a2, + int32_t *res) +{ + if (n1 == 0 || n2 == 0) { + return 0; + } + size_t i1 = 0, i2 = 0, i = 0; + for(;;) { + if (a1[i1] < a2[i2]) { + i1++; + if (i1 >= n1) { + return i; + } + } else if (a1[i1] > a2[i2]) { + i2++; + if (i2 >= n2) { + return i; + } + } else { // equal + res[i++] = a1[i1++]; + i2++; + if (i1 >= n1 || i2 >= n2) { + return i; + } + } + } +} + +%} + + +%pythoncode %{ + +import numpy as np + +# example additional function that converts the passed-in numpy arrays to +# C++ pointers +def intersect_sorted(a1, a2): + n1, = a1.shape + n2, = a2.shape + res = np.empty(n1 + n2, dtype=a1.dtype) + nres = intersect_sorted_c( + n1, faiss.swig_ptr(a1), + n2, faiss.swig_ptr(a2), + faiss.swig_ptr(res) + ) + return res[:nres] + +%} \ No newline at end of file diff --git a/neurips23/filter/faiss/config.yaml b/neurips23/filter/faiss/config.yaml new file mode 100644 index 00000000..62cc5b24 --- /dev/null +++ b/neurips23/filter/faiss/config.yaml @@ -0,0 +1,74 @@ +random-filter-s: + faiss: + docker-tag: neurips23-filter-faiss + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF1024,SQ8"}] + query-args: | + [{"nprobe": 1}, + {"nprobe":2}, + {"nprobe":4}] +random-s: + faiss: + docker-tag: neurips23-filter-faiss + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF1024,SQ8"}] + query-args: | + [{"nprobe": 1}, + {"nprobe":2}, + {"nprobe":4}] +yfcc-10M-unfiltered: + faiss: + docker-tag: neurips23-filter-faiss + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF16384,SQ8", "binarysig": true, "threads": 16}] + query-args: | + [{"nprobe": 1}, {"nprobe": 4}, {"nprobe": 16}, {"nprobe": 64}] +yfcc-10M: + faiss: + docker-tag: neurips23-filter-faiss + module: neurips23.filter.faiss.faiss + constructor: FAISS + base-args: ["@metric"] + run-groups: + base: + args: | + [{"indexkey": "IVF16384,SQ8", + "binarysig": true, + "threads": 16 + }] + query-args: | + [{"nprobe": 1, "mt_threshold":0.0003}, + {"nprobe": 4, "mt_threshold":0.0003}, + {"nprobe": 16, "mt_threshold":0.0003}, + {"nprobe": 32, "mt_threshold":0.0003}, + {"nprobe": 64, "mt_threshold":0.0003}, + {"nprobe": 96, "mt_threshold":0.0003}, + {"nprobe": 1, "mt_threshold":0.0001}, + {"nprobe": 4, "mt_threshold":0.0001}, + {"nprobe": 16, "mt_threshold":0.0001}, + {"nprobe": 32, "mt_threshold":0.0001}, + {"nprobe": 64, "mt_threshold":0.0001}, + {"nprobe": 96, "mt_threshold":0.0001}, + {"nprobe": 1, "mt_threshold":0.01}, + {"nprobe": 4, "mt_threshold":0.01}, + {"nprobe": 16, "mt_threshold":0.01}, + {"nprobe": 32, "mt_threshold":0.01}, + {"nprobe": 64, "mt_threshold":0.01}, + {"nprobe": 96, "mt_threshold":0.01} + ] + diff --git a/neurips23/filter/faiss/faiss.py b/neurips23/filter/faiss/faiss.py new file mode 100644 index 00000000..02980d12 --- /dev/null +++ b/neurips23/filter/faiss/faiss.py @@ -0,0 +1,287 @@ +import pdb +import pickle +import numpy as np +import os + +from multiprocessing.pool import ThreadPool + +import faiss + +from neurips23.filter.base import BaseFilterANN +from benchmark.datasets import DATASETS +from benchmark.dataset_io import download_accelerated + +import bow_id_selector + +def csr_get_row_indices(m, i): + """ get the non-0 column indices for row i in matrix m """ + return m.indices[m.indptr[i] : m.indptr[i + 1]] + +def make_bow_id_selector(mat, id_mask=0): + sp = faiss.swig_ptr + if id_mask == 0: + return bow_id_selector.IDSelectorBOW(mat.shape[0], sp(mat.indptr), sp(mat.indices)) + else: + return bow_id_selector.IDSelectorBOWBin( + mat.shape[0], sp(mat.indptr), sp(mat.indices), id_mask + ) + +def set_invlist_ids(invlists, l, ids): + n, = ids.shape + ids = np.ascontiguousarray(ids, dtype='int64') + assert invlists.list_size(l) == n + faiss.memcpy( + invlists.get_ids(l), + faiss.swig_ptr(ids), n * 8 + ) + + + +def csr_to_bitcodes(matrix, bitsig): + """ Compute binary codes for the rows of the matrix: each binary code is + the OR of bitsig for non-0 entries of the row. + """ + indptr = matrix.indptr + indices = matrix.indices + n = matrix.shape[0] + bit_codes = np.zeros(n, dtype='int64') + for i in range(n): + # print(bitsig[indices[indptr[i]:indptr[i + 1]]]) + bit_codes[i] = np.bitwise_or.reduce(bitsig[indices[indptr[i]:indptr[i + 1]]]) + return bit_codes + + +class BinarySignatures: + """ binary signatures that encode vectors """ + + def __init__(self, meta_b, proba_1): + nvec, nword = meta_b.shape + # number of bits reserved for the vector ids + self.id_bits = int(np.ceil(np.log2(nvec))) + # number of bits for the binary signature + self.sig_bits = nbits = 63 - self.id_bits + + # select binary signatures for the vocabulary + rs = np.random.RandomState(123) # we rely on this to be reproducible! + bitsig = np.packbits(rs.rand(nword, nbits) < proba_1, axis=1) + bitsig = np.pad(bitsig, ((0, 0), (0, 8 - bitsig.shape[1]))).view("int64").ravel() + self.bitsig = bitsig + + # signatures for all the metadata matrix + self.db_sig = csr_to_bitcodes(meta_b, bitsig) << self.id_bits + + # mask to keep only the ids + self.id_mask = (1 << self.id_bits) - 1 + + def query_signature(self, w1, w2): + """ compute the query signature for 1 or 2 words """ + sig = self.bitsig[w1] + if w2 != -1: + sig |= self.bitsig[w2] + return int(sig << self.id_bits) + +class FAISS(BaseFilterANN): + + def __init__(self, metric, index_params): + self._index_params = index_params + self._metric = metric + print(index_params) + self.indexkey = index_params.get("indexkey", "IVF32768,SQ8") + self.binarysig = index_params.get("binarysig", True) + self.binarysig_proba1 = index_params.get("binarysig_proba1", 0.1) + self.metadata_threshold = 1e-3 + self.nt = index_params.get("threads", 1) + + + def fit(self, dataset): + ds = DATASETS[dataset]() + if ds.search_type() == "knn_filtered" and self.binarysig: + print("preparing binary signatures") + meta_b = ds.get_dataset_metadata() + self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) + print("writing to", self.binarysig_name(dataset)) + pickle.dump(self.binsig, open(self.binarysig_name(dataset), "wb"), -1) + else: + self.binsig = None + + if ds.search_type() == "knn_filtered": + self.meta_b = ds.get_dataset_metadata() + self.meta_b.sort_indices() + + index = faiss.index_factory(ds.d, self.indexkey) + xb = ds.get_dataset() + print("train") + index.train(xb) + print("populate") + if self.binsig is None: + index.add(xb) + else: + ids = np.arange(ds.nb) | self.binsig.db_sig + index.add_with_ids(xb, ids) + + self.index = index + self.nb = ds.nb + self.xb = xb + self.ps = faiss.ParameterSpace() + self.ps.initialize(self.index) + print("store", self.index_name(dataset)) + faiss.write_index(index, self.index_name(dataset)) + + + def index_name(self, name): + return f"data/{name}.{self.indexkey}.faissindex" + + def binarysig_name(self, name): + return f"data/{name}.{self.indexkey}.binarysig" + + + def load_index(self, dataset): + """ + Load the index for dataset. Returns False if index + is not available, True otherwise. + + Checking the index usually involves the dataset name + and the index build paramters passed during construction. + """ + if not os.path.exists(self.index_name(dataset)): + if 'url' not in self._index_params: + return False + + print('Downloading index in background. This can take a while.') + download_accelerated(self._index_params['url'], self.index_name(dataset), quiet=True) + + print("Loading index") + + self.index = faiss.read_index(self.index_name(dataset)) + + self.ps = faiss.ParameterSpace() + self.ps.initialize(self.index) + + ds = DATASETS[dataset]() + + if ds.search_type() == "knn_filtered" and self.binarysig: + if not os.path.exists(self.binarysig_name(dataset)): + print("preparing binary signatures") + meta_b = ds.get_dataset_metadata() + self.binsig = BinarySignatures(meta_b, self.binarysig_proba1) + else: + print("loading binary signatures") + self.binsig = pickle.load(open(self.binarysig_name(dataset), "rb")) + else: + self.binsig = None + + if ds.search_type() == "knn_filtered": + self.meta_b = ds.get_dataset_metadata() + self.meta_b.sort_indices() + + self.nb = ds.nb + self.xb = ds.get_dataset() + + return True + + def index_files_to_store(self, dataset): + """ + Specify a triplet with the local directory path of index files, + the common prefix name of index component(s) and a list of + index components that need to be uploaded to (after build) + or downloaded from (for search) cloud storage. + + For local directory path under docker environment, please use + a directory under + data/indices/track(T1 or T2)/algo.__str__()/DATASETS[dataset]().short_name() + """ + raise NotImplementedError() + + def query(self, X, k): + nq = X.shape[0] + self.I = -np.ones((nq, k), dtype='int32') + bs = 1024 + for i0 in range(0, nq, bs): + _, self.I[i0:i0+bs] = self.index.search(X[i0:i0+bs], k) + + + def filtered_query(self, X, filter, k): + print('running filtered query') + nq = X.shape[0] + self.I = -np.ones((nq, k), dtype='int32') + meta_b = self.meta_b + meta_q = filter + docs_per_word = meta_b.T.tocsr() + ndoc_per_word = docs_per_word.indptr[1:] - docs_per_word.indptr[:-1] + freq_per_word = ndoc_per_word / self.nb + + def process_one_row(q): + faiss.omp_set_num_threads(1) + qwords = csr_get_row_indices(meta_q, q) + assert qwords.size in (1, 2) + w1 = qwords[0] + freq = freq_per_word[w1] + if qwords.size == 2: + w2 = qwords[1] + freq *= freq_per_word[w2] + else: + w2 = -1 + if freq < self.metadata_threshold: + # metadata first + docs = csr_get_row_indices(docs_per_word, w1) + if w2 != -1: + docs = bow_id_selector.intersect_sorted( + docs, csr_get_row_indices(docs_per_word, w2)) + + assert len(docs) >= k, pdb.set_trace() + xb_subset = self.xb[docs] + _, Ii = faiss.knn(X[q : q + 1], xb_subset, k=k) + + self.I[q, :] = docs[Ii.ravel()] + else: + # IVF first, filtered search + sel = make_bow_id_selector(meta_b, self.binsig.id_mask if self.binsig else 0) + if self.binsig is None: + sel.set_query_words(int(w1), int(w2)) + else: + sel.set_query_words_mask( + int(w1), int(w2), self.binsig.query_signature(w1, w2)) + + params = faiss.SearchParametersIVF(sel=sel, nprobe=self.nprobe) + + _, Ii = self.index.search( + X[q:q+1], k, params=params + ) + Ii = Ii.ravel() + if self.binsig is None: + self.I[q] = Ii + else: + # we'll just assume there are enough results + # valid = Ii != -1 + # I[q, valid] = Ii[valid] & binsig.id_mask + self.I[q] = Ii & self.binsig.id_mask + + + if self.nt <= 1: + for q in range(nq): + process_one_row(q) + else: + faiss.omp_set_num_threads(self.nt) + pool = ThreadPool(self.nt) + list(pool.map(process_one_row, range(nq))) + + def get_results(self): + return self.I + + def set_query_arguments(self, query_args): + faiss.cvar.indexIVF_stats.reset() + if "nprobe" in query_args: + self.nprobe = query_args['nprobe'] + self.ps.set_index_parameters(self.index, f"nprobe={query_args['nprobe']}") + self.qas = query_args + else: + self.nprobe = 1 + if "mt_threshold" in query_args: + self.metadata_threshold = query_args['mt_threshold'] + else: + self.metadata_threshold = 1e-3 + + def __str__(self): + return f'Faiss({self.indexkey, self.qas})' + + \ No newline at end of file diff --git a/neurips23/filter/wm_filter/Dockerfile b/neurips23/filter/wm_filter/Dockerfile new file mode 100644 index 00000000..cf63f4a4 --- /dev/null +++ b/neurips23/filter/wm_filter/Dockerfile @@ -0,0 +1,28 @@ +FROM neurips23 + +RUN apt-get update; DEBIAN_FRONTEND=noninteractive apt install intel-mkl python3-setuptools wget python3-matplotlib build-essential checkinstall libssl-dev swig4.0 python3-dev python3-numpy python3-numpy-dev -y +COPY install/requirements_conda.txt ./ +# conda doesn't like some of our packages, use pip +RUN python3 -m pip install -r requirements_conda.txt + + +# CMAKE with good enough version +RUN mkdir /build && wget https://github.com/Kitware/CMake/archive/refs/tags/v3.27.1.tar.gz && mv v3.27.1.tar.gz /build +RUN cd /build; tar -zxvf v3.27.1.tar.gz +RUN cd /build/CMake-3.27.1 && ./bootstrap && make && make install + + +RUN cd / && git clone https://github.com/alemagnani/faiss.git && cd /faiss && git pull && git checkout wm_filter + +RUN cd /faiss && rm -rf ./build +RUN cd /faiss/; cmake -B build /faiss/ -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DFAISS_OPT_LEVEL=avx2 -DBLA_VENDOR=Intel10_64_dyn -DBUILD_TESTING=ON -DPython_EXECUTABLE=/usr/bin/python3 -DMKL_LIBRARIES=/usr/lib/x86_64-linux-gnu/libmkl_rt.so +RUN cd /faiss/; make -C build -j faiss faiss_avx2 swigfaiss swigfaiss_avx2 +RUN (cd /faiss/build/faiss/python && python3 setup.py install) + +#RUN pip install tritonclient[all] +ENV PYTHONPATH=/faiss/build/faiss/python/build/lib/ + +RUN python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)' + + + diff --git a/neurips23/filter/wm_filter/README.md b/neurips23/filter/wm_filter/README.md new file mode 100644 index 00000000..0d451065 --- /dev/null +++ b/neurips23/filter/wm_filter/README.md @@ -0,0 +1,7 @@ + +### Submission for Neurips23 Filter track of WM_filter team +This submission leverages the IVF index to run the filter in a fast way. + +More info to come... + + diff --git a/neurips23/filter/wm_filter/config.yaml b/neurips23/filter/wm_filter/config.yaml new file mode 100644 index 00000000..4397152e --- /dev/null +++ b/neurips23/filter/wm_filter/config.yaml @@ -0,0 +1,50 @@ +random-filter-s: + wm_filter: + docker-tag: neurips23-filter-wm_filter + module: neurips23.filter.wm_filter.wm_filter + constructor: FAISS + base-args: [ "@metric" ] + run-groups: + base: + args: | + [{"indexkey": "IVF1024,SQ8", + "threads": 8, + "train_size": 2000000, + "type": "direct" + }] + query-args: | + [ + {"nprobe": 80, "max_codes": 100, "selector_probe_limit": 80}, + {"nprobe": 100, "max_codes": 500, "selector_probe_limit": 100}, + {"nprobe": 120, "max_codes": 1000, "selector_probe_limit": 120}, + {"nprobe": 140, "max_codes": 1800, "selector_probe_limit": 140}, + {"nprobe": 160, "max_codes": 500, "selector_probe_limit": 160}, + {"nprobe": 70, "max_codes": 1000, "selector_probe_limit": 70} + ] +yfcc-10M: + wm_filter: + docker-tag: neurips23-filter-wm_filter + module: neurips23.filter.wm_filter.wm_filter + constructor: FAISS + base-args: [ "@metric" ] + run-groups: + base: + args: | + [{"indexkey": "IVF1024,SQ8", + "threads": 8, + "train_size": 2000000, + "type": "direct" + }] + query-args: | + [ + {"nprobe": 80, "max_codes": 1800, "selector_probe_limit": 80}, + {"nprobe": 100, "max_codes": 1800, "selector_probe_limit": 100}, + {"nprobe": 120, "max_codes": 1800, "selector_probe_limit": 120}, + {"nprobe": 140, "max_codes": 1800, "selector_probe_limit": 140}, + {"nprobe": 160, "max_codes": 1800, "selector_probe_limit": 160}, + {"nprobe": 70, "max_codes": 2100, "selector_probe_limit": 70}, + {"nprobe": 100, "max_codes": 2100, "selector_probe_limit": 100}, + {"nprobe": 130, "max_codes": 2100, "selector_probe_limit": 130}, + {"nprobe": 160, "max_codes": 2100, "selector_probe_limit": 160}, + {"nprobe": 200, "max_codes": 2100, "selector_probe_limit": 200} + ] diff --git a/neurips23/filter/wm_filter/wm_filter.py b/neurips23/filter/wm_filter/wm_filter.py new file mode 100644 index 00000000..671b19de --- /dev/null +++ b/neurips23/filter/wm_filter/wm_filter.py @@ -0,0 +1,572 @@ +import pdb +import pickle +import numpy as np +import os + +from multiprocessing.pool import ThreadPool +from threading import current_thread + +import faiss + + +from faiss.contrib.inspect_tools import get_invlist +from neurips23.filter.base import BaseFilterANN +from benchmark.datasets import DATASETS +from benchmark.dataset_io import download_accelerated +from math import log10, pow + + +def csr_get_row_indices(m, i): + """ get the non-0 column indices for row i in matrix m """ + return m.indices[m.indptr[i] : m.indptr[i + 1]] + +def make_id_selector_ivf_two(docs_per_word): + sp = faiss.swig_ptr + return faiss.IDSelectorIVFTwo(sp(docs_per_word.indices), sp(docs_per_word.indptr)) + +def make_id_selector_cluster_aware(indices, limits, clusters, cluster_limits): + sp = faiss.swig_ptr + return faiss.IDSelectorIVFClusterAware(sp(indices), sp(limits), sp(clusters), sp(cluster_limits)) + +def make_id_selector_cluster_aware_intersect(indices, limits, clusters, cluster_limits, tmp_size): + sp = faiss.swig_ptr + return faiss.IDSelectorIVFClusterAwareIntersect(sp(indices), sp(limits), sp(clusters), sp(cluster_limits), int(tmp_size)) + +def make_id_selector_cluster_aware_direct(id_position_in_cluster, limits, clusters, cluster_limits, tmp_size): + sp = faiss.swig_ptr + return faiss.IDSelectorIVFClusterAwareIntersectDirect(sp(id_position_in_cluster), sp(limits), sp(clusters), sp(cluster_limits), int(tmp_size)) + +def make_id_selector_cluster_aware_direct_exp(id_position_in_cluster, limits, nprobes, tmp_size): + sp = faiss.swig_ptr + return faiss.IDSelectorIVFClusterAwareIntersectDirectExp(sp(id_position_in_cluster), sp(limits), int(nprobes), int(tmp_size)) + + +def find_invlists(index): + try: + inverted_lists = index.invlists + except: + base_index = faiss.downcast_index(index.base_index) + print('cannot find the inverted list trying one level down') + print('type of index', type(base_index)) + inverted_lists = base_index.invlists + return inverted_lists + +def print_stats(): + m = 1000000. + intersection = faiss.cvar.IDSelectorMy_Stats.intersection/m + find_cluster = faiss.cvar.IDSelectorMy_Stats.find_cluster/m + set_list_time = faiss.cvar.IDSelectorMy_Stats.set_list_time/m + scan_codes = faiss.cvar.IDSelectorMy_Stats.scan_codes/m + one_list = faiss.cvar.IDSelectorMy_Stats.one_list/m + extra = faiss.cvar.IDSelectorMy_Stats.extra / m + inter_plus_find = intersection + find_cluster + print('intersection: {}, find_cluster: {}, intersection+ find cluster: {}, set list time: {}, scan_codes: {}, one list: {}, extra: {}'.format(intersection, find_cluster, inter_plus_find, set_list_time, scan_codes, one_list, extra)) + + +def spot_check_filter(docs_per_word, index, indices, limits, clusters, cluster_limits): + print('running spot check') + + + inverted_lists = find_invlists(index) + + from_id_to_map = dict() + for i in range(inverted_lists.nlist): + list_ids, _ = get_invlist(inverted_lists, i) + for id in list_ids: + from_id_to_map[id] = i + + indptr = docs_per_word.indptr + + ## lets' run some spot check + for word in [0, 5, 7]: + #for word in range(docs_per_word.shape[0]): + #for word in [docs_per_word.shape[0]-1 ]: + c_start = cluster_limits[word] + c_end = cluster_limits[word + 1] + assert c_end >= c_start + + start = indptr[word] + end = indptr[word + 1] + ids_in_word = {id for id in docs_per_word.indices[start:end]} + + cluster_base = -1 + for pos, cluster in enumerate(clusters[c_start: c_end]): + if cluster_base == -1: + cluster_base = cluster + else: + assert cluster != cluster_base + cluster_base = cluster + for id in indices[limits[c_start + pos]: limits[c_start + pos + 1]]: + assert from_id_to_map[id] == cluster + assert id in ids_in_word + ids_in_word.remove(id) + assert len(ids_in_word) == 0 # we should have covered all the ids in the word with the clusters + + +def spot_check_filter_exp(docs_per_word, index, indices, limits): + print('running spot check') + + + inverted_lists = find_invlists(index) + + from_id_to_map = dict() + for i in range(inverted_lists.nlist): + list_ids, _ = get_invlist(inverted_lists, i) + for id in list_ids: + from_id_to_map[id] = i + + indptr = docs_per_word.indptr + + nprobes = inverted_lists.nlist + + ## lets' run some spot check + for word in [0, 5000, 12124, 151123, 198000]: + #for word in range(docs_per_word.shape[0]): + #for word in [docs_per_word.shape[0]-1 ]: + local_ids_to_cluster = dict() + #print(limits[nprobes * word: nprobes * word + nprobes]) + for cluster in range(nprobes): + c_start = limits[word * nprobes + cluster] + c_end = limits[word * nprobes + cluster+1] + + if c_end >=0 and c_start >=0 and c_end > c_start: + for id in indices[c_start: c_end]: + local_ids_to_cluster[id] = cluster + + + + start = indptr[word] + end = indptr[word + 1] + ids_in_word = {id for id in docs_per_word.indices[start:end]} + print(len(ids_in_word), len(local_ids_to_cluster)) + assert len(ids_in_word) == len(local_ids_to_cluster) + for id in ids_in_word: + cluster_found = from_id_to_map[id] + assert cluster_found == local_ids_to_cluster[id] + print('done checking word ', word) + + print('done spot check') + + +def find_max_interval(limits): + + out = -1 + for i in range(len(limits)-1): + delta = limits[i+1] - limits[i] + if delta > out: + out = delta + return out + + +def prepare_filter_by_cluster(docs_per_word, index): + print('creating filter cluster') + inverted_lists = find_invlists(index) + from_id_to_map = dict() + from_id_to_pos = dict() + for i in range(inverted_lists.nlist): + list_ids, _ = get_invlist(inverted_lists, i) + for pos, id in enumerate(list_ids): + #print('list: ', i, "id: ", id, "pos: ",pos) + from_id_to_map[id] = i + from_id_to_pos[id] = pos + print('loaded the mapping with {} entries'.format(len(from_id_to_map))) + + ## reorganize the docs per word + # + cluster_limits = [0] + clusters = list() + limits = list() + id_position_in_cluster = list() + + indices = np.array(docs_per_word.indices) + indptr = docs_per_word.indptr + for word in range(docs_per_word.shape[0]): + start = indptr[word] + end = indptr[word + 1] + if word % 10000 == 0: + print('processed {} words'.format(word)) + array_ind_cluster = [(id, from_id_to_map[id]) for id in indices[start:end]] + array_ind_cluster.sort(key=lambda x: x[1]) + + if len(array_ind_cluster) == 0: + pass + local_clusters = [] + local_limits = [] + current_cluster = -1 + for pos, arr in enumerate(array_ind_cluster): + id, cluster = arr + if current_cluster == -1 or cluster != current_cluster: + current_cluster = cluster + local_clusters.append(cluster) + local_limits.append(start + pos) + indices[start + pos] = id + id_position_in_cluster.append(from_id_to_pos[id]) + + clusters.extend(local_clusters) + limits.extend(local_limits) + new_cluster_limit = len(local_clusters) + cluster_limits[-1] + cluster_limits.append( new_cluster_limit) + limits.append(len(indices)) + + clusters = np.array(clusters, dtype=np.int16) + limits = np.array(limits, dtype=np.int32) + cluster_limits = np.array(cluster_limits, dtype=np.int32) + id_position_in_cluster = np.array(id_position_in_cluster, dtype=np.int32) + + return indices, limits, clusters, cluster_limits, id_position_in_cluster + + +def prepare_filter_by_cluster_exp(docs_per_word, index): + print('creating filter cluster expanded') + inverted_lists = find_invlists(index) + from_id_to_map = dict() + from_id_to_pos = dict() + + nprobes = inverted_lists.nlist + for i in range(inverted_lists.nlist): + list_ids, _ = get_invlist(inverted_lists, i) + for pos, id in enumerate(list_ids): + #print('list: ', i, "id: ", id, "pos: ",pos) + from_id_to_map[id] = i + from_id_to_pos[id] = pos + print('loaded the mapping with {} entries'.format(len(from_id_to_map))) + + ## reorganize the docs per word + # + + limits = -np.ones( (docs_per_word.shape[0] * nprobes + 1,), dtype=np.int32) + id_position_in_cluster = list() + + indices = np.array(docs_per_word.indices) + indptr = docs_per_word.indptr + for word in range(docs_per_word.shape[0]): + start = indptr[word] + end = indptr[word + 1] + if word % 10000 == 0: + print('processed {} words'.format(word)) + array_ind_cluster = [(id, from_id_to_map[id]) for id in indices[start:end]] + array_ind_cluster.sort(key=lambda x: x[1]) + + + + local_limits = [] + current_cluster = -1 + + for pos, arr in enumerate(array_ind_cluster): + id, cluster = arr + if current_cluster == -1 or cluster != current_cluster: + + if current_cluster != -1: + limits[word * nprobes + current_cluster + 1] = start + pos + + + current_cluster = cluster + local_limits.append(start + pos) + + limits[word * nprobes + current_cluster] = start + pos + + indices[start + pos] = id + id_position_in_cluster.append(from_id_to_pos[id]) + + limits[word * nprobes + current_cluster + 1] = start + len(array_ind_cluster) + + + limits = np.array(limits, dtype=np.int32) + + id_position_in_cluster = np.array(id_position_in_cluster, dtype=np.int32) + + return indices, limits, id_position_in_cluster, nprobes + + +class FAISS(BaseFilterANN): + + def __init__(self, metric, index_params): + self._index_params = index_params + self._metric = metric + + self.train_size = index_params.get('train_size', None) + self.indexkey = index_params.get("indexkey", "IVF32768,SQ8") + self.metadata_threshold = 1e-3 + self.nt = index_params.get("threads", 1) + self.type = index_params.get("type", "intersect") + + self.clustet_dist = [] + + + def fit(self, dataset): + faiss.omp_set_num_threads(self.nt) + ds = DATASETS[dataset]() + + print('the size of the index', ds.d) + index = faiss.index_factory(ds.d, self.indexkey) + xb = ds.get_dataset() + + print("train") + print('train_size', self.train_size) + if self.train_size is not None: + x_train = xb[:self.train_size] + else: + x_train = xb + index.train(x_train) + print("populate") + + bs = 1024 + for i0 in range(0, ds.nb, bs): + index.add(xb[i0: i0 + bs]) + + + print('ids added') + self.index = index + self.nb = ds.nb + self.xb = xb + self.ps = faiss.ParameterSpace() + self.ps.initialize(self.index) + print("store", self.index_name(dataset)) + faiss.write_index(index, self.index_name(dataset)) + + if ds.search_type() == "knn_filtered": + words_per_doc = ds.get_dataset_metadata() + words_per_doc.sort_indices() + self.docs_per_word = words_per_doc.T.tocsr() + self.docs_per_word.sort_indices() + self.ndoc_per_word = self.docs_per_word.indptr[1:] - self.docs_per_word.indptr[:-1] + self.freq_per_word = self.ndoc_per_word / self.nb + del words_per_doc + + if self.type == 'exp': + self.indices, self.limits, self.id_position_in_cluster, self.total_clusters = prepare_filter_by_cluster_exp( + self.docs_per_word, self.index) + pickle.dump( + (self.indices, self.limits, self.id_position_in_cluster, self.total_clusters ), + open(self.cluster_sig_name(dataset), "wb"), -1) + #spot_check_filter_exp(self.docs_per_word, self.index, self.indices, self.limits) + else: + self.indices, self.limits, self.clusters, self.cluster_limits, self.id_position_in_cluster = prepare_filter_by_cluster(self.docs_per_word, self.index) + print('dumping cluster map') + pickle.dump((self.indices, self.limits, self.clusters, self.cluster_limits, self.id_position_in_cluster), open(self.cluster_sig_name(dataset), "wb"), -1) + #spot_check_filter(self.docs_per_word, self.index, self.indices, self.limits, self.clusters, + # self.cluster_limits) + + self.max_range = find_max_interval(self.limits) + print('the max range is {}'.format(self.max_range)) + + def index_name(self, name): + + if self.type == 'exp': + return f"data/{name}.{self.indexkey}_exp_wm.faissindex" + else: + return f"data/{name}.{self.indexkey}_wm.faissindex" + + + def cluster_sig_name(self, name): + if self.type == 'exp': + return f"data/{name}.{self.indexkey}_exp_cluster_wm.pickle" + return f"data/{name}.{self.indexkey}_cluster_wm.pickle" + + + def get_probes(self, freq, a, b, min_prob = 4, max_prob=256): + #print("b: ", b) + probes = int( pow(2, - a * log10(freq )+ b)) + probes = max(min_prob, probes) + probes = min(max_prob, probes) + return probes + + def load_index(self, dataset): + """ + Load the index for dataset. Returns False if index + is not available, True otherwise. + + Checking the index usually involves the dataset name + and the index build paramters passed during construction. + """ + if not os.path.exists(self.index_name(dataset)): + if 'url' not in self._index_params: + return False + + print('Downloading index in background. This can take a while.') + download_accelerated(self._index_params['url'], self.index_name(dataset), quiet=True) + + print("Loading index") + ds = DATASETS[dataset]() + self.nb = ds.nb + self.xb = ds.get_dataset() + + if ds.search_type() == "knn_filtered": + words_per_doc = ds.get_dataset_metadata() + words_per_doc.sort_indices() + self.docs_per_word = words_per_doc.T.tocsr() + self.docs_per_word.sort_indices() + self.ndoc_per_word = self.docs_per_word.indptr[1:] - self.docs_per_word.indptr[:-1] + self.freq_per_word = self.ndoc_per_word / self.nb + del words_per_doc + + self.index = faiss.read_index(self.index_name(dataset)) + + if ds.search_type() == "knn_filtered": + if os.path.isfile( self.cluster_sig_name(dataset)): + print('loading cluster file') + if self.type == 'exp': + self.indices, self.limits, self.id_position_in_cluster, self.total_clusters = pickle.load( + open(self.cluster_sig_name(dataset), "rb")) + #spot_check_filter_exp(self.docs_per_word, self.index, self.indices, self.limits) + + else: + self.indices, self.limits, self.clusters, self.cluster_limits, self.id_position_in_cluster = pickle.load(open(self.cluster_sig_name(dataset), "rb")) + else: + print('cluster file not found') + if self.type == 'exp': + self.indices, self.limits, self.id_position_in_cluster, self.total_clusters = prepare_filter_by_cluster_exp( + self.docs_per_word, self.index) + pickle.dump( + (self.indices, self.limits, self.id_position_in_cluster, self.total_clusters ), + open(self.cluster_sig_name(dataset), "wb"), -1) + #spot_check_filter_exp(self.docs_per_word, self.index, self.indices, self.limits) + + else: + self.indices, self.limits, self.clusters, self.cluster_limits, self.id_position_in_cluster = prepare_filter_by_cluster(self.docs_per_word, self.index) + pickle.dump((self.indices, self.limits, self.clusters, self.cluster_limits, self.id_position_in_cluster), open(self.cluster_sig_name(dataset), "wb"), -1) + + #spot_check_filter(self.docs_per_word, self.index, self.indices, self.limits, self.clusters, self.cluster_limits) + + self.max_range = find_max_interval(self.limits) + print('the max range is {}'.format(self.max_range)) + + self.ps = faiss.ParameterSpace() + self.ps.initialize(self.index) + + + # delete not necessary data + del self.xb + del ds + if self.type == "exp" or self.type == 'direct': + print(" deleting indices") + del self.indices + #del self.docs_per_word + return True + + def index_files_to_store(self, dataset): + """ + Specify a triplet with the local directory path of index files, + the common prefix name of index component(s) and a list of + index components that need to be uploaded to (after build) + or downloaded from (for search) cloud storage. + + For local directory path under docker environment, please use + a directory under + data/indices/track(T1 or T2)/algo.__str__()/DATASETS[dataset]().short_name() + """ + raise NotImplementedError() + + def query(self, X, k): + nq = X.shape[0] + self.I = -np.ones((nq, k), dtype='int32') + bs = 1024 + + try: + print('k_factor', self.index.k_factor) + self.index.k_factor = self.k_factor + except Exception as e: + print(e) + pass + for i0 in range(0, nq, bs): + _, self.I[i0:i0+bs] = self.index.search(X[i0:i0+bs], k) + + + + def filtered_query(self, X, filter, k): + + # try: + # self.index.k_factor = self.k_factor + # except Exception as e: + # pass + + nq = X.shape[0] + self.I = -np.ones((nq, k), dtype='int32') + + meta_q = filter + selector_by_thread = dict() + + def process_one_row(q): + faiss.omp_set_num_threads(1) + thread = current_thread() + + qwords = csr_get_row_indices(meta_q, q) + w1 = qwords[0] + if qwords.size == 2: + w2 = qwords[1] + else: + w2 = -1 + + if thread not in selector_by_thread: + + sel = make_id_selector_cluster_aware_direct(self.id_position_in_cluster, self.limits, self.clusters, + self.cluster_limits, self.max_range) + # # IVF first, filtered search + # if self.type == 'simple': + # sel = make_id_selector_ivf_two(self.docs_per_word) + # elif self.type == "aware": + # sel = make_id_selector_cluster_aware(self.indices, self.limits, self.clusters, self.cluster_limits) + # elif self.type == 'intersect': + # sel = make_id_selector_cluster_aware_intersect(self.indices, self.limits, self.clusters, self.cluster_limits, self.max_range) + # elif self.type == 'direct': + # sel = make_id_selector_cluster_aware_direct(self.id_position_in_cluster, self.limits, self.clusters, + # self.cluster_limits, self.max_range) + # elif self.type == 'exp': + # sel = make_id_selector_cluster_aware_direct_exp(self.id_position_in_cluster, self.limits, self.total_clusters, self.max_range) + # else: + # raise Exception('unknown type ', self.type) + selector_by_thread[thread] = sel + else: + sel = selector_by_thread.get(thread) + + sel.set_words(int(w1), int(w2)) + + params = faiss.SearchParametersIVF(sel=sel, nprobe=self.nprobe, max_codes=self.max_codes, selector_probe_limit=self.selector_probe_limit) + _, Ii = self.index.search( X[q:q+1], k, params=params) + Ii = Ii.ravel() + self.I[q] = Ii + + if self.nt <= 1: + for q in range(nq): + process_one_row(q) + else: + faiss.omp_set_num_threads(self.nt) + + pool = ThreadPool(self.nt) + list(pool.map(process_one_row, range(nq))) + + def get_results(self): + return self.I + + def set_query_arguments(self, query_args): + #faiss.cvar.indexIVF_stats.reset() + #faiss.cvar.IDSelectorMy_Stats.reset() + if "nprobe" in query_args: + self.nprobe = query_args['nprobe'] + self.ps.set_index_parameters(self.index, f"nprobe={query_args['nprobe']}") + self.qas = query_args + else: + self.nprobe = 1 + if "max_codes" in query_args: + self.max_codes = query_args["max_codes"] + self.ps.set_index_parameters(self.index, f"max_codes={query_args['max_codes']}") + self.qas = query_args + else: + self.max_codes = -1 + if "selector_probe_limit" in query_args: + self.selector_probe_limit = query_args['selector_probe_limit'] + self.ps.set_index_parameters(self.index, f"selector_probe_limit={query_args['selector_probe_limit']}") + self.qas = query_args + else: + self.selector_probe_limit = 0 + + if "k_factor" in query_args: + self.k_factor = query_args['k_factor'] + self.qas = query_args + + + + def __str__(self): + return f'Faiss({self.indexkey,self.type, self.qas})' + + \ No newline at end of file