From 6ec0eafeea1cc8efb5d18d2e9f2d361e1f3f901c Mon Sep 17 00:00:00 2001 From: Giovanni Cherubin Date: Tue, 8 Nov 2022 14:35:23 +0000 Subject: [PATCH] Initial commit. --- .github/workflows/codeql.yml | 74 ++++ .gitignore | 167 +++++++ CODE_OF_CONDUCT.md | 9 + LICENSE | 21 + README.md | 165 +++++++ SECURITY.md | 41 ++ SUPPORT.md | 13 + codalab-package/README.md | 51 +++ codalab-package/competition.yaml | 172 ++++++++ codalab-package/data.md | 117 +++++ codalab-package/evaluation.md | 37 ++ codalab-package/logo.png | Bin 0 -> 42416 bytes codalab-package/overview.md | 47 ++ codalab-package/terms.md | 10 + .../utilities/make_competition_bundle.sh | 66 +++ environment.yaml | 16 + md5sums.md | 7 + pyproject.toml | 3 + requirements.txt | 8 + setup.cfg | 23 + setup.py | 4 + src/mico-competition/__init__.py | 12 + src/mico-competition/challenge_datasets.py | 99 +++++ src/mico-competition/mico.py | 263 +++++++++++ src/mico-competition/scoring/__init__.py | 10 + src/mico-competition/scoring/metadata | 2 + src/mico-competition/scoring/score.py | 140 ++++++ src/mico-competition/scoring/score_html.py | 128 ++++++ starting-kit/cifar10.ipynb | 302 +++++++++++++ starting-kit/ddp.ipynb | 337 ++++++++++++++ starting-kit/purchase100.ipynb | 301 +++++++++++++ starting-kit/requirements-starting-kit.txt | 3 + starting-kit/sst2.ipynb | 333 ++++++++++++++ training/accountant.py | 54 +++ training/requirements-cifar10.txt | 3 + training/requirements-purchase100.txt | 3 + training/requirements-sst2.txt | 5 + training/train_cifar10.py | 417 ++++++++++++++++++ training/train_cifar10_ddp.sh | 16 + training/train_cifar10_hi.sh | 17 + training/train_cifar10_inf.sh | 13 + training/train_cifar10_lo.sh | 16 + training/train_purchase100.py | 417 ++++++++++++++++++ training/train_purchase100_ddp.sh | 17 + training/train_purchase100_hi.sh | 17 + training/train_purchase100_inf.sh | 14 + training/train_purchase100_lo.sh | 17 + training/train_sst2.py | 212 +++++++++ training/train_sst2_ddp.sh | 23 + training/train_sst2_hi.sh | 23 + training/train_sst2_inf.sh | 21 + training/train_sst2_lo.sh | 23 + 52 files changed, 4309 insertions(+) create mode 100644 .github/workflows/codeql.yml create mode 100644 .gitignore create mode 100644 CODE_OF_CONDUCT.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 SUPPORT.md create mode 100644 codalab-package/README.md create mode 100644 codalab-package/competition.yaml create mode 100644 codalab-package/data.md create mode 100644 codalab-package/evaluation.md create mode 100644 codalab-package/logo.png create mode 100644 codalab-package/overview.md create mode 100644 codalab-package/terms.md create mode 100644 codalab-package/utilities/make_competition_bundle.sh create mode 100644 environment.yaml create mode 100644 md5sums.md create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/mico-competition/__init__.py create mode 100644 src/mico-competition/challenge_datasets.py create mode 100644 src/mico-competition/mico.py create mode 100644 src/mico-competition/scoring/__init__.py create mode 100644 src/mico-competition/scoring/metadata create mode 100644 src/mico-competition/scoring/score.py create mode 100644 src/mico-competition/scoring/score_html.py create mode 100644 starting-kit/cifar10.ipynb create mode 100644 starting-kit/ddp.ipynb create mode 100644 starting-kit/purchase100.ipynb create mode 100644 starting-kit/requirements-starting-kit.txt create mode 100644 starting-kit/sst2.ipynb create mode 100644 training/accountant.py create mode 100644 training/requirements-cifar10.txt create mode 100644 training/requirements-purchase100.txt create mode 100644 training/requirements-sst2.txt create mode 100644 training/train_cifar10.py create mode 100755 training/train_cifar10_ddp.sh create mode 100755 training/train_cifar10_hi.sh create mode 100755 training/train_cifar10_inf.sh create mode 100755 training/train_cifar10_lo.sh create mode 100644 training/train_purchase100.py create mode 100755 training/train_purchase100_ddp.sh create mode 100755 training/train_purchase100_hi.sh create mode 100755 training/train_purchase100_inf.sh create mode 100755 training/train_purchase100_lo.sh create mode 100644 training/train_sst2.py create mode 100755 training/train_sst2_ddp.sh create mode 100755 training/train_sst2_hi.sh create mode 100755 training/train_sst2_inf.sh create mode 100755 training/train_sst2_lo.sh diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..67bb653 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,74 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '35 12 * * 4' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # πŸ“š See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..12e50da --- /dev/null +++ b/.gitignore @@ -0,0 +1,167 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Compressed archives +*.zip +*.tar.gz + +# Folders +tmp/ diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f9ba8cf --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,9 @@ +# Microsoft Open Source Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9e841e7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/README.md b/README.md new file mode 100644 index 0000000..d5f96d5 --- /dev/null +++ b/README.md @@ -0,0 +1,165 @@ +# MICO + +Welcome to the Microsoft Membership Inference Competition (MICO)! +In this competition, you will evaluate the effectiveness of differentially private model training as a mitigation against white-box membership inference attacks. + + +## What is Membership Inference? + +Membership inference is a widely-studied class of threats against Machine Learning (ML) models. +The goal of a membership inference attack is to infer whether a given record was used to train a specific ML model. +An attacker might have full access to the model and its weights (known as "white-box" access), or might only be able to query the model on inputs of their choice ("black-box" access). +In either case, a successful membership inference attack could have negative consequences, especially if the model was trained on sensitive data. + +Membership inference attacks vary in complexity. +In a simple case, the model might have overfitted to its training data, so that it outputs higher confidence predictions when queried on training records than when queried on records that the model has not seen during training. +Recognizing this, an attacker could simply query the model on records of their interest, establish a threshold on the model's confidence, and infer that records with higher confidence are likely members of the training data. +In a white-box setting, as is the case for this competition, the attacker can use more sophisticated strategies that exploit access to the internals of the model. + + +## What is MICO? + +In MICO, your goal is to perform white-box membership inference against a series of trained ML models that we provide. +Specifically, given a model and a set of *challenge points*, the aim is to decide which of these challenge points were used to train the model. + +You can compete on any of four separate membership inference tasks against classification models for image, text, and tabular data, as well as on a special _Differential Privacy Distinguisher_ task spanning all 3 modalities. +Each task will be scored separately. +You do not need to participate in all of them, and can choose to participate in as many as you like. +Throughout the competition, submissions will be scored on a subset of the evaluation data and ranked on a live scoreboard. +When submission closes, the final scores will be computed on a separate subset of the evaluation data. + +The winner of each task will be eligible for an award of **$2,000 USD** from Microsoft and the runner-up of each task for an award of **$1,000 USD** from Microsoft (in the event of tied entries, these awards may be adjusted). +This competition is co-located with the [IEEE Conference on Secure and Trustworthy Machine Learning (SaTML) 2023](https://satml.org/), and the winners will be invited to present their strategies at the conference. + + +## Task Details + +For each of the four tasks, we provide a set of models trained on different splits of a public dataset. +For each of these models, we provide `m` challenge points; exactly half of which are _members_ (i.e., used to train the model) and half are _non-members_ (i.e., they come from the same dataset, but were not used to train the model). +Your goal is to determine which challenge points are members and which are non-members. + +Each of the first three tasks consists of three different _scenarios_ with increasing difficulty, determined by the differential privacy guarantee of the algorithm used to train target models: $\varepsilon = \infty$, high $\varepsilon$, and low $\varepsilon$. +All scenarios share the same model architecture and are trained for the same number of epochs. +The $\varepsilon = \infty$ scenario uses Stochastic Gradient Descent (SGD) without any differential privacy guarantee, while the high $\varepsilon$ and low $\varepsilon$ scenarios use Differentially-Private SGD with a high and low privacy budget $\varepsilon$, respectively. +The lower the privacy budget $\varepsilon$, the more _private_ the model. + +In the fourth task, the target models span all three modalities (image, text, and tabular data) and are trained with a low privacy budget. +The model architectures and hyperparameters are the same as for first three tasks. +However, we reveal the training data of models except for the `m/2` member challenge points. + + +| Task | Scenario | Dataset | Model Architecture | $\varepsilon$ | Other training points given | +| :--- | :----: | :----: | :----: | :----: | :----: | +| Image | I1 | CIFAR-10 | 4-layer CNN | $\infty$ | No | +| | I2 | CIFAR-10 | 4-layer CNN | High | No | +| | I3 | CIFAR-10 | 4-layer CNN | Low | No | +| Text | X1 | SST-2 | Roberta-Base | $\infty$ | No | +| | X2 | SST-2 | Roberta-Base | High | No | +| | X3 | SST-2 | Roberta-Base | Low | No | +| Tabular Data | T1 | Purchase-100 | 3-layer fully connected NN | $\infty$ | No | +| | T2 | Purchase-100 | 3-layer fully connected NN | High | No | +| | T3 | Purchase-100 | 3-layer fully connected NN | Low | No | +| DP Distinguisher | D1 | CIFAR-10 | 4-layer CNN | Low | Yes | +| | D2 | SST-2 | Roberta-Base | Low | Yes | +| | D3 | Purchase-100 | 3-layer fully connected NN | Low | Yes | + + +## Submissions and Scoring + +Submissions will be ranked based on their performance in white-box membership inference against the provided models. + +There are three sets of challenges: `train`, `dev`, and `final`. +For models in `train`, we reveal the full training dataset, and consequently the ground truth membership data for challenge points. +These models can be used by participants to develop their attacks. +For models in the `dev` and `final` sets, no ground truth is revealed and participants must submit their membership predictions for challenge points. + +During the competition, there will be a live scoreboard based on the `dev` challenges. +The final ranking will be decided on the `final` set; scoring for this dataset will be withheld until the competition ends. + +For each challenge point, the submission must provide a value, indicating the confidence level with which the challenge point is a member. +Each value must be a floating point number in the range `[0.0, 1.0]`, where `1.0` indicates certainty that the challenge point is a member, and `0.0` indicates certainty that it is a non-member. + +Submissions will be evaluated according to their **True Positive Rate at 10% False Positive Rate** (i.e. `TPR @ 0.1 FPR`). +In this context, *positive* challenge points are members and *negative* challenge points are non-members. +For each submission, the scoring program concatenates the confidence values for all models (`dev` and `final` treated separately) and compares these to the reference ground truth. +The scoring program determines the minimum confidence threshold for membership such that at most 10% of the non-member challenge points are incorrectly classified as members. +The score is the True Positive Rate achieved by this threshold (i.e., the proportion of correctly classified member challenge points). +The live scoreboard shows additional scores (i.e., TPR at other FPRs, membership inference advantage, accuracy, AUC-ROC score), but these are only informational. + +You are allowed to make multiple submissions, but only your latest submission will be considered. +In order for a submission to be valid, you must submit confidence values for all challenge points in all three scenarios of the task. + +Hints and tips: +- We do realize that the score of a submission leaks some information about the ground truth. +However, using this information to optimize a submission based only on the live scoreboard (i.e., on `dev`) is a bad strategy, as this score has no relevance on the final ranking. +- Pay a special attention to the evaluation metric (`TPR @ 0.1 FPR`). +Your average accuracy at predicting membership in general may be misleading. Your attack should aim to maximize the number of predicted members whilst remaining below the specified FPR. + + +## Winner Selection + +Winners will be selected independently for each task (i.e. if you choose not to participate in certain tasks, this will not affect your rank for the tasks in which you do participate). +For each task, the winner will be the one achieving the highest average score (`TPR @ 0.1 FPR`) across the three scenarios. + + +## Important Dates + +- Submission opens: November 8, 2022 +- Submission closes: **January 12, 2023, 23:59 (Anywhere on Earth)** +- Conference: February 8-10, 2023 + + +## Terms and Conditions +- This challenge is subject to the [Microsoft Bounty Terms and Conditions](https://www.microsoft.com/en-us/msrc/bounty-terms). + +- Microsoft employees and students/employees of Imperial College London may submit solutions, but are not eligible to receive awards. + +- Submissions will be evaluated by a panel of judges according to the aims of the competition. + +- Winners may be asked to provide their code and/or a description of their strategy to the judges for verification purposes. + + +## Getting Started + +First, register on CodaLab for the tasks in which you would like to participate: +- Images (`CIFAR10`): [TBC: fill in URL] +- Text (`SST2`): [TBC: fill in URL] +- Tabular Data (`Purchase-100`): [TBC: fill in URL] +- DP Distinguisher (`DP-distinguisher`): [TBC: fill in URL] + +Once registered, you will be given URLs from which to download the challenge data. + +This repository contains starting kit Jupyter notebooks which will guide you through making your first submission. +To use it, clone this repository and follow the steps below: +- `pip install -r requirements.txt`. You may want to do this in a [virtualenv](https://docs.python.org/3/library/venv.html). +- `pip install -e .` +- `cd starting-kit/` +- `pip install -r requirements-starting-kit.txt` +- The corresponding starting kit notebook illustrates how to load the challenge data, run a basic membership inference attack, and prepare an archive to submit to CodaLab. + + +## Contact + +For any additional queries or suggestions, please contact [mico-competition@microsoft.com](mico-competition@microsoft.com). + + +## Contributing + +This project welcomes contributions and suggestions. +Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. + +When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). +Simply follow the instructions provided by the bot. +You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + + +## Trademarks + +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft trademarks or logos is subject to and must follow +[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). +Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. +Any use of third-party trademarks or logos are subject to those third-party's policies. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..e138ec5 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). + + diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..2c6a20c --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,13 @@ +# Support + +## How to file issues and get help + +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. + +For help and questions about using this project, please send email to mico@micosoft.com + +## Microsoft Support Policy + +Support for this project is limited to the resources listed above. diff --git a/codalab-package/README.md b/codalab-package/README.md new file mode 100644 index 0000000..8292fe4 --- /dev/null +++ b/codalab-package/README.md @@ -0,0 +1,51 @@ +This is the package containing the CodaLab competition template. + +# Create a Competition Bundle + +Let `$COMPETITION_DATA` be the directory containing all the private competition data directories including the ground truth `solution.csv` files, i.e.: + +``` +`$COMPETITION_DATA` +β”œβ”€β”€ {scenario_1} +β”‚Β Β  β”œβ”€β”€ dev +β”‚Β Β  β”œβ”€β”€ final +β”‚Β Β  └── train +β”œβ”€β”€ {scenario_2} +β”‚Β Β  β”œβ”€β”€ dev +β”‚Β Β  β”œβ”€β”€ final +β”‚Β Β  └── train +└── {scenario_3} + β”œβ”€β”€ dev + β”œβ”€β”€ final + └── train +``` + +To create the competition bundle, from the root directory of this repository: + +``` +bash codalab-package/utilities/make_competition_bundle.sh $COMPETITION_DATA codalab-package src/mico-competition/scoring +``` +The second argument is a path to the codalab package (i.e., this directory). +The third argument is the path to the scoring program. + + +## Editing the Web Pages + +Edit the `.md` files. These are converted to HTML by the competition bundling script. + + +## CodaLab Internals + +On CodaLab's side: + +- The `.zip` file containing the predictions of a submission is expanded in `input/res/` on CodaLab's docker. + +- The `reference_data_{phase}/*` files for the appropriate challenge `{phase}` are expanded in `input/ref` on CodaLab's docker. + +- CodaLab runs `scoring_program/score.py`, which takes as input the `input/` folder, and returns a list of scores into the `output/` folder. Results are stored in `scores.txt` as `score-name: score` lines. +This file is parsed by CodaLab to populate the leaderboards. +The usage of the scoring program is specified in `scoring_program/metadata`. + +- The leaderboards are specified in the `competition.yaml` file together with other configuration parameters of the competition. + +When the phase changes from `dev` to `final`, CodaLab will automatically update the predictions with the scores for the `final` challenges. diff --git a/codalab-package/competition.yaml b/codalab-package/competition.yaml new file mode 100644 index 0000000..2707960 --- /dev/null +++ b/codalab-package/competition.yaml @@ -0,0 +1,172 @@ +# Competition YAML file : general challenge definition +admin_names: micochallenge, gchers, szanella, ahmed.salem +allow_public_submissions: false +allow_teams: true +anonymous_leaderboard: false +description: The Microsoft Membership Inference Competition (MICO). +start_date: 2022-11-01 00:00:00+00:00 # YYYY-MM-DD +competition_docker_image: 'codalab/codalab-legacy:py37' +disallow_leaderboard_modifying: true +enable_detailed_results: true +enable_forum: true +enable_per_submission_metadata: true +end_date: null # FIXME +force_submission_to_leaderboard: true +has_registration: true +html: + data: data.html + evaluation: evaluation.html + overview: overview.html + terms: terms.html +image: logo.png + +# Phases +phases: + # Development + 1: + color: green + description: 'Development phase: submit membership inference predictions for `dev` and `final`. + The live scoreboard shows scores on `dev` only.' + is_scoring_only: true + label: Development + max_submissions: 1000 + max_submissions_per_day: 30 + max_submission_size: 2 # 2MB. A typical submission is under 1MB + phasenumber: 1 + reference_data: reference_data_dev.zip + scoring_program: scoring_program.zip + start_date: 2022-11-01 00:00:00+00:00 # YYYY-MM-DD + force_best_submission_to_leaderboard: true + # Final + 2: + color: purple + description: 'Final phase: submissions from the previous phase are automatically + migrated and used to compute the score on `final` and determine the final ranking. + Final scores are revealed when the organizers make them available.' + is_scoring_only: true + label: Final + max_submissions: 1000 + max_submissions_per_day: 30 + max_submission_size: 2 # 2MB. A typical submission is under 1MB + phasenumber: 2 + reference_data: reference_data_final.zip + scoring_program: scoring_program.zip + start_date: 2023-01-13 12:00:00+00:00 # YYYY-MM-DD + force_best_submission_to_leaderboard: true +show_datasets_from_yaml: true +title: MICO + +# Leaderboard +leaderboard: + leaderboards: + Results_rank: &RESULTS_RANK + label: Ranking (average TPR@0.1FPR) + rank: 1 + Results_1: &RESULTS_1 + label: High Ξ΅ + rank: 3 + Results_2: &RESULTS_2 + label: No DP + rank: 2 + Results_3: &RESULTS_3 + label: Low Ξ΅ + rank: 4 + columns: + average_TPR_FPR_1000: + leaderboard: *RESULTS_RANK + label: Average TPR@0.1FPR across the three scenarios + numeric_format: 4 + rank: 1 + + scenario1_TPR_FPR_10: # This corresponds to 1st value provided by the scoring program + leaderboard: *RESULTS_1 # This is a reference to the leaderboard where the column appears + label: TPR@0.001FPR # This is the name of the column + numeric_format: 4 # This is the number of decimals + rank: 3 # This is the number of the column (column 1) + scenario1_TPR_FPR_100: + leaderboard: *RESULTS_1 + label: TPR@0.01FPR + numeric_format: 4 + rank: 2 + scenario1_TPR_FPR_1000: + leaderboard: *RESULTS_1 + label: TPR@0.1FPR + numeric_format: 4 + rank: 1 # Sort by this column primarily + scenario1_AUC: + leaderboard: *RESULTS_1 + label: AUC + numeric_format: 4 + rank: 6 + scenario1_MIA: + leaderboard: *RESULTS_1 + label: MIA + numeric_format: 4 + rank: 7 + scenario1_accuracy: + leaderboard: *RESULTS_1 + label: Accuracy + numeric_format: 4 + rank: 8 + + scenario2_TPR_FPR_10: # This corresponds to 1st value provided by the scoring program + leaderboard: *RESULTS_2 # This is a reference to the leaderboard where the column appears + label: TPR@0.001FPR # This is the name of the column + numeric_format: 4 # This is the number of decimals + rank: 3 # This is the number of the column (column 1) + scenario2_TPR_FPR_100: + leaderboard: *RESULTS_2 + label: TPR@0.01FPR + numeric_format: 4 + rank: 2 + scenario2_TPR_FPR_1000: + leaderboard: *RESULTS_2 + label: TPR@0.1FPR + numeric_format: 4 + rank: 1 # Sort by this column primarily + scenario2_AUC: + leaderboard: *RESULTS_2 + label: AUC + numeric_format: 4 + rank: 6 + scenario2_MIA: + leaderboard: *RESULTS_2 + label: MIA + numeric_format: 4 + rank: 7 + scenario2_accuracy: + leaderboard: *RESULTS_2 + label: Accuracy + numeric_format: 4 + rank: 8 + + scenario3_TPR_FPR_10: # This corresponds to 1st value provided by the scoring program + leaderboard: *RESULTS_3 # This is a reference to the leaderboard where the column appears + label: TPR@0.001FPR # This is the name of the column + numeric_format: 4 # This is the number of decimals + rank: 3 # This is the number of the column (column 1) + scenario3_TPR_FPR_100: + leaderboard: *RESULTS_3 + label: TPR@0.01FPR + numeric_format: 4 + rank: 2 + scenario3_TPR_FPR_1000: + leaderboard: *RESULTS_3 + label: TPR@0.1FPR + numeric_format: 4 + rank: 1 # Sort by this column primarily + scenario3_AUC: + leaderboard: *RESULTS_3 + label: AUC + numeric_format: 4 + rank: 6 + scenario3_MIA: + leaderboard: *RESULTS_3 + label: MIA + numeric_format: 4 + rank: 7 + scenario3_accuracy: + leaderboard: *RESULTS_3 + label: Accuracy + numeric_format: 4 + rank: 8 \ No newline at end of file diff --git a/codalab-package/data.md b/codalab-package/data.md new file mode 100644 index 0000000..4b20d71 --- /dev/null +++ b/codalab-package/data.md @@ -0,0 +1,117 @@ +## Getting Started + +The challenge data for this task can be downloaded from: **TODO: update when live**. + +The starting kit notebook for this task is available at: **TODO: update when live**. + +In the starting kit notebook you will find a walk-through of how to load the data and make your first submission. +We also provide a library for loading the data with the appropriate splits. This section describes the dataset splits, model training, and answer submission format. + + +## Challenge Construction + +For each dataset and each $\varepsilon$ value, we trained 200 different models. +Each model was trained on a different split of the dataset, which is defined by three seed values: `seed_challenge`, `seed_training`, `seed_membership`. +The diagram below illustrates the splits. +Each arrow denotes a call to `torch.utils.data.random_split` and the labels on the arrows indicate the number of records in each split e.g. `N = len(dataset)`: + +``` +Parameters: + - `challenge` : `2m` challenge examples (m = 100) + - `nonmember` : `m` non-members challenge examples from `challenge` + - `member` : `m` member challenge examples, from `challenge` + - `training` : non-challenge examples to use for model training + - `evaluation`: non-challenge examples to use for model evaluation + + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ dataset β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ N + seed_challenge β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ 2m β”‚ N - 2m + β–Ό β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ challenge β”‚ rest β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ 2m β”‚ N - 2m + seed_membership β”‚ seed_training β”‚ + β”Œβ”€β”€β”€β”€β”΄β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ m β”‚ m β”‚ n - m β”‚ N - n - m + β–Ό β–Ό β–Ό β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚nonmemberβ”‚ member β”‚ training β”‚ evaluation β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +Models are trained on `member + training` and evaluated on `evaluation`. +Standard scenarios disclose `challenge` (equivalently, `seed_challenge`). +DP distinguisher scenarios also disclose `training` and `evaluation` (equivalently, `seed_training`). +The ground truth (i.e., `nonmember` and `member`) can be recovered from `seed_membership`. + +The 200 models are split into 3 sets: + +- `train` [`model_0` ... `model_99`]: for these models, we provide *full* information (including `seed_membership`). They can be used for training your attack (e.g., shadow models). +- `dev` [`model_100` ... `model_149`]: these models are used for the live scoreboard. Performance on these models has no effect in the final ranking. +- `final` [`model_150` ... `model_199`]: these models are used for deciding the final winners. Attack performance on these models will be only be revealed at the end of the competition. + + +## Challenge Data + +The challenge data provided to participants is arranged as follows: + +- `train/` + - `model_0/` + - `seed_challenge`: Given this seed, you'll be able to retrieve the challenge points. + - `seed_training`: Given this seed, you'll be able to retrieve the training points (excluding 50% of the challenge points). + - `seed_membership`: Given this seed, you'll be able to retrieve the true membership of the challenge points. + - `model.pt`: The trained model. (Equivalently, `pytorch_model.bin` and `config.json` for text classification models.) + - `solution.csv`: A list of `{0,1}` values, indicating the true membership of the challenge points. + - ... + - `model_99` + - ... + +- `dev/`: Used for live scoring. + - `model_100` + - `seed_challenge` + - `model.pt` (or `pytorch_model.bin` and `config.json`) + - ... + - `model_149` + - ... + +- `final/`: Used for final scoring, which will be used to determine the winner. + - `model_150`: + - `seed_challenge` + - `model.pt` (or `pytorch_model.bin` and `config.json`) + - ... + - `model_199`: + - ... + +`train` data is provided for your convenience: it contains full information about the membership of the challenge points. +You can use it for developing your attack (e.g. as shadow models). + +You can load the public datasets and individual models and their associated challenge data using the functions provided by the `mico-competition` package in the [accompanying repository](https://github.com/microsoft/MICO) (i.e., `loda_cifar10`, `load_model`, `ChallengeDataset.from_path`, etc.) +Please refer to the starting kit for more information. + + +## Predictions + +You must submit predictions for `dev` and `final` data. +These will be used for live scoring and final scoring respectively. + +Predictions should be provided in **a single `.zip` file** containing the following structure: + +- `dev/`: Used for live scoring. + - `model_100` + - `predictions.csv`: Provided by the participant. A list of values between 0 and 1, indicating membership confidence for each challenge point. Each value must be a floating point number in the range `[0.0, 1.0]`, where `1.0` indicates certainty that the challenge point is a member, and `0.0` indicates certainty that it is a non-member. + - `model_101` + - `predictions.csv` + - ... +- `final/`: Used for final scoring, which will be used to determine the winners. + - `model_150` + - `predictions.csv`: Provided by the participant. A list of confidence values between 0 and 1, indicating membership confidence for each challenge point. Each value must be a floating point number in the range `[0.0, 1.0]`, where `1.0` indicates certainty that the challenge point is a member, and `0.0` indicates certainty that it is a non-member. + - ... + +The starting kit notebooks in the [accompanying repository](https://github.com/microsoft/MICO) provide example code for preparing a submission. + +**IMPORTANT: predictions for `dev` and `final` models must be provided for every submission you make.** diff --git a/codalab-package/evaluation.md b/codalab-package/evaluation.md new file mode 100644 index 0000000..3fb7e23 --- /dev/null +++ b/codalab-package/evaluation.md @@ -0,0 +1,37 @@ +# Evaluation + +Submissions will be ranked based on their performance in white-box membership inference against the provided models. + +There are three sets of challenges: `train`, `dev`, and `final`. +For models in `train`, we reveal the full training dataset, and consequently the ground truth membership data for challenge points. +These models can be used by participants to develop their attacks. +For models in the `dev` and `final` sets, no ground truth is revealed and participants must submit their membership predictions for challenge points. + +During the competition, there will be a live scoreboard based on the `dev` challenges. +The final ranking will be decided on the `final` set; scoring for this dataset will be withheld until the competition ends. + +For each challenge point, the submission must provide a value, indicating the confidence level with which the challenge point is a member. +Each value must be a floating point number in the range `[0.0, 1.0]`, where `1.0` indicates certainty that the challenge point is a member, and `0.0` indicates certainty that it is a non-member. + +Submissions will be evaluated according to their **True Positive Rate at 10% False Positive Rate** (i.e. `TPR @ 0.1 FPR`). +In this context, *positive* challenge points are members and *negative* challenge points are non-members. +For each submission, the scoring program concatenates the confidence values for all models (`dev` and `final` treated separately) and compares these to the reference ground truth. +The scoring program determines the minimum confidence threshold for membership such that at most 10% of the non-member challenge points are incorrectly classified as members. +The score is the True Positive Rate achieved by this threshold (i.e., the proportion of correctly classified member challenge points). +The live scoreboard shows additional scores (i.e., TPR at other FPRs, membership inference advantage, accuracy, AUC-ROC score). +These are only informational. + +You are allowed to make multiple submissions, but only your latest submission will be considered. +In order for a submission to be valid, you must submit confidence values for all challenge points in all three scenarios of the task. + +Hints and tips: +- We do realize that the score of a submission leaks some information about the ground truth. +However, using this information to optimize a submission based only on the live scoreboard (i.e., on `dev`) is a bad strategy, as this score has no relevance on the final ranking. +- Pay a special attention to the evaluation metric (`TPR @ 0.1 FPR`). +Your average accuracy at predicting membership in general may be misleading. Your attack should aim to maximize the number of predicted members whilst remaining below the specified FPR. + + +## Winner Selection + +Winners will be selected independently for each task (i.e. if you choose not to participate in certain tasks, this will not affect your rank for the tasks in which you do participate). +For each task, the winner will be the one achieving the highest average score (`TPR @ 0.1 FPR`) across the three scenarios. \ No newline at end of file diff --git a/codalab-package/logo.png b/codalab-package/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..2016b34a6b3024416dd966ecf054344b80cfe34b GIT binary patch literal 42416 zcmbTdcT^K!7%dulks=BTQWcaYND~PVib!t~s(^@w-ix##O#~G~LPse9q=X_M(t8s^ z5ds7O3DN?DB0T|ufH%K;@49!b_t$%GGRay=W@deR=A5tWZ=ZiN{}ut44RrK%08~^| zfH}$w@DB&j0?<%X|F=^%TFOqxNJmFYOUKN>K+nj+%))Ys`O+m;wkwxe**MrPUAlbj zGRIX;E-o$>_Uqi&IJvKIa&i855h@zWIka?4baYIdte03h|37d4x&R!ER87=|G*q_$ z)Erba98~|l0|Ws8Dmsd_{~7$hH!5n1jr0tROw5-k2h?8%P*c&+P}9=HubMq)xaH3&7n5ATDEzRdn+rU?CnEnM;58HTwd>qGH*Vh%6%&_GxUcv? z38<{~NLxo&Pv5}Y!qUpx=7}xD$=Su#&D|sLWl(TPXc#p1O^7@qrIy(JgJ z$oVk2pr)HiSRTB`^&()L`I?Br((V2Kp#4u||L=gk{{MyS{{Z{HxMl$?G*py_N5cV7 z2b@h2LKIrVFwoGm4UIo)=}BkrdVV$mZ4+o+I>bkJ0r4#9gB(q__&{ z^WpqdN5AQiF5YX`lQB~9ZlyQzfA2~xoBYInKTi|#{E_wPg~hnve{3>Y_;dc5{i0U6;I-klAk&SA87}?ybQmb5Bmk?f~_O7@tX4@ISz5tx>=76>$PfThJC_!|0J$7lC2+YsosToV9q2>V6+O5N&&e}HPR;lbhb z#@R+*oFpgB%s+s{bxB=yTLuLiX;r77;_d?FD8#MYi>TAdp2cP}1K@>44mQnIaK-!) z+zcIc;D*~JgW>Jsx}Dudfy0uLAjj6mwhN)=O88JUOCU~O!b#J?V<5-kL@B&6Xj*is= zk5s|#&m$|yW_5YKJ<)8Tf|cUtr{H}Ic~@sbMmBF%gC112HJ|FMPB=iZr?&)7MDrt{ ze$Vy_-PKOkkQH<1VpNI*&eeZ_t;h1`?(5ktS%4SkpiQ?6bL4r1F>g3?K(*kx2VLp3 zL+81Nqfv6+h84T2xnj^-42J|V_Y|CtYZ9C)dQ30=qg@Y}Fa5Ca(x^XiEwNoaSFfVA z)5Ex?=vVO{+N@BO@1~BQg2{A4JUey zDjk(XyC{K19*`%iipq#bBiWI2}s;e*P{Tig;fa8QvL^sN>ekBBnp;ln8heDR@yH027=_Y-9FgIDJfofCek| zNnL>Wg}-W&w=58#kV zL~VoIgIE0f{O2}>`!(_N=>bp@Dh}=IZ*aZxI#o6lj3?t0lC2Mmap&h`M_|EglegCzFO>Tmy5M@92Lr?#J5pve5S=GVQ!D>uDiy1A|hCb~8VLr>&a9 zCd!Gsvw856=*!}ZkOeaKClg3K6fbCyChb2$n`nuCRt$n{^Z_kGw!qsM4_f)*-?O<5 zMVFA`IB1>$;KT6oKLD`E^60qiC{RzBC_vnyE&Yjomh* z84mLi+3kj}Lsci_E-a{+_*q%nWv7a|@r~pSW9Kadwj60sQm-~(raXCpykv?iM7lF0Mg zI;Qx-Hw1ysp=9gTwGv5myUabUg{cqGbC79Hx?RJ@Y97i5QsV9A10=mVrv5=vJMg|V zeS})wM+HWoxoTBxm}pk2I->G^Cde3|koZ%e7eO$GpiqR_SV6&Jm3Ohe!xNc4yFMPq z&D)rECG-NelT@Pr%vbBgJesrx{+8J3;NGg|>ygiGovgn>wrGuX=!|z0N2?x}NG2`j zz+7Gz$9wqRWs60oc^$2N5>X->i)Mrq%NqU$UPRfNN+)W(<@V5E+d=jhADp_RKDS>h zwdR3?HsZpQBnE8}j?2-_G}8WAE{X&}Q-g01a!dHjb1{zNd}(WE*6y>7xG3hj7bPES zrasUOwMKf#&X>56kG71_)l((C!njuMa}SL=E^@@iVqk*CodtAru_eW)C?3X=P>=!; z^db#V;6I>3x@6+RA3R2MEkosHAyn! zuR3zpN-Xh8BTA*jP4O!a%cC$;ITeExLy6od#-YXJsU#NMg|S`*`CR-}8|3U*)gpm1 zs{HZkOpjJnDi><#kW5A_vlVdzS!3orJ++&MM5p@ zFmIK=Qw=@J)C^!x`=z#!R}y}ME->W)U7~Wjmg-(n`(&7q98 zibG!VY>(J*obI21!y-xgR|A@0JE$+=nYe0b>M#%u+%T~?JKs#Xln%4mHnn@PF#Hct zY4s{~@?4zfG@5ZctAZmy*rx(R3HkdaWKSIOr>NAfenV9cN%#P5&+Kk*@!bd5_rTSG zOhUcz(Xn2|uNzaoEf~~%#b;R|Py#=#ZV!Bc?CP%^T5&P(@R?76ehmyz2Ek9Uloj)X z!2yLSglILFe<}kdhAF#d=0Sm4A1i9l++TIH`CBQ5n+Jz%7^VT=y{bKTPZZ>?Z~^sv z918qzHies_$fOcZGW12|$C}**irL0)p@g$Tvmx7LYxhE5^^2A~S8|_#19Y;^)`?Dk zV?>&5CtVJ`-6!@o*x2}OfDrKbc&&)W4UZ%)2|~6^aFfC18uCOgF$AI=(UZzwYihY% znw{o8L=6w!HD#7PBcmi>EPUsl80GJl-v0m~ z%FOu9ZtG``%Q5r2VU0yjMvN(4)Qj%HvP`S_#{Gm>zv&SUiZ3B$!QQ_gRxKQE8!X-` zGfS|4wNR;t4srOL+jx+lB3F1Cf}&I6UMsqURv;mAg16>#2W~M(rpuUo&MA%9rzC^h z;JZs>V6k_?z zF2X0{LPy1BEp^12{%FCF?uu|fb%{*ZW#yofZKXj$qlq;!SO59K=Z7!VM~ex57*+Ep zn%fr<;&Y6=o!tBp7j>{s6xYp%GXwZ`h3XOQt`BZ!f%@>b{FJ<9%OX4Yb=2J~MSrxd zv*LSJ}dE`CRL$({{dKBT3%ocILiiSQGS1vgi*FcR>|IK&gVY` z9#{y8KQ1f)njzl^IxM$TvKx8NFmJ-T7_`yNtOo}g4AZ3>$;L06p5(;k^|sO_ipJa& z@Rau!V|dKS@5gk6NB*oVK6&yScE5J;+QV8&nrOyn**@|yO(9!(2D|t%?YNa6<7?*J zR6V1N zB&QBpQ-TJfq{@f<9S_~tHJ*S>739qJ@7uL7_$I;g8zxk)==IY_wo78`O_@JQ=cq_% z-YQ&NEITO6N;b0z>m$oLoTlU#t zR=_`G@shW`%k^vdza`4Qq{=cBsD9mkJ?C?A)okI_ZS1|T0&{km^H=ok`C)8+-yh6( zGpy^5{hdl(;;Mh`Ui{(59mQa}4>e?6Z8W$Sz-}5TH1SGQ(&3W>kkNllbhzFoo|Z7_ z+;cb=wICIiFdRszv|i+|pJMU7oip@_8w@s59Fs@1S+HrWtW4)_MKRz1eSOVSOYB~p zELH5_08NL|Iw(SN$Jn&TWm(Us5cl?R&vhz;R+g({PV|m`?5O*>6(>^Z@xu46XiT%_ zewXc55{ijU2bmsL-gSm7lItdt-nk%#K7AXqIgClNK-kBUtzLfpcv_`aPeQ;uIA`Vt zuPA8Vr@K3p5F#lJSb=})Ucm$JuUyES==(c+`iFU0E}PEN0y4TpF6*hf0`}N7aanbw z+FRCh2lrv8;n-KT7*gfd2!DPX5&+BQ1*kDShofj=IRbB|N{xMMVUGAt(@GERABTdF zfi1(jYVNhlFqQDAWIf0(oP1Fqd~4+rhmSad4GT-PE`Ysr(1g)jsxBTd4G8WDPQ@dF zY~M@k_-3duD|%BLX?c}`%`?K5{>rNUx^m=w`=UcGMQCK6twtMC-m0y|-Q*47?6@$^ z`j*Wx`*g9S_C-Yb<5oLFvAWL=-Ze~w~Y_>Cn~?Z3{wp%9^&gWZlaiAn_UOBEJ0xR&o=qn zCTi@zv*Z&wCs3V&wzZAbC+4=G8A28k+?;?5B^3?}RLTmx>;-r% z*UsP|;oZ^3{IFfUzoaklG8Qommo>$~bHM57!_U3VH>y>clh}Qy-~>T)6P}~9^{-VH z2cw|y;4;O(*mTWBO2w!ci!eCwf!?65RS5*^R|w+f(FUfs7QAU2s}Res;f^djyzeBROAYaGSKE@J0nv3)?|z+ z2^F9tHbgxBk1fDWrCYBKa-%5TT&<3+C+uZwNMN3?ZK|(5a*BYc=k}|YuzulZ=9||1c5rWd3OFHK3pfUd&qe} zAN3(-j@=}*{862M+=K9Q@$7IbWC1EWjAKlqPi(GSKE1o!KK!!olI2pZ=X>f>pU&Rb z5g@l2=KVfi7Wp)vU-pPH;0BSawJ7w33ff%qOU>29ftf{kc<`xJ!-vJ)`B=ZVG}S+f z5Y>&5^}p<9aaDc4nEm7I|0wm-D{WyLq1v}2244QYgE59llttf@I2omvvq}8}Qu%B3 zv$)0hu8e(oMYE$|XxDQXA6%m!IHEX{r+l9-sSF??6U=v+Q3>-{8L=!t>MYv$2k@|2 zwSqBzauCd9jb`d!u+P{YWrx;e)IC*Qlv9ezzHO#s2G=Jjb@v?Y$UlW=dWhKADn8EZ zz4aQzyhBS;BJG;FD?j^m)B2sRkXf3Q(txU5&MBgoVdJUq+cP!WIlH=mYtQTL7|Is~ znDG3OsCMw9xroY2G|PG`Zq?w`=q)eaOW1acK`Q6wT=vKri0} zwSE%v`G1wU4@v&Lcn6gPfi`BKz$aKP}}g$B42IC*rRO?lI{ zy-r4os3~=p$YRX+9uNr!Y^m~%pe2Z+wOJKB}G5sbDejccuzcJBYB3ce%wyS z>r2Nt-%V+spG@y!^3t1t1O@`eR-t{|ll6+$-zmpe+ql$%yM@ST0z#)xMM_#|k~p)y zeetyU{gdlIM0vdUWH6$%0=oY8?P;>L^SXR&>l1fHUqLc@%9vZ5l+3UtOS?}S3LK|9 zUA=b*Ay%S^T}gWQd>YtkghYKu!i~lLj2~Lp=0@{5U@OT;CXsWw2#M2#S)0RqoS|xV z{hrNw3zGdzFSz+?*)x%GIeR)XP*Rn{arb50_4TJ2PH#E5f2H|Z$PO@wG;(A7=i6rq zMsWRN@DM)Wi&uWKz(nY8EDij(jaB%`!_h|^tzid`^5hzLxrONaL8ks^z|uKoLB5mN zrM`@!LFF#p%pVneD%8(yVEdD=^BclMtgkPd#^3YwI8G8v3mamalX=XUn!--I62y17 z@$5-xYj?x+b!@jc8%MnD^Js>hUYWpO)Ol^D{#*7Y1%=f)d&AUZqaR=6UziTUXfWmo z2Xwn+{bVL5NzJrludPNm5DOpQ^E(NC_^J8d(VuE`{!>>dT)2VmwFecrN%VFWfQv~@{-{s zE(mP_w{9Lb_8P-I+HT?ru$FkZ%>anqN`kVH=od6opv>!M#w=sQ-d?+sw8kTs26%^) z&KCiHn}qzr771JqXl>P3UPFe3UJvk6b(tn)@;ot_Gb(GC+=KN&Pk3zM!t@d zOdp2up@w0Hd8Vin^CzHh9>j(%UT;y_fKc@e*~i}ip#)^Rq!Uc!^s?7R-e@MDlSQ`) zb~ekpgdIBmXyltzCtjx)15LZ<;b%%nr~*rCK#C9g^mI?m8yp1|Dy_vINNzL#8oA$$ zS+(McpjW~2$deed>*}^v(kst(cK!h_%As)Oi^4N^gUJsmpFc3KYWK@Ntdd1X zRYNmg2}a8uZoiS0Srk?Vj{^Vg&i=4J{n36|l; zt0p|0bhJn2JRTmUSL-CS8`e&%rEwko11RM)p4D80qQw3{qw@C3sIlqc=FP$o@>dmX zPdw$awvXGE^BSv5Ddy?#fz6Kh`@lNBRppn*jnP_Igm(`Rl~xYeFcr4B#=Wc>Tqv@9Bb0vMP1V| zJt$7J)t9fweHB@Z(D*OdTKN>i-%WH7w2Jz!dP&OVRteQa5O z;q!}NMEjEy<}Wbr(w(6*K~@4!?34U#@=BS*qm-M&P<{L>3o1z!xx?u|cE zsoGHN6h92%{~=f)>|^35e?TIkg}0u{8WGY{By86M?aq;#OxHA9xh0DBqXXmzXb<%9 z{F2qx%3> z*hzj|YNbRI8x8uZ0$pwk)I2G^8{A>^%|~XiNybrU0>Hw?2Edo?fF#TR0p658eV933 zDx`PKN$mtkqekd_ZdHCWTwEZX|6Q`H+Yn^`-@7YluL3#nbEcsoQnI> zz|rm2ql#1GKi0t>VWtD zw3bPWJZZe$=H_h9tsW|SyEacl>8PsEu*S{heGb{XmIs37rV*X?MS(nRT@vPjGAAsh zO}G^O(NN-s8AcBG(5#qY#bu5Wfc&HiiOu^lVYV~mvoe8`)N`hFBm!X2)V!erBv5U1 z>mX&Tr-Hq8V6bar@Z2+dvQ0Bv(>_*-`=|mc&@ePn_e~hn7%1IKq4(>KxHydh~uY9kW-?4CxXJS!*%djoTb>cVgUi*EhyNavOMLOjQ8z;c;dh({O znS(2PuMNrg;RKN2^sP4CsinKeR7fOFJm%PjI&k>%Z~26AW1VB!mvgf$6)F6zS`Hg1Kw2z zVVYeoZq?~AMd)3#a$RnV2D2WP#YF>s5w6q>uIel;fdi|%x3)rBvQsY2I4qJrJWFHZ7dVL~e`+oOOLw%+>=%jZEhsH^e= zxt!(I{C^>Wd6k|l=7{eSAMf9Lb6~gy5tx=){5mI~vo&~3;xe$osYa{hHlfL9o=sB< zN_eY`YjVO)dwRHmE#!<^{m4_Be6#nuO@kPppWEZ*<=P6NTxH4ul*z6<^SWpfJ;Hni^fgjU>9PqXkAtez_PJ}y@)q$-HX@|pcQ+^0|t zlAxSTKTk=7dfMr1WU&UxGZb=_bnV{9x4yu0_lK|`!CZBpuq^{lNcy?^jI@=o`cx`6 zzFt435iZV1{tAsERjeWWJj4chj*D@~JcH|HM;l~wGU1?zs%}R;9b}C#or-ffr!xSZ zPEZv@`cRrVlxBN>P6d+EDW1yo1%b-uj2ul&qn_=dB!YkxCioT{u&!2DUKB;zNRheM z*x-&cp;y{HE1QF2e`@B_TKogBPZeozp9U9>At`ND`4bYxfR>pO-_KFl>JzZUDTHhh z6Zj7>RqP*+MW*fJ8oeH0=RXB+Q(B1H?~ko_QZJ&?K_X~6Dt`7#!C~$Z-iXsfi~;@o z#)1oJ20A}^@TJAVcMWWnK2WS1JsIWn1M({$ckMte zb#i}?qw2oZDjNin4y zTlM>L;|jxG-roI2NO5=L@v3tZ-;P%7YXF946u@7Bu#%lqD1Q!%eqg{|+RyMC4-X_y zrK1wRD12&b-=clrdoA^K;Aqq-2@Up<)EX}~Hna+7h_5p=tqUti*K@WAcJ5|SqIIoy zD4cEm8#dKf)R>~ztj+dLhtI!D_~b3QFZ53KzNAsd%?u4bHtHte>EYPlX?N%43gFGV zqKEvMKrdHSz6i~cfQPlF<~8G{TEB=Og9U@v#?4L)q53kKv=Dyf*X&WYt=^S%%`q?R z!)ge=N(7PE?NkT-qLZ3k5!vklvz!97PZWD~S>+b!$*a1&NGb+qXcZ$j-`~i39E;|# zs`y5%!(oP>?(X9yna}~jb12;KlaF+5>nl&TE|UN@O^{mpkj>`M4?Q279g$=r`k5x! zc)5W2w_NI~vt=l7hfV!P+46Nw^B$E=@2Bpv^Ts^MpE5Yw1Y2mRV1zN6(`ofGcMc3M zhL%<3;A_>g{wv26{dI=bgihZoJBN*LeX5C9Wxm51(M>c0{{Ta^Rc)3f?nmYW-&m2h z<&w{J`KS>7+ZztfMq175z3fhon@}gH<;N@?Ig=)r-_5^5L2Q2XvM|Kx*Ly+w(nYwS zm&LsqdJx!U6yFNfF@k^J>fV_pBs}1nN?x5h_{qoc%M5%u(Bsk$3|;TtZTA&zZu8YI zmEvA3PSj97pRGy3Vqb%A8M#IhHw1_nSVF6=k zeBRoxp8Y1hVsM2BEP*?xLF=;aLxX)%HP4>qNc-fqJ-b$gy))R2JM4#!iw(M>8wtc* z{*(pN1Y-@|TM>h{Q;w~A!hjmQJI^BQ)z3}y)VTvm-2%aPQae?<217TSV`kEwA#qxb zK-vI9nz7^3sypx@v{Q0lha6Xd(?j}I5&k*8MZV|Za9y6+mUg#4H z7k`?5vuIq71&YF{@QwQd^-CQ>lSSbhMezrEDd5)C_wR$j#@C&K5G$Y#~^u&Qa>iIw_~eag+L zBC>OU(a6r~_5kIq5ud-_c#5j0&ChxRMrpL)SdY7IrO#JeUZ%5hRd>xcx34#&A25Vu zT5I0=Syw-L{YQXW=L%xKB2z#FsWkLnPUBA-8}vpma>M88$LKf5{km8|u;=iFUe`u!-6^m@cLoY6T0Bu;iSw zmjd6cD&l3JRHKMD^B(^{U*@Z2OI}(eL3T$EgQOcp~DpAO^2z3m1$xTZWrNyhJmWxry$pqlw zh-#z-V?f#VDR{AgshE%wwo4(WhuVnfdO8~b4NXTVs{F!%SSk(^d2`D0=GHbrN-Ld|{2T~-_xs-b! zA2w1_(LhSs1r=OZF;8NKwBNE$obR4c``wBq=@>@TBK0-N~oEWC5BsQjvj zryIXa4%f7E&W5L}JGp76pH$dF#E-la>?C?AIRuRVcQ+rTJ@DuTXUm%^7SUn**mnI< zCH1!^>4~~SqX}vU&7;pezxAVr$Pd_LbKoHViMD?L%Y9|7Ad%ql_fJhnL~>xU*er@W z3%$0~rV=9cxpwYh;!Qvh6?fiuDhE->i^LmeZO6ZHlSoOfq4Tv7QLz3&AFmoOrJStj zzw!NsOKhQW<)*xktd0B8$lcv~Ls<*6rX@=RP5yn{St- zX<@AJ{43(0w z;)Kly8CF!8f}zLN)?;VxB1x_<17NREFj#PPPF*{&`j*aHC1-1HqS`$A{z9k;+M?d8 zs5o9R%te@{CTY{tEe@>IxTr4amR{5o#!t5TBE=_yj9plH%_bGX`9DNJhx+FQ zWxpo;p`V=Vib);hvl2x(HcY9SLL0yk(^{9;zimj4e&7FQaGz0aJQ3j74=(m}Ep;>7 zvc3jY%O5zNk{w#+`QaLHUBeG&qBp0s)}EutmX?}^M8~}UArW{`T&R4H?d)M~%j6ZN z&k>3%dsT>O9PlED*x2(Aa9J}#sJ{8`L4Rv{HchEewC1L6<*T4&R%z^yLn*^6-}%b7 z`B;#$TM4Iuivn-UJZ7cRe-D>wL>KhyKMp=TiN&=)w}OF>lGfcHwiS`@L~s;uUOUvk zslp>#l63(HSOdi~JDEPdYq*WU**R;6arm5*i+6z==hM^{bU z?vZh&!!hske|Ev|)k+I}kN!KH<38y3u4A7o4L>V03mK>S} z=~V}>uLK$2ui_=lnwq9@z-LB5?Vj)V1rq7fkWPmRS{)XLC(!Aj%FWEnjlb65b_%dP zJOQD^AmKmu*ZSj0szt`ym1?=B(cohO&j@rTwCNX8=avUFSmr*D)W>vvCVmZmwWco( zzh?HQi;aeP#qSse2~Y1Wm&@|LrctZs7%Pw;|0#lReQJkgn>ZXs2t08YL6o zAumNrTM`5OSyV5W8VZ;#L@_0il`es=%?B;E~$zZs+baB7}eqiP;}gvQ5!+43P-(qO)d z-IGIbAX=4h3Z_c$J_W}yEKDS!{S9NgHpR|l*Tom9=E5JVu;M7<#;80GGjyjHc>C;> zln+hp#t0HB@67pyogIQ-@(1poi_11$P`IMJ!vr@i#>HcPWk6&)sN&XF)^xQ6!bHKU z9)Szx%1zPNM8H83G1qcm_h>w?gR8jIuBE#1#o}=L6^)EeN#(4C^|xj-gy=B77wv(= z!NIyq*5bXFzQv1r3b5&zI`mY1VHa@}2L~p8$2U6UO;z=kyu;t2kgBbDgoIcAbWR7S z#5q$fOk;KFj${c3kRWn&d*aJV7yGRy%pP8?N%f^_$nMkl53ZiRx1KK}A^`|LD@d0p z${u-vDt2@?ywxeB2Mcpa;~0avsx@s&mQUPg-2R!o!>yYm_WGeNZB9}tMIb}7!t1cj#mX0^XiSL-}HnJ%5>a{Da!-?NV>ipQ1)R!Y9;VsR$x9fUXD&hW-v?NHw zybhz_P!73ktt|+81_58lcbrYG9S> zo1L)A`+j}E(T;fX#xeQQ6RB zZ+^UJs*8DR^ez5$5mLesXdCrhyU|}ontK(d6+=?m^n?cG*Vm+do%mJTr^bI`_1E~T zoWq^oame$WKM*AI@yy`el?m=l*q(zc@K_KiT7Cq4#A?d)EV%7CeiW4~^? zE*)W>n@s?y>&DYjMD^_Ro2^#wuEwRlZiB!18$K=Nb}+QQ@tK_ZLCtud;dPKN`G9ba=1D>hqW zPq(3FlYOmwGNo8hq<6H;MEq;E-ha8YD{c_L;uL3}0-Tk!ejmT9^?pe-Tr*GM?CyzW z^=3<4e4}!&KyDC6I@fLeOZiC!Pq=tr0woqgsUNNeK6`p%tsOr)KiWTjOU6!TyfmKf z;aRQkEIVz+E)q}`XfUzs>H@zc%L#C{kP&VOMi{Xh>=1w7evbW-Vv?M(nx#n2gpp`E z`6IScs${3?l|2S#k*sEIYda%lUXGpdcj;1x?3rXtfFd$0d<5Kdd(b>@qt&~x6es%b zgYlYaxa>5y8ifqZ!9pw&@pc(a$=~`?WNTg36=mmAC*ePvDPo)(Xt^Or$sayK61eCA zXJ$=|N#fj#c2(hETLB;G-dAR~YoFR_-JQNRxVEf)xElK1xi+^7T>vDTK!d6u{QOgp zQp$5Ptc#~#-Jz#F^YNQGF?qvxN2`U&m-xT4B$0>Nbj|ELnmrpHT;@!X9h0E} znK1TayFk$8CTEieMnnkMs&;0tlgoYjTA)5*B+h}lHW~_bZQ@#S5NUxMqA$-%&&RSR zubcQUoK82KZ`3$d@wEpnVFzqP*_t^Ta1-9SnJ^z_$&v0~^9 z)zQXdS5RlV`=D=f?7+rYcob5iEu)Q(;%Z64(&%ilg@6Jt|1GV1{Gf9QEhZ~Mf{r=1 zw9?15XpH8vqfOl@-EUiJNg)LS@X=UH}wc@+DG zK8};|?HJs>az`HA=sQ%1M?tf>0@RRribTff`FzRaOM!ahi8K7_=N2|0W3j%)dKG<= zq8FSDOR-qN;)O+fYV;03A@6gUlMUoxiYps<@t5zAuS z`-8Xg;2_f_V@nW&T`rA#+ijsS;>daN?GUTH~ zB7adL@dWvmYbRwWb}Wu4q8dD%~o1%%e>*S3xi z^M3v^VUusC%k5*WV~TvMC^wOz248R0)Yg(EqfN-LK6&)z7v9$~JeEwXvK?;zrJ3vV z0aQ6rApeOcUiOeBc=UH$43c7drJ73Ub3NR#|8`FXMsy4oq@Et8GXHd@RQFJjz22}% zs|k`}(j-i&TX$JEUk6`YGndHfyCyi`ksTYiThM3cDn*V zAaU@&K6XHd(Ecs7j}_+p$(ltHMD-%Rc=EpZ%_a^x+4tWnZ!K;wLIi$?ThZ5ho)9k4S#^$1qpf4zBSz0*J8?+$UkoUcjg^$W{nWc6_-UU z^=|tM5!UW_Aj0M4NX@9qF-Ra)Ups-6)VKfm_EcZ8^=e~4ZqueLQJ{nLCquqx;?~@^ zZOKEOUog8^oR284|w8{I;GGcGWUQ|J^ItmKgb)zR-heN>`ez*wXmw_u;0Z6aY>q z;Ci&2sV2N>?Q5~gh?^@oMAz)O4vuME9uB&y*^xX__0jdduZBPUqq?i#rrMmQQX~%6vjoQe%J17Gx^Jk=liF8 z%KA<3+P>1`v4A8QIh7-RsP&6kqYDd@!}2~x^Q(8eTyys1k@6=l-|jOy|0rH^r_NCm z6iv;E)(Oz$7M*Qb<-f1ABq>jAX!i~;FXxz?9)Jp#G~62Dp{(5g{hNW@%6T7HgHnivSHo_mEW3+D%_}P| zs`eDlaAC|g9Q>;_Ltb_~%aB@nMs~>)Bj7rhJiyux|zS-FIy0*EagBW$;^AuBnmyHZPf?NG?mF6>Dx&B_1rQS~gsR z@%GO4r!2vkZM4(#X*zR9>AoW3vwm5?r{!L*&h&Al&LV|}{svVBPdw2ziA?9IkxXt( z^3)qtE4lK)u26oEwVTiNEX)0^8`|8K)yG=6tb9VpdY`K;XoiripE()%axRjhxV^Vz zglo*2?YF)~!Uzs!S)6<5aDJ2okJ-QLkE|D%b3NFyV%3Frq-~Gc4{}h}qn2z-Vz(xW zoJCQU!qV#L*5axwHmEo}>Yx}mQRI-TZqNY+!n&HNGu_ap{c-&MTPA9Gagm`BRi7aZ zVd#le&nT0RC~uGoVLt5@8rfM4cmohu9dG0ltmK31oTcQq7dn2S>>N!EoJ`z zcj~_ozYXSgqHL*n)HI1|Zv_cw%URD=;sbXi3zhY8$OVJSJ2{$vOJY8$Ju`}|ztwNb zz)=6}F9%*Y55;vi`0gA8yQQDVX~jPy6hbV6G)j%iZ;W`_ZFt8d=RLz_O*D=l;*@}e z;0fSvO%h6Yft3F%<4-7oYqBE+3OPG9J4ak#IPG7dM33bE0XB>O0&&NoRfT_r{sBS| z6h>>ReENo8VnAr$^k5o79?rf z3e_CNXRjO(T)_or^|y6dN9B=k4jA^CtNkM+?XPIvvG%B@-ScA-N%5n3iHLDKw;+aW>2C_t z-8z)hq+P60e>fLovDWcd8}DM&CuqL$haQ@Pz#R0;V$Cb^EP{ut^<08wQ|q?>cE!S# z@a={{)pl|hkP-^x^P_r`~=*6uRh{*AP-bu;zVoQU-0p(18sQPSbJM`~y z-N}w$eOMKkys)5#tdEh(Et-9kVyCalJUk%Pn!k0DRZ)7WmAkix<)TxXJ?f(Vimm)- zaX08FVwv%ISLjFO+_q$RIANl(^3|ge|Eu0lR~7-kx&#T+?V%6OM5Yzho**CA?mY^i zz6AIm6rE*MlMfrlK@b!XR6;rgMJZ_*FeV~h(j79oVdN+g0l_g!Kx%Ze(J?|mq@=q; zHW(wNCI9cj4{?W$4UcJg17X+m&s&6 zyemZUtkgP8zqAwwm1KBJ4M(ViybvWdrTdw?$?A`z03-4@N2$w6_W@GGVj?#r;kQu$(IahMmHhw~Hmu&6<#r=3WBY1% zt~Pn7<;@T{TP>V~rC9aiBZbmZ@|f2#+s{_NDM5FE6=TWaHX!~<`4Y#u$AEWA>B#AO}2SQS#lDsZK zY0tDX32($~80Lt@sW$LkeGQ7a9rjF3tiIX#`369`&jR_`#Epsl+1&D_9`0|($o;!< z{oNZPGA;5I#=W`3nN$}Q?qk_m7nkPw^kOBx=9hzXI0jyFZuS-zk!CY2uGS&Ek-vHhV0?r3my@){+T2KpVDPvx zM8P54vgp_cJ-h>T(Pp{IwJyX^%(;IC7Y9+D_G$h(b${>|6dB=A5(+{>VX4}_jFjfE zyGzSnAEh?So1(R?+PvbwgXNaPifmSzBsM1X1{Z&Bgt~n^gsXr`aB8w522$GH*SB=| zirn~%P4bINz5j@+6~Z2}44Ic64YUDLt_Z3~2a5n#HxLirvggGo;d6+-U@++DW_2P> zluF^(ey}v;oKM1o`bnVGFQYn(0DsielS*`(~9>Bvit zOCo9WAcUnz+0znj+%28z6RZs{EMBX*h{aoxgCzH)iy){~^IzGEy&k(d6rPO@1->!} z1Kl>fwEThi_P|E?0x|<{#w|9eRHU*nNg%pe3Ga#|cLW%M_d6L0QB3asfH~(`wzlDXi zxDH?DiDe|(7tPpiAZbg82y^F?-1Tmx8i#+jhFJLE(az&Ny7{Ir?79oT&l2Uox)p3M zXC4-FBdP)A{4;u+`X_X!)YTI}s5A_f@@p@CB`8bb!z~gcTUf4V0cX*S7U;gW1nZAw z!cQ~KULqU<#MHWhc@CG!IwcqiUg_g7-wy?}Ef$ zVtm*3tHd3>kg|uBRZdL7#fmy;WsndF35P=&umUvbl*5DLSfl(4RQ{v47wx30!Y?8L zpJN=@%PBJW9`oXdOA%a@yH15J*>5c(E6P1viZHpKN^OjxC9;h*`B| z6SHXc(hc6x+q->V{cLf)_a4&9xJC(qt8!4CKJ3ACyWqS9?W~>~*Kikm-^Y}ZQhi^q zOR`Y^>d%vsUsv}A{(!PC?#IU`#G|YsL9?$)ms1Q?R$poMj~#)1ZMR%pF*0)=O;nybX^21gf$+j$Jzy@Y>9z{PW128<7QN55pUMm_ZEZ zf(3(nFtU&PvA68~Q-Fqi8GO6ALvAH+OPbH~-sewC< z1+xN3OgJ_i#YNS{ZNE_bri3WUsy*7Pf8;*hTg*AsD|LdA`*DA(8JCC$$-PeNjd+HB z(y0CU3Zrq;?Lpd9e>d(egTimUoOP&Mm2z=>)!6 z&8c>iq2&_7LZ-54#vTDmkqQN*y#4TU3oQm17zRcudrF`0n?#;SJMQZ_D%4ON>7>A= ztrH+HpMxd6&H7JAl7?Yok%AI**6iFE0G0+GRmZ=dDmDKgO_nY(xmbIdhsTjzs`Hw6 zGe-;oj&`N=LU#hCKyWAy$us|Hs=x!x&e_dL0(hfr?vSI_z*;r*)H75B%07g~zJ#gob(bmTICQk559YksQc#&!7Bq3Rz&tLU~4N+K3Ymz>%I z^7I{9Xk^Q;y8qPr%?g)0a*=OT*|trD@0rgO6~vWnpuQWp*h92p9lkiPu(N{(BlgVg zOY$!QF**ISE!!CqsuvJ*t-Ack<;W@z>ez;4hxji)i!)UYGb==yTN21|`Wa6^qjd8- zOeQfgB-eb1jNzkS6qn9=x~e$>hE&vgeEl-2J)%x}lM7XDw*A9z2QAciXbW5Df_w@W zCqbq)b*wd-M;EvZzbPZGMf~Y{Bz_`m)lvpJX~clPWb&gMGGI9;jC6RN*L>J4LyyRdWwZDgMXiJDinO4Bb5V`-^VTfVw+^WH49ekhK_(KUJ43rUqcS<>>> zeEE2y4kZ>oI%qMKs=z9$Z(oZm(z8{P&aygFS)M>H;<>IG#{MI6h?{n1e%z}Fk?`%6 z5ayitE3F^Z^PY__Y`4p(NUzN{QWY$32w21(VPr$aaflbw6l(qKa`SZ40TsUz%zkdO zXB<_8OEh?zzG5T`p}y#G(UKiG15~J|n1~plmy=Kmrx)e#(%B<8mq4vECAEKtjuPI* zWmzA_F2r}O%RVs*d#@GN+a92}2p;&B7*FZK6`ah1;c(!=uT-qcZm z*1BEtc8vWbpxg4kDWr)(CLhf9!$;6ym3Ya^{1MZ2aY_|)YD>{R=K@jGlIy_PtpRk4 z)cteLL}`6c`x}Z2a5Bj3Di0mef?RtYj2iN9 zBBWptRv|n$jBdXA#}i*1a>bNN7BxMU9^BVC0D37?O+^m4-8#8tcX!@l-WP-^3Dlt; z@n-Q_-z8b)@MqZ{1v=pdFX%?p^Ik7Eb3eLIPIQ-*-0PoVigMcXU+KrTHuH<&qYAUd z{-G)SMR~epDz^5)t_yD#?z1F#1EWt$ixUnKsy9r+WiJzRHYQgJTf$M?dhGqe9}6Ez z4UisA#>TEMrxBmuQT=&C79yQ>n*ytsfae!=_6?rT9V4Vze|*GUzIXdw{-GL@bgy%Z zjOY?qRaV}UB7E#gz5HGz0MIvkIo(`H6X@FWmOvl!>hJHo=HuD);X}smxk0?dCi)HL zVBwvt%6L=hFsUTBTGq~T#BX9LmXl$CX(tM}gVM1_$Vr z8YNh@>Ka=!l?vuoBiYO=8h@GBe%m4w0$jUjG1v<28h@|+{7G59UoK}Bjq;;UN70QE z6$|6LMxlE?&Da1PNa8peu$ME*p3PF0wrBc$>|`KgpXSwEVsaMN9ZM6A|HL=lCh9&T z`qpUBaRt~3U>79k55$imy%sbQO_n*+JW_vsl1603qli(TmqeD-#fWP^{48HfOp}LU zYDjWVS$9-Wy-zYzaUsP^7qdb`rjkRha^@1(y`FFO?{RgQuQwtWR_IzAY~f;}fl0j8 z8oh-|6)_CZPdxPrx7EK%dD zyN$=H{d20M>q-i^_T6yH)quZbd?cDvx81&Ub7<~0@DNq6MuJcuD)fD8__o)gc!gU@ zU&GomzlbvL@R>38L2~^Q`NcR4>+JE>U`P7bT}83SR?kwG1O=wP%n6%3py?-RCVI^< zuma?L-q;q?){ix7$gmmic#uj$&;kQkFZwGU|32g^tVWJtIa9`}Qsud&>DW(>FD05_9#+kswzKYHkiL$`N0(3PU0@q zS6rk{vD?eY6mECd@)@nI@Y|xAxuYqWon@RbE(>trtw4(N%O`gFI4X-)Dd7%iK0K<1dQV zR>L%8O9f-xpr|63ndUv4U&350r>F%3ae%Yl_}t+#L{*>_&KK7)Ib_0N(;lrqsk+?b zmC=~|Dby`2q7RsTC3;vo4aFOR$?s(Prr z`qYwEdTOae|4*$kGnCedYuY#~Wlh(5@MK78Bw)+SY{5O&QvN8*ceBA~`EGjGgN)si zckMLCoTuk`XgV^0E&MCS}0_XFa=@*nL z+uM5!Pk}O&Zi>C9Us!eLQ@f|AKf0Wgf`q(2ICR-#FUPUeV<{ ztQ53UB2)VSdT7L|;jEq#m{ujOP`c#d??8GpMlcl=fxTD)4BSdO8XyvWTg}T^CuaqIw8vNn(x$cqC8RBYQkysxpSMA@ebygwx6^ z{BN8y$v+H!p`lu^Po`9|;>F>|;@3OKI>1L*+o4^)7nt z4D`FGwHQ47@Wms;GqxlHb=H{`sA?h!E6aJ1m=@lU6!Q;*B|mGY}hrXdZPx1vVd@3o!~D z!7!62qo-o-OewFqL|oso_}cH>NK=M3XoC6G>#zgY@6H%Id;hNw7%DPG-0vlWz_rao zL(1EKahkimm-yIrBpN<-6ISH#+#jOS4olZD-s9B{W`lym|HY;*M_CDcTz!on#b+e# zg?d$6-wD7~kscr9%oM-{hV*?2GwiD4>m42%pWa3~OY!GQyH#a<43}9`GmZRnGLS_W z`|+^)!C`}Bv^cw?AIb#`oZ@tkyW;m zULq{#L-N$DQ*m9d`!3G8ri<*BR0_CppLjj_7@M?+d0N*8a-SiSQCF#w`Z-AKOqf@#ji zOt3t}PHdC2r-Land|PaBXYNzmeR!(PV<+&atH6py``>szt{~FB#uQB z3$D8r<@5H3z9(B%>gVy~mN*!3iE^+Fl!`p!N)oxRtA$C%{&2 z)6zZ#^$;0Z;J!TWhjP2T_)uc;tex-=wMKd1Kc3kPQ&@1ux3MzSJ1q`!_V-C21(M6M zQR|p5Z->kM<2GKdWdfoprITS$C{s5Evd6(>TBo>MOl_Nps8RWn=C!S|#57tQ@R&YUaH`e^z@GnR;>>TB=Ws~*Q`Q<>x!T_*JglAvP@eJS`BHw^pN-CPFaXNk5*Zu*JX4213mu!sT+01 zX{j=GJ>ow5;X8So3-Ih#f0h#zSa+hEqz4*zo0+|2Bwp$i%%0j>X3zj<`nDe^mzRHF zL)?V|8BS`rcA|tW+sp8r@HGJIPW+5mTa66lpV8Q#8ci9AfB=|C5|qt*46e$+%3Ghv zT0$5w9$P%zNL0PYMZN_wNuysh#++2yi@!#WKG9$D5#feWk(XTDrZ82piv9~bEys?- z5)Ur)6&#@BDm}lt>Z|$Z7JKEgcl-tw?YTBEc^CquA#XBRT5G&%r3ES3olBg8!CN63 zLw-4(1F|K%v0%Q%zo-(;zyA@**BdF9Y;k%W-KLBiZ~gJ=vBml*rNaluePoZ;Q`Ry-YOb(8MJi|_Ls5?V zMJ)tf9_~&RWH7a7koOk2y%0Z94~zMx+#t_oQ{>K@{VUT5=`^Gkb9cJ)jklM1*q639 zjfZl~jndOMLHbAFP?z2|tM&V~iQj}tcGM)3$Ml>Z=yCN*U#BHWQTgE*28O-VkNdR( z&epR-0w($)1_j@$%%T3`i}x1#5_N+|Hrqr39i*0|*NzP_e(B9?m#_aLLe(NSm%Jk2 zDM;r(0Q^7OSvN7SfegcH_q9}>A4)E7@5TfSzC)nNLGt9I_)>VaC?0=I`;N9-6goyGF<)|UeNVO2%8+fALq`i#lrVK=@l z+&FQv`STGWDV>FC6hr^YD%P-lqRuWI&Bglv5$cR$uK_{Ndx}>zpLu@Yi?Nec`$8=9 zzRnyW`$(q^F7IojlrQiqgH6TvSq!OOWOZ|07}K1S6I7Dg&>WBSk0qNx<#kVp!GA=s zCxJVoM|ZNai9!C=mp+xogEj20j0hQ0?i}>GnQeX_^lMS&6KYI~#0y7yI2!uvw!v^& z#lS-$u9Elwk2n0`bl=bZ=ic2W+ry7U*FYfey1LEF2NA1;4xgOlQ0jWSFtOe{-<^XC z<3c3uL!MKzVN_@{=02*B^no}3)K%hxx_5d(V@m3xh{AgDl= z&c=I&>UprwT|8Od_jYcBCQaV?lco|8Ps&}4fTZW@>s~mSQ^xJmCdeRGmULn6jvF?7 zA+munP06q$ZFlmM8)v_!HI%o0B*RYUPCPhv0)AJw`K}WAPyu3f5|ph>>c+>&U^lAS z$5mxjMRJt?zVQneU!vJ!)QVLJBg4)PVf-B0Fy?rsv;yoGlBe{ij32~bZzJ2wC`y+E z0ocT0w*}>^1nuHYJVG@5B8Xt)*Gu&1H-yhWTQWadNgp8t5nH^7pw&}m;+3MTq(~R> zq-aznh`w1X*d~eBqFl0>b?83aqRorKKbV8 zA+=gbQT*Or!2%ohZ?uLa+)1=q{ufZhbZz6NI+i!r!u@>>WaV5h52d{Vwz=IA z%JJKTq=cY3vO7-(b^~P-CmQRg4Ghy66Ge(rQr>itZ$1d`BK0QsJ*7)yaT0ISbp<@x z_(o8|*dF7v&Te_9>V64u@Gv*KeIL+;CN9zFYYsYWL=RhhcUy%ZS_3|8DFvN_Rw`XO zqlYNbU!@)$r);b0!9U`vN|BCkn=U2}O^Y1S`~x5Tu$aUx)ss_HXwbBYD6cEy!W=MQ z*F|#91szk$-?;4=jhOG`Ldaik)|z^b?|%3ATEiZ6cmp!s6RaQiauJ*_py73=z&`3m z$u*y#gxfM>C|(Ln=cL~sk=mv%(P|jhVf8tmDv?ZM-(M&CGD>6p39>UH)rH^DTX25T z2Ls4m-0&Ji`bD}cYeP{p+Z{evzBhb>75hMH%e1z*uOXgmWFVwfw4N#%WY}99?X|e0 zYt}!{vi@c2$A%iY%fsHyKIgfb1Y0J|34SRtnSeStZaomNnul!?%7rwD`8;QZ_)~PIFBAB>h_R`M*uiynQ1k z4sMijI;^cWvilluUR+d|!W}t^a#_wJ^i;jbC~f_Cv6g5jQkY+5t(G=ftA(EO2&U<8 zCH+Jwxa+!NzQj_%J)q&GCNSQzZj7j}pU-bp`2ifHw)Ir~@o{VoYpK=oM2FNRLF5Zk zysS^%%5x!fF>K3xx)0t9_h_=nC_h8?usZOj&8IP0%9C=~)aSu$DtaYi(d-iiS+nyO zPx4~}*}hGc`Au#%dmDcxK#`FO>j^~SH9xv3h4 zas@tg+ENv1G#@k}(!-lmveN*XK{qY!o`q&O4>P)}ifIOwv1w1R-Jp(pj9%D(K=6gP z$8Q35Eb2exJEX_ty!)Weptqv|2W2A2O;fk_Y^YJZcR`X>!M>89d+KGU25!+!!hH*b! z6MKaX^Ow250M={?3n$%owc4+}iMkOFAluN%?~gK%C#&4E!g!&m4Pad#VZF06g=9SX z8=8|tP}5Y>h+-ns&%vouIMy#~hpU%{8=sj!QAUhq{RNHQsdHNw5yq8~QqQu6xP;#O z+euYuZT**!7b!Se;01GG^V47SMh@2T1KbYySX-gucVRxbCvgaO=-PAn&0ZQ7zIRLBgf&x&6xDRf5-5X+ z-0|YY{oe6&zbY04F%SDk5Yp6cf%LP(5XOVlccAk!enV%|;*!Q}F-CA&Kjs0Ir>77R zYY3J0hUt9v%~;Et zEkx1ZmOv)Ya6+f2?$IFsyd%BNVWhB>hp+>^jJO`QjMC;;>Y`Pv>`DHFK8Qew-rP0l zu_<>plGa%9(aefq`HEn8Bu~D5zcOg{O0g@g&Mw0GmgADW$mi*bdCn`n?RKEd;Q%8D zhrHWJ!TR<(8l}8UaQ47oh$Ze>mhc_)cwM&MbAOk!6^Qg>$@8|j7=7z^_1lr zVS`ZwoH1vQkAgNuo1ilz|5@z!e(xC5pRWa?354w(pXTnDD3kkJCgZ<=5(qtI=q>+O5kWxZGZC?_r-F6H;?bY|T8 zdWIU~j7E>`;V9OS9-)eTcR=v zT?L_00F%d0w9_9=Nd#0f?+B05jZ+{AOb|`E=K&wl+rR@JjDOmwyolmX{Ni3GSz{I>#>EnZ9t7eK)C^Q#wfflty0K;PZ}wwOjX0 zB{Ilo8!e#CR`i_FQ7amF>Dyxrujm^g|3lcFSLn>Xb&!d%k((A-&(oli*3ah46Qz%d z)L^Q3wWo`8ge^Q@KF~v_pTOP_a;^1N*T9WARR*4jvFXUPob+bA@m}Df>`C9r zj-J`jFbAbbnB1vq__)F|2G8N}*_}N-WM?v=c1pPBd&Vf~NfpL4*?6;nJso*-+wPJl z*K0Qyu_UDPu>cj8tpzszk7$v@8eDSGV+el(pd`lkP|jJp9O@GlgU29iE<4!B+QuxTUk;bcrEhQ#L;@-z3Z}KaYpgk9qf{0!l4`g>1Ok z@pMmVNLJY8cIInH|7k8oreC8r>aOvEAMs*|v;FFKhnZ}ieljRH<5LW{vsbQT1ZBqg zUJDAJ)?JkqVjzUdnUuDmx?U+d$0$|WfcG7#@|R4t`hnb+$M}_kg{J^a{E@StTlHTV zn~HuFcrX2m%9b=`2%kOa{4**jmLPG=$|!PRo5qH$3IDsC$#T}|^2p?2=p?}AHj~Dw zOlzlvqKrI$lPwYFcwL7d92kN>Uj*`<>$`Un!PcO@*js}aId_&cz8IMCzFCG*0eBGI z7y83!a4_YOP{8N~BYVmd7S)r9nm*58tlxhhCjmIXIbl;X89!0A^nYmXF()5Co@Bc| zJt=j~qd!Bzg(Oh{=cKrJFC(#k{t7Qc%~jicceOzYH!dJ9E8&@%0?R_zX8QqiaM_2x z_1u}G#h6D!JI7eI^a~{-_Glm<)3nAcX~F|O;^e)LLC`SX9m7@MHaq!|bpXp>Ij(Y2xe-X+K8B(Mj%Gn?^bVhDgrID1}MJy?+iD(*pD~EY=@|x=GnO(q_ps z<{BekL1#kI>=-;x#DdhR#0z#V_sj`AuI_f`E6y_orxI(X(IEN3VW12YGZlJvQr z5uT^=-B_Dzs4tcU?kOd8ihS^LfqkiZuD!3BkZ};-P%?Cit!m^5(8`C&Dl4RfNJOP;Nq{+4v_ z!`Q4{5OJ3jMm(m>>>?HS482ia)8^wZ?4#3331Y${F~3hyU@bPy-pNVvS<{!+K$o5h zDihHy$0^;|rvsQGdGS(Uh0cdj=Z7^PvJ81&AhUMQlmR6sN;V1hjo4bb6!N*tLo;Os zgtC18(%Q>m-&8oajyZkS=mX-|9kIo@-vHE{Xsnk+Ax_!*aYA_I4YtMBBwtT9g2t6-%tO zm89g8_!BK|zwX{PV=#n4xMbJmzpV0Bv|XkHIu8$bZq}j%$z^A#lwzL;RYGFB+q*_w z-fm-7YCHr3WXB3$z(N#_5PS~{Ec&h$ot>XDp6?kCdrGn&-?+47`T<_V?ZjD&-0Z&+ zIDI$-IEY;bPjP!xsm)K?yA6&2D~=JWirewiOI(BKW#c7xx)S!`nQEUI9=djN=QGbM zcHoz|bZ_bE=x_tQ*o>$A_w8OnU@U&z4%e5nP}vz%r83d?1%73^EziKu35XkS1n3m?SM;5^Z5em}BG-bd} zcD;GnhLM{T$8J7Zm?l1-Vs^CU)sjhO^9xJhfDFdXlU(15*C8cSQby&bN0yqtp{+yR zl|VPX`;!eR@mZDGr<|$pD@K9kbs8orqnlxBu})EwNnY;*S8i7tCcM6Gl4f%6yAPTg z4v~e8{xL=`bw)&Fpda=P(P@*wu$L2x%;P`sL@ZSI>B^if&^#m!HVf79$v48L>YhyO z9=%O5lpQ!%SSgl`fW?~aC)Oq}W4D|p)p%^sdP>s3W^K2kL~V8aZ>H&4w|%z;oi?5G z-Ng;8H4&soFaCuFT!s+Q+smhtToXS6$pXk(0U(5nqPJy{DnlDg#+JyI9?Hg*z1`aDKbM}S|CF&mccZw9a-yi?A> z-5;qsE0&(9GBTX>5|EEkgDQ1mGI@{PdKu6^KZ+Pr-Wp$TZ$wr*nZ$Fl%O=CXCELy* z(HpMKF3paBzd&tF#@OVZa`JAUqm(Q*;znkQDH-4R!_tmO+I?gNG7dY!}1g-*<72N>*)Pw~O(^xwsSI(AXfq+*C{Y{E15gay~@6 z^hf#Y3d5OImo(lxs;f=2NC_Dlv`8W0;6xtR;%rAQQeA(ZG43N*IpR9jXiMyQL)FN= zKYgTOIcgI}PyNh4j4pqeJS^_az6b0maw)u{e-YgQEcazp-rfBuAq2tq=(g0_g2NMM z+5eagK6u%6xn7VVu#;j=&|;)^EAs_Wh$+J)wSs0}2e;A|*g}nb1MP>Co@U+f-T9A5 z zQ|W`OmnlL;r@u6VU2Q97@R>^xT*!H4ZM}|&ciNdz+P+aq<3&`dxnk@zKLfI>7 zBkI`aky1AJLS_;L?kb1*-8kEoDyD^wFN#~bps2+SRbz1(1zx`J({4S5#+)`yU;hJ> zr$&fV)}C2Q7vAliD)CJ%)43Dy9rer~4QXEfvpLRD>sxEbCPtDDv#~#S$Mel9M%re? zH)a$dJx?X?)(zX452OVH_xD8l(4sq&zZ(G830xs62Kg}DkM zWx8>vgy8O2RAZ~Qmh?>amj%V(B4Y`(5|NlRMSB#*~KBt%P;lXKm5P z0_qmRy01#dn@c5MqiXq8ddjc0oYy1K#W0CU`vieE$>1MSc!Mgz zkLD+YD2Fwi5^M%`+!zcwKeY&~$bjdzep%f!-#Fb|56nxF_mzN*LN`tayPiB=y*|0F zu-U{ha<3WDtUt&d>TCiMjsD1~6l{RGCpeaIhCh&Dv4=E^ef89q%1uaT)2bn(4h&v@&0>x{9&ak4TK^+7 zCmjhhac0kO4cTjEU*`?Be?#rpiTRs913D=npcO8suC%iggp$EX4Ee>Bv@0nJ3XVU#c6mGOZ?D8Eh5K9PC-B7V_6U2}*MS~As+jR& ze9q0H+$s~VfvZ~gdhaxr3Wk$dpJU)IV9h4Zw^CQD!sI&y)<)f_#0lR`dion|#l!Np z92Bt5f;HxG-$od*AJN%1IbQRtsxQ$8;|g~}l7Ffoq`A5Rj-MmAdy||sa1__0GI(iG z>cq`zMj0Xs|I~0Yr&*>fze#-5<^YtAxbEOBqhL1M>Sw$gh>18w2K>3tll{1ptC;xu zre&hTQxb_)rGIhN`i~sm9)nq${_3Yo=`MuHWB*-ya@FJ5$H>@C6(9?g9Q8ur8GV3$dfD8 z?Mgu9;8zkVA-=H#enVX?z)7Y0u-GFNTJM!VL(?#3#@OnIW@oO<43!@oHu}9x7lK1} zM$6a?p+2}_8s`<(xzh@=*$17)^Ks>Oow4(m3#%V2*~X5W(_9f z1`fbBJ}Z{?tr1 zEBM}yd!HsHdrPNN$n0F0mMNNSTBy`J-qyKee(*(Bhd$>)J@O-20AnetAu@&qix9O` zJf!Jy;%ZSE&a#HB1gFD+iH4F%)Z=f-jDKnaaj|=4@=^;t>6Guj9SFqhPK&=56#WF5 zjKvBTy2Mb+t*2g@$EqlJkQ%)7ND8D=2p&GYoYE;7tYCXqA`z2nDA4AxQnB*-yO3N; z3CqiJ^~X8KJ@!l0nsX@ZC0p35@IXe5H7^qy1c$L0%$zLduc#Ic+w5N1H|9v6+~pae;24TsW1q4q?9ZLU^*;8YVAz%uer9MSB6bvy*-6)P4|eda9k(r_4eIXoK|Sq*~w<9 z8LU5UwMlL|boCrAe>Ip$fUFe_qMct%f8*Nylg5me+9L=%MX$?GB(Qia@v+0?lGlZO zNp&^3rV+_M(aY(tzRR99f_KRJ<=RND2g){?6!$=y5AKM>hgr6hAEQ34OoLnEyu!yG zUzb08uAENGFZW(}3;57&)g&@O7*jw=Z5?o3?rspR?EY{n!X@;toih8EQj|!Y+Q(gA z=5SyLsK{-;d$U}18_js@n^|exkxkqAcjuT|VfAm3=aJ%>4#bUi97XaBFccwPgyy{13NLUfP#=oROG3OqqHdSc9@6s-=SE z;*L=f>aD7eDb_f#uVmwOOk%C==Z-w*=WetAI{ke;_k5ArT2jVa<%CD4qR*l z=}NSB7^30#k|i~Nk`Q6Zs3Z9mS>I2Z{G-2XvH6w=yvXfyTDcSl(3;(&7$IC?>1zHM zvNK||o|?`QF(j(UOj)`2cz$&3iEb0 zyG&g@rZ{#sA?5pSkGiuOTfp9d@y@!s`q!#gJaFaZnzz;5S!*AG@t7Omy*e(Eq`*^@ zX`|zIK~VO?oFV!~fdZKjm#5l1T(U;;9-s2jIx|026ke;1G&e2u#m+y*dKsWNcY5gQ zhU|2aM#wpRQRT)=W;dXVO1e6_hQ&q^hcsxbp)7$-HP$^WKsYZ7*=;s0^?|=m)gPgy zP@%5BJY70|G*VJSHySi$z26jVJp~L#kGRkuqmeTG3ldxncG?3kqN>nd(50bl=uGhV zDP5UoLkE{3YVZoPHY=-XZ>>pfd2?wH#?#=cNd=u^ z(*_~^I@gk4B0EYCbRR%=Vwvq7_`$2nHy?g&)lJ){w=t7Dve#xh3owk zDS&%D`}x{JAWE`lp)-n}vzZXiK`B0yB|{TJWqOGc`iGLRR9f4(F*X7=O=n%edYK0M#7koDW827+zV&b=q*_&bC8xl z7X0AKNQn1+fhm<{D)J!^QlJ=#TT-TJeBkM=&0=U6Jo8%Jg5I6;=FbUXcYNk)Mnbq1 zXeKG`s$-_iultk%D?z?tawVCQ7I7QWzMbI({Zuzs0I`2(OBCXq-P6hs<^tVz{5?nI z&nymoeG;;Zi_a}*Dyo|A>eFyQvh+u@l`#9|NLu~Q|AZM#u>BiY)^I-<%|J!1Eq_wk zlUU2{{zf*7g%bBe`tZ^KN`v(gUZU?CB5W#tgpKeijp)XP@%y5@nZpw!5P%(134dlY z)-WoN^uh}b{NqLpjBVOcZWl6B`zO&cO>y`dwHD#J+plUC-$MHb z8RL)S*wc8VaG)ab;>BlKhdIaB<*$V}fP|X-SwRz-(?^m5d-wqHBz}813(^`p2bZn> zLS=lc@ALi5Jfb~JZLdur_dg=1p<|)DG~qXzPFRO+orVe9#WvKP(2fI)nJs@SFaLTy}Z? z-jJ5}{WWg@JAppZF89ZsE?e34coJT)V{YH%Q(mpk!*WATyGj)h{wL`Tsu(!m>n&qF zWtHbQc7PI0(#%s?8a(FFn09fqW==5(8JGlvLD9`9E6ZuN-`n>&L0pG0=*e}QPRk5u zTbpj>-JhsU`A)gQpa|VnYjrgKb*WNEz-PG|z=4V0j?K)PYj8xu8(CbI*A9V-#T*A2 z`t+jzB1$;R2Xl;~&}MoFHbpmyhVHYDyAKdyC<6P+0HgA!m6_gt%I`R!^ktYx-m+E_ z?gVct(Mh@JYnS9h`kEI-uCaE>1A+yO04Z4v03%Bk;=-=kz|@%c(s)5VU=bpXpa!a5 zLW4F5*V*cZg})P%zOe%@NuWu%q)0xl;@aODLNn@95~pCe>Xn_rwD*cm%wS@hp>%uHDT=AWp?nPMG69xZu0i!F4`E-Hw^Q9#= ziI=^!lqXW`2`I#>rb}Rc6YP6#6wB-H-ogj_DmOvX2WUUjr}Y%6^{>a zzhc~$nM(cfx_NSVC9XuSOUBJ%L6oVKxMp7qWV#`1wq7#%nzODUIbwv~~eWJ%Be?2{1TvVWEC0%Y0JuU(`ON459P33EKB1io=3ZwT0Ns7ZV|PP6k!t7n5Rn) zxo7PnU$}&(Q`%^mUixZ`Jyj>@68X~GnuL=rdN!HGhqvEHkyYfjjc9GO#EvE-ovf4_ z;+=<+cP#y$Nt>T|9&NolH8l90Q9GPb#YdHb6BxOmwbz<9=(B_v75C_VJHPdvCm+Ln z|Jl8eJak4lwL0Nj8ay;}F65An#L4Pc1YXQadjPJc3f}h}hBowor_hr~T=A=&z&=^F z5_0h&R6g;@W!PCp)gh%%(eF73VO|ilra!()+#5hf$Vz+qBk#WU0TrcobizjEZ#8v` zUps-Tnx{VX&GM9#Rf+t|MujAG(Rv>XnUmaxJakhreW67vM(sq7h9Ns66aveW4Qc)h9-9NeZGOWK*llkg!M^VIK|47^FZ@DGF zx}~=Erq{B)<&v;ffd55{0pYO-%g|X(Duq)5QD21~xhBtY57J zekx~|`(b#4e7eqAd{JgPEg5~fq$nz=x2O*CR|c?qaqxu>S`|p`(H%QBE4VfzcIDJf z+H955q9Yhbl|AW{O%}gSKXnoaiQqQ|j0i6>ARkcf-Cg9V3L9ygU`E%^OL9jl#Kpr! z_^NAW3YsVK2R9Wl&2qip+Of&^N*HCb?Y`wLG7OADRFQ9MIXbat*L<0Ev!t)1WKa8V zeO=xk-Wl-7Udw`9P;6{C&JVbbs2l6nu^=D^$pnXA_FVYvSQJu5medhwCdy<3I3=N{P2Zjp`7 zlE&l^0nbnK-nAl^<>2xDfBMzSTg*~U56jxM?XC+pI`{o+T}Vc(ZM&LDrd4tW6qfi{ z%M-!>06xE3&bYV|OxQUcJw|G;Wni%^w;wj#{Wz_?8pWi`Fyp!Wt2av1OwtLY&cJ#c z{#DoM)}}?_c17Pke+sqTko=8365QNmAmDteao0U-*EAhHG@TFS;GPNfuRGFwyEj6K zm*@v!$5Z%MaUIyWhTbsvRpo~xJXA_xlQm|sZ9*8<5Kit(cI5T?(*1@zEsC8xw;B4? zcJTyyeW8jr`=g&z&r0Smd_^VOyBFBGz}v-T%xI(7=x;Ryf<2Ki0ChR!n$5Vr@?s3f zCC*3~&rSt%<5@<=Pu&?OJl1vQur4Mjq>#sj@Abtr!qz)YHt)`NBxeH)gU0OV^sS34 z-?FhB9p5qZ9;Uc!4O8s+QgQ&v^{FQD5im<2I3!>X!yk~(=}cBTD7Cg1XXYMW?2PwP zG5qmOk5+40u%u%+Z_E5EfNgnEqMk;~aiAE+GHNNjPv#j)oEZ+_c+PS={WJN}yDG@+ zFSXUawJ4{|MPSNv&s-6oQN?20>#JpPazlKI%A*+WKtH8&mio7p=!BePmgms%S=Rc& znoE*FP&YB%`XA2}wI7k`nzxG8PzhghCI&Jy{&84RYK3(%a*u%uVZ!UO&&{Qtq`bDO~j1zYrrsg8P15%a3uMmB;8Bptri! zHAor(aAh$^8AM~3BoYA~0Nv1VNcODjjYyV0Cm@sRNzXpDw|#MKVkE`DavNcjpy+Z(Jpd!WIW@&< z-Y3(vyE}igNYTkS+{j1;FmMUM_T!&Q!?M;T)}woulWmM=0Fr-tMn@jm&rZ4Yp+@TE zM_1@|I&PM?o*+7{!gebRECwP%Nhdk#Nj0QKe$0SHhWTGw=S%^~2f<{M70LPzd z^J|Y0CYh(9xI3kpw#2H~+BW^;#&OfWD^o@BOt%ds-NULb8aQDrN%@qF@$HVIxcbp3 zbsC-aH9SEz#; z^;%23cdBzal?a~Q|W*F^OS5h9eL1U;}!Rt`nTn1BvSShLw zdHrc5)W@&uO)^|Z-`s`IHP7E(fsjp2s9tW#2Dx2ZP}--O(&SiOLf|%etZA+no@*}i zTf-1f>sj(@>fQeUT8YI;8bVyI=-u3|!pGuF+AKw*Cu8U@(`$yOCKmBUxF+3XI zT1$d9+%x)dkJ6G^CEdAhSaHJt08`KOtTo#h8$e$Cdv>c!dyoeMw-~OjMcC$5A94$= za{AUsQ!>au+3G%`pohd)a!dowu?g$`ef{by_(alVLx=0Q5IYXO^^-C3Wc}i>Oy@jj z^UYr@ZLyP-yPXx*vm`1u-YdR2P{Oii*DgaA7Y=jie-Fm1tinu&wr~`1&OjdD=N;-X zbgIF8z2AsG(M=Dz*p!EL$>-Vc)YKtE- z0QGz)K{C_&qR_ch2+R>R>q^&Uh<1>(PoPv5*4A)Hy z5CI8~>OHFFt!%MOO3f>r@-jZTtZ6PKX@a6NjCE0-a0O`?reP-TZLYBGYUK$JjFFzT zHlMEu*c_ez0OyM4*5I!IVD&BQ&(^I(su<^?tL0q*N?K}l8iu@=(2tWlM_*5Rt7m+C zf=TPqd-VSRJt~xUA7+if3fsmG;h)Risrk*-PI>xczvV|N=niterr-9AQ!56}Pao%+ zeUR;{q(eEA3N1*=z zKGlT`p-;?ztw}Y~ZTT0sUVl2zmM=oNq`I4JsMOC<{6O~ZHT;z~cXFOw`pI^$UQi(3Cx|__fw1;CdDd2HWP!yBAAE2xc zwE1iZ3^?O)&*RdqL#Sb2TqJ#ZkHmGaHic$`pu2^*yUz-q!9IOwvZ)TBrx+2acqW55I9z)K-PZnd*9Cc&0Q^yS9NY{NjCA0Go;ucU5$al#I|;m9Hr5%*CpqIC zdR1AxNh3zevMghEasL3g<={Hlb8K0qLx>(tVt$|G)?A0pIRhsc=m_gfXy0)e&IUgBALsC;%Da!9_8%?d`KL2^X9t$f zIrjDDp!-D7!y-iB?#5i6ahgdMLZB5~XF0&nPp7Y~M-*+6G5&pM-y3aZWWg!zg$$fbxfruS=2Q^ zwvW0%9H}6ZGyZt4Pi5YdYZc>4fZi(vRA5LM!Q4G@p7qe)TJD78XOHPzW##q6mjJlS zMm)6L&;`H)81|&0*uQ99mtGnwl_P!QjAMckPfl~x{XHseC&af_wsJ)@yV3q~8=a>+ zxb*A#n!&u$WwZM{_cA1&a`~!J)&Br7o)59A@?6fLqfnt3VVST=2aI&BIWCaBt6Q6z z#)Jv&c2EqV<@$Xg1eT1s%so33X(&`u6q;_}C9osk<><6Icx=UXaMWw|O+X-j8 z1gK_E5fqR?0Ci$>*V4SrZylU5kYhaHWd2^Yqi=ZF8=G((^s7|+iPpY}9>Hg-J^auc z$yP1P%xvJuq74qJbcwRtJa^K!;b6uvF zthJuPK35!uRv`U)*EMQQ>2m4Sw?piR{8yyMZ5rL`cJPu%24x+Gy=%wu8^ii~jry&y z0B{yS%y2mX4*vi_Umx3ehD)^!`<;NnW^DZjUY%=B*H{e}931xN{{XLCih@S2w7MTt z-h5>6#4x0v+KNMT#BP5-m%qJW-F$Mrgkw%@Rhy7f4U{qfkA6yM~ZD>i2tF&j^r8S3JG!DGFGwFE#FlbDH zn(?CJC0ZBA_9P5v@u#oFtp;EMY6H+_Hv{lHYvq^m5-B5b&mgWaI`M;=V|bb!yLWZx zpRHfjW6;m*Qto{-ckyRI1(*9pNgRBr%=f{sQl+3jyiYiRrO>aS#PQ8dZ&(b=@_(kVaDumt&dMi%>Mv{;?YK0 zf0Xwsf5N=BO=>1m18zNVGyZc>zNoS&Q^I~YsrtG>^@$$4505ne0QveHb;7sn#aoBQ znswJPUZ7>`yDR+1@~;qksN>Et&pk26_|zZTg&gc|N1?3cgp7?Ni*elg9d+nrj1`b$ z8O~|Tev&Rj0y@`(LE;JGj5118b>NUcooQS6>T7uRnHBvq4u4AImJUl}tz#nhJ$w67 z5aoAsQJrGlPCURX$eYKKeEcqd8iBlBy^9jLKka|@>HSth`qWNG?ZbBmrySE$P(1Yq zuUhlJ_*fAxe{&{(4}STo(tJRdCuBtS$#3ahe$DndRaM;eSnpSNEs@lmnsB_gR(2-^ zxF_`765K|zB#}=D0B1YC zW9l(n$M)3u$Cy`~9-!7FI@>Nuk9o#PVVsqR9Zj7Bo z`kN8z%+eyXtPr=%!=5lQNcHPc$*9XW%2}NN>C-=@X19oun~P}hI$=hBpQR&fkuxUP z0DgqvbH~=7FOaY1X~%Tx*@!q!OJk>}^{E$Bl#sEzIr@@*n8jp&ZYP!$rvs2Xejm!8 z4!IEdj9BM6UJtMapDp?aH)dO#P!ewv7AGB5v7fIVwG&FNK~<4|y@u=G)83(*TY~7} zt-*1~V}eNaVe9mwJ!D2cTX@cR6(-6YYM1A^ApKq{Hr~00T>XffzRF~_4?J7zksL9Zn8cxlFjtT^QhyzxAM>KSh?i$^!KE#*nctd zOgC;Qz!vN%N#k4+&CeJeDe`KO$sWysZRigJ=}-%(`WsaP9d{q)p0v|lfv+;s-3B8j zOpXRg_2@rJv-XJgjoXe!Th~3#Gs&#@uVGm5B2Ylx)Z;&u7aFLN+z+zGAG3x(mOnqv zs{TY-_b;{VZP7^S!N9B5_txgzp^SNi^M8P9I@W<7aO2P~{e zk}nlU zD*~!O>B%GMTsPXh#>q~d^T*V2S{CL+6X;}w-$D#RoAFn}LzuMY;jf3okMq{FA%z)02M0d2ix^dGZowbK zL9G?Djvb%rDoYPuoPUisD6XWXC(zTjlkDxo{n6|GeY;kiddJF1b`9(N#d3`pjlNlw z=byYY^r^hF96QYY@r=|tK)X+&*lGSGFytAFjCbSp=CmR4QY!+}5I(i%RN3G7$d_%fCZ|_xR@dFNr9kX1`w2mVoo94!P)N$SgllP7C_3KBqfW3^~mp5e9 z#H>pkW17#J`QMD4y()#wpyb4UN|M&nOURjn0gvZeD$%i%p)1I;> z4&ptjdq}bSIU{~eMc)AP+x-4@9oG0^42|=BYZ6PRELe00Ij6~}Niy~6M>-!XEXk_R z9HCg?dsJ6ioRdlWvw@DB)n;~=oN_tq`qW}MGxu?t&)OoroUBf_(}9A-{xqL#fA9YQ z7>AwfO(=BHa%$KCyC_K_N;BgxluZ2i-oeqr2m$)*@)VsM9(j&Lg4+qMtq zkF5-{H?}yXXVj~r2;!F`Y7+j7&N=D&RLLBSgoI;`c64FIc42fXt1(0nv358dAM?#S%9WUb6J&p~d*j-XZQCH6ll8?)(6GqMdlG+2c9AzL zW==qjjy-_0OvCs3R%Pq&whA5GjggrC?hoe}!}W=+t+Jz_O_)%72vOq3uYrINCNgbCJhhl_KwB%#2Uhijh^> za!yD*b)hK^@fo^i;iF^UIK?b-Nf^P6x_i})vK1L-PW%(w@uXOvKYhJ=nk_Abc#^|3 ziy&_#vF(t5QBIa=805xL!Oz{}=s)`OC!4#5asL1UKb1%k@q)~A@1FE~!@NpL&&ddX zBTtTJUQA?mJRDWUX*mi`Ivn#&5x#S{lh@jvq${EpW4HxI2N~d0(fNTv*xPz^r+)Qc zJC1RK)}D_lau|`G^lCezCRuuU4Ck#v%W%AynFn5K_nALYD+AMleQ26FCQQiN{<-v} z%Obg=R7s^#!u99$s|hp6pb`3076o2$+qFHR+6FkLqWY7}x)L?U)3hNxRBLqQxl#e^ z-kz|@xNXm^J)xC(In6Fdxm@~MpQeQ{BvO1uO54Ap}) z+d}1ij?q)eF2!@Lu2=h0D{zhed;WBNzD52P=i03P`SFqu`~-XRN<_Osz$fTGI$W|J zIx8jIo3aBgeziPzrA`Vl9e?`uF;`%OZR4N2PG*stJ6EO#7ykgSQ^@yY_K18JX)_)F z04Je5ezfnl2xJ%}fN`3O$!5qZ58=n544-1<>`>vN{kxn5kCq?e0Z39qSk^68%PhTAOix zJhjF>%{xB9%)6EU0JIoz2?M4vQ(NCILB>y9;*>$nuW!#;w z0xm~W{&=dhY63oS$?sDtu_JUj$31dC&os9Xi4+ej1bgTDR(!3!i|0$JT0LvaU8C0m ztgfiDDaaiARbR7!zIDbn zZ>Bx|wCwv6%)66Gbe>+^;d*x!Mr*|Tq;dZM)~P<(ZIhL3{{Xr_#-V4onSsI2PfA>` zv0Ur8<-A2;M?vapNwujNNC)xss?4wg2qzy(M)F^#arCIr+Q~ z6*KttH3&<6pa$pYY59er#&Ua6%KHa8?m4w3ZaRDCpRGgZF6@K#=Aly*Y;xT>_3KLh zPSQhr4zzOl3!QIq?Q%KGAJ5*Ww$_cJ5Ufvc(zD}O$>p#rNs<7+DIUM2E?2dO&bx%# zp?2I42mb)CO#aqwr9*Y+nqtEOgSd)WA-BigUtYh`mn&Td%$UWx4B0Ku<4fiSZVx|N zj!0ut04j0snpo8nCpkIdi!kPoL456_1pc)oR{?*BYK9x`UroXnS=7~|fT_<{UuFlv$U^BSA`QWoI0KMLySUCv%vbuXN5>BqH5v6kEwdW={DS5^W>=DO*rQQdUo|^QX-o;maA&p6M2)`RmB2*9dRYq7F` zszJx$P#tS|!Tz*!#5pC{-?yFNP!-4>M>wfBnWg>gfRoUkqPf-9Liap+3L9Jpo_`uS z*K&Df*y{evZ47&!P40OMnxI(uo11|S%i7Ll5o;|~sU5zD-V}Z>`vHt*ivX{gQ{n~%_{CsEm z)8~l!WNUo9dsLfMj}?kF*uxXlRIB1BasL47qn;tlBV|*cV^hYXZ#9U%AcLlH_|)sI zLf^&5)~lX9#mg?nryl1UD%L~XJ)tI`}Ki=&~-XMj0=8kyxA1sYkhdny`Qiwu~ zaDJ7UABbS7f8ijuTw$s6#9WWDd_p+{b4lg;gUwWrU$%Unn*+&1El$A(M^RsaIT!Z{7#-r_Q^JmRnfZhCvw_P)}O8@+`^#Om?n$ zb=Wi1eg>!i0EB`i!Od4X?i{bNsMf2_MhCqj+l=(8ORmQvXK?lF`c!}Lkwm17sYDcNC#FQjZ6Ok2`pdU?Ms?H%Ga@}C6HcmkIx-xix@w{ zS*OH^#{_};)X{i?-@jTf9^tO`G#1yUDs|Jz!RzZ;-|&%JD)k(U^%W}kq6O%~)831& z<4yK7zQ9+g91S)XjpL(tR ztpgBueiZa4m|0;59C64MU9GZk20K)PQ-^on6s@U4kG;G3RcJ>xw<}3vc8)pxYO7eD zn@?KO{?Lezy~RZ?r4QZRRJlkqf3wd#0r?7zwB}#uT8pVejO2e>X}^iPih2`sH|#9f z03T1SPP$w9dD>5ExVnT2+_0-ls@q`TicKzqOy`=;MI;sW#W^7=56WvwZBE>_ayrvK z-2(&2qR3Y_S1otBb5M zr~_^~{{W3gCb774iqDm|xxYM8^D*YrS$H)`d;3$`KD<@bRZtH~L$^5mJ*s&Sl-X4m3sc}MXV>-mQ~p8tQ(Mbp)Ml3|E18s} zZo7Nb4362)UZS+xIMqqd@~Z8CdsK3w`HIOCt7M<^=|`AXBj&4n6si39rkIyKR+lJq zGt6n;+)+hw9m4JQ?@r@xJJCf1jpL8box%E1MF<7$fz$B+06pmR@86{qQvpvw>(exw zk9sJk0{pBnMkyNrfH@eVidQG7yN7X~{{USmAoGAIqJgm~!MX}4rX(u2J$9D#>CEcmIN*2se>y0wBD5k+=wNPO+%Xh+k}pXQ9CJk#C5h(AkGxGU zn>S6z^P-9b0(nQRFWV&`{Ov^)3>NZ1(niPDp0~1fF5aNz(M2n;BsbHB%F5XMXnjuQ zy2{^$6i}FmzqKw^xS6x{H6(iF scoring program path> () +# +set -e +set -u +set -x + +COMPETITION_DATA_PATH=$(realpath "$1") +CODALAB_PACKAGE_TEMPLATE=$(realpath "$2") +BUNDLE_NAME=bundle-$(date "+%Y-%m-%d-%H-%M-%S")-"$(basename $COMPETITION_DATA_PATH)" +SCORING_PROGRAM_DIR=$(realpath "$3") + +tmp="$PWD/$BUNDLE_NAME.tmp/" + +# Create archives for the reference data (solutions) +mkdir -p "$tmp/reference_data_dev/$(basename $COMPETITION_DATA_PATH)" +mkdir -p "$tmp/reference_data_final/$(basename $COMPETITION_DATA_PATH)" + +# NOTE: We need to to be in $COMPETITION_DATA_PATH for the following to work. +cd "$COMPETITION_DATA_PATH/" +find . -name solution.csv | grep dev | xargs -i cp --parents {} "$tmp/reference_data_dev/$(basename $COMPETITION_DATA_PATH)" +find . -name solution.csv | grep final | xargs -i cp --parents {} "$tmp/reference_data_final/$(basename $COMPETITION_DATA_PATH)" + +cd "$tmp" + +# Convert Markdown to HTML pages +for fname in "$CODALAB_PACKAGE_TEMPLATE"/*.md; do + echo Converting $fname to HTML + pandoc "$fname" -o ./$(basename "${fname%.*}").html +done + +# Tweak and copy competition YAML file +if [ -z "$4" ] + then + cp "$CODALAB_PACKAGE_TEMPLATE/competition.yaml" . +else + CHANGES="(.title = \"MICO - $4\")" + if [[ "$4" == "DP Distinguisher" ]] + then + CHANGES="$CHANGES|\ +(.leaderboard.leaderboards.Results_1.label = \"CIFAR-10\")|\ +(.leaderboard.leaderboards.Results_2.label = \"Purchase-100\")|\ +(.leaderboard.leaderboards.Results_3.label = \"SST-2\")" + fi + cat "$CODALAB_PACKAGE_TEMPLATE/competition.yaml" | yq e "$CHANGES" > competition.yaml +fi + +# Copy logo and scoring program +cp "$CODALAB_PACKAGE_TEMPLATE/logo.png" . +cp -r "$SCORING_PROGRAM_DIR" scoring_program + +# Zip individual bundle components +for dir in reference_data_dev reference_data_final scoring_program; do + echo Zipping $dir + # Zip without including main directory + cd $dir + zip -r ../$dir.zip . + cd .. + rm -rf $dir +done + +# Zip the bundle +zip -r ../$BUNDLE_NAME.zip * +cd .. +rm -rf "$tmp" diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 0000000..0e63f04 --- /dev/null +++ b/environment.yaml @@ -0,0 +1,16 @@ +name: mico-competition +channels: + - pytorch + - conda-forge + - defaults +dependencies: + - python==3.8.13 + - pytorch==1.8.1 + - torchvision==0.9.1 + - torchcsprng==0.2.1 + - scikit-learn==1.1.3 + - matplotlib==3.6.1 + - pandas==1.5.1 + - pip: + - datasets==2.6.1 + - transformers==4.24.0 diff --git a/md5sums.md b/md5sums.md new file mode 100644 index 0000000..e14126e --- /dev/null +++ b/md5sums.md @@ -0,0 +1,7 @@ + +- c615b172eb42aac01f3a0737540944b1 cifar10.zip +- 67eba1f88d112932fe722fef85fb95fd purchase100.zip +- 205414dd0217065dcebe2d8adc1794a3 sst2_lo.tar.gz +- d285958529fcad486994347478feccd2 sst2_hi.tar.gz +- 7dca44b191a0055edbde5c486a8fc671 sst2_inf.tar.gz +- 1b9587c2bdf7867e43fb9da345f395eb ddp.tar.gz \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fed528d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2fe880e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +torch==1.8.1 +torchvision==0.9.1 +torchcsprng==0.2.1 +scikit-learn==1.1.3 +matplotlib==3.6.1 +pandas==1.5.1 +datasets==2.6.1 +transformers==4.23.1 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..62e4041 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,23 @@ +[metadata] + name = mico-competition + version = 0.1.0 + description = MICO Competition utilities + url = https://github.com/microsoft/MICO + license = MIT License + +[options] +package_dir = + mico_competition = src/mico-competition + mico_competition.scoring = src/mico-competition/scoring +packages = + mico_competition + mico_competition.scoring +install_requires = + torch==1.8.1 + torchvision==0.9.1 + torchcsprng==0.2.1 + scikit-learn==1.1.3 + matplotlib==3.6.1 + pandas==1.5.1 + datasets==2.6.1 + transformers==4.23.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7f1a176 --- /dev/null +++ b/setup.py @@ -0,0 +1,4 @@ +from setuptools import setup + +if __name__ == "__main__": + setup() diff --git a/src/mico-competition/__init__.py b/src/mico-competition/__init__.py new file mode 100644 index 0000000..1767efb --- /dev/null +++ b/src/mico-competition/__init__.py @@ -0,0 +1,12 @@ +from .mico import ChallengeDataset, CNN, MLP, load_model +from .challenge_datasets import load_cifar10, load_purchase100, load_sst2 + +__all__ = [ + "ChallengeDataset", + "load_model", + "load_cifar10", + "load_purchase100", + "load_sst2", + "CNN", + "MLP" +] \ No newline at end of file diff --git a/src/mico-competition/challenge_datasets.py b/src/mico-competition/challenge_datasets.py new file mode 100644 index 0000000..177308e --- /dev/null +++ b/src/mico-competition/challenge_datasets.py @@ -0,0 +1,99 @@ +import os +import numpy as np +import torch + +from torch.utils.data import Dataset, ConcatDataset + + +def load_cifar10(dataset_dir: str = ".", download=True) -> Dataset: + """Loads the CIFAR10 dataset. + """ + from torchvision.datasets import CIFAR10 + import torchvision.transforms as transforms + + # Precomputed statistics of CIFAR10 dataset + # Exact values are assumed to be known, but can be estimated with a modest privacy budget + # Opacus wrongly uses CIFAR10_STD = (0.2023, 0.1994, 0.2010) + # This is the _average_ std across all images (see https://github.com/kuangliu/pytorch-cifar/issues/8) + CIFAR10_MEAN = (0.49139968, 0.48215841, 0.44653091) + CIFAR10_STD = (0.24703223, 0.24348513, 0.26158784) + + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(CIFAR10_MEAN, CIFAR10_STD) + ]) + + # NB: torchvision checks the integrity of downloaded files + train_dataset = CIFAR10( + root=f"{dataset_dir}/cifar10", + train=True, + download=download, + transform=transform + ) + + test_dataset = CIFAR10( + root=f"{dataset_dir}/cifar10", + train=False, + download=download, + transform=transform + ) + + return ConcatDataset([train_dataset, test_dataset]) + + +def load_sst2() -> Dataset: + """Loads the SST2 dataset. + """ + import datasets + + # Specify cache_dir as argument? + ds = datasets.load_dataset("glue", "sst2") + return ConcatDataset([ds['train'], ds['validation']]) + + +class Purchase100(Dataset): + """ + Purchase100 dataset pre-processed by Shokri et al. + (https://github.com/privacytrustlab/datasets/blob/master/dataset_purchase.tgz). + We save the dataset in a .pickle version because it is much faster to load + than the original file. + """ + def __init__(self, dataset_dir: str) -> None: + import pickle + + dataset_path = os.path.join(dataset_dir, 'purchase100', 'dataset_purchase') + + # Saving the dataset in pickle format because it is quicker to load. + dataset_path_pickle = dataset_path + '.pickle' + + if not os.path.exists(dataset_path) and not os.path.exists(dataset_path_pickle): + raise ValueError("Purchase-100 dataset not found.\n" + "You may download the dataset from https://www.comp.nus.edu.sg/~reza/files/datasets.html\n" + f"and unzip it in the {dataset_dir}/purchase100 directory") + + if not os.path.exists(dataset_path_pickle): + print('Found the dataset. Saving it in a pickle file that takes less time to load...') + purchase = np.loadtxt(dataset_path, dtype=int, delimiter=',') + with open(dataset_path_pickle, 'wb') as f: + pickle.dump({'dataset': purchase}, f) + + with open(dataset_path_pickle, 'rb') as f: + dataset = pickle.load(f)['dataset'] + + self.labels = list(dataset[:, 0] - 1) + self.records = torch.FloatTensor(dataset[:, 1:]) + assert len(self.labels) == len(self.records), f'ERROR: {len(self.labels)} and {len(self.records)}' + print('Successfully loaded the Purchase-100 dataset consisting of', + f'{len(self.records)} records and {len(self.records[0])}', 'attributes.') + + def __len__(self) -> int: + return len(self.records) + + def __getitem__(self, idx: int): + return self.records[idx], self.labels[idx] + + +def load_purchase100(dataset_dir: str = ".") -> Dataset: + """Loads the Purchase-100 dataset. + """ + return Purchase100(dataset_dir) diff --git a/src/mico-competition/mico.py b/src/mico-competition/mico.py new file mode 100644 index 0000000..da9bd2a --- /dev/null +++ b/src/mico-competition/mico.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import os +import torch +import torch.nn as nn + +from collections import OrderedDict +from typing import List, Optional, Union, Type, TypeVar +from torchcsprng import create_mt19937_generator +from torch.utils.data import Dataset, ConcatDataset, random_split + +D = TypeVar("D", bound="ChallengeDataset") + +LEN_CHALLENGE = 100 + +class ChallengeDataset: + """Reconstructs the data splits associated with a model from stored seeds. + + Given a `torch.utils.Dataset`, the desired length of the training dataset `n`, + and the desired number of members/non-member challenge examples `m`, it uses + `torch.utils.data.random_split` with the stored seeds to produce: + + - `challenge` : `2m` challenge examples + - `nonmember` : `m` non-members challenge examples from `challenge` + - `member` : `m` member challenge examples, from `challenge` + - `training` : non-challenge examples to use for model training + - `evaluation`: non-challenge examples to use for model evaluation + + Use `get_training_dataset` to construct the full training dataset + (the concatenation of `member` and `training`) to train a model. + + Use `get_eval_dataset` to retrieve `evaluation`. Importantly, do not + attempt to use `nonmember` for model evaluation, as releasing the + evaluation results would leak membership information. + + The diagram below details the process, where arrows denote calls to + `torch.utils.data.random_split` and `N = len(dataset)`: + + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ dataset β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚N + seed_challenge β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚2m β”‚N - 2m + β–Ό β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ challenge β”‚ rest β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚2m β”‚N - 2m + seed_membership β”‚ seed_training β”‚ + β”Œβ”€β”€β”€β”€β”΄β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚m β”‚m β”‚n - m β”‚N - n - m + β–Ό β–Ό β–Ό β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚nonmemberβ”‚ member β”‚ training β”‚ evaluation β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + + - Models are trained on `member + training` and evaluated on `evaluation` + - Standard scenarios disclose `challenge` (equivalently, `seed_challenge`) + - DP distinguisher scenarios also disclose `training` and `evaluation` (equivalently, `seed_training`) + - To disclose ground truth, disclose `nonmember` and `member` (equivalently, `seed_membership`) + """ + def __init__(self, dataset: Dataset, len_training: int, len_challenge: int, + seed_challenge: int, seed_training: Optional[int], seed_membership: Optional[int]) -> None: + """Pseudorandomly select examples for `challenge`, `non-member`, `member`, `training`, and `evaluation` + splits from given seeds. Only the seed for `challenge` is mandatory. + + Args: + dataset (Dataset): Dataset to select examples from. + len_training (int): Length of the training dataset. + len_challenge (int): Number of challenge examples (`len_challenge` members and `len_challenge` non-members). + seed_challenge (int): Seed to select challenge examples. + seed_training (Optional[int]): Seed to select non-challenge training examples. + seed_membership (Optional[int]): Seed to split challenge examples into members/non-members. + """ + + challenge_gen = create_mt19937_generator(seed_challenge) + self.challenge, self.rest = random_split( + dataset, + [2 * len_challenge, len(dataset) - 2 * len_challenge], + generator = challenge_gen) + + if seed_training is not None: + training_gen = create_mt19937_generator(seed_training) + self.training, self.evaluation = random_split( + self.rest, + [len_training - len_challenge, len(dataset) - len_training - len_challenge], + generator = training_gen) + + if seed_membership is not None: + membership_gen = create_mt19937_generator(seed_membership) + self.nonmember, self.member = random_split( + self.challenge, + [len_challenge, len_challenge], + generator = membership_gen) + + def get_challenges(self) -> Dataset: + """Returns the challenge dataset. + + Returns: + Dataset: The challenge examples. + """ + return self.challenge + + def get_train_dataset(self) -> Dataset: + """Returns the training dataset. + + Raises: + ValueError: If the seed to select non-challenge training examples has not been set. + ValueError: If the seed to split challenges into members/non-members has not been set. + + Returns: + Dataset: The training dataset. + """ + if self.training is None: + raise ValueError("The seed to generate the training dataset has not been set.") + + if self.member is None: + raise ValueError("The seed to split challenges into members/non-members has not been set.") + + return ConcatDataset([self.member, self.training]) + + def get_eval_dataset(self) -> Dataset: + """Returns the evaluation dataset. + + Raises: + ValueError: If the seed to generate the evaluation dataset has not been set. + + Returns: + Dataset: The evaluation dataset. + """ + if self.evaluation is None: + raise ValueError("The seed to generate the evaluation dataset has not been set.") + + return self.evaluation + + def get_solutions(self) -> List: + """Returns the membership labels of the challenges. + + Raises: + ValueError: If the seed to generate the evaluation dataset has not been set. + + Returns: + List: The list of membership labels for challenges, indexed as in the + Dataset returned by `get_challenges()`. + """ + if self.member is None: + raise ValueError("The seed to split challenges into members/non-members has not been set.") + + member_indices = set(self.challenge.indices[i] for i in self.member.indices) + + labels = [1 if i in member_indices else 0 for i in self.challenge.indices] + + return labels + + @classmethod + def from_path(cls: Type[D], path: Union[str, os.PathLike], dataset: Dataset, len_training: int, len_challenge: int=LEN_CHALLENGE) -> D: + """Loads a ChallengeDataset from a directory `path`. + The directory must contain, at a minimum, the file `seed_challenge`. + + Args: + path (str): Path to the folder containing the dataset. + + Returns: + ChallengeDataset: The loaded ChallengeDataset. + """ + # Load the seeds. + if os.path.exists(os.path.join(path, "seed_challenge")): + with open(os.path.join(path, "seed_challenge"), "r") as f: + seed_challenge = int(f.read()) + else: + raise Exception(f"`seed_challenge` was not found in {path}") + + seed_training = None + if os.path.exists(os.path.join(path, "seed_training")): + with open(os.path.join(path, "seed_training"), "r") as f: + seed_training = int(f.read()) + + seed_membership = None + if os.path.exists(os.path.join(path, "seed_membership")): + with open(os.path.join(path, "seed_membership"), "r") as f: + seed_membership = int(f.read()) + + return cls( + dataset=dataset, + len_training=len_training, + len_challenge=len_challenge, + seed_challenge=seed_challenge, + seed_training=seed_training, + seed_membership=seed_membership + ) + + +X = TypeVar("X", bound="CNN") + +class CNN(nn.Module): + def __init__(self): + super().__init__() + self.cnn = nn.Sequential( + nn.Conv2d(3, 128, kernel_size=8, stride=2, padding=3), nn.Tanh(), + nn.MaxPool2d(kernel_size=3, stride=1), + nn.Conv2d(128, 256, kernel_size=3), nn.Tanh(), + nn.Conv2d(256, 256, kernel_size=3), nn.Tanh(), + nn.AvgPool2d(kernel_size=2, stride=2), + nn.Flatten(), + nn.Linear(in_features=6400, out_features=10) + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # shape of x is [B, 3, 32, 32] for CIFAR10 + logits = self.cnn(x) + return logits + + @classmethod + def load(cls: Type[X], path: Union[str, os.PathLike]) -> X: + model = cls() + state_dict = torch.load(path) + new_state_dict = OrderedDict((k.replace('_module.', ''), v) for k, v in state_dict.items()) + model.load_state_dict(new_state_dict) + model.eval() + return model + + +Y = TypeVar("Y", bound="MLP") + +class MLP(nn.Module): + """ + The fully-connected network architecture from Bao et al. (2022). + """ + def __init__(self): + super().__init__() + self.mlp = nn.Sequential( + nn.Linear(600, 128), nn.Tanh(), + nn.Linear(128, 100) + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.mlp(x) + + @classmethod + def load(cls: Type[Y], path: Union[str, os.PathLike]) -> Y: + model = cls() + state_dict = torch.load(path) + new_state_dict = OrderedDict((k.replace('_module.', ''), v) for k, v in state_dict.items()) + model.load_state_dict(new_state_dict) + model.eval() + return model + + +def load_model(task: str, path: Union[str, os.PathLike]) -> nn.Module: + if task == 'cifar10': + return CNN.load(os.path.join(path, 'model.pt')) + elif task == 'purchase100': + return MLP.load(os.path.join(path, 'model.pt')) + elif task == 'sst2': + from transformers import AutoModelForSequenceClassification + # tokenizer = AutoTokenizer.from_pretrained('roberta-base') + model = AutoModelForSequenceClassification.from_pretrained(path, num_labels=2) + model.eval() + return model + else: + raise ValueError("`task` must be one of {'cifar10', 'purchase100', 'sst2'}") diff --git a/src/mico-competition/scoring/__init__.py b/src/mico-competition/scoring/__init__.py new file mode 100644 index 0000000..da2c550 --- /dev/null +++ b/src/mico-competition/scoring/__init__.py @@ -0,0 +1,10 @@ +from .score import tpr_at_fpr, score +from .score_html import generate_roc, generate_table, generate_html + +__all__ = [ + "tpr_at_fpr", + "score", + "generate_roc", + "generate_table", + "generate_html", +] \ No newline at end of file diff --git a/src/mico-competition/scoring/metadata b/src/mico-competition/scoring/metadata new file mode 100644 index 0000000..04b3b36 --- /dev/null +++ b/src/mico-competition/scoring/metadata @@ -0,0 +1,2 @@ +command: python $program/score.py $input $output +description: Compute scores for the competition diff --git a/src/mico-competition/scoring/score.py b/src/mico-competition/scoring/score.py new file mode 100644 index 0000000..6ee0d3c --- /dev/null +++ b/src/mico-competition/scoring/score.py @@ -0,0 +1,140 @@ +"""Scoring program for the CodaLab competition platform. + +Usage: + score.py + +This program expects the following directory structure for : +- /ref/: Contains the solutions directories + (e.g., cifar10/cifar10_lo, cifar10/cifar10_hi, cifar10/cifar10_inf) for either the dev or the final phase. +- /res/: Contains the predictions directories (containing prediction.csv) for both + phases. +""" +import os +import numpy as np + +from sklearn.metrics import roc_curve, roc_auc_score +from typing import List, Dict + +FPR_THRESHOLD = 0.1 +FPR_THRESHOLD_LIST = [0.001, 0.01, 0.05, 0.1, 0.15, 0.2] + +def tpr_at_fpr(true_membership: List, predictions: List, max_fpr=FPR_THRESHOLD) -> float: + """Calculates the best True Positive Rate when the False Positive Rate is + at most `max_fpr`. + + Args: + true_membership (List): A list of values in {0,1} indicating the membership of a + challenge point. 0: "non-member", 1: "member". + predictions (List): A list of values in the range [0,1] indicating the confidence + that a challenge point is a member. The closer the value to 1, the more + confident the predictor is about the hypothesis that the challenge point is + a member. + max_fpr (float, optional): Threshold on the FPR. Defaults to 0.1. + + Returns: + float: The TPR @ `max_fpr` FPR. + """ + fpr, tpr, _ = roc_curve(true_membership, predictions) + + return max(tpr[fpr < max_fpr]) + + +def score(solutions: List, predictions: List) -> Dict: + scores = {} + for max_fpr in FPR_THRESHOLD_LIST: + scores[f"TPR_FPR_{int(1e4 * max_fpr)}"] = tpr_at_fpr(solutions, predictions, max_fpr=max_fpr) + fpr, tpr, _ = roc_curve(solutions, predictions) + scores["fpr"] = fpr + scores["tpr"] = tpr + scores["AUC"] = roc_auc_score(solutions, predictions) + scores["MIA"] = np.max(tpr - fpr) + # This is the balanced accuracy, which coincides with accuracy for balanced classes + scores["accuracy"] = np.max(1 - (fpr + (1 - tpr)) / 2) + + return scores + + +if __name__ == "__main__": + from score_html import generate_html + + # Parse arguments. + assert len(os.sys.argv) == 3, "Usage: score.py " + solutions_dir = os.path.join(os.sys.argv[1], "ref") + predictions_dir = os.path.join(os.sys.argv[1], "res") + output_dir = os.sys.argv[2] + + current_phase = None + + # Which competition? + dataset = os.listdir(solutions_dir) + assert len(dataset) == 1, f"Wrong content: {solutions_dir}: {dataset}" + dataset = dataset[0] + print(f"[*] Competition: {dataset}") + + # Update solutions and predictions directories. + solutions_dir = os.path.join(solutions_dir, dataset) + assert os.path.exists(solutions_dir), f"Couldn't find soultions directory: {solutions_dir}" + + predictions_dir = os.path.join(predictions_dir, dataset) + assert os.path.exists(predictions_dir), f"Couldn't find predictions directory: {predictions_dir}" + + scenarios = sorted(os.listdir(solutions_dir)) + assert len(scenarios) == 3, f"Found spurious directories in solutions directory: {solutions_dir}: {scenarios}" + + found_scenarios = sorted(os.listdir(predictions_dir)) + assert scenarios == found_scenarios, f"Found spurious directories in predictions directory {solutions_dir}: {found_scenarios}" + + # Compute the scores for each scenario + all_scores = {} + for scenario in scenarios: + print(f"[*] Processing {scenario}...") + + # What phase are we in? + phase = os.listdir(os.path.join(solutions_dir, scenario)) + assert len(phase) == 1, "Corrupted solutions directory" + assert phase[0] in ["dev", "final"], "Corrupted solutions directory" + current_phase = phase[0] + print(f"[**] Scoring `{current_phase}` phase...") + + # We compute the scores globally, across the models. This is somewhat equivalent to having + # one attack (threshold) for all the attacks. + # Load the predictions. + predictions = [] + solutions = [] + for model_id in os.listdir(os.path.join(solutions_dir, scenario, current_phase)): + basedir = os.path.join(scenario, current_phase, model_id) + solutions.append(np.loadtxt(os.path.join(solutions_dir, basedir, "solution.csv"), delimiter=",")) + predictions.append(np.loadtxt(os.path.join(predictions_dir, basedir, "prediction.csv"), delimiter=",")) + + solutions = np.concatenate(solutions) + predictions = np.concatenate(predictions) + + # Verify that the predictions are valid. + assert len(predictions) == len(solutions) + assert np.all(predictions >= 0), "Some predictions are < 0" + assert np.all(predictions <= 1), "Some predictions are > 1" + + scores = score(solutions, predictions) + + print(f"[*] Scores: {scores}") + all_scores[scenario] = scores + + # Store the scores. + os.makedirs(output_dir, exist_ok=True) + with open(os.path.join(output_dir, "scores.txt"), "w") as f: + for i, scenario in enumerate(scenarios): + assert scenario in all_scores, f"Score for scenario {scenario} not found. Corrupted ref/?" + for score in {"AUC", "MIA", "accuracy"}: + f.write(f"scenario{i+1}_{score}: {all_scores[scenario][score]}\n") + for max_fpr in FPR_THRESHOLD_LIST: + score = f"TPR_FPR_{int(1e4 * max_fpr)}" + f.write(f"scenario{i+1}_{score}: {all_scores[scenario][score]}\n") + + # Average TPR@0.1FPR (used for ranking) + avg = np.mean([all_scores[scenario]["TPR_FPR_1000"] for scenario in scenarios]) + f.write(f"average_TPR_FPR_1000: {avg}") + + # Detailed scoring (HTML) + html = generate_html(all_scores) + with open(os.path.join(output_dir, "scores.html"), "w") as f: + f.write(html) diff --git a/src/mico-competition/scoring/score_html.py b/src/mico-competition/scoring/score_html.py new file mode 100644 index 0000000..6f137be --- /dev/null +++ b/src/mico-competition/scoring/score_html.py @@ -0,0 +1,128 @@ +import io +import matplotlib +import pandas as pd +import matplotlib.pyplot as plt + + +def image_to_html(fig): + """Converts a matplotlib plot to SVG""" + iostring = io.StringIO() + fig.savefig(iostring, format="svg", bbox_inches=0, dpi=300) + iostring.seek(0) + + return iostring.read() + + +def generate_roc(fpr, tpr): + fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(8,3.5)) + + ax2.semilogx() + ax2.semilogy() + ax2.set_xlim(1e-5,1) + ax2.set_ylim(1e-5,1) + ax2.set_xlabel("False Positive Rate") + #ax2.set_ylabel("True Positive Rate") + ax2.plot([0, 1], [0, 1], ls=':', color='grey') + + ax1.set_xlim(0,1) + ax1.set_ylim(0,1) + ax1.set_xlabel("False Positive Rate") + ax1.set_ylabel("True Positive Rate") + ax1.plot([0,1], [0,1], ls=':', color='grey') + + ax1.plot(fpr, tpr) + ax2.plot(fpr, tpr) + + return fig + + +def generate_table(scores): + table = pd.DataFrame(scores).T + table.drop(["fpr", "tpr"], axis=1, inplace=True) + # replace = { + # "inf": "No DP", + # "hi": "High ε", + # "lo": "Low ε", + # } + # table.index = [replace[i] for i in table.index] + replace_column = { + "accuracy": "Accuracy", + "AUC": "AUC-ROC", + "MIA": "MIA", + "TPR_FPR_10": "TPR @ 0.001 FPR", + "TPR_FPR_100": "TPR @ 0.01 FPR", + "TPR_FPR_500": "TPR @ 0.05 FPR", + "TPR_FPR_1000": "TPR @ 0.1 FPR", + "TPR_FPR_1500": "TPR @ 0.15 FPR", + "TPR_FPR_2000": "TPR @ 0.2 FPR", + } + table.columns = [replace_column[c] for c in table.columns] + + return table + + +def generate_html(scores): + """Generates the HTML document as a string, containing the various detailed scores""" + matplotlib.use('Agg') + + img = {} + for scenario in scores: + fpr = scores[scenario]["fpr"] + tpr = scores[scenario]["tpr"] + fig = generate_roc(fpr, tpr) + fig.tight_layout(pad=1.0) + + img[scenario] = f"

{scenario}

{image_to_html(fig)}
" + + table = generate_table(scores) + + # Generate the HTML document. + css = ''' + body { + background-color: #ffffff; + } + h1 { + text-align: center; + } + h2 { + text-align: center; + } + div { + white-space: normal; + text-align: center; + } + table { + border-collapse: collapse; + margin: auto; + } + table > :is(thead, tbody) > tr > :is(th, td) { + padding: 5px; + } + table > thead > tr > :is(th, td) { + border-top: 2px solid; /* \toprule */ + border-bottom: 1px solid; /* \midrule */ + } + table > tbody > tr:last-child > :is(th, td) { + border-bottom: 2px solid; /* \bottomrule */ + }''' + + html = f''' + + + MICO - Detailed scores + + + + +
+ {table.to_html(border=0, float_format='{:0.4f}'.format, escape=False)} +
''' + + for scenario in scores: + html += img[scenario] + + html += "" + + return html \ No newline at end of file diff --git a/starting-kit/cifar10.ipynb b/starting-kit/cifar10.ipynb new file mode 100644 index 0000000..0e08345 --- /dev/null +++ b/starting-kit/cifar10.ipynb @@ -0,0 +1,302 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Membership Inference Competition (MICO) @ IEEE SatML 2023: CIFAR-10\n", + "\n", + "Welcome to the MICO competition!\n", + "\n", + "This notebook will walk you through the process of creating and packaging a submission to one of the challenges.\n", + "\n", + "Let's start by downloading and extracting the archive for the CIFAR-10 challenge." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from torchvision.datasets.utils import download_and_extract_archive\n", + "\n", + "url = \"https://membershipinference.blob.core.windows.net/mico/cifar10.zip\" \n", + "filename = \"cifar10.zip\"\n", + "md5 = \"c615b172eb42aac01f3a0737540944b1\"\n", + "\n", + "# WARNING: this will download and extract a 2.1GiB file, if not already present. Please save the file and avoid re-downloading it.\n", + "download_and_extract_archive(url=url, download_root=os.curdir, extract_root=None, filename=filename, md5=md5, remove_finished=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Contents\n", + "\n", + "The archive was extracted under the `cifar10` folder containing 3 sub-folders, one for each of the scenarios in the challenge:\n", + "\n", + "- `cifar10_lo` : Models trained with DP-SGD and a small privacy budget ($\\epsilon \\approx 4$) \n", + "- `cifar10_hi` : Models trained with DP-SGD and a large privacy budget ($\\epsilon \\approx 10$) \n", + "- `cifar10_inf` : Models trained without differential privacy guarantee ($\\epsilon = \\infty$)\n", + "\n", + "Each of these folders contains 3 other folders:\n", + "\n", + "- `train`: Models with metadata allowing to reconstruct their full training datasets. Use these to develop your attacks without having to train your own models.\n", + "- `dev`: Models with metadata allowing to reconstruct just the set of challenge examples. Membership predictions for these challenges will be used to evaluate submissions during the competition and update the live scoreboard in CodaLab. \n", + "- `final`: Models with metadata allowing to reconstruct just the set of challenge examples. Membership predictions for these challenges will be used to evaluate submissions when the competition closes and to determine the final ranking.\n", + "\n", + "Each model folder in `train`, `dev`, and `final` contains a `model.pt` file with the model weights (a serialized PyTorch `state_dict`). There are 100 models in `train`, and 50 models in each of `dev` and `final`.\n", + "\n", + "Models in the `train` folder come with 3 PRNG seeds used to reconstruct the set of member and non-member challenge examples, and the rest of the examples in the training dataset of the model. Additionally (and redundantly), a `solution.csv` file reveals the membership information of the challenge examples.\n", + "\n", + "Models in the `dev` and `final` folders contain just 1 PRNG seed used to reconstruct the set of challenge examples, without revealing which were included in the training dataset.\n", + "\n", + "We provide utilities to reconstruct the different data splits from provided seeds and to load models as classes inheriting from `torch.nn.Module`. If you use TensorFlow, JAX, or any other framework, you can easily convert the models to the appropriate format (e.g. using ONXX).\n", + "\n", + "Here's a summary of how the contents are structured:\n", + "\n", + "- `cifar10_lo`\n", + " - `train`\n", + " - `model_0`\n", + " - `model.pt`: Serialized model weights\n", + " - `seed_challenge`: PRNG seed used to select a list of 100 challenge examples\n", + " - `seed_training`: PRNG seed used to select the non-challenge examples in the training dataset\n", + " - `seed_membership`: PRNG seed used to split the set of challenge examples into members and non-members (100 of each)\n", + " - `solution.csv`: Membership information of the challenge examples (`1` for member, `0` for non-member)\n", + " - ...\n", + " - `dev`\n", + " - `model_100`\n", + " - `model.pt`\n", + " - `seed_challenge`\n", + " - ...\n", + " - `final`\n", + " - `model_150`\n", + " - `model.pt`\n", + " - `seed_challenge`\n", + " - ...\n", + "- `cifar10_hi`\n", + " - ...\n", + "- `cifar10_inf`\n", + " - ..." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Task\n", + "\n", + "Your task as a competitor is to produce, for each model in `dev` and `final`, a CSV file listing your confidence scores (values between 0 and 1) for the membership of the challenge examples. You must save these scores in a `prediction.csv` file and place it in the same folder as the corresponding model. A submission to the challenge is an an archive containing just these `prediction.csv` files.\n", + "\n", + "**You must submit predictions for both `dev` and `final` when you submit to CodaLab.**\n", + "\n", + "In the following, we will show you how to compute predictions from a basic membership inference attack and package them as a submission archive. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import torch\n", + "import csv\n", + "\n", + "from tqdm.notebook import tqdm\n", + "from mico_competition import ChallengeDataset, load_cifar10, load_model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "CHALLENGE = \"cifar10\"\n", + "LEN_TRAINING = 50000\n", + "LEN_CHALLENGE = 100\n", + "\n", + "scenarios = os.listdir(CHALLENGE)\n", + "phases = ['dev', 'final', 'train']\n", + "\n", + "dataset = load_cifar10(dataset_dir=\"/data\")\n", + "\n", + "criterion = torch.nn.CrossEntropyLoss(reduction='none')\n", + "\n", + "for scenario in tqdm(scenarios, desc=\"scenario\"):\n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " challenge_dataset = ChallengeDataset.from_path(path, dataset=dataset, len_training=LEN_TRAINING)\n", + " challenge_points = challenge_dataset.get_challenges()\n", + "\n", + " # This is where you plug in your membership inference attack\n", + " # As an example, here is a simple loss threshold attack\n", + "\n", + " # Loss Threshold Attack\n", + " model = load_model('cifar10', path)\n", + " challenge_dataloader = torch.utils.data.DataLoader(challenge_points, batch_size=2*LEN_CHALLENGE)\n", + " features, labels = next(iter(challenge_dataloader))\n", + " output = model(features)\n", + " predictions = -criterion(output, labels).detach().numpy()\n", + " # Normalize to unit interval\n", + " min_prediction = np.min(predictions)\n", + " max_prediction = np.max(predictions)\n", + " predictions = (predictions - min_prediction) / (max_prediction - min_prediction)\n", + "\n", + " assert np.all((0 <= predictions) & (predictions <= 1))\n", + "\n", + " with open(os.path.join(path, \"prediction.csv\"), \"w\") as f:\n", + " csv.writer(f).writerow(predictions)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scoring\n", + "\n", + "Let's see how the attack does on `train`, for which we have the ground truth. \n", + "When preparing a submission, you can use part of `train` to develop an attack and a held-out part to evaluate your attack. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from mico_competition.scoring import tpr_at_fpr, score, generate_roc, generate_table\n", + "from sklearn.metrics import roc_curve, roc_auc_score\n", + "\n", + "FPR_THRESHOLD = 0.1\n", + "\n", + "all_scores = {}\n", + "phases = ['train']\n", + "\n", + "for scenario in tqdm(scenarios, desc=\"scenario\"): \n", + " all_scores[scenario] = {} \n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " predictions = []\n", + " solutions = []\n", + "\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " predictions.append(np.loadtxt(os.path.join(path, \"prediction.csv\"), delimiter=\",\"))\n", + " solutions.append(np.loadtxt(os.path.join(path, \"solution.csv\"), delimiter=\",\"))\n", + "\n", + " predictions = np.concatenate(predictions)\n", + " solutions = np.concatenate(solutions)\n", + " \n", + " scores = score(solutions, predictions)\n", + " all_scores[scenario][phase] = scores" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's plot the ROC curve for the attack and see how the attack performed on different metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import matplotlib\n", + "\n", + "for scenario in scenarios:\n", + " fpr = all_scores[scenario]['train']['fpr']\n", + " tpr = all_scores[scenario]['train']['tpr']\n", + " fig = generate_roc(fpr, tpr)\n", + " fig.suptitle(f\"{scenario}\", x=-0.1, y=0.5)\n", + " fig.tight_layout(pad=1.0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "for scenario in scenarios:\n", + " print(scenario)\n", + " scores = all_scores[scenario]['train']\n", + " scores.pop('fpr', None)\n", + " scores.pop('tpr', None)\n", + " display(pd.DataFrame([scores]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Packaging the submission\n", + "\n", + "Now we can store the predictions into a zip file, which you can submit to CodaLab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import zipfile\n", + "\n", + "phases = ['dev', 'final']\n", + "\n", + "with zipfile.ZipFile(\"predictions_cifar10.zip\", 'w') as zipf:\n", + " for scenario in tqdm(scenarios, desc=\"scenario\"): \n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " file = os.path.join(path, \"prediction.csv\")\n", + " if os.path.exists(file):\n", + " zipf.write(file)\n", + " else:\n", + " raise FileNotFoundError(f\"`prediction.csv` not found in {path}. You need to provide predictions for all challenges\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.13 ('mico-competition')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "1c823568a0650a753a55947c22141ec594c2fc02bd68b5a71e505ecc57f17796" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/starting-kit/ddp.ipynb b/starting-kit/ddp.ipynb new file mode 100644 index 0000000..ef297a4 --- /dev/null +++ b/starting-kit/ddp.ipynb @@ -0,0 +1,337 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Membership Inference Competition (MICO) @ IEEE SatML 2023: DP Distinguisher\n", + "\n", + "Welcome to the MICO competition!\n", + "\n", + "This notebook will walk you through the process of creating and packaging a submission to one of the challenges.\n", + "\n", + "Let's start by downloading and extracting the archive for the DP Distinguisher (DDP) challenge. \n", + "The archive is 87GiB. \n", + "Downloading, verifying, and extracting it can take a while, so you may want to run the cell below only once." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from torchvision.datasets.utils import download_and_extract_archive\n", + "\n", + "url = \"https://membershipinference.blob.core.windows.net/mico/ddp.tar.gz\" \n", + "filename = \"ddp.tar.gz\"\n", + "md5 = \"1b9587c2bdf7867e43fb9da345f395eb\"\n", + "\n", + "# WARNING: this will download and extract a 87GiB file, if not already present. Please save the file and avoid re-downloading it.\n", + "download_and_extract_archive(url=url, download_root=os.curdir, extract_root=None, filename=filename, md5=md5, remove_finished=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Contents\n", + "\n", + "The archive was extracted under the `ddp` folder containing 3 sub-folders, one for each of the scenarios in the challenge:\n", + "\n", + "- `cifar10_ddp` : `CIFAR-10` models (4-layer CNN) trained with DP-SGD and a small privacy budget ($\\epsilon \\approx 4$) \n", + "- `purchase100_ddp` : `Purchase-100` models (3-layer MLP) trained with DP-SGD and a small privacy budget ($\\epsilon \\approx 4$) \n", + "- `sst2_ddp` : `SST-2` models (RoBERTa-Base) fine-tuned with DP-SGD and a small privacy budget ($\\epsilon = \\approx 4$)\n", + "\n", + "Each of these folders contains 3 other folders:\n", + "\n", + "- `train`: Models with metadata allowing to reconstruct their full training datasets. Use these to develop your attacks without having to train your own models.\n", + "- `dev`: Models with metadata allowing to reconstruct the sets of challenge examples and non-challenge training examples. Membership predictions for these challenges will be used to evaluate submissions during the competition and update the live scoreboard in CodaLab. \n", + "- `final`: Models with metadata allowing to reconstruct the sets of challenge examples and non-challenge training examples. Membership predictions for these challenges will be used to evaluate submissions when the competition closes and to determine the final ranking.\n", + "\n", + "Each model folder in `train`, `dev`, and `final` contains a `model.pt` file with the model weights (a serialized PyTorch `state_dict`) or `pytorch_model.bin`, `config.json` files with the model weights and configuration (for SST-2). There are 100 models in `train`, and 50 models in each of `dev` and `final`.\n", + "\n", + "Models in the `train` folder come with 3 PRNG seeds used to reconstruct the set of member and non-member challenge examples, and the rest of the examples in the training dataset of the model. Additionally (and redundantly), a `solution.csv` file reveals the membership information of the challenge examples.\n", + "\n", + "Models in the `dev` and `final` folders contain 2 PRNG seeds used to reconstruct the sets of challenge examples, without revealing which were included in the training dataset, and the set of non-challenge training examples.\n", + "\n", + "We provide utilities to reconstruct the different data splits from provided seeds and to load models as classes inheriting from `torch.nn.Module`. If you use TensorFlow, JAX, or any other framework, you can easily convert the models to the appropriate format (e.g. using ONXX).\n", + "\n", + "Here's a summary of how the contents are structured:\n", + "\n", + "- `cifar10_ddp`\n", + " - `train`\n", + " - `model_0`\n", + " - `model.pt`: Serialized model weights\n", + " - `seed_challenge`: PRNG seed used to select a list of 100 challenge examples\n", + " - `seed_training`: PRNG seed used to select the non-challenge examples in the training dataset\n", + " - `seed_membership`: PRNG seed used to split the set of challenge examples into members and non-members (100 of each)\n", + " - `solution.csv`: Membership information of the challenge examples (`1` for member, `0` for non-member)\n", + " - ...\n", + " - `dev`\n", + " - `model_100`\n", + " - `model.pt`\n", + " - `seed_challenge`\n", + " - `seed_training`\n", + " - ...\n", + " - `final`\n", + " - `model_150`\n", + " - `model.pt`\n", + " - `seed_challenge`\n", + " - `seed_training`\n", + " - ...\n", + "- `purchase100_ddp`\n", + " - ...\n", + "- `sst2_ddp`\n", + " - ..." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Task\n", + "\n", + "Your task as a competitor is to produce, for each model in `dev` and `final`, a CSV file listing your confidence scores (values between 0 and 1) for the membership of the challenge examples. You must save these scores in a `prediction.csv` file and place it in the same folder as the corresponding model. A submission to the challenge is an an archive containing just these `prediction.csv` files.\n", + "\n", + "**You must submit predictions for both `dev` and `final` when you submit to CodaLab.**\n", + "\n", + "In the following, we will show you how to compute predictions from a basic membership inference attack and package them as a submission archive. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import torch\n", + "import csv\n", + "\n", + "from tqdm.notebook import tqdm\n", + "from mico_competition import ChallengeDataset, load_sst2, load_cifar10, load_purchase100, load_model\n", + "from transformers import AutoTokenizer\n", + "\n", + "assert torch.cuda.is_available(), \"CUDA is not available; the below would only work with CUDA\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "CHALLENGE = \"ddp\"\n", + "LEN_TRAINING = { 'cifar10_ddp': 50000, 'purchase100_ddp': 150000, 'sst2_ddp': 67349 }\n", + "LEN_CHALLENGE = 100\n", + "\n", + "scenarios = os.listdir(CHALLENGE)\n", + "phases = ['dev', 'final','train']\n", + "\n", + "criterion = torch.nn.CrossEntropyLoss(reduction='none')\n", + "tokenizer = AutoTokenizer.from_pretrained('roberta-base')\n", + "\n", + "for scenario in tqdm(scenarios, desc=\"scenario\"):\n", + " if scenario == \"cifar10_ddp\":\n", + " dataset = load_cifar10(dataset_dir=\"/data\")\n", + " datasetName = \"cifar10\"\n", + " batch_size = 2*LEN_CHALLENGE\n", + " elif scenario == \"purchase100_ddp\":\n", + " dataset = load_purchase100(dataset_dir=\"/data\")\n", + " datasetName = \"purchase100\"\n", + " batch_size = 2*LEN_CHALLENGE\n", + " else:\n", + " assert scenario == \"sst2_ddp\"\n", + " dataset = load_sst2()\n", + " datasetName = \"sst2\"\n", + " batch_size = 10\n", + "\n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " challenge_dataset = ChallengeDataset.from_path(path, dataset=dataset, len_training=LEN_TRAINING[scenario])\n", + " challenge_points = challenge_dataset.get_challenges()\n", + "\n", + " # This is where you plug in your membership inference attack\n", + " # As an example, here is a simple loss threshold attack\n", + "\n", + " # Loss Threshold Attack\n", + " model = load_model(datasetName, path)\n", + " challenge_dataloader = torch.utils.data.DataLoader(challenge_points, batch_size=batch_size)\n", + "\n", + " model = model.to(torch.device('cuda'))\n", + " predictions = []\n", + " for batch in challenge_dataloader:\n", + " if scenario == \"sst2_ddp\":\n", + " labels = batch['label'].to(torch.device('cuda'))\n", + " tokenizedSequences = tokenizer(batch['sentence'], return_tensors=\"pt\", padding=\"max_length\", max_length=67)\n", + " tokenizedSequences = tokenizedSequences.to(torch.device('cuda'))\n", + " output = model(**tokenizedSequences).logits\n", + " else:\n", + " features, labels = batch\n", + " features = features.to(torch.device('cuda'))\n", + " labels = labels.to(torch.device('cuda'))\n", + " output = model(features)\n", + "\n", + " batch_predictions = -criterion(output, labels).detach().cpu().numpy()\n", + " predictions.extend(batch_predictions)\n", + "\n", + " # Normalize to unit interval\n", + " min_prediction = np.min(predictions)\n", + " max_prediction = np.max(predictions)\n", + " predictions = (predictions - min_prediction) / (max_prediction - min_prediction)\n", + "\n", + " assert np.all((0 <= predictions) & (predictions <= 1))\n", + "\n", + " with open(os.path.join(path, \"prediction.csv\"), \"w\") as f:\n", + " csv.writer(f).writerow(predictions)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scoring\n", + "\n", + "Let's see how the attack does on `train`, for which we have the ground truth. \n", + "When preparing a submission, you can use part of `train` to develop an attack and a held-out part to evaluate your attack. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from mico_competition.scoring import tpr_at_fpr, score, generate_roc, generate_table\n", + "from sklearn.metrics import roc_curve, roc_auc_score\n", + "\n", + "FPR_THRESHOLD = 0.1\n", + "\n", + "all_scores = {}\n", + "phases = ['train']\n", + "\n", + "for scenario in tqdm(scenarios, desc=\"scenario\"): \n", + " all_scores[scenario] = {} \n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " predictions = []\n", + " solutions = []\n", + "\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " predictions.append(np.loadtxt(os.path.join(path, \"prediction.csv\"), delimiter=\",\"))\n", + " solutions.append(np.loadtxt(os.path.join(path, \"solution.csv\"), delimiter=\",\"))\n", + "\n", + " predictions = np.concatenate(predictions)\n", + " solutions = np.concatenate(solutions)\n", + " \n", + " scores = score(solutions, predictions)\n", + " all_scores[scenario][phase] = scores" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's plot the ROC curve for the attack and see how the attack performed on different metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import matplotlib\n", + "\n", + "for scenario in scenarios:\n", + " fpr = all_scores[scenario]['train']['fpr']\n", + " tpr = all_scores[scenario]['train']['tpr']\n", + " fig = generate_roc(fpr, tpr)\n", + " fig.suptitle(f\"{scenario}\", x=-0.1, y=0.5)\n", + " fig.tight_layout(pad=1.0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "for scenario in scenarios:\n", + " print(scenario)\n", + " scores = all_scores[scenario]['train']\n", + " scores.pop('fpr', None)\n", + " scores.pop('tpr', None)\n", + " display(pd.DataFrame([scores]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Packaging the submission\n", + "\n", + "Now we can store the predictions into a zip file, which you can submit to CodaLab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import zipfile\n", + "\n", + "phases = ['dev', 'final']\n", + "\n", + "with zipfile.ZipFile(\"predictions_ddp.zip\", 'w') as zipf:\n", + " for scenario in tqdm(scenarios, desc=\"scenario\"): \n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " file = os.path.join(path, \"prediction.csv\")\n", + " if os.path.exists(file):\n", + " zipf.write(file)\n", + " else:\n", + " raise FileNotFoundError(f\"`prediction.csv` not found in {path}. You need to provide predictions for all challenges\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.13 ('mico-competition')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "1c823568a0650a753a55947c22141ec594c2fc02bd68b5a71e505ecc57f17796" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/starting-kit/purchase100.ipynb b/starting-kit/purchase100.ipynb new file mode 100644 index 0000000..686e6ef --- /dev/null +++ b/starting-kit/purchase100.ipynb @@ -0,0 +1,301 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Membership Inference Competition (MICO) @ IEEE SatML 2023: Purchase-100\n", + "\n", + "Welcome to the MICO competition!\n", + "\n", + "This notebook will walk you through the process of creating and packaging a submission to one of the challenges.\n", + "\n", + "Let's start by downloading and extracting the archive for the Purchase-100 challenge." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from torchvision.datasets.utils import download_and_extract_archive\n", + "\n", + "url = \"https://membershipinference.blob.core.windows.net/mico/purchase100.zip\"\n", + "filename = \"purchase100.zip\"\n", + "md5 = \"67eba1f88d112932fe722fef85fb95fd\"\n", + "\n", + "download_and_extract_archive(url=url, download_root=os.curdir, extract_root=None, filename=filename, md5=md5, remove_finished=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Contents\n", + "\n", + "The archive was extracted under the `purchase100` folder containing 3 sub-folders, one for each of the scenarios in the challenge:\n", + "\n", + "- `purchase100_lo` : Models trained with DP-SGD and a small privacy budget ($\\epsilon \\approx 4$) \n", + "- `purchase100_hi` : Models trained with DP-SGD and a large privacy budget ($\\epsilon \\approx 10$) \n", + "- `purchase100_inf` : Models trained without differential privacy guarantee ($\\epsilon = \\infty$)\n", + "\n", + "Each of these folders contains 3 other folders:\n", + "\n", + "- `train`: Models with metadata allowing to reconstruct their full training datasets. Use these to develop your attacks without having to train your own models.\n", + "- `dev`: Models with metadata allowing to reconstruct just the set of challenge examples. Membership predictions for these challenges will be used to evaluate submissions during the competition and update the live scoreboard in CodaLab. \n", + "- `final`: Models with metadata allowing to reconstruct just the set of challenge examples. Membership predictions for these challenges will be used to evaluate submissions when the competition closes and to determine the final ranking.\n", + "\n", + "Each model folder in `train`, `dev`, and `final` contains a `model.pt` file with the model weights (a serialized PyTorch `state_dict`). There are 100 models in `train`, and 50 models in each of `dev` and `final`.\n", + "\n", + "Models in the `train` folder come with 3 PRNG seeds used to reconstruct the set of member and non-member challenge examples, and the rest of the examples in the training dataset of the model. Additionally (and redundantly), a `solution.csv` file reveals the membership information of the challenge examples.\n", + "\n", + "Models in the `dev` and `final` folders contain just 1 PRNG seed used to reconstruct the set of challenge examples, without revealing which were included in the training dataset.\n", + "\n", + "We provide utilities to reconstruct the different data splits from provided seeds and to load models as classes inheriting from `torch.nn.Module`. If you use TensorFlow, JAX, or any other framework, you can easily convert the models to the appropriate format (e.g. using ONXX).\n", + "\n", + "Here's a summary of how the contents are structured:\n", + "\n", + "- `purchase100_lo`\n", + " - `train`\n", + " - `model_0`\n", + " - `model.pt`: Serialized model weights\n", + " - `seed_challenge`: PRNG seed used to select a list of 100 challenge examples\n", + " - `seed_training`: PRNG seed used to select the non-challenge examples in the training dataset\n", + " - `seed_membership`: PRNG seed used to split the set of challenge examples into members and non-members (100 of each)\n", + " - `solution.csv`: Membership information of the challenge examples (`1` for member, `0` for non-member)\n", + " - ...\n", + " - `dev`\n", + " - `model_100`\n", + " - `model.pt`\n", + " - `seed_challenge`\n", + " - ...\n", + " - `final`\n", + " - `model_150`\n", + " - `model.pt`\n", + " - `seed_challenge`\n", + " - ...\n", + "- `purchase100_hi`\n", + " - ...\n", + "- `purchase100_inf`\n", + " - ..." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Task\n", + "\n", + "Your task as a competitor is to produce, for each model in `dev` and `final`, a CSV file listing your confidence scores (values between 0 and 1) for the membership of the challenge examples. You must save these scores in a `prediction.csv` file and place it in the same folder as the corresponding model. A submission to the challenge is an an archive containing just these `prediction.csv` files.\n", + "\n", + "**You must submit predictions for both `dev` and `final` when you submit to CodaLab.**\n", + "\n", + "In the following, we will show you how to compute predictions from a basic membership inference attack and package them as a submission archive. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import torch\n", + "import csv\n", + "\n", + "from tqdm.notebook import tqdm\n", + "from mico_competition import ChallengeDataset, load_purchase100, load_model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "CHALLENGE = \"purchase100\"\n", + "LEN_TRAINING = 150000\n", + "LEN_CHALLENGE = 100\n", + "\n", + "scenarios = os.listdir(CHALLENGE)\n", + "phases = ['dev', 'final', 'train']\n", + "\n", + "dataset = load_purchase100(dataset_dir=\"/data\")\n", + "\n", + "criterion = torch.nn.CrossEntropyLoss(reduction='none')\n", + "\n", + "for scenario in tqdm(scenarios, desc=\"scenario\"):\n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " challenge_dataset = ChallengeDataset.from_path(path, dataset=dataset, len_training=LEN_TRAINING)\n", + " challenge_points = challenge_dataset.get_challenges()\n", + "\n", + " # This is where you plug in your membership inference attack\n", + " # As an example, here is a simple loss threshold attack\n", + "\n", + " # Loss Threshold Attack\n", + " model = load_model('purchase100', path)\n", + " challenge_dataloader = torch.utils.data.DataLoader(challenge_points, batch_size=2*LEN_CHALLENGE)\n", + " features, labels = next(iter(challenge_dataloader))\n", + " output = model(features)\n", + " predictions = -criterion(output, labels).detach().numpy()\n", + " # Normalize to unit interval\n", + " min_prediction = np.min(predictions)\n", + " max_prediction = np.max(predictions)\n", + " predictions = (predictions - min_prediction) / (max_prediction - min_prediction)\n", + "\n", + " assert np.all((0 <= predictions) & (predictions <= 1))\n", + "\n", + " with open(os.path.join(path, \"prediction.csv\"), \"w\") as f:\n", + " csv.writer(f).writerow(predictions)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scoring\n", + "\n", + "Let's see how the attack does on `train`, for which we have the ground truth. \n", + "When preparing a submission, you can use part of `train` to develop an attack and a held-out part to evaluate your attack. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from mico_competition.scoring import tpr_at_fpr, score, generate_roc, generate_table\n", + "from sklearn.metrics import roc_curve, roc_auc_score\n", + "\n", + "FPR_THRESHOLD = 0.1\n", + "\n", + "all_scores = {}\n", + "phases = ['train']\n", + "\n", + "for scenario in tqdm(scenarios, desc=\"scenario\"): \n", + " all_scores[scenario] = {} \n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " predictions = []\n", + " solutions = []\n", + "\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " predictions.append(np.loadtxt(os.path.join(path, \"prediction.csv\"), delimiter=\",\"))\n", + " solutions.append(np.loadtxt(os.path.join(path, \"solution.csv\"), delimiter=\",\"))\n", + "\n", + " predictions = np.concatenate(predictions)\n", + " solutions = np.concatenate(solutions)\n", + " \n", + " scores = score(solutions, predictions)\n", + " all_scores[scenario][phase] = scores" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's plot the ROC curve for the attack and see how the attack performed on different metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import matplotlib\n", + "\n", + "for scenario in scenarios:\n", + " fpr = all_scores[scenario]['train']['fpr']\n", + " tpr = all_scores[scenario]['train']['tpr']\n", + " fig = generate_roc(fpr, tpr)\n", + " fig.suptitle(f\"{scenario}\", x=-0.1, y=0.5)\n", + " fig.tight_layout(pad=1.0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "for scenario in scenarios:\n", + " print(scenario)\n", + " scores = all_scores[scenario]['train']\n", + " scores.pop('fpr', None)\n", + " scores.pop('tpr', None)\n", + " display(pd.DataFrame([scores]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Packaging the submission\n", + "\n", + "Now we can store the predictions into a zip file, which you can submit to CodaLab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import zipfile\n", + "\n", + "phases = ['dev', 'final']\n", + "\n", + "with zipfile.ZipFile(\"predictions_purchase100.zip\", 'w') as zipf:\n", + " for scenario in tqdm(scenarios, desc=\"scenario\"): \n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " file = os.path.join(path, \"prediction.csv\")\n", + " if os.path.exists(file):\n", + " zipf.write(file)\n", + " else:\n", + " raise FileNotFoundError(f\"`prediction.csv` not found in {path}. You need to provide predictions for all challenges\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.13 ('mico-competition')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "1c823568a0650a753a55947c22141ec594c2fc02bd68b5a71e505ecc57f17796" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/starting-kit/requirements-starting-kit.txt b/starting-kit/requirements-starting-kit.txt new file mode 100644 index 0000000..2f28939 --- /dev/null +++ b/starting-kit/requirements-starting-kit.txt @@ -0,0 +1,3 @@ +ipykernel +ipywidgets +tqdm diff --git a/starting-kit/sst2.ipynb b/starting-kit/sst2.ipynb new file mode 100644 index 0000000..a69d16e --- /dev/null +++ b/starting-kit/sst2.ipynb @@ -0,0 +1,333 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Membership Inference Competition (MICO) @ IEEE SatML 2023: SST-2\n", + "\n", + "Welcome to the MICO competition!\n", + "\n", + "This notebook will walk you through the process of creating and packaging a submission to one of the challenges.\n", + "\n", + "Let's start by downloading and extracting the archives for the SST-2 challenge.\n", + "We split the challenge data into three archives, one per scenario (~80GiB each).\n", + "Downloading, verifying, and extracting them can take a while, so you may want to run the cell below only once." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from torchvision.datasets.utils import download_and_extract_archive\n", + "\n", + "files = [\n", + " {\n", + " 'filename' : 'sst2_lo.tar.gz',\n", + " 'url': 'https://membershipinference.blob.core.windows.net/mico/sst2_lo.tar.gz',\n", + " 'md5': '205414dd0217065dcebe2d8adc1794a3'\n", + " },\n", + " {\n", + " 'filename' : 'sst2_hi.tar.gz',\n", + " 'url': 'https://membershipinference.blob.core.windows.net/mico/sst2_hi.tar.gz',\n", + " 'md5': 'd285958529fcad486994347478feccd2'\n", + " },\n", + " {\n", + " 'filename' : 'sst2_inf.tar.gz',\n", + " 'url': 'https://membershipinference.blob.core.windows.net/mico/sst2_inf.tar.gz',\n", + " 'md5': '7dca44b191a0055edbde5c486a8fc671'\n", + " }\n", + "]\n", + "\n", + "for f in files:\n", + " url, filename, md5 = f['url'], f['filename'], f['md5']\n", + " print(f\"Downloading and extracting {filename}...\")\n", + " # WARNING: this will download and extract three ~80GiB files, if not already present. Please save the files and avoid re-downloading them.\n", + " download_and_extract_archive(url=url, download_root=os.curdir, extract_root=None, filename=filename, md5=md5, remove_finished=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Contents\n", + "\n", + "The archives were extracted under the `sst2` folder containing 3 sub-folders, one for each of the scenarios in the challenge:\n", + "\n", + "- `sst2_lo` : Models trained with DP-SGD and a small privacy budget ($\\epsilon \\approx 4$) \n", + "- `sst2_hi` : Models trained with DP-SGD and a large privacy budget ($\\epsilon \\approx 10$) \n", + "- `sst2_inf` : Models trained without differential privacy guarantee ($\\epsilon = \\infty$)\n", + "\n", + "Each of these folders contains 3 other folders:\n", + "\n", + "- `train`: Models with metadata allowing to reconstruct their full training datasets. Use these to develop your attacks without having to train your own models.\n", + "- `dev`: Models with metadata allowing to reconstruct just the set of challenge examples. Membership predictions for these challenges will be used to evaluate submissions during the competition and update the live scoreboard in CodaLab. \n", + "- `final`: Models with metadata allowing to reconstruct just the set of challenge examples. Membership predictions for these challenges will be used to evaluate submissions when the competition closes and to determine the final ranking.\n", + "\n", + "Each model folder in `train`, `dev`, and `final` contains `pytorch_model.bin`, `config.json` files with the model weights and configuration. There are 100 models in `train`, and 50 models in each of `dev` and `final`.\n", + "\n", + "Models in the `train` folder come with 3 PRNG seeds used to reconstruct the set of member and non-member challenge examples, and the rest of the examples in the training dataset of the model. Additionally (and redundantly), a `solution.csv` file reveals the membership information of the challenge examples.\n", + "\n", + "Models in the `dev` and `final` folders contain just 1 PRNG seed used to reconstruct the set of challenge examples, without revealing which were included in the training dataset.\n", + "\n", + "We provide utilities to reconstruct the different data splits from provided seeds and to load models as classes inheriting from `torch.nn.Module`. If you use TensorFlow, JAX, or any other framework, you can easily convert the models to the appropriate format (e.g. using ONXX).\n", + "\n", + "Here's a summary of how the contents are structured:\n", + "\n", + "- `sst2_lo`\n", + " - `train`\n", + " - `model_0`\n", + " - `pytorch_model.bin`, `config.json`: Serialized model weights and configuration\n", + " - `seed_challenge`: PRNG seed used to select a list of 100 challenge examples\n", + " - `seed_training`: PRNG seed used to select the non-challenge examples in the training dataset\n", + " - `seed_membership`: PRNG seed used to split the set of challenge examples into members and non-members (100 of each)\n", + " - `solution.csv`: Membership information of the challenge examples (`1` for member, `0` for non-member)\n", + " - ...\n", + " - `dev`\n", + " - `model_100`\n", + " - `pytorch_model.bin`, `config.json`\n", + " - `seed_challenge`\n", + " - ...\n", + " - `final`\n", + " - `model_150`\n", + " - `pytorch_model.bin`, `config.json`\n", + " - `seed_challenge`\n", + " - ...\n", + "- `sst2_hi`\n", + " - ...\n", + "- `sst2_inf`\n", + " - ..." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Task\n", + "\n", + "Your task as a competitor is to produce, for each model in `dev` and `final`, a CSV file listing your confidence scores (values between 0 and 1) for the membership of the challenge examples. You must save these scores in a `prediction.csv` file and place it in the same folder as the corresponding model. A submission to the challenge is an an archive containing just these `prediction.csv` files.\n", + "\n", + "**You must submit predictions for both `dev` and `final` when you submit to CodaLab.**\n", + "\n", + "In the following, we will show you how to compute predictions from a basic membership inference attack and package them as a submission archive. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import torch\n", + "import csv\n", + "\n", + "from tqdm.notebook import tqdm\n", + "from mico_competition import ChallengeDataset, load_sst2, load_model\n", + "from transformers import AutoTokenizer\n", + "\n", + "assert torch.cuda.is_available(), \"CUDA is not available; the below would only work with CUDA\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "CHALLENGE = \"sst2\"\n", + "LEN_TRAINING = 67349\n", + "LEN_CHALLENGE = 100\n", + "\n", + "scenarios = os.listdir(CHALLENGE)\n", + "phases = ['dev', 'final', 'train']\n", + "\n", + "dataset = load_sst2()\n", + "\n", + "criterion = torch.nn.CrossEntropyLoss(reduction='none')\n", + "tokenizer = AutoTokenizer.from_pretrained('roberta-base')\n", + "\n", + "for scenario in tqdm(scenarios, desc=\"scenario\"):\n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " challenge_dataset = ChallengeDataset.from_path(path, dataset=dataset, len_training=LEN_TRAINING)\n", + " challenge_points = challenge_dataset.get_challenges()\n", + "\n", + " # This is where you plug in your membership inference attack\n", + " # As an example, here is a simple loss threshold attack\n", + "\n", + " # Loss Threshold Attack\n", + " model = load_model('sst2', path)\n", + " challenge_dataloader = torch.utils.data.DataLoader(challenge_points, batch_size=10)\n", + " \n", + " model.to(torch.device('cuda'))\n", + " predictions = []\n", + " for batch in challenge_dataloader:\n", + " labels = batch['label'].to(torch.device('cuda'))\n", + " tokenizedSequences = tokenizer(batch['sentence'], return_tensors=\"pt\", padding=\"max_length\", max_length=67)\n", + " tokenizedSequences = tokenizedSequences.to(torch.device('cuda'))\n", + " output = model(**tokenizedSequences)\n", + " batch_predictions = -criterion(output.logits, labels).detach().cpu().numpy()\n", + " predictions.extend(batch_predictions)\n", + " \n", + " # Normalize to unit interval\n", + " min_prediction = np.min(predictions)\n", + " max_prediction = np.max(predictions)\n", + " predictions = (predictions - min_prediction) / (max_prediction - min_prediction)\n", + "\n", + " assert np.all((0 <= predictions) & (predictions <= 1))\n", + "\n", + " with open(os.path.join(path, \"prediction.csv\"), \"w\") as f:\n", + " csv.writer(f).writerow(predictions)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scoring\n", + "\n", + "Let's see how the attack does on `train`, for which we have the ground truth. \n", + "When preparing a submission, you can use part of `train` to develop an attack and a held-out part to evaluate your attack. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from mico_competition.scoring import tpr_at_fpr, score, generate_roc, generate_table\n", + "from sklearn.metrics import roc_curve, roc_auc_score\n", + "\n", + "FPR_THRESHOLD = 0.1\n", + "\n", + "all_scores = {}\n", + "phases = ['train']\n", + "\n", + "for scenario in tqdm(scenarios, desc=\"scenario\"): \n", + " all_scores[scenario] = {} \n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " predictions = []\n", + " solutions = []\n", + "\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " predictions.append(np.loadtxt(os.path.join(path, \"prediction.csv\"), delimiter=\",\"))\n", + " solutions.append(np.loadtxt(os.path.join(path, \"solution.csv\"), delimiter=\",\"))\n", + "\n", + " predictions = np.concatenate(predictions)\n", + " solutions = np.concatenate(solutions)\n", + " \n", + " scores = score(solutions, predictions)\n", + " all_scores[scenario][phase] = scores" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's plot the ROC curve for the attack and see how the attack performed on different metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import matplotlib\n", + "\n", + "for scenario in scenarios:\n", + " fpr = all_scores[scenario]['train']['fpr']\n", + " tpr = all_scores[scenario]['train']['tpr']\n", + " fig = generate_roc(fpr, tpr)\n", + " fig.suptitle(f\"{scenario}\", x=-0.1, y=0.5)\n", + " fig.tight_layout(pad=1.0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "for scenario in scenarios:\n", + " print(scenario)\n", + " scores = all_scores[scenario]['train']\n", + " scores.pop('fpr', None)\n", + " scores.pop('tpr', None)\n", + " display(pd.DataFrame([scores]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Packaging the submission\n", + "\n", + "Now we can store the predictions into a zip file, which you can submit to CodaLab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import zipfile\n", + "\n", + "phases = ['dev', 'final']\n", + "\n", + "with zipfile.ZipFile(\"predictions_sst2.zip\", 'w') as zipf:\n", + " for scenario in tqdm(scenarios, desc=\"scenario\"): \n", + " for phase in tqdm(phases, desc=\"phase\"):\n", + " root = os.path.join(CHALLENGE, scenario, phase)\n", + " for model_folder in tqdm(sorted(os.listdir(root), key=lambda d: int(d.split('_')[1])), desc=\"model\"):\n", + " path = os.path.join(root, model_folder)\n", + " file = os.path.join(path, \"prediction.csv\")\n", + " if os.path.exists(file):\n", + " zipf.write(file)\n", + " else:\n", + " raise FileNotFoundError(f\"`prediction.csv` not found in {path}. You need to provide predictions for all challenges\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.13 ('mico-competition')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "1c823568a0650a753a55947c22141ec594c2fc02bd68b5a71e505ecc57f17796" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/training/accountant.py b/training/accountant.py new file mode 100644 index 0000000..425f9bd --- /dev/null +++ b/training/accountant.py @@ -0,0 +1,54 @@ +from typing import List, Optional + +from prv_accountant.dpsgd import DPSGDAccountant + +from opacus.accountants.accountant import IAccountant + +class PRVAccountant(IAccountant): + def __init__(self, noise_multiplier, sample_rate, max_steps, eps_error = 0.1, delta_error = 1e-9): + super().__init__() + self.noise_multiplier = noise_multiplier + self.sample_rate = sample_rate + self.max_steps = max_steps + self.eps_error = eps_error + self.delta_error = delta_error + self.accountant = DPSGDAccountant( + noise_multiplier=noise_multiplier, + sampling_probability=sample_rate, + max_steps=max_steps, + eps_error=eps_error, + delta_error=delta_error) + + def step(self, *, noise_multiplier: float, sample_rate: float): + if not (noise_multiplier == self.noise_multiplier and sample_rate == self.sample_rate): + raise ValueError("Noise multplier and sample rate must be constant for DPSGDAccountant") + + if len(self.history) > 0: + _, _, num_steps = self.history.pop() + self.history.append((noise_multiplier, sample_rate, num_steps + 1)) + else: + self.history.append((noise_multiplier, sample_rate, 1)) + + def get_epsilon(self, delta: float, *, eps_error: float = 0.1, delta_error: float = 1e-9) -> float: + """ + Compute upper bound for epsilon + :param float delta: Target delta + :return: Returns upper bound for $\varepsilon$ + :rtype: float + """ + if not (eps_error == self.eps_error and delta_error == self.delta_error): + raise ValueError("Attempted to override eps_error and delta_error which are fixed at initialization") + + if len(self.history) == 0: + return 0 + + _, _, num_steps = self.history[-1] + _, _, eps = self.accountant.compute_epsilon(delta=delta, num_steps=num_steps) + return eps + + @classmethod + def mechanism(cls) -> str: + return "PRV" + + def __len__(self): + return len(self.history) \ No newline at end of file diff --git a/training/requirements-cifar10.txt b/training/requirements-cifar10.txt new file mode 100644 index 0000000..761cc81 --- /dev/null +++ b/training/requirements-cifar10.txt @@ -0,0 +1,3 @@ +opacus==1.1.3 +prv-accountant==0.2.0 +gitpython diff --git a/training/requirements-purchase100.txt b/training/requirements-purchase100.txt new file mode 100644 index 0000000..761cc81 --- /dev/null +++ b/training/requirements-purchase100.txt @@ -0,0 +1,3 @@ +opacus==1.1.3 +prv-accountant==0.2.0 +gitpython diff --git a/training/requirements-sst2.txt b/training/requirements-sst2.txt new file mode 100644 index 0000000..cd68fc0 --- /dev/null +++ b/training/requirements-sst2.txt @@ -0,0 +1,5 @@ +opacus==0.15.0 +prv-accountant==0.2.0 +dp-transformers==0.1.0 +gitpython +pyyaml diff --git a/training/train_cifar10.py b/training/train_cifar10.py new file mode 100644 index 0000000..ce57ad5 --- /dev/null +++ b/training/train_cifar10.py @@ -0,0 +1,417 @@ +import os +import argparse +import warnings +import git +import csv +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim + +from torchcsprng import create_mt19937_generator, create_random_device_generator + +from torch.utils.data import DataLoader + +from opacus import PrivacyEngine +from opacus.validators import ModuleValidator +from opacus.utils.batch_memory_manager import BatchMemoryManager + +from prv_accountant.dpsgd import find_noise_multiplier + +from accountant import PRVAccountant + +from mico_competition import ChallengeDataset, CNN, load_cifar10 + +from tqdm import tqdm, trange + +from datetime import datetime + +from typing import Callable, Optional + + +def accuracy(preds: torch.Tensor, labels: torch.Tensor) -> float: + return (preds == labels).mean() + + +def train(args: argparse.Namespace, + model: nn.Module, + device: torch.device, + train_loader: DataLoader, + criterion, + optimizer: optim.Optimizer, + epoch: int, + compute_epsilon: Optional[Callable[[int], float]] = None): + model.train() + + losses = [] + top1_acc = [] + + with BatchMemoryManager( + data_loader=train_loader, + max_physical_batch_size=args.max_physical_batch_size, + optimizer=optimizer + ) as memory_safe_data_loader: + + if args.disable_dp: + data_loader = train_loader + else: + data_loader = memory_safe_data_loader + + # BatchSplittingSampler.__len__() approximates (badly) the length in physical batches + # See https://github.com/pytorch/opacus/issues/516 + # We instead heuristically keep track of logical batches processed + pbar = tqdm(data_loader, desc="Batch", unit="batch", position=1, leave=True, total=len(train_loader), disable=None) + logical_batch_len = 0 + for i, (inputs, target) in enumerate(data_loader): + inputs = inputs.to(device) + target = target.to(device) + + logical_batch_len += len(target) + if logical_batch_len >= args.batch_size: + pbar.update(1) + logical_batch_len = logical_batch_len % args.max_physical_batch_size + + optimizer.zero_grad() + output = model(inputs) + loss = criterion(output, target) + + preds = np.argmax(output.detach().cpu().numpy(), axis=1) + labels = target.detach().cpu().numpy() + acc = accuracy(preds, labels) + + losses.append(loss.item()) + top1_acc.append(acc) + + loss.backward() + optimizer.step() + + if (pbar.n + 1) % args.logging_steps == 0 or (pbar.n + 1) == pbar.total: + if not args.disable_dp: + epsilon = compute_epsilon(delta=args.target_delta) + pbar.set_postfix( + epoch=f"{epoch:02}", + train_loss=f"{np.mean(losses):.3f}", + accuracy=f"{np.mean(top1_acc) * 100:.3f}", + dp=f"(Ξ΅={epsilon:.2f}, Ξ΄={args.target_delta})" + ) + else: + pbar.set_postfix( + epoch=f"{epoch:02}", + train_loss=f"{np.mean(losses):.3f}", + accuracy=f"{np.mean(top1_acc) * 100:.3f}", + dp="(Ξ΅ = ∞, Ξ΄ = 0)" + ) + + pbar.update(pbar.total - pbar.n) + + +def test(args: argparse.Namespace, + model: nn.Module, + device: torch.device, + test_loader: DataLoader, + criterion): + model.eval() + + losses = [] + top1_acc = [] + + with torch.no_grad(): + for inputs, target in tqdm(test_loader, desc="Test ", unit="batch", disable=None): + inputs = inputs.to(device) + target = target.to(device) + + output = model(inputs) + loss = criterion(output, target) + + preds = np.argmax(output.detach().cpu().numpy(), axis=1) + labels = target.detach().cpu().numpy() + + acc = accuracy(preds, labels) + + losses.append(loss.item()) + top1_acc.append(acc) + + top1_avg = np.mean(top1_acc) + loss_avg = np.mean(losses) + + print( + f"Test Loss : {loss_avg:.6f}\n" + f"Test Accuracy: {top1_avg * 100:.6f}" + ) + + return np.mean(top1_acc) + + +def main(args: argparse.Namespace): + noise_generator = None + if not args.secure_mode and args.train_seed is not None: + # Following the advice on https://pytorch.org/docs/1.8.1/notes/randomness.html + + if torch.cuda.is_available(): + os.environ['CUBLAS_WORKSPACE_CONFIG'] = ":4096:8" + torch.use_deterministic_algorithms(True) + torch.cuda.manual_seed(args.train_seed) + torch.cuda.manual_seed_all(args.train_seed) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + + import random + random.seed(args.train_seed) + os.environ['PYTHONHASHSEED'] = str(args.train_seed) + + # Required to get deterministic batches because Opacus uses secure_rng as a generator for + # train_loader when poisson_sampling = True even though secure_mode = False, which sets secure_rng = None + # https://github.com/pytorch/opacus/blob/5e632cdb8d497aade29e5555ad79921c239c78f7/opacus/privacy_engine.py#L206 + torch.manual_seed(args.train_seed) + np.random.seed(args.train_seed) + noise_generator = create_mt19937_generator(args.train_seed) + + if (args.seed_challenge is None or args.seed_training is None or args.seed_membership is None): + + if args.split_seed is None: + seed_generator = create_random_device_generator() + else: + seed_generator = create_mt19937_generator(args.split_seed) + + args.seed_challenge, args.seed_training, args.seed_membership = torch.empty( + 3, dtype=torch.int64).random_(0, to=None, generator=seed_generator) + + print("Using generated seeds\n" + f" seed_challenge = {args.seed_challenge}\n" + f" seed_training = {args.seed_training}\n" + f" seed_membership = {args.seed_membership}\n") + + else: + print("Using specified seeds") + + full_dataset = load_cifar10(dataset_dir=args.dataset_dir, download=False) + + challenge_dataset = ChallengeDataset( + full_dataset, + len_challenge=args.len_challenge, + len_training=args.len_training, + seed_challenge=args.seed_challenge, + seed_training=args.seed_training, + seed_membership=args.seed_membership) + + train_dataset = challenge_dataset.get_train_dataset() + test_dataset = challenge_dataset.get_eval_dataset() + + train_loader = DataLoader( + train_dataset, + batch_size=args.batch_size, + num_workers=args.dataloader_num_workers, + pin_memory=True, + ) + + test_loader = DataLoader( + test_dataset, + batch_size=args.max_physical_batch_size, + num_workers=args.dataloader_num_workers + ) + + # Supress warnings + warnings.filterwarnings(action="ignore", module="opacus", message=".*Secure RNG turned off") + warnings.filterwarnings(action="ignore", module="torch", message=".*Using a non-full backward hook") + + model = CNN() + assert ModuleValidator.is_valid(model) + + device = 'cuda' if torch.cuda.is_available() else 'cpu' + model.to(device) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.SGD(model.parameters(), lr=args.learning_rate, momentum=0) + + # Not the same as args.batch_size / len(train_dataset) + args.sample_rate = 1 / len(train_loader) + num_steps = int(len(train_loader) * args.num_epochs) + + if not args.disable_dp: + args.noise_multiplier = find_noise_multiplier( + sampling_probability=args.sample_rate, + num_steps=num_steps, + target_epsilon=args.target_epsilon, + target_delta=args.target_delta, + eps_error=0.1 + ) + + privacy_engine = PrivacyEngine(secure_mode=args.secure_mode) + + # Override Opacus accountant + # Revise if https://github.com/pytorch/opacus/pull/493 is merged + privacy_engine.accountant = PRVAccountant( + noise_multiplier=args.noise_multiplier, + sample_rate=args.sample_rate, + max_steps=num_steps, + eps_error=0.1, + delta_error=1e-9) + + model, optimizer, train_loader = privacy_engine.make_private( + module=model, + optimizer=optimizer, + data_loader=train_loader, + noise_multiplier=args.noise_multiplier, + max_grad_norm=args.max_grad_norm, + poisson_sampling=True, + noise_generator=noise_generator + ) + + print(f"Training using DP-SGD with {optimizer.original_optimizer.__class__.__name__} optimizer\n" + f" noise multiplier Οƒ = {optimizer.noise_multiplier},\n" + f" clipping norm C = {optimizer.max_grad_norm:},\n" + f" average batch size L = {args.batch_size},\n" + f" sample rate = {args.sample_rate},\n" + f" for {args.num_epochs} epochs ({num_steps} steps)\n" + f" to target Ξ΅ = {args.target_epsilon}, Ξ΄ = {args.target_delta}") + + compute_epsilon: Optional[Callable[[float], float]] = lambda delta: privacy_engine.get_epsilon(delta=delta) + else: + print(f"Training using SGD with {optimizer.__class__.__name__} optimizer\n" + f" batch size L = {args.batch_size},\n" + f" for {args.num_epochs} epochs ({num_steps} steps)") + compute_epsilon = None + + # Must be initialized after attaching the privacy engine. + # See https://discuss.pytorch.org/t/how-to-use-lr-scheduler-in-opacus/111718 + scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.lr_scheduler_step, gamma=args.lr_scheduler_gamma) + + pbar = trange(args.num_epochs, desc="Epoch", unit="epoch", position=0, leave=True, disable=None) + for epoch in pbar: + pbar.set_postfix(lr=f"{scheduler.get_last_lr()}") + train(args, model, device, train_loader, criterion, optimizer, epoch + 1, compute_epsilon=compute_epsilon) + scheduler.step() + + acc = test(args, model, device, test_loader, criterion) + with open(os.path.join(args.output_dir, "accuracy"), "w") as f: + print(f"{acc:.3f}", file=f) + + if not args.disable_dp: + final_epsilon = compute_epsilon(args.target_delta) + print(f"The trained model is (Ξ΅ = {final_epsilon}, Ξ΄ = {args.target_delta})-DP") + with open(os.path.join(args.output_dir, "epsilon"), "w") as f: + print(f"{final_epsilon:.3f}", file=f) + + with open(os.path.join(args.output_dir, "seed_challenge"), "w") as f: + print(f"{args.seed_challenge}", file=f) + + with open(os.path.join(args.output_dir, "seed_training"), "w") as f: + print(f"{args.seed_training}", file=f) + + with open(os.path.join(args.output_dir, "seed_membership"), "w") as f: + print(f"{args.seed_membership}", file=f) + + with open(os.path.join(args.output_dir, "solution.csv"), "w") as f: + solution = challenge_dataset.get_solutions() + csv.writer(f).writerow(solution) + + torch.save(model.state_dict(), os.path.join(args.output_dir, "model.pt")) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model_id", type=int, metavar='ID', + help="an identifier for the trained model") + # Seeds + parser.add_argument("--train_seed", type=int, metavar='TS', + help="seed for reproducibility") + parser.add_argument("--split_seed", type=int, metavar='SS', + help="seed to deterministically generate the 3 seeds for creating splits " + "(--seed_challenge, --seed_trainig, seed_membership)") + parser.add_argument("--seed_challenge", type=int, metavar='SC', + help="seed to select challenge examples") + parser.add_argument("--seed_training", type=int, metavar='ST', + help="seed to select non-challenge training examples") + parser.add_argument("--seed_membership", type=int, metavar='SM', + help="seed to split challenge examples into members/non-members") + # Split lengths + parser.add_argument("--len_training", type=int, metavar="N", required=True, + help="(required) number of examples used for training") + parser.add_argument("--len_challenge", type=int, metavar="m", required=True, + help="(required) number of member and non-member challenge examples " + "(i.e., m members and m non-members)") + # General + parser.add_argument("--secure_mode", action="store_true", default=False, + help="whether to use Opacus secure mode for training (default=True)") + parser.add_argument("--disable_dp", action="store_true", default=False, + help="whether to disable differentially private training altogether (default=False)") + parser.add_argument("--dataloader_num_workers", type=int, metavar='W', default=2, + help="number of workers for data loading. 0 means that the data will be loaded in the main process (default=2). " + "See torch.utils.data.DataLoader") + parser.add_argument("--logging_steps", type=int, metavar='k', default=10, + help="prints accuracy, loss, and privacy accounting information during training every k logical batches " + "(default=10)") + parser.add_argument("--dataset_dir", type=str, metavar="DATA", default=".", + help="root directory for cached dataset (default='.')") + parser.add_argument("--output_dir", type=str, metavar="OUT", + help="output directory. If none given, will pick one based on hyperparameters") + # Training hyperparameters + parser.add_argument("--target_epsilon", type=float, metavar="EPSILON", + help="target DP epsilon. Required unless specifying --disable_dp") + parser.add_argument("--target_delta", type=float, metavar="DELTA", + help="target DP delta. Will use 1/N if unspecified") + parser.add_argument("--batch_size", type=int, metavar="L", + help="expected logical batch size; determines the sample rate of DP-SGD. " + "Actual batch size varies because batches are constructed using Poisson sampling") + parser.add_argument("--max_physical_batch_size", type=int, metavar="B", + help="maximum physical batch size. Use to simulate logical batches larger than available memory and " + "to safeguard against unusually large batches produces by Poisson sampling. " + "See opacus.utils.batch_memory_manager.BatchMemoryManager") + parser.add_argument("--num_epochs", metavar='E', type=int, default=10, + help="number of training epochs (default=10)") + parser.add_argument("--max_grad_norm", type=float, metavar='C', default=1.0, + help="clipping norm for per-sample gradients in DP-SGD (default=1.0)") + parser.add_argument("--learning_rate", type=float, metavar="LR", default=1.0, + help="initial learning rate (default=1.0)") + parser.add_argument("--lr_scheduler_gamma", type=float, metavar="GAMMA", default=1.0, + help="gamma parameter for exponential learning rate scheduler") + parser.add_argument("--lr_scheduler_step", type=int, metavar="S", default=1, + help="step size for exponential learning rate scheduler") + + args = parser.parse_args() + + if args.len_training is None: + raise ValueError("Please specify --len_training") + + if args.len_challenge is None: + raise ValueError("Please specify --len_challenge") + + # Parameter validation + if args.secure_mode and args.train_seed is not None: + raise ValueError("Specify either secure mode or a seed for reproducibility, but not both") + + if args.target_delta is None: + args.target_delta = 1 / args.len_training + + if args.split_seed is not None and (args.seed_challenge is not None or args.seed_training is not None or args.seed_membership is not None): + raise ValueError("A --split_seed was given to generate seeds to construct splits but at least one explicit seed was specified. Bailing out.") + + if args.output_dir is None: + now = datetime.now().strftime("%Y_%m_%d-%H_%M_%S") + if args.disable_dp: + args.output_dir = f"{now}-nodp-lr{args.learning_rate}-gamma{args.lr_scheduler_gamma}-S{args.lr_scheduler_step}-L{args.batch_size}-" + \ + f"E{args.num_epochs}" + else: + args.output_dir = f"{now}-eps{args.target_epsilon}-delta{args.target_delta}-lr{args.learning_rate}-" + \ + f"gamma{args.lr_scheduler_gamma}-S{args.lr_scheduler_step}-L{args.batch_size}-E{args.num_epochs}-C{args.max_grad_norm}" + \ + f"{'-secure' if args.secure_mode else ''}" + + print(f"No --output_dir specified. Will use {args.output_dir}") + + if args.model_id is not None: + args.output_dir = args.output_dir + f"_{args.model_id}" + + os.makedirs(args.output_dir, exist_ok=True) + + with open(os.path.join(args.output_dir, "arguments"), "w") as argfile: + try: + commit_hash = git.Repo(".", search_parent_directories=True).git.rev_parse("HEAD") + except git.exc.InvalidGitRepositoryError: + commit_hash = "unknown" + print(f"Commit hash: {commit_hash}") + print(f"# Commit hash: {commit_hash}", file=argfile) + for arg in vars(args): + print(f"--{arg} {getattr(args, arg)}") + print(f"--{arg} {getattr(args, arg)}", file=argfile) + + main(args) diff --git a/training/train_cifar10_ddp.sh b/training/train_cifar10_ddp.sh new file mode 100755 index 0000000..4fc04ed --- /dev/null +++ b/training/train_cifar10_ddp.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +python train_cifar10.py \ + --len_challenge 100 \ + --len_training 50000 \ + --dataloader_num_workers 4 \ + --dataset_dir /data \ + --batch_size 512 \ + --max_physical_batch_size 128 \ + --num_epochs 50 \ + --max_grad_norm 2.6 \ + --target_epsilon 4.0 \ + --target_delta 1e-5 \ + --learning_rate 0.5 \ + --lr_scheduler_gamma 0.96 \ + --secure_mode diff --git a/training/train_cifar10_hi.sh b/training/train_cifar10_hi.sh new file mode 100755 index 0000000..df30445 --- /dev/null +++ b/training/train_cifar10_hi.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh + +python train_cifar10.py \ + --len_challenge 100 \ + --len_training 50000 \ + --dataloader_num_workers 4 \ + --dataset_dir /data \ + --batch_size 512 \ + --max_physical_batch_size 128 \ + --num_epochs 50 \ + --max_grad_norm 2.6 \ + --target_epsilon 10.0 \ + --target_delta 1e-5 \ + --learning_rate 0.5 \ + --lr_scheduler_gamma 0.96 \ + --secure_mode + diff --git a/training/train_cifar10_inf.sh b/training/train_cifar10_inf.sh new file mode 100755 index 0000000..2f8211b --- /dev/null +++ b/training/train_cifar10_inf.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +python train_cifar10.py \ + --len_challenge 100 \ + --len_training 50000 \ + --dataloader_num_workers 4 \ + --dataset_dir /data \ + --batch_size 32 \ + --max_physical_batch_size 128 \ + --num_epochs 50 \ + --learning_rate 0.005 \ + --lr_scheduler_gamma 0.96 \ + --disable_dp diff --git a/training/train_cifar10_lo.sh b/training/train_cifar10_lo.sh new file mode 100755 index 0000000..4fc04ed --- /dev/null +++ b/training/train_cifar10_lo.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +python train_cifar10.py \ + --len_challenge 100 \ + --len_training 50000 \ + --dataloader_num_workers 4 \ + --dataset_dir /data \ + --batch_size 512 \ + --max_physical_batch_size 128 \ + --num_epochs 50 \ + --max_grad_norm 2.6 \ + --target_epsilon 4.0 \ + --target_delta 1e-5 \ + --learning_rate 0.5 \ + --lr_scheduler_gamma 0.96 \ + --secure_mode diff --git a/training/train_purchase100.py b/training/train_purchase100.py new file mode 100644 index 0000000..267c465 --- /dev/null +++ b/training/train_purchase100.py @@ -0,0 +1,417 @@ +import os +import argparse +import warnings +import git +import csv +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim + +from torchcsprng import create_mt19937_generator, create_random_device_generator + +from torch.utils.data import DataLoader + +from opacus import PrivacyEngine +from opacus.validators import ModuleValidator +from opacus.utils.batch_memory_manager import BatchMemoryManager + +from prv_accountant.dpsgd import find_noise_multiplier + +from accountant import PRVAccountant + +from mico_competition import ChallengeDataset, MLP, load_purchase100 + +from tqdm import tqdm, trange + +from datetime import datetime + +from typing import Callable, Optional + + +def accuracy(preds: torch.Tensor, labels: torch.Tensor) -> float: + return (preds == labels).mean() + + +def train(args: argparse.Namespace, + model: nn.Module, + device: torch.device, + train_loader: DataLoader, + criterion, + optimizer: optim.Optimizer, + epoch: int, + compute_epsilon: Optional[Callable[[int], float]] = None): + model.train() + + losses = [] + top1_acc = [] + + with BatchMemoryManager( + data_loader=train_loader, + max_physical_batch_size=args.max_physical_batch_size, + optimizer=optimizer + ) as memory_safe_data_loader: + + if args.disable_dp: + data_loader = train_loader + else: + data_loader = memory_safe_data_loader + + # BatchSplittingSampler.__len__() approximates (badly) the length in physical batches + # See https://github.com/pytorch/opacus/issues/516 + # We instead heuristically keep track of logical batches processed + pbar = tqdm(data_loader, desc="Batch", unit="batch", position=1, leave=True, total=len(train_loader), disable=None) + logical_batch_len = 0 + for i, (inputs, target) in enumerate(data_loader): + inputs = inputs.to(device) + target = target.to(device) + + logical_batch_len += len(target) + if logical_batch_len >= args.batch_size: + pbar.update(1) + logical_batch_len = logical_batch_len % args.max_physical_batch_size + + optimizer.zero_grad() + output = model(inputs) + loss = criterion(output, target) + + preds = np.argmax(output.detach().cpu().numpy(), axis=1) + labels = target.detach().cpu().numpy() + acc = accuracy(preds, labels) + + losses.append(loss.item()) + top1_acc.append(acc) + + loss.backward() + optimizer.step() + + if (pbar.n + 1) % args.logging_steps == 0 or (pbar.n + 1) == pbar.total: + if not args.disable_dp: + epsilon = compute_epsilon(delta=args.target_delta) + pbar.set_postfix( + epoch=f"{epoch:02}", + train_loss=f"{np.mean(losses):.3f}", + accuracy=f"{np.mean(top1_acc) * 100:.3f}", + dp=f"(Ξ΅={epsilon:.2f}, Ξ΄={args.target_delta})" + ) + else: + pbar.set_postfix( + epoch=f"{epoch:02}", + train_loss=f"{np.mean(losses):.3f}", + accuracy=f"{np.mean(top1_acc) * 100:.3f}", + dp="(Ξ΅ = ∞, Ξ΄ = 0)" + ) + + pbar.update(pbar.total - pbar.n) + + +def test(args: argparse.Namespace, + model: nn.Module, + device: torch.device, + test_loader: DataLoader, + criterion): + model.eval() + + losses = [] + top1_acc = [] + + with torch.no_grad(): + for inputs, target in tqdm(test_loader, desc="Test ", unit="batch", disable=None): + inputs = inputs.to(device) + target = target.to(device) + + output = model(inputs) + loss = criterion(output, target) + + preds = np.argmax(output.detach().cpu().numpy(), axis=1) + labels = target.detach().cpu().numpy() + + acc = accuracy(preds, labels) + + losses.append(loss.item()) + top1_acc.append(acc) + + top1_avg = np.mean(top1_acc) + loss_avg = np.mean(losses) + + print( + f"Test Loss : {loss_avg:.6f}\n" + f"Test Accuracy: {top1_avg * 100:.6f}" + ) + + return np.mean(top1_acc) + + +def main(args: argparse.Namespace): + noise_generator = None + if not args.secure_mode and args.train_seed is not None: + # Following the advice on https://pytorch.org/docs/1.8.1/notes/randomness.html + + if torch.cuda.is_available(): + os.environ['CUBLAS_WORKSPACE_CONFIG'] = ":4096:8" + torch.use_deterministic_algorithms(True) + torch.cuda.manual_seed(args.train_seed) + torch.cuda.manual_seed_all(args.train_seed) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + + import random + random.seed(args.train_seed) + os.environ['PYTHONHASHSEED'] = str(args.train_seed) + + # Required to get deterministic batches because Opacus uses secure_rng as a generator for + # train_loader when poisson_sampling = True even though secure_mode = False, which sets secure_rng = None + # https://github.com/pytorch/opacus/blob/5e632cdb8d497aade29e5555ad79921c239c78f7/opacus/privacy_engine.py#L206 + torch.manual_seed(args.train_seed) + np.random.seed(args.train_seed) + noise_generator = create_mt19937_generator(args.train_seed) + + if (args.seed_challenge is None or args.seed_training is None or args.seed_membership is None): + + if args.split_seed is None: + seed_generator = create_random_device_generator() + else: + seed_generator = create_mt19937_generator(args.split_seed) + + args.seed_challenge, args.seed_training, args.seed_membership = torch.empty( + 3, dtype=torch.int64).random_(0, to=None, generator=seed_generator) + + print("Using generated seeds\n" + f" seed_challenge = {args.seed_challenge}\n" + f" seed_training = {args.seed_training}\n" + f" seed_membership = {args.seed_membership}\n") + + else: + print("Using specified seeds") + + full_dataset = load_purchase100(dataset_dir=args.dataset_dir) + + challenge_dataset = ChallengeDataset( + full_dataset, + len_challenge=args.len_challenge, + len_training=args.len_training, + seed_challenge=args.seed_challenge, + seed_training=args.seed_training, + seed_membership=args.seed_membership) + + train_dataset = challenge_dataset.get_train_dataset() + test_dataset = challenge_dataset.get_eval_dataset() + + train_loader = DataLoader( + train_dataset, + batch_size=args.batch_size, + num_workers=args.dataloader_num_workers, + pin_memory=True, + ) + + test_loader = DataLoader( + test_dataset, + batch_size=args.max_physical_batch_size, + num_workers=args.dataloader_num_workers + ) + + # Supress warnings + warnings.filterwarnings(action="ignore", module="opacus", message=".*Secure RNG turned off") + warnings.filterwarnings(action="ignore", module="torch", message=".*Using a non-full backward hook") + + model = MLP() + assert ModuleValidator.is_valid(model) + + device = 'cuda' if torch.cuda.is_available() else 'cpu' + model.to(device) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=args.learning_rate) + + # Not the same as args.batch_size / len(train_dataset) + args.sample_rate = 1 / len(train_loader) + num_steps = int(len(train_loader) * args.num_epochs) + + if not args.disable_dp: + args.noise_multiplier = find_noise_multiplier( + sampling_probability=args.sample_rate, + num_steps=num_steps, + target_epsilon=args.target_epsilon, + target_delta=args.target_delta, + eps_error=0.1 + ) + + privacy_engine = PrivacyEngine(secure_mode=args.secure_mode) + + # Override Opacus accountant + # Revise if https://github.com/pytorch/opacus/pull/493 is merged + privacy_engine.accountant = PRVAccountant( + noise_multiplier=args.noise_multiplier, + sample_rate=args.sample_rate, + max_steps=num_steps, + eps_error=0.1, + delta_error=1e-9) + + model, optimizer, train_loader = privacy_engine.make_private( + module=model, + optimizer=optimizer, + data_loader=train_loader, + noise_multiplier=args.noise_multiplier, + max_grad_norm=args.max_grad_norm, + poisson_sampling=True, + noise_generator=noise_generator + ) + + print(f"Training using DP-SGD with {optimizer.original_optimizer.__class__.__name__} optimizer\n" + f" noise multiplier Οƒ = {optimizer.noise_multiplier},\n" + f" clipping norm C = {optimizer.max_grad_norm:},\n" + f" average batch size L = {args.batch_size},\n" + f" sample rate = {args.sample_rate},\n" + f" for {args.num_epochs} epochs ({num_steps} steps)\n" + f" to target Ξ΅ = {args.target_epsilon}, Ξ΄ = {args.target_delta}") + + compute_epsilon: Optional[Callable[[float], float]] = lambda delta: privacy_engine.get_epsilon(delta=delta) + else: + print(f"Training using SGD with {optimizer.__class__.__name__} optimizer\n" + f" batch size L = {args.batch_size},\n" + f" for {args.num_epochs} epochs ({num_steps} steps)") + compute_epsilon = None + + # Must be initialized after attaching the privacy engine. + # See https://discuss.pytorch.org/t/how-to-use-lr-scheduler-in-opacus/111718 + scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.lr_scheduler_step, gamma=args.lr_scheduler_gamma) + + pbar = trange(args.num_epochs, desc="Epoch", unit="epoch", position=0, leave=True, disable=None) + for epoch in pbar: + pbar.set_postfix(lr=f"{scheduler.get_last_lr()}") + train(args, model, device, train_loader, criterion, optimizer, epoch + 1, compute_epsilon=compute_epsilon) + scheduler.step() + + acc = test(args, model, device, test_loader, criterion) + with open(os.path.join(args.output_dir, "accuracy"), "w") as f: + print(f"{acc:.3f}", file=f) + + if not args.disable_dp: + final_epsilon = compute_epsilon(args.target_delta) + print(f"The trained model is (Ξ΅ = {final_epsilon}, Ξ΄ = {args.target_delta})-DP") + with open(os.path.join(args.output_dir, "epsilon"), "w") as f: + print(f"{final_epsilon:.3f}", file=f) + + with open(os.path.join(args.output_dir, "seed_challenge"), "w") as f: + print(f"{args.seed_challenge}", file=f) + + with open(os.path.join(args.output_dir, "seed_training"), "w") as f: + print(f"{args.seed_training}", file=f) + + with open(os.path.join(args.output_dir, "seed_membership"), "w") as f: + print(f"{args.seed_membership}", file=f) + + with open(os.path.join(args.output_dir, "solution.csv"), "w") as f: + solution = challenge_dataset.get_solutions() + csv.writer(f).writerow(solution) + + torch.save(model.state_dict(), os.path.join(args.output_dir, "model.pt")) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model_id", type=int, metavar='ID', + help="an identifier for the trained model") + # Seeds + parser.add_argument("--train_seed", type=int, metavar='TS', + help="seed for reproducibility") + parser.add_argument("--split_seed", type=int, metavar='SS', + help="seed to deterministically generate the 3 seeds for creating splits " + "(--seed_challenge, --seed_trainig, seed_membership)") + parser.add_argument("--seed_challenge", type=int, metavar='SC', + help="seed to select challenge examples") + parser.add_argument("--seed_training", type=int, metavar='ST', + help="seed to select non-challenge training examples") + parser.add_argument("--seed_membership", type=int, metavar='SM', + help="seed to split challenge examples into members/non-members") + # Split lengths + parser.add_argument("--len_training", type=int, metavar="N", required=True, + help="(required) number of examples used for training") + parser.add_argument("--len_challenge", type=int, metavar="m", required=True, + help="(required) number of member and non-member challenge examples " + "(i.e., m members and m non-members)") + # General + parser.add_argument("--secure_mode", action="store_true", default=False, + help="whether to use Opacus secure mode for training (default=True)") + parser.add_argument("--disable_dp", action="store_true", default=False, + help="whether to disable differentially private training altogether (default=False)") + parser.add_argument("--dataloader_num_workers", type=int, metavar='W', default=2, + help="number of workers for data loading. 0 means that the data will be loaded in the main process (default=2). " + "See torch.utils.data.DataLoader") + parser.add_argument("--logging_steps", type=int, metavar='k', default=10, + help="prints accuracy, loss, and privacy accounting information during training every k logical batches " + "(default=10)") + parser.add_argument("--dataset_dir", type=str, metavar="DATA", default=".", + help="root directory for cached dataset (default='.')") + parser.add_argument("--output_dir", type=str, metavar="OUT", + help="output directory. If none given, will pick one based on hyperparameters") + # Training hyperparameters + parser.add_argument("--target_epsilon", type=float, metavar="EPSILON", + help="target DP epsilon. Required unless specifying --disable_dp") + parser.add_argument("--target_delta", type=float, metavar="DELTA", + help="target DP delta. Will use 1/N if unspecified") + parser.add_argument("--batch_size", type=int, metavar="L", + help="expected logical batch size; determines the sample rate of DP-SGD. " + "Actual batch size varies because batches are constructed using Poisson sampling") + parser.add_argument("--max_physical_batch_size", type=int, metavar="B", + help="maximum physical batch size. Use to simulate logical batches larger than available memory and " + "to safeguard against unusually large batches produces by Poisson sampling. " + "See opacus.utils.batch_memory_manager.BatchMemoryManager") + parser.add_argument("--num_epochs", metavar='E', type=int, default=10, + help="number of training epochs (default=10)") + parser.add_argument("--max_grad_norm", type=float, metavar='C', default=1.0, + help="clipping norm for per-sample gradients in DP-SGD (default=1.0)") + parser.add_argument("--learning_rate", type=float, metavar="LR", default=1.0, + help="initial learning rate (default=1.0)") + parser.add_argument("--lr_scheduler_gamma", type=float, metavar="GAMMA", default=1.0, + help="gamma parameter for exponential learning rate scheduler") + parser.add_argument("--lr_scheduler_step", type=int, metavar="S", default=1, + help="step size for exponential learning rate scheduler") + + args = parser.parse_args() + + if args.len_training is None: + raise ValueError("Please specify --len_training") + + if args.len_challenge is None: + raise ValueError("Please specify --len_challenge") + + # Parameter validation + if args.secure_mode and args.train_seed is not None: + raise ValueError("Specify either secure mode or a seed for reproducibility, but not both") + + if args.target_delta is None: + args.target_delta = 1 / args.len_training + + if args.split_seed is not None and (args.seed_challenge is not None or args.seed_training is not None or args.seed_membership is not None): + raise ValueError("A --split_seed was given to generate seeds to construct splits but at least one explicit seed was specified. Bailing out.") + + if args.output_dir is None: + now = datetime.now().strftime("%Y_%m_%d-%H_%M_%S") + if args.disable_dp: + args.output_dir = f"{now}-nodp-lr{args.learning_rate}-gamma{args.lr_scheduler_gamma}-S{args.lr_scheduler_step}-L{args.batch_size}-" + \ + f"E{args.num_epochs}" + else: + args.output_dir = f"{now}-eps{args.target_epsilon}-delta{args.target_delta}-lr{args.learning_rate}-" + \ + f"gamma{args.lr_scheduler_gamma}-S{args.lr_scheduler_step}-L{args.batch_size}-E{args.num_epochs}-C{args.max_grad_norm}" + \ + f"{'-secure' if args.secure_mode else ''}" + + print(f"No --output_dir specified. Will use {args.output_dir}") + + if args.model_id is not None: + args.output_dir = args.output_dir + f"_{args.model_id}" + + os.makedirs(args.output_dir, exist_ok=True) + + with open(os.path.join(args.output_dir, "arguments"), "w") as argfile: + try: + commit_hash = git.Repo(".", search_parent_directories=True).git.rev_parse("HEAD") + except git.exc.InvalidGitRepositoryError: + commit_hash = "unknown" + print(f"Commit hash: {commit_hash}") + print(f"# Commit hash: {commit_hash}", file=argfile) + for arg in vars(args): + print(f"--{arg} {getattr(args, arg)}") + print(f"--{arg} {getattr(args, arg)}", file=argfile) + + main(args) diff --git a/training/train_purchase100_ddp.sh b/training/train_purchase100_ddp.sh new file mode 100755 index 0000000..f9e204e --- /dev/null +++ b/training/train_purchase100_ddp.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh + +python train_purchase100.py \ + --len_challenge 100 \ + --len_training 150000 \ + --dataloader_num_workers 4 \ + --dataset_dir /data \ + --batch_size 512 \ + --max_physical_batch_size 128 \ + --num_epochs 30 \ + --learning_rate 0.001 \ + --lr_scheduler_gamma 0.9 \ + --lr_scheduler_step 5 \ + --max_grad_norm 2.6 \ + --target_epsilon 4.0 \ + --target_delta 1e-5 \ + --secure_mode diff --git a/training/train_purchase100_hi.sh b/training/train_purchase100_hi.sh new file mode 100755 index 0000000..14d8bb8 --- /dev/null +++ b/training/train_purchase100_hi.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh + +python train_purchase100.py \ + --len_challenge 100 \ + --len_training 150000 \ + --dataloader_num_workers 4 \ + --dataset_dir /data \ + --batch_size 512 \ + --max_physical_batch_size 128 \ + --num_epochs 30 \ + --learning_rate 0.001 \ + --lr_scheduler_gamma 0.9 \ + --lr_scheduler_step 5 \ + --max_grad_norm 2.6 \ + --target_epsilon 10.0 \ + --target_delta 1e-5 \ + --secure_mode diff --git a/training/train_purchase100_inf.sh b/training/train_purchase100_inf.sh new file mode 100755 index 0000000..4e79193 --- /dev/null +++ b/training/train_purchase100_inf.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env sh + +python train_purchase100.py \ + --len_challenge 100 \ + --len_training 150000 \ + --dataloader_num_workers 4 \ + --dataset_dir /data \ + --batch_size 512 \ + --max_physical_batch_size 128 \ + --num_epochs 30 \ + --learning_rate 0.001 \ + --lr_scheduler_gamma 0.9 \ + --lr_scheduler_step 5 \ + --disable_dp diff --git a/training/train_purchase100_lo.sh b/training/train_purchase100_lo.sh new file mode 100755 index 0000000..f9e204e --- /dev/null +++ b/training/train_purchase100_lo.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh + +python train_purchase100.py \ + --len_challenge 100 \ + --len_training 150000 \ + --dataloader_num_workers 4 \ + --dataset_dir /data \ + --batch_size 512 \ + --max_physical_batch_size 128 \ + --num_epochs 30 \ + --learning_rate 0.001 \ + --lr_scheduler_gamma 0.9 \ + --lr_scheduler_step 5 \ + --max_grad_norm 2.6 \ + --target_epsilon 4.0 \ + --target_delta 1e-5 \ + --secure_mode diff --git a/training/train_sst2.py b/training/train_sst2.py new file mode 100644 index 0000000..18addfb --- /dev/null +++ b/training/train_sst2.py @@ -0,0 +1,212 @@ +import numpy as np +import pandas as pd +import os +import torch +import sys +import csv +import yaml +import warnings +import datasets + +from opacus import PrivacyEngine + +from dp_transformers import TrainingArguments, PrivacyArguments, PrivacyEngineCallback + +from prv_accountant.dpsgd import find_noise_multiplier, DPSGDAccountant + +from torchcsprng import create_mt19937_generator, create_random_device_generator + +from transformers import ( + HfArgumentParser, AutoTokenizer, AutoModelForSequenceClassification, + Trainer, EvalPrediction, PreTrainedTokenizerBase +) + +from dataclasses import dataclass +from pathlib import Path + +from mico_competition import ChallengeDataset, load_sst2 + +from typing import Optional + + +@dataclass +class ModelArguments: + model_name: str + + +@dataclass +class DataArguments: + model_index: int + len_training: int = 67349 + len_challenge: int = 100 + seed_challenge: Optional[int] = None + seed_training: Optional[int] = None + seed_membership: Optional[int] = None + split_seed: Optional[int] = None + + +@dataclass +class SecurePrivacyArguments(PrivacyArguments): + delta: float = None + use_secure_prng: bool = False + + +@dataclass +class Arguments: + training: TrainingArguments + model: ModelArguments + privacy: SecurePrivacyArguments + data: DataArguments + + +def preprocess_text(D: datasets.DatasetDict, tokenizer: PreTrainedTokenizerBase, + max_sequence_length: int = None) -> datasets.DatasetDict: + processed_data = D.map( + lambda batch: tokenizer(batch["sentence"], padding="max_length", max_length=max_sequence_length), + batched=True + ) + return processed_data.remove_columns(["sentence"]) + +def load_dataset() -> datasets.DatasetDict: + if (args.data.seed_challenge is None or args.data.seed_training is None or args.data.seed_membership is None): + if args.data.split_seed is None: + seed_generator = create_random_device_generator() + else: + seed_generator = create_mt19937_generator(args.split_seed) + + args.data.seed_challenge, args.data.seed_training, args.data.seed_membership = torch.empty( + 3, dtype=torch.int64).random_(0, to=None, generator=seed_generator) + + print("Using generated seeds\n" + f" seed_challenge = {args.data.seed_challenge}\n" + f" seed_training = {args.data.seed_training}\n" + f" seed_membership = {args.data.seed_membership}\n") + else: + print("Using specified seeds") + + full_dataset = load_sst2() + + challenge_dataset = ChallengeDataset( + full_dataset, + len_challenge=args.data.len_challenge, + len_training=args.data.len_training, + seed_challenge=args.data.seed_challenge, + seed_training=args.data.seed_training, + seed_membership=args.data.seed_membership) + + with open(os.path.join(args.training.output_dir, "challenge", "seed_challenge"), "w") as f: + print(f"{args.data.seed_challenge}", file=f) + + with open(os.path.join(args.training.output_dir, "challenge", "seed_training"), "w") as f: + print(f"{args.data.seed_training}", file=f) + + with open(os.path.join(args.training.output_dir, "challenge", "seed_membership"), "w") as f: + print(f"{args.data.seed_membership}", file=f) + + with open(os.path.join(args.training.output_dir, "challenge", "solution.csv"), "w") as f: + solution = challenge_dataset.get_solutions() + csv.writer(f).writerow(solution) + + ds_train = pd.DataFrame.from_records(challenge_dataset.get_train_dataset()) + ds_test = pd.DataFrame.from_records(challenge_dataset.get_eval_dataset()) + + return datasets.DatasetDict({ + "train": datasets.Dataset.from_pandas(ds_train), + "test": datasets.Dataset.from_pandas(ds_test) + }).remove_columns("idx") + + +def main(args: Arguments): + output_dir = Path(args.training.output_dir) + + output_dir.mkdir(parents=True, exist_ok=True) + with open(os.path.join(args.training.output_dir, "arguments.yml"), "w") as f: + yaml.dump(args, f) + print(yaml.dump(args)) + + os.mkdir(output_dir/"challenge") + + ds = load_dataset() + + if args.privacy.use_secure_prng: + import torchcsprng as csprng + mt19937_gen = csprng.create_mt19937_generator() + ds['train'] = ds['train'].select(torch.randperm(len(ds['train']), generator=mt19937_gen).tolist()) + + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + warnings.filterwarnings(action="ignore", module="torch", message=".*Using a non-full backward hook") + + model = AutoModelForSequenceClassification.from_pretrained(args.model.model_name, num_labels=2) + tokenizer = AutoTokenizer.from_pretrained(args.model.model_name) + + ds = preprocess_text(ds, tokenizer=tokenizer, max_sequence_length=67) + + model.train() + model = model.to(args.training.device) + + if (not args.training.no_cuda) and (not torch.cuda.is_available()): + raise RuntimeError("CUDA is not available. Please use --no-cuda to run this script.") + + callbacks = [] + if not args.privacy.disable_dp: + sampling_probability = training_args.train_batch_size * training_args.gradient_accumulation_steps / len(ds["train"]) + num_steps = int(np.ceil(1 / sampling_probability) * training_args.num_train_epochs) + noise_multiplier = find_noise_multiplier( + sampling_probability=sampling_probability, num_steps=num_steps, target_epsilon=args.privacy.target_epsilon, + target_delta=args.privacy.delta, + eps_error=0.1 + ) + engine = PrivacyEngine( + module=model, + batch_size=training_args.per_device_train_batch_size*training_args.gradient_accumulation_steps, + sample_size=len(ds['train']), + noise_multiplier=noise_multiplier, + max_grad_norm=args.privacy.per_sample_max_grad_norm, + secure_rng=args.privacy.use_secure_prng, + ) + accountant = DPSGDAccountant( + noise_multiplier=noise_multiplier, sampling_probability=sampling_probability, max_steps=num_steps, + eps_error=0.2 + ) + privacy_callback = PrivacyEngineCallback( + engine, + compute_epsilon=lambda s: accountant.compute_epsilon(num_steps=s, delta=args.privacy.delta)[2] + ) + callbacks.append(privacy_callback) + + def compute_metrics(p: EvalPrediction): + preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions + preds = np.argmax(preds, axis=1) + return {"accuracy": (preds == p.label_ids).astype(np.float32).mean().item()} + + trainer = Trainer( + args=training_args, + train_dataset=ds["train"], + eval_dataset=ds["test"], + model=model, + tokenizer=tokenizer, + compute_metrics=compute_metrics, + callbacks=callbacks + ) + + try: + trainer.train() + finally: + trainer.save_model(output_dir/"challenge") + + if args.privacy.disable_dp: + epsilon_final = float('inf') + else: + epsilon_final = accountant.compute_epsilon(num_steps=engine.steps, delta=args.privacy.delta)[2] + trainer.log({"epsilon_final": epsilon_final}) + assert np.isclose(epsilon_final, args.privacy.target_epsilon, atol=0.2, rtol=0.0) + + print("Training successful. Exiting...") + return 0 + + +if __name__ == "__main__": + parser = HfArgumentParser((TrainingArguments, ModelArguments, SecurePrivacyArguments, DataArguments)) + training_args, model_args, privacy_args, data_args = parser.parse_args_into_dataclasses() + args = Arguments(training=training_args, model=model_args, privacy=privacy_args, data=data_args) + sys.exit(main(args)) diff --git a/training/train_sst2_ddp.sh b/training/train_sst2_ddp.sh new file mode 100755 index 0000000..26f29da --- /dev/null +++ b/training/train_sst2_ddp.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh + +# Does not support DDP. Might need to set e.g. CUDA_VISIBLE_DEVICES to +# force single-GPU training + +python train_sst2.py \ + --model_name roberta-base \ + --dataloader_num_workers 2 \ + --per_device_train_batch_size 16 \ + --gradient_accumulation_steps 64 \ + --num_train_epochs 3.0 \ + --per_sample_max_grad_norm 0.1 \ + --target_epsilon 4.0 \ + --delta 1e-5 \ + --learning_rate 0.0005 \ + --lr_scheduler_type constant \ + --use_secure_prng True \ + --model_index 0 \ + --save_strategy no \ + --evaluation_strategy epoch \ + --logging_steps 10 \ + --seed 42 \ + --output_dir output diff --git a/training/train_sst2_hi.sh b/training/train_sst2_hi.sh new file mode 100755 index 0000000..0d87a38 --- /dev/null +++ b/training/train_sst2_hi.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh + +# Does not support DDP. Might need to set e.g. CUDA_VISIBLE_DEVICES to +# force single-GPU training + +python train_sst2.py \ + --model_name roberta-base \ + --dataloader_num_workers 2 \ + --per_device_train_batch_size 16 \ + --gradient_accumulation_steps 64 \ + --num_train_epochs 3.0 \ + --per_sample_max_grad_norm 0.1 \ + --target_epsilon 10.0 \ + --delta 1e-5 \ + --learning_rate 0.0005 \ + --lr_scheduler_type constant \ + --use_secure_prng True \ + --model_index 0 \ + --save_strategy no \ + --evaluation_strategy epoch \ + --logging_steps 10 \ + --seed 42 \ + --output_dir output diff --git a/training/train_sst2_inf.sh b/training/train_sst2_inf.sh new file mode 100755 index 0000000..a4a2a4a --- /dev/null +++ b/training/train_sst2_inf.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh + +# Does not support DDP. Might need to set e.g. CUDA_VISIBLE_DEVICES to +# force single-GPU training + +python train_sst2.py \ + --model_name roberta-base \ + --lr_scheduler_type constant \ + --learning_rate 5e-5 \ + --use_secure_prng True \ + --num_train_epochs 3.0 \ + --logging_steps 10 \ + --save_strategy no \ + --evaluation_strategy epoch \ + --dataloader_num_workers 2 \ + --per_device_train_batch_size 96 \ + --gradient_accumulation_steps 1 \ + --output_dir output_inf \ + --model_index 0 \ + --seed 42 \ + --disable_dp True diff --git a/training/train_sst2_lo.sh b/training/train_sst2_lo.sh new file mode 100755 index 0000000..26f29da --- /dev/null +++ b/training/train_sst2_lo.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh + +# Does not support DDP. Might need to set e.g. CUDA_VISIBLE_DEVICES to +# force single-GPU training + +python train_sst2.py \ + --model_name roberta-base \ + --dataloader_num_workers 2 \ + --per_device_train_batch_size 16 \ + --gradient_accumulation_steps 64 \ + --num_train_epochs 3.0 \ + --per_sample_max_grad_norm 0.1 \ + --target_epsilon 4.0 \ + --delta 1e-5 \ + --learning_rate 0.0005 \ + --lr_scheduler_type constant \ + --use_secure_prng True \ + --model_index 0 \ + --save_strategy no \ + --evaluation_strategy epoch \ + --logging_steps 10 \ + --seed 42 \ + --output_dir output