diff --git a/3rdParty/tdigest.LICENSE.txt b/3rdParty/fastdigest.LICENSE.txt similarity index 100% rename from 3rdParty/tdigest.LICENSE.txt rename to 3rdParty/fastdigest.LICENSE.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07e05e2264..dd31762704 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,9 +44,6 @@ To collaborate efficiently, please read through this section and follow them. * [Building documentation](#building-the-documentation) * [Signing your work](#signing-your-work) -> Note: - > some package dependencies requires python-dev in local development such as - > python3.12-dev. #### Checking the coding style We check code style using flake8 and isort. diff --git a/examples/advanced/federated-statistics/README.md b/examples/advanced/federated-statistics/README.md index 32b09124ce..0d16d705fe 100644 --- a/examples/advanced/federated-statistics/README.md +++ b/examples/advanced/federated-statistics/README.md @@ -2,7 +2,7 @@ ## Objective NVIDIA FLARE will provide built-in federated statistics operators (controllers and executors) that -can generate global statistics based on local client side statistics. +can generate global statistics based on local client-side statistics. At each client site, we could have one or more datasets (such as "train" and "test" datasets); each dataset may have many features. For each feature in the dataset, we will calculate the statistics and then combine them to produce @@ -19,14 +19,47 @@ The result should be visualized via the visualization utility in the notebook. ## Assumptions -Assume that clients will provide the following: - * user needs to provide target statistics such as count, histogram only - * user needs to provide the local statistics for the target statistics (by implementing the statistic_spec) - * user needs to provide the data sets and dataset features (feature name, data type) - * * Note: count is always required as we use count to enforce data privacy policy -We only support **numerical features**, not categorical features. But user can return all types of features +Assume that clients will provide the following: +* Users need to provide target statistics such as count, histogram only +* Users need to provide the local statistics for the target statistics (by implementing the statistics_spec) +* Users need to provide the datasets and dataset features (feature name, data type) +* Note: count is always required as we use count to enforce data privacy policy + +We only support **numerical features**, not categorical features. However, users can return all types of features; the non-numerical features will be removed. + +## Statistics + + Federated statistics includes numerics statistics measures for + * count + * mean + * sum + * std_dev + * histogram + * quantile + + We did not include min, max value to avoid data privacy concern. + +### Quantile + +Quantile statistics refers to statistical measures that divide a probability distribution or dataset into intervals with equal probabilities or proportions. Quantiles help summarize the distribution of data by providing key points that indicate how values are spread. + +#### Key Quantiles: +1. Median (50th percentile): The middle value of a dataset, dividing it into two equal halves. +2. Quartiles (25th, 50th, 75th percentiles): Divide the data into four equal parts: +* Q1 (25th percentile): Lower quartile, below which 25% of the data falls. +* Q2 (50th percentile): Median. +* Q3 (75th percentile): Upper quartile, below which 75% of the data falls. +3. Deciles (10th, 20th, ..., 90th percentiles): Divide the data into ten equal parts. +4. Percentiles (1st, 2nd, ..., 99th): Divide the data into 100 equal parts. + +#### Usage of Quantiles: +* Descriptive Statistics: Summarizes the spread of data. +* Outlier Detection: Helps identify extreme values. +* Machine Learning: Used in feature engineering, normalization, and decision tree algorithms. +* Risk Analysis: Used in finance (e.g., Value at Risk, VaR). + ## Examples We provide several examples to demonstrate how should the operators be used. @@ -57,20 +90,21 @@ The main steps are The detailed example instructions can be found [Data frame statistics](df_stats/README.md) + ### COVID 19 Radiology Image Examples -The second example provided is image histogram example. Different from **Tabular** data example, +The second example provided is an image histogram example. Unlike the **Tabular** data example: -The image examples show the followings +The image examples show the following: * The [image_statistics.py](image_stats/jobs/image_stats/app/custom/image_statistics.py) only needs -to calculate the count and histogram target statistics, then user only needs to provide the calculation count, failure_count and histogram functions. There is no need to implement other metrics functions - (sum, mean,std_dev etc.) ( get_failure_count by default return 0 ) -* For each site's dataset, there are several thousands of images, the local histogram is aggregate histogram of all the image histograms. -* The image files are large, we can't load everything in memory, then calculate the statistics. -We will need to iterate through files for each calculation. For single feature, such as example. This is ok. If there are multiple features, -such as multiple channels, reload image to memory for each channel to do histogram calculation is really wasteful. -* Unlike [Data frame statistics](df_stats/README.md), the histogram bin's global range is pre-defined by user [0, 256] -where in [Data frame statistics](df_stats/README.md), besides "Age", all other features histogram global bin range +to calculate the count and histogram target statistics. Users only need to provide the calculation count, failure_count and histogram functions. There is no need to implement other metrics functions +(sum, mean, std_dev etc.) (get_failure_count by default returns 0) +* For each site's dataset, there are several thousand images; the local histogram is an aggregate histogram of all the image histograms +* The image files are large, so we can't load everything into memory and then calculate the statistics. +We will need to iterate through files for each calculation. For a single feature, this is acceptable. If there are multiple features, +such as multiple channels, reloading images to memory for each channel to do histogram calculation is wasteful +* Unlike [Data frame statistics](df_stats/README.md), the histogram bin's global range is pre-defined by users [0, 256], +whereas in [Data frame statistics](df_stats/README.md), besides "Age", all other features' histogram global bin range is dynamically estimated based on local min/max values An example of image histogram (the underline image files have only 1 channel) @@ -155,6 +189,7 @@ The main steps are * provide client side configuration to specify data input location * provide hierarchy specification file providing details about all the clients and their hierarchy. + ## Privacy Policy and Privacy Filters NVFLARE provide data privacy protection through privacy filters [privacy-management](https://nvflare.readthedocs.io/en/main/user_guide/security/site_policy_management.html#privacy-management) @@ -178,22 +213,21 @@ defined and job doesn't specify the privacy scope, the job deployment will fail, ### Privacy Policy Instrumentation -There are different ways to set privacy filter depending the use cases +There are different ways to set privacy filters depending on the use cases: #### Set Privacy Policy as researcher You can specify the "task_result_filters" in config_fed_client.json to specify -the privacy control. This is useful when you develop these filters +the privacy control. This is useful when you develop these filters. #### Setup site privacy policy as org admin -Once the company decides to instrument certain privacy policy independent of individual -job, one can copy the local directory privacy.json content to clients' local privacy.json ( merge not overwrite). -in this example, since we only has one app, we can simply copy the private.json from local directory to +Once the company decides to implement certain privacy policies independent of individual +jobs, one can copy the local directory privacy.json content to clients' local privacy.json (merge, not overwrite). +In this example, since we only have one app, we can simply copy the privacy.json from the local directory to: * site-1/local/privacy.json * site-2/local/privacy.json - We need to remove the same filters from the job definition in config_fed_client.json by simply set the "task_result_filters" to empty list to avoid **double filtering** ``` @@ -304,10 +338,7 @@ sequenceDiagram ``` - - ## Summary -We provided federated statistics operators that can easily aggregate and visualize the local statistics for -different data site and features. We hope this feature will make it easier to perform federated data analysis. - +We provided federated statistics operators that can easily aggregate and visualize the local statistics for +different data site and features. We hope this feature will make it easier to perform federated data analysis. \ No newline at end of file diff --git a/examples/advanced/federated-statistics/df_stats/README.md b/examples/advanced/federated-statistics/df_stats/README.md index 65900aeb5f..6c50b8b8d8 100644 --- a/examples/advanced/federated-statistics/df_stats/README.md +++ b/examples/advanced/federated-statistics/df_stats/README.md @@ -17,6 +17,52 @@ cd NVFlare/examples/advanced/federated-statistics/df_stats pip install -r requirements.txt ``` + +## Install fastdigest + +If you intend to calculate quantiles, you need to install fastdigest. + +``` +pip install fastdigest==0.4.0 +``` + +on Ubuntu, you might get the following error: + + Cargo, the Rust package manager, is not installed or is not on PATH. + This package requires Rust and Cargo to compile extensions. Install it through + the system's package manager or via https://rustup.rs/ + + Checking for Rust toolchain.... + +This is because fastdigest (or its dependencies) requires Rust and Cargo to build. + +You need to install Rust and Cargo on your Ubuntu system. Follow these steps: +Install Rust and Cargo +Run the following command to install Rust using rustup: + +``` +cd NVFlare/examples/advanced/federated-statistics/df_stats +./install_cargo.sh +``` + +Then you can install fastdigest again +``` +pip install fastdigest==0.4.0 +``` + +### Quantile Calculation + +To calculate federated quantiles, we needed to select a package that satisfies the following constraints: + +* Works in distributed systems +* Does not copy the original data (avoiding privacy leaks) +* Avoids transmitting large amounts of data +* Ideally, no system-level dependency + +We chose the fastdigest python package, a rust-based package. tdigest only carries the cluster coordinates, initially each data point is in its own cluster. By default, we will compress with max_bin = sqrt(datasize) to compress the coordinates, so the data won't leak. You can always override max_bins if you prefer more or less compression. + + + ## 1. Prepare data In this example, we are using UCI (University of California, Irvine) [adult dataset](https://archive.ics.uci.edu/dataset/2/adult) @@ -165,8 +211,12 @@ statistics computing, we will only need to provide the followings "stddev": {}, "histogram": { "*": {"bins": 10 }, "Age": {"bins": 5, "range":[0,120]} - } + }, + "quantile": { + "*": [25, 50, 75] + } }, + "writer_id": "stats_writer" } } @@ -195,7 +245,8 @@ in FLARE job store. ### 5.2 client side configuration -First, we specify the built-in client side executor: `StatisticsExecutor`, which takes a local stats generator Id +First, we specify the built-in client side executor: `StatisticsExecutor`, which takes a local stats generator ID + ``` "executor": { @@ -248,7 +299,7 @@ In this example, task_result_filters is defined as task privacy filter : `Statis `StatisticsPrivacyFilter` is using three separate the `StatisticsPrivacyCleanser`, you can find more details in [local privacy policy](../local/privacy.json) and in later discussion on privacy. -The privacy cleansers specify policy can be find in +The privacy cleansers specify policies can be found in ``` "components": [ { @@ -311,6 +362,8 @@ to calculate the local statistics, we will need to implements few methods def histogram(self, dataset_name: str, feature_name: str, num_of_bins: int, global_min_value: float, global_max_value: float) -> Histogram: + def quantiles(self, dataset_name: str, feature_name: str, percentiles: List) -> Dict: + ``` since some of features do not provide histogram bin range, we will need to calculate based on local min/max to estimate the global min/max, and then use the global bin/max as the range for all clients' histogram bin range. diff --git a/examples/advanced/federated-statistics/df_stats/demo/visualization.ipynb b/examples/advanced/federated-statistics/df_stats/demo/visualization.ipynb index 283f5279b2..f03648716a 100644 --- a/examples/advanced/federated-statistics/df_stats/demo/visualization.ipynb +++ b/examples/advanced/federated-statistics/df_stats/demo/visualization.ipynb @@ -37,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "c44a0217", "metadata": { "tags": [] @@ -81,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "93c62d5e", "metadata": { "tags": [] @@ -271,9 +271,9 @@ ], "metadata": { "kernelspec": { - "display_name": "nvflare_example", + "display_name": "nvflare-env", "language": "python", - "name": "nvflare_example" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -285,7 +285,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.2" + "version": "3.8.13" } }, "nbformat": 4, diff --git a/examples/advanced/federated-statistics/df_stats/install_cargo.sh b/examples/advanced/federated-statistics/df_stats/install_cargo.sh new file mode 100755 index 0000000000..6fcbc255c6 --- /dev/null +++ b/examples/advanced/federated-statistics/df_stats/install_cargo.sh @@ -0,0 +1,15 @@ + +# fastdigest (or its dependencies) requires Rust and Cargo to build. +# You need to install Rust and Cargo on your Ubuntu system. Follow these steps: +# Install Rust and Cargo +# Run the following command to install Rust using rustup: + + +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +# Then restart your terminal or run: + +source $HOME/.cargo/env +# Verify Installation +# Check if Rust and Cargo are installed correctly: +rustc --version +cargo --version \ No newline at end of file diff --git a/examples/advanced/federated-statistics/df_stats/job_api/df_statistics.py b/examples/advanced/federated-statistics/df_stats/job_api/df_statistics.py index 5078c2f0a9..942bbf692d 100644 --- a/examples/advanced/federated-statistics/df_stats/job_api/df_statistics.py +++ b/examples/advanced/federated-statistics/df_stats/job_api/df_statistics.py @@ -21,10 +21,10 @@ class DFStatistics(DFStatisticsCore): - def __init__(self, data_path): + def __init__(self, filename, data_root_dir="/tmp/nvflare/df_stats/data"): super().__init__() - self.data_root_dir = "/tmp/nvflare/df_stats/data" - self.data_path = data_path + self.data_root_dir = data_root_dir + self.filename = filename self.data: Optional[Dict[str, pd.DataFrame]] = None self.data_features = [ "Age", @@ -57,7 +57,7 @@ def load_data(self, fl_ctx: FLContext) -> Dict[str, pd.DataFrame]: self.log_info(fl_ctx, f"load data for client {client_name}") try: skip_rows = self.skip_rows[client_name] - data_path = f"{self.data_root_dir}/{fl_ctx.get_identity_name()}/{self.data_path}" + data_path = f"{self.data_root_dir}/{fl_ctx.get_identity_name()}/{self.filename}" # example of load data from CSV df: pd.DataFrame = pd.read_csv( data_path, names=self.data_features, sep=r"\s*,\s*", skiprows=skip_rows, engine="python", na_values="?" diff --git a/examples/advanced/federated-statistics/df_stats/job_api/df_stats_job.py b/examples/advanced/federated-statistics/df_stats/job_api/df_stats_job.py index 696d57170c..39aafe312b 100644 --- a/examples/advanced/federated-statistics/df_stats/job_api/df_stats_job.py +++ b/examples/advanced/federated-statistics/df_stats/job_api/df_stats_job.py @@ -20,9 +20,9 @@ def define_parser(): parser = argparse.ArgumentParser() - parser.add_argument("-n", "--n_clients", type=int, default=3) - parser.add_argument("-d", "--data_root_dir", type=str, nargs="?", default="/tmp/nvflare/dataset/output") - parser.add_argument("-o", "--stats_output_path", type=str, nargs="?", default="statistics/stats.json") + parser.add_argument("-n", "--n_clients", type=int, default=2) + parser.add_argument("-d", "--data_root_dir", type=str, nargs="?", default="/tmp/nvflare/df_stats/data") + parser.add_argument("-o", "--stats_output_path", type=str, nargs="?", default="statistics/adults_stats.json") parser.add_argument("-j", "--job_dir", type=str, nargs="?", default="/tmp/nvflare/jobs/stats_df") parser.add_argument("-w", "--work_dir", type=str, nargs="?", default="/tmp/nvflare/jobs/stats_df/work_dir") parser.add_argument("-co", "--export_config", action="store_true", help="config only mode, export config") @@ -45,12 +45,11 @@ def main(): "mean": {}, "sum": {}, "stddev": {}, - "histogram": {"*": {"bins": 20}}, - "Age": {"bins": 20, "range": [0, 10]}, - "percentile": {"*": [25, 50, 75], "Age": [50, 95]}, + "histogram": {"*": {"bins": 20}, "Age": {"bins": 20, "range": [0, 100]}}, + "quantile": {"*": [0.1, 0.5, 0.9], "Age": [0.1, 0.5, 0.9]}, } # define local stats generator - df_stats_generator = DFStatistics(data_root_dir=data_root_dir) + df_stats_generator = DFStatistics(filename="data.csv", data_root_dir=data_root_dir) job = StatsJob( job_name="stats_df", @@ -63,6 +62,7 @@ def main(): job.setup_clients(sites) if export_config: + print("Exporting job config...", job_dir) job.export_job(job_dir) else: job.simulator_run(work_dir) diff --git a/examples/advanced/federated-statistics/df_stats/jobs/df_stats/app/config/config_fed_server.json b/examples/advanced/federated-statistics/df_stats/jobs/df_stats/app/config/config_fed_server.json index 58e4b861fb..62c4d3c8c5 100644 --- a/examples/advanced/federated-statistics/df_stats/jobs/df_stats/app/config/config_fed_server.json +++ b/examples/advanced/federated-statistics/df_stats/jobs/df_stats/app/config/config_fed_server.json @@ -19,7 +19,7 @@ "range": [0,120] } }, - "percentile": { + "quantile": { "*": [25, 50, 75] } }, diff --git a/examples/advanced/federated-statistics/df_stats/requirements.txt b/examples/advanced/federated-statistics/df_stats/requirements.txt index c766bd7827..5ae34037be 100644 --- a/examples/advanced/federated-statistics/df_stats/requirements.txt +++ b/examples/advanced/federated-statistics/df_stats/requirements.txt @@ -2,4 +2,4 @@ numpy pandas matplotlib jupyterlab -tdigest + diff --git a/examples/advanced/streaming/src/simple_controller.py b/examples/advanced/streaming/src/simple_controller.py index 206346e8f2..3c2d5a56d9 100644 --- a/examples/advanced/streaming/src/simple_controller.py +++ b/examples/advanced/streaming/src/simple_controller.py @@ -25,7 +25,6 @@ class SimpleController(Controller): - def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): logger.info(f"Entering control loop of {self.__class__.__name__}") engine = fl_ctx.get_engine() diff --git a/examples/advanced/streaming/src/standalone_file_streaming.py b/examples/advanced/streaming/src/standalone_file_streaming.py index 1c4c44b87b..e714cb237c 100644 --- a/examples/advanced/streaming/src/standalone_file_streaming.py +++ b/examples/advanced/streaming/src/standalone_file_streaming.py @@ -27,7 +27,6 @@ class FileSender(FLComponent): - def __init__(self): super().__init__() self.seq = 0 @@ -73,7 +72,6 @@ def _sending_file(self, fl_ctx): class FileReceiver(FLComponent): - def __init__(self): super().__init__() self.done = False diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.0_introduction/introduction.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.0_introduction/introduction.ipynb index 260449d93b..d1454a14d4 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.0_introduction/introduction.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.0_introduction/introduction.ipynb @@ -4,43 +4,33 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Recap: Runing Federated Learning Applications\n", - "\n", - "\n", - "In this chapter, we will explore the process of running federated learning applications. We will start by setting up the environment and preparing the data, followed by training a classifier using PyTorch. We will then convert deep learning models to federated learning, customize server and client logic, and setup track experiments. Finally, we will delve into the job structure and configurations, including running a simulator, and conclude with a recap of the covered topics.\n", + "# Running Federated Learning Applications\n", "\n", + "In this chapter, we will explore the process of running federated learning applications. We will start by setting up the environment and preparing the data, followed by training a classifier using PyTorch. We will then convert deep learning models to federated learning, customize server and client logic, and set up experiment tracking. Finally, we will delve into the job structure and configurations, including running a simulator, and conclude with a recap of the covered topics.\n", "\n", "1. **Running federated learning job**\n", " * [Installation, prepare data](../01.1_running_federated_learning_job/setup.ipynb)\n", - " * [traing classifier with pytorch](../01.1_running_federated_learning_job/runing_pytorch_fl_job.ipynb)\n", - "\n", - "2. **From stand-alone-deep learning to Federated Learning**\n", - "\n", - " * [Convert deep learning with pytorch to federated leraning](../01.2_convert_deep_learning_to_federated_learning/convert_dl_to_fl.ipynb)\n", - "\n", - "\n", - "2. **How to Customize the Federated Algorithms**\n", + " * [Training classifier with PyTorch](../01.1_running_federated_learning_job/running_pytorch_fl_job.ipynb)\n", "\n", - " * [customize server logics](../01.3_customize_server_logics/customize_server_logics.ipynb)\n", + "2. **From stand-alone deep learning to Federated Learning**\n", + " * [Convert deep learning with PyTorch to federated learning](../01.2_convert_deep_learning_to_federated_learning/convert_dl_to_fl.ipynb)\n", "\n", - "4. **How to make adjustments to different traing parameters** \n", + "3. **Sever Side Customization**\n", + " * [customize server logics](../01.3_server_side_customization/customize_server_logics.ipynb)\n", "\n", - " * [customize client logics](../01.4_customize_client_training/customize_client_training.ipynb)\n", + "4. **Client Side Customization** \n", + " * [customize client training](../01.4_client_side_customization/customize_client_training.ipynb)\n", "\n", "5. **Tracking the training metrics** \n", - "\n", - " * [experiment tracking](../01.5_experiment_tracking/experiment_tracking.ipynb )\n", + " * [Experiment tracking](../01.5_experiment_tracking/experiment_tracking.ipynb)\n", "\n", "6. **Job structure and configurations**\n", + " * [Job structure & configuration](../01.6_job_structure_and_configuration/understanding_fl_job.ipynb)\n", "\n", - " * [job structure & configuration ](../01.6_job_structure_and_configuration/01.1.6.1_understanding_fl_job.ipynb)\n", - "\n", - " \n", - "7. [Recap of the covered topics](../01.7_recap/recap.ipynb)\n", - "\n", + "7. **Logging configurations**\n", + " * [Logging](../01.7_logging/logging.ipynb)\n", "\n", - "\n", - "Let's get started with [Installation & data preparation](.././01.1_running_federated_learning_job/setup.ipynb)\n" + "Let's get started with [Installation & data preparation](../01.1_running_federated_learning_job/setup.ipynb)" ] }, { @@ -50,8 +40,14 @@ } ], "metadata": { + "kernelspec": { + "display_name": "nvflare-env", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "name": "python", + "version": "3.8.13" } }, "nbformat": 4, diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/code/fl_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/code/fl_job.py index 5cd4c6de70..15ed34823e 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/code/fl_job.py +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/code/fl_job.py @@ -32,4 +32,4 @@ ) job.to(executor, f"site-{i + 1}") - job.simulator_run(workspace="/tmp/nvflare/jobs/workdir", log_config="./log_config.json") + job.simulator_run(workspace="/tmp/nvflare/jobs/workdir") diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/code/log_config.json b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/code/log_config.json deleted file mode 100644 index 240e9616eb..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/code/log_config.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "version": 1, - "disable_existing_loggers": false, - "formatters": { - "baseFormatter": { - "()": "nvflare.fuel.utils.log_utils.BaseFormatter", - "fmt": "%(asctime)s - %(name)s - %(levelname)s - %(fl_ctx)s - %(message)s" - }, - "colorFormatter": { - "()": "nvflare.fuel.utils.log_utils.ColorFormatter", - "fmt": "%(asctime)s - %(levelname)s - %(message)s", - "datefmt": "%Y-%m-%d %H:%M:%S" - }, - "jsonFormatter": { - "()": "nvflare.fuel.utils.log_utils.JsonFormatter", - "fmt": "%(asctime)s - %(identity)s - %(name)s - %(fullName)s - %(levelname)s - %(fl_ctx)s - %(message)s" - } - }, - "filters": { - "FLFilter": { - "()": "nvflare.fuel.utils.log_utils.LoggerNameFilter", - "logger_names": ["custom", "nvflare.app_common", "nvflare.app_opt"] - } - }, - "handlers": { - "consoleHandler": { - "class": "logging.StreamHandler", - "level": "INFO", - "formatter": "colorFormatter", - "filters": ["FLFilter"], - "stream": "ext://sys.stdout" - }, - "logFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "DEBUG", - "formatter": "baseFormatter", - "filename": "log.txt", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10 - }, - "errorFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "ERROR", - "formatter": "baseFormatter", - "filename": "log_error.txt", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10 - }, - "jsonFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "DEBUG", - "formatter": "jsonFormatter", - "filename": "log.json", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10 - }, - "FLFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "DEBUG", - "formatter": "baseFormatter", - "filters": ["FLFilter"], - "filename": "log_fl.txt", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10, - "delay": true - } - }, - "loggers": { - "root": { - "level": "INFO", - "handlers": ["consoleHandler", "logFileHandler", "errorFileHandler", "jsonFileHandler", "FLFileHandler"] - }, - "nvflare.app_opt.pt.client_api_launcher_executor.PTClientAPILauncherExecutor": { - "level": "ERROR" - }, - "nvflare.app_opt.pt.in_process_client_api_executor.PTInProcessClientAPIExecutor": { - "level": "ERROR" - } - } -} - - - - - - - - - diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/runing_pytorch_fl_job.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/running_pytorch_fl_job.ipynb similarity index 61% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/runing_pytorch_fl_job.ipynb rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/running_pytorch_fl_job.ipynb index 87e84f39c9..990a62f9db 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/runing_pytorch_fl_job.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/running_pytorch_fl_job.ipynb @@ -5,9 +5,9 @@ "id": "7a5c3d67-a6ea-4f59-84d2-effc3ef016e1", "metadata": {}, "source": [ - "# Runing Federated Learning Job with PyTorch\n", + " # Running Federated Learning Job with PyTorch\n", "\n", - "We have installed the NVIDIA FLARE, dependencies, download the data, look at the data split in [previous step](01.1.1_setup.ipynb), now we are going to look at the training. " + "We have installed NVIDIA FLARE and its dependencies, downloaded the data, and looked at the data split in the [previous step](01.1.1_setup.ipynb). Now we are going to look at the training.\n" ] }, { @@ -15,18 +15,22 @@ "id": "eb3f04b0", "metadata": {}, "source": [ - "## Run Federated Learning Training code\n", + " ## Run Federated Learning Training Code\n", "\n", + "The training code essentially consists of three files:\n", + "- `fl_job.py`: the main job flow\n", + "- `client.py`: the client-side training code\n", + "- `network.py`: the network model definition\n", + "\n", + "We use the `FedAvg` algorithm and workflow.\n", "\n", - "The training code essentially consists of `fl_job.py` code, `client.py` the client-side training code, and `nn.py` network model. We use the `FedAvg` algorithm and workflow.\n", "\n", - "```markdown\n", "## Run Federated Learning Training\n", "\n", - "The training code consists of three main scripts: `fl_job.py` for the overall job flow, `client.py` for the client-side training, and `network.py` for the network model. We use the built-in `FedAvg` algorithm for the server side worklow.\n", - "```\n", + "The training code consists of three main scripts: `fl_job.py` for the overall job flow, `client.py` for the client-side training, and `network.py` for the network model. We use the built-in `FedAvg` algorithm for the server-side workflow.\n", + "\n", "\n", - "to run the training in simulator we can simply execute the fl_job.py" + "To run the training in the simulator, we can simply execute `fl_job.py`.\n" ] }, { @@ -62,18 +66,18 @@ "id": "e823b5d4", "metadata": {}, "source": [ - "## 3. Access the logs and results\n", + " ## 3. Access the Logs and Results\n", "\n", - "You can find the running logs and results inside the simulator's workspace:\n", + "You can find the running logs and results inside the simulator's workspace.\n", "\n", - "noticed the \"fl_job.py\", we used the code \n", + "Notice that in `fl_job.py`, we used the code:\n", "\n", - "```\n", + "```python\n", "job.simulator_run(\"/tmp/nvflare/jobs/workdir\")\n", - "\n", "```\n", "\n", - "The \"/tmp/nvflare/jobs/workdir\" is the workspace directory of simulator\n" + "The \"/tmp/nvflare/jobs/workdir\" is the workspace directory of the simulator.\n", + "\n" ] }, { @@ -92,15 +96,21 @@ "id": "d4b62108", "metadata": {}, "source": [ - "We have successfully train a federated image classification model with pytorch. Next we need to take closer look the training codes and job structure. Let's go to [converting deep learning to federated learning](../01.1.2_convert_deep_learning_to_federated_learning/convert_dl_to_fl.ipynb)\n" + "We have successfully trained a federated image classification model with PyTorch. Next, we need to take a closer look at the training code and job structure. Let's go to [converting deep learning to federated learning](../01.2_convert_deep_learning_to_federated_learning/convert_dl_to_fl.ipynb)." ] + }, + { + "cell_type": "markdown", + "id": "4406a33e", + "metadata": {}, + "source": [] } ], "metadata": { "kernelspec": { "display_name": "nvflare_env", "language": "python", - "name": "python3" + "name": "nvflare_env" }, "language_info": { "codemirror_mode": { diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/setup.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/setup.ipynb index bbf9a7e923..08da12882a 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/setup.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.1_running_federated_learning_job/setup.ipynb @@ -5,11 +5,12 @@ "id": "7a5c3d67-a6ea-4f59-84d2-effc3ef016e1", "metadata": {}, "source": [ - "# Setup and Prepare Data\n", + "# Setup and Preparation\n", "\n", - "This example of using [NVIDIA FLARE](https://nvflare.readthedocs.io/en/main/index.html) to train an image classifier using federated averaging ([FedAvg](https://arxiv.org/abs/1602.05629))\n", + "This is an example of using [NVIDIA FLARE](https://nvflare.readthedocs.io/en/main/index.html) to train an image classifier using federated averaging ([FedAvg](https://arxiv.org/abs/1602.05629))\n", "and [PyTorch](https://pytorch.org/) as the deep learning training framework.\n", "\n", + "\n", "We will use the train script [cifar10_fl.py](src/cifar10_fl.py) and network [net.py](src/net.py) from the src directory.\n", "\n", "The dataset will be [CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html) dataset and will load its data within the client train code." @@ -72,7 +73,7 @@ "\n", "The CIFAR10 data will be downloaded to the common location. To make this process easiler, we wrote an simple download program like the followings\n", "\n", - "```\n", + "```python\n", "\n", "import argparse\n", "import torchvision.datasets as datasets\n", @@ -99,15 +100,26 @@ "id": "bcba6293", "metadata": {}, "source": [ - "The program just take a root dataset_path and download the training and test dataset to the given root directory from torchvision dataset. Let run the code. " + " The program just takes a root dataset_path and downloads the training and test datasets to the given root directory from the torchvision dataset. Let's run the code." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "87a13909", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to /tmp/nvflare/data/cifar10/cifar-10-python.tar.gz\n", + "100%|████████████████████████████████████████| 170M/170M [00:05<00:00, 28.6MB/s]\n", + "Extracting /tmp/nvflare/data/cifar10/cifar-10-python.tar.gz to /tmp/nvflare/data/cifar10\n", + "Files already downloaded and verified\n" + ] + } + ], "source": [ "!python3 code/data/download.py" ] @@ -122,10 +134,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "08bbe572", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34m/tmp/nvflare/data/cifar10/cifar-10-batches-py/\u001b[0m\n", + "├── \u001b[00mbatches.meta\u001b[0m\n", + "├── \u001b[00mdata_batch_1\u001b[0m\n", + "├── \u001b[00mdata_batch_2\u001b[0m\n", + "├── \u001b[00mdata_batch_3\u001b[0m\n", + "├── \u001b[00mdata_batch_4\u001b[0m\n", + "├── \u001b[00mdata_batch_5\u001b[0m\n", + "├── \u001b[00mreadme.html\u001b[0m\n", + "└── \u001b[00mtest_batch\u001b[0m\n", + "\n", + "0 directories, 8 files\n" + ] + } + ], "source": [ "!tree /tmp/nvflare/data/cifar10/cifar-10-batches-py/" ] @@ -137,8 +167,8 @@ "source": [ "### Split the data\n", "\n", - "In real-world scenarios, the data will be distributed among different clients/sides. Since we are simulating the real-world data, we need to split the data into different clients/sites. How to split the data, \n", - "depending on the type of problem or type of data. For simplicity, in this example we assume all clients will have the same data for horizontal federated learning cases.\n", + "In real-world scenarios, the data will be distributed among different clients/sites. Since we are simulating real-world data, we need to split the data into different clients/sites. How to split the data\n", + "depends on the type of problem or type of data. For simplicity, in this example we assume all clients will have the same data for horizontal federated learning cases.\n", "Thus we do not do a data split, but rather point all clients to the same data location.\n", "\n", "\n", @@ -148,6 +178,7 @@ "\n", "\n", "\n", + "\n", "\n" ] }, @@ -156,7 +187,7 @@ "id": "316bae55", "metadata": {}, "source": [ - "Next Step, we will start to run training using simulation: [run pytorch federated learning job](../01.1_running_federated_learning_job/runing_pytorch_fl_job.ipynb)\n" + "Next step, we will start to run training using simulation: [run pytorch federated learning job](../01.1_running_federated_learning_job/running_pytorch_fl_job.ipynb)\n" ] }, { @@ -168,9 +199,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "nvflare_env", "language": "python", - "name": "python3" + "name": "nvflare_env" }, "language_info": { "codemirror_mode": { diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.2_convert_deep_learning_to_federated_learning/convert_dl_to_fl.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.2_convert_deep_learning_to_federated_learning/convert_dl_to_fl.ipynb index 208177e0be..8f980292c6 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.2_convert_deep_learning_to_federated_learning/convert_dl_to_fl.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.2_convert_deep_learning_to_federated_learning/convert_dl_to_fl.ipynb @@ -7,9 +7,10 @@ "source": [ "# PyTorch Deep Learning to Federated Learning Conversion\n", "\n", - "One common question frequently heard from data scientists is how do I wrote a federated learning ? If I already have training code already for deep learning? how do I write an federated learning training code for the same problem?\n", "\n", - "In this section, we will look at the classification training code we ran earlier and see how to convert the existing the pytorch training script to federated Learning client training code\n", + "One common question frequently heard from data scientists is \"how do I write federated learning code? If I already have training code for deep learning, how do I write federated learning training code for the same problem?\"\n", + "\n", + "In this section, we will look at the classification training code we ran earlier and see how to convert the existing PyTorch training script to federated learning client training code.\n", "\n", "\n", "## Orginal Deep learning Training Script" @@ -61,21 +62,24 @@ "\n", "we call \n", "\n", - "```\n", + "```python\n", "flare.init()\n", "```\n", "\n", "Once the flare is initialized, we will recieve some system metadata for example\n", - "```\n", + "\n", + "```python\n", + "\n", " sys_info = flare.system_info()\n", " client_name = sys_info[\"site_name\"]\n", "\n", "```\n", "We can get current client's \"identity\". \n", "\n", - "Next we need to extends the trainig beyond local iterations. Image the Federated Learning is like the following for-loop: \n", + "Next we need to extend the training beyond local iterations. Imagine the Federated Learning is like the following for-loop:\n", + "\n", + "```python\n", "\n", - "```\n", "rounds = 5\n", "for current_round in ranage (rounds):\n", " \n", @@ -94,9 +98,10 @@ "For each round: we need to receive and evaluate the global model. \n", "\n", "\n", - "**Step-4** Recive global model \n", + "**Step-4** Receive global model\n", + "\n", + "```python\n", "\n", - "```\n", " input_model = flare.receive()\n", " round=input_model.current_round\n", "\n", @@ -104,23 +109,23 @@ " model.load_state_dict(input_model.params)\n", "```\n", "\n", - "**Step-5** Eveluate Global Model\n", + "**Step-5** Evaluate Global Model\n", + "\n", + "Since the local model is being updated with global model, the training procedure calculates the loss which evaluates the model\n", "\n", - " Since the local model is being updated with global model, the training procedue caclate the loss which evaluate the model \n", "\n", "**Step-6** Send the local trained model back to aggregator\n", "\n", - " we take the newly trained local model parameters as well as metadata, sned it back to aggregator. \n", + "We take the newly trained local model parameters as well as metadata, send it back to aggregator.\n", "\n", - "```\n", + "```python\n", "\n", " output_model = flare.FLModel( params=model.cpu().state_dict(), meta={\"NUM_STEPS_CURRENT_ROUND\": steps},)\n", "\n", " flare.send(output_model)\n", "```\n", "\n", - "\n", - "With above steps, just a few lines of code changes, no code structural changes, we converted the pytorch deep learning code to federated learning with NVIDIA FLARE\n", + "With above steps, just a few lines of code changes, no code structural changes, we converted the PyTorch deep learning code to federated learning with NVIDIA FLARE.\n", "\n", "The complete code can be found at client.py" ] @@ -140,19 +145,19 @@ "id": "7f1824bf", "metadata": {}, "source": [ - "Now, we converted the client pytorch training script to federated learning code. Lets look further to handle multi-task client code\n", + "Now, we converted the client PyTorch training script to federated learning code. Let's look further to handle multi-task client code.\n", "\n", "\n", "## Multi-Task Client Scripts\n", "\n", - "So far, the client only handles traing, regardless what tasks the server issues to the clients. What if there are many tasks ? Client should take different actions based on the different tasks. Also, in previous version, we did not evaluate the global model. We are also to handle all these in this section. \n", + "So far, the client only handles training, regardless of what tasks the server issues to the clients. What if there are many tasks? Client should take different actions based on the different tasks. Also, in the previous version, we did not evaluate the global model. We are going to handle all these in this section.\n", "\n", "\n", - "In Flare's Client API, by detault, we will issue three different tasks: \"train\", \"evaluate\" and \"submit_model\"\n", + "In Flare's Client API, by default, we will issue three different tasks: \"train\", \"evaluate\" and \"submit_model\"\n", "\n", "These three tasks can be checked by \n", "\n", - "```\n", + "```python\n", "\n", "flare.is_train()\n", "\n", @@ -162,7 +167,7 @@ "\n", "```\n", "\n", - "So we need to motify our existing training code to have both training and evaluation logics\n", + "So we need to modify our existing training code to have both training and evaluation logics.\n", "\n", "### Training logics changes\n", "\n", @@ -171,23 +176,25 @@ "\n", "evaluate the local model: \n", "\n", - "```\n", + "```python\n", " # (5.2) evaluation on local trained model to save best model\n", " local_accuracy = evaluate(net.state_dict())\n", "\n", "\n", "```\n", "\n", - "evalute the global model received \n", + "evaluate the global model received:\n", + "\n", + "```python\n", "\n", - "```\n", " # (5.3) evaluate on received model for model selection\n", " accuracy = evaluate(input_model.params)\n", "```\n", "\n", "Then add the global model accuracy into the metrics parameter of the FLModel before send it back to server. \n", "\n", - "```\n", + "```python\n", + "\n", " output_model = flare.FLModel(\n", " params=net.cpu().state_dict(),\n", " metrics={\"accuracy\": accuracy},\n", @@ -201,7 +208,7 @@ ">Note: the evaluate() function will discussed next\n", "\n", "\n", - "```\n", + "```python\n", " \n", "\n", " # (5.2) evaluation on local trained model to save best model\n", @@ -235,7 +242,7 @@ "The return value is accuracy percentage. \n", "\n", "\n", - "```\n", + "```python\n", "\n", " # wraps evaluation logic into a method to re-use for\n", " # evaluation on both trained and received model\n", @@ -270,7 +277,8 @@ "\n", "The overall logics becomes\n", "\n", - "```\n", + "```python\n", + "\n", "if flare.is_training(): \n", " traing and evaluate metrics\n", " send model and merics back\n", @@ -313,7 +321,7 @@ "source": [ "Now, we know how to convert an existing Deep Learning code to Federated Learning training script. We can now explore how to customize the training logics. \n", "\n", - "Please checkout [customize server logics](../01.1.3_customize_server_logics/customize_server_logics.ipynb)\n", + "Please checkout [server side customization](../01.3_server_side_customization/customize_server_logics.ipynb)\n", "\n" ] }, diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/data/download.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/data/download.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/data/download.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/data/download.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/fl_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/fl_job.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/fl_job.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/fl_job.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/requirements.txt b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/requirements.txt similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/requirements.txt rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/requirements.txt diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/client.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/client.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/client.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/client.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/fedavg_v0.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/fedavg_v0.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/fedavg_v0.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/fedavg_v0.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/fedavg_v1.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/fedavg_v1.py similarity index 99% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/fedavg_v1.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/fedavg_v1.py index b0508d33eb..e8d555bea5 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/fedavg_v1.py +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/fedavg_v1.py @@ -23,7 +23,6 @@ class FedAvgV1(BaseFedAvg): - def __init__( self, *args, diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/fedavg_v2.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/fedavg_v2.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/fedavg_v2.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/fedavg_v2.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/network.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/network.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/code/src/network.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/code/src/network.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/customize_server_logics.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/customize_server_logics.ipynb similarity index 85% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/customize_server_logics.ipynb rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/customize_server_logics.ipynb index ed22affd71..6d436f9d1c 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_customize_server_logics/customize_server_logics.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.3_server_side_customization/customize_server_logics.ipynb @@ -8,16 +8,14 @@ "\n", "# Customizing Federated Learning Server logics\n", "\n", + "In previous sections, we were able to run federated PyTorch image classification code with NVIDIA FLARE's built-in FedAvg algorithm.\n", + "What if we want to build our own algorithms or modify the existing algorithm?\n", "\n", - "In previous sections, we are able to run federated pytorch image classification code with NVIDIA FLARE builtin FedAvg algorithm. \n", - "What if we want to build my own algorithms or modify the existing algorithm ? \n", - "\n", - "In the following, using FedAvg as starting point, we like to make a few changes to FedAvg to fit our needs: \n", - "\n", - "* Add early stopping mechanism so that the training could stop instead of waiting to the total numbers of rounds if the criteria is statisfied\n", - "* Instead of rely on the internal best model selection approach, we want to provide our own best model selection\n", - "* Instead of using building persiste component PTFileModelPersistor, we like to have our own save and loading functions\n", + "In the following, using FedAvg as a starting point, we would like to make a few changes to FedAvg to fit our needs:\n", "\n", + "* Add early stopping mechanism so that the training could stop instead of waiting for the total number of rounds if the criteria is satisfied\n", + "* Instead of relying on the internal best model selection approach, we want to provide our own best model selection\n", + "* Instead of using built-in persist component PTFileModelPersistor, we would like to have our own save and loading functions\n", "\n", "In this section, we will go over these changes step-by-step. \n", "\n", @@ -39,9 +37,10 @@ "FedAvg can be written as very simple for-loop. There are several other factors to consider \n", "\n", "* How to send the model to clients?\n", - "* How to receive the response \n", - "* for the model and response, what's the format ? \n", - "* The model and responses and corresponding objects must be serialized, how to series them ? \n", + "* How to receive the responses\n", + "* For the model and responses, what's the format?\n", + "* The model and responses and corresponding objects must be serialized, how to serialize them?\n", + "\n", "\n", "Let's dive into these questions.\n", "\n", @@ -50,7 +49,7 @@ "\n", "FLARE defined a high-level data structure \"FLModel\" that holds the model parameters, metrics and metadata\n", "\n", - "```\n", + "```python\n", "\n", "class ParamsType(str, Enum):\n", " FULL = \"FULL\"\n", @@ -76,12 +75,13 @@ "\n", "#### Serialization \n", "\n", - "Many of the deep learning machine frameworks using python pickle as default serrialization mechanism. There are enough security concerns that FLARE is not using Pickle. NVIDIA FLARE Object Serializer (FOBS) used a [messagePack](https://msgpack.org/index.html)-based serialization approach. \n", + "Many of the deep learning machine frameworks using python pickle as default serialization mechanism. There are enough security concerns that FLARE is not using Pickle. NVIDIA FLARE Object Serializer (FOBS) used a [messagePack](https://msgpack.org/index.html)-based serialization approach. \n", "User needs to register a component ( \"Decomposer\") to serialize/de-serialize certain project to fobs. \n", "\n", "To PyTorch Tensor, we need to register [TensorDecompressor](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_opt/pt/decomposers.py) component at FOBS. \n", "\n", - "```\n", + "```python\n", + "\n", " # Use FOBS for serializing/deserializing PyTorch tensors\n", " fobs.register(TensorDecomposer)\n", "```\n", @@ -90,10 +90,11 @@ "\n", "For high-level API, we can use the followings\n", "\n", - "```\n", + "```python\n", + "\n", " results = self.send_model_and_wait(targets=clients, data=model)\n", "```\n", - "the function send the FLModel to targeted clients and recieve result. This is synchornized methood like scatter and gather. We broadcast the model to all targeted clients and receive results when required clients send back the results. \n", + "the function send the FLModel to targeted clients and recieve result. This is synchronized method like scatter and gather. We broadcast the model to all targeted clients and receive results when required clients send back the results. \n", "\n", "The BasedFedAvg is derived from ModelController which has the communication component, which allows the component to send the model and wait for result. \n", "\n", @@ -129,11 +130,13 @@ "```stop_cond``` is a string to represent the stop condition, its string literal in the format of \" \" (e.g. \"accuracy >= 80\")\n", "\n", "we need to parse this condition so we can compare. To parse this, we leverage FLARE's math_utils\n", - "```\n", + "\n", + "```python\n", "\n", "math_utils.parse_compare_criteria(compare_expr: Optional[str] = None) -> Tuple[str, float, Callable]\n", "\n", "```\n", + "\n", "the return will be\n", "* key,\n", "* target_value,\n", @@ -163,16 +166,17 @@ "source": [ "#### Integrate the early stop condition\n", "\n", - "This should simple, if the condition is satified and simply break out the for-loop\n", + "This should simple, if the condition is satisfied and simply break out the for-loop\n", "\n", - "```\n", + "```python\n", " if self.should_stop(model.metrics, self.stop_condition):\n", " break\n", "```\n", "\n", "and the ```should_stop``` function is defined as followings\n", "\n", - "```\n", + "```python\n", + "\n", "def should_stop(self, metrics: Optional[Dict] = None, stop_condition: Optional[str] = None):\n", " key, target, op_fn = stop_condition\n", " value = metrics.get(key, None)\n", @@ -205,9 +209,10 @@ "source": [ "### Select best model \n", "\n", - "we simply write the following two functions and put into previus code\n", + "we simply write the following two functions and put into previous code\n", + "\n", + "```python\n", "\n", - "```\n", " def select_best_model(self, curr_model: FLModel):\n", " if self.best_model is None:\n", " self.best_model = curr_model\n", @@ -246,7 +251,7 @@ "source": [ "### Customized save and load model functions\n", " \n", - "The ```BaseFedAvg``` class defined ```save_model()``` and ```load_model()``` functions for user to overwrite. \n", + "The ```BaseFedAvg``` class defined ```save_model()``` and ```load_model()``` functions for user to override. \n", "We use torch save and load functions, and save the FLModel metadata separately with the fobs.dumpf and fobs.loadf serialization utilities.\n", "\n", "\n", @@ -313,7 +318,8 @@ "\n", "### Create Fed Job\n", "\n", - "```\n", + "```python\n", + "\n", " n_clients = 5\n", " num_rounds = 2\n", "\n", @@ -399,11 +405,17 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "e096501e", + "cell_type": "markdown", + "id": "63108356", + "metadata": {}, + "source": [ + "Next step, we are going to see how to customize the cilent side logics: [customize client side logics](../01.4_client_side_customization/customize_client_training.ipynb)" + ] + }, + { + "cell_type": "markdown", + "id": "99daa93c", "metadata": {}, - "outputs": [], "source": [] } ], diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/data/download.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/data/download.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/data/download.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/data/download.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/fl_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/fl_job.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/fl_job.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/fl_job.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/requirements.txt b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/requirements.txt similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/requirements.txt rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/requirements.txt diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/src/client.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/src/client.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/src/client.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/src/client.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/src/fedavg.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/src/fedavg.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/src/fedavg.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/src/fedavg.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/src/fl_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/src/fl_job.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/src/fl_job.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/src/fl_job.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/src/network.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/src/network.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/code/src/network.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/code/src/network.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/customize_client_training.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/customize_client_training.ipynb similarity index 75% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/customize_client_training.ipynb rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/customize_client_training.ipynb index 56f1a33a89..9b37e5e50c 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_customize_client_training/customize_client_training.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.4_client_side_customization/customize_client_training.ipynb @@ -1,24 +1,19 @@ { "cells": [ - { - "cell_type": "markdown", - "id": "38561a0b-a072-41ea-b027-290402cf4582", - "metadata": {}, - "source": [ - "# Customize client training scripts for different sites\n" - ] - }, { "cell_type": "markdown", "id": "3f020f8b", "metadata": {}, "source": [ - "The client training script, so far, assume all sides have the same training parameters. In the real-world applications, each site's data will be different, therefore the training parameters such batch size and learning rate will be different.\n", + "# Customize client training scripts for different sites\n", "\n", + "The client training script, so far, assumes all sites have the same training parameters. In real-world applications, each site's data will be different, therefore the training parameters such as batch size and learning rate will be different.\n", "\n", - "In this section, we will show to set different parameters \n", + "In this section, we will show how to set different parameters.\n", + "\n", + "\n", + "```python\n", "\n", - "```\n", " # Add clients\n", "\n", " executor_1 = ScriptRunner(script=train_script, script_args=\"--learning_rate 0.01 --batch_size 12\")\n", @@ -31,10 +26,10 @@ " job.to(executor_3, \"site-3\")\n", "\n", " executor_4 = ScriptRunner(script=train_script, script_args=\"--learning_rate 0.001 --batch_size 6\")\n", - " job.to(executor_3, \"site-4\")\n", + " job.to(executor_4, \"site-4\")\n", " \n", " executor_5 = ScriptRunner(script=train_script, script_args=\"--learning_rate 0.0001 --batch_size 4\")\n", - " job.to(executor_3, \"site-5\")\n", + " job.to(executor_5, \"site-5\")\n", "\n", "```\n", "\n", @@ -93,11 +88,17 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "7966ab3a", + "cell_type": "markdown", + "id": "dbd5d996", + "metadata": {}, + "source": [ + "Next step, we are going to see how to do federated exeperiment trackinig with different experiment tracking systems: [experiment_tracking](../01.5_experiment_tracking/experiment_tracking.ipynb)" + ] + }, + { + "cell_type": "markdown", + "id": "6d7fb83b", "metadata": {}, - "outputs": [], "source": [] } ], diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/01.5.1_experiment_tracking_with_tensorboard/experiment_tracking_tensorboard.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/01.5.1_experiment_tracking_with_tensorboard/experiment_tracking_tensorboard.ipynb index b22d5e90f2..2ea3c3d23b 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/01.5.1_experiment_tracking_with_tensorboard/experiment_tracking_tensorboard.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/01.5.1_experiment_tracking_with_tensorboard/experiment_tracking_tensorboard.ipynb @@ -7,11 +7,12 @@ "source": [ "# Experiment tracking with TensorBoard\n", "\n", - "NVFlare uses `TBAnalyticsReceiver` for experiment tracking on the FL server by default, allowing for .\n", + "NVFlare uses `TBAnalyticsReceiver` for experiment tracking on the FL server by default, enabling experiment tracking.\n", "\n", "## Default in FedAvgJob\n", "\n", - "The FedJob API makes it easy to create job congifurations, and by default the `TBAnalyticsReceiver` for TensorBoard streaming is included. You can specify your own analytics_receiver of type `AnalyticsReceiver` as a parameter if you want, but if left unspecified, `TBAnalyticsReceiver` is configured to be set up in `BaseFedJob` (nvflare/app_opt/pt/job_config/base_fed_job.py). \n", + "The FedJob API makes it easy to create job configurations, and by default the `TBAnalyticsReceiver` for TensorBoard streaming is included. You can specify your own analytics_receiver of type `AnalyticsReceiver` as a parameter if you want, but if left unspecified, `TBAnalyticsReceiver` is configured to be set up in `BaseFedJob` (nvflare/app_opt/pt/job_config/base_fed_job.py).\n", + "\n", "\n", "The `TBAnalyticsReceiver` for TensorBoard streaming receives and records the logs during the experiment by saving them to Tensoboard event files on the FL server. See [this link](https://nvflare.readthedocs.io/en/main/programming_guide/experiment_tracking/experiment_tracking_log_writer.html#tools-sender-logwriter-and-receivers) for more details on the other available AnalyticsReceivers in NVFlare: MLflowReceiver and WandBReceiver." ] @@ -152,7 +153,9 @@ "source": [ "## View tensorboard results\n", "\n", - "In order to see the results, you can use the following command directed to the location of the tensorboard event files (by default the location for the server should be as follows using the default simulator path provided):\n", + "\n", + "In order to see the results, you can use the following command directed to the location of the TensorBoard event files (by default, the location for the server should be as follows using the default simulator path provided):\n", + "\n", "\n", "```commandline\n", "tensorboard --logdir=/tmp/nvflare/jobs/workdir/server/simulate_job/tb_events\n", @@ -164,7 +167,8 @@ "id": "bdd0eb76", "metadata": {}, "source": [ - "Now, we know how experiment tracking can be achieved through metric logging and can be configured to work in a job with an `AnalyticsReceiver`. With this mechanism, we can stream various types of metric data.\n", + "Now we know how experiment tracking can be achieved through metric logging and can be configured to work in a job with an `AnalyticsReceiver`. With this mechanism, we can stream various types of metric data.\n", + "\n", "\n", "For how to use `MLflowReceiver` to set up experiment tracking for MLflow, see [Experiment Tracking with MLflow](../01.5.2_experiment_tracking_with_mlflow/experiment_tracking_mlflow.ipynb)." ] diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/01.5.2_experiment_tracking_with_mlflow/experiment_tracking_mlflow.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/01.5.2_experiment_tracking_with_mlflow/experiment_tracking_mlflow.ipynb index 9e26e8b38f..f106bcf70d 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/01.5.2_experiment_tracking_with_mlflow/experiment_tracking_mlflow.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/01.5.2_experiment_tracking_with_mlflow/experiment_tracking_mlflow.ipynb @@ -171,7 +171,7 @@ "id": "aec950ee", "metadata": {}, "source": [ - "The num_rounds for this job also 20 for more data for a better looking graph. Note that even though [this job](code/src/client_mlflow.py) uses `MLflowWriter`, if we used the [client code](code/src/client.py) with `SummaryWriter`, the resulting data logged to MLflow would be the same since behind the scenes, there is conversion that occurs to translate the event with the log with SummaryWriter to be the equivalent for MLflow." + "The num_rounds for this job is also 20 for more data for a better looking graph. Note that even though [this job](code/src/client_mlflow.py) uses `MLflowWriter`, if we used the [client code](code/src/client.py) with `SummaryWriter`, the resulting data logged to MLflow would be the same since, behind the scenes, there is conversion that occurs to translate the event with the log with SummaryWriter to be the equivalent for MLflow." ] }, { @@ -195,10 +195,16 @@ "id": "bdd0eb76", "metadata": {}, "source": [ - "Now, we know how experiment tracking can be achieved through metric logging and can different types of `AnalyticsReceiver` can be configured to work in a job. With this mechanism, we can stream various types of metric data.\n", + "Now we know how experiment tracking can be achieved through metric logging and how different types of `AnalyticsReceiver` can be configured to work in a job. With this mechanism, we can stream various types of metric data.\n", "\n", - "To continue, please see [Understanding FLARE federated learning Job structure](../../01.6_job_structure_and_configuration/01.1.6.1_understanding_fl_job.ipynb)." + "To continue, please see [Understanding FLARE federated learning Job structure](../../01.6_job_structure_and_configuration/understanding_fl_job.ipynb)" ] + }, + { + "cell_type": "markdown", + "id": "365e2f89", + "metadata": {}, + "source": [] } ], "metadata": { diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/experiment_tracking.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/experiment_tracking.ipynb index e4a865a3b0..bf54d7a70d 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/experiment_tracking.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.5_experiment_tracking/experiment_tracking.ipynb @@ -15,14 +15,14 @@ "\n", "In a federated computing setting, data is distributed across multiple devices or systems, and training is run on each device independently while preserving each client’s data privacy.\n", "\n", - "Assuming a federated system consisting of one server and many clients and the server coordinating the ML training of clients, we can interact with ML experiment tracking tools in two different ways:\n", + "There are two ways to interact with ML experiment tracking tools:\n", "\n", - "![experiment_tracking](img/metrics-streaming-fl-server-clients.png)\n", + "\"experiment_tracking\"\n", "\n", - "- Decentralized tracking (client-side experiment tracking): Each client will directly send the log metrics/parameters to the ML experiment tracking server (like MLflow or Weights and Biases) or local file system (like tensorboard)\n", - "- Centralized tracking (aggregated experiment trackin): Clients will send the log metrics/parameters to the FL server, and the FL server will send the metrics to ML experiment tracking server or local file system\n", + "- Decentralized tracking (client-side experiment tracking): Each client directly sends the log metrics/parameters to the ML experiment tracking server (like MLflow or Weights and Biases) or local file system (like TensorBoard).\n", + "- Centralized tracking (aggregated experiment tracking): Clients send the log metrics/parameters to the FL server, and the FL server sends the metrics to the ML experiment tracking server or local file system.\n", "\n", - "The NVIDIA FLARE job configuration enables you to choose the tracking scenario or system that best fits your needs. When users need to migrate from one experiment tracking system to another, using NVIDIA FLARE, you can modify the job configuration without re-writing the experiment tracking code." + "The NVIDIA FLARE job configuration enables you to choose the tracking scenario or system that best fits your needs. When users need to migrate from one experiment tracking system to another using NVIDIA FLARE, you can modify the job configuration without rewriting the experiment tracking code.\n" ] }, { @@ -30,7 +30,7 @@ "id": "47399ea6", "metadata": {}, "source": [ - "The `nvflare.client.tracking` API enables you to flexibly redirect your logging metrics to any destination. The use of MLflow, Weights & Biases, or TensorBoard syntax does not matter here as you can stream the collected metrics to any supported experiment tracking system. Choosing to use MLflowWriter, WandBWriter, or TBWriter is based on your existing code and requirements.\n", + "The `nvflare.client.tracking` API enables you to flexibly redirect your logging metrics to any destination. The syntax you use (MLflow, Weights & Biases, or TensorBoard) doesn't matter, as you can stream the collected metrics to any supported experiment tracking system. The choice between MLflowWriter, WandBWriter, or TBWriter depends on your existing code and requirements.\n", "\n", "- MLflowWriter uses the MLflow API operation log_metric.\n", "- TBWriter uses the TensorBoard SummaryWriter operation add_scalar.\n", diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.6_job_structure_and_configuration/01.1.6.1_understanding_fl_job.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.6_job_structure_and_configuration/understanding_fl_job.ipynb similarity index 88% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.6_job_structure_and_configuration/01.1.6.1_understanding_fl_job.ipynb rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.6_job_structure_and_configuration/understanding_fl_job.ipynb index ae93a347f5..1ce1f4b440 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.6_job_structure_and_configuration/01.1.6.1_understanding_fl_job.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.6_job_structure_and_configuration/understanding_fl_job.ipynb @@ -13,17 +13,16 @@ "id": "ab47f2c5", "metadata": {}, "source": [ - "## What is NVFlare Job ? \n", + " ## What is an NVFlare Job?\n", "\n", - "\n", - "NVFlare Job refers to a job configuration used within the NVIDIA FLARE framework. \n", + "NVFlare Job refers to a job configuration used within the NVIDIA FLARE framework.\n", "\n", "In NVFlare, a job is a unit of work that defines the specific tasks to be executed during a federated learning process. It encapsulates all necessary configurations, scripts, and resources needed to run an FL task, such as training, validation, or evaluation, across multiple participants in a federated system.\n", "\n", - "A job may have many apps. Each app consists code specific for the site (client site or server site) as well as configurations. \n", + "A job may have many apps. Each app consists of code specific for the site (client site or server site) as well as configurations.\n", "\n", + "In this section, we will take a look at the Job structure as well as the Job API (aka job construction API).\n", "\n", - "In this section, we will take a look at the Job structure as well as Job API ( akak job construction API). \n", "\n", "## Job creation API\n", "\n", @@ -183,7 +182,8 @@ "id": "f475fa49", "metadata": {}, "source": [ - "The job name \"FedAvg\" is folder structure, with each folder representing one app at one site. \n", + "The job name \"FedAvg\" is a folder structure, with each folder representing one app at one site.\n", + "\n", "\n", "* **\"app_server\"**: is the name for the server app\n", "\n", @@ -199,7 +199,7 @@ "\n", "* meta.json gives additional information related to the each app's deployment. \n", "\n", - "```\n", + "```json\n", "{\n", " \"name\": \"fedavg\",\n", " \"resource_spec\": {},\n", @@ -235,7 +235,8 @@ "source": [ "A simplifed format of job structure can also be used when the client code and configuration is the same for all sites\n", "\n", - "```\n", + "```shell\n", + "\n", "/tmp/nvflare/jobs/job_config/fedavg\n", "├── app_server\n", "│ ├── config\n", @@ -258,7 +259,7 @@ "meta.json needs to be \n", "\n", "\n", - "```\n", + "```json\n", "{\n", " \"name\": \"fedavg\",\n", " \"resource_spec\": {},\n", @@ -277,7 +278,7 @@ "\n", "If we don't mind deploy all code to all sites, we can change the job config into the followings\n", "\n", - "A simplifed format of job structure can also be used when the client code and configuration is the same for all sites\n", + " A simplified format of job structure can also be used when the client code and configuration are the same for all sites\n", "\n", "```\n", "/tmp/nvflare/jobs/job_config/fedavg\n", @@ -297,7 +298,7 @@ "meta.json needs to be \n", "\n", "\n", - "```\n", + "```json\n", "{\n", " \"name\": \"fedavg\",\n", " \"resource_spec\": {},\n", @@ -317,9 +318,10 @@ "source": [ "## Job Configuration\n", "\n", - "We have convered a lot of ground so far. You could stop here, and move to the next chapter of the training materials. \n", "\n", - "But if you like to futher understand how NVIDIA FLARE works, you might want to go through this section: Job Configuration. \n" + "We have covered a lot of ground so far. You could stop here and move to the next chapter of the training materials.\n", + "\n", + "But if you would like to further understand how NVIDIA FLARE works, you might want to go through this section: Job Configuration.\n" ] }, { @@ -379,7 +381,7 @@ "id": "a75c80c4", "metadata": {}, "source": [ - "The server configuration is a json file descripe the workflows. In our case, we defined one workflow, whci has a controller using our defined FedAvg class. \n", + "The server configuration is a JSON file describing the workflows. In our case, we defined one workflow, which has a controller using our defined FedAvg class.\n", "\n", "\n", ">Note: The configuration pattern is like the followings\n", @@ -417,8 +419,7 @@ "id": "d9753aeb", "metadata": {}, "source": [ - "the configuration is simular, it defines an array of \"executors\", a builtin ```PTInProcessClientAPIExecutor``` is used, \n", - "which takes the training script client.py and its corresponding arguments as input. \n", + "The configuration is similar; it defines an array of \"executors\". A built-in `PTInProcessClientAPIExecutor` is used, which takes the training script client.py and its corresponding arguments as input. \n", "\n", "\n", "```\n", @@ -472,8 +473,14 @@ "id": "e8914e76", "metadata": {}, "source": [ - "Hope you have a good standing of working with NVIDIA FLARE job so far. Let's move on to other chapters. " + "Hope you now have a good understanding of working with NVIDIA FLARE jobs. Before we move on to other chapters, let's logging configuration to make it easier to debug in case of errors. [Logging Configuration](../01.7_logging/logging.ipynb)" ] + }, + { + "cell_type": "markdown", + "id": "33630af4", + "metadata": {}, + "source": [] } ], "metadata": { diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.7_logging/logging.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.7_logging/logging.ipynb new file mode 100644 index 0000000000..fa63f128bf --- /dev/null +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.7_logging/logging.ipynb @@ -0,0 +1,148 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NVIDIA FLARE Logging\n", + "\n", + "Before we finish chapter one, we would like to discuss logging. This will help us with debugging in the following chapters.\n", + "\n", + "The detailed logging configuration can be found in the [NVFlare Documentation](https://nvflare.readthedocs.io/en/main/user_guide/configurations/logging_configuration.html)\n", + "\n", + "as well as in the [NVFlare Logging Tutorial](https://github.com/NVIDIA/NVFlare/blob/main/examples/tutorials/logging.ipynb)\n", + "\n", + "Here are few key features of the logging in NVFlare:\n", + "\n", + "## Structured logging:\n", + "\n", + "When defining new loggers, we provide several functions to help adhere to the FLARE package logger hierarchy. For example, say we have the following module at my_package.my_module:\n", + "\n", + "get_obj_logger for classes. Ex:\n", + "\n", + "```python\n", + "\n", + " class MyClass:\n", + " def __init__(self):\n", + " self.logger = get_obj_logger(self) # my_package.my_module.MyClass\n", + "```\n", + "\n", + "get_script_logger for scripts (if not in a package, default to custom.). Ex:\n", + "\n", + "```python\n", + " if __name__ == \"__main__\":\n", + " logger = get_script_logger() # my_package.my_module\n", + "``` \n", + "get_module_logger for modules. Ex:\n", + "\n", + "```python\n", + " def my_function():\n", + " logger = get_module_logger(name=\"my_function\") # my_package.my_module.my_function\n", + "```\n", + "\n", + "If you use these functions to create your loggers, you will have a better hierarchy and the logging configuration can be configured in a structured way.\n", + "\n", + "For example, you can enable a module logging level, which can define all the sub-modules' logging levels if not defined.\n", + "\n", + "## Multiple logging formats:\n", + "\n", + "NVFlare supports and creates multiple logging formats, including JSON and txt formats. The JSON format is more useful for integrating with monitoring systems.\n", + "\n", + "log.txt:\n", + "The logFileHandler uses the baseFormatter to write all logs to log.txt. This is the default log that we see in the console.\n", + "\n", + "log.json:\n", + "The jsonFileHandler uses the jsonFormatter to write JSON formatted logs to log.json. This is useful for leveraging structured logs (i.e., with a 3rd party observability package).\n", + "\n", + "log_error.txt:\n", + "The errorFileHandler uses the baseFormatter and level \"ERROR\" to write error level logs to log_error.txt. This allows users to easily see when errors are logged.\n", + "\n", + "log_fl.txt:\n", + "The FLFileHandler uses the baseFormatter and FLFilter (uses LoggerNameFilter allowing certain logger names) to write FL training and custom logs to log_fl.txt. This removes the system and communication related logs and clearly shows logs related to FL training.\n", + "\n", + "## Logging mode for simulation\n", + "\n", + "We define a few special log configurations for simulation. This will reduce the amount of log information seen by data scientists, so they can focus on the training logs.\n", + "\n", + "The three logging modes are:\n", + "\n", + "* **concise**: filters out server and client process logs, only contains the application logs\n", + "* **full**: is the full log\n", + "* **verbose**: is the debug level of full log\n", + "\n", + "The simulator defaults to ```concise``` mode, which is the most useful for data scientists to see the training logs.\n", + "\n", + "\n", + "## Dynamic Logging Configuration Commands\n", + "\n", + "\n", + "When running the FLARE system (POC mode or production mode), in many cases, we need to change the logging level or configuration dynamically without stopping the system. Dynamic logging configuration provides these capabilities.\n", + "\n", + "There are two sets of logs: the site logs and job logs. The current site log configuration will be used for the site logs as well as the log configuration of any new job started on that site. \n", + " \n", + "We provide two admin commands to enable users to dynamically configure the site or job level logging when running the FLARE system. Note these command effects will last until reconfiguration or as long as the corresponding site or job is running. However these commands do not overwrite the log configuration file in the workspace. The previous the log configuration file can be reloaded using “reload”.\n", + "\n", + " \n", + "\n", + "here are more examples: \n", + "\n", + "```python\n", + "\n", + "configure_site_log server debug\n", + "configure_site_log client site-1 debug\n", + "configure_site_log all info\n", + "\n", + "configure_job_log server debug\n", + "configure_job_log client site-1 debug\n", + "configure_job_log all info\n", + "configure_job_log all //custom_log_config.json\n", + "```\n", + "\n", + "The ```configure_site_log``` is the FLARE Console command used to configure the site log configuration. \n", + "The ```configure_job_log``` is the FLARE Console command used to configure job log configuration. \n", + "\n", + "\n", + "\n", + "## Customizing logging\n", + "You can always customize logging by adding or removing filters, formats and profile your own logging configuration for simulation, job and system. We are not going to cover this in this tutorial.\n", + "\n", + "## FLARE Job Simulator Run Logging\n", + "\n", + "Since the Flare Job API uses the simulator to run, it defaults to concise mode. If you want to use different log configure, you can use the following command:\n", + "\n", + "\n", + "```python\n", + "\n", + "job.simulator_run(job_config_dir, log_config=\"full\")\n", + "job.simulator_run(job_config_dir, log_config=\"verbose\")\n", + "job.simulator_run(job_config_dir, log_config=\"concise\")\n", + "job.simulator_run(job_config_dir, log_config=\"/path/to/log_config.json)\n", + "\n", + "```\n", + "\n", + "Now that we have briefly introduced the logging configuration, let's wrap up this chapter: [wrap up](..//01.8_recap/recap.ipynb) \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.7_recap/recap.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.7_recap/recap.ipynb deleted file mode 100644 index 640b47c170..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.7_recap/recap.ipynb +++ /dev/null @@ -1,82 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7b152728-3366-4432-adb1-29aa3051dc22", - "metadata": {}, - "source": [ - "# Summary of Chapter 1\n", - "\n", - "We cover a lot of materials in Chapter 1. We guide you through the process of running federated learning applications. Here is an overview of the key contents:\n", - "\n", - "1. **Running Federated Learning Job**\n", - " - **Installation and Data Preparation**: Instructions for setting up the environment and preparing the data.\n", - " - [setup.ipynb](../01.1_running_federated_learning_job/setup.ipynb)\n", - " - **Training Classifier with PyTorch**: Steps to train a classifier using PyTorch in a federated learning setup.\n", - " - [runing_pytorch_fl_job.ipynb](../01.1_running_federated_learning_job/runing_pytorch_fl_job.ipynb)\n", - "\n", - "2. **From Stand-Alone Deep Learning to Federated Learning**\n", - " - **Conversion to Federated Learning**: Guide on converting deep learning models with PyTorch to federated learning.\n", - " - [convert_dl_to_fl.ipynb](../01.2_convert_deep_learning_to_federated_learning/convert_dl_to_fl.ipynb)\n", - "\n", - "3. **Customizing the Federated Algorithms**\n", - " - **Server Logic Customization**: Techniques to customize server logic for specific federated learning needs, we built an our own fed avg algorithms with best model seleciton, model saving and loading, as well as early stopping. \n", - " - [customize_server_logics.ipynb](../01.3_customize_server_logics/customize_server_logics.ipynb)\n", - "\n", - "4. **Adjusting Training Parameters**\n", - " - **Client Logic Customization**: Methods to customize client logic to optimize training parameters. Here we show how to customize the training for each site. \n", - " - [customize_client_training.ipynb](../01.4_customize_client_training/customize_client_training.ipynb)\n", - "\n", - "5. **Tracking Training Metrics**\n", - " - **Experiment Tracking**: Tools and methods to track experiments and monitor training metrics effectively.\n", - " - [experiment_tracking.ipynb](../01.5_experiment_tracking/experiment_tracking.ipynb)\n", - "\n", - "6. **Job Structure and Configurations**\n", - " - **Understanding Job Structure and Configuration**: Detailed explanation of the job structure and configurations necessary for running federated learning jobs.\n", - " - [01.1.6.1_understanding_fl_job.ipynb](../01.6_job_structure_and_configuration/01.1.6.1_understanding_fl_job.ipynb)\n", - "\n", - "7. **Recap of Covered Topics**\n", - " - **Summary and Recap**: A recap of the topics covered in the previous sections.\n", - " - [recap.ipynb](../01.7_recap/recap.ipynb)\n", - "\n", - "Each section is designed to provide comprehensive guidance and practical examples to help you implement and customize federated learning in your applications. For detailed instructions and examples, refer to the respective notebooks linked in each section.\n", - "\n", - "\n", - "Now let's move on to the [Chapter 2](../../chapter-2_develop_federated_learning_applications/02.0_introduction/introduction.ipynb\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "4f2e3cb3-e61f-45e9-8dad-ad55ebb3641a", - "metadata": {}, - "source": [ - " \n", - "\n", - "\n", - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nvflare_example", - "language": "python", - "name": "nvflare_example" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.8_recap/recap.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.8_recap/recap.ipynb new file mode 100644 index 0000000000..0eb75a8a34 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-1_running_federated_learning_applications/01.8_recap/recap.ipynb @@ -0,0 +1,70 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7b152728-3366-4432-adb1-29aa3051dc22", + "metadata": {}, + "source": [ + "# Recap: Chapter 1 Summary\n", + "\n", + "Throughout this chapter, we've learned several crucial concepts and practical skills in federated learning. Here are the essential takeaways:\n", + "\n", + "1. **Federated Learning Fundamentals**\n", + " - FL enables collaborative model training while keeping data at source\n", + " - Basic FL workflow: client training → model aggregation → model broadcast → repeat\n", + " - NVIDIA FLARE provides a robust framework for implementing FL systems\n", + "\n", + "2. **Converting Traditional to Federated Learning**\n", + " - Most PyTorch models can be adapted for federated learning\n", + " - Key modifications needed:\n", + " - Separating training logic from model loading. The model can be received and sent back to the server, while keeping the training logic mostly the same.\n", + "\n", + "3. **Customization Capabilities**\n", + " - Server-side customization:\n", + " - Implementing custom aggregation strategies\n", + " - Model selection and persistence\n", + " - Early stopping mechanisms\n", + " - Client-side customization:\n", + " - Local training optimization\n", + " - Site-specific parameters\n", + " - Custom data handling\n", + "\n", + "These fundamentals prepare you for more advanced FL concepts and implementations in the following chapters.\n", + "\n", + "Now let's move on to [Chapter 2](../../chapter-2_develop_federated_learning_applications/02.0_introduction/introduction.ipynb)" + ] + }, + { + "cell_type": "markdown", + "id": "4f2e3cb3-e61f-45e9-8dad-ad55ebb3641a", + "metadata": {}, + "source": [ + " \n", + "\n", + "\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nvflare_example", + "language": "python", + "name": "nvflare_example" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.0_introduction/introduction.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.0_introduction/introduction.ipynb index 85352f1462..bcd503a12b 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.0_introduction/introduction.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.0_introduction/introduction.ipynb @@ -5,36 +5,64 @@ "id": "ed32af6f", "metadata": {}, "source": [ - "# Introduction: Develop Federated Learning Applications\n", + "# Develop Federated Learning Applications\n", "\n", - "In this chapter, we will explore the process of developing federated learning applications. We will start by exploring federated statistics and getting different visualizations from the data. We will then convert PyTorch Lightning code for use with NVFlare. We will then examine machine learning algorithms and look at how to convert logistics regression, kmeans, and survival analysis for use with federated learning. Finally, we will look into the Client API and conclude with a recap of the covered topics.\n", + "In this chapter, we will explore the process of developing federated learning applications. \n", + "We will cover the following topics:\n", "\n", + "* Federated Analytics\n", + " * Discover the federated statistics of local and global data\n", + " * Visualize the federated statistics\n", + "\n", + "* Convert deep learning code to Federated Learning code\n", + "\n", + " We already covered converting PyTorch code to Federated Learning code in the previous chapter. Now we will cover: \n", + " * Converting PyTorch Lightning code to Federated Learning code\n", + " * Converting TensorFlow code to Federated Learning code\n", + " \n", + "\n", + "* Convert machine learning code to Federated Learning code\n", + " * Convert logistic regression to Federated Learning code\n", + " * Convert k-means to Federated Learning code\n", + " * Convert survival analysis to Federated Learning code\n", + "\n", + "* Client API\n", + " Converting ML/DL to federated learning leverages the client API to interact with the federated learning system. We will explore the client API in detail in this chapter.\n", + " \n", "\n", "2.1. **Federated Statistics**\n", - " * [Federated Statistics with image data](../02.1_federated_statistics/0federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb)\n", + "\n", " * [Federated Statistics with tabular data](../02.1_federated_statistics/federated_statistics_with_tabular_data/federated_statistics_with_tabular_data.ipynb)\n", "\n", - "2.2. **Convert PyTorch Lightning to Federated Learning**\n", + " * [Federated Statistics with image data](../02.1_federated_statistics/federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb)\n", "\n", - " * [Convert Torch Lightning to FL](../02.2_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb)\n", + "2.2. **Introduction to Client API** \n", "\n", + " * [Client API](../02.2_client_api/client_api.ipynb)\n", "\n", - "2.3. **How to Convert Machine Learning Algorithms to Federated Algorithms**\n", + "2.3. **Convert PyTorch Lightning to Federated Learning**\n", "\n", - " * [Convert Logistics Regression to federated learning](../02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_Logistics_regression_to_federated_learning/convert_lr_to_fl.ipynb)\n", - " * [Convert KMeans to federated learning](../02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb)\n", - " * [Convert Survival Analysis to federated learning](../02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/convert_survival_analysis_to_fl.ipynb)\n", + " * [Convert PyTorch Lightning to Federated Learning](../02.3_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb)\n", "\n", - "2.4. **Client API** \n", + "2.4. **Convert Machine Learning to Federated Learning**\n", "\n", - " * [NVFlare Client API](../02.4_client_api/Client_api.ipynb)\n", + " * [Convert Logistic Regression to Federated Learning](../02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb)\n", "\n", - "2.5. [Recap of the covered topics](../02.5_recap/recap.ipynb)\n", + " * [Convert K-means to Federated Learning](../02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb)\n", "\n", + " * [Convert Survival Analysis to Federated Learning](../02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/convert_survival_analysis_to_fl.ipynb)\n", + "\n", + "2.5. [Recap of the covered topics](../02.5_recap/recap.ipynb)\n", "\n", "\n", - "Let's get started with [Federated Statistics](../02.1_federated_statistics/0federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb)\n" + "Let's get started with [Federated Statistics](../02.1_federated_statistics/federated_statistics_introduction.ipynb)" ] + }, + { + "cell_type": "markdown", + "id": "abde95b6", + "metadata": {}, + "source": [] } ], "metadata": { diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_introduction.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_introduction.ipynb index 5fcc41dfed..ad2066eb2f 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_introduction.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_introduction.ipynb @@ -4,18 +4,227 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Federated Statistics Introduction\n", + "# Federated Statistics\n", "\n", - "In a federated learning setting, because data is private at each site and we need to ensure data privacy, there are many considerations to take into account when trying to gather statistics on the data. We provide two examples, one for image data and one for tabular data:\n", + "Before training the model, data scientists need to understand the data distribution and data quality of the data at different sites. As we can't access the raw data, we need to perform federated Analytics to get the statistics of the data at global level. That's where Federated Statistics comes in.\n", "\n", - " * [Federated Statistics with image data](./federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb) shows how to compute local and global image statistics with the consideration that data is private at each of the client sites.\n", - " * [Federated Statistics with tabular data](./federated_statistics_with_tabular_data/federated_statistics_with_tabular_data.ipynb) demonstrates how to create federated statistics for data that can be represented as Pandas DataFrames." + "\n", + "## Objective\n", + "NVIDIA FLARE will provide built-in federated statistics operators (controllers and executors) that \n", + "can generate global statistics based on local client side statistics.\n", + "\n", + "At each client site, we could have one or more datasets (such as \"train\" and \"test\" datasets); each dataset may have many \n", + "features. For each feature in the dataset, we will calculate the statistics and then combine them to produce \n", + "global statistics for all the numeric features. The output would be complete statistics for all datasets in clients and global. \n", + "\n", + "The statistics here are commonly used statistics: count, sum, mean, std_dev and histogram for the numerical features.\n", + "The max, min are not included as it might violate the client's data privacy. The mean will be calculated with count and sum. \n", + "\n", + "The quantile can also be included, but requires additional dependency to be installed. We are not going to include it in the default installation.\n", + "\n", + "\n", + "A client will only need to implement the selected methods of \"Statistics\" class from statistics_spec.\n", + "\n", + "The result will be statistics for all features of all datasets at all sites as well as global aggregates. \n", + "The result could be visualized via the visualization utility in the notebook. \n", + "\n", + "\n", + "## Assumptions\n", + " \n", + "Assume that clients will provide the following: \n", + " * users need to provide target statistics such as count, histogram etc. of targeted statistics features\n", + " * users need to provide the local statistics for the **target statistics** (by implementing the statistic_spec), not all statistics. For example, if the target statistics does not include histogram, then users will not need to provide the histogram calculation function.\n", + " * users need to provide the datasets and dataset features (feature name, data type)\n", + " * Note: count is always required as we use count to enforce data privacy policy\n", + " \n", + "We only support **numerical features**, not categorical features. But users can return all types of features;\n", + "the non-numerical features will be removed.\n", + "\n", + "\n", + "\n", + "## Statistics\n", + "\n", + " Federated statistics includes numerics statistics measures for \n", + " * count\n", + " * mean \n", + " * sum\n", + " * std_dev\n", + " * histogram \n", + " * quantile\n", + " \n", + " We did not include min, max value to avoid data privacy concern. \n", + "\n", + "\n", + "#### Quantile\n", + "\n", + "Quantile statistics refers to statistical measures that divide a probability distribution or dataset into intervals with equal probabilities or proportions. Quantiles help summarize the distribution of data by providing key points that indicate how values are spread.\n", + "\n", + "##### Key Quantiles:\n", + "1. Median (50th percentile): The middle value of a dataset, dividing it into two equal halves.\n", + "2. Quartiles (25th, 50th, 75th percentiles): Divide the data into four equal parts:\n", + "* Q1 (25th percentile): Lower quartile, below which 25% of the data falls.\n", + "* Q2 (50th percentile): Median.\n", + "* Q3 (75th percentile): Upper quartile, below which 75% of the data falls.\n", + "3. Deciles (10th, 20th, ..., 90th percentiles): Divide the data into ten equal parts.\n", + "4. Percentiles (1st, 2nd, ..., 99th): Divide the data into 100 equal parts.\n", + "\n", + "##### Usage of Quantiles:\n", + "* Descriptive Statistics: Summarizes the spread of data.\n", + "* Outlier Detection: Helps identify extreme values.\n", + "* Machine Learning: Used in feature engineering, normalization, and decision tree algorithms.\n", + "* Risk Analysis: Used in finance (e.g., Value at Risk, VaR).\n", + "\n", + " \n", + "## Privacy Policy and Privacy Filters\n", + "\n", + "NVFLARE provide data privacy protection through privacy filters [privacy-management](https://nvflare.readthedocs.io/en/main/user_guide/security/site_policy_management.html#privacy-management)\n", + "Each site can have its own privacy policy. \n", + "\n", + "### Local privacy policy\n", + "\n", + "privacy.json provides local site specific privacy policy.\n", + "The policy is likely setup by the company and implemented by organization admin\n", + "for the project. For different type of scope or categories, there are might be different types of policies. \n", + "\n", + "### Privacy configuration\n", + "\n", + "The NVFLARE privacy configuration consists of set of task data filters and task result filters\n", + "* The task data filter applies before client executor executes;\n", + "* The task result filter is applied after the client executor executes and before the data is sent to the server;\n", + "* Both the data filter and result filter are grouped by scope.\n", + "\n", + "Each job will need to have a privacy scope. If not specified, the default scope will be used. If default scope is not\n", + "defined and job doesn't specify the privacy scope, the job deployment will fail, and job will not executed\n", + "\n", + "### Privacy Policy Instrumentation \n", + "\n", + "There are different ways to set privacy filters depending on the use cases\n", + "\n", + "\n", + "#### Set Privacy Policy as researcher\n", + "\n", + "You can specify the \"task_result_filters\" in config_fed_client.json to specify\n", + "the privacy control. This is useful when you develop these filters\n", + "\n", + "#### Setup site privacy policy as org admin\n", + "\n", + "Once the company decides to instrument certain privacy policy independent of individual\n", + "job, one can copy the local directory privacy.json content to clients' local privacy.json ( merge not overwrite).\n", + "in this example, since we only has one app, we can simply copy the private.json from local directory to\n", + "\n", + "* site-1/local/privacy.json\n", + "* site-2/local/privacy.json\n", + "\n", + "We need to remove the same filters from the job definition in config_fed_client.json\n", + "by simply set the \"task_result_filters\" to empty list to avoid **double filtering**\n", + "```\n", + "\"task_result_filters\": []\n", + "```\n", + "#### Job filter vs filters in private.json\n", + "\n", + "Privacy filters are defined within a privacy scope.\n", + "If a job's privacy scope is defined or has default scope, then the scope's filters (if any) are applied\n", + "before the job-specified filters (if any). This rule is enforced during task execution time.\n", + "\n", + "With such rules, if we have both task result filters and privacy scoped filters, we need to understand\n", + "that the privacy filters will be applied first, then the job filters. The filters will be applied by their specified orders\n", + "\n", + "### Statistics Privacy Filters\n", + "Statistics privacy filters are task result filters. We already built one for Statistics. \n", + "\n", + "```\n", + "StatisticsPrivacyFilter\n", + "```\n", + "The StatisticsPrivacyFilter consists of several `StatisticsPrivacyCleanser`s focused on the statistics sent\n", + "from client to server.\n", + "\n", + "`StatisticsPrivacyCleanser` can be considered as an interceptor before the results delivered to server.\n", + "Currently, we use three `StatisticsPrivacyCleanser`s to guard the data privacy. The reason we built \n", + "`StatisticsPrivacyCleanser` instead of separate filters is to avoid repeated data de-serialization.\n", + "\n", + "#### MinCountCleanser:\n", + "Check against the number of count returned from client for each dataset and each feature.\n", + "\n", + "If the min_count is not satisfied, there is potential risk of reveal client's real data. Then remove that feature's statistics\n", + "from the result for this client.\n", + "\n", + "#### HistogramBinsCleanser:\n", + "For histogram calculations, number of bins can't be too large compare to count. if the bins = count, then\n", + "we also reveal the real data. This check to make sure that the number of bins be less than X percent of the count.\n", + "X = max_bins_percent in percentage, for 10 is for 10%\n", + "if the number of bins for the histogram is not satisfy this specified condition, the resulting histogram will be removed\n", + "from statistics before sending to server.\n", + "\n", + "#### AddNoiseToMinMax\n", + "For histogram calculations, if the feature's histogram bin's range is not specified, we will need to use local data's min\n", + "and max values to calculate the global min/max values, then use the global min, max values as the bin range for histogram\n", + "calculation. But send the server the local min, max values will reveal client's real data.\n", + "To protect data privacy, we add noise to the local min/max values.\n", + "\n", + "Min/max random is used to generate random noise between (min_noise_level and max_noise_level).\n", + "for example, the random noise is to be within (0.1 and 0.3),i.e. 10% to 30% level. These noise\n", + "will make local min values smaller than the true local min values, and max values larger than\n", + "the true local max values. As result, the estimate global max and min values (i.e. with noise)\n", + "are still bounded the true global min/max values, in such that\n", + "```\n", + "est. global min value <\n", + " true global min value <\n", + " client's min value <\n", + " client's max value <\n", + " true global max <\n", + " est. global max value\n", + "```" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Summary\n", + "\n", + "We provided federated statistics operators that can easily aggregate and visualize the local statistics for\n", + "different data site and features. We hope this feature will make it easier to perform federated data analysis. \n", + "\n", + "We provide several examples to demonstrate how should the operators be used. \n", + "\n", + "The main steps are \n", + "* provide server side configuration to specify target statistics and their configurations and output location\n", + "* implement the local statistics generator (statistics_spec)\n", + "* provide client side configuration to specify data input location\n", + " \n", + "\n", + "Now lets dive into the examples\n", + "\n", + "\n", + "* [Federated Statistics with tabular data](./federated_statistics_with_tabular_data/federated_statistics_with_tabular_data.ipynb) demonstrates how to create federated statistics for data that can be represented as Pandas DataFrames.\n", + "* [Federated Statistics with image data](./federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb) shows how to compute local and global image statistics with the consideration that data is private at each of the client sites.\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] } ], "metadata": { + "kernelspec": { + "display_name": "nvflare-env", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.2" } }, "nbformat": 4, diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/data/download_and_unzip_data.sh b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/data/download_and_unzip_data.sh deleted file mode 100755 index a50bb1c504..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/data/download_and_unzip_data.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -DATASET_PATH="/tmp/nvflare/image_stats/data" - -if [ ! -d $DATASET_PATH ]; then - mkdir -p $DATASET_PATH -fi - -source_url="$1" -echo "download url = ${source_url}" -if [ -n "${source_url}" ]; then - if [ ! -f "${DATASET_PATH}/COVID-19_Radiography_Dataset.zip" ]; then - wget -O "${DATASET_PATH}/COVID-19_Radiography_Dataset.zip" "${source_url}" - else - echo "zip file exists." - fi - if [ ! -d "${DATASET_PATH}/COVID-19_Radiography_Dataset" ]; then - unzip -d $DATASET_PATH "${DATASET_PATH}/COVID-19_Radiography_Dataset.zip" - else - echo "image files exist." - fi -else - echo "empty URL, nothing downloaded, you need to provide real URL to download" -fi \ No newline at end of file diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/data/prepare_data.sh b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/data/prepare_data.sh new file mode 100755 index 0000000000..c7f677ad3e --- /dev/null +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/data/prepare_data.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +INPUT_DATASET_PATH="/tmp/nvflare/image_stats/data" +OUTPUT_DATASET_PATH="/tmp/nvflare/image_stats/data" + +python3 "${SCRIPT_DIR}"/utils/prepare_data.py --input_dir "${INPUT_DATASET_PATH}" --output_dir "${OUTPUT_DATASET_PATH}" \ No newline at end of file diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/utils/prepare_data.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/data/utils/prepare_data.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/utils/prepare_data.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/data/utils/prepare_data.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/demo/visualization.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/demo/visualization.ipynb new file mode 100644 index 0000000000..965bda266e --- /dev/null +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/demo/visualization.ipynb @@ -0,0 +1,258 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3f851980", + "metadata": {}, + "source": [ + "# NVFLARE Federated Statistics Visualization" + ] + }, + { + "cell_type": "markdown", + "id": "0b71dd55", + "metadata": {}, + "source": [ + "## Image Statistics Visualization\n", + "In this example, we demonstate how to visualize the results from the statistics of image data. The visualization requires json, pandas, matplotlib modules as well as nvflare visualization utlities. " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "85f23acf", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "import json\n", + "import pandas as pd\n", + "from nvflare.app_opt.statistics.visualization.statistics_visualization import Visualization" + ] + }, + { + "cell_type": "markdown", + "id": "151e23a8", + "metadata": {}, + "source": [ + "First, copy the resulting json file to demo directory. In this example, resulting file is called image_statistics.json. Then load json file\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "44f6bed2", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "with open('image_stats.json', 'r') as f:\n", + " data = json.load(f)" + ] + }, + { + "cell_type": "markdown", + "id": "c4b83ddb", + "metadata": {}, + "source": [ + "Initialize the Visualization utilities\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "ab771712", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "vis = Visualization()\n" + ] + }, + { + "cell_type": "markdown", + "id": "49f976aa", + "metadata": {}, + "source": [ + "### Overall Statistics\n", + "vis.show_stats() will show the statistics for each features, at each site for each dataset\n", + "\n", + "vis.show_stats(data = data)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20ea4dff", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "vis.show_stats(data = data)" + ] + }, + { + "cell_type": "markdown", + "id": "521cbf6f", + "metadata": {}, + "source": [ + "### select features statistics using white_list_features \n", + "user can optionally select only show specified features via white_list_features arguments. In these image files, we only have one feature" + ] + }, + { + "cell_type": "markdown", + "id": "9ab23bcc", + "metadata": {}, + "source": [ + "### Histogram Visualization\n", + "We can use vis.show_histograms() to visualize the histogram. Before we do that, we need set some iPython display setting to make sure the graph displayed in full cell. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4bada64b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from IPython.display import display, HTML\n", + "display(HTML(\"\"))" + ] + }, + { + "cell_type": "markdown", + "id": "e415b49e", + "metadata": {}, + "source": [ + "The following command display histograms for numberic features. The result shows both main plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53542cf9", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "vis.show_histograms(data = data, plot_type=\"main\")" + ] + }, + { + "cell_type": "markdown", + "id": "d8d537cc", + "metadata": {}, + "source": [ + "## Display Options\n", + "Similar to other statistics, we can use white_list_features to select only few features to display histograms. We can also use display_format=\"percent\" to allow all dataset and sites to be displayed in the same scale. User can set \n", + "\n", + "* display_format: \"percent\" or \"sample_count\"\n", + "* white_list_features: feature names\n", + "* plot_type : \"both\" or \"main\" or \"subplot\"\n", + "\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "353db4d9", + "metadata": {}, + "source": [ + "#### show default display format with subplot\n", + "In the following, we display only feature \"Intensity\" in default display_format, with \"subplot\" plot_type" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f619729", + "metadata": {}, + "outputs": [], + "source": [ + "vis.show_histograms(data = data, plot_type=\"subplot\")" + ] + }, + { + "cell_type": "markdown", + "id": "fbf7fc73", + "metadata": {}, + "source": [ + "\n", + "#### show percent display format with default plot_type (main)\n", + "In the following, we display only feature \"Intensity\" in \"percent\" display_format, with default plot_type" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8655ad63", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "vis.show_histograms(data = data, display_format=\"percent\")" + ] + }, + { + "cell_type": "markdown", + "id": "48501d37", + "metadata": {}, + "source": [ + "back to [federated_statistics_with_image_data](../../federated_statistics_with_image_data.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd6f59f2", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "918341b9", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "d86bd3e8", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nvflare_example", + "language": "python", + "name": "nvflare_example" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/demo/image_statistics.json b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/demo/image_statistics.json deleted file mode 100644 index 45ba336e50..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/demo/image_statistics.json +++ /dev/null @@ -1 +0,0 @@ -{"intensity": {"count": {"site-4": {"train": 1345}, "site-1": {"train": 3616}, "site-2": {"train": 6012}, "site-3": {"train": 10192}, "Global": {"train": 21165}}, "histogram": {"site-4": {"train": [[0.0, 1.0039, 7529944], [1.0039, 2.0078, 425108], [2.0078, 3.0118, 380736], [3.0118, 4.0157, 334702], [4.0157, 5.0196, 327862], [5.0196, 6.0235, 310277], [6.0235, 7.0275, 313290], [7.0275, 8.0314, 313654], [8.0314, 9.0353, 307805], [9.0353, 10.0392, 305356], [10.0392, 11.0431, 295868], [11.0431, 12.0471, 279067], [12.0471, 13.051, 270320], [13.051, 14.0549, 260414], [14.0549, 15.0588, 262861], [15.0588, 16.0627, 264740], [16.0627, 17.0667, 263759], [17.0667, 18.0706, 266840], [18.0706, 19.0745, 269593], [19.0745, 20.0784, 265030], [20.0784, 21.0824, 260416], [21.0824, 22.0863, 259147], [22.0863, 23.0902, 257797], [23.0902, 24.0941, 259564], [24.0941, 25.098, 259845], [25.098, 26.102, 256981], [26.102, 27.1059, 252589], [27.1059, 28.1098, 244041], [28.1098, 29.1137, 236778], [29.1137, 30.1176, 235530], [30.1176, 31.1216, 233887], [31.1216, 32.1255, 232690], [32.1255, 33.1294, 230496], [33.1294, 34.1333, 227944], [34.1333, 35.1373, 225518], [35.1373, 36.1412, 226460], [36.1412, 37.1451, 228242], [37.1451, 38.149, 230445], [38.149, 39.1529, 234045], [39.1529, 40.1569, 238784], [40.1569, 41.1608, 243710], [41.1608, 42.1647, 251529], [42.1647, 43.1686, 258775], [43.1686, 44.1726, 264193], [44.1726, 45.1765, 271677], [45.1765, 46.1804, 277858], [46.1804, 47.1843, 287293], [47.1843, 48.1882, 295818], [48.1882, 49.1922, 304070], [49.1922, 50.1961, 312974], [50.1961, 51.2, 323260], [51.2, 52.2039, 328831], [52.2039, 53.2078, 335326], [53.2078, 54.2118, 343748], [54.2118, 55.2157, 351160], [55.2157, 56.2196, 356396], [56.2196, 57.2235, 363184], [57.2235, 58.2275, 369794], [58.2275, 59.2314, 373595], [59.2314, 60.2353, 378877], [60.2353, 61.2392, 384616], [61.2392, 62.2431, 391297], [62.2431, 63.2471, 396375], [63.2471, 64.251, 398059], [64.251, 65.2549, 401636], [65.2549, 66.2588, 404846], [66.2588, 67.2627, 409274], [67.2627, 68.2667, 413504], [68.2667, 69.2706, 416709], [69.2706, 70.2745, 422904], [70.2745, 71.2784, 425087], [71.2784, 72.2824, 430026], [72.2824, 73.2863, 434464], [73.2863, 74.2902, 438169], [74.2902, 75.2941, 441835], [75.2941, 76.298, 447908], [76.298, 77.302, 454231], [77.302, 78.3059, 458792], [78.3059, 79.3098, 466910], [79.3098, 80.3137, 473704], [80.3137, 81.3176, 480384], [81.3176, 82.3216, 488587], [82.3216, 83.3255, 493441], [83.3255, 84.3294, 499632], [84.3294, 85.3333, 506173], [85.3333, 86.3373, 508150], [86.3373, 87.3412, 514601], [87.3412, 88.3451, 520894], [88.3451, 89.349, 523951], [89.349, 90.3529, 532177], [90.3529, 91.3569, 539180], [91.3569, 92.3608, 544994], [92.3608, 93.3647, 550783], [93.3647, 94.3686, 556833], [94.3686, 95.3726, 563344], [95.3726, 96.3765, 570413], [96.3765, 97.3804, 578149], [97.3804, 98.3843, 585887], [98.3843, 99.3882, 592452], [99.3882, 100.3922, 599573], [100.3922, 101.3961, 606518], [101.3961, 102.4, 612282], [102.4, 103.4039, 620579], [103.4039, 104.4078, 627741], [104.4078, 105.4118, 635267], [105.4118, 106.4157, 643762], [106.4157, 107.4196, 651374], [107.4196, 108.4235, 658375], [108.4235, 109.4275, 664897], [109.4275, 110.4314, 672678], [110.4314, 111.4353, 680048], [111.4353, 112.4392, 688321], [112.4392, 113.4431, 694140], [113.4431, 114.4471, 702296], [114.4471, 115.451, 710582], [115.451, 116.4549, 715170], [116.4549, 117.4588, 724991], [117.4588, 118.4627, 733420], [118.4627, 119.4667, 741582], [119.4667, 120.4706, 747743], [120.4706, 121.4745, 752486], [121.4745, 122.4784, 759045], [122.4784, 123.4824, 767290], [123.4824, 124.4863, 773984], [124.4863, 125.4902, 779475], [125.4902, 126.4941, 786162], [126.4941, 127.498, 791886], [127.498, 128.502, 798187], [128.502, 129.5059, 807763], [129.5059, 130.5098, 812860], [130.5098, 131.5137, 820695], [131.5137, 132.5177, 830039], [132.5177, 133.5216, 837940], [133.5216, 134.5255, 844831], [134.5255, 135.5294, 853935], [135.5294, 136.5333, 862363], [136.5333, 137.5373, 872252], [137.5373, 138.5412, 875083], [138.5412, 139.5451, 879200], [139.5451, 140.549, 886720], [140.549, 141.5529, 894372], [141.5529, 142.5569, 898412], [142.5569, 143.5608, 907203], [143.5608, 144.5647, 914686], [144.5647, 145.5686, 921733], [145.5686, 146.5726, 930211], [146.5726, 147.5765, 942743], [147.5765, 148.5804, 950464], [148.5804, 149.5843, 964346], [149.5843, 150.5882, 975391], [150.5882, 151.5922, 985204], [151.5922, 152.5961, 993241], [152.5961, 153.6, 1003272], [153.6, 154.6039, 1007841], [154.6039, 155.6078, 1014981], [155.6078, 156.6118, 1025516], [156.6118, 157.6157, 1038689], [157.6157, 158.6196, 1046387], [158.6196, 159.6235, 1055958], [159.6235, 160.6275, 1061917], [160.6275, 161.6314, 1069902], [161.6314, 162.6353, 1074383], [162.6353, 163.6392, 1077711], [163.6392, 164.6431, 1084480], [164.6431, 165.6471, 1092246], [165.6471, 166.651, 1098250], [166.651, 167.6549, 1105551], [167.6549, 168.6588, 1112735], [168.6588, 169.6628, 1118098], [169.6628, 170.6667, 1125523], [170.6667, 171.6706, 1133121], [171.6706, 172.6745, 1142512], [172.6745, 173.6784, 1148704], [173.6784, 174.6824, 1154061], [174.6824, 175.6863, 1163823], [175.6863, 176.6902, 1168052], [176.6902, 177.6941, 1171740], [177.6941, 178.698, 1176882], [178.698, 179.702, 1179433], [179.702, 180.7059, 1176290], [180.7059, 181.7098, 1170622], [181.7098, 182.7137, 1165328], [182.7137, 183.7177, 1156930], [183.7177, 184.7216, 1147879], [184.7216, 185.7255, 1137300], [185.7255, 186.7294, 1120497], [186.7294, 187.7333, 1106738], [187.7333, 188.7373, 1091219], [188.7373, 189.7412, 1070762], [189.7412, 190.7451, 1047710], [190.7451, 191.749, 1024671], [191.749, 192.7529, 999503], [192.7529, 193.7569, 969456], [193.7569, 194.7608, 933606], [194.7608, 195.7647, 898096], [195.7647, 196.7686, 857537], [196.7686, 197.7726, 821693], [197.7726, 198.7765, 775987], [198.7765, 199.7804, 729475], [199.7804, 200.7843, 682130], [200.7843, 201.7882, 640395], [201.7882, 202.7922, 601811], [202.7922, 203.7961, 568228], [203.7961, 204.8, 542387], [204.8, 205.8039, 506923], [205.8039, 206.8078, 477752], [206.8078, 207.8118, 453628], [207.8118, 208.8157, 431648], [208.8157, 209.8196, 412465], [209.8196, 210.8235, 392404], [210.8235, 211.8275, 373984], [211.8275, 212.8314, 352797], [212.8314, 213.8353, 331776], [213.8353, 214.8392, 314056], [214.8392, 215.8431, 299929], [215.8431, 216.8471, 282146], [216.8471, 217.851, 263520], [217.851, 218.8549, 243529], [218.8549, 219.8588, 225895], [219.8588, 220.8627, 211073], [220.8627, 221.8667, 195363], [221.8667, 222.8706, 179534], [222.8706, 223.8745, 163521], [223.8745, 224.8784, 150222], [224.8784, 225.8824, 137280], [225.8824, 226.8863, 124883], [226.8863, 227.8902, 114227], [227.8902, 228.8941, 101407], [228.8941, 229.898, 89917], [229.898, 230.902, 79965], [230.902, 231.9059, 69839], [231.9059, 232.9098, 60326], [232.9098, 233.9137, 51425], [233.9137, 234.9176, 43914], [234.9176, 235.9216, 38433], [235.9216, 236.9255, 32395], [236.9255, 237.9294, 27010], [237.9294, 238.9333, 22764], [238.9333, 239.9373, 18860], [239.9373, 240.9412, 14844], [240.9412, 241.9451, 12068], [241.9451, 242.949, 9001], [242.949, 243.9529, 7004], [243.9529, 244.9569, 5834], [244.9569, 245.9608, 4248], [245.9608, 246.9647, 2930], [246.9647, 247.9686, 2306], [247.9686, 248.9725, 2106], [248.9725, 249.9765, 1872], [249.9765, 250.9804, 1876], [250.9804, 251.9843, 1856], [251.9843, 252.9882, 1814], [252.9882, 253.9922, 1858], [253.9922, 254.9961, 1955], [254.9961, 256.0, 9820]]}, "site-1": {"train": [[0.0, 1.0039, 10894028], [1.0039, 2.0078, 919649], [2.0078, 3.0118, 769280], [3.0118, 4.0157, 693827], [4.0157, 5.0196, 647480], [5.0196, 6.0235, 650549], [6.0235, 7.0275, 534844], [7.0275, 8.0314, 473089], [8.0314, 9.0353, 423891], [9.0353, 10.0392, 407681], [10.0392, 11.0431, 377949], [11.0431, 12.0471, 371994], [12.0471, 13.051, 337606], [13.051, 14.0549, 321416], [14.0549, 15.0588, 314833], [15.0588, 16.0627, 305874], [16.0627, 17.0667, 299324], [17.0667, 18.0706, 299362], [18.0706, 19.0745, 302235], [19.0745, 20.0784, 303270], [20.0784, 21.0824, 304850], [21.0824, 22.0863, 311243], [22.0863, 23.0902, 313018], [23.0902, 24.0941, 322185], [24.0941, 25.098, 323243], [25.098, 26.102, 333137], [26.102, 27.1059, 367039], [27.1059, 28.1098, 330758], [28.1098, 29.1137, 332182], [29.1137, 30.1176, 350388], [30.1176, 31.1216, 343288], [31.1216, 32.1255, 353006], [32.1255, 33.1294, 359859], [33.1294, 34.1333, 397470], [34.1333, 35.1373, 373270], [35.1373, 36.1412, 379933], [36.1412, 37.1451, 396555], [37.1451, 38.149, 398242], [38.149, 39.1529, 405559], [39.1529, 40.1569, 417184], [40.1569, 41.1608, 430439], [41.1608, 42.1647, 441752], [42.1647, 43.1686, 452952], [43.1686, 44.1726, 466667], [44.1726, 45.1765, 484097], [45.1765, 46.1804, 505844], [46.1804, 47.1843, 515014], [47.1843, 48.1882, 527903], [48.1882, 49.1922, 555451], [49.1922, 50.1961, 569528], [50.1961, 51.2, 586813], [51.2, 52.2039, 599347], [52.2039, 53.2078, 618684], [53.2078, 54.2118, 630950], [54.2118, 55.2157, 647808], [55.2157, 56.2196, 673090], [56.2196, 57.2235, 688245], [57.2235, 58.2275, 710112], [58.2275, 59.2314, 718399], [59.2314, 60.2353, 729100], [60.2353, 61.2392, 743367], [61.2392, 62.2431, 758390], [62.2431, 63.2471, 774620], [63.2471, 64.251, 790928], [64.251, 65.2549, 810389], [65.2549, 66.2588, 828657], [66.2588, 67.2627, 847662], [67.2627, 68.2667, 865040], [68.2667, 69.2706, 879144], [69.2706, 70.2745, 895969], [70.2745, 71.2784, 916437], [71.2784, 72.2824, 935822], [72.2824, 73.2863, 953892], [73.2863, 74.2902, 973171], [74.2902, 75.2941, 989698], [75.2941, 76.298, 1006427], [76.298, 77.302, 1024038], [77.302, 78.3059, 1039927], [78.3059, 79.3098, 1055270], [79.3098, 80.3137, 1069199], [80.3137, 81.3176, 1083336], [81.3176, 82.3216, 1097692], [82.3216, 83.3255, 1116214], [83.3255, 84.3294, 1129098], [84.3294, 85.3333, 1141204], [85.3333, 86.3373, 1155065], [86.3373, 87.3412, 1168550], [87.3412, 88.3451, 1185271], [88.3451, 89.349, 1199064], [89.349, 90.3529, 1214772], [90.3529, 91.3569, 1225242], [91.3569, 92.3608, 1239084], [92.3608, 93.3647, 1253750], [93.3647, 94.3686, 1269047], [94.3686, 95.3726, 1284168], [95.3726, 96.3765, 1299647], [96.3765, 97.3804, 1317955], [97.3804, 98.3843, 1335169], [98.3843, 99.3882, 1354612], [99.3882, 100.3922, 1379464], [100.3922, 101.3961, 1400813], [101.3961, 102.4, 1459381], [102.4, 103.4039, 1437747], [103.4039, 104.4078, 1452027], [104.4078, 105.4118, 1472904], [105.4118, 106.4157, 1495774], [106.4157, 107.4196, 1517943], [107.4196, 108.4235, 1540170], [108.4235, 109.4275, 1567816], [109.4275, 110.4314, 1596016], [110.4314, 111.4353, 1623599], [111.4353, 112.4392, 1638923], [112.4392, 113.4431, 1648676], [113.4431, 114.4471, 1660864], [114.4471, 115.451, 1670206], [115.451, 116.4549, 1678346], [116.4549, 117.4588, 1689001], [117.4588, 118.4627, 1697572], [118.4627, 119.4667, 1709890], [119.4667, 120.4706, 1723120], [120.4706, 121.4745, 1733628], [121.4745, 122.4784, 1744632], [122.4784, 123.4824, 1758708], [123.4824, 124.4863, 1769412], [124.4863, 125.4902, 1780412], [125.4902, 126.4941, 1793815], [126.4941, 127.498, 1804310], [127.498, 128.502, 1816440], [128.502, 129.5059, 1826211], [129.5059, 130.5098, 1838735], [130.5098, 131.5137, 1851383], [131.5137, 132.5177, 1864726], [132.5177, 133.5216, 1876103], [133.5216, 134.5255, 1886905], [134.5255, 135.5294, 1901283], [135.5294, 136.5333, 1911711], [136.5333, 137.5373, 1918392], [137.5373, 138.5412, 1927938], [138.5412, 139.5451, 1936607], [139.5451, 140.549, 1944733], [140.549, 141.5529, 1949944], [141.5529, 142.5569, 1956728], [142.5569, 143.5608, 1964468], [143.5608, 144.5647, 1971692], [144.5647, 145.5686, 1978261], [145.5686, 146.5726, 1989150], [146.5726, 147.5765, 1994601], [147.5765, 148.5804, 1998874], [148.5804, 149.5843, 2002253], [149.5843, 150.5882, 2010981], [150.5882, 151.5922, 2019781], [151.5922, 152.5961, 2029319], [152.5961, 153.6, 2041442], [153.6, 154.6039, 2049317], [154.6039, 155.6078, 2052385], [155.6078, 156.6118, 2057924], [156.6118, 157.6157, 2063970], [157.6157, 158.6196, 2070129], [158.6196, 159.6235, 2073822], [159.6235, 160.6275, 2071202], [160.6275, 161.6314, 2074443], [161.6314, 162.6353, 2085518], [162.6353, 163.6392, 2084152], [163.6392, 164.6431, 2088635], [164.6431, 165.6471, 2087421], [165.6471, 166.651, 2089825], [166.651, 167.6549, 2091350], [167.6549, 168.6588, 2098728], [168.6588, 169.6628, 2098083], [169.6628, 170.6667, 2103421], [170.6667, 171.6706, 2106742], [171.6706, 172.6745, 2108895], [172.6745, 173.6784, 2104367], [173.6784, 174.6824, 2097960], [174.6824, 175.6863, 2091572], [175.6863, 176.6902, 2083093], [176.6902, 177.6941, 2073152], [177.6941, 178.698, 2069031], [178.698, 179.702, 2049718], [179.702, 180.7059, 2040335], [180.7059, 181.7098, 2023287], [181.7098, 182.7137, 2022978], [182.7137, 183.7177, 2010025], [183.7177, 184.7216, 1994103], [184.7216, 185.7255, 1965720], [185.7255, 186.7294, 1961279], [186.7294, 187.7333, 1943250], [187.7333, 188.7373, 1936609], [188.7373, 189.7412, 1916980], [189.7412, 190.7451, 1913374], [190.7451, 191.749, 1893539], [191.749, 192.7529, 1882353], [192.7529, 193.7569, 1864142], [193.7569, 194.7608, 1855682], [194.7608, 195.7647, 1847667], [195.7647, 196.7686, 1853185], [196.7686, 197.7726, 1846513], [197.7726, 198.7765, 1826374], [198.7765, 199.7804, 1805915], [199.7804, 200.7843, 1777044], [200.7843, 201.7882, 1775611], [201.7882, 202.7922, 1738088], [202.7922, 203.7961, 1716504], [203.7961, 204.8, 1694891], [204.8, 205.8039, 1667268], [205.8039, 206.8078, 1643096], [206.8078, 207.8118, 1613994], [207.8118, 208.8157, 1578087], [208.8157, 209.8196, 1534606], [209.8196, 210.8235, 1495973], [210.8235, 211.8275, 1449118], [211.8275, 212.8314, 1399671], [212.8314, 213.8353, 1339223], [213.8353, 214.8392, 1287605], [214.8392, 215.8431, 1223187], [215.8431, 216.8471, 1174973], [216.8471, 217.851, 1131599], [217.851, 218.8549, 1090599], [218.8549, 219.8588, 1044891], [219.8588, 220.8627, 1008683], [220.8627, 221.8667, 974052], [221.8667, 222.8706, 944405], [222.8706, 223.8745, 918074], [223.8745, 224.8784, 899155], [224.8784, 225.8824, 878302], [225.8824, 226.8863, 844524], [226.8863, 227.8902, 813378], [227.8902, 228.8941, 787275], [228.8941, 229.898, 764613], [229.898, 230.902, 739253], [230.902, 231.9059, 712397], [231.9059, 232.9098, 696324], [232.9098, 233.9137, 676432], [233.9137, 234.9176, 668208], [234.9176, 235.9216, 651782], [235.9216, 236.9255, 633075], [236.9255, 237.9294, 624123], [237.9294, 238.9333, 618392], [238.9333, 239.9373, 612616], [239.9373, 240.9412, 602796], [240.9412, 241.9451, 603908], [241.9451, 242.949, 612502], [242.949, 243.9529, 616100], [243.9529, 244.9569, 616756], [244.9569, 245.9608, 608695], [245.9608, 246.9647, 612086], [246.9647, 247.9686, 615833], [247.9686, 248.9725, 612640], [248.9725, 249.9765, 611049], [249.9765, 250.9804, 643359], [250.9804, 251.9843, 705574], [251.9843, 252.9882, 716770], [252.9882, 253.9922, 679330], [253.9922, 254.9961, 601071], [254.9961, 256.0, 1052689]]}, "site-2": {"train": [[0.0, 1.0039, 15941521], [1.0039, 2.0078, 3409289], [2.0078, 3.0118, 3292634], [3.0118, 4.0157, 2963028], [4.0157, 5.0196, 2697815], [5.0196, 6.0235, 2472349], [6.0235, 7.0275, 2279512], [7.0275, 8.0314, 2005163], [8.0314, 9.0353, 1802259], [9.0353, 10.0392, 1628386], [10.0392, 11.0431, 1567570], [11.0431, 12.0471, 1431692], [12.0471, 13.051, 1392798], [13.051, 14.0549, 1269837], [14.0549, 15.0588, 1193088], [15.0588, 16.0627, 1130059], [16.0627, 17.0667, 1092008], [17.0667, 18.0706, 1015100], [18.0706, 19.0745, 983700], [19.0745, 20.0784, 960282], [20.0784, 21.0824, 909431], [21.0824, 22.0863, 866762], [22.0863, 23.0902, 836220], [23.0902, 24.0941, 823204], [24.0941, 25.098, 815079], [25.098, 26.102, 816740], [26.102, 27.1059, 816584], [27.1059, 28.1098, 810720], [28.1098, 29.1137, 815188], [29.1137, 30.1176, 817414], [30.1176, 31.1216, 825910], [31.1216, 32.1255, 834258], [32.1255, 33.1294, 842404], [33.1294, 34.1333, 847901], [34.1333, 35.1373, 855877], [35.1373, 36.1412, 865681], [36.1412, 37.1451, 877634], [37.1451, 38.149, 891457], [38.149, 39.1529, 906151], [39.1529, 40.1569, 920792], [40.1569, 41.1608, 937021], [41.1608, 42.1647, 951070], [42.1647, 43.1686, 969154], [43.1686, 44.1726, 986948], [44.1726, 45.1765, 1005515], [45.1765, 46.1804, 1027579], [46.1804, 47.1843, 1048011], [47.1843, 48.1882, 1065838], [48.1882, 49.1922, 1089849], [49.1922, 50.1961, 1112060], [50.1961, 51.2, 1132509], [51.2, 52.2039, 1154043], [52.2039, 53.2078, 1177892], [53.2078, 54.2118, 1200277], [54.2118, 55.2157, 1224671], [55.2157, 56.2196, 1246311], [56.2196, 57.2235, 1273277], [57.2235, 58.2275, 1298370], [58.2275, 59.2314, 1320641], [59.2314, 60.2353, 1343924], [60.2353, 61.2392, 1368803], [61.2392, 62.2431, 1398404], [62.2431, 63.2471, 1418798], [63.2471, 64.251, 1447717], [64.251, 65.2549, 1478809], [65.2549, 66.2588, 1509546], [66.2588, 67.2627, 1544021], [67.2627, 68.2667, 1576975], [68.2667, 69.2706, 1614871], [69.2706, 70.2745, 1647199], [70.2745, 71.2784, 1684707], [71.2784, 72.2824, 1721295], [72.2824, 73.2863, 1754361], [73.2863, 74.2902, 1791169], [74.2902, 75.2941, 1827869], [75.2941, 76.298, 1867136], [76.298, 77.302, 1909117], [77.302, 78.3059, 1948501], [78.3059, 79.3098, 1990679], [79.3098, 80.3137, 2026898], [80.3137, 81.3176, 2064295], [81.3176, 82.3216, 2100536], [82.3216, 83.3255, 2134824], [83.3255, 84.3294, 2173686], [84.3294, 85.3333, 2215018], [85.3333, 86.3373, 2250716], [86.3373, 87.3412, 2289487], [87.3412, 88.3451, 2331344], [88.3451, 89.349, 2370562], [89.349, 90.3529, 2404724], [90.3529, 91.3569, 2446641], [91.3569, 92.3608, 2481625], [92.3608, 93.3647, 2516636], [93.3647, 94.3686, 2557334], [94.3686, 95.3726, 2598710], [95.3726, 96.3765, 2638704], [96.3765, 97.3804, 2670602], [97.3804, 98.3843, 2696413], [98.3843, 99.3882, 2721216], [99.3882, 100.3922, 2749841], [100.3922, 101.3961, 2779287], [101.3961, 102.4, 2806616], [102.4, 103.4039, 2835561], [103.4039, 104.4078, 2863948], [104.4078, 105.4118, 2883549], [105.4118, 106.4157, 2905702], [106.4157, 107.4196, 2932802], [107.4196, 108.4235, 2954641], [108.4235, 109.4275, 2982323], [109.4275, 110.4314, 3004721], [110.4314, 111.4353, 3028188], [111.4353, 112.4392, 3048302], [112.4392, 113.4431, 3060261], [113.4431, 114.4471, 3075460], [114.4471, 115.451, 3091074], [115.451, 116.4549, 3101385], [116.4549, 117.4588, 3108729], [117.4588, 118.4627, 3123015], [118.4627, 119.4667, 3126990], [119.4667, 120.4706, 3134952], [120.4706, 121.4745, 3147843], [121.4745, 122.4784, 3156261], [122.4784, 123.4824, 3163852], [123.4824, 124.4863, 3167825], [124.4863, 125.4902, 3168960], [125.4902, 126.4941, 3161481], [126.4941, 127.498, 3164858], [127.498, 128.502, 3173046], [128.502, 129.5059, 3182439], [129.5059, 130.5098, 3198768], [130.5098, 131.5137, 3196157], [131.5137, 132.5177, 3196652], [132.5177, 133.5216, 3197763], [133.5216, 134.5255, 3199729], [134.5255, 135.5294, 3205524], [135.5294, 136.5333, 3211286], [136.5333, 137.5373, 3213368], [137.5373, 138.5412, 3211412], [138.5412, 139.5451, 3206645], [139.5451, 140.549, 3209193], [140.549, 141.5529, 3214692], [141.5529, 142.5569, 3211823], [142.5569, 143.5608, 3212346], [143.5608, 144.5647, 3208845], [144.5647, 145.5686, 3204488], [145.5686, 146.5726, 3199919], [146.5726, 147.5765, 3189501], [147.5765, 148.5804, 3179552], [148.5804, 149.5843, 3166138], [149.5843, 150.5882, 3153095], [150.5882, 151.5922, 3142765], [151.5922, 152.5961, 3139218], [152.5961, 153.6, 3130360], [153.6, 154.6039, 3134121], [154.6039, 155.6078, 3134447], [155.6078, 156.6118, 3139450], [156.6118, 157.6157, 3145100], [157.6157, 158.6196, 3147758], [158.6196, 159.6235, 3146090], [159.6235, 160.6275, 3138656], [160.6275, 161.6314, 3131494], [161.6314, 162.6353, 3119024], [162.6353, 163.6392, 3111654], [163.6392, 164.6431, 3106116], [164.6431, 165.6471, 3098675], [165.6471, 166.651, 3093899], [166.651, 167.6549, 3085426], [167.6549, 168.6588, 3070005], [168.6588, 169.6628, 3067126], [169.6628, 170.6667, 3070429], [170.6667, 171.6706, 3076652], [171.6706, 172.6745, 3076603], [172.6745, 173.6784, 3063677], [173.6784, 174.6824, 3054089], [174.6824, 175.6863, 3046927], [175.6863, 176.6902, 3030450], [176.6902, 177.6941, 3007018], [177.6941, 178.698, 2980311], [178.698, 179.702, 2962652], [179.702, 180.7059, 2946745], [180.7059, 181.7098, 2924491], [181.7098, 182.7137, 2896582], [182.7137, 183.7177, 2869170], [183.7177, 184.7216, 2843862], [184.7216, 185.7255, 2819849], [185.7255, 186.7294, 2795058], [186.7294, 187.7333, 2777121], [187.7333, 188.7373, 2751346], [188.7373, 189.7412, 2737059], [189.7412, 190.7451, 2708849], [190.7451, 191.749, 2669175], [191.749, 192.7529, 2631722], [192.7529, 193.7569, 2609137], [193.7569, 194.7608, 2588888], [194.7608, 195.7647, 2556796], [195.7647, 196.7686, 2524864], [196.7686, 197.7726, 2489195], [197.7726, 198.7765, 2446870], [198.7765, 199.7804, 2408229], [199.7804, 200.7843, 2377414], [200.7843, 201.7882, 2343879], [201.7882, 202.7922, 2302332], [202.7922, 203.7961, 2265841], [203.7961, 204.8, 2239418], [204.8, 205.8039, 2220973], [205.8039, 206.8078, 2184959], [206.8078, 207.8118, 2150144], [207.8118, 208.8157, 2114186], [208.8157, 209.8196, 2078547], [209.8196, 210.8235, 2049835], [210.8235, 211.8275, 1997943], [211.8275, 212.8314, 1943236], [212.8314, 213.8353, 1899057], [213.8353, 214.8392, 1854857], [214.8392, 215.8431, 1814772], [215.8431, 216.8471, 1769082], [216.8471, 217.851, 1731006], [217.851, 218.8549, 1697932], [218.8549, 219.8588, 1659068], [219.8588, 220.8627, 1627294], [220.8627, 221.8667, 1598942], [221.8667, 222.8706, 1552328], [222.8706, 223.8745, 1493287], [223.8745, 224.8784, 1443311], [224.8784, 225.8824, 1398265], [225.8824, 226.8863, 1350080], [226.8863, 227.8902, 1291945], [227.8902, 228.8941, 1235278], [228.8941, 229.898, 1182931], [229.898, 230.902, 1135305], [230.902, 231.9059, 1083947], [231.9059, 232.9098, 1043830], [232.9098, 233.9137, 989887], [233.9137, 234.9176, 921426], [234.9176, 235.9216, 856490], [235.9216, 236.9255, 794486], [236.9255, 237.9294, 732566], [237.9294, 238.9333, 667414], [238.9333, 239.9373, 583868], [239.9373, 240.9412, 492933], [240.9412, 241.9451, 411886], [241.9451, 242.949, 340969], [242.949, 243.9529, 281630], [243.9529, 244.9569, 232361], [244.9569, 245.9608, 190872], [245.9608, 246.9647, 152602], [246.9647, 247.9686, 126933], [247.9686, 248.9725, 100975], [248.9725, 249.9765, 75090], [249.9765, 250.9804, 54275], [250.9804, 251.9843, 35979], [251.9843, 252.9882, 29189], [252.9882, 253.9922, 38216], [253.9922, 254.9961, 18398], [254.9961, 256.0, 24730]]}, "site-3": {"train": [[0.0, 1.0039, 35420536], [1.0039, 2.0078, 3434579], [2.0078, 3.0118, 3341431], [3.0118, 4.0157, 3092450], [4.0157, 5.0196, 3042051], [5.0196, 6.0235, 2757745], [6.0235, 7.0275, 2490254], [7.0275, 8.0314, 2411643], [8.0314, 9.0353, 2285068], [9.0353, 10.0392, 2156599], [10.0392, 11.0431, 2074827], [11.0431, 12.0471, 2118880], [12.0471, 13.051, 2024748], [13.051, 14.0549, 1964664], [14.0549, 15.0588, 1877950], [15.0588, 16.0627, 1895079], [16.0627, 17.0667, 1777361], [17.0667, 18.0706, 1680089], [18.0706, 19.0745, 1668241], [19.0745, 20.0784, 1609927], [20.0784, 21.0824, 1539886], [21.0824, 22.0863, 1526719], [22.0863, 23.0902, 1444669], [23.0902, 24.0941, 1446578], [24.0941, 25.098, 1415474], [25.098, 26.102, 1430897], [26.102, 27.1059, 1446241], [27.1059, 28.1098, 1467478], [28.1098, 29.1137, 1493183], [29.1137, 30.1176, 1537432], [30.1176, 31.1216, 1586366], [31.1216, 32.1255, 1630488], [32.1255, 33.1294, 1677620], [33.1294, 34.1333, 1716854], [34.1333, 35.1373, 1766773], [35.1373, 36.1412, 1817857], [36.1412, 37.1451, 1867103], [37.1451, 38.149, 1918587], [38.149, 39.1529, 1969687], [39.1529, 40.1569, 2017640], [40.1569, 41.1608, 2067479], [41.1608, 42.1647, 2120295], [42.1647, 43.1686, 2173831], [43.1686, 44.1726, 2225973], [44.1726, 45.1765, 2277126], [45.1765, 46.1804, 2324888], [46.1804, 47.1843, 2375966], [47.1843, 48.1882, 2422688], [48.1882, 49.1922, 2470304], [49.1922, 50.1961, 2515550], [50.1961, 51.2, 2562232], [51.2, 52.2039, 2604402], [52.2039, 53.2078, 2646158], [53.2078, 54.2118, 2685611], [54.2118, 55.2157, 2725405], [55.2157, 56.2196, 2769219], [56.2196, 57.2235, 2801003], [57.2235, 58.2275, 2836516], [58.2275, 59.2314, 2868959], [59.2314, 60.2353, 2899569], [60.2353, 61.2392, 2928114], [61.2392, 62.2431, 2956648], [62.2431, 63.2471, 2985628], [63.2471, 64.251, 3014738], [64.251, 65.2549, 3050418], [65.2549, 66.2588, 3082767], [66.2588, 67.2627, 3113070], [67.2627, 68.2667, 3144182], [68.2667, 69.2706, 3171032], [69.2706, 70.2745, 3197708], [70.2745, 71.2784, 3224877], [71.2784, 72.2824, 3254809], [72.2824, 73.2863, 3282706], [73.2863, 74.2902, 3313171], [74.2902, 75.2941, 3340328], [75.2941, 76.298, 3369138], [76.298, 77.302, 3393056], [77.302, 78.3059, 3420210], [78.3059, 79.3098, 3446986], [79.3098, 80.3137, 3469169], [80.3137, 81.3176, 3490349], [81.3176, 82.3216, 3511811], [82.3216, 83.3255, 3529029], [83.3255, 84.3294, 3551042], [84.3294, 85.3333, 3575271], [85.3333, 86.3373, 3600291], [86.3373, 87.3412, 3623519], [87.3412, 88.3451, 3648167], [88.3451, 89.349, 3666903], [89.349, 90.3529, 3684651], [90.3529, 91.3569, 3703091], [91.3569, 92.3608, 3718889], [92.3608, 93.3647, 3736978], [93.3647, 94.3686, 3759958], [94.3686, 95.3726, 3779436], [95.3726, 96.3765, 3797707], [96.3765, 97.3804, 3819111], [97.3804, 98.3843, 3828612], [98.3843, 99.3882, 3847381], [99.3882, 100.3922, 3866771], [100.3922, 101.3961, 3879339], [101.3961, 102.4, 3902031], [102.4, 103.4039, 3929528], [103.4039, 104.4078, 3948769], [104.4078, 105.4118, 3968417], [105.4118, 106.4157, 3992482], [106.4157, 107.4196, 4016228], [107.4196, 108.4235, 4042653], [108.4235, 109.4275, 4063340], [109.4275, 110.4314, 4087918], [110.4314, 111.4353, 4109329], [111.4353, 112.4392, 4138338], [112.4392, 113.4431, 4162863], [113.4431, 114.4471, 4189909], [114.4471, 115.451, 4215519], [115.451, 116.4549, 4247141], [116.4549, 117.4588, 4273944], [117.4588, 118.4627, 4311672], [118.4627, 119.4667, 4338825], [119.4667, 120.4706, 4370836], [120.4706, 121.4745, 4402997], [121.4745, 122.4784, 4431116], [122.4784, 123.4824, 4460083], [123.4824, 124.4863, 4482474], [124.4863, 125.4902, 4504318], [125.4902, 126.4941, 4519438], [126.4941, 127.498, 4550325], [127.498, 128.502, 4580158], [128.502, 129.5059, 4618214], [129.5059, 130.5098, 4659317], [130.5098, 131.5137, 4686527], [131.5137, 132.5177, 4721523], [132.5177, 133.5216, 4744639], [133.5216, 134.5255, 4762751], [134.5255, 135.5294, 4779756], [135.5294, 136.5333, 4801818], [136.5333, 137.5373, 4821757], [137.5373, 138.5412, 4834936], [138.5412, 139.5451, 4845614], [139.5451, 140.549, 4873763], [140.549, 141.5529, 4898921], [141.5529, 142.5569, 4937001], [142.5569, 143.5608, 4973016], [143.5608, 144.5647, 4993681], [144.5647, 145.5686, 5002594], [145.5686, 146.5726, 4999512], [146.5726, 147.5765, 4994651], [147.5765, 148.5804, 4987965], [148.5804, 149.5843, 4967454], [149.5843, 150.5882, 4942949], [150.5882, 151.5922, 4920523], [151.5922, 152.5961, 4896259], [152.5961, 153.6, 4882397], [153.6, 154.6039, 4869893], [154.6039, 155.6078, 4869395], [155.6078, 156.6118, 4868932], [156.6118, 157.6157, 4866115], [157.6157, 158.6196, 4861393], [158.6196, 159.6235, 4858587], [159.6235, 160.6275, 4849058], [160.6275, 161.6314, 4848421], [161.6314, 162.6353, 4847538], [162.6353, 163.6392, 4853239], [163.6392, 164.6431, 4852638], [164.6431, 165.6471, 4858857], [165.6471, 166.651, 4851576], [166.651, 167.6549, 4844393], [167.6549, 168.6588, 4825204], [168.6588, 169.6628, 4824452], [169.6628, 170.6667, 4832095], [170.6667, 171.6706, 4845295], [171.6706, 172.6745, 4852993], [172.6745, 173.6784, 4852364], [173.6784, 174.6824, 4847783], [174.6824, 175.6863, 4843062], [175.6863, 176.6902, 4833583], [176.6902, 177.6941, 4825035], [177.6941, 178.698, 4813977], [178.698, 179.702, 4809204], [179.702, 180.7059, 4803096], [180.7059, 181.7098, 4794134], [181.7098, 182.7137, 4774045], [182.7137, 183.7177, 4758050], [183.7177, 184.7216, 4748434], [184.7216, 185.7255, 4742331], [185.7255, 186.7294, 4734879], [186.7294, 187.7333, 4727653], [187.7333, 188.7373, 4723189], [188.7373, 189.7412, 4716853], [189.7412, 190.7451, 4695524], [190.7451, 191.749, 4673050], [191.749, 192.7529, 4652493], [192.7529, 193.7569, 4640657], [193.7569, 194.7608, 4630755], [194.7608, 195.7647, 4621774], [195.7647, 196.7686, 4592518], [196.7686, 197.7726, 4554421], [197.7726, 198.7765, 4528396], [198.7765, 199.7804, 4495683], [199.7804, 200.7843, 4456919], [200.7843, 201.7882, 4421014], [201.7882, 202.7922, 4385719], [202.7922, 203.7961, 4354884], [203.7961, 204.8, 4337416], [204.8, 205.8039, 4320880], [205.8039, 206.8078, 4297270], [206.8078, 207.8118, 4265592], [207.8118, 208.8157, 4221109], [208.8157, 209.8196, 4184597], [209.8196, 210.8235, 4145936], [210.8235, 211.8275, 4102365], [211.8275, 212.8314, 4054169], [212.8314, 213.8353, 4014081], [213.8353, 214.8392, 3980732], [214.8392, 215.8431, 3950586], [215.8431, 216.8471, 3909584], [216.8471, 217.851, 3864430], [217.851, 218.8549, 3813607], [218.8549, 219.8588, 3776123], [219.8588, 220.8627, 3735486], [220.8627, 221.8667, 3706293], [221.8667, 222.8706, 3661984], [222.8706, 223.8745, 3606364], [223.8745, 224.8784, 3544293], [224.8784, 225.8824, 3477928], [225.8824, 226.8863, 3406908], [226.8863, 227.8902, 3332751], [227.8902, 228.8941, 3252660], [228.8941, 229.898, 3161730], [229.898, 230.902, 3062149], [230.902, 231.9059, 2953007], [231.9059, 232.9098, 2862108], [232.9098, 233.9137, 2760883], [233.9137, 234.9176, 2633718], [234.9176, 235.9216, 2503331], [235.9216, 236.9255, 2347287], [236.9255, 237.9294, 2194271], [237.9294, 238.9333, 2016997], [238.9333, 239.9373, 1808115], [239.9373, 240.9412, 1593507], [240.9412, 241.9451, 1395255], [241.9451, 242.949, 1210900], [242.949, 243.9529, 1038748], [243.9529, 244.9569, 875010], [244.9569, 245.9608, 714476], [245.9608, 246.9647, 562953], [246.9647, 247.9686, 431427], [247.9686, 248.9725, 326175], [248.9725, 249.9765, 231626], [249.9765, 250.9804, 155845], [250.9804, 251.9843, 104933], [251.9843, 252.9882, 72649], [252.9882, 253.9922, 47831], [253.9922, 254.9961, 34488], [254.9961, 256.0, 100798]]}, "Global": {"train": [[0.0, 1.0039, 69786029], [1.0039, 2.0078, 8188625], [2.0078, 3.0118, 7784081], [3.0118, 4.0157, 7084007], [4.0157, 5.0196, 6715208], [5.0196, 6.0235, 6190920], [6.0235, 7.0275, 5617900], [7.0275, 8.0314, 5203549], [8.0314, 9.0353, 4819023], [9.0353, 10.0392, 4498022], [10.0392, 11.0431, 4316214], [11.0431, 12.0471, 4201633], [12.0471, 13.051, 4025472], [13.051, 14.0549, 3816331], [14.0549, 15.0588, 3648732], [15.0588, 16.0627, 3595752], [16.0627, 17.0667, 3432452], [17.0667, 18.0706, 3261391], [18.0706, 19.0745, 3223769], [19.0745, 20.0784, 3138509], [20.0784, 21.0824, 3014583], [21.0824, 22.0863, 2963871], [22.0863, 23.0902, 2851704], [23.0902, 24.0941, 2851531], [24.0941, 25.098, 2813641], [25.098, 26.102, 2837755], [26.102, 27.1059, 2882453], [27.1059, 28.1098, 2852997], [28.1098, 29.1137, 2877331], [29.1137, 30.1176, 2940764], [30.1176, 31.1216, 2989451], [31.1216, 32.1255, 3050442], [32.1255, 33.1294, 3110379], [33.1294, 34.1333, 3190169], [34.1333, 35.1373, 3221438], [35.1373, 36.1412, 3289931], [36.1412, 37.1451, 3369534], [37.1451, 38.149, 3438731], [38.149, 39.1529, 3515442], [39.1529, 40.1569, 3594400], [40.1569, 41.1608, 3678649], [41.1608, 42.1647, 3764646], [42.1647, 43.1686, 3854712], [43.1686, 44.1726, 3943781], [44.1726, 45.1765, 4038415], [45.1765, 46.1804, 4136169], [46.1804, 47.1843, 4226284], [47.1843, 48.1882, 4312247], [48.1882, 49.1922, 4419674], [49.1922, 50.1961, 4510112], [50.1961, 51.2, 4604814], [51.2, 52.2039, 4686623], [52.2039, 53.2078, 4778060], [53.2078, 54.2118, 4860586], [54.2118, 55.2157, 4949044], [55.2157, 56.2196, 5045016], [56.2196, 57.2235, 5125709], [57.2235, 58.2275, 5214792], [58.2275, 59.2314, 5281594], [59.2314, 60.2353, 5351470], [60.2353, 61.2392, 5424900], [61.2392, 62.2431, 5504739], [62.2431, 63.2471, 5575421], [63.2471, 64.251, 5651442], [64.251, 65.2549, 5741252], [65.2549, 66.2588, 5825816], [66.2588, 67.2627, 5914027], [67.2627, 68.2667, 5999701], [68.2667, 69.2706, 6081756], [69.2706, 70.2745, 6163780], [70.2745, 71.2784, 6251108], [71.2784, 72.2824, 6341952], [72.2824, 73.2863, 6425423], [73.2863, 74.2902, 6515680], [74.2902, 75.2941, 6599730], [75.2941, 76.298, 6690609], [76.298, 77.302, 6780442], [77.302, 78.3059, 6867430], [78.3059, 79.3098, 6959845], [79.3098, 80.3137, 7038970], [80.3137, 81.3176, 7118364], [81.3176, 82.3216, 7198626], [82.3216, 83.3255, 7273508], [83.3255, 84.3294, 7353458], [84.3294, 85.3333, 7437666], [85.3333, 86.3373, 7514222], [86.3373, 87.3412, 7596157], [87.3412, 88.3451, 7685676], [88.3451, 89.349, 7760480], [89.349, 90.3529, 7836324], [90.3529, 91.3569, 7914154], [91.3569, 92.3608, 7984592], [92.3608, 93.3647, 8058147], [93.3647, 94.3686, 8143172], [94.3686, 95.3726, 8225658], [95.3726, 96.3765, 8306471], [96.3765, 97.3804, 8385817], [97.3804, 98.3843, 8446081], [98.3843, 99.3882, 8515661], [99.3882, 100.3922, 8595649], [100.3922, 101.3961, 8665957], [101.3961, 102.4, 8780310], [102.4, 103.4039, 8823415], [103.4039, 104.4078, 8892485], [104.4078, 105.4118, 8960137], [105.4118, 106.4157, 9037720], [106.4157, 107.4196, 9118347], [107.4196, 108.4235, 9195839], [108.4235, 109.4275, 9278376], [109.4275, 110.4314, 9361333], [110.4314, 111.4353, 9441164], [111.4353, 112.4392, 9513884], [112.4392, 113.4431, 9565940], [113.4431, 114.4471, 9628529], [114.4471, 115.451, 9687381], [115.451, 116.4549, 9742042], [116.4549, 117.4588, 9796665], [117.4588, 118.4627, 9865679], [118.4627, 119.4667, 9917287], [119.4667, 120.4706, 9976651], [120.4706, 121.4745, 10036954], [121.4745, 122.4784, 10091054], [122.4784, 123.4824, 10149933], [123.4824, 124.4863, 10193695], [124.4863, 125.4902, 10233165], [125.4902, 126.4941, 10260896], [126.4941, 127.498, 10311379], [127.498, 128.502, 10367831], [128.502, 129.5059, 10434627], [129.5059, 130.5098, 10509680], [130.5098, 131.5137, 10554762], [131.5137, 132.5177, 10612940], [132.5177, 133.5216, 10656445], [133.5216, 134.5255, 10694216], [134.5255, 135.5294, 10740498], [135.5294, 136.5333, 10787178], [136.5333, 137.5373, 10825769], [137.5373, 138.5412, 10849369], [138.5412, 139.5451, 10868066], [139.5451, 140.549, 10914409], [140.549, 141.5529, 10957929], [141.5529, 142.5569, 11003964], [142.5569, 143.5608, 11057033], [143.5608, 144.5647, 11088904], [144.5647, 145.5686, 11107076], [145.5686, 146.5726, 11118792], [146.5726, 147.5765, 11121496], [147.5765, 148.5804, 11116855], [148.5804, 149.5843, 11100191], [149.5843, 150.5882, 11082416], [150.5882, 151.5922, 11068273], [151.5922, 152.5961, 11058037], [152.5961, 153.6, 11057471], [153.6, 154.6039, 11061172], [154.6039, 155.6078, 11071208], [155.6078, 156.6118, 11091822], [156.6118, 157.6157, 11113874], [157.6157, 158.6196, 11125667], [158.6196, 159.6235, 11134457], [159.6235, 160.6275, 11120833], [160.6275, 161.6314, 11124260], [161.6314, 162.6353, 11126463], [162.6353, 163.6392, 11126756], [163.6392, 164.6431, 11131869], [164.6431, 165.6471, 11137199], [165.6471, 166.651, 11133550], [166.651, 167.6549, 11126720], [167.6549, 168.6588, 11106672], [168.6588, 169.6628, 11107759], [169.6628, 170.6667, 11131468], [170.6667, 171.6706, 11161810], [171.6706, 172.6745, 11181003], [172.6745, 173.6784, 11169112], [173.6784, 174.6824, 11153893], [174.6824, 175.6863, 11145384], [175.6863, 176.6902, 11115178], [176.6902, 177.6941, 11076945], [177.6941, 178.698, 11040201], [178.698, 179.702, 11001007], [179.702, 180.7059, 10966466], [180.7059, 181.7098, 10912534], [181.7098, 182.7137, 10858933], [182.7137, 183.7177, 10794175], [183.7177, 184.7216, 10734278], [184.7216, 185.7255, 10665200], [185.7255, 186.7294, 10611713], [186.7294, 187.7333, 10554762], [187.7333, 188.7373, 10502363], [188.7373, 189.7412, 10441654], [189.7412, 190.7451, 10365457], [190.7451, 191.749, 10260435], [191.749, 192.7529, 10166071], [192.7529, 193.7569, 10083392], [193.7569, 194.7608, 10008931], [194.7608, 195.7647, 9924333], [195.7647, 196.7686, 9828104], [196.7686, 197.7726, 9711822], [197.7726, 198.7765, 9577627], [198.7765, 199.7804, 9439302], [199.7804, 200.7843, 9293507], [200.7843, 201.7882, 9180899], [201.7882, 202.7922, 9027950], [202.7922, 203.7961, 8905457], [203.7961, 204.8, 8814112], [204.8, 205.8039, 8716044], [205.8039, 206.8078, 8603077], [206.8078, 207.8118, 8483358], [207.8118, 208.8157, 8345030], [208.8157, 209.8196, 8210215], [209.8196, 210.8235, 8084148], [210.8235, 211.8275, 7923410], [211.8275, 212.8314, 7749873], [212.8314, 213.8353, 7584137], [213.8353, 214.8392, 7437250], [214.8392, 215.8431, 7288474], [215.8431, 216.8471, 7135785], [216.8471, 217.851, 6990555], [217.851, 218.8549, 6845667], [218.8549, 219.8588, 6705977], [219.8588, 220.8627, 6582536], [220.8627, 221.8667, 6474650], [221.8667, 222.8706, 6338251], [222.8706, 223.8745, 6181246], [223.8745, 224.8784, 6036981], [224.8784, 225.8824, 5891775], [225.8824, 226.8863, 5726395], [226.8863, 227.8902, 5552301], [227.8902, 228.8941, 5376620], [228.8941, 229.898, 5199191], [229.898, 230.902, 5016672], [230.902, 231.9059, 4819190], [231.9059, 232.9098, 4662588], [232.9098, 233.9137, 4478627], [233.9137, 234.9176, 4267266], [234.9176, 235.9216, 4050036], [235.9216, 236.9255, 3807243], [236.9255, 237.9294, 3577970], [237.9294, 238.9333, 3325567], [238.9333, 239.9373, 3023459], [239.9373, 240.9412, 2704080], [240.9412, 241.9451, 2423117], [241.9451, 242.949, 2173372], [242.949, 243.9529, 1943482], [243.9529, 244.9569, 1729961], [244.9569, 245.9608, 1518291], [245.9608, 246.9647, 1330571], [246.9647, 247.9686, 1176499], [247.9686, 248.9725, 1041896], [248.9725, 249.9765, 919637], [249.9765, 250.9804, 855355], [250.9804, 251.9843, 848342], [251.9843, 252.9882, 820422], [252.9882, 253.9922, 767235], [253.9922, 254.9961, 655912], [254.9961, 256.0, 1188037]]}}}} \ No newline at end of file diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/demo/visualization.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/demo/visualization.ipynb deleted file mode 100644 index 71f933fb78..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/demo/visualization.ipynb +++ /dev/null @@ -1,440 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "3f851980", - "metadata": {}, - "source": [ - "# NVFLARE Federated Statistics Visualization" - ] - }, - { - "cell_type": "markdown", - "id": "1aea2083", - "metadata": {}, - "source": [ - "#### Dependencies\n", - "\n", - "To run this example, you need to install the following dependencies:\n", - "* monai[itk]\n", - "* numpy\n", - "* pandas\n", - "* kaleido\n", - "* matplotlib\n", - "* jupyter\n", - "* notebook\n", - "\n", - "These are captured in the requirements.txt\n" - ] - }, - { - "cell_type": "markdown", - "id": "0b71dd55", - "metadata": {}, - "source": [ - "## Image Statistics Visualization\n", - "In this example, we demonstate how to visualize the results from the statistics of image data. The visualization requires json, pandas, matplotlib modules as well as nvflare visualization utlities. " - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "85f23acf", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "\n", - "import json\n", - "import pandas as pd\n", - "from nvflare.app_opt.statistics.visualization.statistics_visualization import Visualization" - ] - }, - { - "cell_type": "markdown", - "id": "151e23a8", - "metadata": {}, - "source": [ - "First, copy the resulting json file to demo directory. In this example, resulting file is called image_statistics.json. Then load json file\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "44f6bed2", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "with open('image_statistics.json', 'r') as f:\n", - " data = json.load(f)" - ] - }, - { - "cell_type": "markdown", - "id": "c4b83ddb", - "metadata": {}, - "source": [ - "Initialize the Visualization utilities\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "ab771712", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "vis = Visualization()\n" - ] - }, - { - "cell_type": "markdown", - "id": "49f976aa", - "metadata": {}, - "source": [ - "### Overall Statistics\n", - "vis.show_stats() will show the statistics for each features, at each site for each dataset\n", - "\n", - "vis.show_stats(data = data)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "20ea4dff", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "intensity\n", - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
counthistogram
site-4-train1345[[0.0, 1.0039, 7529944], [1.0039, 2.0078, 4251...
site-1-train3616[[0.0, 1.0039, 10894028], [1.0039, 2.0078, 919...
site-2-train6012[[0.0, 1.0039, 15941521], [1.0039, 2.0078, 340...
site-3-train10192[[0.0, 1.0039, 35420536], [1.0039, 2.0078, 343...
Global-train21165[[0.0, 1.0039, 69786029], [1.0039, 2.0078, 818...
\n", - "
" - ], - "text/plain": [ - " count histogram\n", - "site-4-train 1345 [[0.0, 1.0039, 7529944], [1.0039, 2.0078, 4251...\n", - "site-1-train 3616 [[0.0, 1.0039, 10894028], [1.0039, 2.0078, 919...\n", - "site-2-train 6012 [[0.0, 1.0039, 15941521], [1.0039, 2.0078, 340...\n", - "site-3-train 10192 [[0.0, 1.0039, 35420536], [1.0039, 2.0078, 343...\n", - "Global-train 21165 [[0.0, 1.0039, 69786029], [1.0039, 2.0078, 818..." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "vis.show_stats(data = data)" - ] - }, - { - "cell_type": "markdown", - "id": "521cbf6f", - "metadata": {}, - "source": [ - "### select features statistics using white_list_features \n", - "user can optionally select only show specified features via white_list_features arguments. In these image files, we only have one feature" - ] - }, - { - "cell_type": "markdown", - "id": "9ab23bcc", - "metadata": {}, - "source": [ - "### Histogram Visualization\n", - "We can use vis.show_histograms() to visualize the histogram. Before we do that, we need set some iPython display setting to make sure the graph displayed in full cell. " - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "4bada64b", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from IPython.display import display, HTML\n", - "display(HTML(\"\"))" - ] - }, - { - "cell_type": "markdown", - "id": "e415b49e", - "metadata": {}, - "source": [ - "The following command display histograms for numberic features. The result shows both main plot" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "53542cf9", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAHBCAYAAAA1lPLZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB8v0lEQVR4nO3deXwU5eEG8Gdm781uNvdJSLhBblFR8UBBkCqK1gspilorCla8a7XegtRiRflVK1agiPUEtCgiKniAUEBB5L5CAiTkvnaTveb9/TG7myw5SCBhyOb5tvPZmdmZnTdDzDz7vu+8IwkhBIiIiIhagax1AYiIiChyMFgQERFRq2GwICIiolbDYEFERESthsGCiIiIWg2DBREREbUaBgsiIiJqNQwWRERE1GoYLIiIiKjVMFgQRYD58+dDkiRkZ2drXZSTsnr1akiShNWrV2tdFCI6QQwWRB3Y9u3b8fTTT5/WgeTdd9/FK6+8onUxiKiZJD4rhKj98/v98Hq9MJlMkCSp2ft99NFHuP7667Fq1SoMHz687QrYTIqiwOPxwGg0QpbV7z1XXnklfv3119M6/BBRLb3WBSCik6fT6aDT6bQuxkmTZRlms1nrYhDRSWBTCFEEOLaPRVZWFq688kr88MMPOOecc2A2m9G1a1f8+9//Dtvn+uuvBwBccsklkCSpXv+G5cuX48ILL0RUVBTsdjuuuOIKbNu2LezYkyZNgs1mw+HDhzFu3DjYbDYkJibioYcegt/vD9v2vffew5AhQ2C32xEdHY3+/ftj9uzZofeP7WMxfPhwfPbZZzh48GCofFlZWaiqqkJUVBTuu+++eufi0KFD0Ol0mDFjxsmcUiI6QQwWRBFq7969uO6663DZZZdh1qxZiI2NxaRJk0LB4KKLLsIf//hHAMCf//xnLFy4EAsXLkSfPn0AAAsXLsQVV1wBm82GmTNn4i9/+Qu2b9+OCy64oF6zhN/vx+jRoxEfH4+//e1vuPjiizFr1iy8+eaboW1WrlyJ8ePHIzY2FjNnzsSLL76I4cOHY82aNY3+DI8//jgGDRqEhISEUPleeeUV2Gw2XHPNNXj//ffrhZf//Oc/EEJgwoQJrXEaiailBBG1e/PmzRMAxIEDB4QQQmRmZgoA4rvvvgttU1BQIEwmk3jwwQdD6z788EMBQKxatSrs8yorK0VMTIy48847w9bn5+cLh8MRtv7WW28VAMSzzz4btu3gwYPFkCFDQsv33XefiI6OFj6fr9GfY9WqVfXKc8UVV4jMzMx6265YsUIAEMuXLw9bP2DAAHHxxRc3egwialussSCKUGeccQYuvPDC0HJiYiJ69eqF/fv3H3fflStXoqysDOPHj0dRUVFo0ul0GDp0KFatWlVvn8mTJ4ctX3jhhWHHiomJgdPpxMqVK0/ip6o1cuRIpKWlYdGiRaF1v/76K3755Rf87ne/a5VjEFHLaRYsvvvuO4wdOxZpaWmQJAlLly5t0f5PP/10qM217hQVFdU2BSZqZzp37lxvXWxsLEpLS4+77549ewAAl156KRITE8OmL7/8EgUFBWHbm81mJCYmNnmse+65Bz179sSYMWPQqVMn3H777fjiiy9O5EcDoHb0nDBhApYuXQqXywUAWLRoEcxmc6jvCBGdeprdFeJ0OjFw4EDcfvvtuPbaa1u8/0MPPVTvG9KIESNw9tlnt1YRidq1xu4SEc24w1xRFABqP4uUlJR67+v14X86mnNHSlJSEjZv3owVK1Zg+fLlWL58OebNm4dbbrkFCxYsOO7+Dbnlllvw0ksvYenSpRg/fjzeffddXHnllXA4HCf0eUR08jQLFmPGjMGYMWMafd/tduPxxx/Hf/7zH5SVlaFfv36YOXNm6F57m80Gm80W2n7Lli3Yvn073njjjbYuOlHEaGzMi27dugFQw8DIkSNb7XhGoxFjx47F2LFjoSgK7rnnHvzzn//EX/7yF3Tv3r1FZQSAfv36YfDgwVi0aBE6deqEnJwcvPbaa61WXiJqudO2j8XUqVPx448/4r333sMvv/yC66+/HpdffnmoivZYb731Fnr27BnWpkxETQs2HZaVlYWtHz16NKKjozF9+nR4vd56+xUWFrb4WMXFxWHLsixjwIABANQvEk2Vsby8vNH3J06ciC+//BKvvPIK4uPjm/zCQkRt77QcICsnJwfz5s1DTk4O0tLSAKhNH1988QXmzZuH6dOnh21fU1ODRYsW4U9/+pMWxSVqtwYNGgSdToeZM2eivLwcJpMJl156KZKSkvD6669j4sSJOPPMM3HTTTchMTEROTk5+OyzzzBs2DDMmTOnRcf6/e9/j5KSElx66aXo1KkTDh48iNdeew2DBg0K3eLakCFDhuD999/HAw88gLPPPhs2mw1jx44NvX/zzTfjkUcewZIlS3D33XfDYDCc8PkgopN3WgaLrVu3wu/3o2fPnmHr3W434uPj622/ZMkSVFZW4tZbbz1VRSSKCCkpKXjjjTcwY8YM3HHHHfD7/Vi1ahWSkpJw8803Iy0tDS+++CJeeukluN1upKen48ILL8Rtt93W4mP97ne/w5tvvol//OMfKCsrQ0pKCm688UY8/fTToeG7G3LPPfdg8+bNmDdvHv7+978jMzMzLFgkJydj1KhR+PzzzzFx4sQTOg9E1HpOi2eFSJKEJUuWYNy4cQCA999/HxMmTMC2bdvqdQqz2Wz1OpONGDEC0dHRWLJkyakqMhGdRq655hps3boVe/fu1booRB3eaVljMXjwYPj9fhQUFBy3z8SBAwewatUqfPrpp6eodER0OsnLy8Nnn32Gxx9/XOuiEBE0DBZVVVVh3y4OHDiAzZs3Iy4uDj179sSECRNwyy23YNasWRg8eDAKCwvx9ddfY8CAAbjiiitC+7399ttITU1lhy2iDubAgQNYs2YN3nrrLRgMBtx1111aF4mIoGGw2LhxIy655JLQ8gMPPAAAuPXWWzF//nzMmzcPzz//PB588EEcPnwYCQkJOPfcc3HllVeG9lEUBfPnz8ekSZMi4smORNR83377LW677TZ07twZCxYsaHC8DSI69U6LPhZEREQUGU7bcSyIiIio/WGwICIiolZzyvtYKIqCI0eOwG63NzlULxEREZ0+hBCorKxEWlpak2PPnPJgceTIEWRkZJzqwxIREVEryM3NRadOnRp9/5QHC7vdDkAtWHR09Kk+PBEREZ2AiooKZGRkhK7jjTnlwSLY/BEdHc1gQURE1M4crxsDO28SERFRq2GwICIiolbDYEFERESt5rR8CBkREZ16fr8fXq9X62KQRgwGQ6s8HoPBgoiogxNCID8/H2VlZVoXhTQWExODlJSUkxpnisGCiKiDC4aKpKQkWK1WDl7YAQkh4HK5UFBQAABITU094c9isCAi6sD8fn8oVMTHx2tdHNKQxWIBABQUFCApKemEm0XYeZOIqAML9qmwWq0al4ROB8Hfg5Ppa8NgQUREbP4gAK3ze9CiYJGVlQVJkupNU6ZMOemCEBERUfvXomCxYcMG5OXlhaaVK1cCAK6//vo2KRwREVFLTZo0CePGjdO6GG1m/vz5iImJ0boYjWpRsEhMTERKSkpoWrZsGbp164aLL764rcpHRETUIrNnz8b8+fNDy8OHD8e0adNa/TiTJ0+GJEl45ZVXmtxu9erVkCSp1W7nvfHGG7F79+5W+ay2cMJ3hXg8Hrzzzjt44IEHmmyTcbvdcLvdoeWKiooTPWST8veXw+9TkNjZDqOZN7sQEXVUDoejzY+xZMkSrFu3Dmlpaa32mR6PB0aj8bjbWSyW0B0cp6MT7ry5dOlSlJWVYdKkSU1uN2PGDDgcjtCUkZFxoods0udvbMXSl39GRVF1m3w+ERGdXj766CP0798fFosF8fHxGDlyJJxOZ1hTyKRJk/Dtt99i9uzZoX6B2dnZAIBff/0VY8aMgc1mQ3JyMiZOnIiioqLjHvfw4cO49957sWjRIhgMhia3zc7OxiWXXAIAiI2NhSRJoevm8OHDMXXqVEybNg0JCQkYPXo0AODll19G//79ERUVhYyMDNxzzz2oqqoKfeaxTSFPP/00Bg0ahIULFyIrKwsOhwM33XQTKisrm3kmW9cJB4t//etfGDNmzHHT2mOPPYby8vLQlJube6KHbFKw0kSINvl4IqIOQQgBl8enySRa8Ac8Ly8P48ePx+23344dO3Zg9erVuPbaa+t9xuzZs3HeeefhzjvvDPUPzMjIQFlZGS699FIMHjwYGzduxBdffIGjR4/ihhtuaPK4iqJg4sSJePjhh9G3b9/jljMjIwMff/wxAGDXrl3Iy8vD7NmzQ+8vWLAARqMRa9aswRtvvAEAkGUZr776KrZt24YFCxbgm2++wSOPPNLkcfbt24elS5di2bJlWLZsGb799lu8+OKLxy1fWzihNoODBw/iq6++wuLFi4+7rclkgslkOpHDtEioMYbBgojohFV7/TjjyRWaHHv7s6NhNTbvspSXlwefz4drr70WmZmZAID+/fvX287hcMBoNMJqtSIlJSW0fs6cORg8eDCmT58eWvf2228jIyMDu3fvRs+ePRs87syZM6HX6/HHP/6xWeXU6XSIi4sDACQlJdXrdNmjRw/89a9/DVtXtz9IVlYWnn/+eUyePBn/+Mc/Gj2OoiiYP38+7HY7AGDixIn4+uuv8cILLzSrnK3phILFvHnzkJSUhCuuuKK1y3PCJFmNFi1JvERE1D4NHDgQI0aMQP/+/TF69GiMGjUK1113HWJjY5u1/5YtW7Bq1SrYbLZ67+3btw8bNmzAXXfdFVq3fPlyWK1WzJ49Gz/99FOjfQvHjBmD77//HgCQmZmJbdu2NVmOIUOG1Fv31VdfYcaMGdi5cycqKirg8/lQU1MDl8vV6EBmWVlZoVABqENyB4fnPtVaHCwURcG8efNw6623Qq8/jTpJsimEiOikWQw6bH92tGbHbi6dToeVK1di7dq1+PLLL/Haa6/h8ccfx/r165u1f1VVFcaOHYuZM2fWey81NRWKomDo0KGhdenp6fjnP/+JgoICdO7cObTe7/fjwQcfxCuvvILs7Gy89dZbqK5W+/odr/8FAERFRYUtZ2dn48orr8Tdd9+NF154AXFxcfjhhx9wxx13wOPxNBosjj2WJElQFOW4x28LLU4GX331FXJycnD77be3RXlOmATWWBARnSxJkprdHKE1SZIwbNgwDBs2DE8++SQyMzOxZMmSetsZjUb4/f6wdWeeeSY+/vhjZGVlNfoluW4NAKA2L4wcOTJs3ejRozFx4kTcdtttANQA0tDxAdQrQ0M2bdoERVEwa9YsyLLaDfKDDz447n6nkxZ33hw1ahSEEI22P2lFCv4kzBVERBFv/fr1mD59OjZu3IicnBwsXrwYhYWF6NOnT71ts7KysH79emRnZ6OoqAiKomDKlCkoKSnB+PHjsWHDBuzbtw8rVqzAbbfd1mgAiI+PR79+/cImg8GAlJQU9OrVq9GyZmZmQpIkLFu2DIWFhWF3eByre/fu8Hq9eO2117B//34sXLgw1KmzvYi4Z4WwwoKIKPJFR0fju+++w29+8xv07NkTTzzxBGbNmoUxY8bU2/ahhx6CTqfDGWecgcTEROTk5CAtLQ1r1qyB3+/HqFGj0L9/f0ybNg0xMTGhmoLWkp6ejmeeeQZ/+tOfkJycjKlTpza67cCBA/Hyyy9j5syZ6NevHxYtWoQZM2a0annamiROcdtBRUUFHA4HysvLER0d3Wqf+85ffkR5YTWufehMpHaPabXPJSKKZDU1NThw4AC6dOkCs9msdXFIY039PjT3+h05NRbsvElERKS5iAkWtbf+MFkQERFpJYKChfoqtLm7hoiIiBBBwSKYLFhfQUREpJ2ICRa1zwphtCAiItJKxAULVlkQERFpJ2KCBTjyJhERkeYiJlhw5E0iIiLtRUywCGKFBRERkXYiJlgEx7FgUwgRUcc2adIkjBs3TutitJnVq1dDkiSUlZVpXZQGRVCwCMwwVxARdWizZ8/G/PnzQ8vDhw/HtGnTWuWzFy9ejFGjRiE+Ph6SJGHz5s3H3Sc7O7vZ2zbH+eefj7y8PDgcjlb5vNYWMcGC41gQEREAOBwOxMTEtMlnO51OXHDBBZg5c2arf7bH42nWdkajESkpKXVGnD69REywqB15k9GCiKgj+Oijj9C/f39YLBbEx8dj5MiRcDqdYU0hkyZNwrfffovZs2dDkiRIkoTs7GwAwK+//ooxY8bAZrMhOTkZEydORFFRUZPHnDhxIp588kmMHDmy2eXs0qULAGDw4MGQJAnDhw8PlW3cuHF44YUXkJaWFnr0+sKFC3HWWWfBbrcjJSUFN998MwoKCkKfd2xTyPz58xETE4MVK1agT58+sNlsuPzyy5GXl9fsMramiAsWrLIgIjoJQgAepzZTC/rI5eXlYfz48bj99tuxY8cOrF69Gtdee229fnazZ8/GeeedhzvvvBN5eXnIy8tDRkYGysrKcOmll2Lw4MHYuHEjvvjiCxw9ehQ33HBDa59R/O9//wMAfPXVV8jLy8PixYtD73399dfYtWsXVq5ciWXLlgEAvF4vnnvuOWzZsgVLly5FdnY2Jk2a1OQxXC4X/va3v2HhwoX47rvvkJOTg4ceeqjVf5bm0Gty1DYQ6rzJZEFEdOK8LmB6mjbH/vMRwBjVrE3z8vLg8/lw7bXXIjMzEwDQv3//ets5HA4YjUZYrVakpKSE1s+ZMweDBw/G9OnTQ+vefvttZGRkYPfu3ejZs+dJ/jC1EhMTAQDx8fFhZQCAqKgovPXWWzAajaF1t99+e2i+a9euePXVV3H22WejqqoKNputwWN4vV688cYb6NatGwBg6tSpePbZZ1vtZ2iJiKmxAB9CRkTUYQwcOBAjRoxA//79cf3112Pu3LkoLS1t9v5btmzBqlWrYLPZQlPv3r0BAPv27cOiRYvC3vv++++b9bmTJ08O2+94+vfvHxYqAGDTpk0YO3YsOnfuDLvdjosvvhgAkJOT0+jnWK3WUKgAgNTU1LDmk1Mp4mosiIjoJBisas2BVsduJp1Oh5UrV2Lt2rX48ssv8dprr+Hxxx/H+vXrm7V/VVUVxo4d22AnzNTUVCiKgqFDh4bWpaenN+tzn3322RY1QURFhdfQOJ1OjB49GqNHj8aiRYuQmJiInJwcjB49usnOnQaDIWxZkiTNhl+ImGARxHEsiIhOgiQ1uzlCa5IkYdiwYRg2bBiefPJJZGZmYsmSJfW2MxqN8Pv9YevOPPNMfPzxx8jKyoJe3/Cl0G63t7hMSUlJSEpKqnd8APXK0JCdO3eiuLgYL774IjIyMgAAGzdubHE5tBQxTSEc0puIqONYv349pk+fjo0bNyInJweLFy9GYWEh+vTpU2/brKwsrF+/HtnZ2SgqKoKiKJgyZQpKSkowfvx4bNiwAfv27cOKFStw2223NRkASkpKsHnzZmzfvh0AsGvXLmzevBn5+fmN7pOUlASLxRLqIFpeXt7otp07d4bRaMRrr72G/fv349NPP8Vzzz3XgjOjvYgJFnwIGRFRxxEdHY3vvvsOv/nNb9CzZ0888cQTmDVrFsaMGVNv24ceegg6nQ5nnHFGqGkhLS0Na9asgd/vx6hRo9C/f39MmzYNMTExkOXGL42ffvopBg8ejCuuuAIAcNNNN2Hw4MF44403Gt1Hr9fj1VdfxT//+U+kpaXh6quvbnTbxMREzJ8/Hx9++CHOOOMMvPjii/jb3/7WgjOjPUmc4itxRUUFHA4HysvLER0d3Wqf+8krP+PQzlKMvO0M9BqacvwdiIgINTU1OHDgALp06QKz2ax1cUhjTf0+NPf6HTE1FrXjWLDGgoiISCsRFCw4pDcREZHWIiZYcBwLIiIi7UVMsJA4pjcREZHmIiZYhGosmCuIiIg0EzHBIlRjwWBBRESkmQgKFuorx7EgIiLSTsQEiyDmCiIiIu1ETLCQZHayICIi0lrkBIvAK3MFEVHHNmnSJIwbN07rYrSZ+fPnIyYmRutiNCpigkWwkwWDBRFRxzZ79mzMnz8/tDx8+HBMmzbtpD/X6/Xi0UcfRf/+/REVFYW0tDTccsstOHKk6cfMr169GpIkoays7KTLAAA33ngjdu/e3Sqf1RYiJliw8yYREQGAw+Fok2/0LpcLP/30E/7yl7/gp59+wuLFi7Fr1y5cddVVrfL5Ho+nWdtZLJZ6j2Y/nURcsODtpkREHcNHH32E/v37w2KxID4+HiNHjoTT6QxrCpk0aRK+/fZbzJ49G5IkQZIkZGdnAwB+/fVXjBkzBjabDcnJyZg4cSKKiooaPZ7D4cDKlStxww03oFevXjj33HMxZ84cbNq0CTk5OQ3uk52djUsuuQQAEBsbC0mSMGnSJABqTcrUqVMxbdo0JCQkYPTo0QCAl19+OVQrkpGRgXvuuQdVVVWhzzy2KeTpp5/GoEGDsHDhQmRlZcHhcOCmm25CZWXlCZ7ZkxMxwaK2KYTJgojoRAkh4PK6NJla8vc7Ly8P48ePx+23344dO3Zg9erVuPbaa+t9xuzZs3HeeefhzjvvRF5eHvLy8pCRkYGysjJceumlGDx4MDZu3IgvvvgCR48exQ033NCi81VeXg5JkhqtIcnIyMDHH38MANi1axfy8vIwe/bs0PsLFiyA0WjEmjVrQo9el2UZr776KrZt24YFCxbgm2++wSOPPNJkOfbt24elS5di2bJlWLZsGb799lu8+OKLLfpZWou+pTscPnwYjz76KJYvXw6Xy4Xu3btj3rx5OOuss9qifM0WqrEgIqITVu2rxtB3h2py7PU3r4fVYG3Wtnl5efD5fLj22muRmZkJAOjfv3+97RwOB4xGI6xWK1JSUkLr58yZg8GDB2P69OmhdW+//TYyMjKwe/du9OzZ87hlqKmpwaOPPorx48c3+hhxnU6HuLg4AEBSUlK9ANKjRw/89a9/DVtXtz9IVlYWnn/+eUyePBn/+Mc/Gi2LoiiYP38+7HY7AGDixIn4+uuv8cILLxz352htLaqxKC0txbBhw2AwGLB8+XJs374ds2bNQmxsbFuVr/n4EDIiog5j4MCBGDFiBPr374/rr78ec+fORWlpabP337JlC1atWgWbzRaaevfuDUD99r9o0aKw977//vuw/b1eL2644QYIIfD666+H1gebVmw2G/r27XvccgwZMqTeuq+++gojRoxAeno67HY7Jk6ciOLiYrhcrkY/JysrKxQqACA1NRUFBQXHPX5baFGNxcyZM5GRkYF58+aF1nXp0qXVC3Uiah+bzqYQIqITZdFbsP7m9Zodu7l0Oh1WrlyJtWvX4ssvv8Rrr72Gxx9/HOvXN6/sVVVVGDt2LGbOnFnvvdTUVCiKgqFDa2tu0tPTQ/PBUHHw4EF88803YbUVb731FqqrqwEABoPhuOWIiooKW87OzsaVV16Ju+++Gy+88ALi4uLwww8/4I477oDH44HV2nCNzrHHkiQJiqLNN+0WBYtPP/0Uo0ePxvXXX49vv/0W6enpuOeee3DnnXc2uo/b7Ybb7Q4tV1RUnHhpmxBqCWGuICI6YZIkNbs5QmuSJGHYsGEYNmwYnnzySWRmZmLJkiX1tjMajfD7/WHrzjzzTHz88cfIysqCXt/wpbBuDUBQMFTs2bMHq1atQnx8fNj7dQNI3eMDqFeGhmzatAmKomDWrFmQZbVR4YMPPjjufqeTFjWF7N+/H6+//jp69OiBFStW4O6778Yf//hHLFiwoNF9ZsyYAYfDEZoyMjJOutANktl5k4ioo1i/fj2mT5+OjRs3IicnB4sXL0ZhYSH69OlTb9usrCysX78e2dnZKCoqgqIomDJlCkpKSjB+/Hhs2LAB+/btw4oVK3Dbbbc1GgC8Xi+uu+46bNy4EYsWLYLf70d+fj7y8/ObvFU0MzMTkiRh2bJlKCwsDLvD41jdu3eH1+vFa6+9hv3792PhwoWhTp3tRYuChaIoOPPMMzF9+nQMHjwYf/jDH3DnnXc2+UM/9thjKC8vD025ubknXeiGcORNIqKOIzo6Gt999x1+85vfoGfPnnjiiScwa9YsjBkzpt62Dz30EHQ6Hc444wwkJiYiJycHaWlpWLNmDfx+P0aNGoX+/ftj2rRpiImJCdUUHOvw4cP49NNPcejQIQwaNAipqamhae3atY2WNT09Hc888wz+9Kc/ITk5GVOnTm1024EDB+Lll1/GzJkz0a9fPyxatAgzZsxo+QnSkCRa8BU/MzMTl112Gd56663Qutdffx3PP/88Dh8+3KzPqKiogMPhQHl5eaO9aE/EqoU7sH1NHoZe1RVn/Sar1T6XiCiS1dTU4MCBA+jSpQvMZrPWxSGNNfX70Nzrd4tqLIYNG4Zdu3aFrdu9e3foVh9NsSmEiIhIcy0KFvfffz/WrVuH6dOnY+/evXj33Xfx5ptvYsqUKW1VvmbjMBZERETaa1GwOPvss7FkyRL85z//Qb9+/fDcc8/hlVdewYQJE9qqfM0Wut1UYY0FERGRVlo88uaVV16JK6+8si3KcnJCDyHTthhEREQdWcQ8K0TimN5ERESai5hgUTukN6ssiIiItBIxwSJYYcFYQUREpJ3ICRbsZEFERKS5iAkWwZ+EuYKIiEg7ERMsOKQ3EREBwKRJkzBu3Diti9Fm5s+fj5iYGK2L0ajICRYSm0KIiAiYPXs25s+fH1oePnw4pk2b1iqf/fTTT6N3796IiopCbGwsRo4cedxHta9evRqSJKGsrKxVynDjjTdi9+7drfJZbSFiggW7WBAREQA4HI42+0bfs2dPzJkzB1u3bsUPP/yArKwsjBo1CoWFhSf92U09IbUui8WCpKSkkz5eW4mYYCHxthAiopMmhIDicmkytfRZTx999BH69+8Pi8WC+Ph4jBw5Ek6nM6wpZNKkSfj2228xe/ZsSJIESZKQnZ0NAPj1118xZswY2Gw2JCcnY+LEiSgqKmrymDfffDNGjhyJrl27om/fvnj55ZdRUVGBX375pcHts7OzcckllwAAYmNjIUkSJk2aBECtSZk6dSqmTZuGhIQEjB49GgDw8ssvo3///oiKikJGRgbuueeesEetH9sU8vTTT2PQoEFYuHAhsrKy4HA4cNNNN6GysrJF57O1tHjkzdNWqMaCyYKI6ESJ6mrsOnOIJsfu9dMmSFZrs7bNy8vD+PHj8de//hXXXHMNKisr8f3339e7BsyePRu7d+9Gv3798OyzzwIAEhMTUVZWhksvvRS///3v8fe//x3V1dV49NFHccMNN+Cbb75pVhk8Hg/efPNNOBwODBw4sMFtMjIy8PHHH+O3v/0tdu3ahejoaFgsltD7CxYswN133401a9aE1smyjFdffRVdunTB/v37cc899+CRRx7BP/7xj0bLsm/fPixduhTLli1DaWkpbrjhBrz44ot44YUXmvWztKaICRassCAi6jjy8vLg8/lw7bXXhp6w3b9//3rbORwOGI1GWK1WpKSkhNbPmTMHgwcPxvTp00Pr3n77bWRkZGD37t3o2bNno8detmwZbrrpJrhcLqSmpmLlypVISEhocFudToe4uDgAQFJSUr0mmh49euCvf/1r2Lq6/UGysrLw/PPPY/LkyU0GC0VRMH/+fNjtdgDAxIkT8fXXXzNYnIxQUwhH3iQiOmGSxYJeP23S7NjNNXDgQIwYMQL9+/fH6NGjMWrUKFx33XWIjY1t1v5btmzBqlWrYLPZ6r23b98+bNiwAXfddVdo3fLly3HhhRcCAC655BJs3rwZRUVFmDt3Lm644QasX78eSUlJGDNmDL7//nsAQGZmJrZt29ZkOYYMqV879NVXX2HGjBnYuXMnKioq4PP5UFNTA5fLBWsjNTpZWVmhUAEAqampKCgoOP6JaAMREyzAGgsiopMmSVKzmyO0pNPpsHLlSqxduxZffvklXnvtNTz++OPHvUMjqKqqCmPHjsXMmTPrvZeamgpFUTB06NDQuvT09NB8VFQUunfvju7du+Pcc89Fjx498K9//QuPPfYY3nrrLVRXVwMADAbDccsRFRUVtpydnY0rr7wSd999N1544QXExcXhhx9+wB133AGPx9NosDj2WJIkQVGU4x6/LURMsAg9Np3JgoioQ5AkCcOGDcOwYcPw5JNPIjMzE0uWLKm3ndFohN/vD1t35pln4uOPP0ZWVhb0+oYvhXVrAJqiKArcbjeA8ABS9/gA6pWhIZs2bYKiKJg1axZkWb2/4oMPPmhWOU4XEXRXiPrKzptERJFv/fr1mD59OjZu3IicnBwsXrwYhYWF6NOnT71ts7KysH79emRnZ6OoqAiKomDKlCkoKSnB+PHjsWHDBuzbtw8rVqzAbbfd1mgAcDqd+POf/4x169bh4MGD2LRpE26//XYcPnwY119/faNlzczMhCRJWLZsGQoLC8Pu8DhW9+7d4fV68dprr2H//v1YuHAh3njjjZafIA1FXLBgWwgRUeSLjo7Gd999h9/85jfo2bMnnnjiCcyaNQtjxoypt+1DDz0EnU6HM844A4mJicjJyUFaWhrWrFkDv9+PUaNGoX///pg2bRpiYmJCNQXH0ul02LlzJ37729+iZ8+eGDt2LIqLi/H999+jb9++jZY1PT0dzzzzDP70pz8hOTkZU6dObXTbgQMH4uWXX8bMmTPRr18/LFq0CDNmzGj5CdKQJE7xV/yKigo4HA6Ul5cjOjq61T534+fZWP/pfvQZlopLJ9ZPrEREVF9NTQ0OHDiALl26wGw2a10c0lhTvw/NvX5HTo1F8CdhjQUREZFmIiZYBLGPBRERkXYiJlhwSG8iIiLtRUyw4EPIiIiItBcxwSI0jgWrLIiIiDQTQcFCfRXaDDRGREREiKhgIR1/IyIiImpTERMsgnhXCBERkXYiJlhwHAsiIiLtRUywCN4WwhoLIiKqS5IkLF26tNnbT5o0CePGjTupY2ZnZ0OSJGzevPmkPqclnn76aQwaNOiUHa8xERMs+KwQIqKOJz8/H/fddx+6d+8Os9mM5ORkDBs2DK+//jpcLpfWxWvS/PnzERMT02qf99BDD+Hrr79utc87UZHz2HSZj00nIupI9u/fj2HDhiEmJgbTp09H//79YTKZsHXrVrz55ptIT0/HVVddpXUxT5rH4wk9er0pNpsNNpvtFJSoaRFTYxHEphAiohMnhIDX7ddkaunf73vuuQd6vR4bN27EDTfcgD59+qBr1664+uqr8dlnn2Hs2LEN7rd161ZceumlsFgsiI+Pxx/+8IcGH2X+zDPPIDExEdHR0Zg8eTI8Hk/ovS+++AIXXHABYmJiEB8fjyuvvBL79u1rdtlXr16N2267DeXl5ZAkCZIk4emnnwagPub9ueeewy233ILo6Gj84Q9/AAA8+uij6NmzJ6xWK7p27Yq//OUv8Hq9oc88tikk2KTzt7/9DampqYiPj8eUKVPC9mkLkVNjwZE3iYhOms+j4M37vtXk2H+YfTEMJl2zti0uLsaXX36J6dOnIyoqqsFtGhqGwOl0YvTo0TjvvPOwYcMGFBQU4Pe//z2mTp2K+fPnh7b7+uuvYTabsXr1amRnZ+O2225DfHw8XnjhhdDnPPDAAxgwYACqqqrw5JNP4pprrsHmzZsbfex6Xeeffz5eeeUVPPnkk9i1axcAhNU2/O1vf8OTTz6Jp556KrTObrdj/vz5SEtLw9atW3HnnXfCbrfjkUceafQ4q1atQmpqKlatWoW9e/fixhtvxKBBg3DnnXcet4wnKoKCBZMFEVFHsXfvXggh0KtXr7D1CQkJqKmpAQBMmTIFM2fODHv/3XffRU1NDf7973+HAsmcOXMwduxYzJw5E8nJyQAAo9GIt99+G1arFX379sWzzz6Lhx9+GM899xxkWcZvf/vbsM99++23kZiYiO3bt6Nfv37HLb/RaITD4YAkSUhJSan3/qWXXooHH3wwbN0TTzwRms/KysJDDz2E9957r8lgERsbizlz5kCn06F379644oor8PXXXzNYNAufQUZEdNL0Rhl/mH2xZsc+Wf/73/+gKAomTJgAt9td7/0dO3Zg4MCBYbUcw4YNg6Io2LVrVyhYDBw4EFarNbTNeeedh6qqKuTm5iIzMxN79uzBk08+ifXr16OoqAiKog77nJOT02Cw6Nu3Lw4ePAgAuPDCC7F8+fImf46zzjqr3rr3338fr776Kvbt24eqqir4fD5ER0c3+Tl9+/aFTldbC5SamoqtW7c2uc/JiphgwSG9iYhOniRJzW6O0FL37t0hSVKoGSGoa9euAACLxdKmxx87diwyMzMxd+5cpKWlQVEU9OvXL6wfRl2ff/55qG9Dc8p2bPPOjz/+iAkTJuCZZ57B6NGj4XA48N5772HWrFlNfo7BYAhbliQpFILaSgQFC95vSkTUUcTHx+Oyyy7DnDlzcO+99zbaz+JYffr0wfz58+F0OkP7rFmzBrIshzWrbNmyBdXV1aEQsG7dOthsNmRkZKC4uBi7du3C3LlzceGFFwIAfvjhhyaPm5mZWW+d0WiE3+9vVrnXrl2LzMxMPP7446F1wRqQ003k3BXCLhZERB3KP/7xD/h8Ppx11ll4//33sWPHDuzatQvvvPMOdu7cGdYEEDRhwgSYzWbceuut+PXXX7Fq1Srce++9mDhxYqgZBFBv8bzjjjuwfft2fP7553jqqacwdepUyLKM2NhYxMfH480338TevXvxzTff4IEHHmhx+bOyslBVVYWvv/4aRUVFTY670aNHD+Tk5OC9997Dvn378Oqrr2LJkiUtPuapEDHBgp03iYg6lm7duuHnn3/GyJEj8dhjj2HgwIE466yz8Nprr+Ghhx7Cc889V28fq9WKFStWoKSkBGeffTauu+46jBgxAnPmzAnbbsSIEejRowcuuugi3HjjjbjqqqtCt4PKsoz33nsPmzZtQr9+/XD//ffjpZdeanH5zz//fEyePBk33ngjEhMT8de//rXRba+66ircf//9mDp1KgYNGoS1a9fiL3/5S4uPeSpIogU3Dj/99NN45plnwtb16tULO3fubPYBKyoq4HA4UF5eftxOJy2xa30+vpq3HZ16x+LqaYNb7XOJiCJZTU0NDhw4gC5dusBsNmtdHNJYU78Pzb1+t7iPRd++ffHVV1/VfoD+9OimEXwIGSssiIiItNPiVKDX6xu851ZrEjtZEBERaa7FfSz27NmDtLQ0dO3aFRMmTEBOTk6T27vdblRUVIRNbYK5goiISHMtChZDhw7F/Pnz8cUXX+D111/HgQMHcOGFF6KysrLRfWbMmAGHwxGaMjIyTrrQDQl23uSzQoiIiLTTomAxZswYXH/99RgwYABGjx6Nzz//HGVlZfjggw8a3eexxx5DeXl5aMrNzT3pQjekgSHhiYiomfiljIDW+T04qZ6XMTEx6NmzJ/bu3dvoNiaTCSaT6WQO0yyhGguOvElE1GzBkRldLlebj1ZJp7/gWBrHjtjZEicVLKqqqrBv3z5MnDjxZD6mdYRqLJi6iYiaS6fTISYmBgUFBQDUcR4aeiooRTYhBFwuFwoKChATE9Pg4GLN1aJg8dBDD4XGRz9y5Aieeuop6HQ6jB8//oQL0Fo4PhYR0YkJ3ukXDBfUccXExJz0nZ8tChaHDh3C+PHjUVxcjMTERFxwwQVYt24dEhMTT6oQrSLUeVPjchARtTOSJCE1NRVJSUmhB2VRx2MwGE6qpiKoRcHivffeO+kDtpXaZ5AxWRARnQidTtcqFxbq2CLmWSFBzBVERETaiZhgIckcx4KIiEhrkRMsAq/MFURERNqJnGARui1E23IQERF1ZBETLGqfFcJkQUREpJWICRYcz4WIiEh7ERMsQuNYKKyxICIi0krEBAvWWBAREWkv4oIFu1gQERFpJ2KCRe2Q3kwWREREWomYYCHxWSFERESai6BgEZhhsiAiItJMxAQLsI8FERGR5iImWEjgyJtERERai5xgEfhJ2HmTiIhIOxETLIKYK4iIiLQTMcFC4kAWREREmouYYMHOm0RERNqLmGARGsdC43IQERF1ZBEULAIzrLIgIiLSTMQECzaFEBERaS9igkVwHAvebkpERKSdyAkWwZ+EuYKIiEgzERMsglhhQUREpJ2ICRYcx4KIiEh7kRMsQkN6a1sOIiKijixigkXwthDmCiIiIu1ETLAIjWOhMFoQERFpJWKCBZ+aTkREpL2ICRahIb3ZyYKIiEgzERQsAjPMFURERJqJoGARrLHQuCBEREQdWMQEi9pnhTBZEBERaSVigkXtAFnaloOIiKgji5hgURdrLYiIiLQRMcFCqvuTMFcQERFpInKCRbCTBVhjQUREpJWTChYvvvgiJEnCtGnTWqk4J6E2V7DCgoiISCMnHCw2bNiAf/7znxgwYEBrlueESXWCBRTNikFERNShnVCwqKqqwoQJEzB37lzExsa2dplOiFQnWQjWWRAREWnihILFlClTcMUVV2DkyJHH3dbtdqOioiJsahN1m0KYK4iIiDShb+kO7733Hn766Sds2LChWdvPmDEDzzzzTIsL1lJ1ayxYYUFERKSNFtVY5Obm4r777sOiRYtgNpubtc9jjz2G8vLy0JSbm3tCBT2esFzBKgsiIiJNtKjGYtOmTSgoKMCZZ54ZWuf3+/Hdd99hzpw5cLvd0Ol0YfuYTCaYTKbWKW1T6nbeZK4gIiLSRIuCxYgRI7B169awdbfddht69+6NRx99tF6oOJXCOm+yxoKIiEgTLQoWdrsd/fr1C1sXFRWF+Pj4eutPOXbeJCIi0lzkjLzJzptERESaa/FdIcdavXp1KxTj5IV1sWCVBRERkSYipsaCTSFERETai5hgwc6bRERE2ouYYAEc87wQIiIiOuUiKlgEk4XgQ8iIiIg0EVHBorbGgk0hREREWoioYBHswMkuFkRERNqIqGAR7MDJzptERETaiKxgEZxhriAiItJERAULyMEaC43LQURE1EFFVLCQQn0smCyIiIi0EFnBIjjDXEFERKSJyAoWMjtvEhERaSmiggURERFpK6KChcSRN4mIiDQVMcEid/LdUCorAACCnSyIiIg0ETHBombbNgifV11griAiItJExAQL6HShu0LYeZOIiEgbERMsJFkOda5griAiItJGxAQL6HS18wwWREREmoiYYCHJMqRAVQWbQoiIiLQRMcECsoxgVQVzBRERkTYiJ1joZEjBNhAGCyIiIk1ETLCQZF2oqoJNIURERNqImGChdt5koCAiItJSxASLsM6bCgMGERGRFiImWNStsWCsICIi0kbEBAtJrtt5k9GCiIhICxETLKDTgbmCiIhIWxETLOrWWDBYEBERaSNigkXYXSFMFkRERJqImGChPoSMNRZERERaiphgoT42nSNvEhERaSligkV4jQWTBRERkRYiJliEjWPBXEFERKSJiAkWkq7uXSFMFkRERFqImGABuXYcC/axICIi0kbEBIuwGguNy0JERNRRtShYvP766xgwYACio6MRHR2N8847D8uXL2+rsrWMXKePBR9CRkREpIkWBYtOnTrhxRdfxKZNm7Bx40ZceumluPrqq7Ft27a2Kl+zSTqZvTaJiIg0pm/JxmPHjg1bfuGFF/D6669j3bp16Nu3b6sWrMUkdt4kIiLSWouCRV1+vx8ffvghnE4nzjvvvEa3c7vdcLvdoeWKiooTPWTT6tZYMFcQERFposWdN7du3QqbzQaTyYTJkydjyZIlOOOMMxrdfsaMGXA4HKEpIyPjpArcGEnW8SFkREREGmtxsOjVqxc2b96M9evX4+6778att96K7du3N7r9Y489hvLy8tCUm5t7UgVulE4G2BRCRESkqRY3hRiNRnTv3h0AMGTIEGzYsAGzZ8/GP//5zwa3N5lMMJlMJ1fKZpBkHZtCiIiINHbS41goihLWh0IzOh2kwKxgsiAiItJEi2osHnvsMYwZMwadO3dGZWUl3n33XaxevRorVqxoq/I1m/oQMgVA6IWIiIhOsRYFi4KCAtxyyy3Iy8uDw+HAgAEDsGLFClx22WVtVb7m0+m0LgEREVGH16Jg8a9//autynHS+BAyIiIi7UXMs0JQp/Mmm0KIiIi0ETHBom6NBW8LISIi0kbEBIuwh5AxVxAREWkiYoKFpJMhcRwLIiIiTUVMsFBrLFTsvElERKSNiAkW6mPTA+NYMFcQERFpImKCBeTakTeZLIiIiLQRMcFCCnsImbZlISIi6qgiJlhA1oU6bzJYEBERaSNygkWdGgveFkJERKSNiAkW6kPIOPImERGRliImWECuO/ImERERaSFigoWkqzvyJgMGERGRFiImWKgPIVNnmSuIiIi0ETHBQn0IWXCALCYLIiIiLURMsKg7pDe7WhAREWkjYoJF3YeQscaCiIhIGxETLPjYdCIiIu1FTLBQH0LGx6YTERFpKWKChfoQMjaFEBERaSligoUUNqQ3ERERaSFigoX6EDJ1VigMGERERFqImGCh1lgExrHQtihEREQdVsQEi7qPTWeyICIi0kbEBAu1xkLFzptERETaiJhgAZ0OoaYQ5goiIiJNREywkGQ51HmTTSFERETaiJhgAT42nYiISHOREyzkOuNYMFcQERFpImKChdoUwhoLIiIiLUVMsAh7CJm2JSEiIuqwIiZY1H1seuDmECIiIjrFIiZYqDUWKjaFEBERaSNiggWH9CYiItJexAQL6GofQgY+hIyIiEgTERMspDq3mzJWEBERaSNigkX4AFnaFoWIiKijalGwmDFjBs4++2zY7XYkJSVh3Lhx2LVrV1uVrUXqjmPBZEFERKSNFgWLb7/9FlOmTMG6deuwcuVKeL1ejBo1Ck6ns63K13wc0puIiEhz+pZs/MUXX4Qtz58/H0lJSdi0aRMuuuiiVi1YS4WNvMnOm0RERJo4qT4W5eXlAIC4uLhWKcxJqVtjwWBBRESkiRbVWNSlKAqmTZuGYcOGoV+/fo1u53a74Xa7Q8sVFRUnesgm1b0rBAqH3iQiItLCCddYTJkyBb/++ivee++9JrebMWMGHA5HaMrIyDjRQzZNpws1hSissSAiItLECQWLqVOnYtmyZVi1ahU6derU5LaPPfYYysvLQ1Nubu4JFfR41BqLAHbeJCIi0kSLmkKEELj33nuxZMkSrF69Gl26dDnuPiaTCSaT6YQL2Gx1aiyEn00hREREWmhRsJgyZQreffddfPLJJ7Db7cjPzwcAOBwOWCyWNilgs9UdeZM1FkRERJpoUVPI66+/jvLycgwfPhypqamh6f3332+r8jWbJEmQJHWed4UQERFpo8VNIae1ULI4zctJREQUoSLnWSEAArGCNRZEREQaiahgAVmNFgwWRERE2oioYBGssWBTCBERkTYiK1gEaiw4QBYREZE2IitYhDpZMFgQERFpIaKCRTBZsI8FERGRNiIqWEi83ZSIiEhTkRUsAj8NayyIiIi0EVnBItgUwhoLIiIiTURUsAiNY8FgQUREpImICha1fSy0LQcREVFHFZHBgn0siIiItBExwWLEByNwtLoAAJtCiIiItBIxwQISICQFAIMFERGRViImWJh0JiihkTc1LQoREVGHFTHBwigbIQKJgn0siIiItBE5wUJnhJAYKIiIiLQUkcGCNRZERETaiJhgYdKZaoMFO28SERFpImKChUFnCPWxYOdNIiIibURMsDDJJojQw02ZLIiIiLQQMcHCqKtzVwhzBRERkSYiK1iwjwUREZGmIiZYqANksY8FERGRliImWITXWGhcGCIiog4qooJFaEhvVlkQERFpInKChcwaCyIiIq1FTLAIHyBL48IQERF1UBETLIw6I5TQAFlMFkRERFqIqGAhZNZYEBERaSlygoVcW2PBXEFERKSNyAkWdWosmCyIiIi0EVHBQuGQ3kRERJqKmGCh3hWidSmIiIg6Nr3WBWgtRtkYGtKbzwohovZCKAJetx+yToLOIEOS+A2J2rfICRa6usFC48IQUasTioCrwoPKkprQVFXihtvlhd+rwO9T4Au8Kn4BnV6GziBDp5ehN8jQG2XojTp1MsjQ6SUoivq5QhEQQkDxC/g8CrweP3xuP7weP7xuP3weBT6PH5AkyDIgyVKDAaCxLzWyLEHWydAZ1P081T7UuHxwO73wVPtCf7MkCdAbdTCYdNCbdDAE5i12A2yxZthiTYiKMcEWa1KXY0zQGSKm4pkiROQEix9m13beJKJ2w+9T4Cxzo6q0BlWlblRXelFd6UG104uaSi+qqzyB991Q/JH937gQgNethpnmstgNiIoxISbJitgUK2JToxCXGoWYJCtDB2mixcHiu+++w0svvYRNmzYhLy8PS5Yswbhx49qgaC1jyt0ABVkAWGNBdDpR/ArKC6tRmu9CZXENKktrUFWihojKkhq4KjzNvpNLkiVExRhhjzPDHmeGLc4Mi81Qr3ZCkiUofgG/1x+qxQjVRARqHxSfotY8yBLkwKskS9Ab5VBNQaj2ILAOABQhIPwCiiLCay2OqcAILgqotSJ+nwLFp+5nsuhhsuphshpgitLDZNFDCTSJqDUkfnjdCrxuH7xuP6orPKgqVcNVVZl67pylbvi8SiCIeVGUWxV+fAmITrQgLhA0YlOjEJcWhdgUK/QG3Yn9YxI1Q4uDhdPpxMCBA3H77bfj2muvbYsynRCDvnZIbyI69bwePyqLa1Ca70TJESdK85woyXOi9KgLiq/p/zZ1ejlQvW+CJdoIi80Is80Ai80Ai90Iq0MNE1EOI2Rd5H4LN5qb/ydZCAG3y6eGjZIalB51oTTPidJ8J0rzXXC7fCgvqEZ5QTUObCkK7SdJQHSCBXFp4YEjPi0qos8tnTotDhZjxozBmDFj2qIsJ8WkN9cZeZOdn4ham6fGh/LCalQUVof1cQjO11R5G91Xb5QRmxKF6AQLbHEm2GPN6mucGbZYMyx2AzsttpAkSTBHGWCOMiChky1QX6sSQu2PUpIXCHhH1JBXcsSpBo7CapQXhgcOo1mH9F6xyOgTh069YxGTbOW/CZ2QyOljoTPXPiuEI2QRnRC/X0FFoNmi7GidqcCF6srGg0OQwaRDTLI19G04LvBt2B5nhiTzInWqSJKEKIcJUQ4TMnrHhdY3FjiKDzvhqfbhwJaiUNiw2A1I6epASjcHUrs6kJhpZxMKNUubBwu32w232x1arqioaJPjGA1WKJIPACCObewkojCeGh9K82urzkvyXCjNd6KiqAZCaTyYm20GOBItsMebA7UOZtjjTLDHqzUPJque33JPY40FDkURKMypRO6OEhzaUYL8/RWorvSGBQ1ZLyGpczTSejjQZVAikrOi+W9NDWrzYDFjxgw888wzbX0YmPRmKHIlAI5jQRTk8/hRdKgKxYerQkGiJM+JqlJ3o/voTTrEJlsRE5hik61wJFngSLLCZImYSk6qQ5YlJGdFIzkrGmeNyYLfq6AwtxJ5e8uRt68M+fvLUV3pRf7+cuTvL8dPK3Jgjzej+5Ak9DwnGQmd7Fr/CHQaafO/Eo899hgeeOCB0HJFRQUyMjJa/ThGgxVCapvaEKL2wOf1o/iwE4UHK1BwsBIFBytRkudstAbCEm1EXKoVcSlqB76YFDVERMWY+E20g9MZZLUZpKsDg9EZQgiUF1Yjf185crYV48DWYlQW1+DnL3Pw85c5iE+PQs+hKeh1TgqiYkxaF5801ubBwmQywWRq+1802WAN3RXCCguKdH6fguLDVSjMqQyEiAqUHHZCaSBEWKKNSOxkC41vEBzrwBxl0KDk1B5JkoSYJCtikqzofV4qvB4/Dm4txt6NR3FgaxGKDzvx4+J9WLdkHzr1jkWvoSnoMiixRXe5UORo8b96VVUV9u7dG1o+cOAANm/ejLi4OHTu3LlVC9ciBgvk0GPT+W2LIoffp6DkiBMFBytQkFOJwoOVKD5S1eAtnGabAUmZdiRlRiOxs/oaFWNkDQS1KoNRh+5DktB9SBJqnF7s+6kAu9bnI29vOXJ3lCJ3Ryn0pt3oNigRvYamIL1XDG9l7UBaHCw2btyISy65JLQcbOa49dZbMX/+/FYrWIsZLJAlhTeEULvm9yk4srsMh3eXorKkBqX5rkZDhMmqR1KmHYmZ0aEwYYtlMwadWuYoA/pemI6+F6ajvLAau/+Xj53r8lFRWI1d6/Oxa30+jGYd0nrGolOvWHTqHYu4tCj+nkawFgeL4cOHn56dIw0WQBKAYLag9kEoAmUFLhQcrERhTu3U0HDOJqs+UANhR2JnNUjY483840ynFUeiBWdf0QVn/SYLRw9UYNe6fOzZdBRupw/ZvxQh+xf1DhN7vBldByai6+AEpHSLgcxbkSNK5DSA6WubQpgs6HQjhEBVqRsF2RU4ml2BowcqGg0RlmgjMvvFIzbFiuh4C0MEtTuSJIU6f154U08U5Vbi0K5SHN5VhiO7S1FZXIMt3+Riyze5iHIY0evcVPQ5PxUxyVati06tIHKChcECSVIAsI8Fac9d7UPBQTVABMOEq9xTbzu9QUZChg2JGXYkBmoj4tOiOJgURQxZlpCUGY2kzGicOSoTXrcfudtLsH9LIbJ/KYKz3IOfVhzETysOIrWbA73PT0X3IUns+NmORc6/nMECSXJBEn4okgFbvs7FwBGtf1sr0bGCd2gcPaAGiILsCpTmu+ptJ8kS4tOjkBQYLyA5KxqxKVZ2aqMOxWDSoevgRHQdnAi/V0H21iLsWJuHnG3FyNtXjrx95fj+/d3odmYS+l6QhpRuDtbWtTOREyz0ZsiyC10OLMP+rlfjhw/3wOowosdZyVqXjCJI8H7+ujURRblV8PuUettGJ5hDISIpS71LI/iETCJSx8vodmYSup2ZBGeZG7vW52PH2jyUHXVh17p87FqXj5Su0Rg8KhNdBiSwJq+diJhgsWp/FXQQyMz5Eq5Omcg3DsL37+9GZr94VqnRCauu9IT6RASDhNvlq7edKUofChDJWWq1rzXaqEGJidqnqBgTzhydicGjOiN/fwW2rzmC3f/LR/7+Cix/Yytikq3od3E6eg1N4Rgsp7mIueIu216KTpLauyLD9S2q089DeWE1tnydi7Ov6KJ18agd8Hr86oBTdTpYVhbX1NtOp1f7RSRnRSO5ixomHIkWVtcStQJJkpDazYHUbg6ce3VX/LLqEH799jDKjrrwwwd78OOSfeh2ZiL6XpCO1O5sJjkdRUywgNECXeD3S/J7MfSqrvjyX9uweWUO+l/cCWYbEy7VUhSB0jynGiAC/SKKDzc8/HVsirW2NqJLNOLTbdDp2S+CqK1FOUw4b1w3DLk8E7vW5WPb90dQfLgKu9cfxe71RxGbYsUZF6Sh17kpsNhYQ3i6iJhgIRss0AVH3vT70X1IEn768iCKcquwct42jJncn4/87cD8XgVHD1Ygb29Z4MFK5fBU12/SsDqMYTURSZnRfPAWkcaMZj36D++EfhenoyC7Ett/OIzdGwtQmu/Cmo/24sel+9BtcBLOuCAN6T1jWIuhsYj5iymbrNAFnxWi+CHJEi4e3wuf/P1n5GwrwfI3tuI3dw/gN80Owl3tQ/7+cuTtKcORvWUoyK6s18HSYNIhKdMeChHJWdGwxZo1KjERHY8kSUjuogb/Ydf1wO4NR7H9hyMozKnEng1HsWfDUTiSLOhzfip6nJWM6ASL1kXukCImWBiM1rAaCwBI6erAlVMHYtmcLcjZVoL1n+7H+dd217KY1Eac5W7k7S3Hkb1lyNtbhuJDVfUeRmexG5DWPQap3WOQ2t2BhE423upJ1E4ZLXr0uygd/S5KR8HBCmz/4Qh2bziK8oJqrFu6H+uW7kdiZzu6D1HvOnEkMmScKhETLHTmKOiDtV/+2m+m6b1icdntfbH8n1vx88ocZPaLR3rPWG0KSa1CKAKl+S7k7StT73vfW4aKovqdLKMTzGqQ6BGDtO4xcCSxgyVRJAoOwHX+b7tj76YC7P7fURzZXRoaJv/HJfuQ2NmOvhemoefQFN723cYiJlgYzFHQo7YppK6ugxPRZ1gqdqzJw1fzt2P8k0N5C2o74vcqKDhYERo8J29fGdzOY/pHSEB8mg1p3R2hIBEVY9KmwESkCaNZjzOGpeGMYWlwVXiwf3Mh9v1UgMO71JCxetEu/Lh0H/pekIZ+F3eCPY5Nn20hYq6uJktU6K4Q+Os/f+GC63vg8K5SVBTVYN2SfbhofK9TW0BqthqnF/mBAJG3r7zB/hF6g4zkLtFI7R6DlG7qMwnYyZKIgqzRxlBTSXWlB7vW52Pr6kOoKKrBTyty8PPKXHQbnIgBl2YgpWs0azNbUcT8JTbVqbFoKFgYzXoM/11vfPrKZmz97jB6nJ2M1O4xp7aQVI/iV1Ca70JhTiXy9pcjf185So44621nsRvUvhHdHEjtFoOEzjbo2D+CiJrBYjdi0MjOGHBpBrJ/KcIv3+Ti8O4y7N1UgL2bCpCUaceASzPQfUgSO/i3gogJFlazEaHAqdQfXhkAMnrHoc/5qdixNg8r396O3z4yhNXlp5CiCJTmO9VBqA5WovBgJYpyK+Hz1v/3ikm2IrW7IxQk2D+CiE6WLEvoOigRXQclouhQJX755hB2/+8oCg5W4qt527F28V70uygdfS9M58i5JyFigoXdrIeEQNJsJFgAwPm/7Y4je8pQXliNT1/djGsePJPDw7YBIQQqiqrVwacOVKLgYAUKcyvh89T/tzGYdKGRLIO1EhY7/6MmoraT0MmOS2/pg/Ou6YZt3x/G1m8Pw1Xuwf/+ewCbvjiIMy5Iw5mjOvMW9BMgCXHsTXltq6KiAg6HA+Xl5YiOjm61z132yxFU//ti9FumR3m8Ceeu2dx4GYqqsfilTXCWe5DcJRpXTxsMg4m9hE+G36egMKcSR/ao/SLy95Wjxumtt53epENihg1JnaMDjwm3IybZCpkPF6JmEl4v/OXl8JeWwldaClFTA6EogBCAEJAMBkhGEySjAbLJBMlohGy1QrbZIEdFQdLxv3Wqz+9TsO+nAmz5OhcFBysBALJeQp/zUnHm6EyOiYHmX78jpsYiyqRHTbDG4jhZKTrBgrF/HIQls37C0QMVWP7Prbji7gHQGdi21lyeah/y9peHRrI8ml0B/zFNGrJeQmKGXR18KtOOxMxohogOTni9UJxO+KucUJxVUKqqoDidUKqq4K+qglLlhOJyBabAvNMFf3kZ/MUl8JWUQCkvP6kyyFFRasiw26Cz2SGZzZD0ekh6PaDXQZJkQJIAWQYkdVAmhNZJ9ZYhSbX7NLKNpDdAMhogGY2Q9AZAUSD8PsCvqHex+f0QfkV9DYQkSa+HZNADer0alvSGUDkloyFQ3tr3ZLMJksUCuc4kmUyBoGWEZDAAOh2bFBuh08voeU4KepydjEM7S7Hx82wc2VOGbd8fwfY1eeg1NBlDLs9CTLJV66Ke9iKmxmJjdgn2z7kA/ZcJlMfoce66rcfdJ39/OT555Wf4PArSe8bg8rv6s1mkEdVVHuTtKceRwEiWRbmVDQ5AldpNHXwqpZsDiRl2doSKcEp1NfxlZfUmX2kp/EVF8BUWwVcUmIqLIVyu1jmwJEHncEAXGwvJYoYk60JBQHi9EG4PhMcD4XZDuN1QXC4Ib/0atA5HktSgUSdshCarBTprFKQoK3RRUWrtjsWi1vZYrGpYsVog22zQx8dDl5AAfWKiul2EhpUje9SAkbujFICaE7uflYwhYzIRn2bTuHSnXnOv3xETLHbkVeCXl4dhwH99qIjWYej/fm3Wfod2luDzN7bCW+OHI9GCc8d1Q7fBiZA6+Ldqr9uPI3vKkLujBId2lqD4cP07NUIDUAVGsoxJtkbsH5hIJ/x++CsqjgkI5Q2GhrqTcLtP6HiSyaTWGtiioIuyBebVpgo5ygrZGqVe0KxWyFaLGiLi4qGPj4MuPh666OgWN2koHg+UykoolZVqjUlVJfyVlRA1brX2wOeD8PkAIQK1Bgg0ryiBdSLU3AKhQAgBBNdB1DbHBNYJUbssfD416Hg8ED4fJJ0MyDr1VaeHJMtqbYJOfQWg1l54vOq+Pp8amHxetZxeL4S3znqvF6KmBkp1NZSaGgiXC0p1NYTHc0L/Pi0hmc3QJySoU2KCGjji1WVdXCx00Q7oHNHQRUdDjo5Wg4jcvr5w5B8ox6bPs5G9tTi0rvuQJJx3TbcO1UTS4YJFbokLa2YMw8D/1qDSJuOcjduavW/RoSp89n9bUFWq/pG0xZnQZWAi+pyfisQMe6uV8XTmdfuRv68ch/eU4sjuMhzNroDiD//ViEuLQlr3GKT1UMOELZZ31JyOGqtFCKtRqDOvlJXDX1Fx3CbERhkM0MU4oI+Jgc4RA11sDHQxMeoFJiEB+oRE6BMT1G+5Dod6YTGwZvBUEEIAweDR0OTxhF6V6upA05OzdnJVq+ur1eYp4aqGv6pKrY0qKoLirP+F47hkGbLdDl10NHR2O3SJCTCkpMKQmgJ9SkrYvGw6vf7GFOZUYuPybOz/uRAAoDPIGHxZZ5w5OrND9NPrcMGixOnBF09dgMHLKlFllXD2T9tbtH+N04tfvsnFlm8OhT31MrN/PM69uisSOkVWwPDUqA/pOry7DEd2l6IguxLKMY8Mt8WZkNEnDhl94tCpVyzv1DgFhN9f+8c90P/AH+qHEPyDr/ZN8FdWtWotAgDINpsaClowyVGsqeqoFJcLvuLiQJNXIfyh+SL4CgvV38mKCvgryqGUV7S4BkUXHw9DSgr0qWrg0CcnwZCUBH2dSbbZTvnvX9GhKvzw4W4c3lUGALDFmnDhjT3RdVDiKS3HqdbhgoXb58cHj12Es/5bAqdFwlk/tyxYBHk9fhzaUYLd/zuKfT8VqF/iJIR6BrfXjjuuCk/YaJaFBxsIErEmpPeMRVpPtVbCkcixI5pDKIr6zc5Z1XQgqKoKbeMPW1fntbX6IOj1avNBWAgIX9bHxoa/73CwJoHalOJ2w19eDqWiAv6KSvjLy+ArLIQvPx/evHx48/Pgy8uHNz8foqb+838aIlks0CclqjUdaWmBKRWG1FToA7UfsqX1myuEENj/cyHWfLQXlSVqWbsPScKFN/aM2DEwOlywAIAFjwzHOZ8ehcsEDNmy46Q/r+yoC+v/ux97NxaoKyQgrXsMYlKsyDwjHlkDE07LOxyCA1Hlh56tUY6Kwup629njzEjvGYO0njFI7xkLe7y5QwYJ4fHAH2x7r6yEv6ICSmUV/JV1Xisq1Tb5isr6653OE29GaIxeH+pAF+p7EHqNCrwXqF2IbaAWQYNvcUStRQihNtkdEzh8BQXwFRbAW1AAX0EhlIqKZn2eLiYG+lQ1bBjS02Hq2QPm3r1h6t79pEOHz+PHhs+z8fOXORCKgDnKgAtv7IEeZydH3H+DHTNY/Gk0zlmaA58MyA/ciV7X3gZ93Mk/yTR/fzk2Ls/GwToddwAgOtGCQSMy0Pv8VM2elld3NMvCg+qT/AoPVcHnPmZYcwmIS40KjGTpQGr3mIjqdBTqfFhSoo5vUFICf2mgD0FlBfyVVeprRZ0AEQgIzf1mdFw6XeDib1U7JB4bBkKdE+u+Z61dX+dVMhoj7o8SUWtTqqvVGo+jR+HNy4P3SB68R47Am5cHX34evIePNF0LKEkwZmbC1KsXLP37wTJoEMz9+kE2t3xQrMKcSny9YAeKD1cBALIGJODi8b0iqi9ahwwWC5/+LQZ+uB2GwDXVZTOg8zPPI2H0b9R7vk9S2VEXjh4oR+GhKuxcmwe3S+2LoTfpEB1vhiPRgpSuDmT0iUNCRut+Y6xxelGS50TJESdK85xwlnvgLKtB0aGqBkez1BtlJGdFIyUwJHZK12iYrO2nmltxuQIBoRT+slL4S0rUWxhLSuEvVdf5gvMlJfCXl590rYE6toEdOrsdcrQdOlvg1R6tjnlgjw4s20Odz2SbTV222dQxA07jMOBX/PAoHnj8HngVLzx+D9x+d73l4Hy9ZcUDRSihSQgBBXXmhQIFdeaFAgFRb/tj32/ocxraTxEKZEmGTtJBJ+nUeVkXWg7Oy5IMvawPvRplIwyyAUadEUZdnXlZXZYlGQIC6v/V/wHqt+bQcuBXK/he3eMF5/WSPrROL+tbtI1e1kMv6U/r35/2SAgBpbJSrfXIOwJffj482Qfh3r0LNTt3wV9SUn8nvR7m3r1hGTQI1rOGIOrcc6GLiWnW8fw+BT+tOIiNn2dD8QuYbQaMuasf0nqc/Bfc00GHDBbvz7gdGeVfYFdxOjJ2OJFeFPjR9DqYe/ZCypN/gWXQoFY5ltftx461edjydQ4qiup/441OtCA5Sx0QKqNPHJK7RDfYbOJ1++Gp8cEcZYAkS/C4fCgrdKHkiDMwVaHkiBokGhMczTKxsx1Jne1I7ByNmJTTZyAq4fernbjq1iSUlsJfGh4WfKW1751oLYLsCNydEBsLXVyc2jRgrxsQ7NBF22sDhD1aXT7NR2QUQsDlc6HCXYFyT3mzXis8FXB6nWo48HvhE77jH4g0pZf1MMgG6GU9TDpTaIoyRCHaGA270Y5oYzSiTdFNLtuNdshS+7qlUwu+oiLU7NoF944dqN6yBa7Nm+EvLArfSJJg7tcPUcPOR9T558M6aBAkY9N9KIqPVOGredtRlFsFWZZw4U090e+i9Db8SU6NjhksXpqCG53vIKfLjSgZcSe+euYujFhXDXNwXBy9HkkPPIC42ya12jeDYFOEs8yN4sNO5O0tQ+72knoP1pJ1EiRZgsGkQ5TDCJ9HgavCA++xTRZNsMWaEJcWhbjUKNjjzbDYjYhPt2k2mqUQAv7iYniPHlXbPgsK1WrJgoLw1+LiBp84ezySwaCGg7g46GNjoIuNCwSGWLXjYWxsYF0M9HFx7aLzoV/xo9RdinJ3OSo8FY2+HhsQKtwVrRoMJEhh39rrfps36Uz135ONMOjUC54ECbIkQ5bk8HlJgow688d5v6HPCe4no8584H1JktTfOeGHX/HDL/xQhKIu11lXd96n+EI1MnVrYjyKB16/N1QLEzwnwXKFzlNg+dj1xx4jWA6f4gu95xO+2jIdU7a2DnkSJNgMtlDgCIWOY5cDoSQ47zA54DA6oJNP35DdloQQ8B05Atfmzaj+eTNc69fBvWdv2DaS1Yqos8+GbfjFsI8eDX1cXIOf5fX4serfO7An0Eev70XpuPCGHu160MCOGSxeeQg3ls1FdqerkPX7hThQfgAPrXoAJTl7MGGVgmE71B/VNnw4Up56EobU1FY9fpCnxodDO0tRdtSFwtxK5GwrCbuF9Xis0UbEpUUhPs2mBom0KMSmRsFkObUjsAuvF968PHhyc+E9fFhttzySB29+fqANM79Ft48FR0rUxakBQR8XC11MbHhYCL4XGwvJ2n5uYxRCwOl1osBVgKOuoyisLkSBq6DeVFRdBL9oecgKMsiG0AWgodfghSO4bDPYaoNCnZCgl1ntrqVgk08wfHgVL3yKL/QaDD5uvxtuvxtVnio1YHoqUOmpDIXNhpZr/CfXZ0iChGhTNGJNsYg1x4bChsPkQIIlIWxKtCTCYXJE9O+S9+hRONeshXOtOoU1n+h0iDrvPERfeQWiR42CbA2/a1AIgZ9WHMS6T/YDAkjrEYPL/9Cv3d663yGDxQf/9xfcUPgqDiSNRJd7PgYAuP1uvLLpFSza/g5GbFZw20ol1AdDn5oK28UXwXHFFbAMGdJmo8H5/QqcZW5AAJ4aP1zlbuhNOlijjbBGG2Ew6uB2+aAoAiar/pQmWn9VFbw5OfDkHoI3NweenFx4D+Wqr3l5x69pkCToEuJhSEqGPjFRvbe8odf4uFbp56IFr99bPyhUq6+FrsJQmKj21b/zpiESpLBviC0JChY9bwGmpnn8nlAIqXDXCR7BEBKsBTsmlFR6KlHprWzx8fSyPhQy4i3xSLQkNhhA4i3xMOra5wU1SCgK3Lt2oeqHH1D5xQrUbKsdiFF2OBB7ww2I/d0EGJKTw/bL/qUIX769Dd4aP2xxJoy4pQ869W64puN01iGDxYdzZ+D6wy8iO+4CZP3xs7D3thZuxfPrn4dz+zbc+YUf3fMAuc5Prk9JgX3kSFjPHAzJZIKvsBDGrCxYBg06oR7CpwuhKPAVFMCbq4YFT24OvLmH1FqI3Fz4S0ub3F8ymWDI6ARjeifo01ID94qnBgatSYMhKfG47Y2nKyEESt2lKHQVqrUMdUJC3SBRUtNAB69G2A12JFoTkWRNCp8s6muiVf2jq5fbZ8iKGEIAih9QfIDiBfxedd7vVZcVvzoJvzqkd9i8cpz1wWHAFag9QpXQsOBhr0GShMDTzmrXyTpAZwR0JkBvDMwbAb2p/rzeDBgs4fufIJ/iQ5m7DKU1pSitKUW5pxzlbnUqc5ehqLooNBVWF6Lc3bIHwln1VsSYYuAwORBnjkOiNRGJlsTQfxtJFvU13hIPg3x6N2sCgPvAAVR89jnKly6F99AhdaVej+jLL0fcrbfA0r9/aNuSPCc+/8cvKA/c+n/GsFSc/9vu7apTfYcMFosXvIJrDzyFg9FDkPnAN/XeV4SClQdX4u+b/o6ikkPod0jG7/K6otNPhyCqGhma1mCAKSsLph49YDnzTBjSUiFqamDMyoKpT5/T4tuj4nLBe+RIKCx4cg8FaiFy4T106LjNFbq4OBgzMmDIyICxcwYMGZ1hzOgEQ0Zn6BMT2t24/gBQ7atusCkiOAWDg1dp3oOpDLJB/eNnCQ8NidZEJFuTQ+9ZDe1zALVWpSiA3w34AlPd+WOXw96rAfwe9dXnCV/2ewD/MSEgFATqBIKwZd8x79XdPwIfSGawqgHDEBV4tQDGwLzeDOgMdYKKSQ0moX2stfvozYHJFAgtxyzXed+jeFFcXRwKGnVDR1F1Uei9ouqiZv+3Bqi1enHmOKREpSDdlo50ezo62Tqhk60T0mxpSLImnVb/rQm/H1WrVqFk/gK4Nm4MrbcOHYqkRx6GpW9fAGoz+Y9L9uHXbw8DAKIcRnXEzsGJp8W15Hg6ZLD45L25uHrnQ8ix9kXnR9Y2ul2VpwrP/PgMvsj+AgCQKMdgivt8nH3ICP+2XYCiQJcQD/eOnfAVFDT6OfrERERdfBGihp4LXbQduthYmLp1gxwV1ao/l+J2q/dmHzqs9nU4fAjew4fhCSz7i4ub/gCdDoa0NDU8dM6oEyI6w9CpE3S29vGUPkUoKHeXh/8BcxWGfYsK/lFzepv/DIM4c1xtULDUCQqB0JBoTUSsKbZd/IcfRlEATxXgrgTcFeqrt7rOxbvOBT3sIl/nwh5aXxMeDsJCwDHr2/NFWzaoF2BZrz72XNYBku6Yeanx9aFlGWothFz7+PS6r8EaisADzMJfodaE+D2Bc+8JzHsC/x7e2n8rnNI/3/Xp6oYPU20AMdoAazxgjQOsCRCWWFQYLSjXG1Cm06NMllAifCj0u1BQU4LC6kK1xrC6AEWuomZ1brUZbLW1g4GajmP/O06wJMCgO7U1AtXbtqH03/9G+efLAa8XkCQ4xo1D4rT7Qk0kR/aU4puFO1FeoNZepPeKwQXX90RCp9P7b3GHDBafLXkHV2yZgsOmbkh/7KcmtxVCYHXuavz9p7/jQPkBAGpK7mTvhMFJgzG221j0j+8PY2EZ3Pv2ofrXX1H908/wl5dDMhpRs307RHXDbeqGTp1g6tEjNAKiuU8fmHp0ByQZssUMXVwc4POFD+1c5xkQ3qNH4T18BN5DaoBoKtwEyTabGhYyMtSmi4zOgdqHDBhSU0/L/g0evwdl7jKUucvCqlvrLpfWlIbCQnFNMXxK8zvBWvSWUEgICwqBWget/vA0SVGA8lyg/FDg26MJcB8TDsKmiobfq6kAPC1vL299UuCCY6q9COkD1ffBavxGl0119jWEX/RlfZ11+jrvBZf1TbzXyOfIulZpTjhlhFBrZHw1gMcFeF1qcPTWmfc41fm6gcTvCQTHasBbU2ef6tptQ4Gxps7kVrdp7TBjigai0wFHOhCdBsWejtKoGBSYrMjT63FIuHHYmY/DVYdDU3P7MwHqF4fgf/PptnR0ju6MzvbOyLBnIN2eDpOubQaw8h4+jIK/v4KKZcsAqEOPx99xB+Jvvw2y1Qqfx49NXxzEzytz4PcqkCTgjAvScPaVXRDlOD0H1eqQweLL5Uswav0kHDWkI/nx5j0rxKf48EX2F3j717exp3RPvfcdJgfOTzsfIzqPQL+EfkiLSoMkSVDcbrg2bETVd9+iZpsaMrwFBfAXFTVwlJMnWa0wpqfDkJ4OQ6dOgdd0dV2nTtC18rlsLr/ih9PnRKWnMtRzvcpThUpvJcpq6oQET3loPvjakj8OdcWYYkIdwhIsCUiwJoR1GAvO24xtmP6FaOSPeJ11oT/2jbwfXOepAmrK1am6rPW/8ct69Y+3yaZWkwcv3KGLfd0q7sbeM9Zuo6uzTVhoMIWv1wUCQXu6WFPThFADSmPBIzjvrgRcxYCrBHAWAdWlQE2Z+hqaytQ+Kccj64GYTCC+GxDfHYjrCmdMBgqsMSiQgYLqwlCNR7CvVHObOiVISIlKQWd751Dg6BzdGZnRmehk79QqoaN6yxYcfXEmqn/+GYBa0x3/+zsQc8MNkC0WVBRVY+3ifdj3k/oFUmeQ0feCNPS9MB1xaS2v/a4qrYE5ygB9G4wG3SGDxerVX2L46utRLCcg/sl9LdpXCIGSmhLsLt2Nrw5+hZUHV6LUXb9jo91oR++43uhs74wkaxJ6xfZCr7heSLAkwKw3w1dSAvfuPXDv36fWSBSXoPqXX+A9rLapKdXV6vj2kgTZalVHewwM5ayzqc+G0CUkwBgMD+mdYOiUDl1MTKtXxXsVL1xeF6p91XB5XajyVqnhwFsRCgqVnspQb/HQsrcy9H6Vt+qkyiBLcuhWNofJEerYFZyvGyISrYmIN8c3v4bB7w1cuAOTNzjvUi/mXled9xoKBMeuq1Y/I7i+reiMgKNT4FumBzDZ1clor5032dWgYIoGzI5j1tsDQSJandebeHE/hhACPkXA61fg9QdfFfj8Ap7AvNcn4FUUeH3h2xw771PU/XyKAp8iAvMC/tB6AX/gWH5FBLZRQusbW66dVz87fFltOtHJEvSyDL1Ogk6WYJBl9TWwbNTLsBh0sBh1MBt06nxg2WrUw2ExINqivtad7GYDdG09No4Qak1b5VGg4hBQfhioOKLOVxxRl0uz1ZqVxhiigNhMIDotMKWH5oU9DeVmGwr81SioLsRR51HkVuYipzIHhyoPIacyp8km07qhIzM6E1mOLGRFZyHLkYW0qLQWjfUhhEDlihUoeOlvoWuBLi4Ocbfeitibx0Nnt+PInlL8uGQf8vfXPv8kpasDZ1yQim5nJsFoPn6ts9+vYPGzq+D1Sbh8ytATCiZN6ZDB4sf1a3De8t/AJVlgvW8DEJNxUp/n9Dqxp3QPvjz4JTbkb8Desr1NVsVb9JbQxTB4/3esKRYx5piwV4fOBp3OACEjbIhkv/CHBgCqu76h97yKNzQcc/Be97pDNNf4a0KBweVzodpbDZfPFVp2ep0t6kx1PEbZCLvRHppsBhtiTDGINkWHzknd4BB8tRtskP3ewMW7KnDRrxsCnLXVuQ2+39ByIDT4W/aI5hMW7JVvsNZ2ggt2mguuM1qP6ShXZ50xCjDHqAHBEgPYUtQq+9NU8CIZvADWvbgG13t8Tb8fuij7FXiDF1Z/8EKubh98P7h96H1/45/r8yvwNLBf8PPqhgdqmt2kR7zNiKRoM1KizUhxmJEcmjchya4uG9vy9nhFASrzgOK9QMk+oHifOl+0Rw0dzanxMEQBUfGAJQ6wxKr9PiyxEOZYlBjNyNUBOcKNg74q5HrKcLC6EDlVh5v80mSQDehs74wsR5YaOqKz0D2mO7rFdGuyU6ni8aB86VIUz30L3txcAIBstcJxzTWInTABxi5ZyN1Rgl+/PYzsrcUQgSdQ6/QyMs6IQ1KmHY4kC4xmPQwmHQwmHWyxZljsBkiShPXvbcDG1ZUwSZUYf3cMogZc3KLTfTwdMlhs3L4bZ31wtrogyUDKACBtEBDXFbAF7iu2pwKZ56tVtC3k9Xuxv3w/dpbsRJ4zD0eqjmB78XbsK9/Xorb/041BNsCityDKEFUbDgyBgGC0BZZtsOvMsEl62CU97JBhhwybosAuBEw+TzNDQFWdbQLrT2LAqGaR9erF22irvZAHp+ByY4EgbF3gNRQSAu8385uLoqgXRp9f1Pu26z32Atqc94+5IPvqXKi9vvoX5IYusI0FBM8xF+S6AUDRuL9gWzHoJBh0cmBqfF6vk2E8Zl6tOZACNQdyaF4vq8vBGoR6yzo5sE4KvRrqfF5Dy8EaCgAN1ngEa0a8gX/HGq8fNV4/qj1+VHv9qPGq6yprfKio8aK82ouKavW1vNoLl6dl/z0m2ExIjzEjPdaCNIcFaTEWpMdakB5jQVK0CbFWIwy6NggfPo8aLoI1HBVHgIrD4fPVTd9O3xgBoNRkQ47FhoNmCw4ajMjWSzggKcgRbnia6GeSbopD96g0dLdloHd0FwyK6YYUc0LYbcXCr6Dim7UofvcTuLNzQ/tGnTMIcb8dg6izB8LlFNi52Y2dm2tQVtz0v4nJ4ENqJ4GDB2QI6DCq+3/RY9pMtQmzFbVpsPi///s/vPTSS8jPz8fAgQPx2muv4ZxzzmnVgp2IXw+X42//NwdTTJ/jbPFr4xuaHUD6WUBCD7XNLrbLMRcUs/oa/CZ6nAuHEAJV3iq142FNGUrdpaF7wcvd5epycH2g30HwgUrHTsGHKDU4qQMdQyf86giKeitMsgFGWQ+TpINR0sEk6WCSZBghwyrrYYUOVkmGVdLBKiRYhYBVUWBV/LD4/bD6PDAEO2UdW/1/bEfBNu6BrujMEAYLFEMU/HorFL0Vfr0Ffr0VPp366tVZ4dNb4JMt8OjU9R7ZAo9shlu2wCtb4JYtcEvm0KsHevjrVDMrSm1VtV+BWmWtCChCrWYObucXAv5AtbMi6u4T/llhAeDY4NABLsiAelEOXvAMgYulQdfIcmA79aJcfz998KItyzDoa7c/9v1jP1cv117sDbIEg77++0Zd4DN1Mgxy7bxeltrfXT9txONTQoGjqNKN/IoaHK2owdGKwHx5DfIralBQ4W52zU+0WY94mwmxVgPiokyIjzIiNsqI+Cgjoi16RJn0sAWmY+dPqkbE41JrPFwlQHVJ4LVUna8ubWC59Lidnv0A8vU6ZBsMyDboccBgQLbBgL0GA4r1DV8rknw+DHR7MLDGjQFuN87weGAK3ATkOmpEyZ4oVB02A4Fh4w02HxyZ1bBnVMMY7UOJPxO5nkEo9mai0p8InzDBKyzwCAucSiyA2nPUw74Ro56+HYhKOPHz1og2Cxbvv/8+brnlFrzxxhsYOnQoXnnlFXz44YfYtWsXkpKSWq1gJyKn2IWLXloFAEhDEc4y7MdZpkNIRyFiUQ5IErr6DiBGlLXoc32SAV7ZDJ9sglc2wSeb4ZPN8OvM8OlM8EtGSFAgCQUyFEjCD1kokBB8VddJQtSuEwp0wguztxQGvwvVhnh49FGQhT+wv6/OvD80b1Ba6RHfJ6FGMqFGsqAGJlQHXl0woxomOIUZrsCrUxhRFXxVTKhU1O1cQn11wgRXYPtqmOBHx3s+QfCbbfjFVg5926130ZRlGPTqhVNf5yIafrENXmCPudjWuWCrF+rGPjc8AIQ+V9fw+7wod0xCCJS6vDhSVo3DZdXqa2k1jpRX43BZDQ6XVqPY6T7Zhw7DqJcDIUMHm8kAm0mHKJMeUUY9rEYdrEYdLGHzgVfDsev0tfMGHfSN1aL4vWrHUk9lnTuyApPn2LuyArdw+9S7a0p9Luz1O7FX1GAvvNiqE9itE/Af89+HXgh08wtk+gQy/UCmXyClTCBmKyB2AsJTu73eKhCVJmBNl2DJjIUxIw1Sch/AngboDPA5y1GcU47s/UCNx4Bz77oapqyBJ3fSG9FmwWLo0KE4++yzMWfOHACAoijIyMjAvffeiz/96U+tVrATIYTAIx/9gjV7i5BfUdPgN0QZCgZI+9FLzkVX6Qi6SvlIl4pghhsWyQMzPLDADbN0+t6L7xZ66OGHThJh6zwwwAM9vNDDE1pW13kC62pgVAOBMKIaJlQHlmuEEdUIrBNG1MCISlhRKSyoggVOYYETZlTDCIHWrdaUJYR3QNPJkKXaDmi6OlXFslRb3awL7FdvmzrVxnXX62QZOrmRfaTANjoJOim8ajpYtV33c49dbujbdEPf4o/9ts0LMkUyvyJQXu1FidONEqf6Wuz0oNTpQbHTgxKnB5U1PlS5fXC6w19rvG3bD8aok2tDiFEHo06GUa8GcLVmS61RCzaD1b4XWKeX6+wTvp363zYgSxK8ihtHqvcg17UDOc7tyHHuQJWvrNFymTwC5+yVcfF2Gb2zPTB6wy9kXqMe1bF21MRFozouGt4oM3xmE3xmM2S/gujDBYg6WoIzPl6KGKu9Vc9ZmwQLj8cDq9WKjz76COPGjQutv/XWW1FWVoZPPvmk3j5utxtutzusYBkZGW0SLOry+hXkl6tVeB6/AkVBWM/q2urw+tXjPkXA7/dD8tUAvmpI3hpIvmro/DWQfDWQfS7I/hro/DUwKG7oFA8USYYiZPUVgUnSQUAKvMrwQ4aQdFAgQ0gyfNDDpXPAI1tg85XAqLjgl/RQJB380EGBDn5JV7ss6VAjR8Elq78sRlEDv6SHD3Xuvz/mX7PuohAidDEMtu2G9ygPXER14dvU9jKv21Yshy7Ex/ZED7YJB/epbXsObFtnn+AFmoioLp9fgdPtR5XHh6pA+AgFjxofnB4fXB6174jL40e1V12uXRd43xu+TvsmSQHJUArZdBSysUidDMWQ9JWQ9JWQ9bV3nBl8Aj0PCQzIFuh7UCCrADA2tzvff+ajz+ChrVry5gaLFnU9Lyoqgt/vR/IxD1hJTk7Gzp07G9xnxowZeOaZZ1pymFZh0MnIiLMiI+70GfaViIiaR6+T4bDKcLTiszSEEHD7FDVkeP2o9vjgdKvhw+tX4PGptxF7/CJwm3HtsidsOXA7cnDZp9TephzoYwUAihDqOGaBVyEAAQFFxEERXQPrAtt4BJQaQIEPPlTAL5XDLzmRG+dCTkIN/nt2NWTFifiqUsRUlSOuqgZxVR5Ya3yweP0we/yQBHAowYTcRBMeT2qbp3c3R5vf0/bYY4/hgQceCC0HayyIiIhOJUmSYDaoY3rEal2YCNaiYJGQkACdToejR4+GrT969ChSUlIa3MdkMsFkOj2HJyUiIqLW1aJeeEajEUOGDMHXX38dWqcoCr7++mucd955rV44IiIial9a3BTywAMP4NZbb8VZZ52Fc845B6+88gqcTiduu+22tigfERERtSMtDhY33ngjCgsL8eSTTyI/Px+DBg3CF198Ua9DJxEREXU8ETWkNxEREbWN5l6/2/DpMURERNTRMFgQERFRq2GwICIiolbDYEFERESthsGCiIiIWg2DBREREbUaBgsiIiJqNQwWRERE1GoYLIiIiKjVtPlj048VHOizoqLiVB+aiIiITlDwun28AbtPebCorKwEAGRkZJzqQxMREdFJqqyshMPhaPT9U/6sEEVRcOTIEdjtdkiS1GqfW1FRgYyMDOTm5vIZJG2E57ht8fy2LZ7ftsdz3La0Pr9CCFRWViItLQ2y3HhPilNeYyHLMjp16tRmnx8dHc1f6DbGc9y2eH7bFs9v2+M5bltant+maiqC2HmTiIiIWg2DBREREbWaiAkWJpMJTz31FEwmk9ZFiVg8x22L57dt8fy2PZ7jttVezu8p77xJREREkStiaiyIiIhIewwWRERE1GoYLIiIiKjVMFgQERFRq2GwICIiolbT7oNFUVERH2hGEY03brUtnl+i1nXKh/RuTdOnT8d7772HmpoaDBgwAA888ADOP/98rYsVkb744guYzWaYzWace+65WhenQ8jJyUF8fDyEELDZbBBCtOrzdTo6nt+2tXjxYqxduxYJCQkYPHgwRo8erXWRIs5pe45FO/X888+LxMRE8fbbb4t33nlHnHfeeeKcc84Rn332mdZFizjXXHONSE9PF927dxdGo1Hcf//9YufOnVoXK6I9+OCDok+fPqJ3795i2LBhYtOmTcLv92tdrIjB89u2HnvsMWG328V1110nBg4cKCwWi5g+fbpwuVxaFy1inM7nuF0Gi+rqanH55ZeLv//976F1hw8fFg8++KA444wzxJYtW7QrXIR57rnnxMCBA0Vubq7Izc0Vn3zyiUhLSxMTJ04UP//8s9bFi0iPPPKIyMzMFJ9//rmYO3euGDdunIiOjhYLFy4UTqdT6+K1ezy/bWvnzp2iW7duYsWKFUIIIcrKysTcuXOFLMvi+eefF1VVVRqXsP073c9xuwwWNTU14pxzzhGPPPJI2Pq9e/eKO++8U5x77rmitLRUm8JFAEVRQvOTJk0SN9xwQ9j7S5cuFQMGDBBTp04VR44cOdXFi3gjRowQM2fODFt3yy23iO7du4vFixfzm/VJ4vltW998841ITU0Vhw4dClv/6quvCp1OJz7++GMhRPjfGWqZ0/0ct8vOmzqdDllZWdi9ezeKiopC67t164YJEybA5/NhwYIFGpawfTt69CgAwOPxoKqqCnq92hXH6/UCAK6++mrceeedWL58OdasWQOAHeBagxACRUVFOHjwIGJjYwEANTU1AIAFCxagc+fOePHFF0P/PtQyPp+P57cNBf8GZGZmoqCgAFu2bAGgnncAuPfeezFp0iTcf//9UBSF/VlaSFGU0Pxpf441iTOtYN26dUKSJPH3v/+9Xir73e9+J8477zyNSta+Pf7446J3796iuLhYCCHExx9/LCRJEhs3bhRCqLVFQWPHjhUXXHCBJuWMZDfffLPo169faDl4zouLi4XVahV//etftSpau7R79+6w5d/97nc8v63o6NGjwu12h5arq6vFLbfcIi644AJx8OBBIYQQHo9HCKE2WWdmZoo333xTk7K2V++//37o91JRFOFyucSkSZNO23PcLmssAGDo0KGYMWMG/vSnP+Hjjz+G2+0Ovde9e3ckJSWFJTw6vhtvvBH/+Mc/8OabbyIuLg4AcPnll+Pqq6/Gtddei6qqKphMJng8HgDA7bffjn379uHQoUOssThBixcvxpIlS/D555+H1t1///1wuVy47777AKhPNHS73YiLi8Ndd92Fzz77DNXV1TznzfDwww/j+uuvx9GjR0Pna8qUKXC73Ty/reCpp57CZZddhnPOOQe/+c1vsH37dpjNZkyYMAF+vx9PPfUUXC4XDAYDAPVc6/V6+P1+jUvefjz88MO46aab0L9/fwCAJEmwWCy4+uqrAeC0PMftNlgAwKOPPoo77rgDd9xxB1599VWsW7cOO3bswLvvvotevXpBltv1j3fKeDwenHPOOdi1axe2bduGCy+8EOXl5VAUBVarFc8++yySk5MxfPhwVFdXw2g0AgDy8vLQtWtXJCYmslrzBFx77bW455578Oyzz+LKK6/ETTfdhB9++AFnnXUWJk+ejP/+97+YNWsWAIQek+zxeJCcnAyLxcJzfhxXX3013n77bbz11ltITk4Ona8zzjgDv//97/HZZ5/x/J6Exx57DP/617/w8MMP45577kFBQQFuvPFGvP/++xg1ahR+97vfYevWrZg8eXLYfhaLJfTFhZp2zTXX4N1338XatWtx+eWXh703btw4jBs3Dtu2bTv9zrFmdSWt6JFHHhHnnnuucDgcomvXrmL8+PFaF6ldmTt3rjAYDOKNN94QQgjx73//W1x22WWib9++YuTIkeKTTz4RX331lRgwYIDo27evePDBB8WcOXNEXFyceOKJJzQuffs0Z84cMWDAAJGTkyNcLpdYt26dOPfcc8Vll10m1q5dK1wul/jzn/8srFareP7558X3338vNmzYILp06SKeeeYZrYt/WnM6nWLIkCFi4MCBorKyUgghREFBgaiurg4tHzlyhOf3JLjdbnH++eeLOXPmhNZ5vV5x1VVXifPOO098/vnnwu/3i7feektkZmaKrl27it/+9rciKytLjB49WsOStw9+v19MmDBBGI1GsXnzZiGEEGvXrhUzZ84UTz31lPjPf/4jhFCb8ebOnXvanWNJiMio7zt69Chyc3MhSRKGDBmidXHaFZfLhSeffBJffvklunTpgl9//RW33HILYmJi8Omnn6Kqqgr33nsvrrrqKjz44IPYv38/fD4frrnmGkybNk3r4rdL999/P37++WesXr06tO67777DCy+8AJPJhDlz5iA1NRULFizAU089BZPJBJ/PhyuuuAKvv/66dgVvB/7v//4PTzzxBP785z/j4Ycfxrx587BgwYJQc8iMGTMwduxYeL1eLFq0iOe3hYQQKCwsxIgRIzB16lTcdddd8Hg8MBqNyMvLw8033wybzYZ//vOfSE1NRV5eHubMmQOdToe4uDjcf//9Wv8I7cLf/vY3vP/++5g4cSJqamowZ84c9O7dG4WFhfjll18wbdo0zJw5E7IsIz8///Q6x5rGGjptFBQUiOuvv1706dNHrFy5MrTe7XaLUaNGiZEjRwoh1G8lQqj3TVPL+f1+4ff7xcMPPyxGjx4tnE5n2O2NH374oTj77LPFzJkzQ+c6JydHHDhwgOOzNFNJSYm47777xIUXXiguuugikZWVJV555RUxd+5cMWnSJJGUlCQWLlwYOu88vydm+PDh4vLLLw8tBzsP/vjjj8Jut4t58+ZpVLL2re7NCA8//LBIT08XXbt2FR9++GGoxu2jjz4SkiSJ9957T6tiNonBgkL27NkjPv7449AgQT6fTwghxKJFi4TRaBS5ubm8x/8EFRYWhi1/++23QpZlsXjxYiFE7bkWQojJkyeLvn37hpZ5v//xHXt+9+7dK6677jpx9tlni1WrVoW9d/3114uBAweGlnl+j2/dunXif//7n9i1a1do3Y8//igsFouYNWuWEEL9HQ7+Hk+aNElcdNFFmpS1vWroHHu9XjFt2jTx+uuvh/2NEEKIcePGhb7wnW4YLChM8FtHXc8++6y44oorNChNZPj9738vxo4dK/bv3x+2furUqSI2Nlbs2bNHCCFCoW3jxo0iKipKbN++/ZSXtT1q7Pxu3rxZfPDBB6EhjoN/mJctWybMZrPYu3cvQ0Uz3H777aJ79+4iMzNTWCwW8e9//1sIIURFRYV49tlnhdFoFJ988knYPnfddZeYOHGiFsVtlxo6x8G/B5WVlaKoqChs+5qaGnH55ZeLKVOmaFHc42KwoCatXr1adOvWTbz00ktaF6Xd8fl84s477xSdOnUSer1eTJkyJewPRF5enrjkkktEjx49wkLEu+++K4YMGcLRY4/jeOdXiNqmOyFqayZefPFFMWrUKNa+HYfX6xXjxo0TgwYNEr/88os4cOCAePzxx0VsbKwoKSkRQghx6NAhMWXKFKHT6cS8efPEunXrxI4dO0S3bt3E008/rfFPcPprzjluyNatW8XgwYPFggULTmFpm4/Bghr06aefimnTpgmHw8E/ECdoy5Yt4oYbbhArVqwQ//3vf4UkSWL69OmhdlIh1MFszj33XNGrVy9x4403ihdffFHExsaKhx56SMOStw+Nnd+mnpOwYsUKkZGREaq+p8b95z//EcOHDw8LvVVVVSIzM1N8+OGHoXXV1dXiySefFBkZGSItLU107txZ3HLLLVoUud1p6hx/9NFH9bbftGmTeOedd0RiYqK46667TmVRW4TBghpUXl4urr32WrFs2TKti9JueTwe8fXXX4uKigohhBAvv/yy0Ol04p133gkbwVQI9WFvV199tbjqqqvEa6+9pkVx252mzm/dkSCFUJs/br75ZhETEyNefPFFLYrb7hQXF4s//OEPYb+rNTU1onPnzmL58uX1tt++fbv4+eefxbp1605lMdu1lpzjyspK8dJLL4msrKywB3CejiLmdlNqfT6fL/ScEDo5QghIkoTJkyfjgw8+wAcffIARI0bUG4TJ6XQiKipKo1K2X02dXyEEjh49iscffxzjx4/HyJEjtS5uu+T3+1FdXY2hQ4finXfeweDBg7UuUsQ53jmuqKjA0aNH0aNHD41K2DwcmpIaxVDReoL5/Y033sDgwYMxdepU/Prrr8jJycHUqVPx1VdfAQCsVquWxWy3mjq/U6ZMwcGDBzF37lyGihMQPLc6nQ7V1dUoKSkJDRft8Xjw1ltvIScnR8sitnvHO8dz585FTk4OoqOjT/tQAQCssSA6RerWAPXu3Rt2ux2HDh1CRkYGfvjhh9BQ6XRiGjq/ubm56Ny5M89vK9m7dy+GDh2K/fv3w+l0Yvjw4YiNjcWaNWv4RaSVRMI5Zo0F0Smi1+tDjzd+4IEHsGnTJowdOxb/+9//eNFrBQ2d36uuuorntxUVFBSgZ8+e2Lx5MwYOHIjBgwdj/fr17eaC1x5EwjlmsCA6hfR6Pd5++21MnjwZzz//PN58802tixRReH7bltPpxPr163HppZfizjvvxPvvv691kSJOJJxjNoUQnUJCCHz22Wfw+XwYN26c1sWJODy/bausrAwJCQlYunQprrzySq2LE5Ei4RwzWBARUbPV1NTAbDZrXYyI1t7PMYMFERERtRr2sSAiIqJWw2BBRERErYbBgoiIiFoNgwURERG1GgYLIiIiajUMFkRERNRqGCyIiIio1TBYEBERUathsCAiIqJWw2BBREREreb/Aa9Ba5+o9U42AAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "vis.show_histograms(data = data, plot_type=\"main\")" - ] - }, - { - "cell_type": "markdown", - "id": "d8d537cc", - "metadata": {}, - "source": [ - "## Display Options\n", - "Similar to other statistics, we can use white_list_features to select only few features to display histograms. We can also use display_format=\"percent\" to allow all dataset and sites to be displayed in the same scale. User can set \n", - "\n", - "* display_format: \"percent\" or \"sample_count\"\n", - "* white_list_features: feature names\n", - "* plot_type : \"both\" or \"main\" or \"subplot\"\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "353db4d9", - "metadata": {}, - "source": [ - "#### show default display format with subplot\n", - "In the following, we display only feature \"Intensity\" in default display_format, with \"subplot\" plot_type" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "8f619729", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAHbCAYAAACX2dMkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB+EUlEQVR4nO3deXwU9f0/8Nfsnc2xOUhCAiHhBiVCRECLtKAIoqJoVUTlUL8eFfiJiFarIihnFb8geFSsBiktiIj6xYoiBQ9UKrR4ISAYSISEJECOzWbP+fz+mN3Jbi5y7GaT5fV8OI+Z+ewcnx3izns+10hCCAEiIiKiINCEOwNEREQUORhYEBERUdAwsCAiIqKgYWBBREREQcPAgoiIiIKGgQUREREFDQMLIiIiChoGFkRERBQ0DCyIiIgoaBhYEHUAubm5kCQJR48eDXdWWmXnzp2QJAk7d+4Md1aIKEQYWBBFsP3792PevHntOiD5+9//juXLl4c7G0QUJBLfFULU/nk8HrhcLhiNRkiS1OT93n77bdx0003YsWMHRo4cGboMNpEsy3A6nTAYDNBolOeaa665Bj/88EO7Dn6IqOl04c4AEZ2dVquFVqsNdzZaTaPRwGQyhTsbRBRCrAoh6gBqt7HIysrCNddcgy+++AJDhw6FyWRCjx498Oabbwbsc9NNNwEARo0aBUmS6rRv+PDDDzFixAhER0cjNjYWV199NX788ceAc0+bNg0xMTE4fvw4JkyYgJiYGCQnJ2POnDnweDwB265fvx6DBw9GbGws4uLikJ2djRUrVqif125jMXLkSHzwwQc4duyYmr+srCxYrVZER0fjgQceqHMtfv31V2i1WixevLg1l5SIQoSBBVEHdfjwYdx444244oorsGzZMiQkJGDatGlqYPDb3/4W/+///T8AwJ/+9CesXbsWa9euRf/+/QEAa9euxdVXX42YmBgsXboUTz75JPbv349LL720TrWEx+PB2LFjkZSUhOeeew6/+93vsGzZMrz66qvqNtu2bcOkSZOQkJCApUuXYsmSJRg5ciR27drV4Hd4/PHHMWjQIHTq1EnN3/LlyxETE4Prr78eGzZsqBO8/OMf/4AQArfddlswLiMRBZsgonbvjTfeEABEXl6eEEKIzMxMAUB89tln6jbFxcXCaDSKhx56SE3buHGjACB27NgRcLzKykoRHx8v7r777oD0oqIiYbFYAtKnTp0qAIinn346YNucnBwxePBgdf2BBx4QcXFxwu12N/g9duzYUSc/V199tcjMzKyz7UcffSQAiA8//DAg/YILLhC/+93vGjwHEYUXSyyIOqjzzjsPI0aMUNeTk5PRt29f/PLLL2fdd9u2bSgrK8OkSZNQWlqqTlqtFsOGDcOOHTvq7HPfffcFrI8YMSLgXPHx8aiqqsK2bdta8a1qjB49Gunp6Vi3bp2a9sMPP+C7777D7bffHpRzEFHwsfEmUQfVrVu3OmkJCQk4c+bMWff9+eefAQCXXXZZvZ/HxcUFrJtMJiQnJzd6rvvvvx9vvfUWxo0bhy5dumDMmDG4+eabceWVV541P/XRaDS47bbb8PLLL8Nms8FsNmPdunUwmUxq2xEian8YWBB1UA31EhFN6EEuyzIApZ1F586d63yu0wX+NDSlR0pKSgr27duHjz76CB9++CE+/PBDvPHGG5gyZQrWrFlz1v3rM2XKFDz77LN49913MWnSJPz973/HNddcA4vF0qLjEVHoMbAgimANjXnRs2dPAEowMHr06KCdz2AwYPz48Rg/fjxkWcb999+Pv/zlL3jyySfRq1evZuURAAYMGICcnBysW7cOXbt2RX5+PlauXBm0/BJR8LGNBVEEi46OBgCUlZUFpI8dOxZxcXFYtGgRXC5Xnf1KSkqafa5Tp04FrGs0GlxwwQUAAIfD0Wgey8vLG/x88uTJ+Pjjj7F8+XIkJSVh3Lhxzc4bEbUdllgQRbBBgwZBq9Vi6dKlKC8vh9FoxGWXXYaUlBS8/PLLmDx5Mi688ELccsstSE5ORn5+Pj744AMMHz4cq1atata5/ud//genT5/GZZddhq5du+LYsWNYuXIlBg0apHZxrc/gwYOxYcMGzJ49G0OGDEFMTAzGjx+vfn7rrbfikUcewebNm/GHP/wBer2+xdeDiEKPJRZEEaxz58545ZVXUFxcjLvuuguTJk3C/v37ASg37O3bt6NLly549tln8cADD2D9+vUYNGgQ7rjjjmaf6/bbb4fJZMJLL72E+++/H2vWrMHEiRPx4YcfqsN31+f+++/HrbfeijfeeAO33norZs6cGfB5amoqxowZA0ApvSCi9o3vCiGidu/666/H999/j8OHD4c7K0R0FiyxIKJ2rbCwEB988AFLK4g6CLaxIKJ2KS8vD7t27cJrr70GvV6Pe++9N9xZIqImYIkFEbVLn376KSZPnoy8vDysWbOm3vE2iKj9YRsLIiIiChqWWBAREVHQMLAgIiKioGFgQUREREHDwIKIiIiChoEFERERBQ0DCyIiIgoaBhZEREQUNAwsiIiIKGgYWBAREVHQMLAgIiKioGFgQUREREHDwIKIiIiChoEFERERBQ0DCyIiIgoaBhZEREQUNGELLD777DOMHz8e6enpkCQJ7777brOPIYTAc889hz59+sBoNKJLly5YuHBh8DNLRERETaIL14mrqqowcOBA3HnnnbjhhhtadIwHHngAH3/8MZ577jlkZ2fj9OnTOH36dJBzSkRERE0lCSFE2DMhSdi8eTMmTJigpjkcDjz++OP4xz/+gbKyMgwYMABLly7FyJEjAQA//fQTLrjgAvzwww/o27dveDJOREREAdptG4sZM2bgq6++wvr16/Hdd9/hpptuwpVXXomff/4ZAPB///d/6NGjB7Zs2YLu3bsjKysL//M//8MSCyIiojBql4FFfn4+3njjDWzcuBEjRoxAz549MWfOHFx66aV44403AAC//PILjh07ho0bN+LNN99Ebm4u9u7dixtvvDHMuSciIjp3ha2NRWO+//57eDwe9OnTJyDd4XAgKSkJACDLMhwOB9588011u7/+9a8YPHgwDh48yOoRIiKiMGiXgYXVaoVWq8XevXuh1WoDPouJiQEApKWlQafTBQQf/fv3B6CUeDCwICIianvtMrDIycmBx+NBcXExRowYUe82w4cPh9vtxpEjR9CzZ08AwKFDhwAAmZmZbZZXIiIiqhG2XiFWqxWHDx8GoAQSzz//PEaNGoXExER069YNt99+O3bt2oVly5YhJycHJSUl2L59Oy644AJcffXVkGUZQ4YMQUxMDJYvXw5ZljF9+nTExcXh448/DsdXIiIiOueFLbDYuXMnRo0aVSd96tSpyM3NhcvlwoIFC/Dmm2/i+PHj6NSpEy6++GLMnz8f2dnZAIATJ05g5syZ+PjjjxEdHY1x48Zh2bJlSExMbOuvQ0RERGgn41gQERFRZGiX3U2JiIioY2JgQUREREHT5r1CZFnGiRMnEBsbC0mS2vr0RERE1AJCCFRWViI9PR0aTcPlEm0eWJw4cQIZGRltfVoiIiIKgoKCAnTt2rXBz9s8sIiNjQWgZCwuLi5oxz1+xga724OuCWYYddqz70BERERNVlFRgYyMDPU+3pA2Dyx81R9xcXFBDSxGrtiN01VOfPzgb9EnsfEvTURERC1ztmYMEdN406RTvord5QlzToiIiM5dERNYGPVK9YfDLYc5J0REROeuyAksvCUWDhcDCyIionBply8hawlfiQWrQoiIgkeWZTidznBng9qAXq+v80bxloicwMJXYsGqECKioHA6ncjLy4Ms83f1XBEfH4/OnTu3apypCAwsWGJBRNRaQggUFhZCq9UiIyOj0QGRqOMTQsBms6G4uBgAkJaW1uJjRUxgYVKrQhhZExG1ltvths1mQ3p6Osxmc7izQ20gKioKAFBcXIyUlJQWV4tETAjKEgsiouDxeJTfUoPBEOacUFvyBZEul6vFx4igwILdTYmIgo3vdDq3BOPfO2ICC5OeA2QREVHDpk2bhgkTJoQ7GyGTm5uL+Pj4cGcjcgILllgQEVFjVqxYgdzcXHV95MiRmDVrVtDPc99990GSJCxfvrzR7Xbu3AlJklBWVhaU806cOBGHDh0KyrFaI2Iabxr1HCCLiIgaZrFYQn6OzZs34+uvv0Z6enrQjul0OpvU1iUqKkptgBlOEVRi4a0KYeNNIqJz2ttvv43s7GxERUUhKSkJo0ePRlVVVUBVyLRp0/Dpp59ixYoVkCQJkiTh6NGjAIAffvgB48aNQ0xMDFJTUzF58mSUlpae9bzHjx/HzJkzsW7dOuj1+ka3PXr0KEaNGgUASEhIgCRJmDZtGgClJGXGjBmYNWsWOnXqhLFjxwIAnn/+eWRnZyM6OhoZGRm4//77YbVa1WPWrgqZN28eBg0ahLVr1yIrKwsWiwW33HILKisrm3glWyZiAgtfd1OWWBARnbsKCwsxadIk3Hnnnfjpp5+wc+dO3HDDDRBCBGy3YsUKXHLJJbj77rtRWFiIwsJCZGRkoKysDJdddhlycnKwZ88ebN26FSdPnsTNN9/c6HllWcbkyZPx8MMP4/zzzz9rPjMyMrBp0yYAwMGDB1FYWIgVK1aon69ZswYGgwG7du3CK6+8AgDQaDR44YUX8OOPP2LNmjX417/+hUceeaTR8xw5cgTvvvsutmzZgi1btuDTTz/FkiVLzpq/1oicqhB2NyUiChkhBKrD1Dg+Sq9tcm+FwsJCuN1u3HDDDcjMzAQAZGdn19nOYrHAYDDAbDajc+fOavqqVauQk5ODRYsWqWmvv/46MjIycOjQIfTp06fe8y5duhQ6nQ7/7//9vyblU6vVIjExEQCQkpJSp9Fl79698ec//zkgzb89SFZWFhYsWID77rsPL730UoPnkWUZubm5iI2NBQBMnjwZ27dvx8KFC5uUz5ZoVmAxb948zJ8/PyCtb9++OHDgQFAz1RK+xpscIIuIKPiqXR6cN/ejsJx7/9NjYTY07XY1cOBAXH755cjOzsbYsWMxZswY3HjjjUhISGjS/t9++y127NiBmJiYOp8dOXIE33zzDe6991417cMPP4TZbMaKFSvwn//8p8EAaNy4cfj8888BAJmZmfjxxx8bzcfgwYPrpH3yySdYvHgxDhw4gIqKCrjdbtjtdthstgYHMcvKylKDCkAZUdM3umaoNLvE4vzzz8cnn3xScwBd+yj08HU3ZYkFEdG5S6vVYtu2bfjyyy/x8ccfY+XKlXj88cexe/fuJu1vtVoxfvx4LF26tM5naWlpkGUZw4YNU9O6dOmCv/zlLyguLka3bt3UdI/Hg4ceegjLly/H0aNH8dprr6G6uhoAztr+AgCio6MD1o8ePYprrrkGf/jDH7Bw4UIkJibiiy++wF133QWn09lgYFH7XJIkhfzdL82OCnQ6XUCxUXvB7qZERKETpddi/9Njw3bu5pAkCcOHD8fw4cMxd+5cZGZmYvPmzXW2MxgM6gijPhdeeCE2bdqErKysBh+c/UsAAKV6YfTo0QFpY8eOxeTJk3HHHXcAUAKQ+s4PoE4e6rN3717Isoxly5ap72156623zrpfODQ7sPj555+Rnp4Ok8mESy65BIsXLw6I0mpzOBxwOBzqekVFRctyehZqGwsOkEVEFHSSJDW5OiKcdu/eje3bt2PMmDFISUnB7t27UVJSgv79++O7774L2DYrKwu7d+/G0aNHERMTg8TEREyfPh2rV6/GpEmT8MgjjyAxMRGHDx/G+vXr8dprr9X7/oykpCQkJSUFpOn1enTu3Bl9+/ZtMK+ZmZmQJAlbtmzBVVddhaioqHqrYACgV69ecLlcWLlyJcaPHx/QqLO9aVavkGHDhiE3Nxdbt27Fyy+/jLy8PIwYMaLRriuLFy+GxWJRp4yMjFZnuj5qrxCWWBARnbPi4uLw2Wef4aqrrkKfPn3wxBNPYNmyZRg3blydbefMmQOtVovzzjsPycnJyM/PR3p6Onbt2gWPx4MxY8YgOzsbs2bNQnx8fNDf8NqlSxfMnz8fjz76KFJTUzFjxowGtx04cCCef/55LF26FAMGDMC6deuwePHioOYnWCRRuw9OM5SVlSEzMxPPP/887rrrrnq3qa/EIiMjA+Xl5YiLi2vpqev45uhp3PTKV+jeKRo75owM2nGJiM5FdrsdeXl56N69O0wmU7izQ22ksX/3iooKWCyWs96/W1WuFR8fjz59+uDw4cMNbmM0GmE0GltzmiZhVQgREVH4tapcx2q14siRI0hLSwtWflrMVxViZ1UIERFR2DQrsJgzZw4+/fRTHD16FF9++SWuv/56aLVaTJo0KVT5azKWWBAREYVfs6pCfv31V0yaNAmnTp1CcnIyLr30Unz99ddITk4OVf6ajN1NiYiIwq9ZgcX69etDlY9W85VYuGUBt0eGThsxr0EhIiLqMCLm7mvyG0CFpRZERMHRio6D1AEF4987YgILg67mqzCwICJqHd9AUE6nM8w5obZks9kANG3Y8Ya0/2HUmkirkaDXSnB5BOxswElE1Co6nQ5msxklJSXQ6/VBHxyK2hchBGw2G4qLixEfH1/vCKNNFTGBBQCYdFq4PG6WWBARtZIkSUhLS0NeXh6OHTsW7uxQG4mPj2/1+8AiKrAw6jWodPANp0REwWAwGNC7d29Wh5wj9Hp9q0oqfCIrsPB2ObW7WGJBRBQMGo2GQ3pTs0RUpZlRz0GyiIiIwimyAgsOkkVERBRWERZYKF+HvUKIiIjCI6ICC5OvKoQlFkRERGERUYEFq0KIiIjCK8ICC1aFEBERhVNEBRa+94WwxIKIiCg8Iiqw8JVYcIAsIiKi8IiswELvqwphiQUREVE4RFZgoTbeZIkFERFROERUYKF2N2WJBRERUVhEVGDBEgsiIqLwirDAgiUWRERE4RRRgQW7mxIREYVXRAUWHCCLiIgovHThzkAwGfmuECLqwIQQcHkEnB4ZTrff5Kk1d8sQENBrNdBrNTDqNN5lCXqtBia9FjFGHUx6DSRJCvfXonNMRAUWJjbeJKIWkmUBh1tGtcsDu8ujzpVJRrUzMM3hluHw3uRdfjd+l6eBdLeAQw0MPGpaQBDhCe5DkVYjIcaoQ4xRh1hTzTwx2ohOMQYkxRiQFG1EUowBqXEmpFuiEBelYzBCrRJRgQUHyCKKPC6P92av3tiV9WqnB3Z3TbrvM7v3s9oBQrWzZt+AgMEvUGhvtBoJBq0GBp130tbMJUm5Ni6PqAli/AIbIQCPLFBe7UJ5tavJ5zQbtEizmJAeH4WMRDP6pMSgT2oseqfGolOMgUEHnVVkBRYssSAKKt9TvMPtfUJ3+S27PXC4ZNi984a3U272tdMctfZTbo7K+Vx+N0iPLNr8exu0Gpj0SpVClEELk04Lk0ELk05Ts65XbvJ6vxu/URu47lv2VVUYtBro/QIEo1/AUDtdr9VAq2nZTVyWBWwuD6x2N6wO72R3w+pQgoxTVU6csjpxusqJUqsDJZUOnKyw44zNBZvTgyMlVThSUlXnuAlmPXqnxqJPqjfYSIlFv86xSIg2tPaSUwSJqMDCxDYWFGF8de4Ot/J03dCN2+F94vYvoq994/bfz17P/s5a+9ndHrg8bX9Tb4hGAqL0Wpi8U5RB6133BgC+dF8w4P0sqoHgwP9Yvu186y29obcXGr8qkOaodnpQWF6NwnI7TpRVI6+0CodOWvFzcSXyT9twxubCv/NO4995pwP2S7eYcF56HM5Li8N56XE4P92CrglRLN04R0VUYOErsWCvEGoqIQQ8slCLk31Fyb7J6RZ+y77PA9N863X3rymi9l8P2MZbx+57Oq8vGBDt5N6ukaDeeI3ep2qjTguj3m9Zp4FRr4FJTffbtt79vHOtsp+vMaLv6d6g897wDb7if96oQinKoEWP5Bj0SI6p81m104MjJVYcOlmJn4ut+PlkJQ6erETB6WqcKLfjRLkdn/xUrG4fa9Khd0oMMpOikZFoRrdEMzKTlHlKrJH/lhGsRYHFiy++iGeffRZFRUUYOHAgVq5ciaFDhwY7b83m625a5fBg/4kK9E6NgV4bUT1qOxTlhi0HFG3730j9b95KQzb/euJ6buj+N3m/47k8Mhx++/ta1dc+Z52bvHe5vdy4m6L2TbopN3mjt9i+/pu8XzBQT5r/Pjr+v3ROizJoMaCLBQO6WALSK+0u/FRYif0nyrG/sAI/nqjAoZOVqLS78Z/8Mvwnv6zOsUx6DdLjo5AcY0SnWCOSY4xIjvVOMUZ0ijEiIVqPBLMBZoOWQUgHIwnRvJ/VDRs2YMqUKXjllVcwbNgwLF++HBs3bsTBgweRkpJy1v0rKipgsVhQXl6OuLi4Fme8PkXldly8eLu6btBp0Cs5BjEmXQN1nFK99Z4Gb7ctg07rnfvSaupNdVoJGgmQJAkaybsMCZIEZV2DmnS/bTSS3zbeZd+6VpKg1XrnGgk6jTJvyf9UQtRtbe5bdtS37q4pIre7vI3iXEoRua+Rm5Lmv03g577gwHfDDkPVeFDU/Pv7PT3rarry+T9Nq2l+fzd1uwBqoNdJ6t+Q3u/4vjSdVqoTGJj0gcECn9ipo3C6ZRwutiKvtAr5p23IP63Mj52y4URZdbN+Gww6DRLMSpCRYDYgMdqAeLPeOzcgMVqvzL2fJ0TrEWNkz5ZQaOr9u9mBxbBhwzBkyBCsWrUKACDLMjIyMjBz5kw8+uijQctYS/31izxs21+EH49XoNLhDvrxqeV8N12Df5DndyNWbs6NbBMQ9AUWmeu1EvS62jd977beff3X6+zvDTJ1LQzkiKhpXB4Zx88o7Th8DUdLvPNSv/kZmwvOFraX02slxJsNAQFJQrSyHm/WI9rb/iTaoKtZNmq9cx1LSRoQksDC6XTCbDbj7bffxoQJE9T0qVOnoqysDO+9917QMtZasixQcMaGn09avY3QavqRq0/Vvid3tbjcU1NsXqvoXHnqF2ofdCEAWQjI3nnNum9ZKTVoaJuaz2s+CyX/G2pAi3VtTct09SlZX9PqvXbDNuUzTUCjN5N3+8Bj131C5/+oRNRUQghUuzw4XeVEmc2F01VOnLE5cabKiTM2l7Jsc3nXa9Krg9DGTpLgDTq0AUFI7VLExqoNtRql9FmjkaD1lmAHpHnXNRqlNLtmW2VdCEDAOxdCXQYAAeVD9XMIv+2VjS7pmaS2OwyWpt6/m9XGorS0FB6PB6mpqQHpqampOHDgQL37OBwOOByOgIy1BY1GQmZSNDKTotvkfMHgkYU6uWUZsgy45Zb3cFGrd7QaaDp4K3ciOrdIkgSzQQezQYeuCU3fz+7y4IzNGRCQlNmcOF2lBCPl1S5YHW5UeSdl2aMsO93eGznUbrqA46znbI/+/fjlSIkNbmDRVCHvFbJ48WLMnz8/1KeJCFpvtOpdC2teiIg6IpNeizRLFNIsUc3e11dKEhBs+IIQp0ft1n227twujwyPrJRGe2ShzgOWhVKy7kur2VbZT4ISXEkAIAG+O4MvTfK264N3ufZnOk34Gls3K7Do1KkTtFotTp48GZB+8uRJdO7cud59HnvsMcyePVtdr6ioQEZGBnbt2oWXXnoJe/fuRWFhITZv3hxQvXI28+bNqzdgMZvNqKqqO7ALERFRY/xLSRAb7tx0XM0KLAwGAwYPHozt27erQYAsy9i+fTtmzJhR7z5GoxFGo1Fd99X/lJaWol+/frjllltw++23w2azNaua5J577sFtt90WkHbttdfiwgsvbLPqFiIionOF79561qaZopnWr18vjEajyM3NFfv37xf33HOPiI+PF0VFRU3av6CgQMDbxoQTJ06cOHHi1LGmgoKCRu/zzW5jMXHiRJSUlGDu3LkoKirCoEGDsHXr1joNOhuSnp6OgoICxMbGqr0ELBYL1q1bh2uuuUbdbubMmTh48CDmzZuHzp07Y8uWLViwYAG++uor9OzZs85xH3jgAeTm5qKgoCCkvU3OZb5qLF7j0OD1DS1e39DjNQ6tcF9fIQQqKyuRnp7e6HbNHsciFCRJCmhjkZ+fjx49eiA/Pz/gC4wePRpDhw7FokWLAva32+1IS0tDWVlZyLuxnsvaqqvwuYrXN7R4fUOP1zi0Osr1bZfvCvn+++/h8XjQp0+fgHSHw4GkpKQ622/evBlWq7WtskdEREQNaJeBhdVqhVarxd69e6HVBna7jImp+3Kc1157DVdeeSW2bNnSVlkkIiKierTLwCInJwcejwfFxcUYMWJEo9vm5eVhx44d2LRpEwYPHhzQA4WCy2g04qmnnuI1DhFe39Di9Q09XuPQ6ijXN2xtLKxWKw4fPgxACSSef/55jBo1ComJiejWrRtuv/127Nq1C8uWLUNOTg5KSkqwfft2XHDBBbj66qvV4zz55JN4/fXXkZ+fX6d0g4iIiNpW2AKLnTt3YtSoUXXSp06ditzcXLhcLixYsABvvvkmjh8/jk6dOuHiiy/G/PnzkZ2dDUAZQyMzMxNTpkzBwoUL2/orEBERUS3tolcIERERRYbwDSZOREREEYeBBREREQVNm/cKkWUZJ06cCBh5k4iIiNo3/5E3NY28PbXNA4sTJ04gIyOjrU9LREREQVBQUICuXbs2+HmbBxaxscq7aDmWPBERUcfhe1eJ7z7ekDYPLHzVH3FxccENLN6bAdhOAVc9C1gajqSIiIio5c7WjCFyGm/+vA04+E/AdjrcOSEiIjpnNTuw+OyzzzB+/Hikp6dDkiS8++67IchWC+i8Q5y6HeHNBxER0Tms2YFFVVUVBg4ciBdffDEU+Wk5nUmZu+3hzQcREdE5rNltLMaNG4dx48aFIi+to2dgQUQUbLIsw+l0hjsb1Ab0en1Q3rkV8sabDocDDkdN9URFRUVoTsQSCyKioHI6ncjLy4Msy+HOCrWR+Ph4dO7cuVXjTIU8sFi8eDHmz58f6tOwjQURURAJIVBYWAitVouMjIxGB0Sijk8IAZvNhuLiYgBAWlpai48V8sDisccew+zZs9V1Xz/YoNNFKXNXdfCPTUR0jnG73bDZbEhPT4fZbA53dqgNREUp99Hi4mKkpKS0uFok5IGF0WiE0WgM9Wn8SixYFUJE1FoejwcAYDAYwpwTaku+INLlcrU4sIicsi21jQWrQoiIgoXvdDq3BOPfu9klFlarFYcPH1bX8/LysG/fPiQmJqJbt26tzlCLqb1CWBVCREQULs0usdizZw9ycnKQk5MDAJg9ezZycnIwd+7coGeuWVhiQUREjZg2bRomTJgQ7myEzM6dOyFJEsrKysKaj2YHFiNHjoQQos6Um5sbguw1A9tYEBFRI1asWBFwrxo5ciRmzZoVlGO/8847GDNmDJKSkiBJEvbt23fWfY4ePdrkbZviN7/5DQoLC2GxWIJyvJaKoDYWvl4hDCyIiKgui8WC+Pj4kBy7qqoKl156KZYuXRr0Yzd1gDKDwdDqMSiCIYICC5ZYEBER8PbbbyM7OxtRUVFISkrC6NGjUVVVFVAVMm3aNHz66adYsWIFJEmCJEk4evQoAOCHH37AuHHjEBMTg9TUVEyePBmlpaWNnnPy5MmYO3cuRo8e3eR8du/eHQCQk5MDSZIwcuRINW8TJkzAwoULkZ6ejr59+wIA1q5di4suugixsbHo3Lkzbr31VnXcCaBuVUhubi7i4+Px0UcfoX///oiJicGVV16JwsLCJuexJSIosGAbCyKikBECcFaFZxKiydksLCzEpEmTcOedd+Knn37Czp07ccMNN0DUOsaKFStwySWX4O6770ZhYSEKCwuRkZGBsrIyXHbZZcjJycGePXuwdetWnDx5EjfffHOwryj+/e9/AwA++eQTFBYW4p133lE/2759Ow4ePIht27Zhy5YtAJQuoM888wy+/fZbvPvuuzh69CimTZvW6DlsNhuee+45rF27Fp999hny8/MxZ86coH8XfyEfx6LNsFcIEVHouGzAovTwnPtPJwBDdJM2LSwshNvtxg033IDMzEwAQHZ2dp3tLBYLDAYDzGYzOnfurKavWrUKOTk5WLRokZr2+uuvIyMjA4cOHUKfPn1a+WVqJCcnAwCSkpIC8gAA0dHReO211wLGEbnzzjvV5R49euCFF17AkCFDYLVaERMTU+85XC4XXnnlFfTs2RMAMGPGDDz99NNB+w71YYkFERFFjIEDB+Lyyy9HdnY2brrpJqxevRpnzpxp8v7ffvstduzYgZiYGHXq168fAODIkSNYt25dwGeff/55k4573333Bex3NtnZ2XUGJ9u7dy/Gjx+Pbt26ITY2Fr/73e8AAPn5+Q0ex2w2q0EFoAzV7V99EgqRU2LBl5AREYWO3qyUHITr3E2k1Wqxbds2fPnll/j444+xcuVKPP7449i9e3eT9rdarRg/fny9jTDT0tIgyzKGDRumpnXp0qVJx3366aebVQURHR1YQlNVVYWxY8di7NixWLduHZKTk5Gfn4+xY8c22rhTr9cHrEuSVKdaKNgiL7BgrxAiouCTpCZXR4SbJEkYPnw4hg8fjrlz5yIzMxObN2+us53BYFCHLve58MILsWnTJmRlZUGnq/8WGRsb2+w8paSkICUlpc75AdTJQ30OHDiAU6dOYcmSJer7tvbs2dPsfLSFCKwKYWBBRHSu2r17NxYtWoQ9e/YgPz8f77zzDkpKStC/f/8622ZlZWH37t04evQoSktLIcsypk+fjtOnT2PSpEn45ptvcOTIEXz00Ue44447Gg0ATp8+jX379mH//v0AgIMHD2Lfvn0oKipqcJ+UlBRERUWpDUTLy8sb3LZbt24wGAxYuXIlfvnlF7z//vt45plnmnFl2k4EBRZ8bToR0bkuLi4On332Ga666ir06dMHTzzxBJYtW4Zx48bV2XbOnDnQarU477zz1KqF9PR07Nq1Cx6PB2PGjEF2djZmzZqF+Pj4Rl8d//777yMnJwdXX301AOCWW25BTk4OXnnllQb30el0eOGFF/CXv/wF6enpuO666xrcNjk5Gbm5udi4cSPOO+88LFmyBM8991wzrkzbkUSoK1tqqaiogMViQXl5OeLi4oJ34F/3AK9dDsR3A2Z9H7zjEhGdg+x2O/Ly8tC9e3eYTKZwZ4faSGP/7k29f7PEgoiIiIImggILtrEgIiIKt8gLLNgrhIiIKGwiL7DwOJo1/CsREREFTwQFFsaaZVaHEBERhUXkBBb6qJplBhZEREHRxh0HKcyC8e8dOYGFRgdI3q/DniFERK2i1WoBoNHhoiny2Gw2AHWHAm+OyBnSW5KUdhYuG+DiG06JiFpDp9PBbDajpKQEer2+0cGhqOMTQsBms6G4uBjx8fFqYNkSkRNYADWBBUssiIhaRZIkpKWlIS8vD8eOHQt3dqiNxMfH13mFe3NFXmABsI0FEVEQGAwG9O7dm9Uh5wi9Xt+qkgqfCAssfKNvMrAgIgoGjUbDIb2pWSKr0szXM4SBBRERUVhEVmDB94UQERGFVYQFFr5hvdkrhIiIKBwiM7BgiQUREVFYRGhgwTYWRERE4RBhgQV7hRAREYVTZAUW7BVCREQUVpEVWLBXCBERUVhFWGDBXiFEREThFJmBBUssiIiIwiJCAwu2sSAiIgqHCAss2CuEiIgonCIrsGCvECIiorCKrMCCvUKIiIjCKsJem85eIURUD1kGZBfgcSlz2VOz7PGuq8vumrnsAjzeuX/62Y4ju2vOLYRfRkTz0jVa5XdNZ/SbR/mt1/7MO9dHAXqzMmki6/mR2r/IDCxYYkHU9mRZqYZ0VQPuamXuqvam2QCXd+6/rm5nBzzOlt/IzxYQCDncVyd89GbAEO2dxwAG33q0Mjd4033b+SZfYKIz+k0mQGvwC2q8yxodIEnh/qbUTkRoYME2FkQAlKdfpxWwlyuT03tjVwMAh3JzdztqggC33XvTt9da99/OERg8uKoBTwcL6CWtckPU6gPnGj2g9c41uppl9XPftr7t6tmnzo3Wb7mhG3B928se5boG/Js4mjD3K7V12ZQplCQtEJUAmJO8pSxGIDoZiE4BYpKVZXNS3ckQzYAkAkVYYMFeIRQBhACqzyhPkVq9X2BQATgqlLm9HLCXeSdv0FDtt6x+XgEIT9t/B61BKbLXRwF6k/fJ1zvXmwKXfZ9pDfXc5P1u4me7yTcnCNDoIruKQJaV4MJpU/5+XDbAWVUzubzpTm+6y/dZre19QaTH4Q1YHDUBjOyqOZ/wALZSZWoOnQmISQXiuymTJQOIzwASsoBOfZSAhIFHhxNZgQV7hVB74nF5g4IKwFGpBAWOypoAIWC9UgkGbKeAkoOAo9x7EAkB9e8tpTUAJov3Zh5VU1fvu8n7pnrXo2rq7f3r+PVR9QcI+ijlqZXCR6OpqdJAcmjOIcs1AYerWvnbrT6tVDu5qgFrMVBVDFhLvEHHKe90GqgqrSmJKTumTPUxxQPJfZWpU1+gU28gqRcQn6kEitQuRda/DHuFUHPJst/TWpX3Kc5/2eb39OY/t3n3q/25X7r/E12LeYMKjR4wxQHGOGVuilcCBZMFiPItxwem+3+mM/HJj4JLowE03lKpqHggLq3p+wqh/D9jKwUqTgBlBUB5vndeAJz+BThzTCl1K9itTAHn1gOJ3ZUgI6lXTcCR1BuI7nRu/63bTiu/E2EMvCIssGCvkHOC2xlYAhAw1ZMWECz4Bw/W0Nc9A8rfpTFW+Z/dGFsTIPiCBGOsN90bHPh+LN3Vync1xTEwoMgiSYAxRpkSsoDMerZxVQOnDisleCUHgdKDwKkjSprbDpQeUqbaTBYgoTsQ21mpZgmYdwZiUrztO8yh/pZtT/YAb01RGkLfsBpIqO/Chl5kBhbVZ4DtzwAX3KzU0/EHOfzcTu8N3XtTd1hr1n3LvvYDZwsWQtFIUNJ4W8xH121BH9Cq3uxtTV87vaHPY5SW8y2hNwX3OxJ1JPoooHO2MvmTZaDiOHDqZ6D0sBJo+JbLC5QqxcJ9QOFZjq8zKQFGVCJgTgDMnZT16FpzcyelYarJouSpvdxPhFB6PPk3FP7yBeDo58pvkCcYJaYtIwkhml2B++KLL+LZZ59FUVERBg4ciJUrV2Lo0KFN2reiogIWiwXl5eWIi4trdoYb5XEBr10OFH5bk+aro4tLV9bjugADbgDSL2w/fyDtkSwDzsqaBoGOCm9AUBn41F97XQ0UqgIDh6BUC9Sij6552g+Y/EoCDNHeAMH7dKSuRwcus0SAqONzVSvVKGX5QGURYD1Zd24tbvnvka+tksmi/G5oDTWNhbV6vwbIfssNpUsaqFWd6m241rokKdtJGiWIsBYrpTZF3yvtV4Ss/IZZMoC0C4AfNinbXfcikHN7a65kvZp6/252YLFhwwZMmTIFr7zyCoYNG4bly5dj48aNOHjwIFJSUoKWsRaTPcCBD4BvVgMF/264IacpHkg5D0jpp5RqmCw1NyaDd25OVLYLZV2VrxW2r5uWPyGUPxz/fvu200DliZo/cNld01K7dsvtgPXqxtsD1Em3ISiNBmvTGpX/IY0xNTd8Q7RflUBc/YFC7XRDDBtvEVHz+bpg207XNDitOuXXuLRUaVzqW68qVR6uwtG7qiXOvx648Y2QPCiFLLAYNmwYhgwZglWrVgEAZFlGRkYGZs6ciUcffTRoGQsKtxMo3q9EsNZi5UIX7AYO/DOwn/fZmOKVG59G5+37rvXONYHrkqQENkJW/ghl71zI3nRPzecepzeoqFTOodEpN0yPu2aAH//R+8JFZ/IGXXF+wYDfk35AqYD/erQSoKlBRHRN90kioo7EfzyY6jKlUamrWnnY8zj9Hv68y7502RW4jW87txM1D25SwKxmXfI+XHofMCVJ6X4bnwF0vkDpnqs1KMHPqSNA/ldKCfLlc5XGtCEQksDC6XTCbDbj7bffxoQJE9T0qVOnoqysDO+9917QMhZSLrtSJ1f8kzfwyAts6Oer17eXn/1Ybc0Qo1TreFxK/rR6pRTAf3S8htbrbQdQqz2Af9sBYxzr+YmICEDT79/NKksuLS2Fx+NBampqQHpqaioOHDhQ7z4OhwMOR01ju4qKiuacMjT0pvobBdXmcSuRqe1UzUBDASUPfqUSsgeA8CvB8NaLqaUZtZZ13gGEYlKUG7n1pBLU1B4BUKMNHORHo2VbACIiardCXkm9ePFizJ8/v076xx9/jL/+9a/Yt28fioqKsG7dOlxzzTXNOu6SJUvqpJvNZhQWnq05cHMYAGOaMoWCE0qViORtO+BP9k4AAJd3IiIianu+goGzVXSEvCqkdonF8ePHcd555zX1lERERNSOFBQUoGvXrg1+3qwSC4PBgMGDB2P79u1qYCHLMrZv344ZM2bUu4/RaITRaFTXY2JiUFBQgNjYWEjeIn2LxVKnxMLhcODpp5/Gpk2bUF5ejv79+2P+/PkYMWJEvef5+uuvMXbsWLz99tu44oormvO1qIkqKiqQkZGBgoKC8LWPiWC8vqHF6xt6vMahFe7rK4RAZWUl0tPTG92u2VUhs2fPxtSpU3HRRRdh6NChWL58OaqqqnDHHXc0aX+NRlNvpGM2mwMu1N133439+/djw4YNSE9Px+bNm/H73/8e33//PXr37l1n/02bNgEArrjiCv5Bh1hcXByvcQjx+oYWr2/o8RqHVjivr8ViOes2zQ4sJk6ciJKSEsydOxdFRUUYNGgQtm7dWqdBZ2vk5+fjjTfeQH5+vhoZzZkzB1u3bsUbb7yBRYsWBWxvt9vx1ltvBe38RERE1DItarw5Y8aMBqs+guH777+Hx+NBnz59AtIdDgeSkpLqbL9582ZYrdaQ5YeIiIiapl0OXWi1WqHVarF3715otYGjUcbExNTZ/rXXXsNVV12FnJycgPYcFFxGoxFPPfUUr3GI8PqGFq9v6PEah1ZHub4teldI0DMhSdi8ebPaIPTQoUPo27cvPvvsswYba/rk5eWhZ8+eeP/995vVXZWIiIiCL2wlFlarFYcPH1bX8/LysG/fPiQmJqJPnz647bbbMGXKFCxbtgw5OTkoKSnB9u3bccEFF+Dqq69W93v99deRlpaGcePGheNrEBERkZ+wlVjs3LkTo0aNqpM+depU5ObmwuVyYcGCBXjzzTdx/PhxdOrUCRdffDHmz5+P7GxlxExZlpGZmYkpU6Zg4cKFbf0ViIiIqJZ2URVCREREkUET7gwQERFR5GBgQUREREHT5o03ZVnGiRMnAob0JiIiovbNf0hvjabhcok2DyxOnDiBjIyMtj4tERERBUFQX0IWDLGxyqvBg/0SlcNnDsPmtqF3Qm9E6aKCdlwiIiKqeQma7z7ekDYPLHzVH8F+icqDHz6I0/bTeOfad5AaF7z3lhAREVGNszVjiJjGmyatCQBgd9vDnBMiIqJzV+QEFjpvYOFhYEFERBQuERdYVLurw5wTIiKic1e7fLtpS7AqhIgo+GRZhtPpDHc2qA3o9fo6bxRviYgJLHw9QVgVQkQUHE6nE3l5eZBlOdxZoTYSHx+Pzp07t2qcqYgJLNQ2FiyxICJqNSEECgsLodVqkZGR0eiASNTxCSFgs9lQXFwMAEhLS2vxsSIusGAbCyKi1nO73bDZbEhPT4fZbA53dqgNREUpJf/FxcVISUlpcbVIxISgbGNBRBQ8Ho8HAGAwGMKcE2pLviDS5XK1+BgRE1iwjQURUfDxnU7nlmD8e0dMYME2FkRE1Jhp06ZhwoQJ4c5GyOTm5iI+Pj7c2YigwELLNhZERNSwFStWIDc3V10fOXIkZs2a1erjulwu/PGPf0R2djaio6ORnp6OKVOm4MSJE43ut3PnTkiShLKyslbnAQAmTpyIQ4cOBeVYrRE5gQVH3iQiokZYLJaQPNHbbDb85z//wZNPPon//Oc/eOedd3Dw4EFce+21QTl+U8cRiYqKQkpKSlDO2RoRE1iobSxYFUJEdE57++23kZ2djaioKCQlJWH06NGoqqoKqAqZNm0aPv30U6xYsQKSJEGSJBw9ehQA8MMPP2DcuHGIiYlBamoqJk+ejNLS0gbPZ7FYsG3bNtx8883o27cvLr74YqxatQp79+5Ffn5+vfscPXoUo0aNAgAkJCRAkiRMmzYNgFKSMmPGDMyaNQudOnXC2LFjAQDPP/+8WiqSkZGB+++/H1arVT1m7aqQefPmYdCgQVi7di2ysrJgsVhwyy23oLKysoVXtmkiJrBgGwsiIiosLMSkSZNw55134qeffsLOnTtxww03QAgRsN2KFStwySWX4O6770ZhYSEKCwuRkZGBsrIyXHbZZcjJycGePXuwdetWnDx5EjfffHOz8lFeXg5JkhosIcnIyMCmTZsAAAcPHkRhYSFWrFihfr5mzRoYDAbs2rULr7zyCgBAo9HghRdewI8//og1a9bgX//6Fx555JFG83HkyBG8++672LJlC7Zs2YJPP/0US5YsadZ3aa7IGcdCy6oQIqJQEUKErQ1blC6qyb0VCgsL4Xa7ccMNNyAzMxMAkJ2dXWc7i8UCg8EAs9mMzp07q+mrVq1CTk4OFi1apKa9/vrryMjIwKFDh9CnT5+z5sFut+OPf/wjJk2ahLi4uHq30Wq1SExMBACkpKTUCUB69+6NP//5zwFp/u1BsrKysGDBAtx333146aWXGsyLLMvIzc1FbGwsAGDy5MnYvn07Fi5ceNbv0VKRE1iwxIKIKGSq3dUY9vdhYTn37lt3w6xv2iBdAwcOxOWXX47s7GyMHTsWY8aMwY033oiEhIQm7f/tt99ix44diImJqfPZkSNH8M033+Dee+9V0z788EOMGDFCXXe5XLj55pshhMDLL7+spo8bNw6ff/45ACAzMxM//vhjo/kYPHhwnbRPPvkEixcvxoEDB1BRUQG32w273Q6bzdbgIGZZWVlqUAEoI2r6RtcMlYgJLNjGgoiItFottm3bhi+//BIff/wxVq5ciccffxy7d+9u0v5WqxXjx4/H0qVL63yWlpYGWZYxbFhNgNWlSxd12RdUHDt2DP/6178CSitee+01VFcrJT56vf6s+YiOjg5YP3r0KK655hr84Q9/wMKFC5GYmIgvvvgCd911F5xOZ4OBRe1zSZIU8ne/NDuw+Oyzz/Dss89i7969KCwsxObNm9tFv2BWhRARhU6ULgq7b23azTkU524OSZIwfPhwDB8+HHPnzkVmZiY2b95cZzuDwaCOMOpz4YUXYtOmTcjKyoJOV/8t0r8EwMcXVPz888/YsWMHkpKSAj73D0D8zw+gTh7qs3fvXsiyjGXLlqnvbXnrrbfOul84NLvxZlVVFQYOHIgXX3wxFPlpMb4rhIgodCRJgllvDsvUnNEgd+/ejUWLFmHPnj3Iz8/HO++8g5KSEvTv37/OtllZWdi9ezeOHj2K0tJSyLKM6dOn4/Tp05g0aRK++eYbHDlyBB999BHuuOOOBgMAl8uFG2+8EXv27MG6devg8XhQVFSEoqKiRruKZmZmQpIkbNmyBSUlJQE9PGrr1asXXC4XVq5ciV9++QVr165VG3W2N80OLMaNG4cFCxbg+uuvD0V+WoxtLIiIKC4uDp999hmuuuoq9OnTB0888QSWLVuGcePG1dl2zpw50Gq1OO+885CcnIz8/Hykp6dj165d8Hg8GDNmDLKzszFr1izEx8c3+IbX48eP4/3338evv/6KQYMGIS0tTZ2+/PLLBvPapUsXzJ8/H48++ihSU1MxY8aMBrcdOHAgnn/+eSxduhQDBgzAunXrsHjx4uZfoDYgidp9cJqzsySdtSrE4XDA4XCo6xUVFcjIyEB5eXmDrWVborS6FKPeGgWNpMG+yfs4vj0RUSvY7Xbk5eWhe/fuMJlM4c4OtZHG/t0rKipgsVjOev8O+TgWixcvhsViUaeMjIyQnMfXxkIWMlxyy9/KRkRERC0X8sDiscceQ3l5uToVFBSE5DxGnVFdZjsLIiKi8Ah5d1Oj0Qij0Xj2DVtJr9FDJ+ngFm7Y3XZYjJaQn5OIiIgCRcyQ3gBfREZERBRuzS6xsFqtOHz4sLqel5eHffv2ITExEd26dQtq5prLpDPB6rKyZwgREVGYNDuw2LNnj/pGNgCYPXs2AGDq1KkB77kPB18DTraxICIKjlZ0HKQOKBj/3s0OLEaOHNlu/9BYFUJEFBxarRYA4HQ6ERXVvJEvqeOy2WwAmjbseEMi5l0hAN8XQkQULDqdDmazGSUlJdDr9Q0ODkWRQQgBm82G4uJixMfHq4FlS0RUYMHRN4mIgkOSJKSlpSEvLw/Hjh0Ld3aojcTHxwe8Rr4lIiuwYBsLIqKgMRgM6N27d6Pvu6DIodfrW1VS4RNZgQXbWBARBZVGo+GQ3tQsEVVpxjYWRERE4RVRgYWvKoSBBRERUXhEVmDhrQqp9rCNBRERUThEZGDBEgsiIqLwiKjAgm0siIiIwiuiAgu2sSAiIgqvyAos2MaCiIgorCIysGCJBRERUXhEVGARpWUbCyIionCKqMCCI28SERGFV2QGFiyxICIiCouIDCz4EjIiIqLwiKjAwtfGosJZga9OfAWXxxXmHBEREZ1bIurtpvGmeEiQUO2uxj3b7kFPS08svHQhzu90frizRhRyQgi4ZBeqXFWwuW2wuWx15y4bqt3VcAs33LIbHuGBR/bALdzwyB54hAduueHPaq8LCAghlPN7lwUEZCHXnSBDCKHsJ5Rt/PdR537Lyn/KtgCgkTTQSBpoJS00kgY6jS4gTavRqp/51jWSBjpJB51GB62khU6jC5gkSC263pIkQStpodfoA46r1QSm+eZ6jV7dx5dn/6m+dK2khQQJUboomPVmROujEa2PRpQuChopop4LKYJIwver0EYqKipgsVhQXl6OuLi4oB//X/n/wrZj2/DF8S9Q5iiDVtLi5r43494L7kVSVFLQz0fUUg6PAzaXrU4gUO2qVter3FV1AoRqV3WD6W7hDvfXojbgCzZ8gYZZb0aMPqYm+NDVpPtv40uvPRm0hnB/JeoAmnr/jrjAwqfMXoZFuxfhw6MfAlCG+76xz42Y3H8y0mLSQnZeOvfIQkalsxJn7GdQ5iirmTvOoMxehnJnOcod5ShzlKHcUY4KRwXKneVweBwhy5NJa4JZb1afdM06ZYrWR8OkMylP1BptwJO872nff9335F3fZ76naeU/5alfglTnqVuC8pQuSVKddEmSAud+y77j+rb1XWtZyPAIT8DcLbvrTVdLWrylLS7ZpZbG+EpmBFr2E+g7j0dWjuWSXepxfXOX7FI/dwt3nVIcX+mN/7y+z+weO6qcVahyV6mlN8Gk0+jUgMQ/GPGfzDozLEYLLEYL4o3xiDfGI84Yp8wNcdBpIqoAnOpxzgcWPrsLd+P5vc9j/6n9AACdpMOV3a/EtPOnoW9i35CfnzoWIQRsblvdIMF+BmccZ+qk+6bW/NjXDgJ8P+5mnVlN9/2w+6erc7/Awbc9f+QjkxBCCTJcVWppl6/Ey7fs+8zqstZs566qd59gNnSP1ceqgYYv6LAYagIRi9GCGH0MYgwxiNZHI86gbButj4Yktaw6itoWAws/QgjsOrELuT/kYnfRbjV9ePpwTBswDcM6D+MfdoRyeByBwYE3QAgIDuw1pQtnHGfgklvW6DdGH4N4YzwSTAnqXP1R9f7A+k+xhlhE66Kh1WiD/K2JmsYje+oEJbWDEf9ApdxRrk5lDqU0rtJZ2ao86DQ6JBiV/1d8/+/4pqSoJHSK6oTkqGQkRyWjk7mT+rJJansMLBrw46kfkftDLj4+9rH6lNk/sT+u63UdLu92OVLMKWwU1Q65ZTcqnZVKVYKzQvlx81Yx+KoW6gsSWvpEZtQakWBKQILR+0NnileWvXP1M1+6MR56rT7I35qo/XPL7pr/Jx2B1X61/x+1Oq2wuqywOq2ocFa0aDDDaH00kkxJSDQlKlNUorpcO91isDBwDyIGFmdRUFmAtfvXYvPPmwP+uA0aA/om9sVv0n+DLEsWkkxJGNBpAGINsWHLa0cnC7mmCNYdWBzr/3RkdVrVHyPfD5HvR6nS1fKnIp2kQ7wpPqA0IdGUGFi64B8kmOL5VETUBqrd1Sh3lNepWvQ9GJyqPoXS6lKUVJegtLq02Q8KGkmj/v+uBh1RifWup5hTYNQaQ/RNIwMDiyY6Yz+DLb9swQe/fICfTv9Ub125RtKgd3xvZFmy0C22G7rFdUOKOQWJpkRkxGYgWh8dhpwHnxACDo8D1e5qVLurYXfbUe2uVuti/dN8U31Fp/49HapcVUGtx43RxyDOEAeL0RJQh2sxWmAxKEWptYOFGH0Mq7qIOjghBKpcVSipLsEZ+xmctp/GaftpnLKfwunq0+q6bypzlDX7HAnGBKRGpyLFnIJUc6oyRdfMk6OSz+nfk5AGFi+++CKeffZZFBUVYeDAgVi5ciWGDh0a1IyFg0t2ochahL3Fe/FN0Tc4aTuJE9YTKKgsaHS/JFMS4oxxiNXHIsYQgxh9DGINsWpDpYBl7zZaSav27Zflmj7+vhbhvr77tfv8q597W4s7ZSdcHhecHiecshMOj0NZl51KmscJl+z3ubsmcKgdMNg99pC0OPfRSto63eGi9FHqcowhRm1h7gsW/JdjDbHQa1jdQERn55JdKLOX1QQf9tP1BiCn7adRWl3a5F5aJq0JSVFJSItOQ2ZcJrLispAZl4nMuEx0je0a0V13QxZYbNiwAVOmTMErr7yCYcOGYfny5di4cSMOHjyIlJSUoGWsPSmqKsJPp35CfmU+8ivykV+Zj1P2UzhVrfyxRiKDxoAofRSidIGTSWeCWWdW1/37xvt6KPj3lff1WIjWR8OoNZ6zkT4RtV9CCFQ4K3DSdhInq04qc/9l79zqsjZ6HI2kQXp0OjItgQFHVlwWOkd37vDt90IWWAwbNgxDhgzBqlWrAACyLCMjIwMzZ87Eo48+GrSMdRTljnKcsJ5ApbMSla5KtXFSpbPWsrfBUqWrElXOKsiQoYGmTt9+X599ddk7Ul/tNA2UZb1WD4PGAIPWO9Va9n1u1Bqh1+ph0poCggQ1QNCZ1UDCpDWxwRMRUS3V7mqUVpeitLoUx63HcaziGI6VH8PRiqM4VnEMNretwX0NGgO6xXVDt9huyIjNQNfYrsiIzVCr1eON8e3+d7ep9+9mdXZ3Op3Yu3cvHnvsMTVNo9Fg9OjR+Oqrr1qe2w7MV0xPRESRLUoXhYzYDGTEZiAnJSfgMyEESqtLcbTiKPIr8nGsoibgyK/Mh1N24nDZYRwuO1zvsX0NTZOikmAxWALHrqk1fk20PlqpFpZQ84AKTcD6JemXhK0xarMCi9LSUng8HqSmpgakp6am4sCBA/Xu43A44HDU1F1VVFS0IJtERETtlyRJSDYnI9mcjCGdhwR85pbdKKwqVEo4Ko7huPU4CioL8GvlrzhVfUodZM/X5iMYdty8A8aoDhBYtMTixYsxf/78UJ+GiIioXdJpdGpJx6VdLq3zuVt2o8xRhlPVp3DKfgoVjoqA9wD5etv5GtxXuargkl1qw36g5mV9vpf4hbOhe7MCi06dOkGr1eLkyZMB6SdPnkTnzp3r3eexxx7D7Nmz1fWKigpkZGRg165deOmll7B3714UFhZi8+bNmDBhQpPzMm/evHoDFrPZjKqqqiYfh4iIKJx0Gh06RXVCp6hO4c5KUDQrsDAYDBg8eDC2b9+uBgGyLGP79u2YMWNGvfsYjUYYjTXFMb62oqWlpejXrx9uueUW3H777bDZbM2qJrnnnntw2223BaRde+21uPDCC1ndQkREFGS+e+tZ+3yIZlq/fr0wGo0iNzdX7N+/X9xzzz0iPj5eFBUVNWn/goICAYATJ06cOHHi1AGngoKCRu/zzW5jMXHiRJSUlGDu3LkoKirCoEGDsHXr1joNOhuSnp6OgoICxMbGqmMaWCwWrFu3Dtdcc4263cyZM3Hw4EHMmzcPnTt3xpYtW7BgwQJ89dVX6NmzZ53jPvDAA8jNzUVBQUFEdGNtj3zVWLzGocHrG1q8vqHHaxxa4b6+QghUVlYiPT290e3afEjvejMhSQFtLPLz89GjRw/k5+cHfIHRo0dj6NChWLRoUcD+drsdaWlpKCsri5jxMdqjSBuDpL3h9Q0tXt/Q4zUOrY5yfUPeK6Qlvv/+e3g8HvTp0ycg3eFwICkpqc72mzdvhtXa+IhoREREFHrtMrCwWq3QarXYu3cvtNrAkchiYmLqbP/aa6/hyiuvxJYtW9oqi0RERFSPdhlY5OTkwOPxoLi4GCNGjGh027y8POzYsQObNm3C4MGDA3qgUHAZjUY89dRTvMYhwusbWry+ocdrHFod5fqGrY2F1WrF4cPK0KY5OTl4/vnnMWrUKCQmJqJbt264/fbbsWvXLixbtgw5OTkoKSnB9u3bccEFF+Dqq69Wj/Pkk0/i9ddfR35+fp3SDSIiImpbYQssdu7ciVGjRtVJnzp1KnJzc+FyubBgwQK8+eabOH78ODp16oSLL74Y8+fPR3Z2NgBlDI3MzExMmTIFCxcubOuvQERERLW0i14hREREFBk69svhiYiIqF1hYEFERERB0+a9QmRZxokTJwJG3iQiIqL2zX/kTY2m4XKJNg8sTpw4gYyMjLY+LREREQVBQUEBunbt2uDnbR5YxMbGAgDHkiciIupAfO8q8d3HG9LmgYWv+iMuLi6ogcWvD8yC59QppC1ZAkPXLkE7LhEREdU4WzOGdjnyZktU//e/cBcXQ64oB8DAgoiIKBwipleI5B3iVLY7wpwTIiKic1fEBBYakxJYCCcDCyIionCJmKoQyWgCAMh2e5hzQkQUOWRZhtPpDHc2qA3o9fqgvHMrggILb4kFq0KIiILC6XQiLy8PsiyHOyvURuLj49G5c+dWjTMVMYGFxsiqECKiYBFCoLCwEFqtFhkZGY0OiEQdnxACNpsNxcXFAIC0tLQWHytiAgvJxKoQIqJgcbvdsNlsSE9Ph9lsDnd2qA1ERUUBAIqLi5GSktLiapGICUElowEAq0KIiILB4/EAAAwGQ5hzQm3JF0S6XK4WHyNiAguNt/Emq0KIiIKH73Q6twTj3ztiAgvJxHEsiIiIwi1iAgu18aaDbSyIiKiuadOmYcKECeHORsjk5uYiPj4+3NmInMBCHcfCwRILIiKqa8WKFcjNzVXXR44ciVmzZgXl2PPmzUO/fv0QHR2NhIQEjB49Grt37250n507d0KSJJSVlQUlDxMnTsShQ4eCcqzWiJzAwsRxLIiIqGEWiyVkT/R9+vTBqlWr8P333+OLL75AVlYWxowZg5KSklYfu6kDlEVFRSElJaXV52utiAksWBVCREQA8PbbbyM7OxtRUVFISkrC6NGjUVVVFVAVMm3aNHz66adYsWIFJEmCJEk4evQoAOCHH37AuHHjEBMTg9TUVEyePBmlpaWNnvPWW2/F6NGj0aNHD5x//vl4/vnnUVFRge+++67e7Y8ePYpRo0YBABISEiBJEqZNmwZAKUmZMWMGZs2ahU6dOmHs2LEAgOeffx7Z2dmIjo5GRkYG7r//flitVvWYtatC5s2bh0GDBmHt2rXIysqCxWLBLbfcgsrKyhZc1aaLmMCipiqEQ88SEQWbEAKyzRaWSQjR5HwWFhZi0qRJuPPOO/HTTz9h586duOGGG+ocY8WKFbjkkktw9913o7CwEIWFhcjIyEBZWRkuu+wy5OTkYM+ePdi6dStOnjyJm2++ucl5cDqdePXVV2GxWDBw4MB6t8nIyMCmTZsAAAcPHkRhYSFWrFihfr5mzRoYDAbs2rULr7zyCgBAo9HghRdewI8//og1a9bgX//6Fx555JFG83LkyBG8++672LJlC7Zs2YJPP/0US5YsafJ3aYmIGSBLfQkZB8giIgo6UV2NgxcODsu5+/5nL6QmDtJVWFgIt9uNG264AZmZmQCA7OzsOttZLBYYDAaYzWZ07txZTV+1ahVycnKwaNEiNe31119HRkYGDh06hD59+jR47i1btuCWW26BzWZDWloatm3bhk6dOtW7rVarRWJiIgAgJSWlThVN79698ec//zkgzb89SFZWFhYsWID77rsPL730UoN5kmUZubm5iI2NBQBMnjwZ27dvx8KFCxvcp7UiqMTC292UVSFEROesgQMH4vLLL0d2djZuuukmrF69GmfOnGny/t9++y127NiBmJgYderXrx8A5el/3bp1AZ99/vnn6r6jRo3Cvn378OWXX+LKK6/EzTffrA6R7ataiYmJwfnnn3/WfAweXDeI++STT3D55ZejS5cuiI2NxeTJk3Hq1CnYbLYGj5OVlaUGFYAyVLcvT6ESMSUW6kvIWBVCRBR0UlQU+v5nb9jO3VRarRbbtm3Dl19+iY8//hgrV67E448/ftYeGj5WqxXjx4/H0qVL63yWlpYGWZYxbNgwNa1Lly7qcnR0NHr16oVevXrh4osvRu/evfHXv/4Vjz32GF577TVUV1cDUN4iejbR0dEB60ePHsU111yDP/zhD1i4cCESExPxxRdf4K677oLT6Wxw2PXa55IkKeQvlYuYwELjfVcIq0KIiIJPkqQmV0eEmyRJGD58OIYPH465c+ciMzMTmzdvrrOdwWBQhy73ufDCC7Fp0yZkZWVBp6v/FulfAtAYWZbh8A6B4B+A+J8fQJ081Gfv3r2QZRnLli1TXwj31ltvNSkfbS1yqkIMvqoQdjclIjpX7d69G4sWLcKePXuQn5+Pd955ByUlJejfv3+dbbOysrB7924cPXoUpaWlkGUZ06dPx+nTpzFp0iR88803OHLkCD766CPccccdDQYAVVVV+NOf/oSvv/4ax44dw969e3HnnXfi+PHjuOmmmxrMa2ZmJiRJwpYtW1BSUhLQw6O2Xr16weVyYeXKlfjll1+wdu1atVFnexMxgYXaeJOBBRHROSsuLg6fffYZrrrqKvTp0wdPPPEEli1bhnHjxtXZds6cOdBqtTjvvPOQnJyM/Px8pKenY9euXfB4PBgzZgyys7Mxa9YsxMfHN/jqeK1WiwMHDuD3v/89+vTpg/Hjx+PUqVP4/PPPG21P0aVLF8yfPx+PPvooUlNTMWPGjAa3HThwIJ5//nksXboUAwYMwLp167B48eLmX6A2IIlm9ONZvHgx3nnnHRw4cABRUVH4zW9+g6VLl6Jv375NPmFFRQUsFgvKy8sRFxfXokzXx37gAPImXA9tcif08WtMQ0REzWe325GXl4fu3bvD5K1qpsjX2L97U+/fzSqx+PTTTzF9+nR8/fXX2LZtG1wuF8aMGYOqqqqWfYMg8lWFcORNIiKi8GlW482tW7cGrOfm5iIlJQV79+7Fb3/726BmrLlYFUJERBR+rWpjUV5eDgDqIB/hJPl6hTidECHuSkNERET1a3F3U1mWMWvWLAwfPhwDBgxocDuHw6F2twGUOppQ8FWFAEqpRXP6PRMREVFwtLjEYvr06fjhhx+wfv36RrdbvHgxLBaLOmVkZLT0lI3yVYUArA4hIiIKlxYFFjNmzMCWLVuwY8cOdO3atdFtH3vsMZSXl6tTQUFBizJ6NpJOB3gHM+FYFkREwdGcF4BRxxeMUTmbVRUihMDMmTOxefNm7Ny5E927dz/rPkajEUaj8azbBYPGYIDsdnP0TSKiVtLr9ZAkCSUlJUhOToYkSeHOEoWQEAJOpxMlJSXQaDTqqKAt0azAYvr06fj73/+O9957D7GxsSgqKgKgvCUuqh20aZBMJsBmY4kFEVErabVadO3aFb/++iuOHj0a7uxQGzGbzejWrVuDg4E1RbMCi5dffhkAMHLkyID0N954A9OmTWtxJoJFYpdTIqKgiYmJQe/eveFyucKdFWoDWq0WOp2u1aVTza4Kac806iBZrAohIgoGrVYLrVYb7mxQBxIx7woBasaykPnqdCIiorCIqMBCY/RVhbDEgoiIKBwiKrCQvIGFzKoQIiKisIiswEJtvMmqECIionCIqMBCY/S+L4RVIURERGERUYFFTVUIu5sSERGFQ2QFFhzHgoiIKKwiKrDwVYXIrAohIiIKi4gKLHxVIYJVIURERGERUYGFhlUhREREYRVRgYXEqhAiIqKwirDAQnnNK6tCiIiIwiOiAguN910hwsnAgoiIKBwiKrBQq0JYYkFERBQWERVYaNSqELaxICIiCoeICizU16azKoSIiCgsIiuw4DgWREREYaULdwaCSaMGFqwKIQomIQSEywXhdCqT/3KtNDlg3VXvNsLlVD6TZcDjgZA9ytzjW5cBj1tZl/3SPZ4668q+sjJ311r3yBAeDyRJAnQ6SFotoNVA0nqXvWlKuncuScqXliS/ZWVdQn2fBW4n+ZYR+m18y5IkAZJG+W4aLaDRQNJolLlWA2i0gEZSPtNqlO+p0dZcD03NdfGtSzrfNhpAr4dUZzIoc4MeGpMJktEIjdEIyWiEpIuoWws1U0T966uNN518bTq1P0II5WbodiuTywX4lt1uCJcbwn2WNJfvM5dyLP/1+rYJWK9Ja/Dm31DA4HKF+/JRR6LTQWMwQPIFHP7LvuDDZITGYPSmGwKXjd5tTd5t1f18n3u3NdRsozEalQBIDcgoXCIqsPCNvClbrbB98w2MvXtDGx8f3kxRSPlu0OrkdyM867r/jdXlqnmKbuqxAm7WnsD1gJu5MiGSbs46HSSDARq9HpLB9+RqCJwC0vTQGAyAXpkrT7s6QKtTnoh9pQW+J2T/p2mt9wnc9zTdyHrNvoHHBITyb+DxQLh9pR6egDThUZYhBABvICi831cINR0QtbaptZ1vRQjlc3X/2seqbzvUnKe+bRrJj1LKI0MIZQ4h15TseNNql+b4Sn+U7y7XX/rj9l4r/799//8nnE4Iuz0w+HS7IbvdgM3W4j+xFpEkSCYTNEYjNGYzNNFmSGYzNFFmZT1gilLmcXHQxsdDl5AAbXw8tAkJ0FoskPT6ts17BImswCI6GgAgV1bi2OQpAABDZia0iYnQJiXCfOFgmIcNhalfP+UHh1pECAHhcECuroaw2SDb7cq6w+F9+nXUrDucyrrTWbPucEA4HcpTs2+9ucGA7ylalsN9OVpHkpRiY70ekk4XMEGvg6Tzpmu1gev1baPTKTdrXRPS6rnx+9I0jQYI3nVNRDXPoiAQsqz8v6z+v+/w/jYovwHqssOufG73/g7YHTW/Cb5lhx2ywxuw+KXLDrv6m+E7R0DVtxAQ1dXwVFfDU1bWqu+jiY1VAw1dSjL0qZ2h65wKfec06DunQte5M3SpqUrATAEkoYbKbaOiogIWiwXl5eWIi4sL+vGL/3c5bHv2wH3yJFy//lrvNhqLBcYePaDr1Am65GTokjtBqy4nQ9LpIRx2aMxm5Q+rA0av6s3fZoNsq4Zsq4KorlbWq6uVtGqbN61aSatvXV2urgkkqqv9npbaF7X+13cDPNu6IbCuWK07NjSyr15fc6PW+93kG0qrFTTAfxsGuEStorb/8X/A8f1m2WwBv4G+dVFdDbnKBrmqCp6KCnjKyuA5c0aZKiqa9fumTUxUAo7UztCnpcGQlQVD9+4wdO8OfXpaRAXhTb1/tyiwePHFF/Hss8+iqKgIAwcOxMqVKzF06NCgZiwY3GfOwHHgADxWK1z5Baj6925U79kLuaqq2cfSxMRAGx8PyWQE3B5oYmOVQCQlGbrEJEhRSvGbZDAqRWzR0QGTZDACEMoTtixDyAIQck0RphDedNkbEPgFA77/CdTAwKbc4G3+gYKSLttqgoG2eJqXvHWdar2pr67UYKhZ914XyWhUisT91401T8Qavxt4TZF5E4ID7zJ0OtavElGrCI9HCTZ8gcaZM3CdPAl30Um4ThYFzM/2wkvJbIapd28Y+/aFsW8fmPr2hbFvX2hjY9vo2wRXyAKLDRs2YMqUKXjllVcwbNgwLF++HBs3bsTBgweRkpIStIyFinC7Yf/pAFwnTsBdWgJ3iXcqLVWXIQtojEY1mm2vT+dNJZlM0ERFKVO0GVKUuWbdHAUpKkqpg6y9bla2CVg3m71pZmiiTHziJqJzkhACnrIypXS8yBtwHP8VzqNH4cjLg+tYfoONnvXp6WqwYezRA4Zu3aDPzFQeXtvxw1HIAothw4ZhyJAhWLVqFQBAlmVkZGRg5syZePTRR4OWsfZCjV7LyuApK4NwuiBpNfBUVMBdXKxMp08H1vtV2+CpqoJcVaUWtwmHQ2lgJkmArxuY3zI0EiRJo9S5+xoeRUUFNDKSoqKgMUcHpEu+5SiloZLvs5ptefMnImprwu2GMz8fjgMHYD94CI6DB2E/dBDuE4UN7qOJjYUhIwP6zG4wdMuEvnMqtIlJ0CUlKlUuiYnQxMWFrXolJIGF0+mE2WzG22+/jQkTJqjpU6dORVlZGd577706+zgcDjj8iosqKiqQkZHRYQILIiKiYPFUVMBx6BDsBw/CcfAQnMeOwZmfD3dhwwFHbWoVtK8br8GgPEAKGZ5KK+SKCvT6dGfQq1yaGlg0q1dIaWkpPB4PUlNTA9JTU1Nx4MCBevdZvHgx5s+f35zTEBERRSRtXBzMF10E80UXBaTLdjtcv/4KZ34+nMfy4cw/BndJCTynTsNz+jTcp09DrqwEgJreN42cR66sDFtbjpB3N33ssccwe/Zsdd1XYkFEREQKjckEY69eMPbq1eA2stMJubISwu7tjuvw9oSxOwDZA0gSNDGx0MYpnQvCpVmBRadOnaDVanHy5MmA9JMnT6Jz58717mM0GmH0DrUNQB0I5uOPP8Zf//pX7Nu3D0VFRVi3bh2uueaaJudl8eLFWLJkSZ10s9mMwmYUKREREXUY3l5zaKAwwgPABcBeXQ1UVwf11BUVFQBq7uMNEs00dOhQMWPGDHXd4/GILl26iMWLFzdp/4KCAt9wcpw4ceLEiROnDjYVFBQ0ep9vdlXI7NmzMXXqVFx00UUYOnQoli9fjqqqKtxxxx1N2j89PR0FBQWIjY1Vu9VYLJY6JRYOhwNPP/00Nm3ahPLycvTv3x/z58/HiBEj6j3u119/jbFjx+Ltt9/GFVdc0dyvRU3gq8YqKChgw9sQ4PUNLV7f0OM1Dq1wX18hBCorK5Gent7ods0OLCZOnIiSkhLMnTsXRUVFGDRoELZu3VqnQWdDNBoNunbtWifdbDYHXKi7774b+/fvx4YNG5Ceno7Nmzfj97//Pb7//nv07t27zv6bNm0CAFxxxRX8gw6xuLg4XuMQ4vUNLV7f0OM1Dq1wXl+LxXLWbdp8SO96MyFJ2Lx5s9qFNT8/Hz169EB+fn5AZDR69GgMHToUixYtCtjfbrcjLS0NZWVl7MYaQh1tDJKOhtc3tHh9Q4/XOLQ6yvVtly8h+/777+HxeNCnT5+AdIfDgaSkpDrbb968GVarta2yR0RERA1ol4GF1WqFVqvF3r17oa01amRMTEyd7V977TVcddVVyMnJCeiBQsFlNBrx1FNP8RqHCK9vaPH6hh6vcWh1lOvbLqtCDh06hL59++Kzzz5rsLGmT15eHnr27In333+/Wd1ViYiIKPjCVmJhtVpx+PBhdT0vLw/79u1DYmIi+vTpg9tuuw1TpkzBsmXLkJOTg5KSEmzfvh0XXHABrr76anW/119/HWlpaRg3blw4vgYRERH5CVuJxc6dOzFq1Kg66VOnTkVubi5cLhcWLFiAN998E8ePH0enTp1w8cUXY/78+cjOzgagvAAtMzMTU6ZMwcKFC9v6KxAREVEt7aIqhIiIiCJDeN69SkRERBGJgQUREREFTZs33pRlGSdOnAgY0puIiIjaN/8hvTWahssl2jywOHHiBF+bTkRE1EEVFBTU+2oOnzYPLGJjlXe9BvslKoVHyuByyOjcIw4GU7sc94uIiKjD8r0EzXcfb0ib34F91R/BfonK23//FtWVLtzy5FDEpdQdnZOIiIha72zNGCKm8aZWr3wVt1MOc06IiIjOXRETWOj0yjtF3C5PmHNCRER07oqcwMKgfBWPiyUWRERE4RIxrRx1vqoQBhZERC3i8XjgcrnCnQ0KE71eX+eN4i0RMYGFllUhREQtIoRAUVERysrKwp0VCrP4+Hh07ty5VeNMRUxgoWPjTSKiFvEFFSkpKTCbzRy88BwkhIDNZkNxcTEAIC0trcXHirjAgm0siIiazuPxqEFFUlJSuLNDYRQVFQUAKC4uRkpKSourRSKm8abWwDYWRETN5WtTYTabw5wTag98fwetaWsTMYGFr7uph20siIiajdUfBATn7yBiAgsOkEVERPWRJAnvvvtuk7efNm0aJkyY0KpzHj16FJIkYd++fa06TnPMmzcPgwYNarPzNSRiAgt2NyUiOvcUFRXhgQceQK9evWAymZCamorhw4fj5Zdfhs1mC3f2GpWbm4v4+PigHW/OnDnYvn170I7XUhHXeJOBBRHRueGXX37B8OHDER8fj0WLFiE7OxtGoxHff/89Xn31VXTp0gXXXnttuLPZak6nEwaD4azbxcTEICYm/O/KalaJxbx58yBJUsDUr1+/UOWtWXQGtrEgIjqX3H///dDpdNizZw9uvvlm9O/fHz169MB1112HDz74AOPHj693v++//x6XXXYZoqKikJSUhHvuuQdWq7XOdvPnz0dycjLi4uJw3333wel0qp9t3boVl156KeLj45GUlIRrrrkGR44caXLed+7ciTvuuAPl5eXq/XTevHkAgKysLDzzzDOYMmUK4uLicM899wAA/vjHP6JPnz4wm83o0aMHnnzyyYBGlrWrQnxVOs899xzS0tKQlJSE6dOnh3wQtGZXhZx//vkoLCxUpy+++CIU+Wo2LUssiIjOGadOncLHH3+M6dOnIzo6ut5t6muIWFVVhbFjxyIhIQHffPMNNm7ciE8++QQzZswI2G779u346aefsHPnTvzjH//AO++8g/nz5wccZ/bs2dizZw+2b98OjUaD66+/HrLctHvQb37zGyxfvhxxcXHq/XTOnDnq58899xwGDhyI//73v3jyyScBALGxscjNzcX+/fuxYsUKrF69Gv/7v//b6Hl27NiBI0eOYMeOHVizZg1yc3ORm5vbpDy2VLOrQnQ6HTp37hyKvLQKB8giImo9IUTYfkd1Bk2TeyUcPnwYQgj07ds3IL1Tp06w2+0AgOnTp2Pp0qUBn//973+H3W7Hm2++qQYkq1atwvjx47F06VKkpqYCAAwGA15//XWYzWacf/75ePrpp/Hwww/jmWeegUajwe9///uA477++utITk7G/v37MWDAgLPm32AwwGKxQJKkeu+pl112GR566KGAtCeeeEJdzsrKwpw5c7B+/Xo88sgjDZ4nISEBq1atglarRb9+/XD11Vdj+/btuPvuu8+ax5ZqdmDx888/Iz09HSaTCZdccgkWL16Mbt26Nbi9w+GAw+FQ1ysqKlqW07OoGSCLVSFERC3ldsp49YFPw3Lue1b8Dnpj695V8e9//xuyLOO2224LuPf4/PTTTxg4cGBAKcfw4cMhyzIOHjyoBhYDBw4MGNvjkksugdVqRUFBATIzM/Hzzz9j7ty52L17N0pLS9WSivz8/HoDi/PPPx/Hjh0DAIwYMQIffvhho9/joosuqpO2YcMGvPDCCzhy5AisVivcbjfi4uIaPc75558fMNBVWloavv/++0b3aa1mBRbDhg1Dbm4u+vbti8LCQsyfPx8jRozADz/8gNjY2Hr3Wbx4cUDxUajUvCuEJRZERJGuV69ekCQJBw8eDEjv0aMHgJpRJENl/PjxyMzMxOrVq5Geng5ZljFgwICAdhj+/vnPf6ptG5qSt9rVO1999RVuu+02zJ8/H2PHjoXFYsH69euxbNmyRo+j1+sD1iVJanJ1TUs1K7AYN26cunzBBRdg2LBhyMzMxFtvvYW77rqr3n0ee+wxzJ49W12vqKhARkZGC7PbML42nYio9XQGDe5Z8buwnbupkpKScMUVV2DVqlWYOXNmg+0sauvfvz9yc3NRVVWl7rNr1y5oNJqAapVvv/0W1dXVahDw9ddfIyYmBhkZGTh16hQOHjyI1atXY8SIEQBw1vaGmZmZddIMBgM8nqaVsn/55ZfIzMzE448/rqb5SkDam1aNYxEfH48+ffrg8OHDDW5jNBoRFxcXMIUCu5sSEbWeJEnQG7VhmZo76uNLL70Et9uNiy66CBs2bMBPP/2EgwcP4m9/+xsOHDhQ77subrvtNphMJkydOhU//PADduzYgZkzZ2Ly5MlqNQigdPG86667sH//fvzzn//EU089hRkzZkCj0SAhIQFJSUl49dVXcfjwYfzrX/8KeIBuqqysLFitVmzfvh2lpaWNjrvRu3dv5OfnY/369Thy5AheeOEFbN68udnnbAutCiysViuOHDnSqregBYtaFeJkGwsionNBz5498d///hejR4/GY489hoEDB+Kiiy7CypUrMWfOHDzzzDN19jGbzfjoo49w+vRpDBkyBDfeeCMuv/xyrFq1KmC7yy+/HL1798Zvf/tbTJw4Eddee63aHVSj0WD9+vXYu3cvBgwYgAcffBDPPvtss/P/m9/8Bvfddx8mTpyI5ORk/PnPf25w22uvvRYPPvggZsyYgUGDBuHLL79Ue4u0N5IQQjR14zlz5qj1SidOnMBTTz2Fffv2Yf/+/UhOTm7SMSoqKmCxWFBeXh7U0ouS/Eq8tegbmC0G3LH00qAdl4goktntduTl5aF79+4wmUzhzg6FWWN/D029fzerjcWvv/6KSZMm4dSpU0hOTsall16Kr7/+uslBRSixjQUREVH4NSuwWL9+fajy0WocIIuIiCj8IuglZL4hvWU0o3aHiIiIgiiCAouar8LqECIiovCImMBC69f/mdUhRERE4RE5gYVWA0mj9IHm+0KIiJqHVcgEBOfvIGICC8DvfSFujmVBRNQUviGfGxucic4dvr+D2kOBN0ezX0LWnukMGrgcHpZYEBE1kVarRXx8PIqLiwEoA0g1dwRM6viEELDZbCguLkZ8fHy9o5Y2VUQFFlodu5wSETWX77XdvuCCzl3x8fH1vsa9OSIqsNAZfF1OWRVCRNRUkiQhLS0NKSkp6hs46dyj1+tbVVLhE1GBhTpIFqtCiIiaTavVBuXGQue2iGy8yaoQIiKi8IjIwIIDZBEREYVHRAUW6qvT2caCiIgoLCIqsPC94ZRtLIiIiMIjsgILtrEgIiIKq4gMLNjGgoiIKDwiKrBgGwsiIqLwiqjAglUhRERE4RVRgYXv1ekeNt4kIiIKi4gKLGpKLFgVQkREFA4RFlj43hXCEgsiIqJwiKjAQss2FkRERGEVUYGFOkAWAwsiIqKwiKzAwtfd1Mk2FkREROEQYYEFB8giIiIKJ124MxBMWlaFEJ0zhCwgeyfh8S57BIRQ5u2dEAIQ/uuAf4KkkaDRaCBpAI1WgkYjedMkSL51SWrzfBOdTUQFFjodAwui5vLdiD1uue7cLeDx+M9leDwCsm8b32f17VvnGDX7ejzKOZXlmmMEfO49hiwLJYjwX5YDb8rnKo1WglangVangUZXs6xMkjddA51embTqXKvMDb7PtH6faaAzaKE31jOZtNBqI6qgm0IgsgILg7e7KdtYUAfm8cjwOGW4nB54XDVzt9MDt1NWJpcncO77zLdd7bkzcF0JCmpu3pFE0kiQNICEID/Nh6BwQPI/rrf0QYISMwm/kpiGyB4B2eOBy9F2v3kanVQTaBi0METpYDB5577JpIMxSgdDlBYGU026MUoHvUkLY5QOWr2GJS4RKqICC193U4fNjQNfFaJrv0TEJBjDnCuKBLLHd9P2v4nXvWHXfyNvYNt6AgGPU270RtImJChPvFoJGp1GeSrW1jwR+56Sa+bK07E699tXq1XSNTq/Y9S7Xa1tvMdXqgCUZUlTUz2gVg145xpNzeeReLPyldL4zz1uAdkjKyU9Ll+Jj7dkyG9d+Vz5O/S4a/4ePS7v359LVgNX9XOnBy6nDJfDDZdDCVxkt/J3KbsFHG43HFXuVn0njU6CMUoHo1kPo1nnXdbBYNary0azDqYYPUzRgZPvt57ap4gKLKJi9ZAkpSpk+5qfAACp3eOQ2j0OnbrGIK1nPCwpURH5w3MuqnOzr+/m7qr7lO9xKT+aHu+Pp6cJT/lheaqXlAbJOoO2Zm5Qiq11Bv/0Wmn+6979tHoN9Aatt+hbW1NMXs9NXqPh/x/tjaSRoA3zv4vHLcPl8ChBhzfYcNo9cNndcFa74aj2wOlddla74bR71GVHtdv7mbINhBKgVFe6UF3panZedEYtTNG6OgGH0S/NGK2HyRuYGM16mKJ10LAap01IQog2/cWsqKiAxWJBeXk54uLign78gv2ncfSHUpzMq8DJoxV16mGj4gxI72lBWq94pPWyICk9htFvEMmyUJ58vEX4bu+N2+V9Gq9dvF+3mL/pT/nhKsIPuMmrN22l3jpgbgi8uTceHNROV+rJGQRTpBGygMvpgcPmDTpsLjhsbr/JBUe1d7nKBXuVG/YqF+xVLjiqXGjNHctg0ioBR7QSaBj9gxJf6YjZG5R4gxSjmQGJT1Pv3xEXWPirKnMgf/8pnDpRhZJjlTiZVwGPO7Bhp6SRYEmOQmJaNCwpUYiKMSAqVg9TjB5RMQZlHquH3qiNiB95IYRaLKrc+D3ep5CadbfDVwzq97nDA5fTA5dD9tvH++Ti9MDtqAkSwsHXEE3vvdEH4ym/vv1YL0wUPkIW3qDDBbu1JuCwW12w21xweIMQhy/dG5w4bK2rtjHF6BFtMSLaYoA53ojoOAOi440wWwyItnjnccaIf0gNaWDx4osv4tlnn0VRUREGDhyIlStXYujQoUHNWCh4XDKKj1XgxOEyFB4uR9Ev5U3+g9PoJETFGJQ6QJMWepMy9xW56fQ1rbF1Bg30JqVhk87byEmr06ilJ0IICFHT3UwIAHXSlMZ1Hm/9p8ftrQt116zXnrt9dakuWQ0O3E7/AEBZb6vW9L5W5nVu8gE38bpP6v7b+xfh6/3X/Yv2dRpILL4nogbIsoDT5heIqMGHWwlIrEoQEhCUVCklKs1hitYjOt4AszcIiUkwIa5TFOI6KfPoeGOHrmoMWWCxYcMGTJkyBa+88gqGDRuG5cuXY+PGjTh48CBSUlKClrG2IIRAVZkTZwqrcLqwCpWn7Ki2OmG3ulBtdSnLla6I7b6q0UlK8OPtWqbz3rx9N3e90RsY+X9urNleDZwMWuiMmprtjFre7Imow5M9Mhw2N2wVTlSVOVBV7kRVuQM2de5AVZkTVRUOtXFrYzRaCbFJvmAjCnFJJqXkI84Ac5wBUbFKKXlrgw8hREhKVkMWWAwbNgxDhgzBqlWrAACyLCMjIwMzZ87Eo48+GrSMtScupwfVlUrA4ah2w+VrsGT3oNqqRLhqy2tvq2pf4ya30wOXXWltDQmQJEnpVRawrMzVdXgba/n6ovuVhqjL3rnOP02ngVYvKTd+X4Dgu/EbagID32esNyQiaj0hBBxVblSVO5SpTAk8Kk/bUVlajfJSO6yn7E3q8SVJgCnWAHOsAaZoHfQmHfRGrVpSrjNoAgZLU3pMKV2AXXYPTvxchlMnrLht3sVBf7hr6v27Wb1CnE4n9u7di8cee0xN02g0GD16NL766qt693E4HHA4HAEZ62j0Bi30SVGIS4oKd1aIiKidkSRJafgZo0dSl5h6t5FlAesZOypL7ag4VY2KUjsqSqtRVe5EdaUTtgrl4VUIoLrCieoKZ6vyVHrciuSM2FYdo6WaFViUlpbC4/EgNTU1ID01NRUHDhyod5/Fixdj/vz5Lc8hERFRB6fRSIjzPqB2QUK928geGdVWF2wVSqDh9JaQO70l5C67G26nDFmImgHUhIDsFnA5PJAkoHNPC7r0SUBienQbf8MaIR/H4rHHHsPs2bPV9YqKCmRkZIT6tERERB2KRqvx9j7p2AM7Niuw6NSpE7RaLU6ePBmQfvLkSXTu3LnefYxGI4zGmovka9LREatEiIiIzlW++/bZmmY2K7AwGAwYPHgwtm/fjgkTJgBQGm9u374dM2bMaNIxKisrAYClFkRERB1QZWUlLBZLg583uypk9uzZmDp1Ki666CIMHToUy5cvR1VVFe64444m7Z+eno6CggLExsYGtTuMr4qloKCgw/Q26Wh4jUOL1ze0eH1Dj9c4tMJ9fYUQqKysRHp6eqPbNTuwmDhxIkpKSjB37lwUFRVh0KBB2Lp1a50GnQ3RaDTo2rVrc0/bZHFxcfyDDjFe49Di9Q0tXt/Q4zUOrXBe38ZKKnxa1HhzxowZTa76ICIionMHR0giIiKioImYwMJoNOKpp54K6IFCwcVrHFq8vqHF6xt6vMah1VGub5u/3ZSIiIgiV8SUWBAREVH4MbAgIiKioGFgQUREREHDwIKIiIiCpsMHFqWlpXzvCBERUTvRoQOLRYsW4bLLLsNFF12EG2+8EV9++WW4s0QUdOy4FVq8vkTB1WG7my5cuBArVqzA0qVLYTAY8OKLL8Lj8eCpp57CVVddFe7sRZytW7fCZDLBZDLh4osvDnd2zgn5+flISkqCEAIxMTEQQgT1/TrnOl7f0HrnnXfw5ZdfolOnTsjJycHYsWPDnaWI026vseiAqqurxZVXXin+93//V007fvy4eOihh8R5550nvv322/BlLgJdf/31okuXLqJXr17CYDCIBx98UBw4cCDc2YpoDz30kOjfv7/o16+fGD58uNi7d6/weDzhzlbE4PUNrccee0zExsaKG2+8UQwcOFBERUWJRYsWCZvNFu6sRYz2fI07ZGBht9vF0KFDxSOPPBKQfvjwYXH33XeLiy++WJw5cyY8mYswzzzzjBg4cKAoKCgQBQUF4r333hPp6eli8uTJ4r///W+4sxeRHnnkEZGZmSn++c9/itWrV4sJEyaIuLg4sXbtWlFVVRXu7HV4vL6hdeDAAdGzZ0/x0UcfCSGEKCsrE6tXrxYajUYsWLBAWK3WMOew42vv17hDBhYul0vcfPPNYsKECaKkpCTgs507d4qLLrpILF++PEy56/hkWVaXp02bJm6++eaAz999911xwQUXiBkzZogTJ060dfYi3uWXXy6WLl0akDZlyhTRq1cv8c477/DJupV4fUPrX//6l0hLSxO//vprQPoLL7wgtFqt2LRpkxAi8HeGmqe9X+MO2XhTp9Nh9uzZeO+99/C3v/0toPHV7373O/Tr1w8bNmwIYw47tpMnTwIAnE4nrFYrdDrlJbgulwsAcN111+Huu+/Ghx9+iF27dgFgA7hgEEKgtLQUx44dQ0JCAgDAbrcDANasWYNu3bphyZIl6r8PNY/b7eb1DSHfb0BmZiaKi4vx7bffAlCuOwDMnDkT06ZNw4MPPghZltmepZlkWVaX2/01Dks4EyRLliwRRqNRbNy4UdjtdjV93rx54rrrruOTRws8/vjjol+/fuLUqVNCCCE2bdokJEkSe/bsEUKIgOs8fvx4cemll4Yln5Hs1ltvFQMGDFDXfdf81KlTwmw2iz//+c/hylqHdOjQoYD122+/ndc3iE6ePCkcDoe6Xl1dLaZMmSIuvfRScezYMSGEEE6nUwihtIXLzMwUr776aljy2lFt2LBB/buUZVnYbDYxbdq0dnuNO2SJhc8f//hH3HXXXbjrrrvwwgsv4Ouvv8ZPP/2Ev//97+jbty80mg799drcxIkT8dJLL+HVV19FYmIiAODKK6/EddddhxtuuAFWqxVGoxFOpxMAcOedd+LIkSP49ddfWWLRQu+88w42b96Mf/7zn2ragw8+CJvNhgceeACA8kZDh8OBxMRE3Hvvvfjggw9QXV3Na94EDz/8MG666SacPHlSvV7Tp0+Hw+Hg9Q2Cp556CldccQWGDh2Kq666Cvv374fJZMJtt92m9tKz2WzQ6/UAlGut0+ng8XjCnPOO4+GHH8Ytt9yC7OxsAIAkSYiKisJ1110HAO3zGoctpAmiRx55RFx88cXCYrGIHj16iEmTJoU7Sx2Kw+EQQ4YMEQMHDlTbTJSVlaklPt99950YMmSIGDx4cECL45deekkMHz48oBSDmu76668XqampYtCgQUKSJDFx4kTx+eefCyGE+POf/yy6d+8unnvuuYB9pk+fXqfNC9Xv2muvFYmJieKbb74JSC8vLxdLly4VPXv25PVthUcffVR06dJFrF27VvzlL38RgwcPFgMGDBDr168XQgjx4osvisGDB4vJkyer+5SWlooBAwaIDRs2hCvbHcqECRNEenq6+Oqrr+r9/LnnnhNDhgxpd9c4IgILIYQoKioS33zzjVpkT023evVqodfrxSuvvCKEEOLNN98UV1xxhTj//PPF6NGjxXvvvSc++eQTccEFF4jzzz9fPPTQQ2LVqlUiMTFRPPHEE2HOfce0atUqccEFF4j8/Hxhs9nE119/LS6++GJxxRVXiC+//FLYbDbxpz/9SZjNZrFgwQLx+eefi2+++UZ0795dzJ8/P9zZb9eqqqrE4MGDxcCBA0VlZaUQQoji4mJRXV2trp84cYLXtxUcDof4zW9+I1atWqWmuVwuce2114pLLrlE/POf/xQej0e89tprIjMzU/To0UP8/ve/F1lZWWLs2LFhzHnH4PF4xG233SYMBoPYt2+fEEKIL7/8UixdulQ89dRT4h//+IcQQqnGW716dbu7xh12gCwKHpvNhrlz5+Ljjz9G9+7d8cMPP2DKlCmIj4/H+++/D6vVipkzZ+Laa6/FQw89hF9++QVutxvXX389Zs2aFe7sd0gPPvgg/vvf/2Lnzp1q2meffYaFCxfCaDRi1apVSEtLw5o1a/DUU0/BaDTC7Xbj6quvxssvvxy+jHcAL774Ip544gn86U9/wsMPP4w33ngDa9asUatDFi9ejPHjx8PlcmHdunW8vs0khEBJSQkuv/xyzJgxA/feey+cTicMBgMKCwtx6623IiYmBn/5y1+QlpaGwsJCrFq1ClqtFomJiXjwwQfD/RU6hOeeew4bNmzA5MmTYbfbsWrVKvTr1w8lJSX47rvvMGvWLCxduhQajQZFRUXt6xqHNayhdqO4uFjcdNNNon///mLbtm1qusPhEGPGjBGjR48WQihPJUIoVSXUfB6PR3g8HvHwww+LsWPHiqqqqoBGxhs3bhRDhgwRS5cuVa91fn6+yMvL48BvTXT69GnxwAMPiBEjRojf/va3IisrSyxfvlysXr1aTJs2TaSkpIi1a9eq153Xt2VGjhwprrzySnXd13jwq6++ErGxseKNN94IU846Nv8uog8//LDo0qWL6NGjh9i4caNa4vb2228LSZLUaqf2hoEFqX7++WexadMmdZAgt9sthBBi3bp1wmAwiIKCAva0aaHa4618+umnQqPRiHfeeUcIUXOthRDivvvuE+eff766zv7+Z1f7+h4+fFjceOONYsiQIWLHjh0Bn910001i4MCB6jqv79l9/fXX4t///rc4ePCgmvbVV1+JqKgosWzZMiGE8jfs+zueNm2a+O1vfxuWvHZU9V1jl8slZs2aJV5++eWA3wghlPYXvge+9oaBBQXwPXX4e/rpp8XVV18dhtxEhv/5n/8R48ePF7/88ktA+owZM0RCQoL4+eefhRBCDdr27NkjoqOjxf79+9s8rx1RQ9d337594q233lIbHPt+mLds2SJMJpM4fPgwg4omuPPOO0WvXr1EZmamiIqKEm+++aYQQoiKigrx9NNPC4PBIN57772Afe69996ABoXUuPquse/3oLKyUpSWlgZsb7fbxZVXXimmT58ejuyeFQMLatTOnTtFz549xbPPPhvurHQ4brdb3H333aJr165Cp9OJ6dOnB/xAFBYWilGjRonevXsHBBF///vfxeDBgzks/Vmc7foKUVN1J0RNycSSJUvEmDFjWPp2Fi6XS0yYMEEMGjRIfPfddyIvL088/vjjIiEhQZw+fVoIIcSvv/4qpk+fLrRarXjjjTfE119/LX766SfRs2dPMW/evDB/g/avKde4Pt9//73IyckRa9asacPcNh0DC6rX+++/L2bNmiUsFgt/IFro22+/FTfffLP46KOPxP/93/8JSZLEokWL1HpSIZTBbC6++GLRt29fMXHiRLFkyRKRkJAg5syZE8acdwwNXd/G3pPw0UcfiYyMDLX4nhr2j3/8Q4wcOTIg6LVarSIzM1Ns3LhRTauurhZz584VGRkZIj09XXTr1k1MmTIlHFnucBq7xm+//Xad7ffu3Sv+9re/ieTkZHHvvfe2ZVabhYEF1au8vFzccMMNYsuWLeHOSofldDrF9u3bRUVFhRBCiOeff15otVrxt7/9rc7YH88884y47rrrxLXXXitWrlwZjux2OI1dX/+RIIVQqj9uvfVWER8fL5YsWRKO7HY4p06dEvfcc0/A36rdbhfdunUTH374YZ3t9+/fL/773/+Kr7/+ui2z2aE15xpXVlaKZ599VmRlZQW82bs9YndTapDb7VbfE0KtI4SAJEm477778NZbb+Gtt97C5ZdfXmcs/6qqKkRHR4cplx1XY9dXCIGTJ0/i8ccfx6RJkzB69OhwZ7dD8ng8qK6uxrBhw/C3v/0NOTk54c5SxDnbNa6oqMDJkyfRu3fvMOWwaTjmNTWIQUXw+OL3V155BTk5OZgxYwZ++OEH5OfnY8aMGfjkk08AAGazOZzZ7LAau77Tp0/HsWPHsHr1agYVLeC7tlqtFtXV1Th9+rQ6XLTT6cRrr72G/Pz8cGaxwzvbNV69ejXy8/MRFxfX7oMKAGCJBVEb8S8B6tevH2JjY/Hrr78iIyMDX3zxBQwGQ5hz2LHVd30LCgrQrVs3Xt8gOXz4MIYNG4ZffvkFVVVVGDlyJBISErBr1y4+iARJJFxjllgQtRGdTqe+3nj27NnYu3cvxo8fj3//+9+86QVBfdf32muv5fUNouLiYvTp0wf79u3DwIEDkZOTg927d3eYG15HEAnXmIEFURvS6XR4/fXXcd9992HBggV49dVXw52liMLrG1pVVVXYvXs3LrvsMtx9993YsGFDuLMUcSLhGrMqhKgNCSHwwQcfwO12Y8KECeHOTsTh9Q2tsrIydOrUCe+++y6uueaacGcnIkXCNWZgQURETWa322EymcKdjYjW0a8xAwsiIiIKGraxICIioqBhYEFERERBw8CCiIiIgoaBBREREQUNAwsiIiIKGgYWREREFDQMLIiIiChoGFgQERFR0DCwICIioqBhYEFERERB8/8BtAPbg90AZHEAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "vis.show_histograms(data = data, plot_type=\"subplot\")" - ] - }, - { - "cell_type": "markdown", - "id": "fbf7fc73", - "metadata": {}, - "source": [ - "\n", - "#### show percent display format with default plot_type (main)\n", - "In the following, we display only feature \"Intensity\" in \"percent\" display_format, with default plot_type" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "8655ad63", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAHBCAYAAABKReAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAACFIUlEQVR4nOzdd3wUdf7H8ddsT930SkjoNRCkg10koqLYQFQQ9Lyz4KnYG3YQTzxQVE49D+Tg5FTk/KGiiGADkd47hFDSe3azfX5/LFmISdgEEjfA5/l4jLs7+52Z744h+873+53vKKqqqgghhBBCtGCaQFdACCGEEMIfCSxCCCGEaPEksAghhBCixZPAIoQQQogWTwKLEEIIIVo8CSxCCCGEaPEksAghhBCixZPAIoQQQogWTwKLEEIIIVo8CSxCiHrNnj0bRVHIysoKdFVOy4oVK1AUhRUrVgS6KkKIUySBRQjR5LZv387zzz/fooPO/PnzmT59eqCrIYRoIEXuJSSEqI/b7cbpdGI0GlEUpcHbffrpp9x0000sX76ciy++uPkq2EAejweHw4HBYECj8f6ddvXVV7N169YWHaqEEMfpAl0BIUTLpdVq0Wq1ga7GadNoNJhMpkBXQwhxGqRLSAhRr9+PYUlLS+Pqq6/m559/pl+/fphMJtq2bctHH31UY5ubbroJgEsuuQRFUWqNH/n666+54IILCAkJISwsjKuuuopt27bVOPa4ceMIDQ3lyJEjjBgxgtDQUGJjY3nkkUdwu901yn788cf07t2bsLAwwsPDSU9PZ8aMGb73fz+G5eKLL+bLL7/k4MGDvvqlpaVRWVlJSEgIDzzwQK1zcfjwYbRaLVOmTDmdUyqEOEUSWIQQjbJ3715uvPFGLr/8cqZNm0ZkZCTjxo3zBY4LL7yQv/71rwA89dRTzJ07l7lz59KlSxcA5s6dy1VXXUVoaChTp07l2WefZfv27Zx//vm1umfcbjeZmZlER0fz+uuvc9FFFzFt2jTee+89X5mlS5cyevRoIiMjmTp1Kq+++ioXX3wxv/zyS72f4emnnyYjI4OYmBhf/aZPn05oaCjXXXcdCxYsqBWK/vOf/6CqKrfeemtTnEYhRGOpQghRj3/9618qoB44cEBVVVVNTU1VAfXHH3/0lcnPz1eNRqP68MMP+9Z98sknKqAuX768xv4qKirUiIgI9a677qqxPjc3VzWbzTXW33777SqgvvjiizXK9urVS+3du7fv9QMPPKCGh4erLper3s+xfPnyWvW56qqr1NTU1Fplv/nmGxVQv/766xrre/TooV500UX1HkMI0bykhUUI0Shdu3blggsu8L2OjY2lU6dO7N+/3++2S5cupbS0lNGjR1NYWOhbtFot/fv3Z/ny5bW2ufvuu2u8vuCCC2ocKyIiAovFwtKlS0/jUx03ZMgQkpKSmDdvnm/d1q1b2bx5M7fddluTHEMI0Xgy6FYI0SitW7eutS4yMpKSkhK/2+7ZsweASy+9tM73w8PDa7w2mUzExsae9Fj33nsv//3vfxk2bBjJyckMHTqUkSNHcsUVV/itT100Gg233nor7777LlarleDgYObNm4fJZPKNzRFC/PEksAghGqW+q4bUBsyQ4PF4AO84loSEhFrv63Q1fyU15AqluLg4Nm7cyDfffMPXX3/N119/zb/+9S/Gjh3LnDlz/G5fl7Fjx/K3v/2NRYsWMXr0aObPn8/VV1+N2Ww+pf0JIU6fBBYhRJOrb86Wdu3aAd6QMWTIkCY7nsFgYPjw4QwfPhyPx8O9997LP/7xD5599lnat2/fqDoCdO/enV69ejFv3jxatWpFdnY2b731VpPVVwjReDKGRQjR5EJCQgAoLS2tsT4zM5Pw8HAmT56M0+mstV1BQUGjj1VUVFTjtUajoUePHgDY7faT1rGsrKze98eMGcO3337L9OnTiY6OZtiwYY2umxCi6UgLixCiyWVkZKDVapk6dSplZWUYjUYuvfRS4uLiePfddxkzZgznnXceN998M7GxsWRnZ/Pll18yePBgZs6c2ahj/elPf6K4uJhLL72UVq1acfDgQd566y0yMjJ8l1LXpXfv3ixYsICJEyfSt29fQkNDGT58uO/9W265hccee4zPP/+ce+65B71ef8rnQwhx+qSFRQjR5BISEpg1axb5+fnceeedjB49mu3btwPeILBs2TKSk5P529/+xgMPPMDHH39MRkYG48ePb/SxbrvtNkwmE++88w733nsvc+bMYdSoUXz99de+afjrcu+993LLLbfwr3/9i1tuuYX777+/xvvx8fEMHToU8La2CCECS+4lJIQQ9bjuuuvYsmULe/fuDXRVhDjnSQuLEELUIScnhy+//FJaV4RoIWQMixBCnODAgQP88ssvfPDBB+j1ev7yl78EukpCCKSFRQghavjhhx8YM2YMBw4cYM6cOXXOFyOE+OPJGBYhhBBCtHjSwiKEEEKIFk8CixBCCCFavLNi0K3H4+Ho0aOEhYWddLptIYQQQrQcqqpSUVFBUlLSSedNgrMksBw9epSUlJRAV0MIIYQQp+DQoUO0atXqpGXOisASFhYGeD/w729PL4QQQoiWqby8nJSUFN/3+MmcFYGluhsoPDxcAosQQghxhmnIcA4ZdCuEEEKIFk8CixBCCCFaPAksQgghhGjxzooxLEIIIVomj8eDw+EIdDVEAOn1erRa7WnvRwKLEEKIZuFwODhw4AAejyfQVREBFhERQUJCwmnNlSaBRQghRJNTVZWcnBy0Wi0pKSl+JwUTZydVVbFareTn5wOQmJh4yvuSwCKEEKLJuVwurFYrSUlJBAcHB7o6IoCCgoIAyM/PJy4u7pS7hyTyCiGEaHJutxsAg8EQ4JqIlqA6tDqdzlPehwQWIYQQzUbu7yagaX4OJLAIIYQQosWTwCKEEEI0wLhx4xgxYkSgq9FsZs+eTURERKCrUS8JLEIIIUQDzJgxg9mzZ/teX3zxxTz44INNfpy7774bRVGYPn36ScutWLECRVEoLS1tkuOOGjWK3bt3N8m+moNcJXQSHo/KmqxiVKBPaiQ6reQ7IYQ4V5nN5mY/xueff86vv/5KUlJSk+3T4XA0aPBzUFCQ74qelki+gU/C4fYw6r1fufm9X7E63YGujhBCiD/Ap59+Snp6OkFBQURHRzNkyBAsFkuNLqFx48bxww8/MGPGDBRFQVEUsrKyANi6dSvDhg0jNDSU+Ph4xowZQ2Fhod/jHjlyhPvvv5958+ah1+tPWjYrK4tLLrkEgMjISBRFYdy4cYC35WfChAk8+OCDxMTEkJmZCcAbb7xBeno6ISEhpKSkcO+991JZWenb5++7hJ5//nkyMjKYO3cuaWlpmM1mbr75ZioqKhp4JpuWBJaT0JwwqllVA1gRIYQ4w6mqitXhCsiiNuIXeE5ODqNHj+aOO+5gx44drFixguuvv77WPmbMmMHAgQO56667yMnJIScnh5SUFEpLS7n00kvp1asXa9euZcmSJeTl5TFy5MiTHtfj8TBmzBgeffRRunXr5reeKSkpfPbZZwDs2rWLnJwcZsyY4Xt/zpw5GAwGfvnlF2bNmgWARqPhzTffZNu2bcyZM4fvv/+exx577KTH2bdvH4sWLWLx4sUsXryYH374gVdffdVv/ZqDdAmdxIlXYTXmB14IIURNVU43XSd9E5Bjb38xk2BDw77ucnJycLlcXH/99aSmpgKQnp5eq5zZbMZgMBAcHExCQoJv/cyZM+nVqxeTJ0/2rfvwww9JSUlh9+7ddOzYsc7jTp06FZ1Ox1//+tcG1VOr1RIVFQVAXFxcrcGyHTp04LXXXqux7sTxNmlpabz88svcfffdvPPOO/Uex+PxMHv2bMLCwgAYM2YMy5Yt45VXXmlQPZuSBJaTkBYWIYQ4t/Ts2ZPLLruM9PR0MjMzGTp0KDfeeCORkZEN2n7Tpk0sX76c0NDQWu/t27ePNWvW8Je//MW37uuvvyY4OJgZM2awfv36eucrGTZsGD/99BMAqampbNu27aT16N27d6113333HVOmTGHnzp2Ul5fjcrmw2WxYrdZ6ZyNOS0vzhRXwTq1fPc3+H00Cy0mc+GPjkcQihBCnLEivZfuLmQE7dkNptVqWLl3KypUr+fbbb3nrrbd4+umnWb16dYO2r6ysZPjw4UydOrXWe4mJiXg8Hvr37+9bl5yczD/+8Q/y8/Np3bq1b73b7ebhhx9m+vTpZGVl8cEHH1BVVQXgd3wLQEhISI3XWVlZXH311dxzzz288sorREVF8fPPP3PnnXficDjqDSy/P5aiKAG7maUElpOo0SUUuGoIIcQZT1GUBnfLBJqiKAwePJjBgwczadIkUlNT+fzzz2uVMxgMvlsQVDvvvPP47LPPSEtLQ6er+/Oe2GIB3m6WIUOG1FiXmZnJmDFjGD9+POANNnUdH6hVh7qsW7cOj8fDtGnTfDei/O9//+t3u5ZEBt2ehCJdQkIIcU5ZvXo1kydPZu3atWRnZ7Nw4UIKCgro0qVLrbJpaWmsXr2arKwsCgsL8Xg83HfffRQXFzN69GjWrFnDvn37+Oabbxg/fny9wSI6Opru3bvXWPR6PQkJCXTq1KneuqampqIoCosXL6agoKDGFT+/1759e5xOJ2+99Rb79+9n7ty5vsG4ZwoJLH5UZxYZdCuEEGe/8PBwfvzxR6688ko6duzIM888w7Rp0xg2bFitso888gharZauXbsSGxtLdnY2SUlJ/PLLL7jdboYOHUp6ejoPPvggERERvpaNppKcnMwLL7zAE088QXx8PBMmTKi3bM+ePXnjjTeYOnUq3bt3Z968eUyZMqVJ69PcFPUs+CYuLy/HbDZTVlZGeHh4k+673VNf4faorH7qMuLDTU26byGEOFvZbDYOHDhAmzZtMJnkd+e5rr6fh8Z8f59S3Hv77bdJS0vDZDLRv39/fvvtt5OW/+STT+jcuTMmk4n09HS++uqrGu+PGzfON/FO9XLFFVecStWaXHWnkAy6FUIIIQKn0YFlwYIFTJw4keeee47169fTs2dPMjMz673MaeXKlYwePZo777yTDRs2MGLECEaMGMHWrVtrlLviiit8k+/k5OTwn//859Q+UROrvrRZ8ooQQggROI0OLG+88QZ33XUX48ePp2vXrsyaNYvg4GA+/PDDOsvPmDGDK664gkcffZQuXbrw0ksvcd555zFz5swa5YxGIwkJCb6lode8N7tjTSzSwiKEEEIETqMCi8PhYN26dTUuv9JoNAwZMoRVq1bVuc2qVavqvFzr9+VXrFhBXFwcnTp14p577qGoqKjeetjtdsrLy2sszaW6S0jyihBCCBE4jQoshYWFuN1u4uPja6yPj48nNze3zm1yc3P9lr/iiiv46KOPWLZsGVOnTuWHH35g2LBh9V4CNmXKFMxms29JSUlpzMdoFE09sw4KIYQQ4o/TImbxufnmm33P09PT6dGjB+3atWPFihVcdtlltco/+eSTTJw40fe6vLy82UKLIl1CQgghRMA1qoUlJiYGrVZLXl5ejfV5eXk1bv50ooSEhEaVB2jbti0xMTHs3bu3zveNRiPh4eE1luYig26FEEKIwGtUYDEYDPTu3Ztly5b51nk8HpYtW8bAgQPr3GbgwIE1ygMsXbq03vIAhw8fpqioiMTExMZUr1nIZc1CCCFE4DX6KqGJEyfy/vvvM2fOHHbs2ME999yDxWLx3e9g7NixPPnkk77yDzzwAEuWLGHatGns3LmT559/nrVr1/pm5KusrOTRRx/l119/JSsri2XLlnHttdfSvn17MjMDc6OsE/lmug1sNYQQQohzWqMDy6hRo3j99deZNGkSGRkZbNy4kSVLlvgG1mZnZ5OTk+MrP2jQIObPn897771Hz549+fTTT1m0aBHdu3cHvHfG3Lx5M9dccw0dO3bkzjvvpHfv3vz0008YjcYm+pinTvF1CUlkEUKIc9m4ceMYMWJEoKvRbFasWIGiKJSWlga6KnWSqfn96PXit5RYnSx96EI6xIf530AIIcRZOTV/WVkZqqoSEREBwMUXX0xGRgbTp08/7X0vXLiQWbNmsW7dOoqLi9mwYQMZGRkn3SYrK4s2bdo0qGxDOBwOiouLiY+Pr3Hz36YQsKn5zyW+FpYA10MIIURgmc1mX1hpahaLhfPPP5+pU6c2+b4dDkeDyhkMBhISEpo8rDQVCSx+yKBbIYQ4t3z66aekp6cTFBREdHQ0Q4YMwWKx1OgSGjduHD/88AMzZszw3QMvKysLgK1btzJs2DBCQ0OJj49nzJgxFBYWnvSYY8aMYdKkSbUmWj2ZNm3aANCrVy8UReHiiy/21W3EiBG88sorJCUl0alTJwDmzp1Lnz59CAsLIyEhgVtuuaXGbXV+3yU0e/ZsIiIi+Oabb+jSpQuhoaG+2+gEggQWPxS5rFkIIU6fqoLDEpilEb/Ac3JyGD16NHfccQc7duxgxYoVXH/99bXGMc6YMYOBAwdy1113+e6Bl5KSQmlpKZdeeim9evVi7dq1LFmyhLy8PEaOHNnUZ9R34+HvvvuOnJwcFi5c6Htv2bJl7Nq1i6VLl7J48WIAnE4nL730Eps2bWLRokVkZWUxbty4kx7DarXy+uuvM3fuXH788Ueys7N55JFHmvyzNESLmDiuJZOJ44QQogk4rTA5KTDHfuooGEIaVDQnJweXy8X1119Pamoq4J3Q9PfMZjMGg4Hg4OAa84rNnDmTXr16MXnyZN+6Dz/8kJSUFHbv3k3Hjh1P88McFxsbC0B0dHStuc1CQkL44IMPMBgMvnV33HGH73nbtm1588036du3L5WVlYSGhtZ5DKfTyaxZs2jXrh0AEyZM4MUXX2yyz9AY0sLih6b6smbJK0IIcdbr2bMnl112Genp6dx00028//77lJSUNHj7TZs2sXz5ckJDQ31L586dAdi3bx/z5s2r8d5PP/3UoP3efffdNbbzJz09vUZYAVi3bh3Dhw+ndevWhIWFcdFFFwHeq3vrExwc7AsrAImJiTW6kf5I0sLih4J0CQkhxGnTB3tbOgJ17AbSarUsXbqUlStX8u233/LWW2/x9NNPs3r16gZtX1lZyfDhw+scPJuYmIjH46F///6+dcnJyQ3a74svvtiorpiQkJotShaLhczMTDIzM5k3bx6xsbFkZ2eTmZl50kG5er2+xmtFUQI2zYcEFj98LSxynZAQQpw6RWlwt0ygKYrC4MGDGTx4MJMmTSI1NZXPP/+8VjmDwVDrJr3nnXcen332GWlpaeh0dX/FhoU1foqMuLg44uLiah0fqPdGwSfauXMnRUVFvPrqq757761du7bR9Qgk6RLyo3rQrUfyihBCnPVWr17N5MmTWbt2LdnZ2SxcuJCCggK6dOlSq2xaWhqrV68mKyuLwsJCPB4P9913H8XFxYwePZo1a9awb98+vvnmG8aPH3/SYFFcXMzGjRvZvn07ALt27WLjxo3k5ubWu01cXBxBQUG+gb1lZWX1lm3dujUGg4G33nqL/fv388UXX/DSSy814swEngSWBjoL5tcTQgjhR3h4OD/++CNXXnklHTt25JlnnmHatGkMGzasVtlHHnkErVZL165dfV0sSUlJ/PLLL7jdboYOHUp6ejoPPvggERERaDT1f+V+8cUX9OrVi6uuugqAm2++mV69ejFr1qx6t9HpdLz55pv84x//ICkpiWuvvbbesrGxscyePZtPPvmErl278uqrr/L666834swEnsx068cFr33PoeIqFt47iPNaRzbpvoUQ4mx1Ns50K06dzHT7Bzg+6PaMz3VCCCHEGUsCix9yWbMQQggReBJY/JBBt0IIIUTgSWDxQ/G1sEhiEUIIIQJFAosfx29+GNBqCCGEEOc0CSx++G5+KBPHCSGEEAEjgcWP6kG3kleEEEKIwJHA4kf1Zc3SJSSEEEIEjgQWPxS5l5AQQggRcBJY/JDLmoUQQgCMGzeOESNGBLoazWb27NlEREQEuhr1ksDih0YuaxZCCAHMmDGD2bNn+15ffPHFPPjgg6e9X6fTyeOPP056ejohISEkJSUxduxYjh49etLtVqxYgaIolJaWnnYdAEaNGsXu3bubZF/NQQKLH4rMdCuEEAIwm83N0gJhtVpZv349zz77LOvXr2fhwoXs2rWLa665pkn273A4GlQuKCiIuLi4Jjlmc5DA4ofvXkIyhkUIIc4Jn376Kenp6QQFBREdHc2QIUOwWCw1uoTGjRvHDz/8wIwZM1AUBUVRyMrKAmDr1q0MGzaM0NBQ4uPjGTNmDIWFhfUez2w2s3TpUkaOHEmnTp0YMGAAM2fOZN26dWRnZ9e5TVZWFpdccgkAkZGRKIrCuHHjAG/Lz4QJE3jwwQeJiYkhMzMTgDfeeMPXipOSksK9995LZWWlb5+/7xJ6/vnnycjIYO7cuaSlpWE2m7n55pupqKg4xTN7eiSw+CH3EhJCiNOnqipWpzUgS2O69HNychg9ejR33HEHO3bsYMWKFVx//fW19jFjxgwGDhzIXXfdRU5ODjk5OaSkpFBaWsqll15Kr169WLt2LUuWLCEvL4+RI0c26nyVlZWhKEq9LTopKSl89tlnAOzatYucnBxmzJjhe3/OnDkYDAZ++eUXZs2aBYBGo+HNN99k27ZtzJkzh++//57HHnvspPXYt28fixYtYvHixSxevJgffviBV199tVGfpanoAnLUM4kMuhVCiNNW5aqi//z+ATn26ltWE6wPblDZnJwcXC4X119/PampqQCkp6fXKmc2mzEYDAQHB5OQkOBbP3PmTHr16sXkyZN96z788ENSUlLYvXs3HTt29FsHm83G448/zujRowkPD6+zjFarJSoqCoC4uLhawaZDhw689tprNdadON4mLS2Nl19+mbvvvpt33nmn3rp4PB5mz55NWFgYAGPGjGHZsmW88sorfj9HU5MWFj9k0K0QQpw7evbsyWWXXUZ6ejo33XQT77//PiUlJQ3eftOmTSxfvpzQ0FDf0rlzZ8DbWjFv3rwa7/300081tnc6nYwcORJVVXn33Xd966u7mEJDQ+nWrZvfevTu3bvWuu+++47LLruM5ORkwsLCGDNmDEVFRVit1nr3k5aW5gsrAImJieTn5/s9fnOQFhY/5F5CQghx+oJ0Qay+ZXXAjt1QWq2WpUuXsnLlSr799lveeustnn76aVavbljdKysrGT58OFOnTq31XmJiIh6Ph/79j7c0JScn+55Xh5WDBw/y/fff12hd+eCDD6iqqgJAr9f7rUdISEiN11lZWVx99dXcc889vPLKK0RFRfHzzz9z55134nA4CA6uuwXq98dSFAWPx+P3+M1BAosfGkXm5hdCiNOlKEqDu2UCTVEUBg8ezODBg5k0aRKpqal8/vnntcoZDAbcbneNdeeddx6fffYZaWlp6HR1f8We2GJRrTqs7Nmzh+XLlxMdHV3j/RODzYnHB2rVoS7r1q3D4/Ewbdo0NBpv58p///tfv9u1JNIl5Ed1XpEWFiGEOPutXr2ayZMns3btWrKzs1m4cCEFBQV06dKlVtm0tDRWr15NVlYWhYWFeDwe7rvvPoqLixk9ejRr1qxh3759fPPNN4wfP77eYOF0OrnxxhtZu3Yt8+bNw+12k5ubS25u7kkvSU5NTUVRFBYvXkxBQUGNK35+r3379jidTt566y3279/P3LlzfYNxzxQSWPzw3a1ZAosQQpz1wsPD+fHHH7nyyivp2LEjzzzzDNOmTWPYsGG1yj7yyCNotVq6du1KbGws2dnZJCUl8csvv+B2uxk6dCjp6ek8+OCDRERE+Fo2fu/IkSN88cUXHD58mIyMDBITE33LypUr661rcnIyL7zwAk888QTx8fFMmDCh3rI9e/bkjTfeYOrUqXTv3p158+YxZcqUxp+gAFLUs2A0aXl5OWazmbKysnpHVJ+qUf9YxeoDxcy8pRdX90hq0n0LIcTZymazceDAAdq0aYPJZAp0dUSA1ffz0Jjvb2lh8UO6hIQQQojAk8Dih8bXJSSJRQghhAgUCSx+yL2EhBBCiMCTwOKHr4VFLmsWQgghAkYCSwMFaJ4cIYQQQiCBxa/jLSxCCCGECBQJLH4oci8hIYQQIuAksPjhm5hf8ooQQggRMBJY/JBBt0IIIUTgSWDxQyaOE0IIATBu3DhGjBgR6Go0m9mzZxMRERHoatRLAosfci8hIYQQADNmzGD27Nm+1xdffDEPPvhgk+z7+eefp3PnzoSEhBAZGcmQIUNYvXr1SbdZsWIFiqJQWlraJHUYNWoUu3fvbpJ9NQcJLH5Uj2HxSGIRQohzmtlsbrYWiI4dOzJz5ky2bNnCzz//TFpaGkOHDqWgoOC0932yOz6fKCgoiLi4uNM+XnORwOKHXNYshBCnT1VVPFZrQJbGXuX56aefkp6eTlBQENHR0QwZMgSLxVKjS2jcuHH88MMPzJgxA0VRUBSFrKwsALZu3cqwYcMIDQ0lPj6eMWPGUFhYeNJj3nLLLQwZMoS2bdvSrVs33njjDcrLy9m8eXOd5bOysrjkkksAiIyMRFEUxo0bB3hbfiZMmMCDDz5ITEwMmZmZALzxxhukp6cTEhJCSkoK9957L5WVlb59/r5L6PnnnycjI4O5c+eSlpaG2Wzm5ptvpqKiolHns6noAnLUM4gilwkJIcRpU6uq2HVe74Acu9P6dSjBwQ0qm5OTw+jRo3nttde47rrrqKio4KeffqoVembMmMHu3bvp3r07L774IgCxsbGUlpZy6aWX8qc//Ym///3vVFVV8fjjjzNy5Ei+//77BtXB4XDw3nvvYTab6dmzZ51lUlJS+Oyzz7jhhhvYtWsX4eHhBAUF+d6fM2cO99xzD7/88otvnUaj4c0336RNmzbs37+fe++9l8cee4x33nmn3rrs27ePRYsWsXjxYkpKShg5ciSvvvoqr7zySoM+S1OSwOKHDLoVQohzR05ODi6Xi+uvv57U1FQA0tPTa5Uzm80YDAaCg4NJSEjwrZ85cya9evVi8uTJvnUffvghKSkp7N69m44dO9Z77MWLF3PzzTdjtVpJTExk6dKlxMTE1FlWq9USFRUFQFxcXK2uqg4dOvDaa6/VWHfieJu0tDRefvll7r777pMGFo/Hw+zZswkLCwNgzJgxLFu2TAJLS6TI3ZqFEOK0KUFBdFq/LmDHbqiePXty2WWXkZ6eTmZmJkOHDuXGG28kMjKyQdtv2rSJ5cuXExoaWuu9ffv2sWbNGv7yl7/41n399ddccMEFAFxyySVs3LiRwsJC3n//fUaOHMnq1auJi4tj2LBh/PTTTwCkpqaybdu2k9ajd+/arVnfffcdU6ZMYefOnZSXl+NyubDZbFitVoLraYFKS0vzhRWAxMRE8vPz/Z+IZiCBxY/jg24DWg0hhDijKYrS4G6ZQNJqtSxdupSVK1fy7bff8tZbb/H000/7vWKnWmVlJcOHD2fq1Km13ktMTMTj8dC/f3/fuuTkZN/zkJAQ2rdvT/v27RkwYAAdOnTgn//8J08++SQffPABVVVVAOj1er/1CAkJqfE6KyuLq6++mnvuuYdXXnmFqKgofv75Z+68804cDke9geX3x1IUBU+Abq4ngcUPGXQrhBDnFkVRGDx4MIMHD2bSpEmkpqby+eef1ypnMBhwu9011p133nl89tlnpKWlodPV/RV7YovFyXg8Hux2O1Az2Jx4fKBWHeqybt06PB4P06ZNQ6PxXm/z3//+t0H1aCnkKiE/5F5CQghx7li9ejWTJ09m7dq1ZGdns3DhQgoKCujSpUutsmlpaaxevZqsrCwKCwvxeDzcd999FBcXM3r0aNasWcO+ffv45ptvGD9+fL3BwmKx8NRTT/Hrr79y8OBB1q1bxx133MGRI0e46aab6q1ramoqiqKwePFiCgoKalzx83vt27fH6XTy1ltvsX//fubOncusWbMaf4ICSAKLHxqZOE4IIc4Z4eHh/Pjjj1x55ZV07NiRZ555hmnTpjFs2LBaZR955BG0Wi1du3YlNjaW7OxskpKS+OWXX3C73QwdOpT09HQefPBBIiIifC0bv6fVatm5cyc33HADHTt2ZPjw4RQVFfHTTz/RrVu3euuanJzMCy+8wBNPPEF8fDwTJkyot2zPnj154403mDp1Kt27d2fevHlMmTKl8ScogBT1LGg6KC8vx2w2U1ZWRnh4eJPue+KCjSzccIQnh3XmLxe1a9J9CyHE2cpms3HgwAHatGmDyWQKdHVEgNX389CY729pYfGnuksosLUQQgghzmmnFFjefvtt0tLSMJlM9O/fn99+++2k5T/55BM6d+6MyWQiPT2dr776qt6yd999N4qiMH369FOpWpOTLiEhhBAi8BodWBYsWMDEiRN57rnnWL9+PT179iQzM7Pe67JXrlzJ6NGjufPOO9mwYQMjRoxgxIgRbN26tVbZzz//nF9//ZWkpKTGf5JmIvcSEkIIIQKv0YHljTfe4K677mL8+PF07dqVWbNmERwczIcfflhn+RkzZnDFFVfw6KOP0qVLF1566SXOO+88Zs6cWaPckSNHuP/++5k3b16DrjH/o2h8c/MLIYQQIlAaFVgcDgfr1q1jyJAhx3eg0TBkyBBWrVpV5zarVq2qUR4gMzOzRnmPx8OYMWN49NFHTzoiOhB8U/PLzHFCCCFEwDRq4rjCwkLcbjfx8fE11sfHx7Nz5846t8nNza2zfG5uru/11KlT0el0/PWvf21QPex2u28yHfCOMm4uikwcJ4QQQgRcwK8SWrduHTNmzGD27Nm+cODPlClTMJvNviUlJaXZ6nf85ocSWYQQQohAaVRgiYmJQavVkpeXV2N9Xl5ejbtVnighIeGk5X/66Sfy8/Np3bo1Op0OnU7HwYMHefjhh0lLS6tzn08++SRlZWW+5dChQ435GI1SHaEkrwghhBCB06jAYjAY6N27N8uWLfOt83g8LFu2jIEDB9a5zcCBA2uUB1i6dKmv/JgxY9i8eTMbN270LUlJSTz66KN88803de7TaDQSHh5eY2kuci8hIYQQIvAa3SU0ceJE3n//febMmcOOHTu45557sFgsjB8/HoCxY8fy5JNP+so/8MADLFmyhGnTprFz506ef/551q5d65tCODo6mu7du9dY9Ho9CQkJdOrUqYk+5qmTewkJIYSoi6IoLFq0qMHlx40bx4gRI07rmFlZWSiKwsaNG09rP43x/PPPk5GR8Ycdrz6NDiyjRo3i9ddfZ9KkSWRkZLBx40aWLFniG1ibnZ1NTk6Or/ygQYOYP38+7733Hj179uTTTz9l0aJFdO/evek+RTOSieOEEOLck5ubywMPPED79u0xmUzEx8czePBg3n33XaxWa6Crd1KzZ88mIiKiyfb3yCOP1OopCYRGXSVUbcKECfXeZGnFihW11t10000nvePk72VlZZ1KtZqc6vEQfXA3nYuP4nGnBbo6Qggh/gD79+9n8ODBREREMHnyZNLT0zEajWzZsoX33nuP5ORkrrnmmkBX87Q5HA4MBoPfcqGhoYSGhv4BNTq5gF8l1JKpLhdD3n6Kv/84E63D7n8DIYQQdVJVFafdHZClsV369957LzqdjrVr1zJy5Ei6dOlC27Ztufbaa/nyyy8ZPnx4ndtt2bKFSy+9lKCgIKKjo/nzn/9MZWVlrXIvvPACsbGxhIeHc/fdd+NwOHzvLVmyhPPPP5+IiAiio6O5+uqr2bdvX4PrvmLFCsaPH09ZWRmKoqAoCs8//zwAaWlpvPTSS4wdO5bw8HD+/Oc/A/D444/TsWNHgoODadu2Lc8++yxOp9O3z993CVV3bb3++uskJiYSHR3NfffdV2Ob5nBKLSznihMvs1Y9ngDWRAghzmwuh4f3HvghIMf+84yL0Bu1DSpbVFTEt99+y+TJkwkJCamzTF1TcFgsFjIzMxk4cCBr1qwhPz+fP/3pT0yYMIHZs2f7yi1btgyTycSKFSvIyspi/PjxREdH88orr/j2M3HiRHr06EFlZSWTJk3iuuuuY+PGjWg0/tsYBg0axPTp05k0aRK7du0CqNE6Uj2k47nnnvOtCwsLY/bs2SQlJbFlyxbuuusuwsLCeOyxx+o9zvLly0lMTGT58uXs3buXUaNGkZGRwV133eW3jqdKAsvJnPjD4ZbAIoQQZ7u9e/eiqmqtiz5iYmKw2WwA3HfffUydOrXG+/Pnz8dms/HRRx/5gs7MmTMZPnw4U6dO9Y3zNBgMfPjhhwQHB9OtWzdefPFFHn30UV566SU0Gg033HBDjf1++OGHxMbGsn379gaN/TQYDJjNZhRFqXO6kUsvvZSHH364xrpnnnnG9zwtLY1HHnmEjz/++KSBJTIykpkzZ6LVauncuTNXXXUVy5Ytk8ASMCcGFhl1K4QQp0xn0PDnGRcF7Nin67fffsPj8XDrrbfWmGm92o4dO+jZs2eNVpnBgwfj8XjYtWuXL7D07NmT4OBgX5mBAwdSWVnJoUOHSE1NZc+ePUyaNInVq1dTWFiI51jrfnZ2dp2BpVu3bhw8eBCACy64gK+//vqkn6NPnz611i1YsIA333yTffv2UVlZicvl8jtdSLdu3dBqj7daJSYmsmXLlpNuc7oksJxEjS4haWERQohTpihKg7tlAql9+/YoiuLrTqnWtm1bAIKCgpr1+MOHDyc1NZX333+fpKQkPB4P3bt3rzHO5URfffWVb+xIQ+r2+26uVatWceutt/LCCy+QmZmJ2Wzm448/Ztq0aSfdz+9vUqwoii9cNRcJLH6oigZF9aCqEliEEOJsFx0dzeWXX87MmTO5//776x3H8ntdunRh9uzZWCwW3za//PILGo2mRvfSpk2bqKqq8oWLX3/9ldDQUFJSUigqKmLXrl28//77XHDBBQD8/PPPJz1uampqrXUGgwG3292geq9cuZLU1FSefvpp37rqFpuWRq4S8kOtnodFBt0KIcQ54Z133sHlctGnTx8WLFjAjh072LVrF//+97/ZuXNnja6Qarfeeismk4nbb7+drVu3snz5cu6//37GjBlT4wbADoeDO++8k+3bt/PVV1/x3HPPMWHCBDQaDZGRkURHR/Pee++xd+9evv/+eyZOnNjo+qelpVFZWcmyZcsoLCw86bwxHTp0IDs7m48//ph9+/bx5ptv8vnnnzf6mH8ECSx+qMfvfhjYigghhPhDtGvXjg0bNjBkyBCefPJJevbsSZ8+fXjrrbd45JFHeOmll2ptExwczDfffENxcTF9+/blxhtv5LLLLmPmzJk1yl122WV06NCBCy+8kFGjRnHNNdf4LjvWaDR8/PHHrFu3ju7du/PQQw/xt7/9rdH1HzRoEHfffTejRo0iNjaW1157rd6y11xzDQ899BATJkwgIyODlStX8uyzzzb6mH8ERT0L5pwvLy/HbDZTVlbW5PcV2preE63TwRdPvMPj4y5p0n0LIcTZymazceDAAdq0aYPJZAp0dUSA1ffz0Jjvb2lh8UM9fjOhwFZECCGEOIdJYPHHN4ZFAosQQggRKBJY/FAV7ymSQbdCCCFE4Ehg8cfXJSSBRQghhAgUCSx+qNWz3UoLixBCNNpZcF2HaAJN8XMggcUfGXQrhBCNVj1XSX0ztIpzS/VcML+fIbcxZKbbk1A9KpbgROyEy9T8QgjRCDqdjuDgYAoKCtDr9Q2607A4+6iqitVqJT8/n4iIiDon3WsoCSwn4XGrbOj6V+8LV2VgKyOEEGcQRVFITEzkwIEDLXaqd/HHiYiIqPPu0Y0hgeVkTvyDQLqEhBCiUQwGAx06dJBuoXOcXq8/rZaVahJYTuLEuzXL1PxCCNF4Go1GZroVTUI6FU/ixLyiyBgWIYQQImAksJyEoii+riCZOE4IIYQIHAksfh3rCpIxLEIIIUTASGDxQ/EFlsDWQwghhDiXSWDxp7plRbqEhBBCiICRwOJHdQuLIl1CQgghRMBIYPFLWliEEEKIQJPA4oevZUXmYRFCCCECRgKLX8cua5YuISGEECJgJLD44btKSFpYhBBCiICRwOJXdWAJbC2EEEKIc5kEFj+qZ+dXVEksQgghRKBIYPGnOqjIGBYhhBAiYCSw+OFrYZExLEIIIUTASGA5GbcTRXUCoPU4A1wZIYQQ4twlgeVkVBWN6gZAOfYohBBCiD+eBJaTUTTHp+aXmW6FEEKIgJHAcjIaLfgCS2CrIoQQQpzLJLCcjKL4LmdWZCIWIYQQImAksPhxvEtIrhISQgghAkUCi1/SwiKEEEIEmgQWP6SFRQghhAg8CSx++G5+KDPdCiGEEAEjgcWfY0FFI/cSEkIIIQJGAosf1WNXFGlgEUIIIQJGAosfx7uEpIVFCCGECBQJLH6pNR6EEEII8ceTwOJHdQuLRgbdCiGEEAEjgcWvY5c1SxOLEEIIETASWPzwTRgn87AIIYQQASOBxQ9FWliEEEKIgJPA4ocvsEheEUIIIQJGAotfklSEEEKIQJPA4odS/ShXCQkhhBABI4HFD9+gW8krQgghRMCcUmB5++23SUtLw2Qy0b9/f3777beTlv/kk0/o3LkzJpOJ9PR0vvrqqxrvP//883Tu3JmQkBAiIyMZMmQIq1evPpWqNTmZh0UIIYQIvEYHlgULFjBx4kSee+451q9fT8+ePcnMzCQ/P7/O8itXrmT06NHceeedbNiwgREjRjBixAi2bt3qK9OxY0dmzpzJli1b+Pnnn0lLS2Po0KEUFBSc+idrKjLaVgghhAg4RVUb13TQv39/+vbty8yZMwHweDykpKRw//3388QTT9QqP2rUKCwWC4sXL/atGzBgABkZGcyaNavOY5SXl2M2m/nuu++47LLL/NapunxZWRnh4eGN+Th+fTZ+OrnGHgRZf+GOj55t0n0LIYQQ57LGfH83qoXF4XCwbt06hgwZcnwHGg1Dhgxh1apVdW6zatWqGuUBMjMz6y3vcDh47733MJvN9OzZs84ydrud8vLyGkvzkXsJCSGEEIHWqMBSWFiI2+0mPj6+xvr4+Hhyc3Pr3CY3N7dB5RcvXkxoaCgmk4m///3vLF26lJiYmDr3OWXKFMxms29JSUlpzMdoFMX3RDlZMSGEEEI0oxZzldAll1zCxo0bWblyJVdccQUjR46sd1zMk08+SVlZmW85dOhQs9TJ5XFRobi9L1RPsxxDCCGEEP41KrDExMSg1WrJy8ursT4vL4+EhIQ6t0lISGhQ+ZCQENq3b8+AAQP45z//iU6n45///Ged+zQajYSHh9dYmoNH9VCqqQ4q0ickhBBCBEqjAovBYKB3794sW7bMt87j8bBs2TIGDhxY5zYDBw6sUR5g6dKl9ZY/cb92u70x1WtyGkUDMjW/EEIIEXC6xm4wceJEbr/9dvr06UO/fv2YPn06FouF8ePHAzB27FiSk5OZMmUKAA888AAXXXQR06ZN46qrruLjjz9m7dq1vPfeewBYLBZeeeUVrrnmGhITEyksLOTtt9/myJEj3HTTTU34URtPq2hRjwUWyStCCCFE4DQ6sIwaNYqCggImTZpEbm4uGRkZLFmyxDewNjs7G43meMPNoEGDmD9/Ps888wxPPfUUHTp0YNGiRXTv3h0ArVbLzp07mTNnDoWFhURHR9O3b19++uknunXr1kQf89QoioKvhSWgNRFCCCHObY2eh6Ulas55WKb/aTJ63QCCK39i/L+fa9J9CyGEEOeyZpuH5VykKt5BtzKGRQghhAgcCSx+yRgWIYQQItAksPglY1iEEEKIQJPA4o/0BQkhhBABJ4HFr+rAIm0sQgghRKBIYPFDVWTiOCGEECLQJLD4oeC9SkiVBhYhhBAiYCSw+HMsqCiSWIQQQoiAkcDiV/XNDyWwCCGEEIEigcUPGboihBBCBJ4EFn9ktK0QQggRcBJY/JIuISGEECLQJLD4oUgLixBCCBFwElgaTFpYhBBCiECRwOKHqshMt0IIIUSgSWDxQ5HAIoQQQgScBJYGk8AihBBCBIoEFn8Uj/8yQgghhGhWElj8UWo9EUIIIcQfTAKLHzKGRQghhAg8CSwNpqCqMieLEEIIEQgSWPw4sYVF8ooQQggRGBJY/FBPGMPikcQihBBCBIQEFj98LSyKInduFkIIIQJEAosfirSwCCGEEAEngcWPE7uEJK8IIYQQgSGBxQ9F5mERQgghAk4Cix+K5vhVQtIlJIQQQgSGBBZ/pEtICCGECDgJLH7IoFshhBAi8CSw+FEdWFRFI5c1CyGEEAEigcUPxXeGFFS5cbMQQggREBJY/PH1CSmo0sYihBBCBIQEFj80J7awSF4RQgghAkICix8n3vxQBt0KIYQQgSGBxZ/qJha5l5AQQggRMBJY/NBUXyUkLSxCCCFEwEhg8UOpTixokCYWIYQQIjAksPhzwqBbjwQWIYQQIiAksPhR3cKiKnJZsxBCCBEoElj80Jw4D4vkFSGEECIgJLD4cXymW40MuhVCCCECRAKLH8qxy5pVRVpYhBBCiECRwOLH8auEJLAIIYQQgSKBxQ+NRu4lJIQQQgSaBBY/NCdcJSSXNQshhBCBIYHFD0V7fOI4VfqEhBBCiICQwOKH4rtds7SwCCGEEIEigcUPzQlXCcnc/EIIIURgSGDxQ3NCC4v0CAkhhBCBIYHFD4322CmSQbdCCCFEwEhg8UM5FlhUNHJZsxBCCBEgElj8OHEMi8cT4MoIIYQQ5ygJLH5oNNpjz2TiOCGEECJQTimwvP3226SlpWEymejfvz+//fbbSct/8skndO7cGZPJRHp6Ol999ZXvPafTyeOPP056ejohISEkJSUxduxYjh49eipVa3K+MSxoZNCtEEIIESCNDiwLFixg4sSJPPfcc6xfv56ePXuSmZlJfn5+neVXrlzJ6NGjufPOO9mwYQMjRoxgxIgRbN26FQCr1cr69et59tlnWb9+PQsXLmTXrl1cc801p/fJmohG521hkZsfCiGEEIGjqI2cvrV///707duXmTNnAuDxeEhJSeH+++/niSeeqFV+1KhRWCwWFi9e7Fs3YMAAMjIymDVrVp3HWLNmDf369ePgwYO0bt3ab53Ky8sxm82UlZURHh7emI/j1zeLprN3SQ+0LhuDnh9Ej1YRTbp/IYQQ4lzVmO/vRrWwOBwO1q1bx5AhQ47vQKNhyJAhrFq1qs5tVq1aVaM8QGZmZr3lAcrKylAUhYiIiDrft9vtlJeX11iai0anA0BVNHjkumYhhBAiIBoVWAoLC3G73cTHx9dYHx8fT25ubp3b5ObmNqq8zWbj8ccfZ/To0fWmrSlTpmA2m31LSkpKYz5Go2i0xwbdKgqq291sxxFCCCFE/VrUVUJOp5ORI0eiqirvvvtuveWefPJJysrKfMuhQ4earU7a6suaUaSFRQghhAgQXWMKx8TEoNVqycvLq7E+Ly+PhISEOrdJSEhoUPnqsHLw4EG+//77k/ZlGY1GjEZjY6p+yhTd8RYWPNLCIoQQQgRCo1pYDAYDvXv3ZtmyZb51Ho+HZcuWMXDgwDq3GThwYI3yAEuXLq1Rvjqs7Nmzh++++47o6OjGVKtZ6bR6QFpYhBBCiEBqVAsLwMSJE7n99tvp06cP/fr1Y/r06VgsFsaPHw/A2LFjSU5OZsqUKQA88MADXHTRRUybNo2rrrqKjz/+mLVr1/Lee+8B3rBy4403sn79ehYvXozb7faNb4mKisJgMDTVZz0lGn11C4sGj4xhEUIIIQKi0YFl1KhRFBQUMGnSJHJzc8nIyGDJkiW+gbXZ2dkn3OEYBg0axPz583nmmWd46qmn6NChA4sWLaJ79+4AHDlyhC+++AKAjIyMGsdavnw5F1988Sl+tKahrR50C6gumZtfCCGECIRGz8PSEjXnPCzr1nzCr//0dlH1+ktbBvVKa9L9CyGEEOeqZpuH5Vyk1R9vhFLl7odCCCFEQEhg8UNzbNAtIPOwCCGEEAEigcUPre54YPG4pYVFCCGECAQJLH5odce7hDwy6FYIIYQICAksfuhOuEoIGcMihBBCBIQEFj802hNaWCSwCCGEEAEhgcUP3QmBRXWd8VeACyGEEGckCSx+aJQT5tZT5SohIYQQIhAksPih0+pB9XYFeaSFRQghhAgICSx+aDQ6lGOTAcsYFiGEECIwJLD4odHoAG9QkYnjhBBCiMCQwOKHVqsHjrWwuF2BrYwQQghxjpLA4seJXUK4HYGtjBBCCHGOksDih1ajw9fCIjPdCiGEEAEhgcUPbwuLN6i43c4A10YIIYQ4N0lg8UOrOXEMiwQWIYQQIhAksPihOSGwqB65SkgIIYQIBAksfmg1Oqieh8UlVwkJIYQQgSCBxQ+NRofia2GRwCKEEEIEggQWfzRa39T8yEy3QgghREBIYPFHUTh+WbMMuhVCCCECQQJLA1R3CUkLixBCCBEYElga5NjdmiWwCCGEEAEhgaUhVLmsWQghhAgkCSwNciywyN2ahRBCiICQwNIg3q4gVbqEhBBCiICQwNIAinQJCSGEEAElgaVB5CohIYQQIpAksDTIsRYWVQKLEEIIEQgSWBpEWliEEEKIQJLA0iDVg27VANdDCCGEODdJYGmA4zc/lEG3QgghRCBIYGmQ6i4haWERQgghAkECS0McG2yrqhJYhBBCiECQwNIgx4KKBBYhhBAiICSwNIhcJSSEEEIEkgSWBqm+W7O0sAghhBCBIIGlQdTfPQohhBDijySBpUHkKiEhhBAikCSwNIAig26FEEKIgJLA0iDV9xIKcDWEEEKIc5QElgZRazwIIYQQ4o8lgaVBjl3OLE0sQgghREBIYGkQGcMihBBCBJIElgaRLiEhhBAikCSwNIAig26FEEKIgJLA0gCqTBwnhBBCBJQElgZQqgfdepTAVkQIIYQ4R0lgaRBpWRFCCCECSQJLY8ggFiGEECIgJLA0iCfQFRBCCCHOaRJYGkBR5LJmIYQQIpAksDSCqsqgWyGEECIQJLA0iLdLSJEWFiGEECIgTimwvP3226SlpWEymejfvz+//fbbSct/8skndO7cGZPJRHp6Ol999VWN9xcuXMjQoUOJjo5GURQ2btx4KtVqRscmjkNaWIQQQohAaHRgWbBgARMnTuS5555j/fr19OzZk8zMTPLz8+ssv3LlSkaPHs2dd97Jhg0bGDFiBCNGjGDr1q2+MhaLhfPPP5+pU6ee+idpTseaVqSFRQghhAgMRVUbd61u//796du3LzNnzgTA4/GQkpLC/fffzxNPPFGr/KhRo7BYLCxevNi3bsCAAWRkZDBr1qwaZbOysmjTpg0bNmwgIyOjwXUqLy/HbDZTVlZGeHh4Yz5Og/xr3ItYTeejd6/gz++/2OT7F0IIIc5Fjfn+blQLi8PhYN26dQwZMuT4DjQahgwZwqpVq+rcZtWqVTXKA2RmZtZbviHsdjvl5eU1luakVPcESQuLEEIIERCNCiyFhYW43W7i4+NrrI+Pjyc3N7fObXJzcxtVviGmTJmC2Wz2LSkpKae8r4apTioyhkUIIYQIhDPyKqEnn3ySsrIy33Lo0KFmPmL1PCwSWIQQQohA0DWmcExMDFqtlry8vBrr8/LySEhIqHObhISERpVvCKPRiNFoPOXtG0uR0bZCCCFEQDWqhcVgMNC7d2+WLVvmW+fxeFi2bBkDBw6sc5uBAwfWKA+wdOnSesu3TNIlJIQQQgRSo1pYACZOnMjtt99Onz596NevH9OnT8disTB+/HgAxo4dS3JyMlOmTAHggQce4KKLLmLatGlcddVVfPzxx6xdu5b33nvPt8/i4mKys7M5evQoALt27QK8rTOn0xLTdKRLSAghhAikRgeWUaNGUVBQwKRJk8jNzSUjI4MlS5b4BtZmZ2ej0RxvuBk0aBDz58/nmWee4amnnqJDhw4sWrSI7t27+8p88cUXvsADcPPNNwPw3HPP8fzzz5/qZ2s6SvWDBBYhhBAiEBo9D0tL1NzzsHz0p2eo0F2K3vELf/7w2SbfvxBCCHEuarZ5WM5d0iUkhBBCBJIElgZQfI8SWIQQQohAkMDSEEqtJ0IIIYT4A0lgaZBjd2s+40f7CCGEEGemRl8ldC5SfFcJSb4TQpzZVFUlr9zO7rwKcsttFFTYfUt+hQ2PCpHBBlKigmgfF0qHuDA6xIUSGWIIdNXFOU4CS4Mca2GRLiEhxBmkrMrJ7rwKduWesORVUFblbPS+okMMdEkMp1+bKPq3iSKjdQRGnbYZai1E3SSwNIQ0rAghWjC7y82+fAu78srZeSyY7M6t4GiZrc7yWo1CWnQwrSKDiQ0zEhdmPPZoQqNAkcVBVqGFvQWV7Mmr5EhpFUUWBz/vLeTnvYUAGHUahnSNZ9ygNPqkRqIo8gedaF4SWBpAOdbCIlcJCSECyeNROVRi9YWSXcdaTw4UWnB76h5kl2Q20TEhjE4JYXROCKNjfBjtYkMx6RveOmKxu9hXUMmmQ6X8eqCY1fuLKay08+XmHL7cnEPbmBCuPy+ZEb2SaRUZ3FQfV4gaJLA0gKIAqnQJCSGan8ejUlBp53CJlcMlVRwprSK7yBtSdudVYHW469wu3KSjc0I4nY6Fk07Hwok5SH/adQox6ujRKoIerSIYMzANVVXZdrScf/96kP9tPMr+Qguvf7ub17/dzUUdY3k0sxPdk82nfVwhTiSBpUGUY/+VviEhRNOptLvYfrScrUfKvMvRMrIKrTjcnnq3Meg0tI8NpXNCGJ0TvaGkc0I48eHGP6xbRlEUuiebefWGHjxzdVeWbM1l4frDrNpfxA+7C/hhdwHX9Urm4aEdpcVFNBkJLA2gKKq0sAghTltumY1vtuWy7mAJW4+UcaDIUud0CVqNQkK4ieTIIFpFBNEqMoiOx7p00qJD0Glbzh9PoUYdN/ZuxY29W3GwyMIbS3fzv41H+XzDEb7cnMNdF7ZhwiUdCDLIAF1xeiSwNET1Zc0yNb8QooFsTjc7cyvYcriULUfK2Hy4jJ25FbXKJZpNdE820z3JTPfkcDrGh5FoNrWoUNJQqdEhzLi5F386vy2Tv9rBqv1FvL18H4s2HOX5a7pxedf4QFdRnMEksDRAdSurtLAIIericnvYmVvB5sNlbDlSyubDZezKrcBVx0DY3qmRXNIplvRWEXRLCicm1BiAGjev9FZm5t/Vn2+25fHi/23jSGkVd320liFd4pl6QzrRZ+FnFs1PAktD+CaOk8AihPBOvrY3v5If9xSycm8hqw8UU2l31SoXFWKgRyszPZLNdE82k9E6grgwUwBq/MdTFIUruidwYccY3vp+Lx/8tJ/vduRx1ZtlzLylF33SogJdRXGGkcDSAMcHsklgEeJcVVRpZ01WCVuOlPLttjz25FfWeD/cpKNnSgTpyWZ6tDKT3iqCJLPp9AfCul3gtILq9t4fRFW9z90OqCqBqlJvOdUDzipwWryPDiu4bGAIgaAIMB1bgiIgOMr7/A8YpBts0PH4FZ25NiOJe+etZ3+BhVveX83fR2VwVY/EZj++OHtIYGkARW5+KMQ5RVVVDpdU+a7cWXOghLUHizmxh8eg1TCgXTSD20UzuH0MXRPD0WiO/Y5wVkHZEcg6CrYysFeArdz7aK9+PPbcYQWtDjQ6cNrAUgBlh8BlB0UDnsbPStsgGh0Ex0BILITEQHgyJPeC+O4QlgjmVqBpuoGynRPC+b8J5zPxvxv5ZlseE/6znoKKrowb3KbJjiHObhJYGqD6d5AHhTW5a+ib0DewFRJCNJkSi4OcMhvZxZZjY1C8A2RPnL5ei5soKukd6+a8aDc9opxkRLsJcmyEyiJYWQiWQrAWQUUuVBU3TeXUuudcAUCj97aWBEV6gw0K6INAHwyGYO9znQkcFm8rjK30+KOjEjwuqMz1LtU2/vv4c30wxHXxPupMEN0eYjt5l4QeYAxt9McJMep459bePP/FNub+epDn/287+RV2Hs3sJDPlCr8ksDRAuMZEHoCicNe3dzHtomlclnpZoKslhGgAVVWP3djPTpHFQWG5jUP5+Rw+fJiCvByUqiKiKCdSqSBKqWAYFdymVBBtqCBBV0mkUkGIu9y7s4pjS1YDDqwPgfAkb/eLMeyEJbzmoyEYPG5vgNCZvCEkItUbFFS391Ef7G3tqA4mmtO8gshl9wYsSwFYj4Wton1wZK33sSLH2w11ZN3xbfYuPf5ca4R2l0BUO29giu8G8V29LTaGkJN2NWk1Ci9e2434cCOvf7ubd1bso8Lm4sVru0loESclgaUBtIr3l4NO1eBW3byx7g0uaX0JGuXMu+xQiEByuB2U2kupdFRS5arC6rJ6H53eR6fHicvj8i6q6/jzY4vFaSHXmovVaQWOjy+rHhDvVt24nXasVZU47FZczircLicKLjS40ShuNHhQFRW3BtREBbcCh/C2oHoU0KhgUj2YVJUgj4pJNWFSjZhUlWBFT5jORLguBLMhjHBjBOGmKMJDYgkPSSA8LBGzOQ19ZGqtMSJujxuX6sLtcfs+p1t141E9mLQmgvXB6DX6P+ZLW2cEc7J3qYvH7Q0uBTu9Y2UclVC4Bwp2Qf52KD8Cu5fUvW1wNLTq6+1SMoR6u5iSz/OGMK33K0dRFCZc2oGYUCNPfr6Fub8eRKdVmHR1Vwktol4SWBriWC4xqlrCDGFkV2Sz4tAKLm19aUCrJUSgqKqK1WWl1F5Kqb2UMlsZJfYS73N7mW99qa20xjqry/rHVlR7bPHReBdVweA2YXQFY3KFYHJ6H6tfG10hmJwhGN1BaFTtsTmYFFRVoRwoV1QOKm48ihu3xoVHceNRynErJXg0m9FoFRQ0KB4NeBQ0Hg2KqkHr0aFVdb5HzbFH9ViIAhVFo0GjaNAqGjRo0Gg0aNCiUTS1F42CVqtBq9OiRQMuDW4HuB0e3HYVjxM0WgWtXoNer0Wn16LVa9DpNWj1GozBeoLC9ASF6jGFGo49N2AK1RMUmkJQu3bojb8bx6KqkL/D2+JiOdY6k7MJivaC2+7tFqsrzChaiEyFpF4Q3QFC47g5LJ64i+zM/XEHh1et5d3iNvz52svQRdQTpMQ5TQJLA5x4ldDIjiP559Z/8tH2jySwiLOaqqrkWfM4UHaAg+UHySrPIqs8i4NlB8mz5uE8xcGgGkVDqD6UYH0wwbpggnRBBOuDMWlN6DV6dBodOtWDzlmF3mFF57Cis1eisVdgtJaTWFVKuOf4JcQejwm3OwyXOwzVHYLHHYbVE42DODxKNFpNBFrC8LhMuJ0G3A4tbvuZ/1e8CriPLTXX1OR2q7gdbhx1vNcQOr2GoDADoZHGY4uJ0KgwQiNvIbSV93VQmN7bxuWweMPMkXXeK5isRXB0PeRu9YaZ4v3e5QSXApcajr04AEwH1ZyC0noApPSHjpkQ0fqU6i7OLhJYGuLYqFsVhdGdRzNn2xzW5a1jS8EW0mPTA1w5IU6PqqoUVhWSVZ7FnpI97C3d63usdFaedFuDxkCEKYIIY83FbDR7n5tqrw8zhKHxuKHsMJQcgJIs71Lkfa6WZKHYy1FVsKshVHnMVHnMVLpjqHB3pNIdQ5k7jhJ3AlZPNB715POanCxW6fQajMFajHoVg+JA77Ghd1SiqypF57Rg8NhRVBeoKhqtBkWjQdFpwWiC4DAIDkEJCkENCgFjEB6DEYdWQxUeUECn03oXvRad1vuo1+t8i06vRaNVcHvc2Fx2bE4bNpcdh9uB3W2vsTh+99rusmN3O3A4HThdLhxuB1YqsStVYFBR9CoerYtKm4VKWyW4Neg8erSqHq1Hh86jx+gKJsgZiskZSpArlCBXGGHuCIJcYejtJhSPBpfTQ0WxjYpiW73nUavTEBJpJCzSSFhUCGExmYTHmAhPCSLq4hBMQVqozPN2Jx3d4P1/X5nvXWcrBWMYpTYPZUW5tCIfbdkh2HIItnwCXz0CMZ28g3wjUqH3OGhz4R9ySbZoWSSwNMCJLSzxIfFc2fZKvtj3BdPXT+eDoR9In6s4Y5TYSthRtIPtxdvZUbSDrPIsDlUcospVVWd5naIjJTyF1PBU2oS3ITU8ldTwVJJCk4gwRhCkC6r759/j8Q7orDgKJQehZL0vnDiLjlBebMXijsDijsTiicbijsLqaY/V0xubJ5yqY4vawF9ROr0GU6geY7AOk0nBoPNgVJzoVTt6lxW9swKdrQJdVSnayhI0FUUopQWoOUfwWCync0rrptWijYhAGxGBotejaDTegbIaje959aOq0aDV6QjV6QjT644NrlVA+d0cUIpy/EtaUVA0CpqQUDThYd51bjeqOwrV4cBTZUWtsuGx2UAJB60GJ27sqhM7LmwaF1atm0pNKWVKLgVqObnuUsq1Tgr04DCBPQJcOhNoQtEQRqgmkURDO2KUZMJcMRiqgnFXaKiqcOJ2eSgvqKK8oO6fo9BII9HJoUQnpxHdqhvR7UKJiAtGqz8+DjACWLM9j6vn/kQPZS9/bV9Ef2UbZK+Cwl3eQkfWwbaF3suuUwdB6mBIOx9iOkqAOQcoqlrXrbfOLOXl5ZjNZsrKyggPD2/y/S9/ajLbiwcQZs1i7Ed3cKTyCMM/H47T4+Tty97mwlYXNvkxhTgdbo+b7IpsdpXsYnfxbnaV7GJn8U7yrfl1ltcoGhJDEukQ0YEOkR1oH9GeDpEdSAtPQ6/V197AVn4sgBz0zhlSegjKDuEsLcRSaqey3EOlO/JYq0gMle4YKj0xWNzR2NVGXg6rV9AH64mMCSIiUk+wUoXJWYqxPA9dYTb6o/ugOB93cTHusjLqvJugH9rYGAxJyegSE9EnJKBPTEATGgZaDYpWCyiobheq0+kNBBYr7pIS3MXFuEpLcJeUel+XlOCpPHmr1NnAqQW7Hhw6cBgNuEJicYXGowbF4zbF4tBH49RGYCWMKnfd0/ArCoRHGYhMDCEqOZzo5BAS20ewaFcuz/5vGwDTburJDZ0MkLsZXA7Ytww2/sc7Od6JIlKh05XQaZg3yNT1MytapMZ8f0tgaYAVz0xhW2F/wqoOMnbOeADeWPsG/9r2L9qa2/LZNZ+h00hjlQgMVVXZW7qX9Xnr2Vmyk93Fu9lTuqfeVpPU8FS6RHWhS3QX2ke0p3VYa5JDk+sOJuBtLTmyDnXHYir3bqO0wE6FxUClO5pKTzSWEx4bGkaceKjQgE0LulA9YREG4sI1JJhcxOvthLkq0JUXoC3LRy3Mx5WXhyM7G3dxw+Y30ZjN6CIj0UZGoo2KQhsZgS4iAk24Ga25eglHF5+APikRjanppstXHQ5cJaW4jwUZ1eUC1YPqdoNHrf3c5faFIVwuVLfneOhSvQNxq5/7fl2rgMeDu7ICT1m5t8VGqwWtFsWgRxMUjCbIhGIyHSvrRnW5fY+q04GnyoZqq8JTZcNjq/K2yFRVHX9us6FWVeGxederNtsphUGnLghLSBKVIUlYQpKpDE3CEpKISxdcZ/lgnR2XxsIWSwUHTFoev+dSzu/e6oQdVsHhtXDwF++Svdo7Pqaa0QwdLof0G6HD0Cad/E40PQksTWzFs1PZVtCXsKpsxs4Z5z2mo5yrFl5Fqb2UB857gD+l/6nJjytEXVRV5XDFYVbnrmZ1zmp+y/2NYlvtL3KT1kSHyA50jOxIp6hOdIrsRMfIjoQaTh4q7FYnpTnllG7bSOmePZTmlFNii6bMnYjLz3gRABduKjUeyjQK5RqoUFSCDG5aBTtJMVhJUUuItRYSXl6EJj8XV24urqIicDdsUKg2JgZD69beJbU1+pTW6GJj0UUdCygRESg6+QOiqamq6m1dslpRbTY8VTaslcUcLcwit/ggBSWHKSo9iqWiBFtlKQ5rJSaHSqgNQqsgrAoiHXrMNg1GB3gIoUoXgzUkAUtwAhVhrakIS0FVagYMo62YyKpsEkIqSUjQEtE2Hv2x//+62Fi0QTqUgz/Brq+9VydZC49vHN4KBk3wjnvRB/2xJ0w0iASWJrbiualsy+tLaNUhbp9zu2/9F/u+4Omfn8agMfDJNZ/Q1ty2yY8tBEC+Nd8XTlbnrCbHklPj/SBdEBmxGXSL6eYNJlEdSQ1LRVvHX5cej0pVhQNLqZ2ygirK8q2U5looO1JIaaEdm/1kzekeHAY3eSoUKwoVige3uwq9sxyTrYSwqkLiHaW0V6wkOSswW0rQFhWAw+H/QyoK2uhob/iIifE+nvDc0DoFfevWaEMbP8Oq+OOV2cvYVrjNF6y3F21HpebXjaKqhDp09FPa0MuTTFtLFEFFkZQUGyiwh1Oqi0f9Xet1kDWfyNLdRJTuxlyehclWhDY8HG1kBFpzBPpwA4agcgy2rRhNZRjCXGij4uGCiXDe7aA/N24+eaaQwNLEfnjhNbbm9CGk6jDj5oz1rVdVlXuW3cMvR36hR2wPZl8xG71G+k7F6Suzl7Emdw2/5vzKb7m/caDsQI33dRodPWJ60D+xP/0T+9MjpkeNLh1VVbFbXJQVVlFeWEVpnpW83XkUHanAYtWiek5+/GBNMaG6fKpwctiuIc+uYLAWYbbkEGMrJaaqjDhbGVG2crSehreM6OPi0CcnoU/yLrrERPSJSeji49BFRUnLyFmszF7GzuKd7C3d611K9rKvdB8VzopaZTtHdWZg0kDSQ/owf14p5nw7qYoRAyH8/p5uOpeV0MrDhFUcJrTyEOHlWQRX5dcopTO5MYS7MMYYMfS+BOMlt2Ds0g1dZGTzfmjhlwSWJvbDS6+z9ch5hFQdZdyc22q8l1OZw/VfXE+ls5Jx3cbxcJ+Hm/z44uxndVpZl7fO14Kys3hnjb9GFRS6RHfxBpSE/vSK60WQLghruYPSPKt3yfeGk/JC79UaDlv9QULBg4lSQj0FhLryCLHnYbAV4bRaUMvLMViqCLNZ0dKAXw8ajbclJCEefXxCzceEBHTxCejiYtEYDP73Jc4pqqpypPIIGws2sjF/I+vz17OnZE+NMhpFi2pPwG5pRafgHkzqOQzbITi6u5Sio5V4XLV/Ro0aO9GOo5jztxJ+aH2tAFNNFxuLsUMHDGmpx7qZUr3djK1aoTHWPVhYNC0JLE3sx5enseVwL0yOIsb944Yal+IBfHfwOx5a8RAAb17yJpe0vqTJ6yDOLg63g00Fm3wBZUvBFlyqq0aZduZ29EvsR9/o/rRXuuIq0RwPJ8eWk4USAKO7jGBbPkFVBYSWHyKs9BAmWzF6ZwUaf80sgKrVoo2Lx5iYgD4h3jtI9cTHhAR0MTHSMiKaTFFVEatyVrHq6Cp+zfm1zivb2oS3ZVDyQLpFdifZ2QZDiZmSw1YKDlWQn1WB21XzZzsoSCE2zEZM6Xqi9nyJNr8Ap+UkP7OKgj4lBWPHDhg7dMDUseOxYJMmP+tNTAJLE1vz+mTW7O6LqtESlxrGsLvTCY2s2Q/62prXmLt9LgkhCSy+bjFGraRzcZzb42ZnyU5W53j789fnrcfmPj4Rl6IqdFY70c/Tm9a2NoRaIqgqUyirgCrHSX5Bqh5MtiKCrfkEV3mDSVBVIUG2Qky2YrR1zEbrRqHCGEy5KRx3uBltVBQh8bFEJsUT1zoBU1ysL5Boo6O984UIESC5llw2F2xmyd7VfLtvJRiPoCg1v7b0Gr3vUvyu5m6k2buizQkjd08ZufvLawWY6Eg7rV1LSbauIqzyMC5zXxwk4zh8BOfB7Hrn5lH0egzt2mFs2xZDWiqGtDQMbdpi6twJRS/DAU6FBJYmtvWtFyj9z2a2dR2HSxdCTEooNzzWG53++IBGu9vOVQuvIs+axyN9HuH2brefZI/ibOfxeDiQt4ONu39g5/7fOJS9DV25lQhLMOH2OEKc8QSpCeg1ibi0MdgMUXhOMv5J76wk2JpHkDWf4Ko8Qqz5BFnzCbLlYzA40Bk9aI0eNHoPWr2KVWckTx/JbmMrNunbYYltR1xKIq3aJNGhXRJdUiJJMptk0kNxRll3sJhxc1ZQpd2NOSqbNkllHLbsw/L7eVmAYF0w6bHp9Io6jw6OHoQUxZK3q5LcfWU1rs4O0pSRalhHasR+Wo/6M/rOF+EuKsK+dy/23bux79mDbfdu7Hv2olrrvheWYjIR1LMnwb3PI7hfP4LPOw9FukAbRAJLE9v69sto35pHVVQMGwa8jM3qJv2iZC4c3alGuc/3fM6klZMIN4Tz9Q1fE25o+rqIwPNUVeHKz8eVn48zPx9XfgGu/Hwqjh6k7MgBHPlFqPYQnPo4rMHxWIOOPQbH4dLXf4WLxuMkqKqQEFcRoe4iwtQCQj2HiXDvJ1RfgvZYKNEZPWhNHnRGN069lgMksldNZrenFXu07bHF9SApuTVdEsPpmhhGp4RwQo3SjC3ODnvyKrhjzhoOFVeh1yrcc1FbrusXzP6yPews2cmWgi1sKthU67YSCgrdorsxNH4YXa39KN3jIntbMY6q412xGpwkxVaQNjidtN4pmGOPzxWjejw4jx7Fvns3jgNZOLKycBw8iH3XLu+EhSceKziYkAEDCL3gfEIuuBBDK7mZY30ksDSxLR9OQ/faBwAUxfZgU7e/AJBxeWsGXNPWN6bF7XFzwxc3sK9sHzd1vIlJAyc1eV1E8/I4HLhycnDm5OA8ctT7mHPUuy7PG1I8Fd6rGtwaA5aQBCzBiVhCErAGJ2ANjqcqKKbWXBInCtLaCTfaCDdWYjYUEqo5RLh7KzHuzZh0VXXOMG5XdexXk9ittmKPJ5kDmhTsER0ISuhAmzgznRPC6JIYTmpUMBqNtJqIs1tRpZ3HP9vCdzvyALiwYyzv3noeIceCudvjZm/pXjbmb2RDwQY25m/kSOWRGvvIiM0gs/UV9PIMonRnFVmr91BqCatRJiLORErXGFp1jiSpQwSmkNqtoKrHg2P/fqzr1mNduxbLypW4i4pqlDG0a0f4FVcQftWVGNvK9BcnksDSxLZ8/zGtPnyI3L1RUOrhQOoVHGgzHICYlFCG359BcLi3+W91zmru+vYuVFT+fvHfGZI6pMnrI06Nqqp4yspwHj36u0CSc2zdUdwFhbW2qxlMvOGkMiQRuymm3mNp9CpR8cFERSsE64vRu7MItm0nyraOWNse9Grdt+Szq3r2qknsOdZickTfGmI6E5rYjjZxEbSLDaV9XChJEUFoJZiIc5iqqvzf5hwe/3QzVU433ZPDeffW3qRE1T2Dbp4ljxWHVrAkawnr8tb5rsJTUOga3ZXzk8+nd3Ewxl+2k13YihxHVzwn3stKgdiUMFp1iiSpYwQJbc31Bhjbjh1YfvqZyp9+omrjxhqTIho7d8Z8zTWYrxshl1UjgaXJ979l+Sek//AndmnaERUxjsJ33qUgOp2dnW/DqQ8lMs7EiEf6+ELL39f9nQ+3fki4IZwFVy+gVVgrP0cQTUF1Or3dNNWB5GjOCc+9j/X1QQO4tEZv1425NVVx7agMSaRcF3nS6eYdBiuaKBcxcQbahKuEuA9irNxIROV6oquy0Ndzr2CbqmevmsweNZl9aivKQtvhjukMkanEhgfTJTGcbknhtIqs5+aCQggANh4q5Y7Zayi2OAgz6XjlunSG90g86b+bfGs+32Z9y5KsJWwq2FTjPbPBzKCQFAYdOkDa4UjK7N054uhBiat2t05UUghJ7SNITY+mVadIdIbaLavu8nIqV6yg/MuvqPzlF3B5u6AUvZ6wzEwiRt5EcN++Lf/f+eZPIK4zxHVt0tsdSGBpYlt/WEj35ePZq21Lu2fWU/T+BxS++y4WQtnQ8wHspkgiE4O5/pHemEL0OD1Obv/6drYUbqGduR1zr5xLmCHM/4HESamqiru01Nt/fPAgjqwsnIcP+1pJXHl53vve+OGJS8ae3Jmq6DZYghOp1Jgpd5iwWOv/hVGlq6A4OBdHeDkRUZBostPJfZRW5TuIsewh1FN78iuAKtXAXjWJQ7pUSkPa4ojshCmpKxHJ7Qg1mYgPN5IaHYJBJ1fiCHGqDhVb+evHG9iQXQrAwLbRTBrelS6J/r8PCqwF/HzkZ34+8jOrjq6qNZFdG4+GnpZy0iuDSKjojNN4JTm29pQV1/xdozNoSOkSRZueMaSlxxAUVnvQraukhIpvvqX0v//Ftn27b72hTRsiRo7EPOLaltnqUlkAr7cHFHjyEBib7vtMAksT2/rTIrovu539mjTaTvKmcdXlwrp2Hbsfepa1ne/GYYwgvk041z7YC71RS54lj1u+vIX8qnx6xPbgvoz7GJA4AI0iX0z+eBwOHPv3Y9+3zxdMHFkHcRw8iOd3g9t+T9Hr0SUloktMwpOQhi0iBWtQLBWEU1ZloLTEjaWs/mnirfpySoJyvUtwHp6QKqJ0FXSzF3FBRQ49nHlolNr/ZNyqwgE1kf3aNEpC2+OK6UpwSg/adehCxwQzJr3cgE2I5uR0e3h7+V7eXbEPu8uDRoFb+rfmr5d1IC6sYdPxuzwuNhds9gWYHcU7apUJc3voYbfTw51Al9Dr0HARWbvsVJaccANGBRLbmknrEUObnjFEJoTU2k/V1m2ULlhA2Zdf+lp+FYOB8GuGE3P3PS1roO6e72DeDRDdAe5f26S7lsDSxLb9/D+6fTeWvbQm6akNBBuO92taVq5k+4PPsz79flz6EOLbhJN5V3fCokxsL9rOuCXjfHfN7Z/Yn+kXT/d787lzhep248jOxr5nD/bde7yPe/bgOHjwpDfC0yUlYkhNRZ+ahjs+jarQBKz6SCzuIMorVcoKbJQVVOGy178Pq6Gc4qAcSoJyKQ4+FlBMeRg9OjpVKVxkK+Uq+xFi1Nr7yFcjOKBJpSi0A47ozhiT0olt24N2iTFEhsiljEIE0qFiK69+vZMvt3jvt2XSa7i1fyp/uahtg4NLtRJbCZsLNrOpYBMbCzayNX8TVZ6af/DEu1xc5QnhkrDhOGwDyTocRsHhmndKj4gPJq1HDG17xpDQ1oxywvgzd2Ul5YsXU7Lgv9h3HAtIej2RN91I9F/uRh8fdwpnoYn9+Dp8/xKk3wQ3fNCku5bA0sQq960mdO5QLKqRSa3n8tr4oTUGPJZ9+SU7XnibTen34tIHY9SrDLmzO2kZ8RwsP8j8HfP5fO/nVLmq6BLVhacHPE236G7oNOfOpaYeqxXbzl3Ytm/Htm0btl07cezbj2q311leYzZjbNcOQ5s0NCltqIpsjcUYS4U7mLIiB6V53pv2uZz1dwF5UKnQWyg15VMSmk1pUA4lwbmUBOXh0FWh8Si0shs4z2bnMlsRfe1WQn73z6FUH0dBeDqW2Ay0yRmY0zKIT0zGqJMWEyFaslX7inh1yU42HSoFwKjTcEv/1tx9UTviw0/tBoguj4vdJbvZdHQ1a/Z9yc+lu6k6ocW1u93OVZVWBtABW/BIDlSmc/iAE4/7eJnQSCPte8fRvk88calhvrErqqpStWEjhTPfwrJyFQCK0UjMPXcTfeedgZ2YbsEY2PEFDH0ZBt3fpLuWwNLUPB4q37mY0MJNfOa+gI29X+XFa7vVGCRV/NFcst74B1u73UlFWCqoKt1SrZz/4OXogr2tLfd8dw/FtmIAIo2RvDDohbNyGn93RQW27Tu84eTY4ti/H+r4UVOCgjC2b4+xQwe0bTtSFdsWiymOMouWkhwLxTkWyots1HdLGw8qZTobZYYyyk0FlAcfoSwkm/KgfCqMxXg0x1tIgt0aOjs8DKoqo4/NRneHHeMJ+1V1QZCUgdKqL7TqA8l9wNyCmmWFEI2iqio/7C5gxrI9vvEtBp2GW/p5g0uC+fTu3Gx32/lx72K+2D6Pn8v34jrhF1UXu4MhFiuXG9vjjridA2Wdydplr3E7jfAYE+37xNOhTzzRySG+7xTL6t8omDGDqvXrATB26ULiyy8R1K3badX3lE3vAaUH4fb/gzYXNumuJbA0h8Pr4INLAbjJPolLM0dw90Vta4QW286dlHz1DWt+tXEoqi8AIfZC+vTW0HHUxRytymH+ihl8o2yn2FOBgsL9ve5nXPdxZ+xdnj0WC7YdO6jauhXb1m3YtmzxdunUQRcXh6lrVwxduuJI6UJlUCKlNiNFRy0UH7VQUWSrczuAKsVFkaGSElMRpcE5lIUcpCz0ABZjER5N7VaWcFVHB7dKD2s53aqsdHc4SHK5j98ALTQekntDYk/vqPf4bhCZ1qSj34UQLYOqqvy8t5AZ3+1h7cESAAxab4vLhEvbExN6+rdSKaoqYknWEpbu/5oNhZvxnBBe+lbZuK6ykos1CRSE38zeivM4cECPy3H8d1dkQjDt+8TTsV88EXHBqKpK+f/9H3mvTPZOTKfVEn3HHcTcdy8a0+kFrUapKoGpad7njx+EoIgm3b0Elubyv/tgw7+pVE1McN7PJlM/erWOZMzAVM5rHUmZ1YnD7UHjdFDy8Xes22HCqfOOVzE4yokq3o65/ACxjoMc6mFiYcQ+dqQoRMSl8Ocef+bKtle26HsQeex27Dt3esPJlq3Ytm3Fvm9/nVfm6JOSMHXrCh3SscZ3pNIYR0kpFB6ppCTHWuveHtUsGifFhkqKTQWUBB+hNGw/pWH7sOnrvrdHBFraeTS0r6qkXZWF9k4nbR1Ook+skzEcknpB8nnekJJ0HoQnUecMbUKIs5aqqqzcV8SM7/bwW5a3tTvYoGVknxTGD04jNbr24NhTUWwrZsWhFXy99wtW56/zRZcQj4dMi5WrKy2k2w0cCruZvVWDOXjUXGPYXqvOkXS/KJm0HjGopSXkvvwyFV8vAcCQlkbSq1MIyshokrrWcHgd5GzwXhWUdr53OfAjfHSN9w+6Bzb53UVjSWBpLrYy+PhWyPoJt6rwifsi/u0egopCthpPBccnLFIUaGvUcNXhfIKVGNzaoOP7UT1ElO0lPm8t0UUb2JlcRU4UhLuNaPr2JPHaG7mszVAM2sAN4FQdDux791K1ZSu2rVup2rYV++49vjkETqSLj0fbLQNHuwysUW2o0EZRWuyi6KiFqvK6r8hxatwUGsooDMqnOCSbkrC9lIYcrjeYxCt62ro8tLWU0dbhoO2xYBJ1YjBRtBDdHuK7Qly3Y49dICIN5AZ+QohjVFXll71FvPbNTjYfPn7lYZ/USEb2SeHaXklNNk4tpzKH/+37H//bs4jDluOz7Ua53d4uI4uVdKuWQ/Z+7HEOIdvaFY61BYeEaUi/JIXuF7fGsfIHcl94EVdBAej1JL7wAhHXX9ckdcReAV8/ARv/XXN9XDdISIfNH0PXETByTtMc7wQSWJqTywFfPQzrP6qx+pAnlpHqFKw6M063B6vjeFzWqNDVaicaI50NIYRbjn/JKh4X0cXbiSjdi9Fegrl8P+WmUn7tZybxqtEkJvQmNaErraOb79JYd2kptp27sO/aiW3HTmw7d2Lftw+cNSc9UwFPTDKurn1xtOqKJTSZCk8oJUXOmpf01dhGpdJUQb4pj6LgbIrDDlAUfJQKYzH87vJgDRqS9WG0VbW0tdtoW15AO2sFbZxOQk/8MQ1PhlZ9ISIFwpIgPNEbVGI6gq7ltlAJIVoWVVX5aU8hH/x8gJ/2FPiG2cWGGRk3KI3b+qdiDm6a7nqP6mFd3jq+2PcF32d/T7mj3PdehAcutVRyucVKl4pwdlddzo6qIVR5IgDQa+x0jdtMeuJuKr7bR8UW7y0Joob1Jm7slShBZu+Oqkq8i60MPC5QNBAcA4YQcNnAYQFnlbfrW2vw/r4sOQhbPwVrEaBAh8u986zs+hqcJ0y0edlzcMHEJjkXJ5LA8kc49Bssnwy5m1GdVShOK7S/HG75L6qiUGRxcLS0ivIqFztzy/l5byE/7Pb+gwjzKHRxaOnm0hHjqt0tEV6eRUzhRqKLd2JwlGPTVrAj2cj2pBRyUwahhGdAcBhGowEF7/wDDrcHl1slPEhPbJiRmFAjyREmuiWZ6RgfhkGn8d686/DhY6FkB/adu7Dt3IkrJ8d3bBUFpz6UqqAYqiJb40zpjC0iBYsugsoqLQ57/VflWPTlFAcfpTi4+nLhHIqDc3Bpa7aymJQQWmvD6ajV095po015IWmlR0hxOqn1q0HRehN+h8u9ISWuK5hbSXeOEKJJ5ZRVsWjDUT5alUVOmXc8XYhByy39W/OnC9qe8pVFdXF6nKzJWcO3B7/l++zvKbGX+N6L0YVwsyGRG8ptFB4ws6E0k2JXKgAaXHQw/USbo1/j2OxtjQ5NspE0sASt/jS/yiNSYcS7kDbY+9paDJ+MgwM/eF/fthDaX3Z6x6iDBJY/Wu5W+OAyb4Lt+yfInFznX/qHiq18sekoqw8Us/5gCZV2F9FuhU5OLVFuhQiPhgS3gkLNL2PF4yK4qoCwioOYy/YTZCtC4yxhR6SW7dFtUD16KvThFBpj0bs9hDkthDqqCHNaCXVUEe2w0MpVTnxJLihGbKZobMZIbKYo7MYobKYorCHR2A2ReDT+WygqDCWUmQooORZIqucycei8cw+oqoLijiRUiSVVa6KjTkt3vYauqo2UoizM+dvr3nFQpHd8SXJvb3dObGeIagc6mdtECPHHcLg8LN58lPd+3M/OXO+stwathht6J3PXBW1pG9u082i5PC7W5a1j6cGlLD241HclqVFrZHjb4dyWMgTdLicbVtk5csT7J51G8dAxeCtxy/6NzmbBGK2j1YhIDImx3t+jJrO3BcXjBkuBt6VEH3xsMYHq8fYWuO2gD4Iu10Lbi0H7u6k2XA749hko3g+j/u3dtolJYAmEDfPgf/d6n8d3h6unQ0rfeou7PSo7c8sprHQQatQRZtIRbNBSVe4gZ3sxBTtKKT1qwW51otbTqKF1VRFUVQioaN0OdO4qPBoDTl0QLl0wHo0Oncv7l4JbZ8RhCD/pXYSrqXioNJRSFlRAmamQMlMB5aZjz42luNxBeFxmVKcZjcdMYlAUPUIMZBhVeugq6WgvIKQ8C/K2ef9B/J6i9baWJHT3hpLYThDbBUJipOVECNEiqKrKit0FvLt8n2+ALsCAtlGM6pvCsO6JTd5N73Q7WZK1hLnb59aYZXdw8mBu63Ib7R3dWfd1NlmbvTdpNRoV2mQtJmH3EvThYSS8/BLhl1/epHVqbhJYAmXnl/DF/cf6AoGOw7zNayn9vXN6nMLAT9WjUllqp+hIJbn7y8jLKqOwsBRbkRs8jf/H4lHcVBpKsRjLsBorsBotVBltWA0O7DobFeiwqTp0umCMmlDMGh3RikK4zkg7vYaOOgdJmlJiXLmEWw9iKN2PUn64/gOGxHkvHY5MhYjW3pHmaRdAcFSj6y6EEIGwJquYWSv28f2ufN84lzCTjhEZyYzqm0L3ZHOTHk9VVdblrWPu9rksP7Tcd2fptPA0bulyC32dF7Pm82yKjx7rFnIV0277f4gq3k7kyJHEP/E4muC671rd0khgCaTKfFj2grfFhd8NFE1I9w6A6nA5dLrytLo6PG4PxTlWrGV2VMBuc1BWXonWoGA0qBiLN6A6S7HEnIcmKIKgIBORYQYiKEAXFO5tBrQUQGUelGbDxvmQvQqCorytH64q72Cs0rrnVKklKOrYwNcOxx9ju0B0O2k1EUKcFY6WVvHpusMsWHOII6XHp9/v0crMqL4pXNEtgegmmNPlRIfKDzF/p3e2dIvTG1BigmK4r8cEOuT0Ze3ig9gs3gskQisO0frQMlqFFNPqtSkEpac3aV2agwSWliBvm7fFJWcT7P8BHL+7m6/J7G15iG4Ppgjva1P4sUezd53O6O1DdNm8XSsuu/f5iY9VpVCR4+2jdDu9gSlvi3eUOIBG770luMsBxfu8I8cbSx/s3U9ItDd4hSV6Z4CNbu+9GVZMB2kxEUKcMzwelV/2FbJgzSG+2ZaL89jU+xoF+rWJYlj3RDK7JZz2TLonsjgtLNq7iLnb53Kk0nt5dPuI9jzQbSK6DQls++mIbyI6o62Y5Jyf6XJ+Cq0n3o0mpGnml2kOElhaGqcNsn6C8iPewUub/+sNGc0pvBWExsLRDTXXG8K8Ycfj9LaKhMZ5l9YDoddtUH4Ucrd4A1NYgncGWAkjQghRp6JKO5+tP8wXm46y9Uh5jffOax3BkK7xXNo5jk7xYTVmRj9VTreTj3d9zKxNs3yXRg9MHMj9XR/EvjmIzd8foqrS+4eporqJseyj+5Wd6TzqQjTaljcflQSWls7t8raC5G6FskNgK/e2iJy42Mu8QUdn8ra06EzeLqQar43e6+XDEsEQChqdN3xEpHpnddVoIX+HNyhpdBDVFswp3jp4XKA9M28HIIQQLdGhYivfbMvl6625rDtYUuO95IggLuoUywXtYxjULua053cps5fx/ub3mb9zPk6PEwWFa9pdw93d7qFih4atX+8gv+B4eZNio8OgVqSdl0xShwh0hsaNgTy6t5TgMAMR8U07NkYCixBCCBFAeeU2vt2ex/Kd+fyytxD7Cbcj0SiQ3iqCAW2j6JcWRZ/UqFMOMIcqDvHm+jdZkrXk2L41XJh8IVe1vYpuzm7s/OBHDhSG4jSE+bbR6hSSOkSQ1CGC+DZmwmOCCA43oNNrUDS1W4GO7i3l/97ahNGk5fpHexMeE1SrzKlq9sDy9ttv87e//Y3c3Fx69uzJW2+9Rb9+/eot/8knn/Dss8+SlZVFhw4dmDp1KldeeaXvfVVVee6553j//fcpLS1l8ODBvPvuu3To0KFB9ZHAIoQQoqWqcrhZtb+QH3cX8tOeAvYV1LwFiaJAp/gw+rWJokerCDrEhdIuLpRQo66ePda2qWATb61/i9W5q33rdBodfeP7ckVVJ2LmHaLQHkNxVBfsxsh696PRKugMWoLC9IRGmohOCmHHyhycdjcpXSK58p4ejW6dOZlmDSwLFixg7NixzJo1i/79+zN9+nQ++eQTdu3aRVxcXK3yK1eu5MILL2TKlClcffXVzJ8/n6lTp7J+/Xq6d+8OwNSpU5kyZQpz5syhTZs2PPvss2zZsoXt27djasBdKSWwCCGEOFPklFXxy94i1hwoZk1WMfsL676HWpLZROvoYFIig2kdFUzr6GBaRQaTYDYRHWKocx6Y/WX7+d/e//F99vdklWf51iuqyqV7TFz/s5vgigiKIztRZm5HZVQbbPoIPOrJx7ckd4pk2F2dMYY2XesKNHNg6d+/P3379mXmzJkAeDweUlJSuP/++3niiSdqlR81ahQWi4XFixf71g0YMICMjAxmzZqFqqokJSXx8MMP88gjjwBQVlZGfHw8s2fP5uabb27SDyyEEEK0JAUVdtZkecPLrtwK9uRXUlBR9/3ZThRq1BEdaiA6xEB0qJGYUANhJj0mvZYgvRarepRs2xqyrBvItmzDpTpBVelyCIZs8DBgl4re7b0li0ejx6E3UB5jxhoRQVl4JJbgGFRda3QON613fYbGbeeSH1Y26WdvzPd3w9ubAIfDwbp163jyySd96zQaDUOGDGHVqlV1brNq1SomTqx5w6TMzEwWLVoEwIEDB8jNzWXIkCG+981mM/3792fVqlV1Bha73Y7dfvx/Znl5ea0yQgghxJkgNszIlemJXJme6FtXanWwr6CSQ8VVZBdbOVRsJbvYyuGSKgoq7DjcHirtLirtLg4WWU+y907eRXGiMRSgMeaxMTifzZfmYb44h/4Hiumz10PnQw7CbA6CjlTCkSP17s2aX0BwXGzTffhGaFRgKSwsxO12Ex8fX2N9fHw8O3furHOb3NzcOsvn5ub63q9eV1+Z35syZQovvPBCY6ouhBBCnDEigg30To2id2rt91RVpcLuoqjSQWGlnaJKO4XHnlvsLmxOD1VON1VONzaH2/fc5Y5BpTOq6r3NkKqq7Ep1sKVNLi4lH7PlMLHlRURVWIiqcBJhsWN0WnEZVLLbxlDSvRVvRNQ//qW5NSqwtBRPPvlkjVab8vJyUlJSAlgjIYQQ4o+hKArhJj3hJj1tYlrupHBNrVGzyMTExKDVasnLy6uxPi8vj4SEhDq3SUhIOGn56sfG7NNoNBIeHl5jEUIIIcTZq1GBxWAw0Lt3b5YtW+Zb5/F4WLZsGQMHDqxzm4EDB9YoD7B06VJf+TZt2pCQkFCjTHl5OatXr653n0IIIYQ4tzS6S2jixIncfvvt9OnTh379+jF9+nQsFgvjx48HYOzYsSQnJzNlyhQAHnjgAS666CKmTZvGVVddxccff8zatWt57733AG/T1oMPPsjLL79Mhw4dfJc1JyUlMWLEiKb7pEIIIYQ4YzU6sIwaNYqCggImTZpEbm4uGRkZLFmyxDdoNjs7G43meMPNoEGDmD9/Ps888wxPPfUUHTp0YNGiRb45WAAee+wxLBYLf/7znyktLeX8889nyZIlDZqDRQghhBBnP5maXwghhBAB0Zjv75Z360YhhBBCiN+RwCKEEEKIFk8CixBCCCFaPAksQgghhGjxJLAIIYQQosWTwCKEEEKIFk8CixBCCCFaPAksQgghhGjxJLAIIYQQosVr9NT8LVH1ZL3l5eUBrokQQgghGqr6e7shk+6fFYGloqICgJSUlADXRAghhBCNVVFRgdlsPmmZs+JeQh6Ph6NHjxIWFoaiKE267/LyclJSUjh06JDcp6gZyPltfnKOm5ec3+Yn57h5BfL8qqpKRUUFSUlJNW6cXJezooVFo9HQqlWrZj1GeHi4/ENpRnJ+m5+c4+Yl57f5yTluXoE6v/5aVqrJoFshhBBCtHgSWIQQQgjR4klg8cNoNPLcc89hNBoDXZWzkpzf5ifnuHnJ+W1+co6b15lyfs+KQbdCCCGEOLtJC4sQQgghWjwJLEIIIYRo8SSwCCGEEKLFk8AihBBCiBZPAosQQgghWjwJLPUoLCyUmymKs5pcINi85PwK0bTOiqn5m9rkyZP5+OOPsdls9OjRg4kTJzJo0KBAV+ustGTJEkwmEyaTiQEDBgS6OueE7OxsoqOjUVWV0NBQVFVt8ntwncvk/DavhQsXsnLlSmJiYujVqxeZmZmBrtJZp8WeY1XU8PLLL6uxsbHqhx9+qP773/9WBw4cqPbr10/98ssvA121s851112nJicnq+3bt1cNBoP60EMPqTt37gx0tc5qDz/8sNqlSxe1c+fO6uDBg9V169apbrc70NU6a8j5bV5PPvmkGhYWpt54441qz5491aCgIHXy5Mmq1WoNdNXOGi35HEtgOUFVVZV6xRVXqH//+999644cOaI+/PDDateuXdVNmzYFrnJnmZdeeknt2bOneujQIfXQoUPq//73PzUpKUkdM2aMumHDhkBX76z02GOPqampqepXX32lvv/+++qIESPU8PBwde7cuarFYgl09c54cn6b186dO9V27dqp33zzjaqqqlpaWqq+//77qkajUV9++WW1srIywDU887X0cyyB5QQ2m03t16+f+thjj9VYv3fvXvWuu+5SBwwYoJaUlASmcmcBj8fjez5u3Dh15MiRNd5ftGiR2qNHD3XChAnq0aNH/+jqnfUuu+wyderUqTXWjR07Vm3fvr26cOFCaQk4TXJ+m9f333+vJiYmqocPH66x/s0331S1Wq362Wefqapa8/eMaJyWfo5l0O0JtFotaWlp7N69m8LCQt/6du3aceutt+JyuZgzZ04Aa3hmy8vLA8DhcFBZWYlO5x1C5XQ6Abj22mu56667+Prrr/nll18AGbjYFFRVpbCwkIMHDxIZGQmAzWYDYM6cObRu3ZpXX33V9/9HNI7L5ZLz24yqfwekpqaSn5/Ppk2bAO95B7j//vsZN24cDz30EB6PR8YLNZLH4/E9b/HnOCAxqQX79ddfVUVR1L///e+1UuRtt92mDhw4MEA1O7M9/fTTaufOndWioiJVVVX1s88+UxVFUdeuXauqqrd1q9rw4cPV888/PyD1PJvdcsstavfu3X2vq895UVGRGhwcrL722muBqtoZaffu3TVe33bbbXJ+m1BeXp5qt9t9r6uqqtSxY8eq559/vnrw4EFVVVXV4XCoqurtuk9NTVXfe++9gNT1TLVgwQLfz6XH41GtVqs6bty4FnuOpYXld/r378+UKVN44okn+Oyzz7Db7b732rdvT1xcXI1EKvwbNWoU77zzDu+99x5RUVEAXHHFFVx77bVcf/31VFZWYjQacTgcANxxxx3s27ePw4cPSwvLKVq4cCGff/45X331lW/dQw89hNVq5YEHHgC8d2i12+1ERUXxl7/8hS+//JKqqio55w3w6KOPctNNN5GXl+c7X/fddx92u13ObxN47rnnuPzyy+nXrx9XXnkl27dvx2Qyceutt+J2u3nuueewWq3o9XrAe651Oh1utzvANT9zPProo9x8882kp6cDoCgKQUFBXHvttQAt8hxLYKnD448/zp133smdd97Jm2++ya+//sqOHTuYP38+nTp1QqOR09YQDoeDfv36sWvXLrZt28YFF1xAWVkZHo+H4OBgXnzxReLj47n44oupqqrCYDAAkJOTQ9u2bYmNjZXm3VNw/fXXc++99/Liiy9y9dVXc/PNN/Pzzz/Tp08f7r77bv7v//6PadOmAfhuJ+9wOIiPjycoKEjOuR/XXnstH374IR988AHx8fG+89W1a1f+9Kc/8eWXX8r5PQ1PPvkk//znP3n00Ue59957yc/PZ9SoUSxYsIChQ4dy2223sWXLFu6+++4a2wUFBfn+IBInd9111zF//nxWrlzJFVdcUeO9ESNGMGLECLZt29byznHA2nbOAI899pg6YMAA1Ww2q23btlVHjx4d6CqdUd5//31Vr9ers2bNUlVVVT/66CP18ssvV7t166YOGTJE/d///qd+9913ao8ePdRu3bqpDz/8sDpz5kw1KipKfeaZZwJc+zPTzJkz1R49eqjZ2dmq1WpVf/31V3XAgAHq5Zdfrq5cuVK1Wq3qU089pQYHB6svv/yy+tNPP6lr1qxR27Rpo77wwguBrn6LZrFY1N69e6s9e/ZUKyoqVFVV1fz8fLWqqsr3+ujRo3J+T4PdblcHDRqkzpw507fO6XSq11xzjTpw4ED1q6++Ut1ut/rBBx+oqampatu2bdUbbrhBTUtLUzMzMwNY8zOD2+1Wb731VtVgMKgbN25UVVVVV65cqU6dOlV97rnn1P/85z+qqnq7M99///0Wd44VVZX2yZPJy8vj0KFDKIpC7969A12dM4rVamXSpEl8++23tGnThq1btzJ27FgiIiL44osvqKys5P777+eaa67h4YcfZv/+/bhcLq677joefPDBQFf/jPTQQw+xYcMGVqxY4Vv3448/8sorr2A0Gpk5cyaJiYnMmTOH5557DqPRiMvl4qqrruLdd98NXMXPAG+//TbPPPMMTz31FI8++ij/+te/mDNnjq9baMqUKQwfPhyn08m8efPk/DaSqqoUFBRw2WWXMWHCBP7yl7/gcDgwGAzk5ORwyy23EBoayj/+8Q8SExPJyclh5syZaLVaoqKieOihhwL9Ec4Ir7/+OgsWLGDMmDHYbDZmzpxJ586dKSgoYPPmzTz44INMnToVjUZDbm5uyzrHAY1L4qyXn5+v3nTTTWqXLl3UpUuX+tbb7XZ16NCh6pAhQ1RV9f4Vpare6/5F47ndbtXtdquPPvqompmZqVoslhqX0X7yySdq37591alTp/rOdXZ2tnrgwAGZX6iBiouL1QceeEC94IIL1AsvvFBNS0tTp0+frr7//vvquHHj1Li4OHXu3Lm+8y7n99RcfPHF6hVXXOF7XT3oc9WqVWpYWJj6r3/9K0A1O7OdeBHJo48+qiYnJ6tt27ZVP/nkE18L4aeffqoqiqJ+/PHHgarmSUlgEc1uz5496meffeabPMvlcqmqqqrz5s1TDQaDeujQIZmj4hQVFBTUeP3DDz+oGo1GXbhwoaqqx8+1qqrq3XffrXbr1s33Wuar8O/353fv3r3qjTfeqPbt21ddvnx5jfduuukmtWfPnr7Xcn79+/XXX9XffvtN3bVrl2/dqlWr1KCgIHXatGmqqnp/hqt/jseNG6deeOGFAanrmaquc+x0OtUHH3xQfffdd2v8jlBVVR0xYoTvD8mWRgKL+ENU/5V0ohdffFG96qqrAlCbs8Of/vQndfjw4er+/ftrrJ8wYYIaGRmp7tmzR1VV1RcG165dq4aEhKjbt2//w+t6Jqrv/G7cuFH973//65uqvPoX/uLFi1WTyaTu3btXwkoD3HHHHWr79u3V1NRUNSgoSP3oo49UVVXV8vJy9cUXX1QNBoP6v//9r8Y2f/nLX9QxY8YEorpnpLrOcfXvg4qKCrWwsLBGeZvNpl5xxRXqfffdF4jq+iWBRQTEihUr1Hbt2ql/+9vfAl2VM47L5VLvuusutVWrVqpOp1Pvu+++Gr94cnJy1EsuuUTt0KFDjXAyf/58tXfv3jJbsx/+zq+qHu/CVNXjLSmvvvqqOnToUGkt9MPpdKojRoxQMzIy1M2bN6sHDhxQn376aTUyMlItLi5WVVVVDx8+rN53332qVqtV//Wvf6m//vrr/7d39yCNg2EcwJ/SDqL4UdBFaBTEj00y1bGoo1ZxUHRwUwt2EHETxEGkILi4iBVBCIjVQfxACropWKGoKLqIYHSpoGg1pJTKc4NcOdHreV6vbxL+v7Fk+PdPaZ4k7fvyxcUFV1VV8fj4uOB3YHxf6fgzp6enLMsyLy4u5jDt12FggZxaX1/noaEhLi4uxhfPN52cnHBnZyeHw2He2Nhgm83Gk5OT6efQzG+LPDU0NHBtbS13dXVxIBBgp9PJIyMjApObw+/6zbSPSjgcZpfLlX6MAb+3tLTEHo/n3TD98vLCFRUVvLKykn5N13UeGxtjl8vF5eXlLEkS9/b2iohsOpk6Xl1d/XB8NBplRVG4rKyMBwYGchn1r2BggZx6enrijo4O3tzcFB3FtJLJJO/u7nI8Hmdm5unpabbb7awoyrsVg5nfNplsa2tjr9fLMzMzIuKaTqZ+f115lfntMVBPTw+XlJRwIBAQEdd07u/vub+//91nNZFIsCRJvL29/eH48/NzPjo64oODg1zGNLW/6fj5+Zmnpqa4srLy3ca/RoS/NUPOpVKp9D5C8G+YmWw2G/l8PgqFQhQKhaipqenD4mSaplFBQYGglOaVqV9mplgsRqOjo9Td3U3Nzc2i45rS6+sr6bpObrebFEUhWZZFR7KcP3Ucj8cpFotRdXW1oIRfgyVbIecwrGTPz+uN2dlZkmWZ/H4/nZ2dkaqq5Pf7aWdnh4iI8vPzRcY0rUz9Dg4O0vX1NQWDQQwr3/CzW7vdTrqu08PDQ3rZ92QySfPz86SqqsiIpvenjoPBIKmqSkVFRYYfVoiIcIcFwOR+vWNVV1dHhYWFdHt7Sy6Xi/b29tJbHsD3fNbvzc0NSZKEfrPk8vKS3G43XV1dkaZp5PF4yOl00v7+Pi5wssQKHeMOC4DJORyO9Dbww8PDFI1GqbW1lQ4PD3EyzYLP+vV6veg3i+7u7qimpoaOj4+pvr6eZFmmSCRimhOpGVihYwwsABbgcDhoYWGBfD4fTUxM0NzcnOhIloJ+/y9N0ygSiVBjYyP19fXR8vKy6EiWY4WO8UgIwAKYmba2tiiVSlF7e7voOJaDfv+vx8dHKi0tpbW1NWppaREdx5Ks0DEGFgAAEC6RSFBeXp7oGJZm9o4xsAAAAIDh4TcsAAAAYHgYWAAAAMDwMLAAAACA4WFgAQAAAMPDwAIAAACGh4EFAAAADA8DCwAAABgeBhYAAAAwPAwsAAAAYHgYWAAAAMDwfgDkhTVQy96EeQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAHbCAYAAAAzs2v3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAACKhUlEQVR4nO3dd2AUZd4H8O9sTw9JSMOQoAZQyRGOEgELao4oRdFTih6I+mKFF0REQIo95hRfEDxz4CmcyoFY0EPFwwjeKREFzkpRkJAIqUDaJtn6vH/M7mQ3jYTMsinfz904M888M/vsJGR/+7SRhBACRERERJ2cxt8FICIiIlIDgxoiIiLqEhjUEBERUZfAoIaIiIi6BAY1RERE1CUwqCEiIqIugUENERERdQkMaoiIiKhLYFBDREREXQKDGiJq0bp16yBJEvLy8vxdlHbZuXMnJEnCzp07/V0UIvIRBjVE5BP79+/HY4891qGDoQ0bNmDFihX+LgYRqUTis5+IqCUOhwM2mw1GoxGSJLX6vLfffhu33HILduzYgVGjRvmugK3kdDphtVphMBig0cjf58aNG4cff/yxQwdeRNR6On8XgIg6Nq1WC61W6+9itJtGo4HJZPJ3MYjIh9j8REQtatinJikpCePGjcMXX3yBYcOGwWQy4fzzz8ff//53r3NuueUWAMBVV10FSZIa9Wf5+OOPcfnllyMoKAghISEYO3YsfvrpJ6/Xnj59OoKDg3H8+HFMmDABwcHB6NmzJ+bNmweHw+GVd+PGjRg8eDBCQkIQGhqKlJQUrFy5UjnesE/NqFGj8OGHH+LYsWNK+ZKSklBdXY2goCDMnj270b347bffoNVqkZmZ2Z5bSkQ+wqCGiNrs8OHDuPnmm/GHP/wBy5cvR48ePTB9+nQlKLniiivwv//7vwCARYsW4fXXX8frr7+Oiy66CADw+uuvY+zYsQgODkZWVhaWLFmC/fv347LLLmvUFORwOJCRkYHIyEg8//zzuPLKK7F8+XKsWbNGybN9+3ZMmTIFPXr0QFZWFp599lmMGjUKX375ZbPv4dFHH0VqaiqioqKU8q1YsQLBwcG48cYbsWnTpkaB0z/+8Q8IIXDbbbepcRuJSG2CiKgFr732mgAgjh49KoQQIjExUQAQ//73v5U8JSUlwmg0ioceekhJ27x5swAgduzY4XW9qqoqER4eLmbMmOGVXlRUJMLCwrzSb7/9dgFAPPHEE155Bw0aJAYPHqzsz549W4SGhgq73d7s+9ixY0ej8owdO1YkJiY2yvvJJ58IAOLjjz/2Sv/d734nrrzyymZfg4j8izU1RNRmF198MS6//HJlv2fPnujXrx9+/fXXM567fft2lJeXY8qUKSgrK1MWrVaLtLQ07Nixo9E59957r9f+5Zdf7vVa4eHhMJvN2L59ezveVb309HTEx8fjzTffVNJ+/PFHfP/99/jTn/6kymsQkfrYUZiI2qx3796N0nr06IHTp0+f8dxffvkFAHD11Vc3eTw0NNRr32QyoWfPni2+1v3334+33noL1113HXr16oXRo0dj4sSJuPbaa89YnqZoNBrcdtttePnll1FTU4PAwEC8+eabMJlMSl8hIup4GNQQUZs1NxpKtGKGCKfTCUDuVxMbG9vouE7n/WepNSOvoqOj8e233+KTTz7Bxx9/jI8//hivvfYapk2bhvXr15/x/KZMmzYNzz33HLZs2YIpU6Zgw4YNGDduHMLCws7qekTkewxqiMgnmpvT5oILLgAgByLp6emqvZ7BYMD48eMxfvx4OJ1O3H///fjrX/+KJUuW4MILL2xTGQFgwIABGDRoEN58802cd955yM/Px6pVq1QrLxGpj31qiMgngoKCAADl5eVe6RkZGQgNDcUzzzwDm83W6LzS0tI2v9bJkye99jUaDX73u98BACwWS4tlrKioaPb41KlT8a9//QsrVqxAZGQkrrvuujaXjYjOHdbUEJFPpKamQqvVIisrCxUVFTAajbj66qsRHR2Nl19+GVOnTsXvf/97TJ48GT179kR+fj4+/PBDjBw5EqtXr27Ta/3P//wPTp06hauvvhrnnXcejh07hlWrViE1NVUZRt6UwYMHY9OmTZg7dy6GDh2K4OBgjB8/Xjl+6623Yv78+Xjvvfdw3333Qa/Xn/X9ICLfY00NEflEbGwssrOzUVJSgrvuugtTpkzB/v37AcjBQk5ODnr16oXnnnsOs2fPxsaNG5Gamoo77rijza/1pz/9CSaTCX/5y19w//33Y/369Zg0aRI+/vhj5ZEITbn//vtx66234rXXXsOtt96KWbNmeR2PiYnB6NGjAci1NkTUsfHZT0RELbjxxhvxww8/4PDhw/4uChGdAWtqiIiaUVhYiA8//JC1NESdBPvUEBE1cPToUXz55Zd45ZVXoNfrcc899/i7SETUCqypISJq4PPPP8fUqVNx9OhRrF+/vsn5dIio42GfGiIiIuoSWFNDREREXQKDGiIiIuoSGNQQERFRl8CghoiIiLoEBjVERETUJTCoISIioi6BQQ0RERF1CQxqiIiIqEtgUENERERdAoMaIiIi6hIY1BAREVGXwKCGiIiIugQGNURERNQlMKghIiKiLoFBDREREXUJDGqIiIioS2BQQ0RERF0CgxoiIiLqEhjUEBERUZfAoIaIiIi6BAY1RERE1CUwqCEiIqIugUENERERdQkMaoiIiKhLYFBDREREXQKDGiIiIuoSGNQQERFRl8CghoiIiLoEnb8LcK44nU6cOHECISEhkCTJ38UhIiKiVhBCoKqqCvHx8dBozlAXI3xk9erVIjExURiNRjFs2DCxe/fuFvO/9dZbol+/fsJoNIoBAwaIDz/80Ov47bffLgB4LRkZGa0uT0FBQaPzuXDhwoULFy6dYykoKDjjZ71Pamo2bdqEuXPnIjs7G2lpaVixYgUyMjJw6NAhREdHN8q/a9cuTJkyBZmZmRg3bhw2bNiACRMmYN++fRgwYICS79prr8Vrr72m7BuNxlaXKSQkBABQUFCA0NDQdrw7b1V1Npw0W2HUahAXHqDadYmIiAiorKxEQkKC8jneEkkIIdQuQFpaGoYOHYrVq1cDkJt+EhISMGvWLCxYsKBR/kmTJsFsNmPr1q1K2qWXXorU1FRkZ2cDAKZPn47y8nJs2bLlrMpUWVmJsLAwVFRUqBrU/D03D0vf/wljUmLxl9sGq3ZdIiIiatvnt+odha1WK/bu3Yv09PT6F9FokJ6ejtzc3CbPyc3N9coPABkZGY3y79y5E9HR0ejXrx/uu+8+nDx5Uu3it5lJpwUA1Nmcfi4JERFR96Z681NZWRkcDgdiYmK80mNiYnDw4MEmzykqKmoyf1FRkbJ/7bXX4qabbkKfPn1w5MgRLFq0CNdddx1yc3Oh1WobXdNiscBisSj7lZWV7XlbzTLq5bjQYnf45PpERETUOp1m9NPkyZOV7ZSUFPzud7/DBRdcgJ07d+Kaa65plD8zMxOPP/64z8tl1LmCGtbUEBER+ZXqQU1UVBS0Wi2Ki4u90ouLixEbG9vkObGxsW3KDwDnn38+oqKicPjw4SaDmoULF2Lu3LnKvrujkdqMelfzE2tqiIhU5XA4YLPZ/F0M8jG9Xt9ki8vZUD2oMRgMGDx4MHJycjBhwgQAckfhnJwczJw5s8lzhg8fjpycHMyZM0dJ2759O4YPH97s6/z22284efIk4uLimjxuNBrbNDrqbLGmhohIXUIIFBUVoby83N9FoXMkPDwcsbGx7Z5HzifNT3PnzsXtt9+OIUOGYNiwYVixYgXMZjPuuOMOAMC0adPQq1cvZGZmAgBmz56NK6+8EsuXL8fYsWOxceNG7NmzB2vWrAEAVFdX4/HHH8cf//hHxMbG4siRI5g/fz4uvPBCZGRk+OIttJrR1VHYYmdQQ0SkBndAEx0djcDAQE6Y2oUJIVBTU4OSkhIAaLaiorV8EtRMmjQJpaWlWLp0KYqKipCamopt27YpnYHz8/O9ZgUcMWIENmzYgMWLF2PRokVITk7Gli1blDlqtFotvv/+e6xfvx7l5eWIj4/H6NGj8eSTT56T2piWmFwdhetsbH4iImovh8OhBDSRkZH+Lg6dAwEB8hxvJSUliI6ObldTlE/mqemIfDVPzeGSaqS/8DnCAvT4btlo1a5LRNQd1dXV4ejRo0hKSlI+7Kjrq62tRV5eHvr06QOTyeR1zK/z1HQ3Sp8adhQmIlINm5y6F7V+3gxq2smkr598r5tUehERURtMnz5dGTjTFa1btw7h4eH+LgYABjXt5p58DwCsDnYWJiIibytXrsS6deuU/VGjRnmN9lXLvffeC0mSsGLFihbz7dy5E5IkqTa6bNKkSfj5559VuVZ7dZrJ9zoqd/MTII+Aco+GIiIiAoCwsDCfv8Z7772Hr776CvHx8apd02q1wmAwnDFfQEBAh+n/xJqadjJoNXA3BXIEFBFR9/X2228jJSUFAQEBiIyMRHp6Osxms1fz0/Tp0/H5559j5cqVkCQJkiQhLy8PAPDjjz/iuuuuQ3BwMGJiYjB16lSUlZWd8XWPHz+OWbNm4c0334Rer28xb15eHq666ioAQI8ePSBJEqZPnw5ArkGaOXMm5syZg6ioKGXKlBdeeAEpKSkICgpCQkIC7r//flRXVyvXbNj89NhjjyE1NRWvv/46kpKSEBYWhsmTJ6OqqqqVd/LsMahpJ0mSOAEfEVE3V1hYiClTpuDOO+/EgQMHsHPnTtx0002N+lquXLkSw4cPx4wZM1BYWIjCwkIkJCSgvLwcV199NQYNGoQ9e/Zg27ZtKC4uxsSJE1t8XafTialTp+Lhhx/GJZdccsZyJiQk4J133gEAHDp0CIWFhVi5cqVyfP369TAYDPjyyy+RnZ0NQH4o9YsvvoiffvoJ69evx2effYb58+e3+DpHjhzBli1bsHXrVmzduhWff/45nn322TOWr73Y/KQCo06LOpuTE/AREfmAEAK1fqgJD9BrWz0qp7CwEHa7HTfddBMSExMByM8pbCgsLAwGgwGBgYFejwJavXo1Bg0ahGeeeUZJe/XVV5GQkICff/4Zffv2bfJ1s7KyoNPp8L//+7+tKqdWq0VERAQAIDo6ulEH3+TkZPz5z3/2SvPs/5OUlISnnnoK9957L/7yl780+zpOpxPr1q1DSEgIAGDq1KnIycnB008/3apyni0GNSpw19Sw+YmISH21NgcuXvrJOX/d/U9kINDQuo/JgQMH4pprrkFKSgoyMjIwevRo3HzzzejRo0erzv/uu++wY8cOBAcHNzp25MgRfPPNN7jnnnuUtI8//hiBgYFYuXIl9u3b12zwdd111+E///kPACAxMRE//fRTi+UYPHhwo7RPP/0UmZmZOHjwICorK2G321FXV4eamhoEBgY2eZ2kpCQloAHkmYLdswb7EoMaFbiHdbOmhoioe9Jqtdi+fTt27dqFf/3rX1i1ahUeffRR7N69u1XnV1dXY/z48cjKymp0LC4uDk6nE2lpaUpar1698Ne//hUlJSXo3bu3ku5wOPDQQw9hxYoVyMvLwyuvvILa2loAOGN/GwAICgry2s/Ly8O4ceNw33334emnn0ZERAS++OIL3HXXXbBarc0GNQ1fS5IkOJ2+/4xkUKMCTsBHROQ7AXot9j9x7p/zF6Bv22hWSZIwcuRIjBw5EkuXLkViYiLee++9RvkMBgMcDu/Pi9///vd45513kJSUBJ2u6Y9mz5oPQG7SSU9P90rLyMjA1KlTlWct9urVq8nXB9CoDE3Zu3cvnE4nli9frjze6K233jrjef7CoEYF7rlq2FGYiEh9kiS1uhnIX3bv3o2cnByMHj0a0dHR2L17N0pLS3HRRRfh+++/98qblJSE3bt3Iy8vD8HBwYiIiMADDzyAtWvXYsqUKZg/fz4iIiJw+PBhbNy4Ea+88kqTz0OKjIxs9HwsvV6P2NhY9OvXr9myJiYmQpIkbN26FWPGjEFAQECTzV4AcOGFF8Jms2HVqlUYP368Vwfijoijn1RgUp7UzZoaIqLuKDQ0FP/+978xZswY9O3bF4sXL8by5ctx3XXXNco7b948aLVaXHzxxejZsyfy8/MRHx+PL7/8Eg6HA6NHj0ZKSgrmzJmD8PBwrwdAq6FXr154/PHHsWDBAsTExGDmzJnN5h04cCBeeOEFZGVlYcCAAXjzzTeRmZmpannUxAdaquC2V77Cl4dPYuXkVNyQ2riqj4iIWsf9QMumHmxIXVdLP3c+0PIcc88izOYnIiIi/2FQowKTq09NHZufiIiI/IZBjQpYU0NEROR/DGpUwCHdRERE/segRgX1MwqzpoaIiMhfGNSooH5GYdbUEBGpoZsMzCUXtX7eDGpUUN/8xJoaIqL2cE+vX1NT4+eS0Lnk/nm35lEOLenYUzR2EkZXTQ0faElE1D5arRbh4eHKww8DAwNb/aRs6nyEEKipqUFJSQnCw8ObnDm5LRjUqIA1NURE6omNjQWAc/JUZ+oYwsPDlZ97ezCoUYG7poZDuomI2k+SJMTFxSE6Oho2m83fxSEf0+v17a6hcWNQowJl9BM7ChMRqUar1ar2YUfdAzsKq8DEmhoiIiK/Y1CjAk6+R0RE5H8MalTAyfeIiIj8j0GNCjj5HhERkf8xqFEBh3QTERH5H4MaFbif0s3mJyIiIv9hUKMCk54dhYmIiPyNQY0KlMn32PxERETkNwxqVODuU2O1O+F08smyRERE/sCgRgXuoAYArA7W1hAREfkDgxoVuId0A5xVmIiIyF8Y1KhAp5GgkeRtPv+JiIjIPxjUqECSJGVYN2tqiIiI/INBjUo4rJuIiMi/GNSohBPwERER+ZfO3wXoKoysqSGiTszhFLDanfLicC12p3ea3QmbwwmdRoJep4Feq4FBq4FBJ8nbOg0C9ToEGbXQafmdmc49BjUqMek4AR8RtZ0QAjaHQJ3dgTqrA3U2J2ptDtTZHMpaXurTLXYnbB6BhmcAYvMMSBwCVrvDlS688lrsTljtDjnd4YRD5Tm2AvRaBJt0CDHqEGzSIdioQ3igHpFBRkQGGxAZbERUkAFRIUbEhZkQE2qCnoEQtRODGpW4a2rqbKypIeoKnE4Bi10OJJQgw+oZbLiCDKvDKwiptTlgsTlRa3V4nesZmNRaHbDYXdezqx9QqMGgc9fC1K/1WrlGxukKxNxBks1RH2TZHPJ7cb/30ipLq15PkoCewUbEhQegV7gJF/QMRnJMCPrGBOP8qGAYdAx46MwY1KiET+omUo+79sJil2slLHYnLDbv7bqGaXY5mFC27U5YbE7UKenNXMvucH0oyzUW7g9nf/xbliS5hiNAr4VJr4VJr4HJa19OM+g0MLqCDXezj7J4BCJexxpsu48ZPdOUYxIkSTqr92CxO2C2OGC22FFVZ0e1xY5qiw1VdXacNltx0mxFWbUVp8wWnKy2oriqDkUVdbA5BEqqLCipsuC7Au9rajUSkiID0TcmRAl0+sWEoE9UEJu5yAuDGpWYlOc/saaGugZ3H4s6JQCoDxTqA4T6QMIrXxPBhcUu11R4BRct5O1IlRcGnQYmnQYBBu8AI0CvVdKMeo1XQBJg0HoEJJpGgUmAQQuTTqusTQY5oDjbYKKjMOq0MOq0iAgytPocp1OgzGxBYXkdCitq8dvpWvxSXI1fSqrwS3E1qix2HCk140ipGR//WOTxWhr0jw3BxfGhuDguFBfHh6F/bAiCjPxo6674k1eJu6aGo5+otZxOAZvTqfR3sHl0xFT2vdKcsNqFsi0fF/XV/naPNIf3Od7HG7xGs7UXHSeqcAcVRr0WRlftglEnBxLKtuu4nM8jrYl8Jvd19J5NK/VrvVbyCkK0ms4daHR0Go2E6BATokNMGJgQ7nVMCIGiyjr8XFyNX4qr8HNxlbJttjrw3W8V+O63CiW/JAFJkUFIigxEYmQQEiICkRgRiN6RgUjoEYgAgxbUdfksqHnppZfw3HPPoaioCAMHDsSqVaswbNiwZvNv3rwZS5YsQV5eHpKTk5GVlYUxY8Yox4UQWLZsGdauXYvy8nKMHDkSL7/8MpKTk331FtrEPaT75+IqlFVbEBVs9HOJui9304VnkGB1NB04WBt90Hs0Q3idL5R+Aw0DB6889gYBR6OgpL4c9o5UFXEGOo2kBA3uoEIJDJoMLjzTPQKRZoISr2s1CEoMWg00DCq6LUmSEBcWgLiwAFzZt6eS7nQKHDtVg/0nKrG/sAI/najE/hOVKKmy4GiZGUfLzABKG12vZ4gRsaEm9AwxIirYgJ4hRvQMNqJniAlRwQZEBhsQHmhAeICeTVudkCSEUP0v66ZNmzBt2jRkZ2cjLS0NK1aswObNm3Ho0CFER0c3yr9r1y5cccUVyMzMxLhx47BhwwZkZWVh3759GDBgAAAgKysLmZmZWL9+Pfr06YMlS5bghx9+wP79+2Eymc5YpsrKSoSFhaGiogKhoaFqv2UseOd7bPymviE4NtSE83oEeH37M7o62nl+KzR4fTt0p0mNvjl6toNrJPkfukYCNJIEqcFa4zrmmUc5rvE4D/V5dBoNtFoJWkmCViPJj344yw8Se4PhoJYGozOaOlZnc8j9JGzydp3dY1vpF9H8cc+aiY5Uw9AWWo0k/340+J1wd85UfheUvg+SR576YbXeeT3yeJzjdS2txisoMem9gwuDVsM/7tRplFZZcKioCvmnalyLGfmnanDsZA2q6uxtulaoSYeIIDnIkdd6RAQa0CPIgB6BBkQE6b2O9Qg0cASXD7Tl89snQU1aWhqGDh2K1atXAwCcTicSEhIwa9YsLFiwoFH+SZMmwWw2Y+vWrUrapZdeitTUVGRnZ0MIgfj4eDz00EOYN28eAKCiogIxMTFYt24dJk+efMYy+Tqo+bW0Gi/m/ILvj1fgaJkZ6t9VOluSBDlQ8AwW3PNqNBE4GDyCAHeQ0TDorD/eVODgHWDUByL1r+nZzOHOwyYOIt8RQqCi1oaCU7UoqapDWbUFpVXyUlZtlberLThltqKi1nbWrxNi1LmCHr0S/MiLHiEmHYKM8vD2IKPntlZJY1DUWFs+v1VvfrJardi7dy8WLlyopGk0GqSnpyM3N7fJc3JzczF37lyvtIyMDGzZsgUAcPToURQVFSE9PV05HhYWhrS0NOTm5rYqqPG183sGY8XkQQCAaosdBworcbLaIs8n4dHMYXONqvBqtrA3bqpoONmV5zUEAKcQ8uKU/7E6BSDgWrv25eMCwr3tWtfvC593xlQCCo/RGk2N1DDpXZ0lXaM9TK6Ol3KaZ7pnWn2tQqNhpx7BCYMFIpIkSW5WCjQACGsxr93hREWtDadrbDhdY8Vps1Ve19iU7VNmG8prrDjlOl5ea4MQQJXFjiqLHfmnzq6cBp1GCXSCDHLQE2DQttzMqzTpaqDVaKDV1NfQa1217nItPJpIq8+n1QBCAAJQvpgLIZR9Afmg13EIj3ME4sMD0Dcm5OzevApUD2rKysrgcDgQExPjlR4TE4ODBw82eU5RUVGT+YuKipTj7rTm8jRksVhgsdTPj1BZWdm2N9IOwUYdhiZFnLPXay8hBBxOAbtTXjuEgN0hcLaVeFqNpAQZWs3ZDw0lIvIHnVaDyGAjItvQN9LhFKisdQVBNVacNttwqsYqBz6uAMg9xN1sca2tdpgtDlRb7LC6phCw2p04ZbfilNlX7863bk3rjWduTPHb63fZ0U+ZmZl4/PHH/V2MTkGSJOi0EnQcFEBEdFa0GklubmrDUHZPNoezPthxBTpm11LXcOqDZqZBsNjlAQhO15dTh6um3v1l1dlg7XCiUZq7ryUASK7/uNPcX08lCZAg99NEg/zxYWfu4+pLqgc1UVFR0Gq1KC4u9kovLi5GbGxsk+fExsa2mN+9Li4uRlxcnFee1NTUJq+5cOFCryatyspKJCQktPn9EBER+Zpeq/FoHqOzpXpQYzAYMHjwYOTk5GDChAkA5I7COTk5mDlzZpPnDB8+HDk5OZgzZ46Stn37dgwfPhwA0KdPH8TGxiInJ0cJYiorK7F7927cd999TV7TaDTCaKyvOnQ3pZzLZigiIiJqH/fndqu6RAgf2LhxozAajWLdunVi//794u677xbh4eGiqKhICCHE1KlTxYIFC5T8X375pdDpdOL5558XBw4cEMuWLRN6vV788MMPSp5nn31WhIeHi/fff198//334oYbbhB9+vQRtbW1rSpTQUGBgKsvExcuXLhw4cKlcy0FBQVn/Kz3SZ+aSZMmobS0FEuXLkVRURFSU1Oxbds2paNvfn4+NJr6YWsjRozAhg0bsHjxYixatAjJycnYsmWLMkcNAMyfPx9msxl33303ysvLcdlll2Hbtm2tmqMGAOLj41FQUICQkBDVO666m7YKCgp8Mly8u+P99S3eX9/jPfYt3l/f8+c9FkKgqqoK8fHxZ8zrk3lquhtfz4HT3fH++hbvr+/xHvsW76/vdZZ7zFl+iIiIqEtgUENERERdAoMaFRiNRixbtsxrtBWph/fXt3h/fY/32Ld4f32vs9xj9qkhIiKiLoE1NURERNQlMKghIiKiLoFBDREREXUJDGqIiIioS+iyT+luyOl04sSJEz6ZUZiIiIh8w3NGYc+nETSl2wQ1J06c4FO6iYiIOqmCggKcd955LebpNkFNSEgIAPDZIERERJ2I+7lT7s/xlnSboMbd5BQaGqpuUHP4U2DvOqDXYOCyB9W7LhERESla03Wk2wQ1PlNeABz4J+B0+rskRERE3RpHP7WXziSv7XX+LQcREVE3x6CmvXSu52DYLf4tBxERUTfH5qf20gfIa3utf8tBRNTFOBwO2Gw2fxeDfEyv10Or1apyLQY17cWaGiIiVQkhUFRUhPLycn8Xhc6R8PBwxMbGtnseOQY17cU+NUREqnIHNNHR0QgMDOSEqV2YEAI1NTUoKSkBAMTFxbXregxq2ssd1NgY1BARtZfD4VACmsjISH8Xh86BgAC5G0dJSQmio6Pb1RTFjsLtxZoaIiLVuPvQBAYG+rkkdC65f97t7UPFoKa92KeGiEh1bHLqXtT6eTOoaS+OfiIiIuoQGNS0l7v5yWkHHHb/loWIiDqc6dOnY8KECf4uhs/s3LkTkiR1iNFqDGray938BAAONkEREZG3lStXYt26dcr+qFGjMGfOHFWu/e6772L06NGIjIyEJEn49ttvz3hOXl5eq/O2xogRI1BYWIiwsDBVrtceDGray11TA3AEFBERNRIWFobw8HCfXNtsNuOyyy5DVlaW6te2Wq2tymcwGFSZY0YNDGraS6MFNHp5myOgiIi6rbfffhspKSkICAhAZGQk0tPTYTabvZqfpk+fjs8//xwrV66EJEmQJAl5eXkAgB9//BHXXXcdgoODERMTg6lTp6KsrKzF15w6dSqWLl2K9PT0VpezT58+AIBBgwZBkiSMGjVKKduECRPw9NNPIz4+Hv369QMAvP766xgyZAhCQkIQGxuLW2+9VZlXBmjc/LRu3TqEh4fjk08+wUUXXYTg4GBce+21KCwsbHUZzxaDGjVwWDcRke8IAVjN534RotVFLCwsxJQpU3DnnXfiwIED2LlzJ2666SaIBtdYuXIlhg8fjhkzZqCwsBCFhYVISEhAeXk5rr76agwaNAh79uzBtm3bUFxcjIkTJ6p9N/H1118DAD799FMUFhbi3XffVY7l5OTg0KFD2L59O7Zu3QpAHmb95JNP4rvvvsOWLVuQl5eH6dOnt/gaNTU1eP755/H666/j3//+N/Lz8zFv3jzV30tDnHxPDXoTYK1iUENE5Au2GuCZ+HP/uotOAIagVmUtLCyE3W7HTTfdhMTERABASkpKo3xhYWEwGAwIDAxEbGyskr569WoMGjQIzzzzjJL26quvIiEhAT///DP69u3bzjdTr2fPngCAyMhIrzIAQFBQEF555RUYDAYl7c4771S2zz//fLz44osYOnQoqqurERwc3ORr2Gw2ZGdn44ILLgAAzJw5E0888YRq76E5rKlRA2tqiIi6tYEDB+Kaa65BSkoKbrnlFqxduxanT59u9fnfffcdduzYgeDgYGXp378/AODIkSN48803vY795z//adV17733Xq/zziQlJcUroAGAvXv3Yvz48ejduzdCQkJw5ZVXAgDy8/ObvU5gYKAS0ADy4w88m6x8hTU1auAEfEREvqMPlGtN/PG6raTVarF9+3bs2rUL//rXv7Bq1So8+uij2L17d6vOr66uxvjx45vs8BsXFwen04m0tDQlrVevXq267hNPPNGmZp+gIO+aKbPZjIyMDGRkZODNN99Ez549kZ+fj4yMjBY7Euv1eq99SZIaNcX5AoMaNehcE/DZOAEfEZHqJKnVzUD+JEkSRo4ciZEjR2Lp0qVITEzEe++91yifwWCAw+HwSvv973+Pd955B0lJSdDpmv5oDgkJaXOZoqOjER0d3ej1ATQqQ1MOHjyIkydP4tlnn0VCQgIAYM+ePW0ux7nC5ic1sKaGiKhb2717N5555hns2bMH+fn5ePfdd1FaWoqLLrqoUd6kpCTs3r0beXl5KCsrg9PpxAMPPIBTp05hypQp+Oabb3DkyBF88sknuOOOO1oMPk6dOoVvv/0W+/fvBwAcOnQI3377LYqKipo9Jzo6GgEBAUpn5IqKimbz9u7dGwaDAatWrcKvv/6KDz74AE8++WQb7sy55bOg5qWXXkJSUhJMJhPS0tKU3tbN2bx5M/r37w+TyYSUlBR89NFHyjGbzYZHHnkEKSkpCAoKQnx8PKZNm4YTJ/xQHdkU9qkhIurWQkND8e9//xtjxoxB3759sXjxYixfvhzXXXddo7zz5s2DVqvFxRdfrDTnxMfH48svv4TD4cDo0aORkpKCOXPmIDw8HBpN8x/VH3zwAQYNGoSxY8cCACZPnoxBgwYhOzu72XN0Oh1efPFF/PWvf0V8fDxuuOGGZvP27NkT69atw+bNm3HxxRfj2WefxfPPP9+GO3NuScIHjVybNm3CtGnTkJ2djbS0NKxYsQKbN2/GoUOHGlWDAcCuXbtwxRVXIDMzE+PGjcOGDRuQlZWFffv2YcCAAaioqMDNN9+MGTNmYODAgTh9+jRmz54Nh8PR6mqwyspKhIWFoaKiAqGhoeq+4Tf+CBz+FJjwMpB6q7rXJiLqRurq6nD06FH06dMHJpPpzCdQl9DSz70tn98+CWrS0tIwdOhQrF69GgDgdDqRkJCAWbNmYcGCBY3yT5o0CWazWRkTDwCXXnopUlNTm402v/nmGwwbNgzHjh1D7969z1gmnwY1G28DDm4Fxv0fMOTOM+cnIqImMajpntQKalRvfrJardi7d6/X7IYajQbp6enIzc1t8pzc3NxGsyFmZGQ0mx8AKioqIEmSz6aebhP2qSEiIvI71Uc/lZWVweFwICYmxis9JiYGBw8ebPKcoqKiJvM319Gprq4OjzzyCKZMmdJs1GaxWGCx1AcZlZWVbXkbbcPRT0RERH7X6UY/2Ww2TJw4EUIIvPzyy83my8zMRFhYmLK4h6L5BGtqiIiI/E71oCYqKgparRbFxcVe6cXFxY2mY3aLjY1tVX53QHPs2DFs3769xba1hQsXoqKiQlkKCgrO8h21gjL6iTU1RERE/qJ6UGMwGDB48GDk5OQoaU6nEzk5ORg+fHiT5wwfPtwrPwBs377dK787oPnll1/w6aefIjIyssVyGI1GhIaGei0+o3cHNaypISJSw7mYfZY6DrV+3j6ZUXju3Lm4/fbbMWTIEAwbNgwrVqyA2WzGHXfcAQCYNm0aevXqhczMTADA7NmzceWVV2L58uUYO3YsNm7ciD179mDNmjUA5IDm5ptvxr59+7B161Y4HA6lv01ERESj51Scc5ynhohIFe7p9WtqahAQEODn0tC5UlNTA6Dx4xXayidBzaRJk1BaWoqlS5eiqKgIqamp2LZtm9IZOD8/32syoREjRmDDhg1YvHgxFi1ahOTkZGzZsgUDBgwAABw/fhwffPABACA1NdXrtXbs2IFRo0b54m20nrtPjY1BDRFRe2i1WoSHhysPPwwMDIQkSX4uFfmKEAI1NTUoKSlBeHg4tFptu67nk3lqOiKfzlOzew3w8cPAxROAievVvTYRUTcjhEBRURHKy8v9XRQ6R8LDwxEbG9tkANuWz28+0FINHP1ERKQaSZIQFxeH6Oho2Gw2fxeHfEyv17e7hsaNQY0aOPqJiEh1Wq1WtQ876h463Tw1HRJHPxEREfkdgxo1cPQTERGR3zGoUQNHPxEREfkdgxo1uJ/9xJoaIiIiv2FQowaOfiIiIvI7BjVq4OgnIiIiv2NQowaOfiIiIvI7BjVq8Bz91D0maCYiIupwGNSowd2nRjgBB2e/JCIi8gcGNWrQeTxJliOgiIiI/IJBjRrcNTUA+9UQERH5CYMaNUgSoHUP6+YIKCIiIn9gUKMWjoAiIiLyKwY1auHzn4iIiPxK5+8CdBl8/hMRNSQE4LTLi8PmsXZv2+Vt9zHP4w7XfquO2+uvKZz1r11fEO8ynTEd8t80nanBOqCZdI+1IRDQBwE6g9p3k+iMGNSohc9/IvIPIeQPenstYHMt9jrAViN/ybDVuo650ryO1chNxkrg4DhDENHGYMRp9/fd8R+NXg5wDMGAPhAwBNUvele6IdB72xAkB0SGQFeg5A6WXAGT1uCdpjUCGjY4UD0GNWrh85+IvDlsQF2FvFgq5SDCXiv/G7G51u5gw+6xtCafO487iBEOf7/bttHoAa1eXmu09dtaHaDReWzr5X1tg7Wy7U73yKvReryQ5LEpNSpG43T3tjtQrPO475ZWrGvrAzmnx8/fl/RBQFCUHBBJGsAUDgT3BIKi5XVgpGuJqt8O6CHfM+py+FNVC5//RF2BrRZwWAFjqPyhZqmsD0rqKoA697q8/gOrrgKobbBfVy7XgpxzkvzNX2+S1zpTg+2Gx1zNKVqDHAx4BhteAUbDYMIzwGhjAKLRNh9gdAUOG2A1y4utBrBWA9Ya174r3epKt9XU522Y324BHJ4Bk6U+gPJsNrOZgXJzGwspyYFN2HlAeG8gLAEIT5C3Iy4AIi/wnqqDOg0GNWrh6CfqKJxOObi2VMvBiKVSDkYsVa79Kte+x7G6cqDiN+DkEcgfGBK8PjjawxgKGENcAYSpftGbmt9vdMzdl8O1VoKTADnNHahoDV07YOgMtHogIFxefMHdT8leB9itgKUCqC511RI5gNrTgLkUqC6W1zWngJqT9UvtaQACqD0lL0XfN34NSQtE9AGi+gE9+8rryAuBqAvlYIg6LAY1auHoJ2oLIeQaEavZ9c3U3GC7xvWt1nNd0+AbrXvdII8qNSQeAY0hBDCFysGJKVSu3jeFyR9apjCPxWPffcwY2qA5hKidJEkOnLR6wAggKBKIOL/15zvscmBTXSwH8hUFQHm+vD59DDh5WA72Tx6Wl0Mfep8fGFUf4EReCEQmy+uIPqzdMZfJTYF+xKBGLcroJzY/dVlOpxw8WKo8lsoG+55p1U0ELe796nPQiVSqD0SMIfU1Jl77HoFKcDQQc4mcVlchj15hUEJdjVYn97UJ7gnEDmh8XAigqhAoPQSU/Vy/PnkEqDoB1JTJS8FX3udJGrn5KvQ8ICQGCI71WLuWwCg54O+K/6Z+/RzYMAn4w+PAsLv9VmPKoEYt7tFP/30dCI0Hki6XPyzIv5xOj3Z6V0CiBBeuwMMrEGkqSPFY1GqS8aQzeYwMcY8UCawfBeIeOdIw/Ux59YFnPzLE3ZxK1N1Ikvw3PDQeuOAq72OW6voanJOHgbJf6ret1cDpPHlp+QVczXMRro7LEXKwE+TRmTkoyrXdo74GtCMFQk4nAFFfpppTwHv3yE2AJfv92gQsCdFwcgJ1vPTSS3juuedQVFSEgQMHYtWqVRg2bFiz+Tdv3owlS5YgLy8PycnJyMrKwpgxY5TjQggsW7YMa9euRXl5OUaOHImXX34ZycnJrSpPZWUlwsLCUFFRgdBQHwQbP74DvHt3/bdvSeOqkjzf9eGiBfpcCVx8vfwLSs2zW7w7n1ob1nq4A5TqZvYb1I6oHYhI2sa1H8risW8IlgMMY3D9tjtwUYa2BnEUBlFnJwRQVQScOgJUFspNW9VFQFWDdXtGghlD65t3tQZXE5zBo2O7x757W0lvkFcudH3ZvbZdJI1rkeR+d5XHgeIfgbLDcvAiaeRaqKhk+Yvjb98AUX2Bu3fKf9tU1JbPb58ENZs2bcK0adOQnZ2NtLQ0rFixAps3b8ahQ4cQHR3dKP+uXbtwxRVXIDMzE+PGjcOGDRuQlZWFffv2YcAAuXowKysLmZmZWL9+Pfr06YMlS5bghx9+wP79+2Eynflbpc+DGkBun/3qZeDAP4HyY03nkTRy7/ro/kD0xfK3AUOwx4dhsKtPQoRriKKPIl6nU+40pw+QX7Mh99wf7vk2bLXyL7WlSv7moDPWd9Sz18n9QzxHKTgs9cea6v9hq226v4itRr6W6iTXfQ72CCyCGwQkTQQmTQUvOhM7oxJR2zlscn8ez87LtafkvijufbOrecvsOuaXUYRnQWsA/udTIG6g6pf2e1CTlpaGoUOHYvXq1QAAp9OJhIQEzJo1CwsWLGiUf9KkSTCbzdi6dauSdumllyI1NRXZ2dkQQiA+Ph4PPfQQ5s2bBwCoqKhATEwM1q1bh8mTJ5+xTOckqPF6wUKg9CBw+mj9L/JPW4DSA62/htYoV03qjHLtgEbrWuvkZgXPNOGU5+pwOuS1EB7bzvptp0MONOoq6muVjKEApPpJxJy2+llJ/UZyBRRh9cGeVzDiWfMR0qBGpIkaEn0gAxEi6nzsVrlZXJk2odxjoker3PHZYZUXp8e2O909SaTD6lrbPP4WSh4r97ZUX3vj/hwxBst9gnq6voybwuTXqjgOFP4XOL4P6JsBXHyDT25BWz6/Va/3tlqt2Lt3LxYuXKikaTQapKenIzc3t8lzcnNzMXfuXK+0jIwMbNmyBQBw9OhRFBUVIT09XTkeFhaGtLQ05ObmtiqoOedC4+QFHm2yoxa4gp0DQIlrqTnZuN9GXYVc0+GwyB3WfM1S2YpMEhASJwcateXyPxCv2T5ds3s2mv3TeIY+Is30FTGGcqZQIiKdAdBF+X1UUZNCYoHzBgND/V2QeqoHNWVlZXA4HIiJifFKj4mJwcGDB5s8p6ioqMn8RUVFynF3WnN5GrJYLLBY6ueMqaxszQf3OeAOdi64uvk8QshVju7qSKUZyKO2xbP2RTg92j+1rlqcBjU5ksY1uZhWDjiMoUBwjPw61SXy6yqTjbknEtM2mDiMQQYREXVcXbaHYmZmJh5//PFG6R0muGkNTTgQHO6765tdbbWGBv2cBACHa4ETgMW1EBERnVvuz+3W9JZRPaiJioqCVqtFcXGxV3pxcTFiY2ObPCc2NrbF/O51cXEx4uLivPKkpqY2ec2FCxd6NWkdP34cF198MRISEtr8noiIiMi/qqqqEBbW8uhh1YMag8GAwYMHIycnBxMmTAAgdxTOycnBzJkzmzxn+PDhyMnJwZw5c5S07du3Y/jw4QCAPn36IDY2Fjk5OUoQU1lZid27d+O+++5r8ppGoxFGY/3sjsHBwSgoKEBISAgklTuMVlZWIiEhAQUFBeemE3I3w/vrW7y/vsd77Fu8v77nz3sshEBVVRXi4+PPmNcnzU9z587F7bffjiFDhmDYsGFYsWIFzGYz7rjjDgDAtGnT0KtXL2RmZgIAZs+ejSuvvBLLly/H2LFjsXHjRuzZswdr1qwBAEiShDlz5uCpp55CcnKyMqQ7Pj5eCZzORKPR4LzzzvPF21WEhobyH5QP8f76Fu+v7/Ee+xbvr+/56x6fqYbGzSdBzaRJk1BaWoqlS5eiqKgIqamp2LZtm9LRNz8/HxqPTqcjRozAhg0bsHjxYixatAjJycnYsmWLMkcNAMyfPx9msxl33303ysvLcdlll2Hbtm2tmqOGiIiIuj6fzSjcnZzzOXC6Gd5f3+L99T3eY9/i/fW9znKPOUZXBUajEcuWLfPqw0Pq4f31Ld5f3+M99i3eX9/rLPeYNTVERETUJbCmhoiIiLoEBjVERETUJTCoISIioi6BQQ0RERF1CQxqiIiIqEvosg+0bMjpdOLEiRM+eUwCERER+YbnYxI8J+5tSrcJak6cOMGHWRIREXVSBQUFZ3zcUbcJakJCQgBA9Ydxna47jUJzIQJ1gUgKS1LtukRERFT/ME3353hLuk1Q425yUvthXB+d+AhP734af0j8A14Y9YJq1yUiIqJ6rek6wo7C7WTSyQ/UrLXX+rkkRERE3RuDmnZyBzV19jo/l4SIiKh7Y1DTTgHaAAAMaoiIiPyt2/Sp8RWlpsbBoIaISE0OhwM2m83fxSAf0+v10Gq1qlyLQU07sU8NEZG6hBAoKipCeXm5v4tC50h4eDhiY2PbPY8cg5p2MmnZp4aISE3ugCY6OhqBgYGcMLULE0KgpqYGJSUlAIC4uLh2XY9BTTsF6Fx9atj8RETUbg6HQwloIiMj/V0cOgcCAuTP0ZKSEkRHR7erKYodhdvJc/STEMLPpSEi6tzcfWgCAwP9XBI6l9w/7/b2oWJQ007uoMYhHLA77X4uDRFR18Amp+5FrZ83g5p2cg/pBoBaBzsLExGRt+nTp2PChAn+LobPrFu3DuHh4f4uBgAGNe2m0+igleT2P3YWJiKihlauXIl169Yp+6NGjcKcOXPafV2bzYZHHnkEKSkpCAoKQnx8PKZNm4YTJ060eN7OnTshSZJqo8smTZqEn3/+WZVrtReDmnaSJImzChMRUbPCwsJ8UpNRU1ODffv2YcmSJdi3bx/effddHDp0CNdff70q17dara3KFxAQgOjoaFVes70Y1KjAPaybc9UQEXVfb7/9NlJSUhAQEIDIyEikp6fDbDZ7NT9Nnz4dn3/+OVauXAlJkiBJEvLy8gAAP/74I6677joEBwcjJiYGU6dORVlZWbOvFxYWhu3bt2PixIno168fLr30UqxevRp79+5Ffn5+k+fk5eXhqquuAgD06NEDkiRh+vTpAOQapJkzZ2LOnDmIiopCRkYGAOCFF15QaoMSEhJw//33o7q6Wrlmw+anxx57DKmpqXj99deRlJSEsLAwTJ48GVVVVWd5Z1uPQY0KOKswEVH3VlhYiClTpuDOO+/EgQMHsHPnTtx0002NRsWuXLkSw4cPx4wZM1BYWIjCwkIkJCSgvLwcV199NQYNGoQ9e/Zg27ZtKC4uxsSJE9tUjoqKCkiS1GzNUEJCAt555x0AwKFDh1BYWIiVK1cqx9evXw+DwYAvv/wS2dnZAACNRoMXX3wRP/30E9avX4/PPvsM8+fPb7EcR44cwZYtW7B161Zs3boVn3/+OZ599tk2vZez4bOg5qWXXkJSUhJMJhPS0tLw9ddft5h/8+bN6N+/P0wmE1JSUvDRRx8px8623fBccc9VY7Fb/FwSIqKuRwiBGlvNOV/aMk1HYWEh7HY7brrpJiQlJSElJQX3338/goODvfKFhYXBYDAgMDAQsbGxiI2NhVarxerVqzFo0CA888wz6N+/PwYNGoRXX30VO3bsaHV/lbq6OjzyyCOYMmUKQkNDm8yj1WoREREBAIiOjkZsbCzCwsKU48nJyfjzn/+Mfv36oV+/fgCAOXPm4KqrrkJSUhKuvvpqPPXUU3jrrbdaLIvT6cS6deswYMAAXH755Zg6dSpycnJa9T7awyeT723atAlz585FdnY20tLSsGLFCmRkZODQoUNNtrvt2rULU6ZMQWZmJsaNG4cNGzZgwoQJ2LdvHwYMGODVbjhw4ECcPn0as2fPxvXXX489e/b44i20iTKrMGtqiIhUV2uvRdqGtHP+urtv3Y1Afevmyxk4cCCuueYapKSkICMjA6NHj8bNN9+MHj16tOr87777Djt27GgUBAFyrcc333yDe+65R0n7+OOPcfnllyv7NpsNEydOhBACL7/8spJ+3XXX4T//+Q8AIDExET/99FOL5Rg8eHCjtE8//RSZmZk4ePAgKisrYbfbUVdXh5qammbnE0pKSkJISIiyHxcXp8wa7Es+CWpeeOEFzJgxA3fccQcAIDs7Gx9++CFeffVVLFiwoFH+lStX4tprr8XDDz8MAHjyySexfft2rF69GtnZ2Uq7oafVq1dj2LBhyM/PR+/evX3xNlqNz38iIuretFottm/fjl27duFf//oXVq1ahUcffRS7d+9u1fnV1dUYP348srKyGh2Li4uD0+lEWlp9YNerVy9l2x3QHDt2DJ999plXLc0rr7yC2lr5s0mv15+xHEFBQV77eXl5GDduHO677z48/fTTiIiIwBdffIG77roLVqu12aCm4WtJkgSn03nG128v1YMaq9WKvXv3YuHChUqaRqNBeno6cnNzmzwnNzcXc+fO9UrLyMjAli1bmn2dM7Ubnksc/URE5DsBugDsvrV1wYHar9sWkiRh5MiRGDlyJJYuXYrExES89957jfIZDAY4HA6vtN///vd45513kJSUBJ2u6Y9mz5oPN3dA88svv2DHjh2NHi3hGfx4vj6ARmVoyt69e+F0OrF8+XJoNHKPlTM1PfmT6kFNWVkZHA4HYmJivNJjYmJw8ODBJs8pKipqMn9RUVGT+VvTbmixWGCx1PdxqaysbMvbaBPl+U8MaoiIVCdJUqubgfxl9+7dyMnJwejRoxEdHY3du3ejtLQUF110Eb7//nuvvElJSdi9ezfy8vIQHByMiIgIPPDAA1i7di2mTJmC+fPnIyIiAocPH8bGjRvxyiuvNPk8JJvNhptvvhn79u3D1q1b4XA4lM/NiIgIJXhpKDExEZIkYevWrRgzZgwCAgKabPYCgAsvvBA2mw2rVq3C+PHjvToQd0SdbvRTc+2GDWVmZiIsLExZEhISfFYm9qkhIureQkND8e9//xtjxoxB3759sXjxYixfvhzXXXddo7zz5s2DVqvFxRdfjJ49eyI/Px/x8fH48ssv4XA4MHr0aKSkpGDOnDkIDw9XakgaOn78OD744AP89ttvSE1NRVxcnLLs2rWr2bL26tULjz/+OBYsWICYmBjMnDmz2bwDBw7ECy+8gKysLAwYMABvvvkmMjMz236DzhFJqPwURncb29tvv+01LfTtt9+O8vJyvP/++43O6d27N+bOnes1w+KyZcuwZcsWfPfdd0qaO6D59ddf8dlnn7X4BNemamoSEhJQUVHRbO3O2Xoi9wls/nkzHkh9APcOvFfVaxMRdSd1dXU4evQo+vTpA5PJ5O/i0DnS0s+9srISYWFhrfr8Vr2mxmAwYPDgwV5Dt5xOJ3JycjB8+PAmzxk+fHijoV7bt2/3yu/Zbvjpp5+e8ZH0RqMRoaGhXouvsE8NERGR//lk9NPcuXNx++23Y8iQIRg2bBhWrFgBs9msjIaaNm0aevXqpVRhzZ49G1deeSWWL1+OsWPHYuPGjdizZw/WrFkD4OzbDc8VNj8RERH5n0+CmkmTJqG0tBRLly5FUVERUlNTsW3bNqUzcH5+vlcb4YgRI7BhwwYsXrwYixYtQnJyMrZs2YIBAwYAqG83BIDU1FSv19qxYwdGjRrli7fRaqypISIi8j+fBDUAMHPmzGY7H+3cubNR2i233IJbbrmlyfxJSUltmtnxXOOzn4iIiPyv041+6ohYU0NEROR/DGpUoMxTwz41RESq6Mi186Q+tX7eDGpUwJoaIiJ1uKfXr6mp8XNJ6Fxy/7xb8yiHlvisT013wj41RETq0Gq1CA8PVx5+GBgYCEmS/Fwq8hUhBGpqalBSUoLw8PAmZ05uCwY1KlBqatj8RETUbrGxsQBwTp7qTB1DeHi48nNvDwY1KuCzn4iI1CNJEuLi4hAdHQ2bzebv4pCP6fX6dtfQuDGoUYEy+R6DGiIi1Wi1WtU+7Kh7YEdhFbD5iYiIyP8Y1KjAHdTU2ms5DJGIiMhPGNSowN2nBgAsDksLOYmIiMhXGNSowKg1KtvsV0NEROQfDGpUoNPooNfIEwaxXw0REZF/MKhRiWe/GiIiIjr3GNSoJEDLuWqIiIj8iUGNSjism4iIyL8Y1KiEzU9ERET+xaBGJXxSNxERkX8xqFEJ+9QQERH5F4MalbBPDRERkX8xqFGJO6j5tuRbHK8+7ufSEBERdT98SrdKIkwRAID3Dr+H94+8jzsuuQP3p94Pg9bg55IR+Z5TOFFjq0GNvabx2r1tq4HFYYFDOOBwOmAXdjicDjiEA3anvcl1w+Pu84QQEBDKGgCEEHAKJ5xwwumU10IIOIQDTlG/LSCUfc9zlXQIyP+vfw0A0EpaaDVaaCQNtJK81ml0XvtajbZ+W9Iq57gn6NRK8rZOo4NWOvunT7uv4762Tqq/pvv67nR3mbQaLSRIcvk0Gmggl1GSJKXMDRe9Ro8AXQCC9EEI0gchWB8MvVbf/l8YIh9hUKOSe353D4L1wfim6Bt8W/ot/vbj3/BZwWd48PcPYlTCKEiS5O8iEgEAHE4Hau21MNvMXkFHrb1WCUTMNnOj4MR9XDnPI52j/roPvUavBDmB+kAE6Ty23em6QAQbghGk805v6hyNxAYDUo8kusljpSsrKxEWFoaKigqEhob69LVyjuXgia+ewKm6UwCAlKgU3DHgDlydcDW0mrP/dkbUkMVhwem60yi3lDdal1vKUWGpqF+s8rrKWqXUUKhNI2kQpAtCgD4AgbpABOoDEagLVD7o9Fo99Bq9Uqug1Wgb1Sa49xuu3bUQ7loQAIAESO7/SZJcwwC5lsFdA9FUukbSQIL8RUOS5POVtAbXdOdz1/h41iC5953C2WSaV82Tq5bJ7qxfzubLjrtGyn2tpq7rWbtlc9rgcDrqa7BctVkOp6NRzZa77O5aLZvDpgSwvuovGKALQLA+2DswahAMhRhCEGYIQ5gpDGGGMIQbwxFmlNcBugB+aezi2vL5zaDGV69nrcSrP7yKNw68oTy5u3dIb9x+ye24/oLrlT44RG42pw0VlgqU15XjtKU+QFGCFcvp+mOudXtqSBoGIO4PlUBdoBKQuJselPQGxxumG7VGfsB0UXanXQlwzDYzqm3VSo2e2WZWavDc217H7GaYra61K90hHKqUS6/RKwFOqCFUCXg8lxBDCEL0IfUBkivd/cw+6tgY1DThXAc1bmW1ZfjHwX9g48GNqLRWApD730zuPxlT+k1BuCn8nJWFzh2ncKLKWqUEJKfqTjVZm+IZoFRZq87qtXSSDuGmcIQbw9HD1APhxnBlUf6wG8IQbgpHmCEMocZQhBhCYNAYGICQXwghYHFY6gMfu3cg1DBgqrJVyQG/R+1juaUcNqetXeUI0csBjte/G1M4ehh7ICogClEBUegZ2BNRAVGIMEWwqcxPGNQ0wV9BjVuNrQbvHX4Pf//p7zhhPgFArnYd02cMxp4/FgOiBiBAF3DOy0UtE0Kg1l6LSmslKiwVytqzOcf9B9az2afcUg6ncLb59SRIyrdO9x9aZW3sgR6mHl774aZwBOuDGZxQt+P+t+n+t+j+d1dp8f43Wm4pR5W1Sq5dslajylaFSktlm5tgtZIWEaaI+iWgfjvSFNkonX/P1cOgpgn+Dmrc7E47th/bjtd+fA0HTh3wOtYzoCeGxA7B4OjBiAqIQu/Q3rgg/AJ+OzhLQghYnVavb4Ce1eOe1ehK0GKprA9cXEFLe74NBuuDGwUo7mCkqXWoIZT9roh8zOF0yDWpltOosFR4fSE5bZGbfEtrS1FWU4bS2lKcrjvd5iAoQBfgHfAERDS5HxUQhR7GHvxi0gIGNU3oKEGNmxACe4r34J9H/onPCj5DhaWiyXw9jD3QN6IvEkMS0Tu0N84LPg8RARGIDoxGXFBclwl47E476ux1ykiaMy0NR+G42+292vdtNbALuyrl00k6hBnlppswQ31bfaghFKHGUEQYIxoHKsZwDn8l6gLsTjtO1p7EqbpTXsvJupM4VdsgrfYkrE5rm66v1+gRExiD6MBoxATFIDYwFjFBMYgJlJeegT0RGRDZbfsAdYig5qWXXsJzzz2HoqIiDBw4EKtWrcKwYcOazb9582YsWbIEeXl5SE5ORlZWFsaMGaMcF0Jg2bJlWLt2LcrLyzFy5Ei8/PLLSE5OblV5OlpQ40kIgUprJX45/Qt2ndiFX07/glOWU/jl9C8tdgQ1aU2IDIhEsD4YwYZghOhDEGwIRrA+GCGGBtuudYAuwGueDmUOD1ea13weQigjIrzmAHGN6rA6rLA6rbA6rLA5bMq21enad1hhcViU/ZaClPa2jZ9JgC7Aa74Nd8dY9xJqDEWoIVTpf+LZFyXMGMYRFkTUKkIImG1m78Cn7lSj4McdAJ22nG7VdSVI6GGS+/r0DumNxNBEJIYmIiksCb1DeiPCFNFl/0b5PajZtGkTpk2bhuzsbKSlpWHFihXYvHkzDh06hOjo6Eb5d+3ahSuuuAKZmZkYN24cNmzYgKysLOzbtw8DBgwAAGRlZSEzMxPr169Hnz59sGTJEvzwww/Yv38/TKYzjyTqyEFNc2wOG/af2o+jFUeRX5mPY5XHUGguxOm60yiuKfZ5IOAPGkmjBCABugCYdCav/QCdPFInQBfQ7PBPz5E87n026RBRR2Rz2FBaW4rimmIUm4tRXFOMInORvO/aPll78oyjxUL0IXKgE+YKdkKTlMAnSB90jt6Nb/g9qElLS8PQoUOxevVqAIDT6URCQgJmzZqFBQsWNMo/adIkmM1mbN26VUm79NJLkZqaiuzsbAghEB8fj4ceegjz5s0DAFRUVCAmJgbr1q3D5MmTz1imzhjUtMTutONE9QmcqjuFalu10gGu2lqNKmtVo7Rqm5xeZ6/zmqtDK2mVOTqUdGi8Zhl1pzU8z6g1Qq/Vw6AxwKCVF71GD6PWKO9rDF7HGwYnAXrXWlu/zxE5RETenMKJ03WnUVZbhuKaYhRUFSCvIg/HKo8pX3Zb6vPTM6Aneof2Ru+Q3jgv5DwkhCQgPjhe6d8TqA88h++m7dry+a36jMJWqxV79+7FwoULlTSNRoP09HTk5uY2eU5ubi7mzp3rlZaRkYEtW7YAAI4ePYqioiKkp6crx8PCwpCWlobc3NxWBTVdjU6jk39JQ3v7uyhERORDGkmDyIBIRAZEol9Ev0bH6+x1KKgqUIIc95JXmYdTdadQWluK0tpS7C3e2+T1PTs1BxuCveekarAO0AVAo9F4TVjpOYFlfHA8+vbo6+tb0izVg5qysjI4HA7ExMR4pcfExODgwYNNnlNUVNRk/qKiIuW4O625PA1ZLBZYLBZlv7Kysm1vhIiIqBMw6UxI7pGM5B6N+5hWWiuRX5mPvMo8/Fb1GwqqCvBb1W9ys1bdSVgcFtTaa3G8+rgqD2O+pe8tWDp8abuvc7a67LOfMjMz8fjjj/u7GERERH4TagjFgKgBGBA1oNExIQRq7DXKyK6TdSeVh8+6nwvnOR1GrU0e2OE5gMT90FcnnIAAegX38sO7rKd6UBMVFQWtVovi4mKv9OLiYsTGxjZ5TmxsbIv53evi4mLExcV55UlNTW3ymgsXLvRq0qqsrERCQkKb3w8REVFXJEmSMsCiq3RlUD2oMRgMGDx4MHJycjBhwgQAckfhnJwczJw5s8lzhg8fjpycHMyZM0dJ2759O4YPHw4A6NOnD2JjY5GTk6MEMZWVldi9ezfuu+++Jq9pNBphNBqVfXd/aDZDERERdR7uz+1WjWsSPrBx40ZhNBrFunXrxP79+8Xdd98twsPDRVFRkRBCiKlTp4oFCxYo+b/88kuh0+nE888/Lw4cOCCWLVsm9Hq9+OGHH5Q8zz77rAgPDxfvv/+++P7778UNN9wg+vTpI2pra1tVpoKCAgGACxcuXLhw4dIJl4KCgjN+1vukT82kSZNQWlqKpUuXoqioCKmpqdi2bZvS0Tc/Px8aTf1MuCNGjMCGDRuwePFiLFq0CMnJydiyZYsyRw0AzJ8/H2azGXfffTfKy8tx2WWXYdu2ba2aowYA4uPjUVBQgJCQENWHDLubtgoKCrrEcPGOhvfXt3h/fY/32Ld4f33Pn/dYCIGqqirEx8efMW+3eUyCL3W1OXA6Gt5f3+L99T3eY9/i/fW9znKPu8aDg4iIiKjbY1BDREREXQKDGhUYjUYsW7bMa7QVqYf317d4f32P99i3eH99r7PcY/apISIioi6BNTVERETUJTCoISIioi6BQQ0RERF1CQxqiIiIqEvosk/pbsjpdOLEiRM+mVGYiIiIfMNzRmHPpxE0pdsENSdOnOBTuomIiDqpgoICnHfeeS3m6TZBTUhICADw2SBERESdiPu5U+7P8ZZ0m6DG3eQUGhqqalBT9dkOnPr73xH4+0Ho+b//q9p1iYiIqF5ruo50m6DGV+xlpaj56itoAgP9XRQiIqJujaOf2knjmjJa1NX5uSRERETdG4OadpKMJgCA02rxc0mIiIi6NzY/tZNkctfUMKghIlKTw+GAzWbzdzHIx/R6PbRarSrXYlDTTkrzk4XNT0REahBCoKioCOXl5f4uCp0j4eHhiI2Nbfc8cgxq2klpfrJY/VwSIqKuwR3QREdHIzAwkBOmdmFCCNTU1KCkpAQAEBcX167rMahpJ42JHYWJiNTicDiUgCYyMtLfxaFzICAgAABQUlKC6OjodjVFsaNwO0mu5ienhX1qiIjay92HJpDTZHQr7p93e/tQMahpJ3fzk2BQQ0SkGjY5dS9q/bwZ1LSTZ/OTEMLPpSEiIuq+GNS0k7v5CQCElZ2FiYjI2/Tp0zFhwgR/F8Nn1q1bh/DwcH8XAwCDmnbTeAY1bIIiIqIGVq5ciXXr1in7o0aNwpw5c1S59mOPPYb+/fsjKCgIPXr0QHp6Onbv3t3iOTt37oQkSaoNmZ80aRJ+/vlnVa7VXgxq2kuvBzTybXRyBBQRETUQFhbms5qMvn37YvXq1fjhhx/wxRdfICkpCaNHj0ZpaWm7r21tZetDQEAAoqOj2/16amBQ006SJClNUKypISLqvt5++22kpKQgICAAkZGRSE9Ph9ls9mp+mj59Oj7//HOsXLlS/vyQJOTl5QEAfvzxR1x33XUIDg5GTEwMpk6dirKyshZf89Zbb0V6ejrOP/98XHLJJXjhhRdQWVmJ77//vsn8eXl5uOqqqwAAPXr0gCRJmD59OgC5BmnmzJmYM2cOoqKikJGRAQB44YUXkJKSgqCgICQkJOD+++9HdXW1cs2GzU+PPfYYUlNT8frrryMpKQlhYWGYPHkyqqqqzuKutg2DGhVoGNQQEfmMEALOmppzvrRl8EdhYSGmTJmCO++8EwcOHMDOnTtx0003NbrGypUrMXz4cMyYMQOFhYUoLCxEQkICysvLcfXVV2PQoEHYs2cPtm3bhuLiYkycOLHVZbBarVizZg3CwsIwcODAJvMkJCTgnXfeAQAcOnQIhYWFWLlypXJ8/fr1MBgM+PLLL5GdnQ0A0Gg0ePHFF/HTTz9h/fr1+OyzzzB//vwWy3LkyBFs2bIFW7duxdatW/H555/j2WefbfV7OVucfE8Fksk1qzCf/0REpDpRW4tDvx98zl+33769kFo5X05hYSHsdjtuuukmJCYmAgBSUlIa5QsLC4PBYEBgYCBiY2OV9NWrV2PQoEF45plnlLRXX30VCQkJ+Pnnn9G3b99mX3vr1q2YPHkyampqEBcXh+3btyMqKqrJvFqtFhEREQCA6OjoRs1iycnJ+POf/+yV5tn/JykpCU899RTuvfde/OUvf2m2TE6nE+vWrUNISAgAYOrUqcjJycHTTz/d7DlqOKuampdeeglJSUkwmUxIS0vD119/3WL+zZs3o3///jCZTEhJScFHH32kHLPZbHjkkUeUqq34+HhMmzYNJ06c8LpGUlKSUlXnXs5F1NcaktEAgM9/IiLqrgYOHIhrrrkGKSkpuOWWW7B27VqcPn261ed/99132LFjB4KDg5Wlf//+AORajzfffNPr2H/+8x/l3Kuuugrffvstdu3ahWuvvRYTJ05UHjvgbs4KDg7GJZdccsZyDB7cOHj89NNPcc0116BXr14ICQnB1KlTcfLkSdTU1DR7naSkJCWgAeTHH7jL5EttrqnZtGkT5s6di+zsbKSlpWHFihXIyMjAoUOHmuwotGvXLkyZMgWZmZkYN24cNmzYgAkTJmDfvn0YMGAAampqsG/fPixZsgQDBw7E6dOnMXv2bFx//fXYs2eP17WeeOIJzJgxQ9n3vGH+pOEEfEREPiMFBKDfvr1+ed3W0mq12L59O3bt2oV//etfWLVqFR599NEzjkRyq66uxvjx45GVldXoWFxcHJxOJ9LS0pS0Xr16KdtBQUG48MILceGFF+LSSy9FcnIy/va3v2HhwoV45ZVXUFtbC0B+GvaZBAUFee3n5eVh3LhxuO+++/D0008jIiICX3zxBe666y5YrdZmZ35u+FqSJMHpdJ7x9durzUHNCy+8gBkzZuCOO+4AAGRnZ+PDDz/Eq6++igULFjTKv3LlSlx77bV4+OGHAQBPPvkktm/fjtWrVyM7OxthYWHYvn271zmrV6/GsGHDkJ+fj969eyvpISEhXtV1HQWbn4iIfEeSpFY3A/mTJEkYOXIkRo4ciaVLlyIxMRHvvfdeo3wGgwEOh8Mr7fe//z3eeecdJCUlQadr+qO5tV/knU4nLK4v2Z7Bj+frA2hUhqbs3bsXTqcTy5cvh8Y10vett95qVTn8oU3NT1arFXv37kV6enr9BTQapKenIzc3t8lzcnNzvfIDQEZGRrP5AaCiogKSJDVq63v22WcRGRmJQYMG4bnnnoPdbm9L8X1GY2DzExFRd7Z7924888wz2LNnD/Lz8/Huu++itLQUF110UaO8SUlJ2L17N/Ly8lBWVgan04kHHngAp06dwpQpU/DNN9/gyJEj+OSTT3DHHXc0G3yYzWYsWrQIX331FY4dO4a9e/fizjvvxPHjx3HLLbc0W9bExERIkoStW7eitLTUayRTQxdeeCFsNhtWrVqFX3/9Fa+//rrSgbgjalNQU1ZWBofDgZiYGK/0mJgYFBUVNXlOUVFRm/LX1dXhkUcewZQpUxAaGqqk/+///i82btyIHTt24J577sEzzzzTYu9ri8WCyspKr8VXlJoaNj8REXVLoaGh+Pe//40xY8agb9++WLx4MZYvX47rrruuUd558+ZBq9Xi4osvRs+ePZGfn4/4+Hh8+eWXcDgcGD16NFJSUjBnzhyEh4crNSQNabVaHDx4EH/84x/Rt29fjB8/HidPnsR//vOfFvvP9OrVC48//jgWLFiAmJgYzJw5s9m8AwcOxAsvvICsrCwMGDAAb775JjIzM9t+g84V0QbHjx8XAMSuXbu80h9++GExbNiwJs/R6/Viw4YNXmkvvfSSiI6ObpTXarWK8ePHi0GDBomKiooWy/K3v/1N6HQ6UVdX1+TxZcuWCQCNljNd92zkP/CA2N+vvzj1j42qX5uIqDupra0V+/fvF7W1tf4uCp1DLf3cKyoqWv353aaamqioKGi1WhQXF3ulFxcXN9vXJTY2tlX5bTYbJk6ciGPHjmH79u1etTRNSUtLg91uVyYtamjhwoWoqKhQloKCgjO8u7OnMbjnqWHzExERkb+0KagxGAwYPHgwcnJylDSn04mcnBwMHz68yXOGDx/ulR8Atm/f7pXfHdD88ssv+PTTTxEZGXnGsnz77bfQaDTNTs1sNBoRGhrqtfhKffMTH2hJRETkL20e/TR37lzcfvvtGDJkCIYNG4YVK1bAbDYro6GmTZuGXr16KW1us2fPxpVXXonly5dj7Nix2LhxI/bs2YM1a9YAkAOam2++Gfv27cPWrVvhcDiU/jYREREwGAzIzc3F7t27cdVVVyEkJAS5ubl48MEH8ac//Qk9evRQ616cNY3JVVPDZz8RERH5TZuDmkmTJqG0tBRLly5FUVERUlNTsW3bNqUzcH5+vlenphEjRmDDhg1YvHgxFi1ahOTkZGzZsgUDBgwAABw/fhwffPABACA1NdXrtXbs2IFRo0bBaDRi48aNeOyxx2CxWNCnTx88+OCDmDt37tm+b1VJruYnJ5ufiIiI/EYSog0Pt+jEKisrERYWhoqKCtWbokpWrMDJ7L+ix5/+hNjFj6p6bSKi7qSurg5Hjx5Fnz59YHI17VPX19LPvS2f33ygpQo0JveMwqypISJSw7mYfZY6DrV+3nygpQqU5ifOKExE1C4GgwEajQYnTpxAz549YTAYIEmSv4tFPiKEgNVqRWlpKTQajTLb8dliUKMCyd1RmJPvERG1i0ajQZ8+fVBYWNjowcbUdQUGBqJ3797NTjTYWgxqVOB+oCU7ChMRtZ/BYEDv3r1ht9tb9Xwi6ty0Wi10Op0qNXIMalQgGd1DullTQ0SkBkmSoNfrW/VkaSI3dhRWgYbNT0RERH7HoEYFkpEPtCQiIvI3BjUqkIxyb23OKExEROQ/DGpUUD9PDWtqiIiI/IVBjQrY/EREROR/DGpUoGHzExERkd8xqFGB5Gp+clqtfi4JERFR98WgRgXueWpgs0FwoigiIiK/YFCjAo07qAGboIiIiPyFQY0KJI/HpLMJioiIyD8Y1KhA0mgguabyZk0NERGRfzCoUYm7X42TQQ0REZFfMKhRibsJSrD5iYiIyC8Y1KhEozypmzU1RERE/sCgRiX1zU+cVZiIiMgfGNSoRDK5amqsDGqIiIj8gUGNSjTu5z+x+YmIiMgvdP4uQFchKX1qWFNDpCZht0PYbBBWq7I4rVZXmke6zQZhs3rlc5/nVNJsEDYb4LBD2B0QTgfgcMrrhvsOpzxDuMPRxL6zfm23N78vBCStFpJWC7jXOl19mmsbWg0kSQNIkrwArm0o+xIaHmuQD4AkSfDY8WkeSFJ9Pq0GkkYDaLSQtBpA0jSfptXJ++50z32d1iNdK+c16CHpm1lMJkgGAzQmEySjEZLB4CoTdVcMalSidBRm8xN1QMLphLDb5Ud52O2uxQHYPfftEDa7d5rNDuHaR3NpNvf5LaTZPF6jiaBD2NyBh61RAAOn09+3jzoRyWiEZDRC41pLJiM0BqMcABkN3ttGORjSmIyQXOkao8F1Dde2yQTJ4MrjeW13QOV+Ha3W32+dwKBGNe4h3XUHDqLu559hvOAC/pJ3YUKI+poCm8eHsc3qnW7z+JB2b59pv9G53tf2ChBcgUPjNO8ApssEBpIkfxvX6+W1sugh6eW1Ru+R3jCfe1+rBXRaSBpXLUnDWoKm9hvWKii1CQ1rF7SARiOvJQ3gdEA4HMrPQdgdck2RO83hkNOEExACQgj5vQoAyrZwJaBBHuGdD0LZFkJ4nO/O21w+eFzPO8+ZyiOcTsApXO/TKb9Hp6PpNK9aMad3rVcTtWDK77bnvxX3YrHIgXBdnUeZAWGxyMfa83t2NvR6aIxGaAICoAkMhBQUCE1AIDSBDRf5uCYoCNoePaAND4c2vAe0PeS1JiiQtU3twKBGJZqgIADA6TfewOk33oAmOBiGpCRoAgNhTE5GYNowBA4dCl2PHn4uaecm7HY4a2vhrKmFqK2B02KFsMp/xITVCqfFAuFKc7rSvPYtViWvsFrqmzFaFXjUb8Nm8/etaD+tFpJOpyzQ6732Jb0O0DWVpoPkmd5EGnTa+n29O02nBBWaZoMOzwDFoDQ9uPNDp+MffPIihADsdte/bwtEXZ38d8FSJwc3dRb533pdXf3fAve2pU4+T8njCogsdcrfCmW7rg5OqyuvRf57Aru9viA2G5w2G5zV1e17Q3o9dOHhcrATGQl9TAx0sbHQx7rXsdDFxkIbHs5/C02QhPAIcbuwyspKhIWFoaKiAqGhoapfv27/fpS+uAq2kmLY8o7BWVPTOJMkwXD++fIvZVQUdNE95XXPntD17AlteLj87CgBaMPDoA0PhyYoqNP94noGHs4aM0RtLZw1NXKa2bWurXGl19bvK9tN7ddC1NTIAUVHpNXWt/O7P6wbbrd332BoHCQ0FTjo9Mp+fZqucQDTyX6viDoaYbfX1xi5Ayr337uaGtffwBo4a8xKmnClOaqr4Cgvh6O8Ao7Tp+E4fRrC0vruC5LBIAc5MTHQxcXCcN55MPTpA0NSHxj69IE2OMiH7/zcasvn91kFNS+99BKee+45FBUVYeDAgVi1ahWGDRvWbP7NmzdjyZIlyMvLQ3JyMrKysjBmzBjluBACy5Ytw9q1a1FeXo6RI0fi5ZdfRnJyspLn1KlTmDVrFv75z39Co9Hgj3/8I1auXIng4OBWldnXQY0n4XDA8vPPsBUXw1lZidpvv4P5692wHj7S9ovp9dCGhUHrLrNGgi6qZ30gFBIMyehqHzaZlGpNzwWSRq7adjrlql13FbPTCeEU9cccjsaBSI1HQFJT4wo2GqTXev9jPSezKms00AQEeLeTuzsKeu4bjXK7uMEgt5kbja62dGN9bUCj4MFVU+C5bWgh8NDr2dRIRO3mrK2F4/Rp2E+flgOesjLYiophLy6S10VFsBUXw3Hy5BmvpT/vPBj79YOpX18Y+/aDqX8/6BMSOuXfKp8GNZs2bcK0adOQnZ2NtLQ0rFixAps3b8ahQ4cQHR3dKP+uXbtwxRVXIDMzE+PGjcOGDRuQlZWFffv2YcCAAQCArKwsZGZmYv369ejTpw+WLFmCH374Afv374fJ1VfluuuuQ2FhIf7617/CZrPhjjvuwNChQ7FhwwbVb4qv2MvKUHfgIOxlZbCXlcJeKi+O0jJ5XVGhjKJylJe3KWrvkLRapX1ZExBQ38YcEOBKD4AUEFCf5rkfKOdpcj8wkKMciKjbclqtsJeUyEFOUTHsRYWwHjsGy9GjsB7NazbokQICYExOlgOd5GQYEhOh790bhl695ObdDsqnQU1aWhqGDh2K1atXAwCcTicSEhIwa9YsLFiwoFH+SZMmwWw2Y+vWrUrapZdeitTUVGRnZ0MIgfj4eDz00EOYN28eAKCiogIxMTFYt24dJk+ejAMHDuDiiy/GN998gyFDhgAAtm3bhjFjxuC3335DfHz8GcvdEYKatnLW1rqqJ8vhqKyCpJEg7HY5KCopgb2kVK7WdPUVcdbVQphr4Kgxw2k2w1ktV3lCCLnjoiQBGk3z21pNfcc2d+ARFOgKJILqAxRXRzdJCVgCoQmqP0dydYhj4EFEdO45ystR9/PPsBw8hLqfD8Fy6GdYfvml+cf4aDTQx8XBkNhbDnLOOw/aqCjoIiOhjYiALiIC2ogIZZTvudaWz+82dRS2Wq3Yu3cvFi5cqKRpNBqkp6cjNze3yXNyc3Mxd+5cr7SMjAxs2bIFAHD06FEUFRUhPT1dOR4WFoa0tDTk5uZi8uTJyM3NRXh4uBLQAEB6ejo0Gg12796NG2+8sdHrWiwWWDxqOiorK9vyVjsEd5Cgj4vzd1GIiKiT0IaHI2jYMAR5dAsRDges+fmwHPoZdYcOwnr4CKwFBbDm50PU1MB2/Dhsx48Du5r+LAcA6HRyp33XvEByM74O0GjhrKuFs7IKYTfdiJiHHz4H77KZIrYlc1lZGRwOB2JiYrzSY2JicPDgwSbPKSoqajJ/UVGRctyd1lKehk1bOp0OERERSp6GMjMz8fjjj7fynREREXVdklYLY58+MPbpg9BrM5R0IQQcZWWw5ufDeiwf1vxjsP12HI5Tp2A/dUpZwzUHldNuB5oaCOPiOF1+Dt5N87rskO6FCxd61RBVVlYiISHBjyUiIiLqWCRJUgaeBA4e3GQeIQScVVVw1tbJw+A9hsQLmw3C4YQmwARNSAh0UVHn+B14a1NQExUVBa1Wi+LiYq/04uJixMbGNnlObGxsi/nd6+LiYsR5NLMUFxcjNTVVyVNSUuJ1DbvdjlOnTjX7ukajEUaP9j9316HO2AxFRETkdwEmeWmCe7JDKwCo/Dnr/txuVRdg0UbDhg0TM2fOVPYdDofo1auXyMzMbDL/xIkTxbhx47zShg8fLu655x4hhBBOp1PExsaK559/XjleUVEhjEaj+Mc//iGEEGL//v0CgNizZ4+S55NPPhGSJInjx4+3qtwFBQXuKTC5cOHChQsXLp1sKSgoOONnfZubn+bOnYvbb78dQ4YMwbBhw7BixQqYzWbccccdAIBp06ahV69eyMzMBADMnj0bV155JZYvX46xY8di48aN2LNnD9asWQNArvqaM2cOnnrqKSQnJytDuuPj4zFhwgQAwEUXXYRrr70WM2bMQHZ2Nmw2G2bOnInJkye3auQTAMTHx6OgoAAhISGqj8hxN20VFBR0mpFVnQnvr2/x/voe77Fv8f76nj/vsRACVVVVrfq8b3NQM2nSJJSWlmLp0qUoKipCamoqtm3bpnT0zc/Ph0ajUfKPGDECGzZswOLFi7Fo0SIkJydjy5Ytyhw1ADB//nyYzWbcfffdKC8vx2WXXYZt27Ypc9QAwJtvvomZM2fimmuuUSbfe/HFF1tdbo1Gg/POO6+tb7dNQkND+Q/Kh3h/fYv31/d4j32L99f3/HWPw8LCWpWv2zwmwZc64xw4nQnvr2/x/voe77Fv8f76Xme5x5ozZyEiIiLq+BjUqMBoNGLZsmVeo61IPby/vsX763u8x77F++t7neUes/mJiIiIugTW1BAREVGXwKCGiIiIugQGNURERNQlMKghIiKiLoFBDREREXUJXfYp3Q05nU6cOHHCJ49JICIiIt/wfEyC5xMLmtJtgpoTJ04gISHB38UgIiKis1BQUHDGxx11m6AmJCQEAFR/GFf1aQvKi80wBekRlRCi2nWJiIio/mGa7s/xlnSboMbd5KT2w7jy//sbPv/Hrzh/UE+cf08v1a5LRERE9VrTdYQdhdtJq5dvod3q9HNJiIiIujcGNe2k02sBAA6bw88lISIi6t4Y1LSTUlNjY00NERGRP3WbPjW+ojMwqCEiag+HwwGbzebvYpCf6PV6aLVaVa7FoKad6pufGNQQEbWFEAJFRUUoLy/3d1HIz8LDwxEbG9vueeQY1LRTfUdh9qkhImoLd0ATHR2NwMBATozaDQkhUFNTg5KSEgBAXFxcu67HoKaddOxTQ0TUZg6HQwloIiMj/V0c8qOAgAAAQElJCaKjo9vVFMWOwu3k7lPD5iciotZz96EJDAz0c0moI3D/HrS3bxWDmnZy96lhTQ0RUduxyYkA9X4PGNS0k7tPjXAKOBwMbIiISCZJErZs2dLq/NOnT8eECRPa9Zp5eXmQJAnffvttu67TFo899hhSU1PP2eu1hEFNO7n71ACAg7MKExF1C0VFRZg9ezYuvPBCmEwmxMTEYOTIkXj55ZdRU1Pj7+K1aN26dQgPD1ftevPmzUNOTo5q12sPdhRuJ61HUGO3OWEI8GNhiIjI53799VeMHDkS4eHheOaZZ5CSkgKj0YgffvgBa9asQa9evXD99df7u5jtZrVaYTAYzpgvODgYwcHB56BEZ8aamnaSJMljVmEO6yYi6uruv/9+6HQ67NmzBxMnTsRFF12E888/HzfccAM+/PBDjB8/vsnzfvjhB1x99dUICAhAZGQk7r77blRXVzfK9/jjj6Nnz54IDQ3FvffeC6vVqhzbtm0bLrvsMoSHhyMyMhLjxo3DkSNHWl32nTt34o477kBFRQUkSYIkSXjssccAAElJSXjyyScxbdo0hIaG4u677wYAPPLII+jbty8CAwNx/vnnY8mSJV4dehs2P7mb0Z5//nnExcUhMjISDzzwwDmZYJFBjQrcTVAcAUVE1LWdPHkS//rXv/DAAw8gKCioyTxNdXo1m83IyMhAjx498M0332Dz5s349NNPMXPmTK98OTk5OHDgAHbu3Il//OMfePfdd/H44497XWfu3LnYs2cPcnJyoNFocOONN8LpbN3nz4gRI7BixQqEhoaisLAQhYWFmDdvnnL8+eefx8CBA/Hf//4XS5YsAQCEhIRg3bp12L9/P1auXIm1a9fi//7v/1p8nR07duDIkSPYsWMH1q9fj3Xr1mHdunWtKmN7sPlJBXxSNxFR+wkh/PZ3VGfQtGoEzuHDhyGEQL9+/bzSo6KiUFdXBwB44IEHkJWV5XV8w4YNqKurw9///nclGFq9ejXGjx+PrKwsxMTEAAAMBgNeffVVBAYG4pJLLsETTzyBhx9+GE8++SQ0Gg3++Mc/el331VdfRc+ePbF//34MGDDgjOU3GAwICwuDJEmIjY1tdPzqq6/GQw895JW2ePFiZTspKQnz5s3Dxo0bMX/+/GZfp0ePHli9ejW0Wi369++PsWPHIicnBzNmzDhjGduDQY0KOAEfEVH72a1OrJn9uV9e++6VV0JvPPtJ377++ms4nU7cdtttsFgsjY4fOHAAAwcO9KrdGTlyJJxOJw4dOqQENQMHDvSau2f48OGorq5GQUEBEhMT8csvv2Dp0qXYvXs3ysrKlBqa/Pz8JoOaSy65BMeOHQMAXH755fj4449bfB9DhgxplLZp0ya8+OKLOHLkCKqrq2G32xEaGtridS655BKvSfTi4uLwww8/tHiOGhjUqEBncM9Vwz41RERd2YUXXghJknDo0CGv9PPPPx9A/ey4vjJ+/HgkJiZi7dq1iI+Ph9PpxIABA7z63Xj66KOPlL4srSlbwya13Nxc3HbbbXj88ceRkZGBsLAwbNy4EcuXL2/xOnq93mtfkqRWN5G1B4MaFbBPDRFR++kMGty98kq/vXZrREZG4g9/+ANWr16NWbNmNduvpqGLLroI69atg9lsVs758ssvodFovJqyvvvuO9TW1ioByFdffYXg4GAkJCTg5MmTOHToENauXYvLL78cAPDFF1+0+LqJiYmN0gwGAxyO1n0J37VrFxITE/Hoo48qae6an46IHYVVwD41RETtJ0kS9EatX5a2zGj7l7/8BXa7HUOGDMGmTZtw4MABHDp0CG+88QYOHjzY5LOLbrvtNphMJtx+++348ccfsWPHDsyaNQtTp05Vmp4AeRj1XXfdhf379+Ojjz7CsmXLMHPmTGg0GvTo0QORkZFYs2YNDh8+jM8++wxz585t831OSkpCdXU1cnJyUFZW1uK8OsnJycjPz8fGjRtx5MgRvPjii3jvvffa/JrnCoMaFdTX1LD5iYioq7vgggvw3//+F+np6Vi4cCEGDhyIIUOGYNWqVZg3bx6efPLJRucEBgbik08+walTpzB06FDcfPPNuOaaa7B69WqvfNdccw2Sk5NxxRVXYNKkSbj++uuVIdcajQYbN27E3r17MWDAADz44IN47rnn2lz+ESNG4N5778WkSZPQs2dP/PnPf2427/XXX48HH3wQM2fORGpqKnbt2qWMiuqIJCGE8HchzoXKykqEhYWhoqLijB2c2urDv3yPvO/LMOq2frjk8l6qXpuIqCuqq6vD0aNH0adPH5hMJn8Xh/yspd+Htnx+s6ZGBe62WDY/ERER+Y/PgpqXXnoJSUlJMJlMSEtLw9dff91i/s2bN6N///4wmUxISUnBRx99pByz2Wx45JFHkJKSgqCgIMTHx2PatGk4ceKEr4rfJkrzk51BDRERkb/4JKjZtGkT5s6di2XLlmHfvn0YOHAgMjIyUFJS0mT+Xbt2YcqUKbjrrrvw3//+FxMmTMCECRPw448/AgBqamqwb98+LFmyBPv27cO7776LQ4cOdZhna+j0riHdVvapISIi8hef9KlJS0vD0KFDlQ5QTqcTCQkJmDVrFhYsWNAo/6RJk2A2m7F161Yl7dJLL0Vqaiqys7ObfI1vvvkGw4YNw7Fjx9C7d+8zlsmXfWq+2PwLvsspwKDRvTHipgtVvTYRUVfEPjXkqcP2qbFardi7dy/S09PrX0SjQXp6OnJzc5s8Jzc31ys/AGRkZDSbH4DyMK7mHp9usVhQWVnptfgKZxQmIiLyP9WDmrKyMjgcDq9x9wAQExODoqKiJs8pKipqU/66ujo88sgjmDJlSrNRW2ZmJsLCwpQlISHhLN5N67g7CjvY/ERE1CbdZAAunYFavwedbvSTzWbDxIkTIYTAyy+/3Gy+hQsXoqKiQlkKCgp8Viatu08NOwoTEbWKexr9liZ+o+7D/XvQ8PEKbaX6YxKioqKg1WpRXFzslV5cXNzkE0EBIDY2tlX53QHNsWPH8Nlnn7XYtmY0GmE0Gs/yXbSNMvqJQ7qJiFpFq9UiPDxcGUASGBjYpll9qWsQQqCmpgYlJSUIDw9vcjbmtlA9qDEYDBg8eDBycnIwYcIEAHJH4ZycHMycObPJc4YPH46cnBzMmTNHSdu+fTuGDx+u7LsDml9++QU7duxAZGSk2kU/a1r2qSEiajP3F9fmRsZS9xEeHt5sxUdb+OSBlnPnzsXtt9+OIUOGYNiwYVixYgXMZjPuuOMOAMC0adPQq1cvZGZmAgBmz56NK6+8EsuXL8fYsWOxceNG7NmzB2vWrAEgBzQ333wz9u3bh61bt8LhcCj9bSIiImAwGHzxNlpNmXyPj0kgImo1SZIQFxeH6Oho5UnS1P3o9fp219C4+SSomTRpEkpLS7F06VIUFRUhNTUV27ZtUzoD5+fnQ6Op784zYsQIbNiwAYsXL8aiRYuQnJyMLVu2YMCAAQCA48eP44MPPgAApKamer3Wjh07MGrUKF+8jVarn6eGNTVERG2l1WpV+1Cj7o3PflJB/k8n8c9V3yEqIRiTHh2m6rWJiIi6Mz776RxT+tSwpoaIiMhvGNSoQGl+Yp8aIiIiv2FQowJl8j2OfiIiIvIbBjUqYPMTERGR/zGoUYG7+Yk1NURERP7DoEYF7hmFnU4Bp4OBDRERkT8wqFGB1lB/GzmrMBERkX8wqFGBTucR1LBfDRERkV8wqFGBpJGg1fFRCURERP7EoEYlHNZNRETkXwxqVFJfU8OghoiIyB8Y1KiENTVERET+xaBGJVrlSd3sU0NEROQPDGpU4p6rhs1PRERE/qHzdwG6CjY/EXUPQggIp3BNtlm/LZyQ94XwdxFbpJTPvVKKK29IkgRJI0GjlSBJ8lqjcaVpJEiufaKOiEGNSthRmKjtnE4Bp90Jh8O1tsuzcjvsTjgdwnttF3A4PNbNHve4XsO16/qNzm10LadcNqeAcIj6bScgnB07aDkXJEn+m6fRaaDVyVNaNLWv1ctrnUEDnV4DrV7rWsv7Or1W3jbU59MbtNAbtdCbXGujDnqjFjqDBpLEYIpaxqBGJToD+9RQ5yaEgMPmhN3qhN3m8F5b3fvubYfHdjPn2DyOufI7bE4lIHHanejglRpt5q7RUJ3Kl5QabkiSsilEfTDX3M9HCNcXuHP5JU5CfcDjCnoMJh0MAToYTFp5HaCDscG+kidAC2OADnqTjjVNXRiDGpW4+9QUHDiFsOhAxF8YBo2WXZaofZRAwzOA8Aw0GgYhDfO0eJ4DDpsTNqsDDldef9NoJfnbvlZu9tDqNB5ruRZAWbvzNbnWQKNrxTWavJac370ozS6aBk0xWo/mGMlHwYyfuZvahBNeNVcOhxygumu2HHZ3wCrqt101XnbX769DWbt+D+1O5ffOYfP+XbbVOWCz1C9yYeC93w4GkxaGQB2MAXoYA3X1S4AexiA5CDIF6euXYHnfEKBjbVEHx6BGJYGhBgDAkX2lOLKvFKYgPRIu6oGIXsGISQxFzPmhMJh4u7sCIeQ/3A1rIRrVXjQRRLj/cDusDtis8h9z77Vn7YfDb4GGRitBZ5CbCnQGjce21tWUoFWaFJQ0g9yUoG9i7T6u8WyqaBRwSPzA6GAkSQ7aoAW0fiqDcArYbU6vIMdmccBWZ4e1zgFrrR2WWjustXZY61zrWoeyXX/MofR5tNY5YK1zoBqWNpVF0kgwBnoEPMF6mAJ1MAbrYQqU9z2PG4Pkbb1Ry9/tc4SfsioZOq4PgnoYUZpfheOHTqO2yoZf9pQAe0oAyP8YeiYEI+7CcMRfGI6Y80MRGGrgL7pKhBBw2oVc6+CuffCohXDvq1G7Ybc5lU6W55JGKzUKIpR9vWdQoYHWoJXX+gZBiNd24/PdgQprGamjkDSS0uTUXg6bE9Y6Oyw1rqXWpmxba+2w1Mj7dWZ5u85sQ121DXU1dtgtDginkPerbW16XY1WgjFIDoDkwEcPkyvgMXrUCLmDIFOQHBwxGGo7SXT0rvoqqaysRFhYGCoqKhAaGurT13I6nDhxuAIleZUo+60aRUcqUHWqrlE+Y6AOEfFB6BEXhOBwI0xBegSEGGAK1iMgWK+su8oHjNMpYLc4YHP3ybC6vn25AwiPYzZLg+MWd5r7m5qzPt219sdvsqSRmgkQWl+70XRg4XGcgQaR39ltDljMdo9Ax7U22+R0174cFNmUxWk/+z9MWr0GQWEGBIYaERRuQGCYEUFhBgSFGREUZkRgmAFB4UYYA7t2s1hbPr99FtS89NJLeO6551BUVISBAwdi1apVGDZsWLP5N2/ejCVLliAvLw/JycnIysrCmDFjlONCCCxbtgxr165FeXk5Ro4ciZdffhnJycmtKs+5DGqaUnWqDoWHy3HicAUKD5fjdKG51R/C7upMQ4AcubvbgwOCDdCbtEp1vk6vcY0ScI0ccHWqgwRAuNrHRf02IFftCgBwHRMCcpu5qy3cq73cM62JtWcNh80jaLFb5cCjPf+420LSSM0HFk0FEZ55vYIPj9qNpppWDHL/CyKipgghN53JwY4NdWa7x7a8b/EIgDwDorb8vdTqNHKA4w54wo0IiTQhNCrAtZg6dfcHvwc1mzZtwrRp05CdnY20tDSsWLECmzdvxqFDhxAdHd0o/65du3DFFVcgMzMT48aNw4YNG5CVlYV9+/ZhwIABAICsrCxkZmZi/fr16NOnD5YsWYIffvgB+/fvh8lkOmOZ/B3UNGS3OnC6uAanTphRXlyDmior6qpsqK22oq7ahlrXNwB/NHP4nCSPFtO7ggW9UVsfkBm10Om10BtdfTFc1c7ufHqj1pXucdx9DXe/DfbNIKJOTAgBu9WJ2iorzBVW1FRYYK6wwFzusV1hhbnCAovZ3qprmoL1CI00IbRnAEIjAxASaUJgqAGBoQYEhMhrNZr4hBCq//31e1CTlpaGoUOHYvXq1QAAp9OJhIQEzJo1CwsWLGiUf9KkSTCbzdi6dauSdumllyI1NRXZ2dkQQiA+Ph4PPfQQ5s2bBwCoqKhATEwM1q1bh8mTJ5+xTB0tqGkNp1PAUmNDbZVcrWmts7s6yDmUqk67xSGPInCNJLBZHY061AFyZY17dIY8uRaUbUiuNAmAJEGjgfccE3rvOSeUtbItyfNN6CQlyJCDEHftR33Q4a7pYNBBRNR+dpsDNRVW1FRaYS6Xg53q03WoOlmHyrJaVJbVyV+QW0Fv1CIg1ICAYLllwOCeL8gkb2t0GtfoP0Cj0UDSQKnxLvutGsd/Po2LRsThd1clqPoe2/L5rXp9lNVqxd69e7Fw4UIlTaPRID09Hbm5uU2ek5ubi7lz53qlZWRkYMuWLQCAo0ePoqioCOnp6crxsLAwpKWlITc3t8mgxmKxwGKp79leWVnZnrflFxqNhIBgAwKCDf4uChERdUA6vVZpZmqOtdaOypO1qCytc61rUXXagppKK2orraipssoDKywO2Erl42frt4OnVQ9q2kL1oKasrAwOhwMxMTFe6TExMTh48GCT5xQVFTWZv6ioSDnuTmsuT0OZmZl4/PHHz+o9EBERdRWGAB2izgtB1HkhTR4XQsBW50CNK8Cpq7LBarHDVicPjbfVOWC1OBo8FkR+TIgcDNkREhWA8/r2QHzf8HP75hrovD2HzmDhwoVetT+VlZVISPBf9EhERNQRSZKkzMAcHhPo7+K0i+pBTVRUFLRaLYqLi73Si4uLERsb2+Q5sbGxLeZ3r4uLixEXF+eVJzU1tclrGo1GGI1GZd/ddagzNkMRERF1V+7P7dZ0AVY9qDEYDBg8eDBycnIwYcIEAHJH4ZycHMycObPJc4YPH46cnBzMmTNHSdu+fTuGDx8OAOjTpw9iY2ORk5OjBDGVlZXYvXs37rvvvlaVq6qqCgBYW0NERNQJVVVVISwsrMU8Pml+mjt3Lm6//XYMGTIEw4YNw4oVK2A2m3HHHXcAAKZNm4ZevXohMzMTADB79mxceeWVWL58OcaOHYuNGzdiz549WLNmDQC5amzOnDl46qmnkJycrAzpjo+PVwKnM4mPj0dBQQFCQkJ8MtwsISEBBQUFnWZkVWfC++tbvL++x3vsW7y/vufPeyyEQFVVFeLj48+Y1ydBzaRJk1BaWoqlS5eiqKgIqamp2LZtm9LRNz8/HxpN/aRlI0aMwIYNG7B48WIsWrQIycnJ2LJlizJHDQDMnz8fZrMZd999N8rLy3HZZZdh27ZtrZqjBpBHYJ133nnqvtEGQkND+Q/Kh3h/fYv31/d4j32L99f3/HWPz1RD49ZtHpPgS51xDpzOhPfXt3h/fY/32Ld4f32vs9xjzvFOREREXQKDGhUYjUYsW7bMa7QVqYf317d4f32P99i3eH99r7PcYzY/ERERUZfAmhoiIiLqEhjUEBERUZfAoIaIiIi6BAY1RERE1CUwqGmHsrIyPkuKiIiog2BQc5aeeeYZXH311RgyZAhuvvlm7Nq1y99FIlIdB0f6Fu8vkbo4pPssPP3001i5ciWysrJgMBjw0ksvweFwYNmyZRgzZoy/i9fluB+HYTKZcOmll/q7ON1Cfn4+IiMjIYRAcHAwhBCqPzOtO+P99a13330Xu3btQlRUFAYNGoSMjAx/F6lL6dD3V1Cb1NbWimuvvVb83//9n5J2/Phx8dBDD4mLL75YfPfdd/4rXBd04403il69eokLL7xQGAwG8eCDD4qDBw/6u1hd2kMPPSQuuugi0b9/fzFy5Eixd+9e4XA4/F2sLoP317cWLlwoQkJCxM033ywGDhwoAgICxDPPPCNqamr8XbQuoaPfXwY1bVRXVyeGDRsm5s+f75V++PBhMWPGDHHppZeK06dP+6dwXcyTTz4pBg4cKAoKCkRBQYF4//33RXx8vJg6dar473//6+/idUnz588XiYmJ4qOPPhJr164VEyZMEKGhoeL1118XZrPZ38Xr9Hh/fevgwYPiggsuEJ988okQQojy8nKxdu1aodFoxFNPPSWqq6v9XMLOrTPcXwY1bWSz2cTEiRPFhAkTRGlpqdexnTt3iiFDhogVK1b4qXSdn9PpVLanT58uJk6c6HV8y5Yt4ne/+52YOXOmOHHixLkuXpd3zTXXiKysLK+0adOmiQsvvFC8++67rFFoJ95f3/rss89EXFyc+O2337zSX3zxRaHVasU777wjhPD+O0Ot1xnuLzsKt5FOp8PcuXPx/vvv44033vDq6HfllVeif//+2LRpkx9L2LkVFxcDAKxWK6qrq6HT6QAANpsNAHDDDTdgxowZ+Pjjj/Hll18CYGdLNQghUFZWhmPHjqFHjx4AgLq6OgDA+vXr0bt3bzz77LPKz4faxm638/76kPtvQGJiIkpKSvDdd98BkO87AMyaNQvTp0/Hgw8+CKfTyf5LbeB0OpXtTnF//RZOdXLPPvusMBqNYvPmzaKurk5Jf+yxx8QNN9zAb1xn4dFHHxX9+/cXJ0+eFEII8c477whJksSePXuEEMLrPo8fP15cdtllfilnV3brrbeKAQMGKPvue37y5EkRGBgo/vznP/uraJ3Szz//7LX/pz/9ifdXRcXFxcJisSj7tbW1Ytq0aeKyyy4Tx44dE0IIYbVahRBy38fExESxZs0av5S1M9q0aZPyO+l0OkVNTY2YPn16h76/rKk5S4888gjuuusu3HXXXXjxxRfx1Vdf4cCBA9iwYQP69esHjYa3ti0mTZqEv/zlL1izZg0iIiIAANdeey1uuOEG3HTTTaiurobRaITVagUA3HnnnThy5Ah+++031tScpXfffRfvvfcePvroIyXtwQcfRE1NDWbPng1AfjKvxWJBREQE7rnnHnz44Yeora3lPW+Fhx9+GLfccguKi4uV+/XAAw/AYrHw/qpg2bJl+MMf/oBhw4ZhzJgx2L9/P0wmE2677TZlNGpNTQ30ej0A+V7rdDo4HA4/l7xzePjhhzF58mSkpKQAACRJQkBAAG644QYA6LD3l5+87fDSSy/h3nvvxbvvvotrr70W48aNw+DBg5GVleXvonUaVqsVw4YNw6FDh/DTTz/h8ssvR0VFBZxOJwIDA/HEE08gJiYGo0aNQm1tLQwGAwCgsLAQ559/Pnr27Mmq5LNw00034f7778cTTzyBcePGYfLkyfjiiy8wZMgQ3HvvvfjnP/+J5cuXA5D/WAHyzyomJgYBAQG852dwww034NVXX8Urr7yCmJgY5X5dfPHF+J//+R98+OGHvL/tsHDhQvztb3/Dww8/jPvvvx8lJSWYNGkSNm3ahNGjR+NPf/oTfvjhB9x7771e5wUEBChfmqh5N954IzZs2IBdu3bh2muv9To2YcIETJgwAT/99FPHvL9+rSfqIoqKisQ333yjNJNQ661du1bo9XqRnZ0thBDi73//u/jDH/4gLrnkEpGeni7ef/998emnn4rf/e534pJLLhEPPfSQWL16tYiIiBCLFy/2c+k7p9WrV4vf/e53Ij8/X9TU1IivvvpKXHrppeIPf/iD2LVrl6ipqRGLFi0SgYGB4qmnnhL/+c9/xDfffCP69OkjHn/8cX8Xv0Mzm81i8ODBYuDAgaKqqkoIIURJSYmora1V9k+cOMH72w4Wi0WMGDFCrF69Wkmz2Wzi+uuvF8OHDxcfffSRcDgc4pVXXhGJiYni/PPPF3/84x9FUlKSyMjI8GPJOz6HwyFuu+02YTAYxLfffiuEEGLXrl0iKytLLFu2TPzjH/8QQsjNpmvXru2Q95eT75Ff1dTUYOnSpfjXv/6FPn364Mcff8S0adMQHh6ODz74ANXV1Zg1axauv/56PPTQQ/j1119ht9tx4403Ys6cOf4ufqf04IMP4r///S927typpP373//G008/DaPRiNWrVyMuLg7r16/HsmXLYDQaYbfbMXbsWLz88sv+K3gn8NJLL2Hx4sVYtGgRHn74Ybz22mtYv3690gSVmZmJ8ePHw2az4c033+T9bSMhBEpLS3HNNddg5syZuOeee2C1WmEwGFBYWIhbb70VwcHB+Otf/4q4uDgUFhZi9erV0Gq1iIiIwIMPPujvt9DhPf/889i0aROmTp2Kuro6rF69Gv3790dpaSm+//57zJkzB1lZWdBoNCgqKup499e/MRWR/E32lltuERdddJHYvn27km6xWMTo0aNFenq6EEL+NiaEPDcCtZ3D4RAOh0M8/PDDIiMjQ5jNZq8O7Zs3bxZDhw4VWVlZyr3Oz88XR48e5aSSrXTq1Ckxe/Zscfnll4srrrhCJCUliRUrVoi1a9eK6dOni+joaPH6668r95339+yMGjVKXHvttcq+u7Nqbm6uCAkJEa+99pqfStZ5eQ7Dfvjhh0WvXr3E+eefLzZv3qzUMr799ttCkiSxceNGfxXzjBjUUIfwyy+/iHfeeUeZgMxutwshhHjzzTeFwWAQBQUFHFF2lhrOp/T5558LjUYj3n33XSFE/b0WQoh7771XXHLJJco+5/M4s4b39/Dhw+Lmm28WQ4cOFTt27PA6dsstt4iBAwcq+7y/Z/bVV1+Jr7/+Whw6dEhJy83NFQEBAWL58uVCCPl32P17PH36dHHFFVf4paydUVP312aziTlz5oiXX37Z6++DEEJMmDBB+aLZETGooQ7D/W3L0xNPPCHGjh3rh9J0Df/zP/8jxo8fL3799Vev9JkzZ4oePXqIX375RQghlIBxz549IigoSOzfv/+cl7Uzau7+fvvtt+Ktt95Spo53fzBs3bpVmEwmcfjwYQY0rXDnnXeKCy+8UCQmJoqAgADx97//XQghRGVlpXjiiSeEwWAQ77//vtc599xzj5g6dao/itvpNHV/3X8LqqqqRFlZmVf+uro6ce2114oHHnjAH8VtFQY11GHt3LlTXHDBBeK5557zd1E6HbvdLmbMmCHOO+88odPpxAMPPOD1B6qwsFBcddVVIjk52SuA2bBhgxg8eDAf9XEGZ7q/QtQ3lwpRXyPz7LPPitGjR7PW8QxsNpuYMGGCSE1NFd9//704evSoePTRR0WPHj3EqVOnhBBC/Pbbb+KBBx4QWq1WvPbaa+Krr74SBw4cEBdccIF47LHH/PwOOrbW3N+m/PDDD2LQoEFi/fr157C0bcOghjqcDz74QMyZM0eEhYXxj9NZ+u6778TEiRPFJ598Iv75z38KSZLEM888o7SNCyFPlnXppZeKfv36iUmTJolnn31W9OjRQ8ybN8+PJe8cmru/LT375pNPPhEJCQlKkwk17x//+IcYNWqUV8BdXV0tEhMTxebNm5W02tpasXTpUpGQkCDi4+NF7969xbRp0/xR5E6lpfv79ttvN8q/d+9e8cYbb4iePXuKe+6551wWtc0Y1FCHU1FRIW666SaxdetWfxel07JarSInJ0dUVlYKIYR44YUXhFarFW+88YbXzMxCyA8OveGGG8T1118vVq1a5Y/idjot3V/PGW6FkJucbr31VhEeHi6effZZfxS30zl58qS4++67vX5X6+rqRO/evcXHH3/cKP/+/fvFf//7X/HVV1+dy2J2Wm25v1VVVeK5554TSUlJ4v/+7//OcUnbjkO6qUOy2+3Kc5+ofYQQkCQJ9957L9566y289dZbuOaaaxpN8GY2mxEUFOSnUnZeLd1fIQSKi4vx6KOPYsqUKUhPT/d3cTslh8OB2tpapKWl4Y033sCgQYP8XaQu5Uz3t7KyEsXFxUhOTvZTCVuPMwpTh8SARj3u7y3Z2dkYNGgQZs6ciR9//BH5+fmYOXMmPv30UwBAYGCgP4vZabV0fx944AEcO3YMa9euZUBzFtz3VqvVora2FqdOnVKm4bdarXjllVeQn5/vzyJ2ame6v2vXrkV+fj5CQ0M7RUADAKypIeoGPGu++vfvj5CQEPz2229ISEjAF198oTx+gs5OU/e3oKAAvXv35v1VyeHDh5GWloZff/0VZrMZo0aNQo8ePfDll1/yS5AKusr9ZU0NUTeg0+lgt9sBAHPnzsXevXsxfvx4fP311/zAVUFT9/f666/n/VVRSUkJ+vbti2+//RYDBw7EoEGDsHv37k71gduRdZX7y6CGqJvQ6XR49dVXce+99+Kpp57CmjVr/F2kLoX317fMZjN2796Nq6++GjNmzMCmTZv8XaQupavcXzY/EXUTQgh8+OGHsNvtmDBhgr+L0+Xw/vpWeXk5oqKisGXLFowbN87fxelyusr9ZVBDRESdQl1dHUwmk7+L0WV1hfvLoIaIiIi6BPapISIioi6BQQ0RERF1CQxqiIiIqEtgUENERERdAoMaIiIi6hIY1BAREVGXwKCGiIiIugQGNURERNQlMKghIiKiLoFBDREREXUJ/w9/WC++YvfufQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "vis.show_histograms(data = data, display_format=\"percent\")" - ] - }, - { - "cell_type": "markdown", - "id": "fe527f64", - "metadata": {}, - "source": [ - "### Tip: Avoid repeated calculation\n" - ] - }, - { - "cell_type": "markdown", - "id": "4640b9aa", - "metadata": {}, - "source": [ - "If you intend to plot histogram main plot and subplot separately, repeated calling show_histogram with different plot_types is not efficicent, as it repeatewd calculate the same set of Dataframes. To do it efficiently, you can use the following functions instead show_histogram methods. This avoid the duplicated calculation in show_histograms. But if you intend to show both plots, the show_histogram() should be used" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "a8e6722f", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAHBCAYAAABKReAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAACFIUlEQVR4nOzdd3wUdf7H8ddsT930SkjoNRCkg10koqLYQFQQ9Lyz4KnYG3YQTzxQVE49D+Tg5FTk/KGiiGADkd47hFDSe3azfX5/LFmISdgEEjfA5/l4jLs7+52Z744h+873+53vKKqqqgghhBBCtGCaQFdACCGEEMIfCSxCCCGEaPEksAghhBCixZPAIoQQQogWTwKLEEIIIVo8CSxCCCGEaPEksAghhBCixZPAIoQQQogWTwKLEEIIIVo8CSxCiHrNnj0bRVHIysoKdFVOy4oVK1AUhRUrVgS6KkKIUySBRQjR5LZv387zzz/fooPO/PnzmT59eqCrIYRoIEXuJSSEqI/b7cbpdGI0GlEUpcHbffrpp9x0000sX76ciy++uPkq2EAejweHw4HBYECj8f6ddvXVV7N169YWHaqEEMfpAl0BIUTLpdVq0Wq1ga7GadNoNJhMpkBXQwhxGqRLSAhRr9+PYUlLS+Pqq6/m559/pl+/fphMJtq2bctHH31UY5ubbroJgEsuuQRFUWqNH/n666+54IILCAkJISwsjKuuuopt27bVOPa4ceMIDQ3lyJEjjBgxgtDQUGJjY3nkkUdwu901yn788cf07t2bsLAwwsPDSU9PZ8aMGb73fz+G5eKLL+bLL7/k4MGDvvqlpaVRWVlJSEgIDzzwQK1zcfjwYbRaLVOmTDmdUyqEOEUSWIQQjbJ3715uvPFGLr/8cqZNm0ZkZCTjxo3zBY4LL7yQv/71rwA89dRTzJ07l7lz59KlSxcA5s6dy1VXXUVoaChTp07l2WefZfv27Zx//vm1umfcbjeZmZlER0fz+uuvc9FFFzFt2jTee+89X5mlS5cyevRoIiMjmTp1Kq+++ioXX3wxv/zyS72f4emnnyYjI4OYmBhf/aZPn05oaCjXXXcdCxYsqBWK/vOf/6CqKrfeemtTnEYhRGOpQghRj3/9618qoB44cEBVVVVNTU1VAfXHH3/0lcnPz1eNRqP68MMP+9Z98sknKqAuX768xv4qKirUiIgI9a677qqxPjc3VzWbzTXW33777SqgvvjiizXK9urVS+3du7fv9QMPPKCGh4erLper3s+xfPnyWvW56qqr1NTU1Fplv/nmGxVQv/766xrre/TooV500UX1HkMI0bykhUUI0Shdu3blggsu8L2OjY2lU6dO7N+/3++2S5cupbS0lNGjR1NYWOhbtFot/fv3Z/ny5bW2ufvuu2u8vuCCC2ocKyIiAovFwtKlS0/jUx03ZMgQkpKSmDdvnm/d1q1b2bx5M7fddluTHEMI0Xgy6FYI0SitW7eutS4yMpKSkhK/2+7ZsweASy+9tM73w8PDa7w2mUzExsae9Fj33nsv//3vfxk2bBjJyckMHTqUkSNHcsUVV/itT100Gg233nor7777LlarleDgYObNm4fJZPKNzRFC/PEksAghGqW+q4bUBsyQ4PF4AO84loSEhFrv63Q1fyU15AqluLg4Nm7cyDfffMPXX3/N119/zb/+9S/Gjh3LnDlz/G5fl7Fjx/K3v/2NRYsWMXr0aObPn8/VV1+N2Ww+pf0JIU6fBBYhRJOrb86Wdu3aAd6QMWTIkCY7nsFgYPjw4QwfPhyPx8O9997LP/7xD5599lnat2/fqDoCdO/enV69ejFv3jxatWpFdnY2b731VpPVVwjReDKGRQjR5EJCQgAoLS2tsT4zM5Pw8HAmT56M0+mstV1BQUGjj1VUVFTjtUajoUePHgDY7faT1rGsrKze98eMGcO3337L9OnTiY6OZtiwYY2umxCi6UgLixCiyWVkZKDVapk6dSplZWUYjUYuvfRS4uLiePfddxkzZgznnXceN998M7GxsWRnZ/Pll18yePBgZs6c2ahj/elPf6K4uJhLL72UVq1acfDgQd566y0yMjJ8l1LXpXfv3ixYsICJEyfSt29fQkNDGT58uO/9W265hccee4zPP/+ce+65B71ef8rnQwhx+qSFRQjR5BISEpg1axb5+fnceeedjB49mu3btwPeILBs2TKSk5P529/+xgMPPMDHH39MRkYG48ePb/SxbrvtNkwmE++88w733nsvc+bMYdSoUXz99de+afjrcu+993LLLbfwr3/9i1tuuYX777+/xvvx8fEMHToU8La2CCECS+4lJIQQ9bjuuuvYsmULe/fuDXRVhDjnSQuLEELUIScnhy+//FJaV4RoIWQMixBCnODAgQP88ssvfPDBB+j1ev7yl78EukpCCKSFRQghavjhhx8YM2YMBw4cYM6cOXXOFyOE+OPJGBYhhBBCtHjSwiKEEEKIFk8CixBCCCFavLNi0K3H4+Ho0aOEhYWddLptIYQQQrQcqqpSUVFBUlLSSedNgrMksBw9epSUlJRAV0MIIYQQp+DQoUO0atXqpGXOisASFhYGeD/w729PL4QQQoiWqby8nJSUFN/3+MmcFYGluhsoPDxcAosQQghxhmnIcA4ZdCuEEEKIFk8CixBCCCFaPAksQgghhGjxzooxLEIIIVomj8eDw+EIdDVEAOn1erRa7WnvRwKLEEKIZuFwODhw4AAejyfQVREBFhERQUJCwmnNlSaBRQghRJNTVZWcnBy0Wi0pKSl+JwUTZydVVbFareTn5wOQmJh4yvuSwCKEEKLJuVwurFYrSUlJBAcHB7o6IoCCgoIAyM/PJy4u7pS7hyTyCiGEaHJutxsAg8EQ4JqIlqA6tDqdzlPehwQWIYQQzUbu7yagaX4OJLAIIYQQosWTwCKEEEI0wLhx4xgxYkSgq9FsZs+eTURERKCrUS8JLEIIIUQDzJgxg9mzZ/teX3zxxTz44INNfpy7774bRVGYPn36ScutWLECRVEoLS1tkuOOGjWK3bt3N8m+moNcJXQSHo/KmqxiVKBPaiQ6reQ7IYQ4V5nN5mY/xueff86vv/5KUlJSk+3T4XA0aPBzUFCQ74qelki+gU/C4fYw6r1fufm9X7E63YGujhBCiD/Ap59+Snp6OkFBQURHRzNkyBAsFkuNLqFx48bxww8/MGPGDBRFQVEUsrKyANi6dSvDhg0jNDSU+Ph4xowZQ2Fhod/jHjlyhPvvv5958+ah1+tPWjYrK4tLLrkEgMjISBRFYdy4cYC35WfChAk8+OCDxMTEkJmZCcAbb7xBeno6ISEhpKSkcO+991JZWenb5++7hJ5//nkyMjKYO3cuaWlpmM1mbr75ZioqKhp4JpuWBJaT0JwwqllVA1gRIYQ4w6mqitXhCsiiNuIXeE5ODqNHj+aOO+5gx44drFixguuvv77WPmbMmMHAgQO56667yMnJIScnh5SUFEpLS7n00kvp1asXa9euZcmSJeTl5TFy5MiTHtfj8TBmzBgeffRRunXr5reeKSkpfPbZZwDs2rWLnJwcZsyY4Xt/zpw5GAwGfvnlF2bNmgWARqPhzTffZNu2bcyZM4fvv/+exx577KTH2bdvH4sWLWLx4sUsXryYH374gVdffdVv/ZqDdAmdxIlXYTXmB14IIURNVU43XSd9E5Bjb38xk2BDw77ucnJycLlcXH/99aSmpgKQnp5eq5zZbMZgMBAcHExCQoJv/cyZM+nVqxeTJ0/2rfvwww9JSUlh9+7ddOzYsc7jTp06FZ1Ox1//+tcG1VOr1RIVFQVAXFxcrcGyHTp04LXXXqux7sTxNmlpabz88svcfffdvPPOO/Uex+PxMHv2bMLCwgAYM2YMy5Yt45VXXmlQPZuSBJaTkBYWIYQ4t/Ts2ZPLLruM9PR0MjMzGTp0KDfeeCORkZEN2n7Tpk0sX76c0NDQWu/t27ePNWvW8Je//MW37uuvvyY4OJgZM2awfv36eucrGTZsGD/99BMAqampbNu27aT16N27d6113333HVOmTGHnzp2Ul5fjcrmw2WxYrdZ6ZyNOS0vzhRXwTq1fPc3+H00Cy0mc+GPjkcQihBCnLEivZfuLmQE7dkNptVqWLl3KypUr+fbbb3nrrbd4+umnWb16dYO2r6ysZPjw4UydOrXWe4mJiXg8Hvr37+9bl5yczD/+8Q/y8/Np3bq1b73b7ebhhx9m+vTpZGVl8cEHH1BVVQXgd3wLQEhISI3XWVlZXH311dxzzz288sorREVF8fPPP3PnnXficDjqDSy/P5aiKAG7maUElpOo0SUUuGoIIcQZT1GUBnfLBJqiKAwePJjBgwczadIkUlNT+fzzz2uVMxgMvlsQVDvvvPP47LPPSEtLQ6er+/Oe2GIB3m6WIUOG1FiXmZnJmDFjGD9+POANNnUdH6hVh7qsW7cOj8fDtGnTfDei/O9//+t3u5ZEBt2ehCJdQkIIcU5ZvXo1kydPZu3atWRnZ7Nw4UIKCgro0qVLrbJpaWmsXr2arKwsCgsL8Xg83HfffRQXFzN69GjWrFnDvn37+Oabbxg/fny9wSI6Opru3bvXWPR6PQkJCXTq1KneuqampqIoCosXL6agoKDGFT+/1759e5xOJ2+99Rb79+9n7ty5vsG4ZwoJLH5UZxYZdCuEEGe/8PBwfvzxR6688ko6duzIM888w7Rp0xg2bFitso888gharZauXbsSGxtLdnY2SUlJ/PLLL7jdboYOHUp6ejoPPvggERERvpaNppKcnMwLL7zAE088QXx8PBMmTKi3bM+ePXnjjTeYOnUq3bt3Z968eUyZMqVJ69PcFPUs+CYuLy/HbDZTVlZGeHh4k+673VNf4faorH7qMuLDTU26byGEOFvZbDYOHDhAmzZtMJnkd+e5rr6fh8Z8f59S3Hv77bdJS0vDZDLRv39/fvvtt5OW/+STT+jcuTMmk4n09HS++uqrGu+PGzfON/FO9XLFFVecStWaXHWnkAy6FUIIIQKn0YFlwYIFTJw4keeee47169fTs2dPMjMz673MaeXKlYwePZo777yTDRs2MGLECEaMGMHWrVtrlLviiit8k+/k5OTwn//859Q+UROrvrRZ8ooQQggROI0OLG+88QZ33XUX48ePp2vXrsyaNYvg4GA+/PDDOsvPmDGDK664gkcffZQuXbrw0ksvcd555zFz5swa5YxGIwkJCb6lode8N7tjTSzSwiKEEEIETqMCi8PhYN26dTUuv9JoNAwZMoRVq1bVuc2qVavqvFzr9+VXrFhBXFwcnTp14p577qGoqKjeetjtdsrLy2sszaW6S0jyihBCCBE4jQoshYWFuN1u4uPja6yPj48nNze3zm1yc3P9lr/iiiv46KOPWLZsGVOnTuWHH35g2LBh9V4CNmXKFMxms29JSUlpzMdoFE09sw4KIYQQ4o/TImbxufnmm33P09PT6dGjB+3atWPFihVcdtlltco/+eSTTJw40fe6vLy82UKLIl1CQgghRMA1qoUlJiYGrVZLXl5ejfV5eXk1bv50ooSEhEaVB2jbti0xMTHs3bu3zveNRiPh4eE1luYig26FEEKIwGtUYDEYDPTu3Ztly5b51nk8HpYtW8bAgQPr3GbgwIE1ygMsXbq03vIAhw8fpqioiMTExMZUr1nIZc1CCCFE4DX6KqGJEyfy/vvvM2fOHHbs2ME999yDxWLx3e9g7NixPPnkk77yDzzwAEuWLGHatGns3LmT559/nrVr1/pm5KusrOTRRx/l119/JSsri2XLlnHttdfSvn17MjMDc6OsE/lmug1sNYQQQohzWqMDy6hRo3j99deZNGkSGRkZbNy4kSVLlvgG1mZnZ5OTk+MrP2jQIObPn897771Hz549+fTTT1m0aBHdu3cHvHfG3Lx5M9dccw0dO3bkzjvvpHfv3vz0008YjcYm+pinTvF1CUlkEUKIc9m4ceMYMWJEoKvRbFasWIGiKJSWlga6KnWSqfn96PXit5RYnSx96EI6xIf530AIIcRZOTV/WVkZqqoSEREBwMUXX0xGRgbTp08/7X0vXLiQWbNmsW7dOoqLi9mwYQMZGRkn3SYrK4s2bdo0qGxDOBwOiouLiY+Pr3Hz36YQsKn5zyW+FpYA10MIIURgmc1mX1hpahaLhfPPP5+pU6c2+b4dDkeDyhkMBhISEpo8rDQVCSx+yKBbIYQ4t3z66aekp6cTFBREdHQ0Q4YMwWKx1OgSGjduHD/88AMzZszw3QMvKysLgK1btzJs2DBCQ0OJj49nzJgxFBYWnvSYY8aMYdKkSbUmWj2ZNm3aANCrVy8UReHiiy/21W3EiBG88sorJCUl0alTJwDmzp1Lnz59CAsLIyEhgVtuuaXGbXV+3yU0e/ZsIiIi+Oabb+jSpQuhoaG+2+gEggQWPxS5rFkIIU6fqoLDEpilEb/Ac3JyGD16NHfccQc7duxgxYoVXH/99bXGMc6YMYOBAwdy1113+e6Bl5KSQmlpKZdeeim9evVi7dq1LFmyhLy8PEaOHNnUZ9R34+HvvvuOnJwcFi5c6Htv2bJl7Nq1i6VLl7J48WIAnE4nL730Eps2bWLRokVkZWUxbty4kx7DarXy+uuvM3fuXH788Ueys7N55JFHmvyzNESLmDiuJZOJ44QQogk4rTA5KTDHfuooGEIaVDQnJweXy8X1119Pamoq4J3Q9PfMZjMGg4Hg4OAa84rNnDmTXr16MXnyZN+6Dz/8kJSUFHbv3k3Hjh1P88McFxsbC0B0dHStuc1CQkL44IMPMBgMvnV33HGH73nbtm1588036du3L5WVlYSGhtZ5DKfTyaxZs2jXrh0AEyZM4MUXX2yyz9AY0sLih6b6smbJK0IIcdbr2bMnl112Genp6dx00028//77lJSUNHj7TZs2sXz5ckJDQ31L586dAdi3bx/z5s2r8d5PP/3UoP3efffdNbbzJz09vUZYAVi3bh3Dhw+ndevWhIWFcdFFFwHeq3vrExwc7AsrAImJiTW6kf5I0sLih4J0CQkhxGnTB3tbOgJ17AbSarUsXbqUlStX8u233/LWW2/x9NNPs3r16gZtX1lZyfDhw+scPJuYmIjH46F///6+dcnJyQ3a74svvtiorpiQkJotShaLhczMTDIzM5k3bx6xsbFkZ2eTmZl50kG5er2+xmtFUQI2zYcEFj98LSxynZAQQpw6RWlwt0ygKYrC4MGDGTx4MJMmTSI1NZXPP/+8VjmDwVDrJr3nnXcen332GWlpaeh0dX/FhoU1foqMuLg44uLiah0fqPdGwSfauXMnRUVFvPrqq757761du7bR9Qgk6RLyo3rQrUfyihBCnPVWr17N5MmTWbt2LdnZ2SxcuJCCggK6dOlSq2xaWhqrV68mKyuLwsJCPB4P9913H8XFxYwePZo1a9awb98+vvnmG8aPH3/SYFFcXMzGjRvZvn07ALt27WLjxo3k5ubWu01cXBxBQUG+gb1lZWX1lm3dujUGg4G33nqL/fv388UXX/DSSy814swEngSWBjoL5tcTQgjhR3h4OD/++CNXXnklHTt25JlnnmHatGkMGzasVtlHHnkErVZL165dfV0sSUlJ/PLLL7jdboYOHUp6ejoPPvggERERaDT1f+V+8cUX9OrVi6uuugqAm2++mV69ejFr1qx6t9HpdLz55pv84x//ICkpiWuvvbbesrGxscyePZtPPvmErl278uqrr/L666834swEnsx068cFr33PoeIqFt47iPNaRzbpvoUQ4mx1Ns50K06dzHT7Bzg+6PaMz3VCCCHEGUsCix9yWbMQQggReBJY/JBBt0IIIUTgSWDxQ/G1sEhiEUIIIQJFAosfx29+GNBqCCGEEOc0CSx++G5+KBPHCSGEEAEjgcWP6kG3kleEEEKIwJHA4kf1Zc3SJSSEEEIEjgQWPxS5l5AQQggRcBJY/JDLmoUQQgCMGzeOESNGBLoazWb27NlEREQEuhr1ksDih0YuaxZCCAHMmDGD2bNn+15ffPHFPPjgg6e9X6fTyeOPP056ejohISEkJSUxduxYjh49etLtVqxYgaIolJaWnnYdAEaNGsXu3bubZF/NQQKLH4rMdCuEEAIwm83N0gJhtVpZv349zz77LOvXr2fhwoXs2rWLa665pkn273A4GlQuKCiIuLi4Jjlmc5DA4ofvXkIyhkUIIc4Jn376Kenp6QQFBREdHc2QIUOwWCw1uoTGjRvHDz/8wIwZM1AUBUVRyMrKAmDr1q0MGzaM0NBQ4uPjGTNmDIWFhfUez2w2s3TpUkaOHEmnTp0YMGAAM2fOZN26dWRnZ9e5TVZWFpdccgkAkZGRKIrCuHHjAG/Lz4QJE3jwwQeJiYkhMzMTgDfeeMPXipOSksK9995LZWWlb5+/7xJ6/vnnycjIYO7cuaSlpWE2m7n55pupqKg4xTN7eiSw+CH3EhJCiNOnqipWpzUgS2O69HNychg9ejR33HEHO3bsYMWKFVx//fW19jFjxgwGDhzIXXfdRU5ODjk5OaSkpFBaWsqll15Kr169WLt2LUuWLCEvL4+RI0c26nyVlZWhKEq9LTopKSl89tlnAOzatYucnBxmzJjhe3/OnDkYDAZ++eUXZs2aBYBGo+HNN99k27ZtzJkzh++//57HHnvspPXYt28fixYtYvHixSxevJgffviBV199tVGfpanoAnLUM4kMuhVCiNNW5aqi//z+ATn26ltWE6wPblDZnJwcXC4X119/PampqQCkp6fXKmc2mzEYDAQHB5OQkOBbP3PmTHr16sXkyZN96z788ENSUlLYvXs3HTt29FsHm83G448/zujRowkPD6+zjFarJSoqCoC4uLhawaZDhw689tprNdadON4mLS2Nl19+mbvvvpt33nmn3rp4PB5mz55NWFgYAGPGjGHZsmW88sorfj9HU5MWFj9k0K0QQpw7evbsyWWXXUZ6ejo33XQT77//PiUlJQ3eftOmTSxfvpzQ0FDf0rlzZ8DbWjFv3rwa7/300081tnc6nYwcORJVVXn33Xd966u7mEJDQ+nWrZvfevTu3bvWuu+++47LLruM5ORkwsLCGDNmDEVFRVit1nr3k5aW5gsrAImJieTn5/s9fnOQFhY/5F5CQghx+oJ0Qay+ZXXAjt1QWq2WpUuXsnLlSr799lveeustnn76aVavbljdKysrGT58OFOnTq31XmJiIh6Ph/79j7c0JScn+55Xh5WDBw/y/fff12hd+eCDD6iqqgJAr9f7rUdISEiN11lZWVx99dXcc889vPLKK0RFRfHzzz9z55134nA4CA6uuwXq98dSFAWPx+P3+M1BAosfGkXm5hdCiNOlKEqDu2UCTVEUBg8ezODBg5k0aRKpqal8/vnntcoZDAbcbneNdeeddx6fffYZaWlp6HR1f8We2GJRrTqs7Nmzh+XLlxMdHV3j/RODzYnHB2rVoS7r1q3D4/Ewbdo0NBpv58p///tfv9u1JNIl5Ed1XpEWFiGEOPutXr2ayZMns3btWrKzs1m4cCEFBQV06dKlVtm0tDRWr15NVlYWhYWFeDwe7rvvPoqLixk9ejRr1qxh3759fPPNN4wfP77eYOF0OrnxxhtZu3Yt8+bNw+12k5ubS25u7kkvSU5NTUVRFBYvXkxBQUGNK35+r3379jidTt566y3279/P3LlzfYNxzxQSWPzw3a1ZAosQQpz1wsPD+fHHH7nyyivp2LEjzzzzDNOmTWPYsGG1yj7yyCNotVq6du1KbGws2dnZJCUl8csvv+B2uxk6dCjp6ek8+OCDRERE+Fo2fu/IkSN88cUXHD58mIyMDBITE33LypUr661rcnIyL7zwAk888QTx8fFMmDCh3rI9e/bkjTfeYOrUqXTv3p158+YxZcqUxp+gAFLUs2A0aXl5OWazmbKysnpHVJ+qUf9YxeoDxcy8pRdX90hq0n0LIcTZymazceDAAdq0aYPJZAp0dUSA1ffz0Jjvb2lh8UO6hIQQQojAk8Dih8bXJSSJRQghhAgUCSx+yL2EhBBCiMCTwOKHr4VFLmsWQgghAkYCSwMFaJ4cIYQQQiCBxa/jLSxCCCGECBQJLH4oci8hIYQQIuAksPjhm5hf8ooQQggRMBJY/JBBt0IIIUTgSWDxQyaOE0IIATBu3DhGjBgR6Go0m9mzZxMRERHoatRLAosfci8hIYQQADNmzGD27Nm+1xdffDEPPvhgk+z7+eefp3PnzoSEhBAZGcmQIUNYvXr1SbdZsWIFiqJQWlraJHUYNWoUu3fvbpJ9NQcJLH5Uj2HxSGIRQohzmtlsbrYWiI4dOzJz5ky2bNnCzz//TFpaGkOHDqWgoOC0932yOz6fKCgoiLi4uNM+XnORwOKHXNYshBCnT1VVPFZrQJbGXuX56aefkp6eTlBQENHR0QwZMgSLxVKjS2jcuHH88MMPzJgxA0VRUBSFrKwsALZu3cqwYcMIDQ0lPj6eMWPGUFhYeNJj3nLLLQwZMoS2bdvSrVs33njjDcrLy9m8eXOd5bOysrjkkksAiIyMRFEUxo0bB3hbfiZMmMCDDz5ITEwMmZmZALzxxhukp6cTEhJCSkoK9957L5WVlb59/r5L6PnnnycjI4O5c+eSlpaG2Wzm5ptvpqKiolHns6noAnLUM4gilwkJIcRpU6uq2HVe74Acu9P6dSjBwQ0qm5OTw+jRo3nttde47rrrqKio4KeffqoVembMmMHu3bvp3r07L774IgCxsbGUlpZy6aWX8qc//Ym///3vVFVV8fjjjzNy5Ei+//77BtXB4XDw3nvvYTab6dmzZ51lUlJS+Oyzz7jhhhvYtWsX4eHhBAUF+d6fM2cO99xzD7/88otvnUaj4c0336RNmzbs37+fe++9l8cee4x33nmn3rrs27ePRYsWsXjxYkpKShg5ciSvvvoqr7zySoM+S1OSwOKHDLoVQohzR05ODi6Xi+uvv57U1FQA0tPTa5Uzm80YDAaCg4NJSEjwrZ85cya9evVi8uTJvnUffvghKSkp7N69m44dO9Z77MWLF3PzzTdjtVpJTExk6dKlxMTE1FlWq9USFRUFQFxcXK2uqg4dOvDaa6/VWHfieJu0tDRefvll7r777pMGFo/Hw+zZswkLCwNgzJgxLFu2TAJLS6TI3ZqFEOK0KUFBdFq/LmDHbqiePXty2WWXkZ6eTmZmJkOHDuXGG28kMjKyQdtv2rSJ5cuXExoaWuu9ffv2sWbNGv7yl7/41n399ddccMEFAFxyySVs3LiRwsJC3n//fUaOHMnq1auJi4tj2LBh/PTTTwCkpqaybdu2k9ajd+/arVnfffcdU6ZMYefOnZSXl+NyubDZbFitVoLraYFKS0vzhRWAxMRE8vPz/Z+IZiCBxY/jg24DWg0hhDijKYrS4G6ZQNJqtSxdupSVK1fy7bff8tZbb/H000/7vWKnWmVlJcOHD2fq1Km13ktMTMTj8dC/f3/fuuTkZN/zkJAQ2rdvT/v27RkwYAAdOnTgn//8J08++SQffPABVVVVAOj1er/1CAkJqfE6KyuLq6++mnvuuYdXXnmFqKgofv75Z+68804cDke9geX3x1IUBU+Abq4ngcUPGXQrhBDnFkVRGDx4MIMHD2bSpEmkpqby+eef1ypnMBhwu9011p133nl89tlnpKWlodPV/RV7YovFyXg8Hux2O1Az2Jx4fKBWHeqybt06PB4P06ZNQ6PxXm/z3//+t0H1aCnkKiE/5F5CQghx7li9ejWTJ09m7dq1ZGdns3DhQgoKCujSpUutsmlpaaxevZqsrCwKCwvxeDzcd999FBcXM3r0aNasWcO+ffv45ptvGD9+fL3BwmKx8NRTT/Hrr79y8OBB1q1bxx133MGRI0e46aab6q1ramoqiqKwePFiCgoKalzx83vt27fH6XTy1ltvsX//fubOncusWbMaf4ICSAKLHxqZOE4IIc4Z4eHh/Pjjj1x55ZV07NiRZ555hmnTpjFs2LBaZR955BG0Wi1du3YlNjaW7OxskpKS+OWXX3C73QwdOpT09HQefPBBIiIifC0bv6fVatm5cyc33HADHTt2ZPjw4RQVFfHTTz/RrVu3euuanJzMCy+8wBNPPEF8fDwTJkyot2zPnj154403mDp1Kt27d2fevHlMmTKl8ScogBT1LGg6KC8vx2w2U1ZWRnh4eJPue+KCjSzccIQnh3XmLxe1a9J9CyHE2cpms3HgwAHatGmDyWQKdHVEgNX389CY729pYfGnuksosLUQQgghzmmnFFjefvtt0tLSMJlM9O/fn99+++2k5T/55BM6d+6MyWQiPT2dr776qt6yd999N4qiMH369FOpWpOTLiEhhBAi8BodWBYsWMDEiRN57rnnWL9+PT179iQzM7Pe67JXrlzJ6NGjufPOO9mwYQMjRoxgxIgRbN26tVbZzz//nF9//ZWkpKTGf5JmIvcSEkIIIQKv0YHljTfe4K677mL8+PF07dqVWbNmERwczIcfflhn+RkzZnDFFVfw6KOP0qVLF1566SXOO+88Zs6cWaPckSNHuP/++5k3b16DrjH/o2h8c/MLIYQQIlAaFVgcDgfr1q1jyJAhx3eg0TBkyBBWrVpV5zarVq2qUR4gMzOzRnmPx8OYMWN49NFHTzoiOhB8U/PLzHFCCCFEwDRq4rjCwkLcbjfx8fE11sfHx7Nz5846t8nNza2zfG5uru/11KlT0el0/PWvf21QPex2u28yHfCOMm4uikwcJ4QQQgRcwK8SWrduHTNmzGD27Nm+cODPlClTMJvNviUlJaXZ6nf85ocSWYQQQohAaVRgiYmJQavVkpeXV2N9Xl5ejbtVnighIeGk5X/66Sfy8/Np3bo1Op0OnU7HwYMHefjhh0lLS6tzn08++SRlZWW+5dChQ435GI1SHaEkrwghhBCB06jAYjAY6N27N8uWLfOt83g8LFu2jIEDB9a5zcCBA2uUB1i6dKmv/JgxY9i8eTMbN270LUlJSTz66KN88803de7TaDQSHh5eY2kuci8hIYQQIvAa3SU0ceJE3n//febMmcOOHTu45557sFgsjB8/HoCxY8fy5JNP+so/8MADLFmyhGnTprFz506ef/551q5d65tCODo6mu7du9dY9Ho9CQkJdOrUqYk+5qmTewkJIYSoi6IoLFq0qMHlx40bx4gRI07rmFlZWSiKwsaNG09rP43x/PPPk5GR8Ycdrz6NDiyjRo3i9ddfZ9KkSWRkZLBx40aWLFniG1ibnZ1NTk6Or/ygQYOYP38+7733Hj179uTTTz9l0aJFdO/evek+RTOSieOEEOLck5ubywMPPED79u0xmUzEx8czePBg3n33XaxWa6Crd1KzZ88mIiKiyfb3yCOP1OopCYRGXSVUbcKECfXeZGnFihW11t10000nvePk72VlZZ1KtZqc6vEQfXA3nYuP4nGnBbo6Qggh/gD79+9n8ODBREREMHnyZNLT0zEajWzZsoX33nuP5ORkrrnmmkBX87Q5HA4MBoPfcqGhoYSGhv4BNTq5gF8l1JKpLhdD3n6Kv/84E63D7n8DIYQQdVJVFafdHZClsV369957LzqdjrVr1zJy5Ei6dOlC27Ztufbaa/nyyy8ZPnx4ndtt2bKFSy+9lKCgIKKjo/nzn/9MZWVlrXIvvPACsbGxhIeHc/fdd+NwOHzvLVmyhPPPP5+IiAiio6O5+uqr2bdvX4PrvmLFCsaPH09ZWRmKoqAoCs8//zwAaWlpvPTSS4wdO5bw8HD+/Oc/A/D444/TsWNHgoODadu2Lc8++yxOp9O3z993CVV3bb3++uskJiYSHR3NfffdV2Ob5nBKLSznihMvs1Y9ngDWRAghzmwuh4f3HvghIMf+84yL0Bu1DSpbVFTEt99+y+TJkwkJCamzTF1TcFgsFjIzMxk4cCBr1qwhPz+fP/3pT0yYMIHZs2f7yi1btgyTycSKFSvIyspi/PjxREdH88orr/j2M3HiRHr06EFlZSWTJk3iuuuuY+PGjWg0/tsYBg0axPTp05k0aRK7du0CqNE6Uj2k47nnnvOtCwsLY/bs2SQlJbFlyxbuuusuwsLCeOyxx+o9zvLly0lMTGT58uXs3buXUaNGkZGRwV133eW3jqdKAsvJnPjD4ZbAIoQQZ7u9e/eiqmqtiz5iYmKw2WwA3HfffUydOrXG+/Pnz8dms/HRRx/5gs7MmTMZPnw4U6dO9Y3zNBgMfPjhhwQHB9OtWzdefPFFHn30UV566SU0Gg033HBDjf1++OGHxMbGsn379gaN/TQYDJjNZhRFqXO6kUsvvZSHH364xrpnnnnG9zwtLY1HHnmEjz/++KSBJTIykpkzZ6LVauncuTNXXXUVy5Ytk8ASMCcGFhl1K4QQp0xn0PDnGRcF7Nin67fffsPj8XDrrbfWmGm92o4dO+jZs2eNVpnBgwfj8XjYtWuXL7D07NmT4OBgX5mBAwdSWVnJoUOHSE1NZc+ePUyaNInVq1dTWFiI51jrfnZ2dp2BpVu3bhw8eBCACy64gK+//vqkn6NPnz611i1YsIA333yTffv2UVlZicvl8jtdSLdu3dBqj7daJSYmsmXLlpNuc7oksJxEjS4haWERQohTpihKg7tlAql9+/YoiuLrTqnWtm1bAIKCgpr1+MOHDyc1NZX333+fpKQkPB4P3bt3rzHO5URfffWVb+xIQ+r2+26uVatWceutt/LCCy+QmZmJ2Wzm448/Ztq0aSfdz+9vUqwoii9cNRcJLH6oigZF9aCqEliEEOJsFx0dzeWXX87MmTO5//776x3H8ntdunRh9uzZWCwW3za//PILGo2mRvfSpk2bqKqq8oWLX3/9ldDQUFJSUigqKmLXrl28//77XHDBBQD8/PPPJz1uampqrXUGgwG3292geq9cuZLU1FSefvpp37rqFpuWRq4S8kOtnodFBt0KIcQ54Z133sHlctGnTx8WLFjAjh072LVrF//+97/ZuXNnja6Qarfeeismk4nbb7+drVu3snz5cu6//37GjBlT4wbADoeDO++8k+3bt/PVV1/x3HPPMWHCBDQaDZGRkURHR/Pee++xd+9evv/+eyZOnNjo+qelpVFZWcmyZcsoLCw86bwxHTp0IDs7m48//ph9+/bx5ptv8vnnnzf6mH8ECSx+qMfvfhjYigghhPhDtGvXjg0bNjBkyBCefPJJevbsSZ8+fXjrrbd45JFHeOmll2ptExwczDfffENxcTF9+/blxhtv5LLLLmPmzJk1yl122WV06NCBCy+8kFGjRnHNNdf4LjvWaDR8/PHHrFu3ju7du/PQQw/xt7/9rdH1HzRoEHfffTejRo0iNjaW1157rd6y11xzDQ899BATJkwgIyODlStX8uyzzzb6mH8ERT0L5pwvLy/HbDZTVlbW5PcV2preE63TwRdPvMPj4y5p0n0LIcTZymazceDAAdq0aYPJZAp0dUSA1ffz0Jjvb2lh8UM9fjOhwFZECCGEOIdJYPHHN4ZFAosQQggRKBJY/FAV7ymSQbdCCCFE4Ehg8cfXJSSBRQghhAgUCSx+qNWz3UoLixBCNNpZcF2HaAJN8XMggcUfGXQrhBCNVj1XSX0ztIpzS/VcML+fIbcxZKbbk1A9KpbgROyEy9T8QgjRCDqdjuDgYAoKCtDr9Q2607A4+6iqitVqJT8/n4iIiDon3WsoCSwn4XGrbOj6V+8LV2VgKyOEEGcQRVFITEzkwIEDLXaqd/HHiYiIqPPu0Y0hgeVkTvyDQLqEhBCiUQwGAx06dJBuoXOcXq8/rZaVahJYTuLEuzXL1PxCCNF4Go1GZroVTUI6FU/ixLyiyBgWIYQQImAksJyEoii+riCZOE4IIYQIHAksfh3rCpIxLEIIIUTASGDxQ/EFlsDWQwghhDiXSWDxp7plRbqEhBBCiICRwOJHdQuLIl1CQgghRMBIYPFLWliEEEKIQJPA4oevZUXmYRFCCCECRgKLX8cua5YuISGEECJgJLD44btKSFpYhBBCiICRwOJXdWAJbC2EEEKIc5kEFj+qZ+dXVEksQgghRKBIYPGnOqjIGBYhhBAiYCSw+OFrYZExLEIIIUTASGA5GbcTRXUCoPU4A1wZIYQQ4twlgeVkVBWN6gZAOfYohBBCiD+eBJaTUTTHp+aXmW6FEEKIgJHAcjIaLfgCS2CrIoQQQpzLJLCcjKL4LmdWZCIWIYQQImAksPhxvEtIrhISQgghAkUCi1/SwiKEEEIEmgQWP6SFRQghhAg8CSx++G5+KDPdCiGEEAEjgcWfY0FFI/cSEkIIIQJGAosf1WNXFGlgEUIIIQJGAosfx7uEpIVFCCGECBQJLH6pNR6EEEII8ceTwOJHdQuLRgbdCiGEEAEjgcWvY5c1SxOLEEIIETASWPzwTRgn87AIIYQQASOBxQ9FWliEEEKIgJPA4ocvsEheEUIIIQJGAotfklSEEEKIQJPA4odS/ShXCQkhhBABI4HFD9+gW8krQgghRMCcUmB5++23SUtLw2Qy0b9/f3777beTlv/kk0/o3LkzJpOJ9PR0vvrqqxrvP//883Tu3JmQkBAiIyMZMmQIq1evPpWqNTmZh0UIIYQIvEYHlgULFjBx4kSee+451q9fT8+ePcnMzCQ/P7/O8itXrmT06NHceeedbNiwgREjRjBixAi2bt3qK9OxY0dmzpzJli1b+Pnnn0lLS2Po0KEUFBSc+idrKjLaVgghhAg4RVUb13TQv39/+vbty8yZMwHweDykpKRw//3388QTT9QqP2rUKCwWC4sXL/atGzBgABkZGcyaNavOY5SXl2M2m/nuu++47LLL/NapunxZWRnh4eGN+Th+fTZ+OrnGHgRZf+GOj55t0n0LIYQQ57LGfH83qoXF4XCwbt06hgwZcnwHGg1Dhgxh1apVdW6zatWqGuUBMjMz6y3vcDh47733MJvN9OzZs84ydrud8vLyGkvzkXsJCSGEEIHWqMBSWFiI2+0mPj6+xvr4+Hhyc3Pr3CY3N7dB5RcvXkxoaCgmk4m///3vLF26lJiYmDr3OWXKFMxms29JSUlpzMdoFMX3RDlZMSGEEEI0oxZzldAll1zCxo0bWblyJVdccQUjR46sd1zMk08+SVlZmW85dOhQs9TJ5XFRobi9L1RPsxxDCCGEEP41KrDExMSg1WrJy8ursT4vL4+EhIQ6t0lISGhQ+ZCQENq3b8+AAQP45z//iU6n45///Ged+zQajYSHh9dYmoNH9VCqqQ4q0ickhBBCBEqjAovBYKB3794sW7bMt87j8bBs2TIGDhxY5zYDBw6sUR5g6dKl9ZY/cb92u70x1WtyGkUDMjW/EEIIEXC6xm4wceJEbr/9dvr06UO/fv2YPn06FouF8ePHAzB27FiSk5OZMmUKAA888AAXXXQR06ZN46qrruLjjz9m7dq1vPfeewBYLBZeeeUVrrnmGhITEyksLOTtt9/myJEj3HTTTU34URtPq2hRjwUWyStCCCFE4DQ6sIwaNYqCggImTZpEbm4uGRkZLFmyxDewNjs7G43meMPNoEGDmD9/Ps888wxPPfUUHTp0YNGiRXTv3h0ArVbLzp07mTNnDoWFhURHR9O3b19++uknunXr1kQf89QoioKvhSWgNRFCCCHObY2eh6Ulas55WKb/aTJ63QCCK39i/L+fa9J9CyGEEOeyZpuH5VykKt5BtzKGRQghhAgcCSx+yRgWIYQQItAksPglY1iEEEKIQJPA4o/0BQkhhBABJ4HFr+rAIm0sQgghRKBIYPFDVWTiOCGEECLQJLD4oeC9SkiVBhYhhBAiYCSw+HMsqCiSWIQQQoiAkcDiV/XNDyWwCCGEEIEigcUPGboihBBCBJ4EFn9ktK0QQggRcBJY/JIuISGEECLQJLD4oUgLixBCCBFwElgaTFpYhBBCiECRwOKHqshMt0IIIUSgSWDxQ5HAIoQQQgScBJYGk8AihBBCBIoEFn8Uj/8yQgghhGhWElj8UWo9EUIIIcQfTAKLHzKGRQghhAg8CSwNpqCqMieLEEIIEQgSWPw4sYVF8ooQQggRGBJY/FBPGMPikcQihBBCBIQEFj98LSyKInduFkIIIQJEAosfirSwCCGEEAEngcWPE7uEJK8IIYQQgSGBxQ9F5mERQgghAk4Cix+K5vhVQtIlJIQQQgSGBBZ/pEtICCGECDgJLH7IoFshhBAi8CSw+FEdWFRFI5c1CyGEEAEigcUPxXeGFFS5cbMQQggREBJY/PH1CSmo0sYihBBCBIQEFj80J7awSF4RQgghAkICix8n3vxQBt0KIYQQgSGBxZ/qJha5l5AQQggRMBJY/NBUXyUkLSxCCCFEwEhg8UOpTixokCYWIYQQIjAksPhzwqBbjwQWIYQQIiAksPhR3cKiKnJZsxBCCBEoElj80Jw4D4vkFSGEECIgJLD4cXymW40MuhVCCCECRAKLH8qxy5pVRVpYhBBCiECRwOLH8auEJLAIIYQQgSKBxQ+NRu4lJIQQQgSaBBY/NCdcJSSXNQshhBCBIYHFD0V7fOI4VfqEhBBCiICQwOKH4rtds7SwCCGEEIEigcUPzQlXCcnc/EIIIURgSGDxQ3NCC4v0CAkhhBCBIYHFD4322CmSQbdCCCFEwEhg8UM5FlhUNHJZsxBCCBEgElj8OHEMi8cT4MoIIYQQ5ygJLH5oNNpjz2TiOCGEECJQTimwvP3226SlpWEymejfvz+//fbbSct/8skndO7cGZPJRHp6Ol999ZXvPafTyeOPP056ejohISEkJSUxduxYjh49eipVa3K+MSxoZNCtEEIIESCNDiwLFixg4sSJPPfcc6xfv56ePXuSmZlJfn5+neVXrlzJ6NGjufPOO9mwYQMjRoxgxIgRbN26FQCr1cr69et59tlnWb9+PQsXLmTXrl1cc801p/fJmohG521hkZsfCiGEEIGjqI2cvrV///707duXmTNnAuDxeEhJSeH+++/niSeeqFV+1KhRWCwWFi9e7Fs3YMAAMjIymDVrVp3HWLNmDf369ePgwYO0bt3ab53Ky8sxm82UlZURHh7emI/j1zeLprN3SQ+0LhuDnh9Ej1YRTbp/IYQQ4lzVmO/vRrWwOBwO1q1bx5AhQ47vQKNhyJAhrFq1qs5tVq1aVaM8QGZmZr3lAcrKylAUhYiIiDrft9vtlJeX11iai0anA0BVNHjkumYhhBAiIBoVWAoLC3G73cTHx9dYHx8fT25ubp3b5ObmNqq8zWbj8ccfZ/To0fWmrSlTpmA2m31LSkpKYz5Go2i0xwbdKgqq291sxxFCCCFE/VrUVUJOp5ORI0eiqirvvvtuveWefPJJysrKfMuhQ4earU7a6suaUaSFRQghhAgQXWMKx8TEoNVqycvLq7E+Ly+PhISEOrdJSEhoUPnqsHLw4EG+//77k/ZlGY1GjEZjY6p+yhTd8RYWPNLCIoQQQgRCo1pYDAYDvXv3ZtmyZb51Ho+HZcuWMXDgwDq3GThwYI3yAEuXLq1Rvjqs7Nmzh++++47o6OjGVKtZ6bR6QFpYhBBCiEBqVAsLwMSJE7n99tvp06cP/fr1Y/r06VgsFsaPHw/A2LFjSU5OZsqUKQA88MADXHTRRUybNo2rrrqKjz/+mLVr1/Lee+8B3rBy4403sn79ehYvXozb7faNb4mKisJgMDTVZz0lGn11C4sGj4xhEUIIIQKi0YFl1KhRFBQUMGnSJHJzc8nIyGDJkiW+gbXZ2dkn3OEYBg0axPz583nmmWd46qmn6NChA4sWLaJ79+4AHDlyhC+++AKAjIyMGsdavnw5F1988Sl+tKahrR50C6gumZtfCCGECIRGz8PSEjXnPCzr1nzCr//0dlH1+ktbBvVKa9L9CyGEEOeqZpuH5Vyk1R9vhFLl7odCCCFEQEhg8UNzbNAtIPOwCCGEEAEigcUPre54YPG4pYVFCCGECAQJLH5odce7hDwy6FYIIYQICAksfuhOuEoIGcMihBBCBIQEFj802hNaWCSwCCGEEAEhgcUP3QmBRXWd8VeACyGEEGckCSx+aJQT5tZT5SohIYQQIhAksPih0+pB9XYFeaSFRQghhAgICSx+aDQ6lGOTAcsYFiGEECIwJLD4odHoAG9QkYnjhBBCiMCQwOKHVqsHjrWwuF2BrYwQQghxjpLA4seJXUK4HYGtjBBCCHGOksDih1ajw9fCIjPdCiGEEAEhgcUPbwuLN6i43c4A10YIIYQ4N0lg8UOrOXEMiwQWIYQQIhAksPihOSGwqB65SkgIIYQIBAksfmg1Oqieh8UlVwkJIYQQgSCBxQ+NRofia2GRwCKEEEIEggQWfzRa39T8yEy3QgghREBIYPFHUTh+WbMMuhVCCCECQQJLA1R3CUkLixBCCBEYElga5NjdmiWwCCGEEAEhgaUhVLmsWQghhAgkCSwNciywyN2ahRBCiICQwNIg3q4gVbqEhBBCiICQwNIAinQJCSGEEAElgaVB5CohIYQQIpAksDTIsRYWVQKLEEIIEQgSWBpEWliEEEKIQJLA0iDVg27VANdDCCGEODdJYGmA4zc/lEG3QgghRCBIYGmQ6i4haWERQgghAkECS0McG2yrqhJYhBBCiECQwNIgx4KKBBYhhBAiICSwNIhcJSSEEEIEkgSWBqm+W7O0sAghhBCBIIGlQdTfPQohhBDijySBpUHkKiEhhBAikCSwNIAig26FEEKIgJLA0iDV9xIKcDWEEEKIc5QElgZRazwIIYQQ4o8lgaVBjl3OLE0sQgghREBIYGkQGcMihBBCBJIElgaRLiEhhBAikCSwNIAig26FEEKIgJLA0gCqTBwnhBBCBJQElgZQqgfdepTAVkQIIYQ4R0lgaRBpWRFCCCECSQJLY8ggFiGEECIgJLA0iCfQFRBCCCHOaRJYGkBR5LJmIYQQIpAksDSCqsqgWyGEECIQJLA0iLdLSJEWFiGEECIgTimwvP3226SlpWEymejfvz+//fbbSct/8skndO7cGZPJRHp6Ol999VWN9xcuXMjQoUOJjo5GURQ2btx4KtVqRscmjkNaWIQQQohAaHRgWbBgARMnTuS5555j/fr19OzZk8zMTPLz8+ssv3LlSkaPHs2dd97Jhg0bGDFiBCNGjGDr1q2+MhaLhfPPP5+pU6ee+idpTseaVqSFRQghhAgMRVUbd61u//796du3LzNnzgTA4/GQkpLC/fffzxNPPFGr/KhRo7BYLCxevNi3bsCAAWRkZDBr1qwaZbOysmjTpg0bNmwgIyOjwXUqLy/HbDZTVlZGeHh4Yz5Og/xr3ItYTeejd6/gz++/2OT7F0IIIc5Fjfn+blQLi8PhYN26dQwZMuT4DjQahgwZwqpVq+rcZtWqVTXKA2RmZtZbviHsdjvl5eU1luakVPcESQuLEEIIERCNCiyFhYW43W7i4+NrrI+Pjyc3N7fObXJzcxtVviGmTJmC2Wz2LSkpKae8r4apTioyhkUIIYQIhDPyKqEnn3ySsrIy33Lo0KFmPmL1PCwSWIQQQohA0DWmcExMDFqtlry8vBrr8/LySEhIqHObhISERpVvCKPRiNFoPOXtG0uR0bZCCCFEQDWqhcVgMNC7d2+WLVvmW+fxeFi2bBkDBw6sc5uBAwfWKA+wdOnSesu3TNIlJIQQQgRSo1pYACZOnMjtt99Onz596NevH9OnT8disTB+/HgAxo4dS3JyMlOmTAHggQce4KKLLmLatGlcddVVfPzxx6xdu5b33nvPt8/i4mKys7M5evQoALt27QK8rTOn0xLTdKRLSAghhAikRgeWUaNGUVBQwKRJk8jNzSUjI4MlS5b4BtZmZ2ej0RxvuBk0aBDz58/nmWee4amnnqJDhw4sWrSI7t27+8p88cUXvsADcPPNNwPw3HPP8fzzz5/qZ2s6SvWDBBYhhBAiEBo9D0tL1NzzsHz0p2eo0F2K3vELf/7w2SbfvxBCCHEuarZ5WM5d0iUkhBBCBJIElgZQfI8SWIQQQohAkMDSEEqtJ0IIIYT4A0lgaZBjd2s+40f7CCGEEGemRl8ldC5SfFcJSb4TQpzZVFUlr9zO7rwKcsttFFTYfUt+hQ2PCpHBBlKigmgfF0qHuDA6xIUSGWIIdNXFOU4CS4Mca2GRLiEhxBmkrMrJ7rwKduWesORVUFblbPS+okMMdEkMp1+bKPq3iSKjdQRGnbYZai1E3SSwNIQ0rAghWjC7y82+fAu78srZeSyY7M6t4GiZrc7yWo1CWnQwrSKDiQ0zEhdmPPZoQqNAkcVBVqGFvQWV7Mmr5EhpFUUWBz/vLeTnvYUAGHUahnSNZ9ygNPqkRqIo8gedaF4SWBpAOdbCIlcJCSECyeNROVRi9YWSXcdaTw4UWnB76h5kl2Q20TEhjE4JYXROCKNjfBjtYkMx6RveOmKxu9hXUMmmQ6X8eqCY1fuLKay08+XmHL7cnEPbmBCuPy+ZEb2SaRUZ3FQfV4gaJLA0gKIAqnQJCSGan8ejUlBp53CJlcMlVRwprSK7yBtSdudVYHW469wu3KSjc0I4nY6Fk07Hwok5SH/adQox6ujRKoIerSIYMzANVVXZdrScf/96kP9tPMr+Qguvf7ub17/dzUUdY3k0sxPdk82nfVwhTiSBpUGUY/+VviEhRNOptLvYfrScrUfKvMvRMrIKrTjcnnq3Meg0tI8NpXNCGJ0TvaGkc0I48eHGP6xbRlEUuiebefWGHjxzdVeWbM1l4frDrNpfxA+7C/hhdwHX9Urm4aEdpcVFNBkJLA2gKKq0sAghTltumY1vtuWy7mAJW4+UcaDIUud0CVqNQkK4ieTIIFpFBNEqMoiOx7p00qJD0Glbzh9PoUYdN/ZuxY29W3GwyMIbS3fzv41H+XzDEb7cnMNdF7ZhwiUdCDLIAF1xeiSwNET1Zc0yNb8QooFsTjc7cyvYcriULUfK2Hy4jJ25FbXKJZpNdE820z3JTPfkcDrGh5FoNrWoUNJQqdEhzLi5F386vy2Tv9rBqv1FvL18H4s2HOX5a7pxedf4QFdRnMEksDRAdSurtLAIIericnvYmVvB5sNlbDlSyubDZezKrcBVx0DY3qmRXNIplvRWEXRLCicm1BiAGjev9FZm5t/Vn2+25fHi/23jSGkVd320liFd4pl6QzrRZ+FnFs1PAktD+CaOk8AihPBOvrY3v5If9xSycm8hqw8UU2l31SoXFWKgRyszPZLNdE82k9E6grgwUwBq/MdTFIUruidwYccY3vp+Lx/8tJ/vduRx1ZtlzLylF33SogJdRXGGkcDSAMcHsklgEeJcVVRpZ01WCVuOlPLttjz25FfWeD/cpKNnSgTpyWZ6tDKT3iqCJLPp9AfCul3gtILq9t4fRFW9z90OqCqBqlJvOdUDzipwWryPDiu4bGAIgaAIMB1bgiIgOMr7/A8YpBts0PH4FZ25NiOJe+etZ3+BhVveX83fR2VwVY/EZj++OHtIYGkARW5+KMQ5RVVVDpdU+a7cWXOghLUHizmxh8eg1TCgXTSD20UzuH0MXRPD0WiO/Y5wVkHZEcg6CrYysFeArdz7aK9+PPbcYQWtDjQ6cNrAUgBlh8BlB0UDnsbPStsgGh0Ex0BILITEQHgyJPeC+O4QlgjmVqBpuoGynRPC+b8J5zPxvxv5ZlseE/6znoKKrowb3KbJjiHObhJYGqD6d5AHhTW5a+ib0DewFRJCNJkSi4OcMhvZxZZjY1C8A2RPnL5ei5soKukd6+a8aDc9opxkRLsJcmyEyiJYWQiWQrAWQUUuVBU3TeXUuudcAUCj97aWBEV6gw0K6INAHwyGYO9znQkcFm8rjK30+KOjEjwuqMz1LtU2/vv4c30wxHXxPupMEN0eYjt5l4QeYAxt9McJMep459bePP/FNub+epDn/287+RV2Hs3sJDPlCr8ksDRAuMZEHoCicNe3dzHtomlclnpZoKslhGgAVVWP3djPTpHFQWG5jUP5+Rw+fJiCvByUqiKiKCdSqSBKqWAYFdymVBBtqCBBV0mkUkGIu9y7s4pjS1YDDqwPgfAkb/eLMeyEJbzmoyEYPG5vgNCZvCEkItUbFFS391Ef7G3tqA4mmtO8gshl9wYsSwFYj4Wton1wZK33sSLH2w11ZN3xbfYuPf5ca4R2l0BUO29giu8G8V29LTaGkJN2NWk1Ci9e2434cCOvf7ubd1bso8Lm4sVru0loESclgaUBtIr3l4NO1eBW3byx7g0uaX0JGuXMu+xQiEByuB2U2kupdFRS5arC6rJ6H53eR6fHicvj8i6q6/jzY4vFaSHXmovVaQWOjy+rHhDvVt24nXasVZU47FZczircLicKLjS40ShuNHhQFRW3BtREBbcCh/C2oHoU0KhgUj2YVJUgj4pJNWFSjZhUlWBFT5jORLguBLMhjHBjBOGmKMJDYgkPSSA8LBGzOQ19ZGqtMSJujxuX6sLtcfs+p1t141E9mLQmgvXB6DX6P+ZLW2cEc7J3qYvH7Q0uBTu9Y2UclVC4Bwp2Qf52KD8Cu5fUvW1wNLTq6+1SMoR6u5iSz/OGMK33K0dRFCZc2oGYUCNPfr6Fub8eRKdVmHR1Vwktol4SWBriWC4xqlrCDGFkV2Sz4tAKLm19aUCrJUSgqKqK1WWl1F5Kqb2UMlsZJfYS73N7mW99qa20xjqry/rHVlR7bPHReBdVweA2YXQFY3KFYHJ6H6tfG10hmJwhGN1BaFTtsTmYFFRVoRwoV1QOKm48ihu3xoVHceNRynErJXg0m9FoFRQ0KB4NeBQ0Hg2KqkHr0aFVdb5HzbFH9ViIAhVFo0GjaNAqGjRo0Gg0aNCiUTS1F42CVqtBq9OiRQMuDW4HuB0e3HYVjxM0WgWtXoNer0Wn16LVa9DpNWj1GozBeoLC9ASF6jGFGo49N2AK1RMUmkJQu3bojb8bx6KqkL/D2+JiOdY6k7MJivaC2+7tFqsrzChaiEyFpF4Q3QFC47g5LJ64i+zM/XEHh1et5d3iNvz52svQRdQTpMQ5TQJLA5x4ldDIjiP559Z/8tH2jySwiLOaqqrkWfM4UHaAg+UHySrPIqs8i4NlB8mz5uE8xcGgGkVDqD6UYH0wwbpggnRBBOuDMWlN6DV6dBodOtWDzlmF3mFF57Cis1eisVdgtJaTWFVKuOf4JcQejwm3OwyXOwzVHYLHHYbVE42DODxKNFpNBFrC8LhMuJ0G3A4tbvuZ/1e8CriPLTXX1OR2q7gdbhx1vNcQOr2GoDADoZHGY4uJ0KgwQiNvIbSV93VQmN7bxuWweMPMkXXeK5isRXB0PeRu9YaZ4v3e5QSXApcajr04AEwH1ZyC0noApPSHjpkQ0fqU6i7OLhJYGuLYqFsVhdGdRzNn2xzW5a1jS8EW0mPTA1w5IU6PqqoUVhWSVZ7FnpI97C3d63usdFaedFuDxkCEKYIIY83FbDR7n5tqrw8zhKHxuKHsMJQcgJIs71Lkfa6WZKHYy1FVsKshVHnMVHnMVLpjqHB3pNIdQ5k7jhJ3AlZPNB715POanCxW6fQajMFajHoVg+JA77Ghd1SiqypF57Rg8NhRVBeoKhqtBkWjQdFpwWiC4DAIDkEJCkENCgFjEB6DEYdWQxUeUECn03oXvRad1vuo1+t8i06vRaNVcHvc2Fx2bE4bNpcdh9uB3W2vsTh+99rusmN3O3A4HThdLhxuB1YqsStVYFBR9CoerYtKm4VKWyW4Neg8erSqHq1Hh86jx+gKJsgZiskZSpArlCBXGGHuCIJcYejtJhSPBpfTQ0WxjYpiW73nUavTEBJpJCzSSFhUCGExmYTHmAhPCSLq4hBMQVqozPN2Jx3d4P1/X5nvXWcrBWMYpTYPZUW5tCIfbdkh2HIItnwCXz0CMZ28g3wjUqH3OGhz4R9ySbZoWSSwNMCJLSzxIfFc2fZKvtj3BdPXT+eDoR9In6s4Y5TYSthRtIPtxdvZUbSDrPIsDlUcospVVWd5naIjJTyF1PBU2oS3ITU8ldTwVJJCk4gwRhCkC6r759/j8Q7orDgKJQehZL0vnDiLjlBebMXijsDijsTiicbijsLqaY/V0xubJ5yqY4vawF9ROr0GU6geY7AOk0nBoPNgVJzoVTt6lxW9swKdrQJdVSnayhI0FUUopQWoOUfwWCync0rrptWijYhAGxGBotejaDTegbIaje959aOq0aDV6QjV6QjT644NrlVA+d0cUIpy/EtaUVA0CpqQUDThYd51bjeqOwrV4cBTZUWtsuGx2UAJB60GJ27sqhM7LmwaF1atm0pNKWVKLgVqObnuUsq1Tgr04DCBPQJcOhNoQtEQRqgmkURDO2KUZMJcMRiqgnFXaKiqcOJ2eSgvqKK8oO6fo9BII9HJoUQnpxHdqhvR7UKJiAtGqz8+DjACWLM9j6vn/kQPZS9/bV9Ef2UbZK+Cwl3eQkfWwbaF3suuUwdB6mBIOx9iOkqAOQcoqlrXrbfOLOXl5ZjNZsrKyggPD2/y/S9/ajLbiwcQZs1i7Ed3cKTyCMM/H47T4+Tty97mwlYXNvkxhTgdbo+b7IpsdpXsYnfxbnaV7GJn8U7yrfl1ltcoGhJDEukQ0YEOkR1oH9GeDpEdSAtPQ6/V197AVn4sgBz0zhlSegjKDuEsLcRSaqey3EOlO/JYq0gMle4YKj0xWNzR2NVGXg6rV9AH64mMCSIiUk+wUoXJWYqxPA9dYTb6o/ugOB93cTHusjLqvJugH9rYGAxJyegSE9EnJKBPTEATGgZaDYpWCyiobheq0+kNBBYr7pIS3MXFuEpLcJeUel+XlOCpPHmr1NnAqQW7Hhw6cBgNuEJicYXGowbF4zbF4tBH49RGYCWMKnfd0/ArCoRHGYhMDCEqOZzo5BAS20ewaFcuz/5vGwDTburJDZ0MkLsZXA7Ytww2/sc7Od6JIlKh05XQaZg3yNT1MytapMZ8f0tgaYAVz0xhW2F/wqoOMnbOeADeWPsG/9r2L9qa2/LZNZ+h00hjlQgMVVXZW7qX9Xnr2Vmyk93Fu9lTuqfeVpPU8FS6RHWhS3QX2ke0p3VYa5JDk+sOJuBtLTmyDnXHYir3bqO0wE6FxUClO5pKTzSWEx4bGkaceKjQgE0LulA9YREG4sI1JJhcxOvthLkq0JUXoC3LRy3Mx5WXhyM7G3dxw+Y30ZjN6CIj0UZGoo2KQhsZgS4iAk24Ga25eglHF5+APikRjanppstXHQ5cJaW4jwUZ1eUC1YPqdoNHrf3c5faFIVwuVLfneOhSvQNxq5/7fl2rgMeDu7ICT1m5t8VGqwWtFsWgRxMUjCbIhGIyHSvrRnW5fY+q04GnyoZqq8JTZcNjq/K2yFRVHX9us6FWVeGxederNtsphUGnLghLSBKVIUlYQpKpDE3CEpKISxdcZ/lgnR2XxsIWSwUHTFoev+dSzu/e6oQdVsHhtXDwF++Svdo7Pqaa0QwdLof0G6HD0Cad/E40PQksTWzFs1PZVtCXsKpsxs4Z5z2mo5yrFl5Fqb2UB857gD+l/6nJjytEXVRV5XDFYVbnrmZ1zmp+y/2NYlvtL3KT1kSHyA50jOxIp6hOdIrsRMfIjoQaTh4q7FYnpTnllG7bSOmePZTmlFNii6bMnYjLz3gRABduKjUeyjQK5RqoUFSCDG5aBTtJMVhJUUuItRYSXl6EJj8XV24urqIicDdsUKg2JgZD69beJbU1+pTW6GJj0UUdCygRESg6+QOiqamq6m1dslpRbTY8VTaslcUcLcwit/ggBSWHKSo9iqWiBFtlKQ5rJSaHSqgNQqsgrAoiHXrMNg1GB3gIoUoXgzUkAUtwAhVhrakIS0FVagYMo62YyKpsEkIqSUjQEtE2Hv2x//+62Fi0QTqUgz/Brq+9VydZC49vHN4KBk3wjnvRB/2xJ0w0iASWJrbiualsy+tLaNUhbp9zu2/9F/u+4Omfn8agMfDJNZ/Q1ty2yY8tBEC+Nd8XTlbnrCbHklPj/SBdEBmxGXSL6eYNJlEdSQ1LRVvHX5cej0pVhQNLqZ2ygirK8q2U5looO1JIaaEdm/1kzekeHAY3eSoUKwoVige3uwq9sxyTrYSwqkLiHaW0V6wkOSswW0rQFhWAw+H/QyoK2uhob/iIifE+nvDc0DoFfevWaEMbP8Oq+OOV2cvYVrjNF6y3F21HpebXjaKqhDp09FPa0MuTTFtLFEFFkZQUGyiwh1Oqi0f9Xet1kDWfyNLdRJTuxlyehclWhDY8HG1kBFpzBPpwA4agcgy2rRhNZRjCXGij4uGCiXDe7aA/N24+eaaQwNLEfnjhNbbm9CGk6jDj5oz1rVdVlXuW3cMvR36hR2wPZl8xG71G+k7F6Suzl7Emdw2/5vzKb7m/caDsQI33dRodPWJ60D+xP/0T+9MjpkeNLh1VVbFbXJQVVlFeWEVpnpW83XkUHanAYtWiek5+/GBNMaG6fKpwctiuIc+uYLAWYbbkEGMrJaaqjDhbGVG2crSehreM6OPi0CcnoU/yLrrERPSJSeji49BFRUnLyFmszF7GzuKd7C3d611K9rKvdB8VzopaZTtHdWZg0kDSQ/owf14p5nw7qYoRAyH8/p5uOpeV0MrDhFUcJrTyEOHlWQRX5dcopTO5MYS7MMYYMfS+BOMlt2Ds0g1dZGTzfmjhlwSWJvbDS6+z9ch5hFQdZdyc22q8l1OZw/VfXE+ls5Jx3cbxcJ+Hm/z44uxndVpZl7fO14Kys3hnjb9GFRS6RHfxBpSE/vSK60WQLghruYPSPKt3yfeGk/JC79UaDlv9QULBg4lSQj0FhLryCLHnYbAV4bRaUMvLMViqCLNZ0dKAXw8ajbclJCEefXxCzceEBHTxCejiYtEYDP73Jc4pqqpypPIIGws2sjF/I+vz17OnZE+NMhpFi2pPwG5pRafgHkzqOQzbITi6u5Sio5V4XLV/Ro0aO9GOo5jztxJ+aH2tAFNNFxuLsUMHDGmpx7qZUr3djK1aoTHWPVhYNC0JLE3sx5enseVwL0yOIsb944Yal+IBfHfwOx5a8RAAb17yJpe0vqTJ6yDOLg63g00Fm3wBZUvBFlyqq0aZduZ29EvsR9/o/rRXuuIq0RwPJ8eWk4USAKO7jGBbPkFVBYSWHyKs9BAmWzF6ZwUaf80sgKrVoo2Lx5iYgD4h3jtI9cTHhAR0MTHSMiKaTFFVEatyVrHq6Cp+zfm1zivb2oS3ZVDyQLpFdifZ2QZDiZmSw1YKDlWQn1WB21XzZzsoSCE2zEZM6Xqi9nyJNr8Ap+UkP7OKgj4lBWPHDhg7dMDUseOxYJMmP+tNTAJLE1vz+mTW7O6LqtESlxrGsLvTCY2s2Q/62prXmLt9LgkhCSy+bjFGraRzcZzb42ZnyU5W53j789fnrcfmPj4Rl6IqdFY70c/Tm9a2NoRaIqgqUyirgCrHSX5Bqh5MtiKCrfkEV3mDSVBVIUG2Qky2YrR1zEbrRqHCGEy5KRx3uBltVBQh8bFEJsUT1zoBU1ysL5Boo6O984UIESC5llw2F2xmyd7VfLtvJRiPoCg1v7b0Gr3vUvyu5m6k2buizQkjd08ZufvLawWY6Eg7rV1LSbauIqzyMC5zXxwk4zh8BOfB7Hrn5lH0egzt2mFs2xZDWiqGtDQMbdpi6twJRS/DAU6FBJYmtvWtFyj9z2a2dR2HSxdCTEooNzzWG53++IBGu9vOVQuvIs+axyN9HuH2brefZI/ibOfxeDiQt4ONu39g5/7fOJS9DV25lQhLMOH2OEKc8QSpCeg1ibi0MdgMUXhOMv5J76wk2JpHkDWf4Ko8Qqz5BFnzCbLlYzA40Bk9aI0eNHoPWr2KVWckTx/JbmMrNunbYYltR1xKIq3aJNGhXRJdUiJJMptk0kNxRll3sJhxc1ZQpd2NOSqbNkllHLbsw/L7eVmAYF0w6bHp9Io6jw6OHoQUxZK3q5LcfWU1rs4O0pSRalhHasR+Wo/6M/rOF+EuKsK+dy/23bux79mDbfdu7Hv2olrrvheWYjIR1LMnwb3PI7hfP4LPOw9FukAbRAJLE9v69sto35pHVVQMGwa8jM3qJv2iZC4c3alGuc/3fM6klZMIN4Tz9Q1fE25o+rqIwPNUVeHKz8eVn48zPx9XfgGu/Hwqjh6k7MgBHPlFqPYQnPo4rMHxWIOOPQbH4dLXf4WLxuMkqKqQEFcRoe4iwtQCQj2HiXDvJ1RfgvZYKNEZPWhNHnRGN069lgMksldNZrenFXu07bHF9SApuTVdEsPpmhhGp4RwQo3SjC3ODnvyKrhjzhoOFVeh1yrcc1FbrusXzP6yPews2cmWgi1sKthU67YSCgrdorsxNH4YXa39KN3jIntbMY6q412xGpwkxVaQNjidtN4pmGOPzxWjejw4jx7Fvns3jgNZOLKycBw8iH3XLu+EhSceKziYkAEDCL3gfEIuuBBDK7mZY30ksDSxLR9OQ/faBwAUxfZgU7e/AJBxeWsGXNPWN6bF7XFzwxc3sK9sHzd1vIlJAyc1eV1E8/I4HLhycnDm5OA8ctT7mHPUuy7PG1I8Fd6rGtwaA5aQBCzBiVhCErAGJ2ANjqcqKKbWXBInCtLaCTfaCDdWYjYUEqo5RLh7KzHuzZh0VXXOMG5XdexXk9ittmKPJ5kDmhTsER0ISuhAmzgznRPC6JIYTmpUMBqNtJqIs1tRpZ3HP9vCdzvyALiwYyzv3noeIceCudvjZm/pXjbmb2RDwQY25m/kSOWRGvvIiM0gs/UV9PIMonRnFVmr91BqCatRJiLORErXGFp1jiSpQwSmkNqtoKrHg2P/fqzr1mNduxbLypW4i4pqlDG0a0f4FVcQftWVGNvK9BcnksDSxLZ8/zGtPnyI3L1RUOrhQOoVHGgzHICYlFCG359BcLi3+W91zmru+vYuVFT+fvHfGZI6pMnrI06Nqqp4yspwHj36u0CSc2zdUdwFhbW2qxlMvOGkMiQRuymm3mNp9CpR8cFERSsE64vRu7MItm0nyraOWNse9Grdt+Szq3r2qknsOdZickTfGmI6E5rYjjZxEbSLDaV9XChJEUFoJZiIc5iqqvzf5hwe/3QzVU433ZPDeffW3qRE1T2Dbp4ljxWHVrAkawnr8tb5rsJTUOga3ZXzk8+nd3Ewxl+2k13YihxHVzwn3stKgdiUMFp1iiSpYwQJbc31Bhjbjh1YfvqZyp9+omrjxhqTIho7d8Z8zTWYrxshl1UjgaXJ979l+Sek//AndmnaERUxjsJ33qUgOp2dnW/DqQ8lMs7EiEf6+ELL39f9nQ+3fki4IZwFVy+gVVgrP0cQTUF1Or3dNNWB5GjOCc+9j/X1QQO4tEZv1425NVVx7agMSaRcF3nS6eYdBiuaKBcxcQbahKuEuA9irNxIROV6oquy0Ndzr2CbqmevmsweNZl9aivKQtvhjukMkanEhgfTJTGcbknhtIqs5+aCQggANh4q5Y7Zayi2OAgz6XjlunSG90g86b+bfGs+32Z9y5KsJWwq2FTjPbPBzKCQFAYdOkDa4UjK7N054uhBiat2t05UUghJ7SNITY+mVadIdIbaLavu8nIqV6yg/MuvqPzlF3B5u6AUvZ6wzEwiRt5EcN++Lf/f+eZPIK4zxHVt0tsdSGBpYlt/WEj35ePZq21Lu2fWU/T+BxS++y4WQtnQ8wHspkgiE4O5/pHemEL0OD1Obv/6drYUbqGduR1zr5xLmCHM/4HESamqiru01Nt/fPAgjqwsnIcP+1pJXHl53vve+OGJS8ae3Jmq6DZYghOp1Jgpd5iwWOv/hVGlq6A4OBdHeDkRUZBostPJfZRW5TuIsewh1FN78iuAKtXAXjWJQ7pUSkPa4ojshCmpKxHJ7Qg1mYgPN5IaHYJBJ1fiCHGqDhVb+evHG9iQXQrAwLbRTBrelS6J/r8PCqwF/HzkZ34+8jOrjq6qNZFdG4+GnpZy0iuDSKjojNN4JTm29pQV1/xdozNoSOkSRZueMaSlxxAUVnvQraukhIpvvqX0v//Ftn27b72hTRsiRo7EPOLaltnqUlkAr7cHFHjyEBib7vtMAksT2/rTIrovu539mjTaTvKmcdXlwrp2Hbsfepa1ne/GYYwgvk041z7YC71RS54lj1u+vIX8qnx6xPbgvoz7GJA4AI0iX0z+eBwOHPv3Y9+3zxdMHFkHcRw8iOd3g9t+T9Hr0SUloktMwpOQhi0iBWtQLBWEU1ZloLTEjaWs/mnirfpySoJyvUtwHp6QKqJ0FXSzF3FBRQ49nHlolNr/ZNyqwgE1kf3aNEpC2+OK6UpwSg/adehCxwQzJr3cgE2I5uR0e3h7+V7eXbEPu8uDRoFb+rfmr5d1IC6sYdPxuzwuNhds9gWYHcU7apUJc3voYbfTw51Al9Dr0HARWbvsVJaccANGBRLbmknrEUObnjFEJoTU2k/V1m2ULlhA2Zdf+lp+FYOB8GuGE3P3PS1roO6e72DeDRDdAe5f26S7lsDSxLb9/D+6fTeWvbQm6akNBBuO92taVq5k+4PPsz79flz6EOLbhJN5V3fCokxsL9rOuCXjfHfN7Z/Yn+kXT/d787lzhep248jOxr5nD/bde7yPe/bgOHjwpDfC0yUlYkhNRZ+ahjs+jarQBKz6SCzuIMorVcoKbJQVVOGy178Pq6Gc4qAcSoJyKQ4+FlBMeRg9OjpVKVxkK+Uq+xFi1Nr7yFcjOKBJpSi0A47ozhiT0olt24N2iTFEhsiljEIE0qFiK69+vZMvt3jvt2XSa7i1fyp/uahtg4NLtRJbCZsLNrOpYBMbCzayNX8TVZ6af/DEu1xc5QnhkrDhOGwDyTocRsHhmndKj4gPJq1HDG17xpDQ1oxywvgzd2Ul5YsXU7Lgv9h3HAtIej2RN91I9F/uRh8fdwpnoYn9+Dp8/xKk3wQ3fNCku5bA0sQq960mdO5QLKqRSa3n8tr4oTUGPJZ9+SU7XnibTen34tIHY9SrDLmzO2kZ8RwsP8j8HfP5fO/nVLmq6BLVhacHPE236G7oNOfOpaYeqxXbzl3Ytm/Htm0btl07cezbj2q311leYzZjbNcOQ5s0NCltqIpsjcUYS4U7mLIiB6V53pv2uZz1dwF5UKnQWyg15VMSmk1pUA4lwbmUBOXh0FWh8Si0shs4z2bnMlsRfe1WQn73z6FUH0dBeDqW2Ay0yRmY0zKIT0zGqJMWEyFaslX7inh1yU42HSoFwKjTcEv/1tx9UTviw0/tBoguj4vdJbvZdHQ1a/Z9yc+lu6k6ocW1u93OVZVWBtABW/BIDlSmc/iAE4/7eJnQSCPte8fRvk88calhvrErqqpStWEjhTPfwrJyFQCK0UjMPXcTfeedgZ2YbsEY2PEFDH0ZBt3fpLuWwNLUPB4q37mY0MJNfOa+gI29X+XFa7vVGCRV/NFcst74B1u73UlFWCqoKt1SrZz/4OXogr2tLfd8dw/FtmIAIo2RvDDohbNyGn93RQW27Tu84eTY4ti/H+r4UVOCgjC2b4+xQwe0bTtSFdsWiymOMouWkhwLxTkWyots1HdLGw8qZTobZYYyyk0FlAcfoSwkm/KgfCqMxXg0x1tIgt0aOjs8DKoqo4/NRneHHeMJ+1V1QZCUgdKqL7TqA8l9wNyCmmWFEI2iqio/7C5gxrI9vvEtBp2GW/p5g0uC+fTu3Gx32/lx72K+2D6Pn8v34jrhF1UXu4MhFiuXG9vjjridA2Wdydplr3E7jfAYE+37xNOhTzzRySG+7xTL6t8omDGDqvXrATB26ULiyy8R1K3badX3lE3vAaUH4fb/gzYXNumuJbA0h8Pr4INLAbjJPolLM0dw90Vta4QW286dlHz1DWt+tXEoqi8AIfZC+vTW0HHUxRytymH+ihl8o2yn2FOBgsL9ve5nXPdxZ+xdnj0WC7YdO6jauhXb1m3YtmzxdunUQRcXh6lrVwxduuJI6UJlUCKlNiNFRy0UH7VQUWSrczuAKsVFkaGSElMRpcE5lIUcpCz0ABZjER5N7VaWcFVHB7dKD2s53aqsdHc4SHK5j98ALTQekntDYk/vqPf4bhCZ1qSj34UQLYOqqvy8t5AZ3+1h7cESAAxab4vLhEvbExN6+rdSKaoqYknWEpbu/5oNhZvxnBBe+lbZuK6ykos1CRSE38zeivM4cECPy3H8d1dkQjDt+8TTsV88EXHBqKpK+f/9H3mvTPZOTKfVEn3HHcTcdy8a0+kFrUapKoGpad7njx+EoIgm3b0Elubyv/tgw7+pVE1McN7PJlM/erWOZMzAVM5rHUmZ1YnD7UHjdFDy8Xes22HCqfOOVzE4yokq3o65/ACxjoMc6mFiYcQ+dqQoRMSl8Ocef+bKtle26HsQeex27Dt3esPJlq3Ytm3Fvm9/nVfm6JOSMHXrCh3SscZ3pNIYR0kpFB6ppCTHWuveHtUsGifFhkqKTQWUBB+hNGw/pWH7sOnrvrdHBFraeTS0r6qkXZWF9k4nbR1Ook+skzEcknpB8nnekJJ0HoQnUecMbUKIs5aqqqzcV8SM7/bwW5a3tTvYoGVknxTGD04jNbr24NhTUWwrZsWhFXy99wtW56/zRZcQj4dMi5WrKy2k2w0cCruZvVWDOXjUXGPYXqvOkXS/KJm0HjGopSXkvvwyFV8vAcCQlkbSq1MIyshokrrWcHgd5GzwXhWUdr53OfAjfHSN9w+6Bzb53UVjSWBpLrYy+PhWyPoJt6rwifsi/u0egopCthpPBccnLFIUaGvUcNXhfIKVGNzaoOP7UT1ElO0lPm8t0UUb2JlcRU4UhLuNaPr2JPHaG7mszVAM2sAN4FQdDux791K1ZSu2rVup2rYV++49vjkETqSLj0fbLQNHuwysUW2o0EZRWuyi6KiFqvK6r8hxatwUGsooDMqnOCSbkrC9lIYcrjeYxCt62ro8tLWU0dbhoO2xYBJ1YjBRtBDdHuK7Qly3Y49dICIN5AZ+QohjVFXll71FvPbNTjYfPn7lYZ/USEb2SeHaXklNNk4tpzKH/+37H//bs4jDluOz7Ua53d4uI4uVdKuWQ/Z+7HEOIdvaFY61BYeEaUi/JIXuF7fGsfIHcl94EVdBAej1JL7wAhHXX9ckdcReAV8/ARv/XXN9XDdISIfNH0PXETByTtMc7wQSWJqTywFfPQzrP6qx+pAnlpHqFKw6M063B6vjeFzWqNDVaicaI50NIYRbjn/JKh4X0cXbiSjdi9Fegrl8P+WmUn7tZybxqtEkJvQmNaErraOb79JYd2kptp27sO/aiW3HTmw7d2Lftw+cNSc9UwFPTDKurn1xtOqKJTSZCk8oJUXOmpf01dhGpdJUQb4pj6LgbIrDDlAUfJQKYzH87vJgDRqS9WG0VbW0tdtoW15AO2sFbZxOQk/8MQ1PhlZ9ISIFwpIgPNEbVGI6gq7ltlAJIVoWVVX5aU8hH/x8gJ/2FPiG2cWGGRk3KI3b+qdiDm6a7nqP6mFd3jq+2PcF32d/T7mj3PdehAcutVRyucVKl4pwdlddzo6qIVR5IgDQa+x0jdtMeuJuKr7bR8UW7y0Joob1Jm7slShBZu+Oqkq8i60MPC5QNBAcA4YQcNnAYQFnlbfrW2vw/r4sOQhbPwVrEaBAh8u986zs+hqcJ0y0edlzcMHEJjkXJ5LA8kc49Bssnwy5m1GdVShOK7S/HG75L6qiUGRxcLS0ivIqFztzy/l5byE/7Pb+gwjzKHRxaOnm0hHjqt0tEV6eRUzhRqKLd2JwlGPTVrAj2cj2pBRyUwahhGdAcBhGowEF7/wDDrcHl1slPEhPbJiRmFAjyREmuiWZ6RgfhkGn8d686/DhY6FkB/adu7Dt3IkrJ8d3bBUFpz6UqqAYqiJb40zpjC0iBYsugsoqLQ57/VflWPTlFAcfpTi4+nLhHIqDc3Bpa7aymJQQWmvD6ajV095po015IWmlR0hxOqn1q0HRehN+h8u9ISWuK5hbSXeOEKJJ5ZRVsWjDUT5alUVOmXc8XYhByy39W/OnC9qe8pVFdXF6nKzJWcO3B7/l++zvKbGX+N6L0YVwsyGRG8ptFB4ws6E0k2JXKgAaXHQw/USbo1/j2OxtjQ5NspE0sASt/jS/yiNSYcS7kDbY+9paDJ+MgwM/eF/fthDaX3Z6x6iDBJY/Wu5W+OAyb4Lt+yfInFznX/qHiq18sekoqw8Us/5gCZV2F9FuhU5OLVFuhQiPhgS3gkLNL2PF4yK4qoCwioOYy/YTZCtC4yxhR6SW7dFtUD16KvThFBpj0bs9hDkthDqqCHNaCXVUEe2w0MpVTnxJLihGbKZobMZIbKYo7MYobKYorCHR2A2ReDT+WygqDCWUmQooORZIqucycei8cw+oqoLijiRUiSVVa6KjTkt3vYauqo2UoizM+dvr3nFQpHd8SXJvb3dObGeIagc6mdtECPHHcLg8LN58lPd+3M/OXO+stwathht6J3PXBW1pG9u082i5PC7W5a1j6cGlLD241HclqVFrZHjb4dyWMgTdLicbVtk5csT7J51G8dAxeCtxy/6NzmbBGK2j1YhIDImx3t+jJrO3BcXjBkuBt6VEH3xsMYHq8fYWuO2gD4Iu10Lbi0H7u6k2XA749hko3g+j/u3dtolJYAmEDfPgf/d6n8d3h6unQ0rfeou7PSo7c8sprHQQatQRZtIRbNBSVe4gZ3sxBTtKKT1qwW51otbTqKF1VRFUVQioaN0OdO4qPBoDTl0QLl0wHo0Oncv7l4JbZ8RhCD/pXYSrqXioNJRSFlRAmamQMlMB5aZjz42luNxBeFxmVKcZjcdMYlAUPUIMZBhVeugq6WgvIKQ8C/K2ef9B/J6i9baWJHT3hpLYThDbBUJipOVECNEiqKrKit0FvLt8n2+ALsCAtlGM6pvCsO6JTd5N73Q7WZK1hLnb59aYZXdw8mBu63Ib7R3dWfd1NlmbvTdpNRoV2mQtJmH3EvThYSS8/BLhl1/epHVqbhJYAmXnl/DF/cf6AoGOw7zNayn9vXN6nMLAT9WjUllqp+hIJbn7y8jLKqOwsBRbkRs8jf/H4lHcVBpKsRjLsBorsBotVBltWA0O7DobFeiwqTp0umCMmlDMGh3RikK4zkg7vYaOOgdJmlJiXLmEWw9iKN2PUn64/gOGxHkvHY5MhYjW3pHmaRdAcFSj6y6EEIGwJquYWSv28f2ufN84lzCTjhEZyYzqm0L3ZHOTHk9VVdblrWPu9rksP7Tcd2fptPA0bulyC32dF7Pm82yKjx7rFnIV0277f4gq3k7kyJHEP/E4muC671rd0khgCaTKfFj2grfFhd8NFE1I9w6A6nA5dLrytLo6PG4PxTlWrGV2VMBuc1BWXonWoGA0qBiLN6A6S7HEnIcmKIKgIBORYQYiKEAXFO5tBrQUQGUelGbDxvmQvQqCorytH64q72Cs0rrnVKklKOrYwNcOxx9ju0B0O2k1EUKcFY6WVvHpusMsWHOII6XHp9/v0crMqL4pXNEtgegmmNPlRIfKDzF/p3e2dIvTG1BigmK4r8cEOuT0Ze3ig9gs3gskQisO0frQMlqFFNPqtSkEpac3aV2agwSWliBvm7fFJWcT7P8BHL+7m6/J7G15iG4Ppgjva1P4sUezd53O6O1DdNm8XSsuu/f5iY9VpVCR4+2jdDu9gSlvi3eUOIBG770luMsBxfu8I8cbSx/s3U9ItDd4hSV6Z4CNbu+9GVZMB2kxEUKcMzwelV/2FbJgzSG+2ZaL89jU+xoF+rWJYlj3RDK7JZz2TLonsjgtLNq7iLnb53Kk0nt5dPuI9jzQbSK6DQls++mIbyI6o62Y5Jyf6XJ+Cq0n3o0mpGnml2kOElhaGqcNsn6C8iPewUub/+sNGc0pvBWExsLRDTXXG8K8Ycfj9LaKhMZ5l9YDoddtUH4Ucrd4A1NYgncGWAkjQghRp6JKO5+tP8wXm46y9Uh5jffOax3BkK7xXNo5jk7xYTVmRj9VTreTj3d9zKxNs3yXRg9MHMj9XR/EvjmIzd8foqrS+4eporqJseyj+5Wd6TzqQjTaljcflQSWls7t8raC5G6FskNgK/e2iJy42Mu8QUdn8ra06EzeLqQar43e6+XDEsEQChqdN3xEpHpnddVoIX+HNyhpdBDVFswp3jp4XKA9M28HIIQQLdGhYivfbMvl6625rDtYUuO95IggLuoUywXtYxjULua053cps5fx/ub3mb9zPk6PEwWFa9pdw93d7qFih4atX+8gv+B4eZNio8OgVqSdl0xShwh0hsaNgTy6t5TgMAMR8U07NkYCixBCCBFAeeU2vt2ex/Kd+fyytxD7Cbcj0SiQ3iqCAW2j6JcWRZ/UqFMOMIcqDvHm+jdZkrXk2L41XJh8IVe1vYpuzm7s/OBHDhSG4jSE+bbR6hSSOkSQ1CGC+DZmwmOCCA43oNNrUDS1W4GO7i3l/97ahNGk5fpHexMeE1SrzKlq9sDy9ttv87e//Y3c3Fx69uzJW2+9Rb9+/eot/8knn/Dss8+SlZVFhw4dmDp1KldeeaXvfVVVee6553j//fcpLS1l8ODBvPvuu3To0KFB9ZHAIoQQoqWqcrhZtb+QH3cX8tOeAvYV1LwFiaJAp/gw+rWJokerCDrEhdIuLpRQo66ePda2qWATb61/i9W5q33rdBodfeP7ckVVJ2LmHaLQHkNxVBfsxsh696PRKugMWoLC9IRGmohOCmHHyhycdjcpXSK58p4ejW6dOZlmDSwLFixg7NixzJo1i/79+zN9+nQ++eQTdu3aRVxcXK3yK1eu5MILL2TKlClcffXVzJ8/n6lTp7J+/Xq6d+8OwNSpU5kyZQpz5syhTZs2PPvss2zZsoXt27djasBdKSWwCCGEOFPklFXxy94i1hwoZk1WMfsL676HWpLZROvoYFIig2kdFUzr6GBaRQaTYDYRHWKocx6Y/WX7+d/e//F99vdklWf51iuqyqV7TFz/s5vgigiKIztRZm5HZVQbbPoIPOrJx7ckd4pk2F2dMYY2XesKNHNg6d+/P3379mXmzJkAeDweUlJSuP/++3niiSdqlR81ahQWi4XFixf71g0YMICMjAxmzZqFqqokJSXx8MMP88gjjwBQVlZGfHw8s2fP5uabb27SDyyEEEK0JAUVdtZkecPLrtwK9uRXUlBR9/3ZThRq1BEdaiA6xEB0qJGYUANhJj0mvZYgvRarepRs2xqyrBvItmzDpTpBVelyCIZs8DBgl4re7b0li0ejx6E3UB5jxhoRQVl4JJbgGFRda3QON613fYbGbeeSH1Y26WdvzPd3w9ubAIfDwbp163jyySd96zQaDUOGDGHVqlV1brNq1SomTqx5w6TMzEwWLVoEwIEDB8jNzWXIkCG+981mM/3792fVqlV1Bha73Y7dfvx/Znl5ea0yQgghxJkgNszIlemJXJme6FtXanWwr6CSQ8VVZBdbOVRsJbvYyuGSKgoq7DjcHirtLirtLg4WWU+y907eRXGiMRSgMeaxMTifzZfmYb44h/4Hiumz10PnQw7CbA6CjlTCkSP17s2aX0BwXGzTffhGaFRgKSwsxO12Ex8fX2N9fHw8O3furHOb3NzcOsvn5ub63q9eV1+Z35syZQovvPBCY6ouhBBCnDEigg30To2id2rt91RVpcLuoqjSQWGlnaJKO4XHnlvsLmxOD1VON1VONzaH2/fc5Y5BpTOq6r3NkKqq7Ep1sKVNLi4lH7PlMLHlRURVWIiqcBJhsWN0WnEZVLLbxlDSvRVvRNQ//qW5NSqwtBRPPvlkjVab8vJyUlJSAlgjIYQQ4o+hKArhJj3hJj1tYlrupHBNrVGzyMTExKDVasnLy6uxPi8vj4SEhDq3SUhIOGn56sfG7NNoNBIeHl5jEUIIIcTZq1GBxWAw0Lt3b5YtW+Zb5/F4WLZsGQMHDqxzm4EDB9YoD7B06VJf+TZt2pCQkFCjTHl5OatXr653n0IIIYQ4tzS6S2jixIncfvvt9OnTh379+jF9+nQsFgvjx48HYOzYsSQnJzNlyhQAHnjgAS666CKmTZvGVVddxccff8zatWt57733AG/T1oMPPsjLL79Mhw4dfJc1JyUlMWLEiKb7pEIIIYQ4YzU6sIwaNYqCggImTZpEbm4uGRkZLFmyxDdoNjs7G43meMPNoEGDmD9/Ps888wxPPfUUHTp0YNGiRb45WAAee+wxLBYLf/7znyktLeX8889nyZIlDZqDRQghhBBnP5maXwghhBAB0Zjv75Z360YhhBBCiN+RwCKEEEKIFk8CixBCCCFaPAksQgghhGjxJLAIIYQQosWTwCKEEEKIFk8CixBCCCFaPAksQgghhGjxJLAIIYQQosVr9NT8LVH1ZL3l5eUBrokQQgghGqr6e7shk+6fFYGloqICgJSUlADXRAghhBCNVVFRgdlsPmmZs+JeQh6Ph6NHjxIWFoaiKE267/LyclJSUjh06JDcp6gZyPltfnKOm5ec3+Yn57h5BfL8qqpKRUUFSUlJNW6cXJezooVFo9HQqlWrZj1GeHi4/ENpRnJ+m5+c4+Yl57f5yTluXoE6v/5aVqrJoFshhBBCtHgSWIQQQgjR4klg8cNoNPLcc89hNBoDXZWzkpzf5ifnuHnJ+W1+co6b15lyfs+KQbdCCCGEOLtJC4sQQgghWjwJLEIIIYRo8SSwCCGEEKLFk8AihBBCiBZPAosQQgghWjwJLPUoLCyUmymKs5pcINi85PwK0bTOiqn5m9rkyZP5+OOPsdls9OjRg4kTJzJo0KBAV+ustGTJEkwmEyaTiQEDBgS6OueE7OxsoqOjUVWV0NBQVFVt8ntwncvk/DavhQsXsnLlSmJiYujVqxeZmZmBrtJZp8WeY1XU8PLLL6uxsbHqhx9+qP773/9WBw4cqPbr10/98ssvA121s851112nJicnq+3bt1cNBoP60EMPqTt37gx0tc5qDz/8sNqlSxe1c+fO6uDBg9V169apbrc70NU6a8j5bV5PPvmkGhYWpt54441qz5491aCgIHXy5Mmq1WoNdNXOGi35HEtgOUFVVZV6xRVXqH//+999644cOaI+/PDDateuXdVNmzYFrnJnmZdeeknt2bOneujQIfXQoUPq//73PzUpKUkdM2aMumHDhkBX76z02GOPqampqepXX32lvv/+++qIESPU8PBwde7cuarFYgl09c54cn6b186dO9V27dqp33zzjaqqqlpaWqq+//77qkajUV9++WW1srIywDU887X0cyyB5QQ2m03t16+f+thjj9VYv3fvXvWuu+5SBwwYoJaUlASmcmcBj8fjez5u3Dh15MiRNd5ftGiR2qNHD3XChAnq0aNH/+jqnfUuu+wyderUqTXWjR07Vm3fvr26cOFCaQk4TXJ+m9f333+vJiYmqocPH66x/s0331S1Wq362Wefqapa8/eMaJyWfo5l0O0JtFotaWlp7N69m8LCQt/6du3aceutt+JyuZgzZ04Aa3hmy8vLA8DhcFBZWYlO5x1C5XQ6Abj22mu56667+Prrr/nll18AGbjYFFRVpbCwkIMHDxIZGQmAzWYDYM6cObRu3ZpXX33V9/9HNI7L5ZLz24yqfwekpqaSn5/Ppk2bAO95B7j//vsZN24cDz30EB6PR8YLNZLH4/E9b/HnOCAxqQX79ddfVUVR1L///e+1UuRtt92mDhw4MEA1O7M9/fTTaufOndWioiJVVVX1s88+UxVFUdeuXauqqrd1q9rw4cPV888/PyD1PJvdcsstavfu3X2vq895UVGRGhwcrL722muBqtoZaffu3TVe33bbbXJ+m1BeXp5qt9t9r6uqqtSxY8eq559/vnrw4EFVVVXV4XCoqurtuk9NTVXfe++9gNT1TLVgwQLfz6XH41GtVqs6bty4FnuOpYXld/r378+UKVN44okn+Oyzz7Db7b732rdvT1xcXI1EKvwbNWoU77zzDu+99x5RUVEAXHHFFVx77bVcf/31VFZWYjQacTgcANxxxx3s27ePw4cPSwvLKVq4cCGff/45X331lW/dQw89hNVq5YEHHgC8d2i12+1ERUXxl7/8hS+//JKqqio55w3w6KOPctNNN5GXl+c7X/fddx92u13ObxN47rnnuPzyy+nXrx9XXnkl27dvx2Qyceutt+J2u3nuueewWq3o9XrAe651Oh1utzvANT9zPProo9x8882kp6cDoCgKQUFBXHvttQAt8hxLYKnD448/zp133smdd97Jm2++ya+//sqOHTuYP38+nTp1QqOR09YQDoeDfv36sWvXLrZt28YFF1xAWVkZHo+H4OBgXnzxReLj47n44oupqqrCYDAAkJOTQ9u2bYmNjZXm3VNw/fXXc++99/Liiy9y9dVXc/PNN/Pzzz/Tp08f7r77bv7v//6PadOmAfhuJ+9wOIiPjycoKEjOuR/XXnstH374IR988AHx8fG+89W1a1f+9Kc/8eWXX8r5PQ1PPvkk//znP3n00Ue59957yc/PZ9SoUSxYsIChQ4dy2223sWXLFu6+++4a2wUFBfn+IBInd9111zF//nxWrlzJFVdcUeO9ESNGMGLECLZt29byznHA2nbOAI899pg6YMAA1Ww2q23btlVHjx4d6CqdUd5//31Vr9ers2bNUlVVVT/66CP18ssvV7t166YOGTJE/d///qd+9913ao8ePdRu3bqpDz/8sDpz5kw1KipKfeaZZwJc+zPTzJkz1R49eqjZ2dmq1WpVf/31V3XAgAHq5Zdfrq5cuVK1Wq3qU089pQYHB6svv/yy+tNPP6lr1qxR27Rpo77wwguBrn6LZrFY1N69e6s9e/ZUKyoqVFVV1fz8fLWqqsr3+ujRo3J+T4PdblcHDRqkzpw507fO6XSq11xzjTpw4ED1q6++Ut1ut/rBBx+oqampatu2bdUbbrhBTUtLUzMzMwNY8zOD2+1Wb731VtVgMKgbN25UVVVVV65cqU6dOlV97rnn1P/85z+qqnq7M99///0Wd44VVZX2yZPJy8vj0KFDKIpC7969A12dM4rVamXSpEl8++23tGnThq1btzJ27FgiIiL44osvqKys5P777+eaa67h4YcfZv/+/bhcLq677joefPDBQFf/jPTQQw+xYcMGVqxY4Vv3448/8sorr2A0Gpk5cyaJiYnMmTOH5557DqPRiMvl4qqrruLdd98NXMXPAG+//TbPPPMMTz31FI8++ij/+te/mDNnjq9baMqUKQwfPhyn08m8efPk/DaSqqoUFBRw2WWXMWHCBP7yl7/gcDgwGAzk5ORwyy23EBoayj/+8Q8SExPJyclh5syZaLVaoqKieOihhwL9Ec4Ir7/+OgsWLGDMmDHYbDZmzpxJ586dKSgoYPPmzTz44INMnToVjUZDbm5uyzrHAY1L4qyXn5+v3nTTTWqXLl3UpUuX+tbb7XZ16NCh6pAhQ1RV9f4Vpare6/5F47ndbtXtdquPPvqompmZqVoslhqX0X7yySdq37591alTp/rOdXZ2tnrgwAGZX6iBiouL1QceeEC94IIL1AsvvFBNS0tTp0+frr7//vvquHHj1Li4OHXu3Lm+8y7n99RcfPHF6hVXXOF7XT3oc9WqVWpYWJj6r3/9K0A1O7OdeBHJo48+qiYnJ6tt27ZVP/nkE18L4aeffqoqiqJ+/PHHgarmSUlgEc1uz5496meffeabPMvlcqmqqqrz5s1TDQaDeujQIZmj4hQVFBTUeP3DDz+oGo1GXbhwoaqqx8+1qqrq3XffrXbr1s33Wuar8O/353fv3r3qjTfeqPbt21ddvnx5jfduuukmtWfPnr7Xcn79+/XXX9XffvtN3bVrl2/dqlWr1KCgIHXatGmqqnp/hqt/jseNG6deeOGFAanrmaquc+x0OtUHH3xQfffdd2v8jlBVVR0xYoTvD8mWRgKL+ENU/5V0ohdffFG96qqrAlCbs8Of/vQndfjw4er+/ftrrJ8wYYIaGRmp7tmzR1VV1RcG165dq4aEhKjbt2//w+t6Jqrv/G7cuFH973//65uqvPoX/uLFi1WTyaTu3btXwkoD3HHHHWr79u3V1NRUNSgoSP3oo49UVVXV8vJy9cUXX1QNBoP6v//9r8Y2f/nLX9QxY8YEorpnpLrOcfXvg4qKCrWwsLBGeZvNpl5xxRXqfffdF4jq+iWBRQTEihUr1Hbt2ql/+9vfAl2VM47L5VLvuusutVWrVqpOp1Pvu+++Gr94cnJy1EsuuUTt0KFDjXAyf/58tXfv3jJbsx/+zq+qHu/CVNXjLSmvvvqqOnToUGkt9MPpdKojRoxQMzIy1M2bN6sHDhxQn376aTUyMlItLi5WVVVVDx8+rN53332qVqtV//Wvf6m//vrr/7d39yCNg2EcwJ/SDqL4UdBFaBTEj00y1bGoo1ZxUHRwUwt2EHETxEGkILi4iBVBCIjVQfxACropWKGoKLqIYHSpoGg1pJTKc4NcOdHreV6vbxL+v7Fk+PdPaZ4k7fvyxcUFV1VV8fj4uOB3YHxf6fgzp6enLMsyLy4u5jDt12FggZxaX1/noaEhLi4uxhfPN52cnHBnZyeHw2He2Nhgm83Gk5OT6efQzG+LPDU0NHBtbS13dXVxIBBgp9PJIyMjApObw+/6zbSPSjgcZpfLlX6MAb+3tLTEHo/n3TD98vLCFRUVvLKykn5N13UeGxtjl8vF5eXlLEkS9/b2iohsOpk6Xl1d/XB8NBplRVG4rKyMBwYGchn1r2BggZx6enrijo4O3tzcFB3FtJLJJO/u7nI8Hmdm5unpabbb7awoyrsVg5nfNplsa2tjr9fLMzMzIuKaTqZ+f115lfntMVBPTw+XlJRwIBAQEdd07u/vub+//91nNZFIsCRJvL29/eH48/NzPjo64oODg1zGNLW/6fj5+Zmnpqa4srLy3ca/RoS/NUPOpVKp9D5C8G+YmWw2G/l8PgqFQhQKhaipqenD4mSaplFBQYGglOaVqV9mplgsRqOjo9Td3U3Nzc2i45rS6+sr6bpObrebFEUhWZZFR7KcP3Ucj8cpFotRdXW1oIRfgyVbIecwrGTPz+uN2dlZkmWZ/H4/nZ2dkaqq5Pf7aWdnh4iI8vPzRcY0rUz9Dg4O0vX1NQWDQQwr3/CzW7vdTrqu08PDQ3rZ92QySfPz86SqqsiIpvenjoPBIKmqSkVFRYYfVoiIcIcFwOR+vWNVV1dHhYWFdHt7Sy6Xi/b29tJbHsD3fNbvzc0NSZKEfrPk8vKS3G43XV1dkaZp5PF4yOl00v7+Pi5wssQKHeMOC4DJORyO9Dbww8PDFI1GqbW1lQ4PD3EyzYLP+vV6veg3i+7u7qimpoaOj4+pvr6eZFmmSCRimhOpGVihYwwsABbgcDhoYWGBfD4fTUxM0NzcnOhIloJ+/y9N0ygSiVBjYyP19fXR8vKy6EiWY4WO8UgIwAKYmba2tiiVSlF7e7voOJaDfv+vx8dHKi0tpbW1NWppaREdx5Ks0DEGFgAAEC6RSFBeXp7oGJZm9o4xsAAAAIDh4TcsAAAAYHgYWAAAAMDwMLAAAACA4WFgAQAAAMPDwAIAAACGh4EFAAAADA8DCwAAABgeBhYAAAAwPAwsAAAAYHgYWAAAAMDwfgDkhTVQy96EeQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "feature_dfs = vis.get_histogram_dataframes(data, display_format=\"percent\" )\n", - " \n", - "vis.show_dataframe_plots(feature_dfs, plot_type=\"main\")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "17fbd7fd", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAHbCAYAAAAzs2v3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAACKhUlEQVR4nO3dd2AUZd4H8O9sTw9JSMOQoAZQyRGOEgELao4oRdFTih6I+mKFF0REQIo95hRfEDxz4CmcyoFY0EPFwwjeKREFzkpRkJAIqUDaJtn6vH/M7mQ3jYTMsinfz904M888M/vsJGR/+7SRhBACRERERJ2cxt8FICIiIlIDgxoiIiLqEhjUEBERUZfAoIaIiIi6BAY1RERE1CUwqCEiIqIugUENERERdQkMaoiIiKhLYFBDREREXQKDGiJq0bp16yBJEvLy8vxdlHbZuXMnJEnCzp07/V0UIvIRBjVE5BP79+/HY4891qGDoQ0bNmDFihX+LgYRqUTis5+IqCUOhwM2mw1GoxGSJLX6vLfffhu33HILduzYgVGjRvmugK3kdDphtVphMBig0cjf58aNG4cff/yxQwdeRNR6On8XgIg6Nq1WC61W6+9itJtGo4HJZPJ3MYjIh9j8REQtatinJikpCePGjcMXX3yBYcOGwWQy4fzzz8ff//53r3NuueUWAMBVV10FSZIa9Wf5+OOPcfnllyMoKAghISEYO3YsfvrpJ6/Xnj59OoKDg3H8+HFMmDABwcHB6NmzJ+bNmweHw+GVd+PGjRg8eDBCQkIQGhqKlJQUrFy5UjnesE/NqFGj8OGHH+LYsWNK+ZKSklBdXY2goCDMnj270b347bffoNVqkZmZ2Z5bSkQ+wqCGiNrs8OHDuPnmm/GHP/wBy5cvR48ePTB9+nQlKLniiivwv//7vwCARYsW4fXXX8frr7+Oiy66CADw+uuvY+zYsQgODkZWVhaWLFmC/fv347LLLmvUFORwOJCRkYHIyEg8//zzuPLKK7F8+XKsWbNGybN9+3ZMmTIFPXr0QFZWFp599lmMGjUKX375ZbPv4dFHH0VqaiqioqKU8q1YsQLBwcG48cYbsWnTpkaB0z/+8Q8IIXDbbbepcRuJSG2CiKgFr732mgAgjh49KoQQIjExUQAQ//73v5U8JSUlwmg0ioceekhJ27x5swAgduzY4XW9qqoqER4eLmbMmOGVXlRUJMLCwrzSb7/9dgFAPPHEE155Bw0aJAYPHqzsz549W4SGhgq73d7s+9ixY0ej8owdO1YkJiY2yvvJJ58IAOLjjz/2Sv/d734nrrzyymZfg4j8izU1RNRmF198MS6//HJlv2fPnujXrx9+/fXXM567fft2lJeXY8qUKSgrK1MWrVaLtLQ07Nixo9E59957r9f+5Zdf7vVa4eHhMJvN2L59ezveVb309HTEx8fjzTffVNJ+/PFHfP/99/jTn/6kymsQkfrYUZiI2qx3796N0nr06IHTp0+f8dxffvkFAHD11Vc3eTw0NNRr32QyoWfPni2+1v3334+33noL1113HXr16oXRo0dj4sSJuPbaa89YnqZoNBrcdtttePnll1FTU4PAwEC8+eabMJlMSl8hIup4GNQQUZs1NxpKtGKGCKfTCUDuVxMbG9vouE7n/WepNSOvoqOj8e233+KTTz7Bxx9/jI8//hivvfYapk2bhvXr15/x/KZMmzYNzz33HLZs2YIpU6Zgw4YNGDduHMLCws7qekTkewxqiMgnmpvT5oILLgAgByLp6emqvZ7BYMD48eMxfvx4OJ1O3H///fjrX/+KJUuW4MILL2xTGQFgwIABGDRoEN58802cd955yM/Px6pVq1QrLxGpj31qiMgngoKCAADl5eVe6RkZGQgNDcUzzzwDm83W6LzS0tI2v9bJkye99jUaDX73u98BACwWS4tlrKioaPb41KlT8a9//QsrVqxAZGQkrrvuujaXjYjOHdbUEJFPpKamQqvVIisrCxUVFTAajbj66qsRHR2Nl19+GVOnTsXvf/97TJ48GT179kR+fj4+/PBDjBw5EqtXr27Ta/3P//wPTp06hauvvhrnnXcejh07hlWrViE1NVUZRt6UwYMHY9OmTZg7dy6GDh2K4OBgjB8/Xjl+6623Yv78+Xjvvfdw3333Qa/Xn/X9ICLfY00NEflEbGwssrOzUVJSgrvuugtTpkzB/v37AcjBQk5ODnr16oXnnnsOs2fPxsaNG5Gamoo77rijza/1pz/9CSaTCX/5y19w//33Y/369Zg0aRI+/vhj5ZEITbn//vtx66234rXXXsOtt96KWbNmeR2PiYnB6NGjAci1NkTUsfHZT0RELbjxxhvxww8/4PDhw/4uChGdAWtqiIiaUVhYiA8//JC1NESdBPvUEBE1cPToUXz55Zd45ZVXoNfrcc899/i7SETUCqypISJq4PPPP8fUqVNx9OhRrF+/vsn5dIio42GfGiIiIuoSWFNDREREXQKDGiIiIuoSGNQQERFRl8CghoiIiLoEBjVERETUJTCoISIioi6BQQ0RERF1CQxqiIiIqEtgUENERERdAoMaIiIi6hIY1BAREVGXwKCGiIiIugQGNURERNQlMKghIiKiLoFBDREREXUJDGqIiIioS2BQQ0RERF0CgxoiIiLqEhjUEBERUZfAoIaIiIi6BAY1RERE1CUwqCEiIqIugUENERERdQkMaoiIiKhLYFBDREREXQKDGiIiIuoSGNQQERFRl8CghoiIiLoEnb8LcK44nU6cOHECISEhkCTJ38UhIiKiVhBCoKqqCvHx8dBozlAXI3xk9erVIjExURiNRjFs2DCxe/fuFvO/9dZbol+/fsJoNIoBAwaIDz/80Ov47bffLgB4LRkZGa0uT0FBQaPzuXDhwoULFy6dYykoKDjjZ71Pamo2bdqEuXPnIjs7G2lpaVixYgUyMjJw6NAhREdHN8q/a9cuTJkyBZmZmRg3bhw2bNiACRMmYN++fRgwYICS79prr8Vrr72m7BuNxlaXKSQkBABQUFCA0NDQdrw7b1V1Npw0W2HUahAXHqDadYmIiAiorKxEQkKC8jneEkkIIdQuQFpaGoYOHYrVq1cDkJt+EhISMGvWLCxYsKBR/kmTJsFsNmPr1q1K2qWXXorU1FRkZ2cDAKZPn47y8nJs2bLlrMpUWVmJsLAwVFRUqBrU/D03D0vf/wljUmLxl9sGq3ZdIiIiatvnt+odha1WK/bu3Yv09PT6F9FokJ6ejtzc3CbPyc3N9coPABkZGY3y79y5E9HR0ejXrx/uu+8+nDx5Uu3it5lJpwUA1Nmcfi4JERFR96Z681NZWRkcDgdiYmK80mNiYnDw4MEmzykqKmoyf1FRkbJ/7bXX4qabbkKfPn1w5MgRLFq0CNdddx1yc3Oh1WobXdNiscBisSj7lZWV7XlbzTLq5bjQYnf45PpERETUOp1m9NPkyZOV7ZSUFPzud7/DBRdcgJ07d+Kaa65plD8zMxOPP/64z8tl1LmCGtbUEBER+ZXqQU1UVBS0Wi2Ki4u90ouLixEbG9vkObGxsW3KDwDnn38+oqKicPjw4SaDmoULF2Lu3LnKvrujkdqMelfzE2tqiIhU5XA4YLPZ/F0M8jG9Xt9ki8vZUD2oMRgMGDx4MHJycjBhwgQAckfhnJwczJw5s8lzhg8fjpycHMyZM0dJ2759O4YPH97s6/z22284efIk4uLimjxuNBrbNDrqbLGmhohIXUIIFBUVoby83N9FoXMkPDwcsbGx7Z5HzifNT3PnzsXtt9+OIUOGYNiwYVixYgXMZjPuuOMOAMC0adPQq1cvZGZmAgBmz56NK6+8EsuXL8fYsWOxceNG7NmzB2vWrAEAVFdX4/HHH8cf//hHxMbG4siRI5g/fz4uvPBCZGRk+OIttJrR1VHYYmdQQ0SkBndAEx0djcDAQE6Y2oUJIVBTU4OSkhIAaLaiorV8EtRMmjQJpaWlWLp0KYqKipCamopt27YpnYHz8/O9ZgUcMWIENmzYgMWLF2PRokVITk7Gli1blDlqtFotvv/+e6xfvx7l5eWIj4/H6NGj8eSTT56T2piWmFwdhetsbH4iImovh8OhBDSRkZH+Lg6dAwEB8hxvJSUliI6ObldTlE/mqemIfDVPzeGSaqS/8DnCAvT4btlo1a5LRNQd1dXV4ejRo0hKSlI+7Kjrq62tRV5eHvr06QOTyeR1zK/z1HQ3Sp8adhQmIlINm5y6F7V+3gxq2smkr598r5tUehERURtMnz5dGTjTFa1btw7h4eH+LgYABjXt5p58DwCsDnYWJiIibytXrsS6deuU/VGjRnmN9lXLvffeC0mSsGLFihbz7dy5E5IkqTa6bNKkSfj5559VuVZ7dZrJ9zoqd/MTII+Aco+GIiIiAoCwsDCfv8Z7772Hr776CvHx8apd02q1wmAwnDFfQEBAh+n/xJqadjJoNXA3BXIEFBFR9/X2228jJSUFAQEBiIyMRHp6Osxms1fz0/Tp0/H5559j5cqVkCQJkiQhLy8PAPDjjz/iuuuuQ3BwMGJiYjB16lSUlZWd8XWPHz+OWbNm4c0334Rer28xb15eHq666ioAQI8ePSBJEqZPnw5ArkGaOXMm5syZg6ioKGXKlBdeeAEpKSkICgpCQkIC7r//flRXVyvXbNj89NhjjyE1NRWvv/46kpKSEBYWhsmTJ6OqqqqVd/LsMahpJ0mSOAEfEVE3V1hYiClTpuDOO+/EgQMHsHPnTtx0002N+lquXLkSw4cPx4wZM1BYWIjCwkIkJCSgvLwcV199NQYNGoQ9e/Zg27ZtKC4uxsSJE1t8XafTialTp+Lhhx/GJZdccsZyJiQk4J133gEAHDp0CIWFhVi5cqVyfP369TAYDPjyyy+RnZ0NQH4o9YsvvoiffvoJ69evx2effYb58+e3+DpHjhzBli1bsHXrVmzduhWff/45nn322TOWr73Y/KQCo06LOpuTE/AREfmAEAK1fqgJD9BrWz0qp7CwEHa7HTfddBMSExMByM8pbCgsLAwGgwGBgYFejwJavXo1Bg0ahGeeeUZJe/XVV5GQkICff/4Zffv2bfJ1s7KyoNPp8L//+7+tKqdWq0VERAQAIDo6ulEH3+TkZPz5z3/2SvPs/5OUlISnnnoK9957L/7yl780+zpOpxPr1q1DSEgIAGDq1KnIycnB008/3apyni0GNSpw19Sw+YmISH21NgcuXvrJOX/d/U9kINDQuo/JgQMH4pprrkFKSgoyMjIwevRo3HzzzejRo0erzv/uu++wY8cOBAcHNzp25MgRfPPNN7jnnnuUtI8//hiBgYFYuXIl9u3b12zwdd111+E///kPACAxMRE//fRTi+UYPHhwo7RPP/0UmZmZOHjwICorK2G321FXV4eamhoEBgY2eZ2kpCQloAHkmYLdswb7EoMaFbiHdbOmhoioe9Jqtdi+fTt27dqFf/3rX1i1ahUeffRR7N69u1XnV1dXY/z48cjKymp0LC4uDk6nE2lpaUpar1698Ne//hUlJSXo3bu3ku5wOPDQQw9hxYoVyMvLwyuvvILa2loAOGN/GwAICgry2s/Ly8O4ceNw33334emnn0ZERAS++OIL3HXXXbBarc0GNQ1fS5IkOJ2+/4xkUKMCTsBHROQ7AXot9j9x7p/zF6Bv22hWSZIwcuRIjBw5EkuXLkViYiLee++9RvkMBgMcDu/Pi9///vd45513kJSUBJ2u6Y9mz5oPQG7SSU9P90rLyMjA1KlTlWct9urVq8nXB9CoDE3Zu3cvnE4nli9frjze6K233jrjef7CoEYF7rlq2FGYiEh9kiS1uhnIX3bv3o2cnByMHj0a0dHR2L17N0pLS3HRRRfh+++/98qblJSE3bt3Iy8vD8HBwYiIiMADDzyAtWvXYsqUKZg/fz4iIiJw+PBhbNy4Ea+88kqTz0OKjIxs9HwsvV6P2NhY9OvXr9myJiYmQpIkbN26FWPGjEFAQECTzV4AcOGFF8Jms2HVqlUYP368Vwfijoijn1RgUp7UzZoaIqLuKDQ0FP/+978xZswY9O3bF4sXL8by5ctx3XXXNco7b948aLVaXHzxxejZsyfy8/MRHx+PL7/8Eg6HA6NHj0ZKSgrmzJmD8PBwrwdAq6FXr154/PHHsWDBAsTExGDmzJnN5h04cCBeeOEFZGVlYcCAAXjzzTeRmZmpannUxAdaquC2V77Cl4dPYuXkVNyQ2riqj4iIWsf9QMumHmxIXVdLP3c+0PIcc88izOYnIiIi/2FQowKTq09NHZufiIiI/IZBjQpYU0NEROR/DGpUwCHdRERE/segRgX1MwqzpoaIiMhfGNSooH5GYdbUEBGpoZsMzCUXtX7eDGpUUN/8xJoaIqL2cE+vX1NT4+eS0Lnk/nm35lEOLenYUzR2EkZXTQ0faElE1D5arRbh4eHKww8DAwNb/aRs6nyEEKipqUFJSQnCw8ObnDm5LRjUqIA1NURE6omNjQWAc/JUZ+oYwsPDlZ97ezCoUYG7poZDuomI2k+SJMTFxSE6Oho2m83fxSEf0+v17a6hcWNQowJl9BM7ChMRqUar1ar2YUfdAzsKq8DEmhoiIiK/Y1CjAk6+R0RE5H8MalTAyfeIiIj8j0GNCjj5HhERkf8xqFEBh3QTERH5H4MaFbif0s3mJyIiIv9hUKMCk54dhYmIiPyNQY0KlMn32PxERETkNwxqVODuU2O1O+F08smyRERE/sCgRgXuoAYArA7W1hAREfkDgxoVuId0A5xVmIiIyF8Y1KhAp5GgkeRtPv+JiIjIPxjUqECSJGVYN2tqiIiI/INBjUo4rJuIiMi/GNSohBPwERER+ZfO3wXoKoysqSGiTszhFLDanfLicC12p3ea3QmbwwmdRoJep4Feq4FBq4FBJ8nbOg0C9ToEGbXQafmdmc49BjUqMek4AR8RtZ0QAjaHQJ3dgTqrA3U2J2ptDtTZHMpaXurTLXYnbB6BhmcAYvMMSBwCVrvDlS688lrsTljtDjnd4YRD5Tm2AvRaBJt0CDHqEGzSIdioQ3igHpFBRkQGGxAZbERUkAFRIUbEhZkQE2qCnoEQtRODGpW4a2rqbKypIeoKnE4Bi10OJJQgw+oZbLiCDKvDKwiptTlgsTlRa3V4nesZmNRaHbDYXdezqx9QqMGgc9fC1K/1WrlGxukKxNxBks1RH2TZHPJ7cb/30ipLq15PkoCewUbEhQegV7gJF/QMRnJMCPrGBOP8qGAYdAx46MwY1KiET+omUo+79sJil2slLHYnLDbv7bqGaXY5mFC27U5YbE7UKenNXMvucH0oyzUW7g9nf/xbliS5hiNAr4VJr4VJr4HJa19OM+g0MLqCDXezj7J4BCJexxpsu48ZPdOUYxIkSTqr92CxO2C2OGC22FFVZ0e1xY5qiw1VdXacNltx0mxFWbUVp8wWnKy2oriqDkUVdbA5BEqqLCipsuC7Au9rajUSkiID0TcmRAl0+sWEoE9UEJu5yAuDGpWYlOc/saaGugZ3H4s6JQCoDxTqA4T6QMIrXxPBhcUu11R4BRct5O1IlRcGnQYmnQYBBu8AI0CvVdKMeo1XQBJg0HoEJJpGgUmAQQuTTqusTQY5oDjbYKKjMOq0MOq0iAgytPocp1OgzGxBYXkdCitq8dvpWvxSXI1fSqrwS3E1qix2HCk140ipGR//WOTxWhr0jw3BxfGhuDguFBfHh6F/bAiCjPxo6674k1eJu6aGo5+otZxOAZvTqfR3sHl0xFT2vdKcsNqFsi0fF/XV/naPNIf3Od7HG7xGs7UXHSeqcAcVRr0WRlftglEnBxLKtuu4nM8jrYl8Jvd19J5NK/VrvVbyCkK0ms4daHR0Go2E6BATokNMGJgQ7nVMCIGiyjr8XFyNX4qr8HNxlbJttjrw3W8V+O63CiW/JAFJkUFIigxEYmQQEiICkRgRiN6RgUjoEYgAgxbUdfksqHnppZfw3HPPoaioCAMHDsSqVaswbNiwZvNv3rwZS5YsQV5eHpKTk5GVlYUxY8Yox4UQWLZsGdauXYvy8nKMHDkSL7/8MpKTk331FtrEPaT75+IqlFVbEBVs9HOJui9304VnkGB1NB04WBt90Hs0Q3idL5R+Aw0DB6889gYBR6OgpL4c9o5UFXEGOo2kBA3uoEIJDJoMLjzTPQKRZoISr2s1CEoMWg00DCq6LUmSEBcWgLiwAFzZt6eS7nQKHDtVg/0nKrG/sAI/najE/hOVKKmy4GiZGUfLzABKG12vZ4gRsaEm9AwxIirYgJ4hRvQMNqJniAlRwQZEBhsQHmhAeICeTVudkCSEUP0v66ZNmzBt2jRkZ2cjLS0NK1aswObNm3Ho0CFER0c3yr9r1y5cccUVyMzMxLhx47BhwwZkZWVh3759GDBgAAAgKysLmZmZWL9+Pfr06YMlS5bghx9+wP79+2Eymc5YpsrKSoSFhaGiogKhoaFqv2UseOd7bPymviE4NtSE83oEeH37M7o62nl+KzR4fTt0p0mNvjl6toNrJPkfukYCNJIEqcFa4zrmmUc5rvE4D/V5dBoNtFoJWkmCViPJj344yw8Se4PhoJYGozOaOlZnc8j9JGzydp3dY1vpF9H8cc+aiY5Uw9AWWo0k/340+J1wd85UfheUvg+SR576YbXeeT3yeJzjdS2txisoMem9gwuDVsM/7tRplFZZcKioCvmnalyLGfmnanDsZA2q6uxtulaoSYeIIDnIkdd6RAQa0CPIgB6BBkQE6b2O9Qg0cASXD7Tl89snQU1aWhqGDh2K1atXAwCcTicSEhIwa9YsLFiwoFH+SZMmwWw2Y+vWrUrapZdeitTUVGRnZ0MIgfj4eDz00EOYN28eAKCiogIxMTFYt24dJk+efMYy+Tqo+bW0Gi/m/ILvj1fgaJkZ6t9VOluSBDlQ8AwW3PNqNBE4GDyCAHeQ0TDorD/eVODgHWDUByL1r+nZzOHOwyYOIt8RQqCi1oaCU7UoqapDWbUFpVXyUlZtlberLThltqKi1nbWrxNi1LmCHr0S/MiLHiEmHYKM8vD2IKPntlZJY1DUWFs+v1VvfrJardi7dy8WLlyopGk0GqSnpyM3N7fJc3JzczF37lyvtIyMDGzZsgUAcPToURQVFSE9PV05HhYWhrS0NOTm5rYqqPG183sGY8XkQQCAaosdBworcbLaIs8n4dHMYXONqvBqtrA3bqpoONmV5zUEAKcQ8uKU/7E6BSDgWrv25eMCwr3tWtfvC593xlQCCo/RGk2N1DDpXZ0lXaM9TK6Ol3KaZ7pnWn2tQqNhpx7BCYMFIpIkSW5WCjQACGsxr93hREWtDadrbDhdY8Vps1Ve19iU7VNmG8prrDjlOl5ea4MQQJXFjiqLHfmnzq6cBp1GCXSCDHLQE2DQttzMqzTpaqDVaKDV1NfQa1217nItPJpIq8+n1QBCAAJQvpgLIZR9Afmg13EIj3ME4sMD0Dcm5OzevApUD2rKysrgcDgQExPjlR4TE4ODBw82eU5RUVGT+YuKipTj7rTm8jRksVhgsdTPj1BZWdm2N9IOwUYdhiZFnLPXay8hBBxOAbtTXjuEgN0hcLaVeFqNpAQZWs3ZDw0lIvIHnVaDyGAjItvQN9LhFKisdQVBNVacNttwqsYqBz6uAMg9xN1sca2tdpgtDlRb7LC6phCw2p04ZbfilNlX7863bk3rjWduTPHb63fZ0U+ZmZl4/PHH/V2MTkGSJOi0EnQcFEBEdFa0GklubmrDUHZPNoezPthxBTpm11LXcOqDZqZBsNjlAQhO15dTh6um3v1l1dlg7XCiUZq7ryUASK7/uNPcX08lCZAg99NEg/zxYWfu4+pLqgc1UVFR0Gq1KC4u9kovLi5GbGxsk+fExsa2mN+9Li4uRlxcnFee1NTUJq+5cOFCryatyspKJCQktPn9EBER+Zpeq/FoHqOzpXpQYzAYMHjwYOTk5GDChAkA5I7COTk5mDlzZpPnDB8+HDk5OZgzZ46Stn37dgwfPhwA0KdPH8TGxiInJ0cJYiorK7F7927cd999TV7TaDTCaKyvOnQ3pZzLZigiIiJqH/fndqu6RAgf2LhxozAajWLdunVi//794u677xbh4eGiqKhICCHE1KlTxYIFC5T8X375pdDpdOL5558XBw4cEMuWLRN6vV788MMPSp5nn31WhIeHi/fff198//334oYbbhB9+vQRtbW1rSpTQUGBgKsvExcuXLhw4cKlcy0FBQVn/Kz3SZ+aSZMmobS0FEuXLkVRURFSU1Oxbds2paNvfn4+NJr6YWsjRozAhg0bsHjxYixatAjJycnYsmWLMkcNAMyfPx9msxl33303ysvLcdlll2Hbtm2tmqMGAOLj41FQUICQkBDVO666m7YKCgp8Mly8u+P99S3eX9/jPfYt3l/f8+c9FkKgqqoK8fHxZ8zrk3lquhtfz4HT3fH++hbvr+/xHvsW76/vdZZ7zFl+iIiIqEtgUENERERdAoMaFRiNRixbtsxrtBWph/fXt3h/fY/32Ld4f32vs9xj9qkhIiKiLoE1NURERNQlMKghIiKiLoFBDREREXUJDGqIiIioS+iyT+luyOl04sSJEz6ZUZiIiIh8w3NGYc+nETSl2wQ1J06c4FO6iYiIOqmCggKcd955LebpNkFNSEgIAPDZIERERJ2I+7lT7s/xlnSboMbd5BQaGqpuUHP4U2DvOqDXYOCyB9W7LhERESla03Wk2wQ1PlNeABz4J+B0+rskRERE3RpHP7WXziSv7XX+LQcREVE3x6CmvXSu52DYLf4tBxERUTfH5qf20gfIa3utf8tBRNTFOBwO2Gw2fxeDfEyv10Or1apyLQY17cWaGiIiVQkhUFRUhPLycn8Xhc6R8PBwxMbGtnseOQY17cU+NUREqnIHNNHR0QgMDOSEqV2YEAI1NTUoKSkBAMTFxbXregxq2ssd1NgY1BARtZfD4VACmsjISH8Xh86BgAC5G0dJSQmio6Pb1RTFjsLtxZoaIiLVuPvQBAYG+rkkdC65f97t7UPFoKa92KeGiEh1bHLqXtT6eTOoaS+OfiIiIuoQGNS0l7v5yWkHHHb/loWIiDqc6dOnY8KECf4uhs/s3LkTkiR1iNFqDGray938BAAONkEREZG3lStXYt26dcr+qFGjMGfOHFWu/e6772L06NGIjIyEJEn49ttvz3hOXl5eq/O2xogRI1BYWIiwsDBVrtceDGray11TA3AEFBERNRIWFobw8HCfXNtsNuOyyy5DVlaW6te2Wq2tymcwGFSZY0YNDGraS6MFNHp5myOgiIi6rbfffhspKSkICAhAZGQk0tPTYTabvZqfpk+fjs8//xwrV66EJEmQJAl5eXkAgB9//BHXXXcdgoODERMTg6lTp6KsrKzF15w6dSqWLl2K9PT0VpezT58+AIBBgwZBkiSMGjVKKduECRPw9NNPIz4+Hv369QMAvP766xgyZAhCQkIQGxuLW2+9VZlXBmjc/LRu3TqEh4fjk08+wUUXXYTg4GBce+21KCwsbHUZzxaDGjVwWDcRke8IAVjN534RotVFLCwsxJQpU3DnnXfiwIED2LlzJ2666SaIBtdYuXIlhg8fjhkzZqCwsBCFhYVISEhAeXk5rr76agwaNAh79uzBtm3bUFxcjIkTJ6p9N/H1118DAD799FMUFhbi3XffVY7l5OTg0KFD2L59O7Zu3QpAHmb95JNP4rvvvsOWLVuQl5eH6dOnt/gaNTU1eP755/H666/j3//+N/Lz8zFv3jzV30tDnHxPDXoTYK1iUENE5Au2GuCZ+HP/uotOAIagVmUtLCyE3W7HTTfdhMTERABASkpKo3xhYWEwGAwIDAxEbGyskr569WoMGjQIzzzzjJL26quvIiEhAT///DP69u3bzjdTr2fPngCAyMhIrzIAQFBQEF555RUYDAYl7c4771S2zz//fLz44osYOnQoqqurERwc3ORr2Gw2ZGdn44ILLgAAzJw5E0888YRq76E5rKlRA2tqiIi6tYEDB+Kaa65BSkoKbrnlFqxduxanT59u9fnfffcdduzYgeDgYGXp378/AODIkSN48803vY795z//adV17733Xq/zziQlJcUroAGAvXv3Yvz48ejduzdCQkJw5ZVXAgDy8/ObvU5gYKAS0ADy4w88m6x8hTU1auAEfEREvqMPlGtN/PG6raTVarF9+3bs2rUL//rXv7Bq1So8+uij2L17d6vOr66uxvjx45vs8BsXFwen04m0tDQlrVevXq267hNPPNGmZp+gIO+aKbPZjIyMDGRkZODNN99Ez549kZ+fj4yMjBY7Euv1eq99SZIaNcX5AoMaNehcE/DZOAEfEZHqJKnVzUD+JEkSRo4ciZEjR2Lp0qVITEzEe++91yifwWCAw+HwSvv973+Pd955B0lJSdDpmv5oDgkJaXOZoqOjER0d3ej1ATQqQ1MOHjyIkydP4tlnn0VCQgIAYM+ePW0ux7nC5ic1sKaGiKhb2717N5555hns2bMH+fn5ePfdd1FaWoqLLrqoUd6kpCTs3r0beXl5KCsrg9PpxAMPPIBTp05hypQp+Oabb3DkyBF88sknuOOOO1oMPk6dOoVvv/0W+/fvBwAcOnQI3377LYqKipo9Jzo6GgEBAUpn5IqKimbz9u7dGwaDAatWrcKvv/6KDz74AE8++WQb7sy55bOg5qWXXkJSUhJMJhPS0tKU3tbN2bx5M/r37w+TyYSUlBR89NFHyjGbzYZHHnkEKSkpCAoKQnx8PKZNm4YTJ/xQHdkU9qkhIurWQkND8e9//xtjxoxB3759sXjxYixfvhzXXXddo7zz5s2DVqvFxRdfrDTnxMfH48svv4TD4cDo0aORkpKCOXPmIDw8HBpN8x/VH3zwAQYNGoSxY8cCACZPnoxBgwYhOzu72XN0Oh1efPFF/PWvf0V8fDxuuOGGZvP27NkT69atw+bNm3HxxRfj2WefxfPPP9+GO3NuScIHjVybNm3CtGnTkJ2djbS0NKxYsQKbN2/GoUOHGlWDAcCuXbtwxRVXIDMzE+PGjcOGDRuQlZWFffv2YcCAAaioqMDNN9+MGTNmYODAgTh9+jRmz54Nh8PR6mqwyspKhIWFoaKiAqGhoeq+4Tf+CBz+FJjwMpB6q7rXJiLqRurq6nD06FH06dMHJpPpzCdQl9DSz70tn98+CWrS0tIwdOhQrF69GgDgdDqRkJCAWbNmYcGCBY3yT5o0CWazWRkTDwCXXnopUlNTm402v/nmGwwbNgzHjh1D7969z1gmnwY1G28DDm4Fxv0fMOTOM+cnIqImMajpntQKalRvfrJardi7d6/X7IYajQbp6enIzc1t8pzc3NxGsyFmZGQ0mx8AKioqIEmSz6aebhP2qSEiIvI71Uc/lZWVweFwICYmxis9JiYGBw8ebPKcoqKiJvM319Gprq4OjzzyCKZMmdJs1GaxWGCx1AcZlZWVbXkbbcPRT0RERH7X6UY/2Ww2TJw4EUIIvPzyy83my8zMRFhYmLK4h6L5BGtqiIiI/E71oCYqKgparRbFxcVe6cXFxY2mY3aLjY1tVX53QHPs2DFs3769xba1hQsXoqKiQlkKCgrO8h21gjL6iTU1RERE/qJ6UGMwGDB48GDk5OQoaU6nEzk5ORg+fHiT5wwfPtwrPwBs377dK787oPnll1/w6aefIjIyssVyGI1GhIaGei0+o3cHNaypISJSw7mYfZY6DrV+3j6ZUXju3Lm4/fbbMWTIEAwbNgwrVqyA2WzGHXfcAQCYNm0aevXqhczMTADA7NmzceWVV2L58uUYO3YsNm7ciD179mDNmjUA5IDm5ptvxr59+7B161Y4HA6lv01ERESj51Scc5ynhohIFe7p9WtqahAQEODn0tC5UlNTA6Dx4xXayidBzaRJk1BaWoqlS5eiqKgIqamp2LZtm9IZOD8/32syoREjRmDDhg1YvHgxFi1ahOTkZGzZsgUDBgwAABw/fhwffPABACA1NdXrtXbs2IFRo0b54m20nrtPjY1BDRFRe2i1WoSHhysPPwwMDIQkSX4uFfmKEAI1NTUoKSlBeHg4tFptu67nk3lqOiKfzlOzew3w8cPAxROAievVvTYRUTcjhEBRURHKy8v9XRQ6R8LDwxEbG9tkANuWz28+0FINHP1ERKQaSZIQFxeH6Oho2Gw2fxeHfEyv17e7hsaNQY0aOPqJiEh1Wq1WtQ876h463Tw1HRJHPxEREfkdgxo1cPQTERGR3zGoUQNHPxEREfkdgxo1uJ/9xJoaIiIiv2FQowaOfiIiIvI7BjVq4OgnIiIiv2NQowaOfiIiIvI7BjVq8Bz91D0maCYiIupwGNSowd2nRjgBB2e/JCIi8gcGNWrQeTxJliOgiIiI/IJBjRrcNTUA+9UQERH5CYMaNUgSoHUP6+YIKCIiIn9gUKMWjoAiIiLyKwY1auHzn4iIiPxK5+8CdBl8/hMRNSQE4LTLi8PmsXZv2+Vt9zHP4w7XfquO2+uvKZz1r11fEO8ynTEd8t80nanBOqCZdI+1IRDQBwE6g9p3k+iMGNSohc9/IvIPIeQPenstYHMt9jrAViN/ybDVuo650ryO1chNxkrg4DhDENHGYMRp9/fd8R+NXg5wDMGAPhAwBNUvele6IdB72xAkB0SGQFeg5A6WXAGT1uCdpjUCGjY4UD0GNWrh85+IvDlsQF2FvFgq5SDCXiv/G7G51u5gw+6xtCafO487iBEOf7/bttHoAa1eXmu09dtaHaDReWzr5X1tg7Wy7U73yKvReryQ5LEpNSpG43T3tjtQrPO475ZWrGvrAzmnx8/fl/RBQFCUHBBJGsAUDgT3BIKi5XVgpGuJqt8O6CHfM+py+FNVC5//RF2BrRZwWAFjqPyhZqmsD0rqKoA697q8/gOrrgKobbBfVy7XgpxzkvzNX2+S1zpTg+2Gx1zNKVqDHAx4BhteAUbDYMIzwGhjAKLRNh9gdAUOG2A1y4utBrBWA9Ya174r3epKt9XU522Y324BHJ4Bk6U+gPJsNrOZgXJzGwspyYFN2HlAeG8gLAEIT5C3Iy4AIi/wnqqDOg0GNWrh6CfqKJxOObi2VMvBiKVSDkYsVa79Kte+x7G6cqDiN+DkEcgfGBK8PjjawxgKGENcAYSpftGbmt9vdMzdl8O1VoKTADnNHahoDV07YOgMtHogIFxefMHdT8leB9itgKUCqC511RI5gNrTgLkUqC6W1zWngJqT9UvtaQACqD0lL0XfN34NSQtE9AGi+gE9+8rryAuBqAvlYIg6LAY1auHoJ2oLIeQaEavZ9c3U3GC7xvWt1nNd0+AbrXvdII8qNSQeAY0hBDCFysGJKVSu3jeFyR9apjCPxWPffcwY2qA5hKidJEkOnLR6wAggKBKIOL/15zvscmBTXSwH8hUFQHm+vD59DDh5WA72Tx6Wl0Mfep8fGFUf4EReCEQmy+uIPqzdMZfJTYF+xKBGLcroJzY/dVlOpxw8WKo8lsoG+55p1U0ELe796nPQiVSqD0SMIfU1Jl77HoFKcDQQc4mcVlchj15hUEJdjVYn97UJ7gnEDmh8XAigqhAoPQSU/Vy/PnkEqDoB1JTJS8FX3udJGrn5KvQ8ICQGCI71WLuWwCg54O+K/6Z+/RzYMAn4w+PAsLv9VmPKoEYt7tFP/30dCI0Hki6XPyzIv5xOj3Z6V0CiBBeuwMMrEGkqSPFY1GqS8aQzeYwMcY8UCawfBeIeOdIw/Ux59YFnPzLE3ZxK1N1Ikvw3PDQeuOAq72OW6voanJOHgbJf6ret1cDpPHlp+QVczXMRro7LEXKwE+TRmTkoyrXdo74GtCMFQk4nAFFfpppTwHv3yE2AJfv92gQsCdFwcgJ1vPTSS3juuedQVFSEgQMHYtWqVRg2bFiz+Tdv3owlS5YgLy8PycnJyMrKwpgxY5TjQggsW7YMa9euRXl5OUaOHImXX34ZycnJrSpPZWUlwsLCUFFRgdBQHwQbP74DvHt3/bdvSeOqkjzf9eGiBfpcCVx8vfwLSs2zW7w7n1ob1nq4A5TqZvYb1I6oHYhI2sa1H8risW8IlgMMY3D9tjtwUYa2BnEUBlFnJwRQVQScOgJUFspNW9VFQFWDdXtGghlD65t3tQZXE5zBo2O7x757W0lvkFcudH3ZvbZdJI1rkeR+d5XHgeIfgbLDcvAiaeRaqKhk+Yvjb98AUX2Bu3fKf9tU1JbPb58ENZs2bcK0adOQnZ2NtLQ0rFixAps3b8ahQ4cQHR3dKP+uXbtwxRVXIDMzE+PGjcOGDRuQlZWFffv2YcAAuXowKysLmZmZWL9+Pfr06YMlS5bghx9+wP79+2Eynflbpc+DGkBun/3qZeDAP4HyY03nkTRy7/ro/kD0xfK3AUOwx4dhsKtPQoRriKKPIl6nU+40pw+QX7Mh99wf7vk2bLXyL7WlSv7moDPWd9Sz18n9QzxHKTgs9cea6v9hq226v4itRr6W6iTXfQ72CCyCGwQkTQQmTQUvOhM7oxJR2zlscn8ez87LtafkvijufbOrecvsOuaXUYRnQWsA/udTIG6g6pf2e1CTlpaGoUOHYvXq1QAAp9OJhIQEzJo1CwsWLGiUf9KkSTCbzdi6dauSdumllyI1NRXZ2dkQQiA+Ph4PPfQQ5s2bBwCoqKhATEwM1q1bh8mTJ5+xTOckqPF6wUKg9CBw+mj9L/JPW4DSA62/htYoV03qjHLtgEbrWuvkZgXPNOGU5+pwOuS1EB7bzvptp0MONOoq6muVjKEApPpJxJy2+llJ/UZyBRRh9cGeVzDiWfMR0qBGpIkaEn0gAxEi6nzsVrlZXJk2odxjoker3PHZYZUXp8e2O909SaTD6lrbPP4WSh4r97ZUX3vj/hwxBst9gnq6voybwuTXqjgOFP4XOL4P6JsBXHyDT25BWz6/Va/3tlqt2Lt3LxYuXKikaTQapKenIzc3t8lzcnNzMXfuXK+0jIwMbNmyBQBw9OhRFBUVIT09XTkeFhaGtLQ05ObmtiqoOedC4+QFHm2yoxa4gp0DQIlrqTnZuN9GXYVc0+GwyB3WfM1S2YpMEhASJwcateXyPxCv2T5ds3s2mv3TeIY+Is30FTGGcqZQIiKdAdBF+X1UUZNCYoHzBgND/V2QeqoHNWVlZXA4HIiJifFKj4mJwcGDB5s8p6ioqMn8RUVFynF3WnN5GrJYLLBY6ueMqaxszQf3OeAOdi64uvk8QshVju7qSKUZyKO2xbP2RTg92j+1rlqcBjU5ksY1uZhWDjiMoUBwjPw61SXy6yqTjbknEtM2mDiMQQYREXVcXbaHYmZmJh5//PFG6R0muGkNTTgQHO6765tdbbWGBv2cBACHa4ETgMW1EBERnVvuz+3W9JZRPaiJioqCVqtFcXGxV3pxcTFiY2ObPCc2NrbF/O51cXEx4uLivPKkpqY2ec2FCxd6NWkdP34cF198MRISEtr8noiIiMi/qqqqEBbW8uhh1YMag8GAwYMHIycnBxMmTAAgdxTOycnBzJkzmzxn+PDhyMnJwZw5c5S07du3Y/jw4QCAPn36IDY2Fjk5OUoQU1lZid27d+O+++5r8ppGoxFGY/3sjsHBwSgoKEBISAgklTuMVlZWIiEhAQUFBeemE3I3w/vrW7y/vsd77Fu8v77nz3sshEBVVRXi4+PPmNcnzU9z587F7bffjiFDhmDYsGFYsWIFzGYz7rjjDgDAtGnT0KtXL2RmZgIAZs+ejSuvvBLLly/H2LFjsXHjRuzZswdr1qwBAEiShDlz5uCpp55CcnKyMqQ7Pj5eCZzORKPR4LzzzvPF21WEhobyH5QP8f76Fu+v7/Ee+xbvr+/56x6fqYbGzSdBzaRJk1BaWoqlS5eiqKgIqamp2LZtm9LRNz8/HxqPTqcjRozAhg0bsHjxYixatAjJycnYsmWLMkcNAMyfPx9msxl33303ysvLcdlll2Hbtm2tmqOGiIiIuj6fzSjcnZzzOXC6Gd5f3+L99T3eY9/i/fW9znKPOUZXBUajEcuWLfPqw0Pq4f31Ld5f3+M99i3eX9/rLPeYNTVERETUJbCmhoiIiLoEBjVERETUJTCoISIioi6BQQ0RERF1CQxqiIiIqEvosg+0bMjpdOLEiRM+eUwCERER+YbnYxI8J+5tSrcJak6cOMGHWRIREXVSBQUFZ3zcUbcJakJCQgBA9Ydxna47jUJzIQJ1gUgKS1LtukRERFT/ME3353hLuk1Q425yUvthXB+d+AhP734af0j8A14Y9YJq1yUiIqJ6rek6wo7C7WTSyQ/UrLXX+rkkRERE3RuDmnZyBzV19jo/l4SIiKh7Y1DTTgHaAAAMaoiIiPyt2/Sp8RWlpsbBoIaISE0OhwM2m83fxSAf0+v10Gq1qlyLQU07sU8NEZG6hBAoKipCeXm5v4tC50h4eDhiY2PbPY8cg5p2MmnZp4aISE3ugCY6OhqBgYGcMLULE0KgpqYGJSUlAIC4uLh2XY9BTTsF6Fx9atj8RETUbg6HQwloIiMj/V0cOgcCAuTP0ZKSEkRHR7erKYodhdvJc/STEMLPpSEi6tzcfWgCAwP9XBI6l9w/7/b2oWJQ007uoMYhHLA77X4uDRFR18Amp+5FrZ83g5p2cg/pBoBaBzsLExGRt+nTp2PChAn+LobPrFu3DuHh4f4uBgAGNe2m0+igleT2P3YWJiKihlauXIl169Yp+6NGjcKcOXPafV2bzYZHHnkEKSkpCAoKQnx8PKZNm4YTJ060eN7OnTshSZJqo8smTZqEn3/+WZVrtReDmnaSJImzChMRUbPCwsJ8UpNRU1ODffv2YcmSJdi3bx/effddHDp0CNdff70q17dara3KFxAQgOjoaFVes70Y1KjAPaybc9UQEXVfb7/9NlJSUhAQEIDIyEikp6fDbDZ7NT9Nnz4dn3/+OVauXAlJkiBJEvLy8gAAP/74I6677joEBwcjJiYGU6dORVlZWbOvFxYWhu3bt2PixIno168fLr30UqxevRp79+5Ffn5+k+fk5eXhqquuAgD06NEDkiRh+vTpAOQapJkzZ2LOnDmIiopCRkYGAOCFF15QaoMSEhJw//33o7q6Wrlmw+anxx57DKmpqXj99deRlJSEsLAwTJ48GVVVVWd5Z1uPQY0KOKswEVH3VlhYiClTpuDOO+/EgQMHsHPnTtx0002NRsWuXLkSw4cPx4wZM1BYWIjCwkIkJCSgvLwcV199NQYNGoQ9e/Zg27ZtKC4uxsSJE9tUjoqKCkiS1GzNUEJCAt555x0AwKFDh1BYWIiVK1cqx9evXw+DwYAvv/wS2dnZAACNRoMXX3wRP/30E9avX4/PPvsM8+fPb7EcR44cwZYtW7B161Zs3boVn3/+OZ599tk2vZez4bOg5qWXXkJSUhJMJhPS0tLw9ddft5h/8+bN6N+/P0wmE1JSUvDRRx8px8623fBccc9VY7Fb/FwSIqKuRwiBGlvNOV/aMk1HYWEh7HY7brrpJiQlJSElJQX3338/goODvfKFhYXBYDAgMDAQsbGxiI2NhVarxerVqzFo0CA888wz6N+/PwYNGoRXX30VO3bsaHV/lbq6OjzyyCOYMmUKQkNDm8yj1WoREREBAIiOjkZsbCzCwsKU48nJyfjzn/+Mfv36oV+/fgCAOXPm4KqrrkJSUhKuvvpqPPXUU3jrrbdaLIvT6cS6deswYMAAXH755Zg6dSpycnJa9T7awyeT723atAlz585FdnY20tLSsGLFCmRkZODQoUNNtrvt2rULU6ZMQWZmJsaNG4cNGzZgwoQJ2LdvHwYMGODVbjhw4ECcPn0as2fPxvXXX489e/b44i20iTKrMGtqiIhUV2uvRdqGtHP+urtv3Y1Afevmyxk4cCCuueYapKSkICMjA6NHj8bNN9+MHj16tOr87777Djt27GgUBAFyrcc333yDe+65R0n7+OOPcfnllyv7NpsNEydOhBACL7/8spJ+3XXX4T//+Q8AIDExET/99FOL5Rg8eHCjtE8//RSZmZk4ePAgKisrYbfbUVdXh5qammbnE0pKSkJISIiyHxcXp8wa7Es+CWpeeOEFzJgxA3fccQcAIDs7Gx9++CFeffVVLFiwoFH+lStX4tprr8XDDz8MAHjyySexfft2rF69GtnZ2Uq7oafVq1dj2LBhyM/PR+/evX3xNlqNz38iIuretFottm/fjl27duFf//oXVq1ahUcffRS7d+9u1fnV1dUYP348srKyGh2Li4uD0+lEWlp9YNerVy9l2x3QHDt2DJ999plXLc0rr7yC2lr5s0mv15+xHEFBQV77eXl5GDduHO677z48/fTTiIiIwBdffIG77roLVqu12aCm4WtJkgSn03nG128v1YMaq9WKvXv3YuHChUqaRqNBeno6cnNzmzwnNzcXc+fO9UrLyMjAli1bmn2dM7Ubnksc/URE5DsBugDsvrV1wYHar9sWkiRh5MiRGDlyJJYuXYrExES89957jfIZDAY4HA6vtN///vd45513kJSUBJ2u6Y9mz5oPN3dA88svv2DHjh2NHi3hGfx4vj6ARmVoyt69e+F0OrF8+XJoNHKPlTM1PfmT6kFNWVkZHA4HYmJivNJjYmJw8ODBJs8pKipqMn9RUVGT+VvTbmixWGCx1PdxqaysbMvbaBPl+U8MaoiIVCdJUqubgfxl9+7dyMnJwejRoxEdHY3du3ejtLQUF110Eb7//nuvvElJSdi9ezfy8vIQHByMiIgIPPDAA1i7di2mTJmC+fPnIyIiAocPH8bGjRvxyiuvNPk8JJvNhptvvhn79u3D1q1b4XA4lM/NiIgIJXhpKDExEZIkYevWrRgzZgwCAgKabPYCgAsvvBA2mw2rVq3C+PHjvToQd0SdbvRTc+2GDWVmZiIsLExZEhISfFYm9qkhIureQkND8e9//xtjxoxB3759sXjxYixfvhzXXXddo7zz5s2DVqvFxRdfjJ49eyI/Px/x8fH48ssv4XA4MHr0aKSkpGDOnDkIDw9XakgaOn78OD744AP89ttvSE1NRVxcnLLs2rWr2bL26tULjz/+OBYsWICYmBjMnDmz2bwDBw7ECy+8gKysLAwYMABvvvkmMjMz236DzhFJqPwURncb29tvv+01LfTtt9+O8vJyvP/++43O6d27N+bOnes1w+KyZcuwZcsWfPfdd0qaO6D59ddf8dlnn7X4BNemamoSEhJQUVHRbO3O2Xoi9wls/nkzHkh9APcOvFfVaxMRdSd1dXU4evQo+vTpA5PJ5O/i0DnS0s+9srISYWFhrfr8Vr2mxmAwYPDgwV5Dt5xOJ3JycjB8+PAmzxk+fHijoV7bt2/3yu/Zbvjpp5+e8ZH0RqMRoaGhXouvsE8NERGR//lk9NPcuXNx++23Y8iQIRg2bBhWrFgBs9msjIaaNm0aevXqpVRhzZ49G1deeSWWL1+OsWPHYuPGjdizZw/WrFkD4OzbDc8VNj8RERH5n0+CmkmTJqG0tBRLly5FUVERUlNTsW3bNqUzcH5+vlcb4YgRI7BhwwYsXrwYixYtQnJyMrZs2YIBAwYAqG83BIDU1FSv19qxYwdGjRrli7fRaqypISIi8j+fBDUAMHPmzGY7H+3cubNR2i233IJbbrmlyfxJSUltmtnxXOOzn4iIiPyv041+6ohYU0NEROR/DGpUoMxTwz41RESq6Mi186Q+tX7eDGpUwJoaIiJ1uKfXr6mp8XNJ6Fxy/7xb8yiHlvisT013wj41RETq0Gq1CA8PVx5+GBgYCEmS/Fwq8hUhBGpqalBSUoLw8PAmZ05uCwY1KlBqatj8RETUbrGxsQBwTp7qTB1DeHi48nNvDwY1KuCzn4iI1CNJEuLi4hAdHQ2bzebv4pCP6fX6dtfQuDGoUYEy+R6DGiIi1Wi1WtU+7Kh7YEdhFbD5iYiIyP8Y1KjAHdTU2ms5DJGIiMhPGNSowN2nBgAsDksLOYmIiMhXGNSowKg1KtvsV0NEROQfDGpUoNPooNfIEwaxXw0REZF/MKhRiWe/GiIiIjr3GNSoJEDLuWqIiIj8iUGNSjism4iIyL8Y1KiEzU9ERET+xaBGJXxSNxERkX8xqFEJ+9QQERH5F4MalbBPDRERkX8xqFGJO6j5tuRbHK8+7ufSEBERdT98SrdKIkwRAID3Dr+H94+8jzsuuQP3p94Pg9bg55IR+Z5TOFFjq0GNvabx2r1tq4HFYYFDOOBwOmAXdjicDjiEA3anvcl1w+Pu84QQEBDKGgCEEHAKJ5xwwumU10IIOIQDTlG/LSCUfc9zlXQIyP+vfw0A0EpaaDVaaCQNtJK81ml0XvtajbZ+W9Iq57gn6NRK8rZOo4NWOvunT7uv4762Tqq/pvv67nR3mbQaLSRIcvk0Gmggl1GSJKXMDRe9Ro8AXQCC9EEI0gchWB8MvVbf/l8YIh9hUKOSe353D4L1wfim6Bt8W/ot/vbj3/BZwWd48PcPYlTCKEiS5O8iEgEAHE4Hau21MNvMXkFHrb1WCUTMNnOj4MR9XDnPI52j/roPvUavBDmB+kAE6Ty23em6QAQbghGk805v6hyNxAYDUo8kusljpSsrKxEWFoaKigqEhob69LVyjuXgia+ewKm6UwCAlKgU3DHgDlydcDW0mrP/dkbUkMVhwem60yi3lDdal1vKUWGpqF+s8rrKWqXUUKhNI2kQpAtCgD4AgbpABOoDEagLVD7o9Fo99Bq9Uqug1Wgb1Sa49xuu3bUQ7loQAIAESO7/SZJcwwC5lsFdA9FUukbSQIL8RUOS5POVtAbXdOdz1/h41iC5953C2WSaV82Tq5bJ7qxfzubLjrtGyn2tpq7rWbtlc9rgcDrqa7BctVkOp6NRzZa77O5aLZvDpgSwvuovGKALQLA+2DswahAMhRhCEGYIQ5gpDGGGMIQbwxFmlNcBugB+aezi2vL5zaDGV69nrcSrP7yKNw68oTy5u3dIb9x+ye24/oLrlT44RG42pw0VlgqU15XjtKU+QFGCFcvp+mOudXtqSBoGIO4PlUBdoBKQuJselPQGxxumG7VGfsB0UXanXQlwzDYzqm3VSo2e2WZWavDc217H7GaYra61K90hHKqUS6/RKwFOqCFUCXg8lxBDCEL0IfUBkivd/cw+6tgY1DThXAc1bmW1ZfjHwX9g48GNqLRWApD730zuPxlT+k1BuCn8nJWFzh2ncKLKWqUEJKfqTjVZm+IZoFRZq87qtXSSDuGmcIQbw9HD1APhxnBlUf6wG8IQbgpHmCEMocZQhBhCYNAYGICQXwghYHFY6gMfu3cg1DBgqrJVyQG/R+1juaUcNqetXeUI0csBjte/G1M4ehh7ICogClEBUegZ2BNRAVGIMEWwqcxPGNQ0wV9BjVuNrQbvHX4Pf//p7zhhPgFArnYd02cMxp4/FgOiBiBAF3DOy0UtE0Kg1l6LSmslKiwVytqzOcf9B9az2afcUg6ncLb59SRIyrdO9x9aZW3sgR6mHl774aZwBOuDGZxQt+P+t+n+t+j+d1dp8f43Wm4pR5W1Sq5dslajylaFSktlm5tgtZIWEaaI+iWgfjvSFNkonX/P1cOgpgn+Dmrc7E47th/bjtd+fA0HTh3wOtYzoCeGxA7B4OjBiAqIQu/Q3rgg/AJ+OzhLQghYnVavb4Ce1eOe1ehK0GKprA9cXEFLe74NBuuDGwUo7mCkqXWoIZT9roh8zOF0yDWpltOosFR4fSE5bZGbfEtrS1FWU4bS2lKcrjvd5iAoQBfgHfAERDS5HxUQhR7GHvxi0gIGNU3oKEGNmxACe4r34J9H/onPCj5DhaWiyXw9jD3QN6IvEkMS0Tu0N84LPg8RARGIDoxGXFBclwl47E476ux1ykiaMy0NR+G42+292vdtNbALuyrl00k6hBnlppswQ31bfaghFKHGUEQYIxoHKsZwDn8l6gLsTjtO1p7EqbpTXsvJupM4VdsgrfYkrE5rm66v1+gRExiD6MBoxATFIDYwFjFBMYgJlJeegT0RGRDZbfsAdYig5qWXXsJzzz2HoqIiDBw4EKtWrcKwYcOazb9582YsWbIEeXl5SE5ORlZWFsaMGaMcF0Jg2bJlWLt2LcrLyzFy5Ei8/PLLSE5OblV5OlpQ40kIgUprJX45/Qt2ndiFX07/glOWU/jl9C8tdgQ1aU2IDIhEsD4YwYZghOhDEGwIRrA+GCGGBtuudYAuwGueDmUOD1ea13weQigjIrzmAHGN6rA6rLA6rbA6rLA5bMq21enad1hhcViU/ZaClPa2jZ9JgC7Aa74Nd8dY9xJqDEWoIVTpf+LZFyXMGMYRFkTUKkIImG1m78Cn7lSj4McdAJ22nG7VdSVI6GGS+/r0DumNxNBEJIYmIiksCb1DeiPCFNFl/0b5PajZtGkTpk2bhuzsbKSlpWHFihXYvHkzDh06hOjo6Eb5d+3ahSuuuAKZmZkYN24cNmzYgKysLOzbtw8DBgwAAGRlZSEzMxPr169Hnz59sGTJEvzwww/Yv38/TKYzjyTqyEFNc2wOG/af2o+jFUeRX5mPY5XHUGguxOm60yiuKfZ5IOAPGkmjBCABugCYdCav/QCdPFInQBfQ7PBPz5E87n026RBRR2Rz2FBaW4rimmIUm4tRXFOMInORvO/aPll78oyjxUL0IXKgE+YKdkKTlMAnSB90jt6Nb/g9qElLS8PQoUOxevVqAIDT6URCQgJmzZqFBQsWNMo/adIkmM1mbN26VUm79NJLkZqaiuzsbAghEB8fj4ceegjz5s0DAFRUVCAmJgbr1q3D5MmTz1imzhjUtMTutONE9QmcqjuFalu10gGu2lqNKmtVo7Rqm5xeZ6/zmqtDK2mVOTqUdGi8Zhl1pzU8z6g1Qq/Vw6AxwKCVF71GD6PWKO9rDF7HGwYnAXrXWlu/zxE5RETenMKJ03WnUVZbhuKaYhRUFSCvIg/HKo8pX3Zb6vPTM6Aneof2Ru+Q3jgv5DwkhCQgPjhe6d8TqA88h++m7dry+a36jMJWqxV79+7FwoULlTSNRoP09HTk5uY2eU5ubi7mzp3rlZaRkYEtW7YAAI4ePYqioiKkp6crx8PCwpCWlobc3NxWBTVdjU6jk39JQ3v7uyhERORDGkmDyIBIRAZEol9Ev0bH6+x1KKgqUIIc95JXmYdTdadQWluK0tpS7C3e2+T1PTs1BxuCveekarAO0AVAo9F4TVjpOYFlfHA8+vbo6+tb0izVg5qysjI4HA7ExMR4pcfExODgwYNNnlNUVNRk/qKiIuW4O625PA1ZLBZYLBZlv7Kysm1vhIiIqBMw6UxI7pGM5B6N+5hWWiuRX5mPvMo8/Fb1GwqqCvBb1W9ys1bdSVgcFtTaa3G8+rgqD2O+pe8tWDp8abuvc7a67LOfMjMz8fjjj/u7GERERH4TagjFgKgBGBA1oNExIQRq7DXKyK6TdSeVh8+6nwvnOR1GrU0e2OE5gMT90FcnnIAAegX38sO7rKd6UBMVFQWtVovi4mKv9OLiYsTGxjZ5TmxsbIv53evi4mLExcV55UlNTW3ymgsXLvRq0qqsrERCQkKb3w8REVFXJEmSMsCiq3RlUD2oMRgMGDx4MHJycjBhwgQAckfhnJwczJw5s8lzhg8fjpycHMyZM0dJ2759O4YPHw4A6NOnD2JjY5GTk6MEMZWVldi9ezfuu+++Jq9pNBphNBqVfXd/aDZDERERdR7uz+1WjWsSPrBx40ZhNBrFunXrxP79+8Xdd98twsPDRVFRkRBCiKlTp4oFCxYo+b/88kuh0+nE888/Lw4cOCCWLVsm9Hq9+OGHH5Q8zz77rAgPDxfvv/+++P7778UNN9wg+vTpI2pra1tVpoKCAgGACxcuXLhw4dIJl4KCgjN+1vukT82kSZNQWlqKpUuXoqioCKmpqdi2bZvS0Tc/Px8aTf1MuCNGjMCGDRuwePFiLFq0CMnJydiyZYsyRw0AzJ8/H2azGXfffTfKy8tx2WWXYdu2ba2aowYA4uPjUVBQgJCQENWHDLubtgoKCrrEcPGOhvfXt3h/fY/32Ld4f33Pn/dYCIGqqirEx8efMW+3eUyCL3W1OXA6Gt5f3+L99T3eY9/i/fW9znKPu8aDg4iIiKjbY1BDREREXQKDGhUYjUYsW7bMa7QVqYf317d4f32P99i3eH99r7PcY/apISIioi6BNTVERETUJTCoISIioi6BQQ0RERF1CQxqiIiIqEvosk/pbsjpdOLEiRM+mVGYiIiIfMNzRmHPpxE0pdsENSdOnOBTuomIiDqpgoICnHfeeS3m6TZBTUhICADw2SBERESdiPu5U+7P8ZZ0m6DG3eQUGhqqalBT9dkOnPr73xH4+0Ho+b//q9p1iYiIqF5ruo50m6DGV+xlpaj56itoAgP9XRQiIqJujaOf2knjmjJa1NX5uSRERETdG4OadpKMJgCA02rxc0mIiIi6NzY/tZNkctfUMKghIlKTw+GAzWbzdzHIx/R6PbRarSrXYlDTTkrzk4XNT0REahBCoKioCOXl5f4uCp0j4eHhiI2Nbfc8cgxq2klpfrJY/VwSIqKuwR3QREdHIzAwkBOmdmFCCNTU1KCkpAQAEBcX167rMahpJ42JHYWJiNTicDiUgCYyMtLfxaFzICAgAABQUlKC6OjodjVFsaNwO0mu5ienhX1qiIjay92HJpDTZHQr7p93e/tQMahpJ3fzk2BQQ0SkGjY5dS9q/bwZ1LSTZ/OTEMLPpSEiIuq+GNS0k7v5CQCElZ2FiYjI2/Tp0zFhwgR/F8Nn1q1bh/DwcH8XAwCDmnbTeAY1bIIiIqIGVq5ciXXr1in7o0aNwpw5c1S59mOPPYb+/fsjKCgIPXr0QHp6Onbv3t3iOTt37oQkSaoNmZ80aRJ+/vlnVa7VXgxq2kuvBzTybXRyBBQRETUQFhbms5qMvn37YvXq1fjhhx/wxRdfICkpCaNHj0ZpaWm7r21tZetDQEAAoqOj2/16amBQ006SJClNUKypISLqvt5++22kpKQgICAAkZGRSE9Ph9ls9mp+mj59Oj7//HOsXLlS/vyQJOTl5QEAfvzxR1x33XUIDg5GTEwMpk6dirKyshZf89Zbb0V6ejrOP/98XHLJJXjhhRdQWVmJ77//vsn8eXl5uOqqqwAAPXr0gCRJmD59OgC5BmnmzJmYM2cOoqKikJGRAQB44YUXkJKSgqCgICQkJOD+++9HdXW1cs2GzU+PPfYYUlNT8frrryMpKQlhYWGYPHkyqqqqzuKutg2DGhVoGNQQEfmMEALOmppzvrRl8EdhYSGmTJmCO++8EwcOHMDOnTtx0003NbrGypUrMXz4cMyYMQOFhYUoLCxEQkICysvLcfXVV2PQoEHYs2cPtm3bhuLiYkycOLHVZbBarVizZg3CwsIwcODAJvMkJCTgnXfeAQAcOnQIhYWFWLlypXJ8/fr1MBgM+PLLL5GdnQ0A0Gg0ePHFF/HTTz9h/fr1+OyzzzB//vwWy3LkyBFs2bIFW7duxdatW/H555/j2WefbfV7OVucfE8Fksk1qzCf/0REpDpRW4tDvx98zl+33769kFo5X05hYSHsdjtuuukmJCYmAgBSUlIa5QsLC4PBYEBgYCBiY2OV9NWrV2PQoEF45plnlLRXX30VCQkJ+Pnnn9G3b99mX3vr1q2YPHkyampqEBcXh+3btyMqKqrJvFqtFhEREQCA6OjoRs1iycnJ+POf/+yV5tn/JykpCU899RTuvfde/OUvf2m2TE6nE+vWrUNISAgAYOrUqcjJycHTTz/d7DlqOKuampdeeglJSUkwmUxIS0vD119/3WL+zZs3o3///jCZTEhJScFHH32kHLPZbHjkkUeUqq34+HhMmzYNJ06c8LpGUlKSUlXnXs5F1NcaktEAgM9/IiLqrgYOHIhrrrkGKSkpuOWWW7B27VqcPn261ed/99132LFjB4KDg5Wlf//+AORajzfffNPr2H/+8x/l3Kuuugrffvstdu3ahWuvvRYTJ05UHjvgbs4KDg7GJZdccsZyDB7cOHj89NNPcc0116BXr14ICQnB1KlTcfLkSdTU1DR7naSkJCWgAeTHH7jL5EttrqnZtGkT5s6di+zsbKSlpWHFihXIyMjAoUOHmuwotGvXLkyZMgWZmZkYN24cNmzYgAkTJmDfvn0YMGAAampqsG/fPixZsgQDBw7E6dOnMXv2bFx//fXYs2eP17WeeOIJzJgxQ9n3vGH+pOEEfEREPiMFBKDfvr1+ed3W0mq12L59O3bt2oV//etfWLVqFR599NEzjkRyq66uxvjx45GVldXoWFxcHJxOJ9LS0pS0Xr16KdtBQUG48MILceGFF+LSSy9FcnIy/va3v2HhwoV45ZVXUFtbC0B+GvaZBAUFee3n5eVh3LhxuO+++/D0008jIiICX3zxBe666y5YrdZmZ35u+FqSJMHpdJ7x9durzUHNCy+8gBkzZuCOO+4AAGRnZ+PDDz/Eq6++igULFjTKv3LlSlx77bV4+OGHAQBPPvkktm/fjtWrVyM7OxthYWHYvn271zmrV6/GsGHDkJ+fj969eyvpISEhXtV1HQWbn4iIfEeSpFY3A/mTJEkYOXIkRo4ciaVLlyIxMRHvvfdeo3wGgwEOh8Mr7fe//z3eeecdJCUlQadr+qO5tV/knU4nLK4v2Z7Bj+frA2hUhqbs3bsXTqcTy5cvh8Y10vett95qVTn8oU3NT1arFXv37kV6enr9BTQapKenIzc3t8lzcnNzvfIDQEZGRrP5AaCiogKSJDVq63v22WcRGRmJQYMG4bnnnoPdbm9L8X1GY2DzExFRd7Z7924888wz2LNnD/Lz8/Huu++itLQUF110UaO8SUlJ2L17N/Ly8lBWVgan04kHHngAp06dwpQpU/DNN9/gyJEj+OSTT3DHHXc0G3yYzWYsWrQIX331FY4dO4a9e/fizjvvxPHjx3HLLbc0W9bExERIkoStW7eitLTUayRTQxdeeCFsNhtWrVqFX3/9Fa+//rrSgbgjalNQU1ZWBofDgZiYGK/0mJgYFBUVNXlOUVFRm/LX1dXhkUcewZQpUxAaGqqk/+///i82btyIHTt24J577sEzzzzTYu9ri8WCyspKr8VXlJoaNj8REXVLoaGh+Pe//40xY8agb9++WLx4MZYvX47rrruuUd558+ZBq9Xi4osvRs+ePZGfn4/4+Hh8+eWXcDgcGD16NFJSUjBnzhyEh4crNSQNabVaHDx4EH/84x/Rt29fjB8/HidPnsR//vOfFvvP9OrVC48//jgWLFiAmJgYzJw5s9m8AwcOxAsvvICsrCwMGDAAb775JjIzM9t+g84V0QbHjx8XAMSuXbu80h9++GExbNiwJs/R6/Viw4YNXmkvvfSSiI6ObpTXarWK8ePHi0GDBomKiooWy/K3v/1N6HQ6UVdX1+TxZcuWCQCNljNd92zkP/CA2N+vvzj1j42qX5uIqDupra0V+/fvF7W1tf4uCp1DLf3cKyoqWv353aaamqioKGi1WhQXF3ulFxcXN9vXJTY2tlX5bTYbJk6ciGPHjmH79u1etTRNSUtLg91uVyYtamjhwoWoqKhQloKCgjO8u7OnMbjnqWHzExERkb+0KagxGAwYPHgwcnJylDSn04mcnBwMHz68yXOGDx/ulR8Atm/f7pXfHdD88ssv+PTTTxEZGXnGsnz77bfQaDTNTs1sNBoRGhrqtfhKffMTH2hJRETkL20e/TR37lzcfvvtGDJkCIYNG4YVK1bAbDYro6GmTZuGXr16KW1us2fPxpVXXonly5dj7Nix2LhxI/bs2YM1a9YAkAOam2++Gfv27cPWrVvhcDiU/jYREREwGAzIzc3F7t27cdVVVyEkJAS5ubl48MEH8ac//Qk9evRQ616cNY3JVVPDZz8RERH5TZuDmkmTJqG0tBRLly5FUVERUlNTsW3bNqUzcH5+vlenphEjRmDDhg1YvHgxFi1ahOTkZGzZsgUDBgwAABw/fhwffPABACA1NdXrtXbs2IFRo0bBaDRi48aNeOyxx2CxWNCnTx88+OCDmDt37tm+b1VJruYnJ5ufiIiI/EYSog0Pt+jEKisrERYWhoqKCtWbokpWrMDJ7L+ix5/+hNjFj6p6bSKi7qSurg5Hjx5Fnz59YHI17VPX19LPvS2f33ygpQo0JveMwqypISJSw7mYfZY6DrV+3nygpQqU5ifOKExE1C4GgwEajQYnTpxAz549YTAYIEmSv4tFPiKEgNVqRWlpKTQajTLb8dliUKMCyd1RmJPvERG1i0ajQZ8+fVBYWNjowcbUdQUGBqJ3797NTjTYWgxqVOB+oCU7ChMRtZ/BYEDv3r1ht9tb9Xwi6ty0Wi10Op0qNXIMalQgGd1DullTQ0SkBkmSoNfrW/VkaSI3dhRWgYbNT0RERH7HoEYFkpEPtCQiIvI3BjUqkIxyb23OKExEROQ/DGpUUD9PDWtqiIiI/IVBjQrY/EREROR/DGpUoGHzExERkd8xqFGB5Gp+clqtfi4JERFR98WgRgXueWpgs0FwoigiIiK/YFCjAo07qAGboIiIiPyFQY0KJI/HpLMJioiIyD8Y1KhA0mgguabyZk0NERGRfzCoUYm7X42TQQ0REZFfMKhRibsJSrD5iYiIyC8Y1KhEozypmzU1RERE/sCgRiX1zU+cVZiIiMgfGNSoRDK5amqsDGqIiIj8gUGNSjTu5z+x+YmIiMgvdP4uQFchKX1qWFNDpCZht0PYbBBWq7I4rVZXmke6zQZhs3rlc5/nVNJsEDYb4LBD2B0QTgfgcMrrhvsOpzxDuMPRxL6zfm23N78vBCStFpJWC7jXOl19mmsbWg0kSQNIkrwArm0o+xIaHmuQD4AkSfDY8WkeSFJ9Pq0GkkYDaLSQtBpA0jSfptXJ++50z32d1iNdK+c16CHpm1lMJkgGAzQmEySjEZLB4CoTdVcMalSidBRm8xN1QMLphLDb5Ud52O2uxQHYPfftEDa7d5rNDuHaR3NpNvf5LaTZPF6jiaBD2NyBh61RAAOn09+3jzoRyWiEZDRC41pLJiM0BqMcABkN3ttGORjSmIyQXOkao8F1Dde2yQTJ4MrjeW13QOV+Ha3W32+dwKBGNe4h3XUHDqLu559hvOAC/pJ3YUKI+poCm8eHsc3qnW7z+JB2b59pv9G53tf2ChBcgUPjNO8ApssEBpIkfxvX6+W1sugh6eW1Ru+R3jCfe1+rBXRaSBpXLUnDWoKm9hvWKii1CQ1rF7SARiOvJQ3gdEA4HMrPQdgdck2RO83hkNOEExACQgj5vQoAyrZwJaBBHuGdD0LZFkJ4nO/O21w+eFzPO8+ZyiOcTsApXO/TKb9Hp6PpNK9aMad3rVcTtWDK77bnvxX3YrHIgXBdnUeZAWGxyMfa83t2NvR6aIxGaAICoAkMhBQUCE1AIDSBDRf5uCYoCNoePaAND4c2vAe0PeS1JiiQtU3twKBGJZqgIADA6TfewOk33oAmOBiGpCRoAgNhTE5GYNowBA4dCl2PHn4uaecm7HY4a2vhrKmFqK2B02KFsMp/xITVCqfFAuFKc7rSvPYtViWvsFrqmzFaFXjUb8Nm8/etaD+tFpJOpyzQ6732Jb0O0DWVpoPkmd5EGnTa+n29O02nBBWaZoMOzwDFoDQ9uPNDp+MffPIihADsdte/bwtEXZ38d8FSJwc3dRb533pdXf3fAve2pU4+T8njCogsdcrfCmW7rg5OqyuvRf57Aru9viA2G5w2G5zV1e17Q3o9dOHhcrATGQl9TAx0sbHQx7rXsdDFxkIbHs5/C02QhPAIcbuwyspKhIWFoaKiAqGhoapfv27/fpS+uAq2kmLY8o7BWVPTOJMkwXD++fIvZVQUdNE95XXPntD17AlteLj87CgBaMPDoA0PhyYoqNP94noGHs4aM0RtLZw1NXKa2bWurXGl19bvK9tN7ddC1NTIAUVHpNXWt/O7P6wbbrd332BoHCQ0FTjo9Mp+fZqucQDTyX6viDoaYbfX1xi5Ayr337uaGtffwBo4a8xKmnClOaqr4Cgvh6O8Ao7Tp+E4fRrC0vruC5LBIAc5MTHQxcXCcN55MPTpA0NSHxj69IE2OMiH7/zcasvn91kFNS+99BKee+45FBUVYeDAgVi1ahWGDRvWbP7NmzdjyZIlyMvLQ3JyMrKysjBmzBjluBACy5Ytw9q1a1FeXo6RI0fi5ZdfRnJyspLn1KlTmDVrFv75z39Co9Hgj3/8I1auXIng4OBWldnXQY0n4XDA8vPPsBUXw1lZidpvv4P5692wHj7S9ovp9dCGhUHrLrNGgi6qZ30gFBIMyehqHzaZlGpNzwWSRq7adjrlql13FbPTCeEU9cccjsaBSI1HQFJT4wo2GqTXev9jPSezKms00AQEeLeTuzsKeu4bjXK7uMEgt5kbja62dGN9bUCj4MFVU+C5bWgh8NDr2dRIRO3mrK2F4/Rp2E+flgOesjLYiophLy6S10VFsBUXw3Hy5BmvpT/vPBj79YOpX18Y+/aDqX8/6BMSOuXfKp8GNZs2bcK0adOQnZ2NtLQ0rFixAps3b8ahQ4cQHR3dKP+uXbtwxRVXIDMzE+PGjcOGDRuQlZWFffv2YcCAAQCArKwsZGZmYv369ejTpw+WLFmCH374Afv374fJ1VfluuuuQ2FhIf7617/CZrPhjjvuwNChQ7FhwwbVb4qv2MvKUHfgIOxlZbCXlcJeKi+O0jJ5XVGhjKJylJe3KWrvkLRapX1ZExBQ38YcEOBKD4AUEFCf5rkfKOdpcj8wkKMciKjbclqtsJeUyEFOUTHsRYWwHjsGy9GjsB7NazbokQICYExOlgOd5GQYEhOh790bhl695ObdDsqnQU1aWhqGDh2K1atXAwCcTicSEhIwa9YsLFiwoFH+SZMmwWw2Y+vWrUrapZdeitTUVGRnZ0MIgfj4eDz00EOYN28eAKCiogIxMTFYt24dJk+ejAMHDuDiiy/GN998gyFDhgAAtm3bhjFjxuC3335DfHz8GcvdEYKatnLW1rqqJ8vhqKyCpJEg7HY5KCopgb2kVK7WdPUVcdbVQphr4Kgxw2k2w1ktV3lCCLnjoiQBGk3z21pNfcc2d+ARFOgKJILqAxRXRzdJCVgCoQmqP0dydYhj4EFEdO45ystR9/PPsBw8hLqfD8Fy6GdYfvml+cf4aDTQx8XBkNhbDnLOOw/aqCjoIiOhjYiALiIC2ogIZZTvudaWz+82dRS2Wq3Yu3cvFi5cqKRpNBqkp6cjNze3yXNyc3Mxd+5cr7SMjAxs2bIFAHD06FEUFRUhPT1dOR4WFoa0tDTk5uZi8uTJyM3NRXh4uBLQAEB6ejo0Gg12796NG2+8sdHrWiwWWDxqOiorK9vyVjsEd5Cgj4vzd1GIiKiT0IaHI2jYMAR5dAsRDges+fmwHPoZdYcOwnr4CKwFBbDm50PU1MB2/Dhsx48Du5r+LAcA6HRyp33XvEByM74O0GjhrKuFs7IKYTfdiJiHHz4H77KZIrYlc1lZGRwOB2JiYrzSY2JicPDgwSbPKSoqajJ/UVGRctyd1lKehk1bOp0OERERSp6GMjMz8fjjj7fynREREXVdklYLY58+MPbpg9BrM5R0IQQcZWWw5ufDeiwf1vxjsP12HI5Tp2A/dUpZwzUHldNuB5oaCOPiOF1+Dt5N87rskO6FCxd61RBVVlYiISHBjyUiIiLqWCRJUgaeBA4e3GQeIQScVVVw1tbJw+A9hsQLmw3C4YQmwARNSAh0UVHn+B14a1NQExUVBa1Wi+LiYq/04uJixMbGNnlObGxsi/nd6+LiYsR5NLMUFxcjNTVVyVNSUuJ1DbvdjlOnTjX7ukajEUaP9j9316HO2AxFRETkdwEmeWmCe7JDKwCo/Dnr/txuVRdg0UbDhg0TM2fOVPYdDofo1auXyMzMbDL/xIkTxbhx47zShg8fLu655x4hhBBOp1PExsaK559/XjleUVEhjEaj+Mc//iGEEGL//v0CgNizZ4+S55NPPhGSJInjx4+3qtwFBQXuKTC5cOHChQsXLp1sKSgoOONnfZubn+bOnYvbb78dQ4YMwbBhw7BixQqYzWbccccdAIBp06ahV69eyMzMBADMnj0bV155JZYvX46xY8di48aN2LNnD9asWQNArvqaM2cOnnrqKSQnJytDuuPj4zFhwgQAwEUXXYRrr70WM2bMQHZ2Nmw2G2bOnInJkye3auQTAMTHx6OgoAAhISGqj8hxN20VFBR0mpFVnQnvr2/x/voe77Fv8f76nj/vsRACVVVVrfq8b3NQM2nSJJSWlmLp0qUoKipCamoqtm3bpnT0zc/Ph0ajUfKPGDECGzZswOLFi7Fo0SIkJydjy5Ytyhw1ADB//nyYzWbcfffdKC8vx2WXXYZt27Ypc9QAwJtvvomZM2fimmuuUSbfe/HFF1tdbo1Gg/POO6+tb7dNQkND+Q/Kh3h/fYv31/d4j32L99f3/HWPw8LCWpWv2zwmwZc64xw4nQnvr2/x/voe77Fv8f76Xme5x5ozZyEiIiLq+BjUqMBoNGLZsmVeo61IPby/vsX763u8x77F++t7neUes/mJiIiIugTW1BAREVGXwKCGiIiIugQGNURERNQlMKghIiKiLoFBDREREXUJXfYp3Q05nU6cOHHCJ49JICIiIt/wfEyC5xMLmtJtgpoTJ04gISHB38UgIiKis1BQUHDGxx11m6AmJCQEAFR/GFf1aQvKi80wBekRlRCi2nWJiIio/mGa7s/xlnSboMbd5KT2w7jy//sbPv/Hrzh/UE+cf08v1a5LRERE9VrTdYQdhdtJq5dvod3q9HNJiIiIujcGNe2k02sBAA6bw88lISIi6t4Y1LSTUlNjY00NERGRP3WbPjW+ojMwqCEiag+HwwGbzebvYpCf6PV6aLVaVa7FoKad6pufGNQQEbWFEAJFRUUoLy/3d1HIz8LDwxEbG9vueeQY1LRTfUdh9qkhImoLd0ATHR2NwMBATozaDQkhUFNTg5KSEgBAXFxcu67HoKaddOxTQ0TUZg6HQwloIiMj/V0c8qOAgAAAQElJCaKjo9vVFMWOwu3k7lPD5iciotZz96EJDAz0c0moI3D/HrS3bxWDmnZy96lhTQ0RUduxyYkA9X4PGNS0k7tPjXAKOBwMbIiISCZJErZs2dLq/NOnT8eECRPa9Zp5eXmQJAnffvttu67TFo899hhSU1PP2eu1hEFNO7n71ACAg7MKExF1C0VFRZg9ezYuvPBCmEwmxMTEYOTIkXj55ZdRU1Pj7+K1aN26dQgPD1ftevPmzUNOTo5q12sPdhRuJ61HUGO3OWEI8GNhiIjI53799VeMHDkS4eHheOaZZ5CSkgKj0YgffvgBa9asQa9evXD99df7u5jtZrVaYTAYzpgvODgYwcHB56BEZ8aamnaSJMljVmEO6yYi6uruv/9+6HQ67NmzBxMnTsRFF12E888/HzfccAM+/PBDjB8/vsnzfvjhB1x99dUICAhAZGQk7r77blRXVzfK9/jjj6Nnz54IDQ3FvffeC6vVqhzbtm0bLrvsMoSHhyMyMhLjxo3DkSNHWl32nTt34o477kBFRQUkSYIkSXjssccAAElJSXjyyScxbdo0hIaG4u677wYAPPLII+jbty8CAwNx/vnnY8mSJV4dehs2P7mb0Z5//nnExcUhMjISDzzwwDmZYJFBjQrcTVAcAUVE1LWdPHkS//rXv/DAAw8gKCioyTxNdXo1m83IyMhAjx498M0332Dz5s349NNPMXPmTK98OTk5OHDgAHbu3Il//OMfePfdd/H44497XWfu3LnYs2cPcnJyoNFocOONN8LpbN3nz4gRI7BixQqEhoaisLAQhYWFmDdvnnL8+eefx8CBA/Hf//4XS5YsAQCEhIRg3bp12L9/P1auXIm1a9fi//7v/1p8nR07duDIkSPYsWMH1q9fj3Xr1mHdunWtKmN7sPlJBXxSNxFR+wkh/PZ3VGfQtGoEzuHDhyGEQL9+/bzSo6KiUFdXBwB44IEHkJWV5XV8w4YNqKurw9///nclGFq9ejXGjx+PrKwsxMTEAAAMBgNeffVVBAYG4pJLLsETTzyBhx9+GE8++SQ0Gg3++Mc/el331VdfRc+ePbF//34MGDDgjOU3GAwICwuDJEmIjY1tdPzqq6/GQw895JW2ePFiZTspKQnz5s3Dxo0bMX/+/GZfp0ePHli9ejW0Wi369++PsWPHIicnBzNmzDhjGduDQY0KOAEfEVH72a1OrJn9uV9e++6VV0JvPPtJ377++ms4nU7cdtttsFgsjY4fOHAAAwcO9KrdGTlyJJxOJw4dOqQENQMHDvSau2f48OGorq5GQUEBEhMT8csvv2Dp0qXYvXs3ysrKlBqa/Pz8JoOaSy65BMeOHQMAXH755fj4449bfB9DhgxplLZp0ya8+OKLOHLkCKqrq2G32xEaGtridS655BKvSfTi4uLwww8/tHiOGhjUqEBncM9Vwz41RERd2YUXXghJknDo0CGv9PPPPx9A/ey4vjJ+/HgkJiZi7dq1iI+Ph9PpxIABA7z63Xj66KOPlL4srSlbwya13Nxc3HbbbXj88ceRkZGBsLAwbNy4EcuXL2/xOnq93mtfkqRWN5G1B4MaFbBPDRFR++kMGty98kq/vXZrREZG4g9/+ANWr16NWbNmNduvpqGLLroI69atg9lsVs758ssvodFovJqyvvvuO9TW1ioByFdffYXg4GAkJCTg5MmTOHToENauXYvLL78cAPDFF1+0+LqJiYmN0gwGAxyO1n0J37VrFxITE/Hoo48qae6an46IHYVVwD41RETtJ0kS9EatX5a2zGj7l7/8BXa7HUOGDMGmTZtw4MABHDp0CG+88QYOHjzY5LOLbrvtNphMJtx+++348ccfsWPHDsyaNQtTp05Vmp4AeRj1XXfdhf379+Ojjz7CsmXLMHPmTGg0GvTo0QORkZFYs2YNDh8+jM8++wxz585t831OSkpCdXU1cnJyUFZW1uK8OsnJycjPz8fGjRtx5MgRvPjii3jvvffa/JrnCoMaFdTX1LD5iYioq7vgggvw3//+F+np6Vi4cCEGDhyIIUOGYNWqVZg3bx6efPLJRucEBgbik08+walTpzB06FDcfPPNuOaaa7B69WqvfNdccw2Sk5NxxRVXYNKkSbj++uuVIdcajQYbN27E3r17MWDAADz44IN47rnn2lz+ESNG4N5778WkSZPQs2dP/PnPf2427/XXX48HH3wQM2fORGpqKnbt2qWMiuqIJCGE8HchzoXKykqEhYWhoqLijB2c2urDv3yPvO/LMOq2frjk8l6qXpuIqCuqq6vD0aNH0adPH5hMJn8Xh/yspd+Htnx+s6ZGBe62WDY/ERER+Y/PgpqXXnoJSUlJMJlMSEtLw9dff91i/s2bN6N///4wmUxISUnBRx99pByz2Wx45JFHkJKSgqCgIMTHx2PatGk4ceKEr4rfJkrzk51BDRERkb/4JKjZtGkT5s6di2XLlmHfvn0YOHAgMjIyUFJS0mT+Xbt2YcqUKbjrrrvw3//+FxMmTMCECRPw448/AgBqamqwb98+LFmyBPv27cO7776LQ4cOdZhna+j0riHdVvapISIi8hef9KlJS0vD0KFDlQ5QTqcTCQkJmDVrFhYsWNAo/6RJk2A2m7F161Yl7dJLL0Vqaiqys7ObfI1vvvkGw4YNw7Fjx9C7d+8zlsmXfWq+2PwLvsspwKDRvTHipgtVvTYRUVfEPjXkqcP2qbFardi7dy/S09PrX0SjQXp6OnJzc5s8Jzc31ys/AGRkZDSbH4DyMK7mHp9usVhQWVnptfgKZxQmIiLyP9WDmrKyMjgcDq9x9wAQExODoqKiJs8pKipqU/66ujo88sgjmDJlSrNRW2ZmJsLCwpQlISHhLN5N67g7CjvY/ERE1CbdZAAunYFavwedbvSTzWbDxIkTIYTAyy+/3Gy+hQsXoqKiQlkKCgp8Viatu08NOwoTEbWKexr9liZ+o+7D/XvQ8PEKbaX6YxKioqKg1WpRXFzslV5cXNzkE0EBIDY2tlX53QHNsWPH8Nlnn7XYtmY0GmE0Gs/yXbSNMvqJQ7qJiFpFq9UiPDxcGUASGBjYpll9qWsQQqCmpgYlJSUIDw9vcjbmtlA9qDEYDBg8eDBycnIwYcIEAHJH4ZycHMycObPJc4YPH46cnBzMmTNHSdu+fTuGDx+u7LsDml9++QU7duxAZGSk2kU/a1r2qSEiajP3F9fmRsZS9xEeHt5sxUdb+OSBlnPnzsXtt9+OIUOGYNiwYVixYgXMZjPuuOMOAMC0adPQq1cvZGZmAgBmz56NK6+8EsuXL8fYsWOxceNG7NmzB2vWrAEgBzQ333wz9u3bh61bt8LhcCj9bSIiImAwGHzxNlpNmXyPj0kgImo1SZIQFxeH6Oho5UnS1P3o9fp219C4+SSomTRpEkpLS7F06VIUFRUhNTUV27ZtUzoD5+fnQ6Op784zYsQIbNiwAYsXL8aiRYuQnJyMLVu2YMCAAQCA48eP44MPPgAApKamer3Wjh07MGrUKF+8jVarn6eGNTVERG2l1WpV+1Cj7o3PflJB/k8n8c9V3yEqIRiTHh2m6rWJiIi6Mz776RxT+tSwpoaIiMhvGNSoQGl+Yp8aIiIiv2FQowJl8j2OfiIiIvIbBjUqYPMTERGR/zGoUYG7+Yk1NURERP7DoEYF7hmFnU4Bp4OBDRERkT8wqFGB1lB/GzmrMBERkX8wqFGBTucR1LBfDRERkV8wqFGBpJGg1fFRCURERP7EoEYlHNZNRETkXwxqVFJfU8OghoiIyB8Y1KiENTVERET+xaBGJVrlSd3sU0NEROQPDGpU4p6rhs1PRERE/qHzdwG6CjY/EXUPQggIp3BNtlm/LZyQ94XwdxFbpJTPvVKKK29IkgRJI0GjlSBJ8lqjcaVpJEiufaKOiEGNSthRmKjtnE4Bp90Jh8O1tsuzcjvsTjgdwnttF3A4PNbNHve4XsO16/qNzm10LadcNqeAcIj6bScgnB07aDkXJEn+m6fRaaDVyVNaNLWv1ctrnUEDnV4DrV7rWsv7Or1W3jbU59MbtNAbtdCbXGujDnqjFjqDBpLEYIpaxqBGJToD+9RQ5yaEgMPmhN3qhN3m8F5b3fvubYfHdjPn2DyOufI7bE4lIHHanejglRpt5q7RUJ3Kl5QabkiSsilEfTDX3M9HCNcXuHP5JU5CfcDjCnoMJh0MAToYTFp5HaCDscG+kidAC2OADnqTjjVNXRiDGpW4+9QUHDiFsOhAxF8YBo2WXZaofZRAwzOA8Aw0GgYhDfO0eJ4DDpsTNqsDDldef9NoJfnbvlZu9tDqNB5ruRZAWbvzNbnWQKNrxTWavJac370ozS6aBk0xWo/mGMlHwYyfuZvahBNeNVcOhxygumu2HHZ3wCrqt101XnbX769DWbt+D+1O5ffOYfP+XbbVOWCz1C9yYeC93w4GkxaGQB2MAXoYA3X1S4AexiA5CDIF6euXYHnfEKBjbVEHx6BGJYGhBgDAkX2lOLKvFKYgPRIu6oGIXsGISQxFzPmhMJh4u7sCIeQ/3A1rIRrVXjQRRLj/cDusDtis8h9z77Vn7YfDb4GGRitBZ5CbCnQGjce21tWUoFWaFJQ0g9yUoG9i7T6u8WyqaBRwSPzA6GAkSQ7aoAW0fiqDcArYbU6vIMdmccBWZ4e1zgFrrR2WWjustXZY61zrWoeyXX/MofR5tNY5YK1zoBqWNpVF0kgwBnoEPMF6mAJ1MAbrYQqU9z2PG4Pkbb1Ry9/tc4SfsioZOq4PgnoYUZpfheOHTqO2yoZf9pQAe0oAyP8YeiYEI+7CcMRfGI6Y80MRGGrgL7pKhBBw2oVc6+CuffCohXDvq1G7Ybc5lU6W55JGKzUKIpR9vWdQoYHWoJXX+gZBiNd24/PdgQprGamjkDSS0uTUXg6bE9Y6Oyw1rqXWpmxba+2w1Mj7dWZ5u85sQ121DXU1dtgtDginkPerbW16XY1WgjFIDoDkwEcPkyvgMXrUCLmDIFOQHBwxGGo7SXT0rvoqqaysRFhYGCoqKhAaGurT13I6nDhxuAIleZUo+60aRUcqUHWqrlE+Y6AOEfFB6BEXhOBwI0xBegSEGGAK1iMgWK+su8oHjNMpYLc4YHP3ybC6vn25AwiPYzZLg+MWd5r7m5qzPt219sdvsqSRmgkQWl+70XRg4XGcgQaR39ltDljMdo9Ax7U22+R0174cFNmUxWk/+z9MWr0GQWEGBIYaERRuQGCYEUFhBgSFGREUZkRgmAFB4UYYA7t2s1hbPr99FtS89NJLeO6551BUVISBAwdi1apVGDZsWLP5N2/ejCVLliAvLw/JycnIysrCmDFjlONCCCxbtgxr165FeXk5Ro4ciZdffhnJycmtKs+5DGqaUnWqDoWHy3HicAUKD5fjdKG51R/C7upMQ4AcubvbgwOCDdCbtEp1vk6vcY0ScI0ccHWqgwRAuNrHRf02IFftCgBwHRMCcpu5qy3cq73cM62JtWcNh80jaLFb5cCjPf+420LSSM0HFk0FEZ55vYIPj9qNpppWDHL/CyKipgghN53JwY4NdWa7x7a8b/EIgDwDorb8vdTqNHKA4w54wo0IiTQhNCrAtZg6dfcHvwc1mzZtwrRp05CdnY20tDSsWLECmzdvxqFDhxAdHd0o/65du3DFFVcgMzMT48aNw4YNG5CVlYV9+/ZhwIABAICsrCxkZmZi/fr16NOnD5YsWYIffvgB+/fvh8lkOmOZ/B3UNGS3OnC6uAanTphRXlyDmior6qpsqK22oq7ahlrXNwB/NHP4nCSPFtO7ggW9UVsfkBm10Om10BtdfTFc1c7ufHqj1pXucdx9DXe/DfbNIKJOTAgBu9WJ2iorzBVW1FRYYK6wwFzusV1hhbnCAovZ3qprmoL1CI00IbRnAEIjAxASaUJgqAGBoQYEhMhrNZr4hBCq//31e1CTlpaGoUOHYvXq1QAAp9OJhIQEzJo1CwsWLGiUf9KkSTCbzdi6dauSdumllyI1NRXZ2dkQQiA+Ph4PPfQQ5s2bBwCoqKhATEwM1q1bh8mTJ5+xTB0tqGkNp1PAUmNDbZVcrWmts7s6yDmUqk67xSGPInCNJLBZHY061AFyZY17dIY8uRaUbUiuNAmAJEGjgfccE3rvOSeUtbItyfNN6CQlyJCDEHftR33Q4a7pYNBBRNR+dpsDNRVW1FRaYS6Xg53q03WoOlmHyrJaVJbVyV+QW0Fv1CIg1ICAYLllwOCeL8gkb2t0GtfoP0Cj0UDSQKnxLvutGsd/Po2LRsThd1clqPoe2/L5rXp9lNVqxd69e7Fw4UIlTaPRID09Hbm5uU2ek5ubi7lz53qlZWRkYMuWLQCAo0ePoqioCOnp6crxsLAwpKWlITc3t8mgxmKxwGKp79leWVnZnrflFxqNhIBgAwKCDf4uChERdUA6vVZpZmqOtdaOypO1qCytc61rUXXagppKK2orraipssoDKywO2Erl42frt4OnVQ9q2kL1oKasrAwOhwMxMTFe6TExMTh48GCT5xQVFTWZv6ioSDnuTmsuT0OZmZl4/PHHz+o9EBERdRWGAB2izgtB1HkhTR4XQsBW50CNK8Cpq7LBarHDVicPjbfVOWC1OBo8FkR+TIgcDNkREhWA8/r2QHzf8HP75hrovD2HzmDhwoVetT+VlZVISPBf9EhERNQRSZKkzMAcHhPo7+K0i+pBTVRUFLRaLYqLi73Si4uLERsb2+Q5sbGxLeZ3r4uLixEXF+eVJzU1tclrGo1GGI1GZd/ddagzNkMRERF1V+7P7dZ0AVY9qDEYDBg8eDBycnIwYcIEAHJH4ZycHMycObPJc4YPH46cnBzMmTNHSdu+fTuGDx8OAOjTpw9iY2ORk5OjBDGVlZXYvXs37rvvvlaVq6qqCgBYW0NERNQJVVVVISwsrMU8Pml+mjt3Lm6//XYMGTIEw4YNw4oVK2A2m3HHHXcAAKZNm4ZevXohMzMTADB79mxceeWVWL58OcaOHYuNGzdiz549WLNmDQC5amzOnDl46qmnkJycrAzpjo+PVwKnM4mPj0dBQQFCQkJ8MtwsISEBBQUFnWZkVWfC++tbvL++x3vsW7y/vufPeyyEQFVVFeLj48+Y1ydBzaRJk1BaWoqlS5eiqKgIqamp2LZtm9LRNz8/HxpN/aRlI0aMwIYNG7B48WIsWrQIycnJ2LJlizJHDQDMnz8fZrMZd999N8rLy3HZZZdh27ZtrZqjBpBHYJ133nnqvtEGQkND+Q/Kh3h/fYv31/d4j32L99f3/HWPz1RD49ZtHpPgS51xDpzOhPfXt3h/fY/32Ld4f32vs9xjzvFOREREXQKDGhUYjUYsW7bMa7QVqYf317d4f32P99i3eH99r7PcYzY/ERERUZfAmhoiIiLqEhjUEBERUZfAoIaIiIi6BAY1RERE1CUwqGmHsrIyPkuKiIiog2BQc5aeeeYZXH311RgyZAhuvvlm7Nq1y99FIlIdB0f6Fu8vkbo4pPssPP3001i5ciWysrJgMBjw0ksvweFwYNmyZRgzZoy/i9fluB+HYTKZcOmll/q7ON1Cfn4+IiMjIYRAcHAwhBCqPzOtO+P99a13330Xu3btQlRUFAYNGoSMjAx/F6lL6dD3V1Cb1NbWimuvvVb83//9n5J2/Phx8dBDD4mLL75YfPfdd/4rXBd04403il69eokLL7xQGAwG8eCDD4qDBw/6u1hd2kMPPSQuuugi0b9/fzFy5Eixd+9e4XA4/F2sLoP317cWLlwoQkJCxM033ywGDhwoAgICxDPPPCNqamr8XbQuoaPfXwY1bVRXVyeGDRsm5s+f75V++PBhMWPGDHHppZeK06dP+6dwXcyTTz4pBg4cKAoKCkRBQYF4//33RXx8vJg6dar473//6+/idUnz588XiYmJ4qOPPhJr164VEyZMEKGhoeL1118XZrPZ38Xr9Hh/fevgwYPiggsuEJ988okQQojy8nKxdu1aodFoxFNPPSWqq6v9XMLOrTPcXwY1bWSz2cTEiRPFhAkTRGlpqdexnTt3iiFDhogVK1b4qXSdn9PpVLanT58uJk6c6HV8y5Yt4ne/+52YOXOmOHHixLkuXpd3zTXXiKysLK+0adOmiQsvvFC8++67rFFoJ95f3/rss89EXFyc+O2337zSX3zxRaHVasU777wjhPD+O0Ot1xnuLzsKt5FOp8PcuXPx/vvv44033vDq6HfllVeif//+2LRpkx9L2LkVFxcDAKxWK6qrq6HT6QAANpsNAHDDDTdgxowZ+Pjjj/Hll18CYGdLNQghUFZWhmPHjqFHjx4AgLq6OgDA+vXr0bt3bzz77LPKz4faxm638/76kPtvQGJiIkpKSvDdd98BkO87AMyaNQvTp0/Hgw8+CKfTyf5LbeB0OpXtTnF//RZOdXLPPvusMBqNYvPmzaKurk5Jf+yxx8QNN9zAb1xn4dFHHxX9+/cXJ0+eFEII8c477whJksSePXuEEMLrPo8fP15cdtllfilnV3brrbeKAQMGKPvue37y5EkRGBgo/vznP/uraJ3Szz//7LX/pz/9ifdXRcXFxcJisSj7tbW1Ytq0aeKyyy4Tx44dE0IIYbVahRBy38fExESxZs0av5S1M9q0aZPyO+l0OkVNTY2YPn16h76/rKk5S4888gjuuusu3HXXXXjxxRfx1Vdf4cCBA9iwYQP69esHjYa3ti0mTZqEv/zlL1izZg0iIiIAANdeey1uuOEG3HTTTaiurobRaITVagUA3HnnnThy5Ah+++031tScpXfffRfvvfcePvroIyXtwQcfRE1NDWbPng1AfjKvxWJBREQE7rnnHnz44Yeora3lPW+Fhx9+GLfccguKi4uV+/XAAw/AYrHw/qpg2bJl+MMf/oBhw4ZhzJgx2L9/P0wmE2677TZlNGpNTQ30ej0A+V7rdDo4HA4/l7xzePjhhzF58mSkpKQAACRJQkBAAG644QYA6LD3l5+87fDSSy/h3nvvxbvvvotrr70W48aNw+DBg5GVleXvonUaVqsVw4YNw6FDh/DTTz/h8ssvR0VFBZxOJwIDA/HEE08gJiYGo0aNQm1tLQwGAwCgsLAQ559/Pnr27Mmq5LNw00034f7778cTTzyBcePGYfLkyfjiiy8wZMgQ3HvvvfjnP/+J5cuXA5D/WAHyzyomJgYBAQG852dwww034NVXX8Urr7yCmJgY5X5dfPHF+J//+R98+OGHvL/tsHDhQvztb3/Dww8/jPvvvx8lJSWYNGkSNm3ahNGjR+NPf/oTfvjhB9x7771e5wUEBChfmqh5N954IzZs2IBdu3bh2muv9To2YcIETJgwAT/99FPHvL9+rSfqIoqKisQ333yjNJNQ661du1bo9XqRnZ0thBDi73//u/jDH/4gLrnkEpGeni7ef/998emnn4rf/e534pJLLhEPPfSQWL16tYiIiBCLFy/2c+k7p9WrV4vf/e53Ij8/X9TU1IivvvpKXHrppeIPf/iD2LVrl6ipqRGLFi0SgYGB4qmnnhL/+c9/xDfffCP69OkjHn/8cX8Xv0Mzm81i8ODBYuDAgaKqqkoIIURJSYmora1V9k+cOMH72w4Wi0WMGDFCrF69Wkmz2Wzi+uuvF8OHDxcfffSRcDgc4pVXXhGJiYni/PPPF3/84x9FUlKSyMjI8GPJOz6HwyFuu+02YTAYxLfffiuEEGLXrl0iKytLLFu2TPzjH/8QQsjNpmvXru2Q95eT75Ff1dTUYOnSpfjXv/6FPn364Mcff8S0adMQHh6ODz74ANXV1Zg1axauv/56PPTQQ/j1119ht9tx4403Ys6cOf4ufqf04IMP4r///S927typpP373//G008/DaPRiNWrVyMuLg7r16/HsmXLYDQaYbfbMXbsWLz88sv+K3gn8NJLL2Hx4sVYtGgRHn74Ybz22mtYv3690gSVmZmJ8ePHw2az4c033+T9bSMhBEpLS3HNNddg5syZuOeee2C1WmEwGFBYWIhbb70VwcHB+Otf/4q4uDgUFhZi9erV0Gq1iIiIwIMPPujvt9DhPf/889i0aROmTp2Kuro6rF69Gv3790dpaSm+//57zJkzB1lZWdBoNCgqKup499e/MRWR/E32lltuERdddJHYvn27km6xWMTo0aNFenq6EEL+NiaEPDcCtZ3D4RAOh0M8/PDDIiMjQ5jNZq8O7Zs3bxZDhw4VWVlZyr3Oz88XR48e5aSSrXTq1Ckxe/Zscfnll4srrrhCJCUliRUrVoi1a9eK6dOni+joaPH6668r95339+yMGjVKXHvttcq+u7Nqbm6uCAkJEa+99pqfStZ5eQ7Dfvjhh0WvXr3E+eefLzZv3qzUMr799ttCkiSxceNGfxXzjBjUUIfwyy+/iHfeeUeZgMxutwshhHjzzTeFwWAQBQUFHFF2lhrOp/T5558LjUYj3n33XSFE/b0WQoh7771XXHLJJco+5/M4s4b39/Dhw+Lmm28WQ4cOFTt27PA6dsstt4iBAwcq+7y/Z/bVV1+Jr7/+Whw6dEhJy83NFQEBAWL58uVCCPl32P17PH36dHHFFVf4paydUVP312aziTlz5oiXX37Z6++DEEJMmDBB+aLZETGooQ7D/W3L0xNPPCHGjh3rh9J0Df/zP/8jxo8fL3799Vev9JkzZ4oePXqIX375RQghlIBxz549IigoSOzfv/+cl7Uzau7+fvvtt+Ktt95Spo53fzBs3bpVmEwmcfjwYQY0rXDnnXeKCy+8UCQmJoqAgADx97//XQghRGVlpXjiiSeEwWAQ77//vtc599xzj5g6dao/itvpNHV/3X8LqqqqRFlZmVf+uro6ce2114oHHnjAH8VtFQY11GHt3LlTXHDBBeK5557zd1E6HbvdLmbMmCHOO+88odPpxAMPPOD1B6qwsFBcddVVIjk52SuA2bBhgxg8eDAf9XEGZ7q/QtQ3lwpRXyPz7LPPitGjR7PW8QxsNpuYMGGCSE1NFd9//704evSoePTRR0WPHj3EqVOnhBBC/Pbbb+KBBx4QWq1WvPbaa+Krr74SBw4cEBdccIF47LHH/PwOOrbW3N+m/PDDD2LQoEFi/fr157C0bcOghjqcDz74QMyZM0eEhYXxj9NZ+u6778TEiRPFJ598Iv75z38KSZLEM888o7SNCyFPlnXppZeKfv36iUmTJolnn31W9OjRQ8ybN8+PJe8cmru/LT375pNPPhEJCQlKkwk17x//+IcYNWqUV8BdXV0tEhMTxebNm5W02tpasXTpUpGQkCDi4+NF7969xbRp0/xR5E6lpfv79ttvN8q/d+9e8cYbb4iePXuKe+6551wWtc0Y1FCHU1FRIW666SaxdetWfxel07JarSInJ0dUVlYKIYR44YUXhFarFW+88YbXzMxCyA8OveGGG8T1118vVq1a5Y/idjot3V/PGW6FkJucbr31VhEeHi6effZZfxS30zl58qS4++67vX5X6+rqRO/evcXHH3/cKP/+/fvFf//7X/HVV1+dy2J2Wm25v1VVVeK5554TSUlJ4v/+7//OcUnbjkO6qUOy2+3Kc5+ofYQQkCQJ9957L9566y289dZbuOaaaxpN8GY2mxEUFOSnUnZeLd1fIQSKi4vx6KOPYsqUKUhPT/d3cTslh8OB2tpapKWl4Y033sCgQYP8XaQu5Uz3t7KyEsXFxUhOTvZTCVuPMwpTh8SARj3u7y3Z2dkYNGgQZs6ciR9//BH5+fmYOXMmPv30UwBAYGCgP4vZabV0fx944AEcO3YMa9euZUBzFtz3VqvVora2FqdOnVKm4bdarXjllVeQn5/vzyJ2ame6v2vXrkV+fj5CQ0M7RUADAKypIeoGPGu++vfvj5CQEPz2229ISEjAF198oTx+gs5OU/e3oKAAvXv35v1VyeHDh5GWloZff/0VZrMZo0aNQo8ePfDll1/yS5AKusr9ZU0NUTeg0+lgt9sBAHPnzsXevXsxfvx4fP311/zAVUFT9/f666/n/VVRSUkJ+vbti2+//RYDBw7EoEGDsHv37k71gduRdZX7y6CGqJvQ6XR49dVXce+99+Kpp57CmjVr/F2kLoX317fMZjN2796Nq6++GjNmzMCmTZv8XaQupavcXzY/EXUTQgh8+OGHsNvtmDBhgr+L0+Xw/vpWeXk5oqKisGXLFowbN87fxelyusr9ZVBDRESdQl1dHUwmk7+L0WV1hfvLoIaIiIi6BPapISIioi6BQQ0RERF1CQxqiIiIqEtgUENERERdAoMaIiIi6hIY1BAREVGXwKCGiIiIugQGNURERNQlMKghIiKiLoFBDREREXUJ/w9/WC++YvfufQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "vis.show_dataframe_plots(feature_dfs, plot_type=\"subplot\") " - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venvpt", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/figs/image_histogram.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/figs/image_histogram.png deleted file mode 100644 index 94503bd053..0000000000 Binary files a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats/figs/image_histogram.png and /dev/null differ diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats_job.py index 0579a82a9b..f8573fe4ba 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats_job.py +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/code/image_stats_job.py @@ -22,7 +22,7 @@ def define_parser(): parser = argparse.ArgumentParser() parser.add_argument("-n", "--n_clients", type=int, default=3) parser.add_argument("-d", "--data_root_dir", type=str, nargs="?", default="/tmp/nvflare/image_stats/data") - parser.add_argument("-o", "--stats_output_path", type=str, nargs="?", default="statistics/stats.json") + parser.add_argument("-o", "--stats_output_path", type=str, nargs="?", default="statistics/image_stats.json") parser.add_argument("-j", "--job_dir", type=str, nargs="?", default="/tmp/nvflare/jobs/image_stats") parser.add_argument("-w", "--work_dir", type=str, nargs="?", default="/tmp/nvflare/workspace/image_stats") diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb index 41852256cb..a2ac9fedd4 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb @@ -7,9 +7,126 @@ "source": [ "# Federated Statistics with image data\n", "\n", - "## Calculate Image Histogram\n", + "In this example, we will compute local and global image statistics with the consideration that data is private at each of the client sites.\n", "\n", - "In this example, we will compute local and global image statistics with the consideration that data is private at each of the client sites." + "## Define target statistics configuration\n", + "\n", + "For Image statistics, we are only interested in histogram of the image intensity, so we ignore all other statistic measures. \n", + "\n", + "```python\n", + "\n", + "statistic_configs = {\"count\": {}, \"histogram\": {\"*\": {\"bins\": 20, \"range\": [0, 256]}}}\n", + "```\n", + "\n", + "## Define the local statistics generator\n", + "\n", + "Based on the above target statistics configuration, we can define the local statistics generator. To do this, we need to write a class that implement \n", + "\n", + "```python\n", + "\n", + "class Statistics(InitFinalComponent, ABC):\n", + "\n", + " def initialize(self, fl_ctx: FLContext):\n", + " def pre_run(self, statistics: List[str], num_of_bins: Optional[Dict[str, Optional[int]]],bin_ranges: Optional[Dict[str, Optional[List[float]]]]):\n", + " def features(self) -> Dict[str, List[Feature]]:\n", + " def count(self, dataset_name: str, feature_name: str) -> int:\n", + " def sum(self, dataset_name: str, feature_name: str) -> float:\n", + " def mean(self, dataset_name: str, feature_name: str) -> float:\n", + " def stddev(self, dataset_name: str, feature_name: str) -> float:\n", + " def variance_with_mean(self, dataset_name: str, feature_name: str, global_mean: float, global_count: float) -> float:\n", + " def histogram(self, dataset_name: str, feature_name: str, num_of_bins: int, global_min_value: float, global_max_value: float) -> Histogram:\n", + " def max_value(self, dataset_name: str, feature_name: str) -> float:\n", + " def min_value(self, dataset_name: str, feature_name: str) -> float:\n", + " def failure_count(self, dataset_name: str, feature_name: str) -> int:\n", + " def quantiles(self, dataset_name: str, feature_name: str, percentiles: List) -> Dict:\n", + " def finalize(self, fl_ctx: FLContext):\n", + "\n", + "```\n", + "\n", + "But since we are only interested in two metrics : Count and Histogram, we can ignore other metrics implementation and only implements count and histogram. Here is the skeleton code for this generator\n", + "\n", + "```python\n", + "\n", + "class ImageStatistics(Statistics):\n", + "\n", + " def __init__(self):\n", + " pass\n", + " \n", + " def initialize(self, fl_ctx: FLContext):\n", + " self.fl_ctx = fl_ctx\n", + " self.client_name = fl_ctx.get_identity_name()\n", + " \n", + " # call load data function \n", + "\n", + " def _load_data_list(self, client_name, fl_ctx: FLContext) -> bool:\n", + " pass\n", + "\n", + "\n", + " def pre_run(\n", + " self,\n", + " statistics: List[str],\n", + " num_of_bins: Optional[Dict[str, Optional[int]]],\n", + " bin_ranges: Optional[Dict[str, Optional[List[float]]]],\n", + " ):\n", + " return {}\n", + "\n", + " def features(self) -> Dict[str, List[Feature]]:\n", + " return {\"train\": [Feature(\"intensity\", DataType.FLOAT)]}\n", + "\n", + " def count(self, dataset_name: str, feature_name: str) -> int:\n", + "\n", + " # return number of images loaded\n", + " pass\n", + " \n", + "\n", + " def failure_count(self, dataset_name: str, feature_name: str) -> int:\n", + "\n", + " return self.failure_images\n", + "\n", + " def histogram(\n", + " self, dataset_name: str, feature_name: str, num_of_bins: int, global_min_value: float, global_max_value: float\n", + " ) -> Histogram:\n", + " # do histogram calculation: \n", + " return Histogram(HistogramType.STANDARD, histogram_bins)\n", + "```\n", + "\n", + "Here ```FLContext``` is the context of the current Job workflow, \"identity\" referring to the site identity, therefore ```get_identity_name()``` will return the site name.\n", + "\n", + "You can take a look of the implementation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31517cdb", + "metadata": {}, + "outputs": [], + "source": [ + "! cat code/src/image_statistics.py" + ] + }, + { + "cell_type": "markdown", + "id": "2bdda4a9", + "metadata": {}, + "source": [ + "# Define Job Configuration\n", + " \n", + "\n", + "```python\n", + "\n", + " statistic_configs = {\"count\": {}, \"histogram\": {\"*\": {\"bins\": 20, \"range\": [0, 256]}}}\n", + " \n", + " # define local stats generator\n", + " stats_generator = ImageStatistics(data_root_dir)\n", + "\n", + " job = StatsJob(\n", + " job_name=\"stats_image\",\n", + " statistic_configs=statistic_configs,\n", + " stats_generator=stats_generator,\n", + " output_path=output_path,\n", + " )\n", + "```" ] }, { @@ -45,12 +162,24 @@ "\n", "As an example, we use the dataset from the [\"COVID-19 Radiography Database\"](https://www.kaggle.com/tawsifurrahman/covid19-radiography-database).\n", "it contains png image files in four different classes: `COVID`, `Lung_Opacity`, `Normal`, and `Viral Pneumonia`.\n", - "First create a temp directory, then we download and extract to `/tmp/nvflare/image_stats/data/.`." + "First create a temp directory, then we download and extract to `/tmp/nvflare/image_stats/data/.`.\n", + "\n", + "\n" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, + "id": "ae0f0f1b", + "metadata": {}, + "outputs": [], + "source": [ + "! pip install kagglehub" + ] + }, + { + "cell_type": "code", + "execution_count": 1, "id": "b4e64769-17f1-4805-9399-1c141e050065", "metadata": { "tags": [] @@ -66,12 +195,28 @@ "fi\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "519789a2", + "metadata": {}, + "outputs": [], + "source": [ + "import kagglehub\n", + "\n", + "# Download latest version\n", + "path = kagglehub.dataset_download(\"tawsifurrahman/covid19-radiography-database\")\n", + "\n", + "print(\"Path to dataset files:\", path)\n", + "\n" + ] + }, { "cell_type": "markdown", "id": "0562f713-5892-43c7-a3d6-d277c337b5ea", "metadata": {}, "source": [ - "Download and unzip the data (you may need to log in to Kaggle or use an API key). Once you have extracted the data from the zip file, you can check the directory to make sure you have the COVID-19_Radiography_Dataset directory at the following location." + "Download and unzip the data (you may need to log in to Kaggle or use an API key). Once you have extracted the data from the zip file, check the directory to make sure you have the COVID-19_Radiography_Dataset directory at the following location." ] }, { @@ -83,7 +228,9 @@ }, "outputs": [], "source": [ - "ls -l /tmp/nvflare/image_stats/data/." + "! mv {path} /tmp/nvflare/image_stats/data/\n", + "\n", + "! tree /tmp/nvflare/image_stats/data" ] }, { @@ -104,17 +251,11 @@ { "cell_type": "code", "execution_count": null, - "id": "e1ea959f-7282-4e55-bb26-11524ec47e99", - "metadata": { - "tags": [] - }, + "id": "79ba087e", + "metadata": {}, "outputs": [], "source": [ - "from code.image_stats.utils.prepare_data import prepare_data\n", - "\n", - "prepare_data(input_dir = \"/tmp/nvflare/image_stats/data\", \n", - " input_ext = \".png\",\n", - " output_dir =\"/tmp/nvflare/image_stats/data\")\n" + "! code/data/prepare_data.sh" ] }, { @@ -140,7 +281,11 @@ "metadata": {}, "outputs": [], "source": [ - "! python3 code/image_stats_job.py" + "%cd code\n", + "\n", + "! python3 image_stats_job.py\n", + "\n", + "%cd -\n" ] }, { @@ -171,7 +316,8 @@ }, "outputs": [], "source": [ - "! ls -al /tmp/nvflare/workspace/image_stats/server/simulate_job/statistics/image_statistics.json" + "! ls -al /tmp/nvflare/workspace/image_stats/server/simulate_job/statistics/image_stats.json\n", + " " ] }, { @@ -182,19 +328,19 @@ }, "source": [ "## Visualization\n", - "We can visualize the results easly via the visualization notebook. Before we do that, we need to copy the data to the notebook directory \n" + "We can visualize the results easily via the visualization notebook. Before we do that, we need to copy the data to the notebook directory\n" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "a3c89693-37b9-450c-85dd-8a2d78fee3fa", "metadata": { "tags": [] }, "outputs": [], "source": [ - "! cp /tmp/nvflare/workspace/image_stats/server/simulate_job/statistics/image_statistics.json image_stats/demo/." + "! cp /tmp/nvflare/workspace/image_stats/server/simulate_job/statistics/image_stats.json code/image_stats/demo/." ] }, { @@ -202,7 +348,7 @@ "id": "d5c6f632-3326-4236-902e-8c0965688d85", "metadata": {}, "source": [ - "now we can visualize via the [visualization notebook](image_stats/demo/visualization.ipynb)" + "now we can visualize via the [visualization notebook](code/image_stats/demo/visualization.ipynb)" ] }, { @@ -213,15 +359,15 @@ }, "source": [ "## We are done !\n", - "Congratulations, you just completed the federated stats image histogram calulation\n" + "Congratulations, you have just completed the federated stats image histogram calculation.\n" ] } ], "metadata": { "kernelspec": { - "display_name": ".venvpt", + "display_name": "nvflare_env", "language": "python", - "name": "python3" + "name": "nvflare_env" }, "language_info": { "codemirror_mode": { @@ -233,7 +379,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.2" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/utils/prepare_data.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/data/prepare_data.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/utils/prepare_data.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/data/prepare_data.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/visualization.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/demo/visualization.ipynb similarity index 82% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/visualization.ipynb rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/demo/visualization.ipynb index c04b07220b..ea51dfe209 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/visualization.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/demo/visualization.ipynb @@ -8,24 +8,6 @@ "# NVFlare Federated Statistics Visualization" ] }, - { - "cell_type": "markdown", - "id": "987e6028", - "metadata": {}, - "source": [ - "#### Dependencies\n", - "\n", - "To run the examples, you will need to install the following dependencies:\n", - "* numpy\n", - "* pandas\n", - "* wget\n", - "* matplotlib\n", - "* jupyter\n", - "* notebook\n", - "\n", - "These are captured in [requirements.txt](../../requirements.txt)." - ] - }, { "cell_type": "markdown", "id": "665dc17e", @@ -37,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "c44a0217", "metadata": { "tags": [] @@ -81,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "93c62d5e", "metadata": { "tags": [] @@ -249,31 +231,24 @@ }, { "cell_type": "markdown", - "id": "2f330eb6", + "id": "ffa1da20", "metadata": {}, "source": [ - "### Tip: Avoid repeated calculation\n", - "If you intend to plot the histogram main plot and subplot separately, repeatedly calling `show_histograms()` with different plot_types is not efficicent, as it repeatedly calculates the same set of Dataframes. To do it efficiently, you can use the following functions instead of `show_histograms()` to avoid the duplicated calculations. If you intend to show both plots, then `show_histograms()` should be used." + "Now, Let's back to [federated_statistics_with_tabular_data](../../../federated_statistics_with_tabular_data/federated_statistics_with_tabular_data.ipynb)" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "395315a4", + "cell_type": "markdown", + "id": "d3f12a4a", "metadata": {}, - "outputs": [], - "source": [ - "feature_dfs = vis.get_histogram_dataframes(data, display_format=\"percent\")\n", - " \n", - "vis.show_dataframe_plots(feature_dfs, plot_type=\"main\")" - ] + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "nvflare_example", + "display_name": "nvflare_env", "language": "python", - "name": "nvflare_example" + "name": "nvflare_env" }, "language_info": { "codemirror_mode": { @@ -285,7 +260,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.2" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/df_stats.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/df_stats.png deleted file mode 100644 index da9835b735..0000000000 Binary files a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/df_stats.png and /dev/null differ diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/hist_plot.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/hist_plot.png deleted file mode 100644 index 9fef1da1ec..0000000000 Binary files a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/hist_plot.png and /dev/null differ diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/stats_df.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/stats_df.png deleted file mode 100644 index 90c797b243..0000000000 Binary files a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats/demo/stats_df.png and /dev/null differ diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats_job.py index 6666b9a533..e2f3637757 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats_job.py +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/df_stats_job.py @@ -20,11 +20,13 @@ def define_parser(): parser = argparse.ArgumentParser() - parser.add_argument("-n", "--n_clients", type=int, default=3) + parser.add_argument("-n", "--n_clients", type=int, default=2) parser.add_argument("-d", "--data_root_dir", type=str, nargs="?", default="/tmp/nvflare/df_stats/data") - parser.add_argument("-o", "--stats_output_path", type=str, nargs="?", default="statistics/adult_stats.json") + parser.add_argument("-o", "--stats_output_path", type=str, nargs="?", default="statistics/adults_stats.json") parser.add_argument("-j", "--job_dir", type=str, nargs="?", default="/tmp/nvflare/jobs/stats_df") - parser.add_argument("-w", "--work_dir", type=str, nargs="?", default="/tmp/nvflare/jobs/stats_df") + parser.add_argument("-w", "--work_dir", type=str, nargs="?", default="/tmp/nvflare/jobs/stats_df/work_dir") + parser.add_argument("-co", "--export_config", action="store_true", help="config only mode, export config") + parser.add_argument("-l", "--log_config", type=str, default="concise") return parser.parse_args() @@ -37,18 +39,19 @@ def main(): output_path = args.stats_output_path job_dir = args.job_dir work_dir = args.work_dir + export_config = args.export_config + log_config = args.log_config statistic_configs = { "count": {}, "mean": {}, "sum": {}, "stddev": {}, - "histogram": {"*": {"bins": 20}}, - "Age": {"bins": 20, "range": [0, 10]}, - "percentile": {"*": [25, 50, 75], "Age": [50, 95]}, + "histogram": {"*": {"bins": 20}, "Age": {"bins": 20, "range": [0, 100]}}, + "quantile": {"*": [0.1, 0.5, 0.9], "Age": [0.5, 0.9]}, } # define local stats generator - df_stats_generator = DFStatistics(data_root_dir=data_root_dir) + df_stats_generator = DFStatistics(filename="data.csv", data_root_dir=data_root_dir) job = StatsJob( job_name="stats_df", @@ -60,9 +63,11 @@ def main(): sites = [f"site-{i + 1}" for i in range(n_clients)] job.setup_clients(sites) - job.export_job(job_dir) - - job.simulator_run(work_dir) + if export_config: + print("Exporting job config...", job_dir) + job.export_job(job_dir) + else: + job.simulator_run(work_dir, log_config=log_config) if __name__ == "__main__": diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/src/df_statistics.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/src/df_statistics.py index 5078c2f0a9..942bbf692d 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/src/df_statistics.py +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/code/src/df_statistics.py @@ -21,10 +21,10 @@ class DFStatistics(DFStatisticsCore): - def __init__(self, data_path): + def __init__(self, filename, data_root_dir="/tmp/nvflare/df_stats/data"): super().__init__() - self.data_root_dir = "/tmp/nvflare/df_stats/data" - self.data_path = data_path + self.data_root_dir = data_root_dir + self.filename = filename self.data: Optional[Dict[str, pd.DataFrame]] = None self.data_features = [ "Age", @@ -57,7 +57,7 @@ def load_data(self, fl_ctx: FLContext) -> Dict[str, pd.DataFrame]: self.log_info(fl_ctx, f"load data for client {client_name}") try: skip_rows = self.skip_rows[client_name] - data_path = f"{self.data_root_dir}/{fl_ctx.get_identity_name()}/{self.data_path}" + data_path = f"{self.data_root_dir}/{fl_ctx.get_identity_name()}/{self.filename}" # example of load data from CSV df: pd.DataFrame = pd.read_csv( data_path, names=self.data_features, sep=r"\s*,\s*", skiprows=skip_rows, engine="python", na_values="?" diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/federated_statistics_with_tabular_data.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/federated_statistics_with_tabular_data.ipynb index a630c65dcc..4e6d4cee41 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/federated_statistics_with_tabular_data.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.1_federated_statistics/federated_statistics_with_tabular_data/federated_statistics_with_tabular_data.ipynb @@ -2,24 +2,28 @@ "cells": [ { "cell_type": "markdown", - "id": "26cb3afa", + "id": "64a17f22-5667-4f99-b4f6-d49116db74b0", "metadata": { "tags": [] }, "source": [ - "# Data Frame Federated Statistics \n", + "# Federated Statistics with Tabular Data\n", + "\n", + "Tabular data is the most common data type in the world; it is easy to understand and manipulate. In this chapter, we will explore federated statistics with tabular data. We will leverage pandas dataframe to calculate the statistics of the local data and the global data.\n", + "\n", + "\n", + "To illustrate with an example, we will prepare the dependencies and prepare the dataset. \n", "\n", - "In this example, we will show how to generate federated statistics for data that can be represented as Pandas Data Frame." + "## Prerequisites" ] }, { "cell_type": "markdown", - "id": "64a17f22-5667-4f99-b4f6-d49116db74b0", - "metadata": { - "tags": [] - }, + "id": "497d44fa", + "metadata": {}, "source": [ - "## Install requirements\n", + "\n", + "### Install dependencies\n", "First, install the required packages:" ] }, @@ -35,6 +39,47 @@ "%pip install -r code/requirements.txt" ] }, + { + "cell_type": "markdown", + "id": "05d9890a", + "metadata": {}, + "source": [ + "\n", + "> Sidebar: \n", + "> **Installing fastdigest**\n", + ">\n", + "> If you intend to calculate quantiles, you need to install fastdigest. the fastdigest not included in the requirements.txt file. If you are not calculating quantiles, you can skip this step.\n", + ">\n", + "> ```bash\n", + "> pip install fastdigest==0.4.0\n", + "> ```\n", + ">\n", + "> On Ubuntu, you might get the following error:\n", + ">\n", + "> ```\n", + "> Cargo, the Rust package manager, is not installed or is not on PATH.\n", + "> This package requires Rust and Cargo to compile extensions. Install it through\n", + "> the system's package manager or via https://rustup.rs/\n", + "> \n", + "> Checking for Rust toolchain....\n", + "> ```\n", + ">\n", + "> This is because fastdigest (or its dependencies) requires Rust and Cargo to build. \n", + ">\n", + "> You need to install Rust and Cargo on your Ubuntu system. Follow these steps:\n", + ">\n", + "> 1. Install Rust and Cargo by running:\n", + "> ```bash\n", + "> cd NVFlare/examples/advanced/federated-statistics/df_stats\n", + "> ./install_cargo.sh\n", + "> ```\n", + ">\n", + "> 2. Then install fastdigest again:\n", + "> ```bash\n", + "> pip install fastdigest==0.4.0\n", + "> ```" + ] + }, { "cell_type": "markdown", "id": "94faaa6b-08fd-485c-87d5-53b4520177fe", @@ -43,10 +88,14 @@ }, "source": [ "\n", - "## Prepare data\n", + "### Prepare data\n", "\n", "In this example, we are using the UCI (University of California, Irvine) [adult dataset](https://archive.ics.uci.edu/dataset/2/adult)\n", "The original dataset already contains \"training\" and \"test\" datasets. Here we simply assume that the \"training\" and \"test\" data set each belong to a client, so we assign the adult.train dataset to site-1 and the adult.test dataset to site-2.\n", + "```\n", + "site-1: adult.train\n", + "site-2: adult.test\n", + "```\n", "\n", "Now we use the data utility to download UCI datasets to separate client package directory to /tmp/nvflare/data/ directory.\n", "Please note that the UCI's website may experience occasional downtime." @@ -61,9 +110,13 @@ }, "outputs": [], "source": [ - "from code.df_stats.utils.prepare_data import prepare_data\n", + "%cd code/data\n", "\n", - "prepare_data(data_root_dir = \"/tmp/nvflare/df_stats/data\")" + "from prepare_data import prepare_data\n", + "\n", + "prepare_data(data_root_dir = \"/tmp/nvflare/df_stats/data\")\n", + "\n", + "%cd -" ] }, { @@ -71,7 +124,7 @@ "id": "c5444d8f-4938-4759-bd43-831013043c23", "metadata": {}, "source": [ - "#### Let's take a look at the data" + "Let's take a look at the data" ] }, { @@ -122,7 +175,175 @@ "jp-MarkdownHeadingCollapsed": true }, "source": [ - "> Note **We will only calculate the statistics of numerical features, categorical features will be skipped**" + "> Note \n", + "We will only calculate the statistics of numerical features; categorical features will be skipped." + ] + }, + { + "cell_type": "markdown", + "id": "ae3bb5a4", + "metadata": {}, + "source": [ + "## Define target statistics configuration\n", + "\n", + "\n", + "Let's see what statistics we want to calculate, we can capture the statistics configuration in a dictionary, the key is the statistic name, the value is the statistic configuration.\n", + "\n", + "```python\n", + "\n", + " statistic_configs = {\n", + " \"count\": {},\n", + " \"mean\": {},\n", + " \"sum\": {},\n", + " \"stddev\": {},\n", + " \"histogram\": {\"*\": {\"bins\": 20}, \"Age\": {\"bins\": 10, \"range\": [0, 100]}},\n", + " \"quantile\": {\"*\": [0.1, 0.5, 0.9], \"Age\": [0.5, 0.9]},\n", + " }\n", + "```\n", + "\n", + "For each statistic, we can configure to give additional instructions for each feature. While count, mean, sum and stddev are defined in such a way that the calculation will be the same for all features, for histogram, we can define different bins for each feature. \"*\" is a wildcard for all features.\n", + "```\n", + "\"histogram\": {\"*\": {\"bins\": 20}, \"Age\": {\"bins\": 20, \"range\": [0, 10]}},\n", + "```\n", + "here, we define a histogram with a histogram with 20 bins for all features, the range is not defined, which means the range will be calculated from the data.\n", + "We also defined 10 bins and range [0, 100] for the feature \"Age\".\n", + "\n", + "Similarly the quantile is defined for different features with different values. This can be specified with Job API\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d17a6a9", + "metadata": {}, + "outputs": [], + "source": [ + "!cat code/df_stats_job.py" + ] + }, + { + "cell_type": "markdown", + "id": "b1857b89", + "metadata": {}, + "source": [ + "If a user only wants to calculate statistics except for quantile and histogram, then the configuration can be simplified as:\n", + "\n", + "```python\n", + "\n", + " statistic_configs = {\n", + " \"count\": {},\n", + " \"mean\": {},\n", + " \"sum\": {},\n", + " \"stddev\": {},\n", + " }\n", + "\n", + "```\n", + "\n", + "Similarly, the code can be changed to \n", + "\n", + "\n", + "```python\n", + "\n", + " skip previous code\n", + "\n", + "\n", + " statistic_configs = {\n", + " \"count\": {},\n", + " \"mean\": {},\n", + " \"sum\": {},\n", + " \"stddev\": {},\n", + "\n", + " }\n", + " # define local stats generator\n", + " df_stats_generator = DFStatistics(data_root_dir=data_root_dir)\n", + "\n", + " job = StatsJob(\n", + " job_name=\"stats_df\",\n", + " statistic_configs=statistic_configs,\n", + " stats_generator=df_stats_generator,\n", + " output_path=output_path,\n", + " )\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "id": "f102f6b9", + "metadata": {}, + "source": [ + "## Define the local statistics generator\n", + "\n", + "Based on the above target statistics configuration, we can define the local statistics generator. To do this, we need to write a class that implement \n", + "\n", + "\n", + "```python\n", + "\n", + "class Statistics(InitFinalComponent, ABC):\n", + "\n", + " def initialize(self, fl_ctx: FLContext):\n", + " def pre_run(self, statistics: List[str], num_of_bins: Optional[Dict[str, Optional[int]]],bin_ranges: Optional[Dict[str, Optional[List[float]]]]):\n", + " def features(self) -> Dict[str, List[Feature]]:\n", + " def count(self, dataset_name: str, feature_name: str) -> int:\n", + " def sum(self, dataset_name: str, feature_name: str) -> float:\n", + " def mean(self, dataset_name: str, feature_name: str) -> float:\n", + " def stddev(self, dataset_name: str, feature_name: str) -> float:\n", + " def variance_with_mean(self, dataset_name: str, feature_name: str, global_mean: float, global_count: float) -> float:\n", + " def histogram(self, dataset_name: str, feature_name: str, num_of_bins: int, global_min_value: float, global_max_value: float) -> Histogram:\n", + " def max_value(self, dataset_name: str, feature_name: str) -> float:\n", + " def min_value(self, dataset_name: str, feature_name: str) -> float:\n", + " def failure_count(self, dataset_name: str, feature_name: str) -> int:\n", + " def quantiles(self, dataset_name: str, feature_name: str, percentiles: List) -> Dict:\n", + " def finalize(self, fl_ctx: FLContext):\n", + "\n", + "```\n", + "\n", + "\n", + "NVIDIA FLARE has implemented ```DFStatisticsCore```, which is a core class for calculating the statistics of the data frame. We can inherit this class and override the methods to calculate the statistics. Here are a few assumptions:\n", + "\n", + "* data can be loaded and cached in the memory.\n", + "* data has the proper column names and can be loaded into a pandas dataframe.\n", + "* The feature names can be obtained from the dataframe.\n", + "\n", + "```\n", + "def load_data(self, fl_ctx: FLContext) -> Dict[str, pd.DataFrame]:\n", + "```\n", + "which loads the data into a dictionary of pandas dataframe, the key is the dataset name, the value is the pandas dataframe.\n", + "\n", + "Let's take a look what the user needs to do to implement the local statistics generator. With ```DFStatisticsCore```, the user needs to implement the following methods:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ad902da", + "metadata": {}, + "outputs": [], + "source": [ + "!cat code/src/df_statistics.py\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "c9eb7c51", + "metadata": {}, + "source": [ + "# Define Job Configuration\n", + "\n", + "Each FLARE job is defined by a job configuration, the configuration includes configurations for the clients and server. Optionally, the job configuration also contains the customized job code. You have seen this in [Job Structure and configuration](../../../chapter-1_running_federated_learning_applications/01.6_job_structure_and_configuration/understanding_fl_job.ipynb)\n", + "\n", + "For now, we can define a FebJob via FLARE's Job API, here is Statistrics Job we havea predefined for you\n", + "\n", + "```python\n", + "\n", + " job = StatsJob(\n", + " job_name=\"\",\n", + " statistic_configs=statistic_configs,\n", + " stats_generator=df_stats_generator,\n", + " output_path=output_path,\n", + " )\n", + "\n", + "```" ] }, { @@ -130,6 +351,9 @@ "id": "5c4da328", "metadata": {}, "source": [ + "For this example, we simply hardcoded the column names, but in practice, users can get the column names from files such as CSV files, parquet files, etc.\n", + "\n", + "\n", "## Run Job with FL Simulator" ] }, @@ -148,7 +372,10 @@ "metadata": {}, "outputs": [], "source": [ - "! python3 code/df_stats_job.py" + "%cd code\n", + "\n", + "! python3 df_stats_job.py \n", + "%cd -" ] }, { @@ -166,9 +393,10 @@ "id": "45bf6e9a-3265-4e45-8b06-c8e543605f21", "metadata": {}, "source": [ - "With the default parameters, the results are stored in workspace \"/tmp/nvflare/jobs/stats_df/\"\n", + "With the default parameters, the results are stored in workspace \"/tmp/nvflare/jobs/stats_df/work_dir\"\n", "```\n", - "/tmp/nvflare/jobs/stats_df/server/simulate_job/statistics/adults_stats.json\n", + "/tmp/nvflare/jobs/stats_df/work_dir/server/simulate_job/statistics/adults_stats.json \n", + "\n", "```" ] }, @@ -181,7 +409,7 @@ }, "outputs": [], "source": [ - "cat /tmp/nvflare/jobs/stats_df/server/simulate_job/statistics/adults_stats.json" + "ls -al /tmp/nvflare/jobs/stats_df/work_dir/server/simulate_job/statistics/adults_stats.json " ] }, { @@ -197,14 +425,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "a3c89693-37b9-450c-85dd-8a2d78fee3fa", "metadata": { "tags": [] }, "outputs": [], "source": [ - "! cp /tmp/nvflare/jobs/stats_df/server/simulate_job/statistics/adults_stats.json code/df_stats/demo/." + "! cp /tmp/nvflare/jobs/stats_df/work_dir/server/simulate_job/statistics/adults_stats.json code/demo/." ] }, { @@ -212,9 +440,17 @@ "id": "d5c6f632-3326-4236-902e-8c0965688d85", "metadata": {}, "source": [ - "now we can visualize the results with the [visualization notebook](code/df_stats/demo/visualization.ipynb)" + "Now we can visualize the results with the [visualization notebook](code/demo/visualization.ipynb)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "ef94e26f", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "fda06c0b-798d-480d-9b4c-a62fab95bcf0", @@ -223,15 +459,23 @@ }, "source": [ "## We are done !\n", - "Congratulations, you just completed the federated stats calulation with data represented by data frame\n" + "Congratulations, you just completed the federated stats calculation with data represented by a data frame.\n", + "\n", + "Let's move on to [federated stats with Image Data](../federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb)" ] + }, + { + "cell_type": "markdown", + "id": "70459712", + "metadata": {}, + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "nvflare_example", + "display_name": "nvflare_env", "language": "python", - "name": "nvflare_example" + "name": "nvflare_env" }, "language_info": { "codemirror_mode": { @@ -243,7 +487,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.2" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_client_api/client_api.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_client_api/client_api.ipynb new file mode 100644 index 0000000000..69eb19ebf2 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_client_api/client_api.ipynb @@ -0,0 +1,248 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "58149c32", + "metadata": {}, + "source": [ + "# Transform Existing Code to FL Easily with the FLARE Client API" + ] + }, + { + "cell_type": "markdown", + "id": "06203527", + "metadata": {}, + "source": [ + "The FLARE Client API offers a straightforward path to transform your existing machine learning or deep learning code into federated learning applications. With just a few lines of code changes, you can adapt your training logic without restructuring your codebase or moving code into different class methods. This flexibility applies to both traditional machine learning and deep learning frameworks. For PyTorch Lightning users, the process is even more streamlined with dedicated Lightning API support.\n", + "\n", + "You can see detailed examples with actual integration across different platforms including PyTorch and TensorFlow [here:](https://github.com/NVIDIA/NVFlare/tree/main/examples/hello-world/ml-to-fl)\n", + "\n", + "In Chapter 1, you have already seen the Client API in action with pytorch. In this section, we will focus on the core concepts of the Client API and explain some of the ways it can be configured to help you use the Client API more effectively.\n", + "\n", + "Then we will see how to use the Client API with PyTorch Lightning, and traditional machine learning algorithms such as Logistic Regression, KMeans and survival analysis." + ] + }, + { + "cell_type": "markdown", + "id": "be7efa36", + "metadata": {}, + "source": [ + "## Core Concept" + ] + }, + { + "cell_type": "markdown", + "id": "76102eac", + "metadata": {}, + "source": [ + "The general workflow of the popular federated learning (FL) follows the following steps:\n", + "\n", + "1. **FL server initializes an initial model**\n", + "2. **For each round (global iteration):**\n", + " * FL server broadcasts the global model to clients\n", + " * Each FL client starts with this global model and perform the local training on their own data\n", + " * Each FL client, then sends back their newly trained model to the FL server\n", + " * FL server aggregates all the local models and produces a new global model\n", + "\n", + "On the client side, the training workflow is as follows:\n", + "\n", + "1. Receive the model from the FL server\n", + "2. Perform local training on the received global model and/or evaluate the received global model for model selection\n", + "3. Send the new model back to the FL server" + ] + }, + { + "cell_type": "markdown", + "id": "50e2b7dd", + "metadata": {}, + "source": [ + "To convert a centralized training code to federated learning, we need to\n", + "adapt the code to do the following steps:\n", + "\n", + "\n", + "1. Obtain the required information from the received `FLModel`\n", + "2. Run local training\n", + "3. Put the results in a new `FLModel` to be sent back\n", + "\n", + "For a general use case, there are three essential methods for the Client API:\n", + "\n", + "* ``init()``: Initializes NVFlare Client API environment.\n", + "* ``receive()``: Receives model from NVFlare side.\n", + "* ``send()``: Sends the model to NVFlare side.\n", + "\n", + "Where `FLModel` is a data structure like this:\n", + "\n", + "```python\n", + "\n", + "class FLModel:\n", + " def __init__(\n", + " self,\n", + " params_type: Union[None, str, ParamsType] = None,\n", + " params: Any = None,\n", + " optimizer_params: Any = None,\n", + " metrics: Optional[Dict] = None,\n", + " start_round: Optional[int] = 0,\n", + " current_round: Optional[int] = None,\n", + " total_rounds: Optional[int] = None,\n", + " meta: Optional[Dict] = None,\n", + " ):\n", + " \"\"\"FLModel is a standardize data structure for NVFlare to communicate with external systems.\n", + "\n", + " Args:\n", + " params_type: type of the parameters. It only describes the \"params\".\n", + " If params_type is None, params need to be None.\n", + " If params is provided but params_type is not provided, then it will be treated as FULL.\n", + " params: model parameters, for example: model weights for deep learning.\n", + " optimizer_params: optimizer parameters.\n", + " For many cases, the optimizer parameters don't need to be transferred during FL training.\n", + " metrics: evaluation metrics such as loss and scores.\n", + " current_round: the current FL rounds. A round means round trip between client/server during training.\n", + " None for inference.\n", + " total_rounds: total number of FL rounds. A round means round trip between client/server during training.\n", + " None for inference.\n", + " meta: metadata dictionary used to contain any key-value pairs to facilitate the process.\n", + " \"\"\"\n", + "```\n", + "\n", + "You can use the Client API to change centralized training code to\n", + "federated learning, for example:\n", + "\n", + "```python\n", + "\n", + "import nvflare.client as flare\n", + "\n", + "flare.init() # 1. Initializes NVFlare Client API environment.\n", + "input_model = flare.receive() # 2. Receives model from NVFlare side.\n", + "params = input_model.params # 3. Obtain the required information from received FLModel\n", + "\n", + "# original local training code begins\n", + "\n", + "new_params = trainer.fit(params)\n", + "\n", + "# original local training code ends\n", + "\n", + "output_model = flare.FLModel(params=new_params) # 4. Put the results in a new FLModel\n", + "flare.send(output_model) # 5. Sends the model to NVFlare side.\n", + "\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "494e4079", + "metadata": {}, + "source": [ + "With 5 lines of code changes, we convert the centralized training code to work in a\n", + "federated learning setting.\n", + "\n", + "After this, we can use the job templates and the Job CLI\n", + "to generate a job and export it to run on a deployed NVFlare system or directly run the job using FL Simulator.\n", + "\n", + "To see a table of the key Client APIs, see the [Client API documentation in the programming guide](https://nvflare.readthedocs.io/en/main/programming_guide/execution_api_type/client_api.html#id2).\n", + "\n", + "Please consult the [Client API Module](https://nvflare.readthedocs.io/en/main/apidocs/nvflare.client.api.html) for more in-depth information about all of the Client API functions.\n", + "\n", + "If you are using PyTorch Lightning in your training code, you can check the [Lightning API Module](https://nvflare.readthedocs.io/en/main/apidocs/nvflare.app_opt.lightning.api.html). Also, be sure to look through the [Convert Torch Lightning to FL notebook](../02.2_client_api/convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb) and related code." + ] + }, + { + "cell_type": "markdown", + "id": "4a09d80e", + "metadata": {}, + "source": [ + "## Client API with Different Implementations\n", + "\n", + "Within the Client API, we offer multiple implementations tailored to diverse requirements:\n", + "\n", + "* In-process Client API: efficient for single GPU training\n", + "* Sub-process Client API: flexible for multi-GPU or distributed PyTorch training\n", + "\n", + "\n", + "\n", + "### In-process Client API\n", + "\n", + "In this setup, the client training script operates within the same process as the NVFlare Client job. This configuration, utilizing the ```InProcessClientAPIExecutor```, offers shared memory usage and is efficient with simple configuration. \n", + "This is the default for `ScriptRunner` since by default `launch_external_process=False`. Use this configuration for development or single GPU training.\n", + "\n", + "### Sub-process Client API: \n", + "\n", + "Here, the client training script runs in a separate subprocess.\n", + "\n", + "Utilizing the ```ClientAPILauncherExecutor```, this option offers flexibility in communication mechanisms:\n", + " * Communication via CellPipe (default)\n", + " * Communication via FilePipe (no capability to stream metrics for experiment tracking) \n", + "\n", + "This configuration is ideal for scenarios requiring multi-GPU or distributed PyTorch training.\n", + "\n", + "Choose the option best suited to your specific requirements and workflow preferences.\n", + "\n", + "\n", + "## Client API communication patterns\n", + "\n", + "We have two different implementations of the Client API tailored to different scenarios, each linked with distinct communication patterns.\n", + "\n", + "Broadly, we present in-process and sub-process executors. The in-process executor entails both training scripts and client executor operating within the same process.\n", + "\n", + "\n", + "On the other hand, the LauncherExecutor employs a sub-process to execute training scripts, leading to the client executor and training scripts residing in separate processes. Communication between them is facilitated by either CellPipe (default) or FilePipe.\n", + "\n", + "\"Client\n", + "\n", + "\n", + "\n", + "### Choice of different Pipes\n", + "We suggest using the default setting with CellPipe for most users.\n", + "\n", + "CellPipe facilitates TCP-based cell-to-cell connections between the Executor and training script processes on the local host. The term cell represents logical endpoints. This communication enables the exchange of models, metrics, and metadata between the two processes.\n", + "\n", + "In contrast, FilePipe offers file-based communication between the Executor and training script processes, utilizing a job-specific file directory for exchanging models and metadata via files. While FilePipe is easier to set up than CellPipe, it’s not suitable for high-frequency metrics exchange. On the other hand, FilePipe might be a better choice for scenarios where the training script is running on a remote machine and the client executor is running on the local machine.\n", + "\n", + "\n", + "## Client API Examples\n", + "\n", + "All implementations can be easily configured using the JobAPI's `ScriptRunner`. By default, the in-process is used, however setting `launch_external_process=True` uses the sub-process with pre-configured CellPipes for communication and metrics streaming.\n", + "\n", + "To find out more about the Client API, and its pipe configurations, please refer to the [Client API](https://nvflare.readthedocs.io/en/2.4/programming_guide/execution_api_type/client_api.html). In this tutorial, we will only use the in-process for simplicity. You can follow the examples in [ML-To-FL](https://github.com/NVIDIA/NVFlare/tree/main/examples/hello-world/ml-to-fl) for sub-processtraining. with different pipe training. \n", + "\n", + "In the following sections, we will see how to use the Client API with PyTorch Lightning and machine learning algorithms.\n", + "\n", + "\n", + "* [Convert PyTorch lightning to federated learning](../02.3_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb)\n", + "\n", + "* [Convert logistic regression to federated learning](../02.4_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb)\n", + "\n", + "* [Convert Kmeans to federated learning](../02.4_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb)\n", + "\n", + "* [Convert survival analysis to federated learning](../02.4_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/convert_survival_analysis_to_fl.ipynb)\n" + ] + }, + { + "cell_type": "markdown", + "id": "fffcd761", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nvflare_example", + "language": "python", + "name": "nvflare_example" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_client_api/client_api_communication_pattern.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_client_api/client_api_communication_pattern.png new file mode 100644 index 0000000000..4a50383778 Binary files /dev/null and b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_client_api/client_api_communication_pattern.png differ diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/log_config.json b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/log_config.json deleted file mode 100644 index e5732b4950..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/log_config.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "version": 1, - "disable_existing_loggers": false, - "formatters": { - "baseFormatter": { - "()": "nvflare.fuel.utils.log_utils.BaseFormatter", - "fmt": "%(asctime)s - %(name)s - %(levelname)s - %(fl_ctx)s - %(message)s" - }, - "colorFormatter": { - "()": "nvflare.fuel.utils.log_utils.ColorFormatter", - "fmt": "%(asctime)s - %(levelname)s - %(message)s", - "datefmt": "%Y-%m-%d %H:%M:%S" - }, - "jsonFormatter": { - "()": "nvflare.fuel.utils.log_utils.JsonFormatter", - "fmt": "%(asctime)s - %(identity)s - %(name)s - %(fullName)s - %(levelname)s - %(fl_ctx)s - %(message)s" - } - }, - "filters": { - "FLFilter": { - "()": "nvflare.fuel.utils.log_utils.LoggerNameFilter", - "logger_names": ["custom", "nvflare.app_common", "nvflare.app_opt"] - } - }, - "handlers": { - "consoleHandler": { - "class": "logging.StreamHandler", - "level": "INFO", - "formatter": "colorFormatter", - "filters": ["FLFilter"], - "stream": "ext://sys.stdout" - }, - "logFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "DEBUG", - "formatter": "baseFormatter", - "filename": "log.txt", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10 - }, - "errorFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "ERROR", - "formatter": "baseFormatter", - "filename": "log_error.txt", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10 - }, - "jsonFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "DEBUG", - "formatter": "jsonFormatter", - "filename": "log.json", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10 - }, - "FLFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "DEBUG", - "formatter": "baseFormatter", - "filters": ["FLFilter"], - "filename": "log_fl.txt", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10, - "delay": true - } - }, - "loggers": { - "root": { - "level": "INFO", - "handlers": ["consoleHandler", "logFileHandler", "errorFileHandler", "jsonFileHandler", "FLFileHandler"] - } - } -} - - - - - - - - - diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/log_config.json b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/log_config.json deleted file mode 100644 index e5732b4950..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/log_config.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "version": 1, - "disable_existing_loggers": false, - "formatters": { - "baseFormatter": { - "()": "nvflare.fuel.utils.log_utils.BaseFormatter", - "fmt": "%(asctime)s - %(name)s - %(levelname)s - %(fl_ctx)s - %(message)s" - }, - "colorFormatter": { - "()": "nvflare.fuel.utils.log_utils.ColorFormatter", - "fmt": "%(asctime)s - %(levelname)s - %(message)s", - "datefmt": "%Y-%m-%d %H:%M:%S" - }, - "jsonFormatter": { - "()": "nvflare.fuel.utils.log_utils.JsonFormatter", - "fmt": "%(asctime)s - %(identity)s - %(name)s - %(fullName)s - %(levelname)s - %(fl_ctx)s - %(message)s" - } - }, - "filters": { - "FLFilter": { - "()": "nvflare.fuel.utils.log_utils.LoggerNameFilter", - "logger_names": ["custom", "nvflare.app_common", "nvflare.app_opt"] - } - }, - "handlers": { - "consoleHandler": { - "class": "logging.StreamHandler", - "level": "INFO", - "formatter": "colorFormatter", - "filters": ["FLFilter"], - "stream": "ext://sys.stdout" - }, - "logFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "DEBUG", - "formatter": "baseFormatter", - "filename": "log.txt", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10 - }, - "errorFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "ERROR", - "formatter": "baseFormatter", - "filename": "log_error.txt", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10 - }, - "jsonFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "DEBUG", - "formatter": "jsonFormatter", - "filename": "log.json", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10 - }, - "FLFileHandler": { - "class": "logging.handlers.RotatingFileHandler", - "level": "DEBUG", - "formatter": "baseFormatter", - "filters": ["FLFilter"], - "filename": "log_fl.txt", - "mode": "a", - "maxBytes": 20971520, - "backupCount": 10, - "delay": true - } - }, - "loggers": { - "root": { - "level": "INFO", - "handlers": ["consoleHandler", "logFileHandler", "errorFileHandler", "jsonFileHandler", "FLFileHandler"] - } - } -} - - - - - - - - - diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/lr_fl_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/lr_fl_job.py deleted file mode 100644 index 4cc8c55e48..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/lr_fl_job.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from src.newton_raphson_persistor import NewtonRaphsonModelPersistor -from src.newton_raphson_workflow import FedAvgNewtonRaphson - -from nvflare.app_opt.pt.job_config.base_fed_job import BaseFedJob -from nvflare.client.config import ExchangeFormat -from nvflare.job_config.script_runner import ScriptRunner - -if __name__ == "__main__": - n_clients = 4 - num_rounds = 5 - - job = BaseFedJob( - name="logistic_regression_fedavg", - model_persistor=NewtonRaphsonModelPersistor(n_features=13), - ) - - controller = FedAvgNewtonRaphson( - num_clients=n_clients, - num_rounds=num_rounds, - damping_factor=0.8, - persistor_id="newton_raphson_persistor", - ) - job.to(controller, "server") - - # Add clients - for i in range(n_clients): - runner = ScriptRunner( - script="src/newton_raphson_train.py", - script_args="--data_root /tmp/flare/dataset/heart_disease_data", - launch_external_process=True, - params_exchange_format=ExchangeFormat.RAW, - ) - job.to(runner, f"site-{i + 1}") - - job.export_job("/tmp/nvflare/jobs/job_config") - job.simulator_run("/tmp/nvflare/jobs/workdir", gpu="0", log_config="./log_config.json") diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_train.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_train.py deleted file mode 100644 index 419b9ed70b..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_train.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import os - -import numpy as np -from sklearn.metrics import accuracy_score, precision_score - -import nvflare.client as flare -from nvflare.apis.fl_constant import FLMetaKey -from nvflare.app_common.abstract.fl_model import FLModel, ParamsType -from nvflare.app_common.np.constants import NPConstants -from nvflare.client.tracking import SummaryWriter - - -def parse_arguments(): - """ - Parse command line args for client side training. - """ - parser = argparse.ArgumentParser(description="Federated Second-Order Newton Raphson") - - parser.add_argument("--data_root", type=str, help="Path to load client side data.") - - return parser.parse_args() - - -def load_data(data_root, site_name): - """ - Load the data for each client. - - Args: - data_root: root directory storing client site data. - site_name: client site name - Returns: - A dict with client site training and validation data. - """ - print("loading data for client {} from: {}".format(site_name, data_root)) - train_x_path = os.path.join(data_root, "{}.train.x.npy".format(site_name)) - train_y_path = os.path.join(data_root, "{}.train.y.npy".format(site_name)) - test_x_path = os.path.join(data_root, "{}.test.x.npy".format(site_name)) - test_y_path = os.path.join(data_root, "{}.test.y.npy".format(site_name)) - - train_X = np.load(train_x_path) - train_y = np.load(train_y_path) - valid_X = np.load(test_x_path) - valid_y = np.load(test_y_path) - - return {"train_X": train_X, "train_y": train_y, "valid_X": valid_X, "valid_y": valid_y} - - -def sigmoid(inp): - return 1.0 / (1.0 + np.exp(-inp)) - - -def train_newton_raphson(data, theta): - """ - Compute gradient and hessian on local data - based on paramters received from server. - - """ - train_X = data["train_X"] - train_y = data["train_y"] - - # Add intercept, pre-pend 1s to as first - # column of train_X - train_X = np.concatenate((np.ones((train_X.shape[0], 1)), train_X), axis=1) - - # Compute probabilities from current weights - proba = sigmoid(np.dot(train_X, theta)) - - # The gradient is X^T . (y - proba) - gradient = np.dot(train_X.T, (train_y - proba)) - - # The hessian is X^T . D . X, where D is the - # diagnoal matrix with values proba * (1 - proba) - D = np.diag((proba * (1 - proba))[:, 0]) - hessian = train_X.T.dot(D).dot(train_X) - - return {"gradient": gradient, "hessian": hessian} - - -def validate(data, theta): - """ - Performs local validation. - Computes accuracy and precision scores. - - """ - valid_X = data["valid_X"] - valid_y = data["valid_y"] - - # Add intercept, pre-pend 1s to as first - # column of valid_X - valid_X = np.concatenate((np.ones((valid_X.shape[0], 1)), valid_X), axis=1) - - # Compute probabilities from current weights - proba = sigmoid(np.dot(valid_X, theta)) - - return {"accuracy": accuracy_score(valid_y, proba.round()), "precision": precision_score(valid_y, proba.round())} - - -def main(): - """ - This is a typical ML training loop, - augmented with Flare Client API to - perform local training on each client - side and send result to server. - - """ - args = parse_arguments() - - flare.init() - - site_name = flare.get_site_name() - print("training on client site: {}".format(site_name)) - - # Load client site data. - data = load_data(args.data_root, site_name) - - # Get metric summary writer - writer = SummaryWriter() - - while flare.is_running(): - - # Receive global model (FLModel) from server. - global_model = flare.receive() - - curr_round = global_model.current_round - print("current_round={}".format(curr_round)) - - print( - ("[ROUND {}] - client site: {}, received " "global model: {}").format(curr_round, site_name, global_model) - ) - - # Get the weights, aka parameter theta for - # logistic regression. - global_weights = global_model.params[NPConstants.NUMPY_KEY] - print("[ROUND {}] - global model weights: {}".format(curr_round, global_weights)) - - # Local validation before training - print(("[ROUND {}] - start validation of global " "model on client: {}").format(curr_round, site_name)) - validation_scores = validate(data, global_weights) - print( - ("[ROUND {}] - validation metric scores on " "client: {} = {}").format( - curr_round, site_name, validation_scores - ) - ) - - # Write validation metric summary - writer.add_scalar("{}/accuracy".format(site_name), validation_scores["accuracy"], curr_round) - - writer.add_scalar("{}/precision".format(site_name), validation_scores["precision"], curr_round) - - # Local training - print(("[ROUND {}] - start local training on client " "site: {}").format(curr_round, site_name)) - result_dict = train_newton_raphson(data, theta=global_weights) - - # Send result to server for aggregation. - result_model = FLModel(params=result_dict, params_type=ParamsType.FULL) - result_model.meta[FLMetaKey.NUM_STEPS_CURRENT_ROUND] = data["train_X"].shape[0] - - print( - ( - "[ROUND {}] - local newton raphson training from " "client: {} complete, sending results to server: {}" - ).format(curr_round, site_name, result_model) - ) - - flare.send(result_model) - - -if __name__ == "__main__": - main() diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_workflow.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_workflow.py deleted file mode 100644 index a4094cb7f6..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_workflow.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from typing import List - -import numpy as np - -from nvflare.apis.fl_constant import FLMetaKey -from nvflare.app_common.abstract.fl_model import FLModel -from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper -from nvflare.app_common.app_constant import AppConstants -from nvflare.app_common.np.constants import NPConstants -from nvflare.app_common.workflows.base_fedavg import BaseFedAvg - - -class FedAvgNewtonRaphson(BaseFedAvg): - def __init__(self, damping_factor, epsilon=1.0, *args, **kwargs): - super().__init__(*args, **kwargs) - """ - Init function for FedAvgNewtonRaphson. - - Args: - damping_factor: damping factor for Newton Raphson updates. - epsilon: a regularization factor to avoid empty hessian for - matrix inversion - """ - self.damping_factor = damping_factor - self.epsilon = epsilon - self.aggregator = WeightedAggregationHelper() - - def run(self) -> None: - """ - The run function executes the logic of federated - second order Newton Raphson optimization. - - """ - self.info("starting Federated Averaging Netwon Raphson ...") - - # First load the model and set up some training params. - # A `persisitor` (NewtonRaphsonModelPersistor) will load - # the model in `ModelLearnable` format, then will be - # converted `FLModel` by `ModelController`. - # - model = self.load_model() - - model.start_round = self.start_round - model.total_rounds = self.num_rounds - - self.info("Server side model loader: {}".format(model)) - - for self.current_round in range(self.start_round, self.start_round + self.num_rounds): - self.info(f"Round {self.current_round} started.") - - # Get the list of clients. - clients = self.sample_clients(self.num_clients) - - model.current_round = self.current_round - - # Send training task and current global model to clients. - # - # A `task` isntance will be created, and sent - # to clients, the model is first converted to a shareable - # and is attached to the task. - # - # After the task is finished, the result (shareable) recieved - # from the task is converted to FLModel, and is returned to the - # server. The `results` below is a list with result (FLModel) - # from all clients. - # - # The full logic of `task` is implemented in: - # https://github.com/NVIDIA/NVFlare/blob/d6827bca96d332adb3402ceceb4b67e876146067/nvflare/app_common/workflows/model_controller.py#L178 - # - self.info("sending server side global model to clients") - results = self.send_model_and_wait(targets=clients, data=model) - - # Aggregate results receieved from clients. - aggregate_results = self.aggregate(results, aggregate_fn=self.newton_raphson_aggregator_fn) - - # Update global model based on the following formula: - # weights = weights + updates, where - # updates = -damping_factor * Hessian^{-1} . Gradient - self.update_model(model, aggregate_results) - - # Save global model. - self.save_model(model) - - self.info("Finished FedAvg.") - - def newton_raphson_aggregator_fn(self, results: List[FLModel]): - """ - Custom aggregator function for second order Newton Raphson - optimization. - - This uses the default thread-safe WeightedAggregationHelper, - which implement a weighted average of all values received from - a `result` dictionary. - - Args: - results: a list of `FLModel`s. Each `FLModel` is received - from a client. The field `params` is a dictionary that - contains values to be aggregated: the gradient and hessian. - """ - self.info("receieved results from clients: {}".format(results)) - - # On client side the `NUM_STEPS_CURRENT_ROUND` key - # is used to track the number of samples for each client. - for curr_result in results: - self.aggregator.add( - data=curr_result.params, - weight=curr_result.meta.get(FLMetaKey.NUM_STEPS_CURRENT_ROUND, 1.0), - contributor_name=curr_result.meta.get("client_name", AppConstants.CLIENT_UNKNOWN), - contribution_round=curr_result.current_round, - ) - - aggregated_dict = self.aggregator.get_result() - self.info("aggregated result: {}".format(aggregated_dict)) - - # Compute global model update: - # update = - damping_factor * Hessian^{-1} . Gradient - # A regularization is added to avoid empty hessian. - # - reg = self.epsilon * np.eye(aggregated_dict["hessian"].shape[0]) - newton_raphson_updates = self.damping_factor * np.linalg.solve( - aggregated_dict["hessian"] + reg, aggregated_dict["gradient"] - ) - self.info("newton raphson updates: {}".format(newton_raphson_updates)) - - # Convert the aggregated result to `FLModel`, this `FLModel` - # will then be used by `update_model` method from the base class, - # to update the global model weights. - # - aggr_result = FLModel( - params={"newton_raphson_updates": newton_raphson_updates}, - params_type=results[0].params_type, - meta={ - "nr_aggregated": len(results), - AppConstants.CURRENT_ROUND: results[0].current_round, - AppConstants.NUM_ROUNDS: self.num_rounds, - }, - ) - return aggr_result - - def update_model(self, model, model_update, replace_meta=True) -> FLModel: - """ - Update logistic regression parameters based on - aggregated gradient and hessian. - - """ - if replace_meta: - model.meta = model_update.meta - else: - model.meta.update(model_update.meta) - - model.metrics = model_update.metrics - model.params[NPConstants.NUMPY_KEY] += model_update.params["newton_raphson_updates"] diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/src/newton_raphson_persistor.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/src/newton_raphson_persistor.py deleted file mode 100644 index 5b324dd50c..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/src/newton_raphson_persistor.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import numpy as np - -from nvflare.app_common.np.np_model_persistor import NPModelPersistor - - -class NewtonRaphsonModelPersistor(NPModelPersistor): - """ - This class defines the persistor for Newton Raphson model. - - A persistor controls the logic behind initializing, loading - and saving of the model / parameters for each round of a - federated learning process. - - In the 2nd order Newton Raphson case, a model is just a - 1-D numpy vector containing the parameters for logistic - regression. The length of the parameter vector is defined - by the number of features in the dataset. - - """ - - def __init__(self, model_dir="models", model_name="weights.npy", n_features=13): - """ - Init function for NewtonRaphsonModelPersistor. - - Args: - model_dir: sub-folder name to save and load the global model - between rounds. - model_name: name to save and load the global model. - n_features: number of features for the logistic regression. - For the UCI ML heart Disease dataset, this is 13. - - """ - - super().__init__() - - self.model_dir = model_dir - self.model_name = model_name - self.n_features = n_features - - # A default model is loaded when no local model is available. - # This happen when training starts. - # - # A `model` for a binary logistic regression is just a matrix, - # with shape (n_features + 1, 1). - # For the UCI ML Heart Disease dataset, the n_features = 13. - # - # A default matrix with value 0s is created. - # - self.default_data = np.zeros((self.n_features + 1, 1), dtype=np.float32) diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb deleted file mode 100644 index 5bf4d3f4f1..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb +++ /dev/null @@ -1,341 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "e8c19632", - "metadata": {}, - "source": [ - "# Converting Logistic Regression to FL" - ] - }, - { - "cell_type": "markdown", - "id": "7f9d96ed", - "metadata": {}, - "source": [ - "## Federated Logistic Regression with Second-Order Newton-Raphson optimization\n", - "This example shows how to implement a federated binary classification via logistic regression with second-order Newton-Raphson optimization.\n", - "\n", - "The [UCI Heart Disease dataset](https://archive.ics.uci.edu/dataset/45/heart+disease) is\n", - "used in this example. Scripts are provided to download and process the\n", - "dataset as described\n", - "[here](https://github.com/owkin/FLamby/tree/main/flamby/datasets/fed_heart_disease).\n", - "\n", - "This dataset contains samples from 4 sites, splitted into training and\n", - "testing sets as described below:\n", - "|site | sample split |\n", - "|-------------|---------------------------------------|\n", - "|Cleveland | train: 199 samples, test: 104 samples |\n", - "|Hungary | train: 172 samples, test: 89 samples |\n", - "|Switzerland | train: 30 samples, test: 16 samples |\n", - "|Long Beach V | train: 85 samples, test: 45 samples |\n", - "\n", - "The number of features in each sample is 13." - ] - }, - { - "cell_type": "markdown", - "id": "e54f0dcc", - "metadata": {}, - "source": [ - "## Introduction\n", - "\n", - "The [Newton-Raphson\n", - "optimization](https://en.wikipedia.org/wiki/Newton%27s_method) problem\n", - "can be described as follows.\n", - "\n", - "In a binary classification task with logistic regression, the\n", - "probability of a data sample $x$ classified as positive is formulated\n", - "as:\n", - "$$p(x) = \\sigma(\\beta \\cdot x + \\beta_{0})$$\n", - "where $\\sigma(.)$ denotes the sigmoid function. We can incorporate\n", - "$\\beta_{0}$ and $\\beta$ into a single parameter vector $\\theta =\n", - "( \\beta_{0}, \\beta)$. Let $d$ be the number\n", - "of features for each data sample $x$ and let $N$ be the number of data\n", - "samples. We then have the matrix version of the above probability\n", - "equation:\n", - "$$p(X) = \\sigma( X \\theta )$$\n", - "Here $X$ is the matrix of all samples, with shape $N \\times (d+1)$,\n", - "having it's first column filled with value 1 to account for the\n", - "intercept $\\theta_{0}$.\n", - "\n", - "The goal is to compute parameter vector $\\theta$ that maximizes the\n", - "below likelihood function:\n", - "$$L_{\\theta} = \\prod_{i=1}^{N} p(x_i)^{y_i} (1 - p(x_i)^{1-y_i})$$\n", - "\n", - "The Newton-Raphson method optimizes the likelihood function via\n", - "quadratic approximation. Omitting the maths, the theoretical update\n", - "formula for parameter vector $\\theta$ is:\n", - "$$\\theta^{n+1} = \\theta^{n} - H_{\\theta^{n}}^{-1} \\nabla L_{\\theta^{n}}$$\n", - "where\n", - "$$\\nabla L_{\\theta^{n}} = X^{T}(y - p(X))$$\n", - "is the gradient of the likelihood function, with $y$ being the vector\n", - "of ground truth for sample data matrix $X$, and\n", - "$$H_{\\theta^{n}} = -X^{T} D X$$\n", - "is the Hessian of the likelihood function, with $D$ a diagonal matrix\n", - "where diagonal value at $(i,i)$ is $D(i,i) = p(x_i) (1 - p(x_i))$.\n", - "\n", - "In federated Newton-Raphson optimization, each client will compute its\n", - "own gradient $\\nabla L_{\\theta^{n}}$ and Hessian $H_{\\theta^{n}}$\n", - "based on local training samples. A server will aggregate the gradients\n", - "and Hessians computed from all clients, and perform the update of\n", - "parameter $\\theta$ based on the theoretical update formula described\n", - "above." - ] - }, - { - "cell_type": "markdown", - "id": "32003ba9", - "metadata": {}, - "source": [ - "## Implementation\n", - "\n", - "Using `nvflare`, The federated logistic regression with Newton-Raphson\n", - "optimization is implemented as follows.\n", - "\n", - "On the server side, all workflow logics are implemented in\n", - "class `FedAvgNewtonRaphson`, which can be found\n", - "[here](code/newton_raphson/app/custom/newton_raphson_workflow.py). The\n", - "`FedAvgNewtonRaphson` class inherits from the\n", - "[`BaseFedAvg`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/base_fedavg.py)\n", - "class, which itself inherits from the **ModelController**\n", - "([`ModelController`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/model_controller.py))\n", - "class. This is the preferrable approach to implement a custom\n", - "workflow, since `ModelController` decouples communication logic from\n", - "actual workflow (training & validation) logic. The mandatory\n", - "method to override in `ModelController` is the\n", - "[`run()`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/model_controller.py#L37)\n", - "method, where the orchestration of server-side workflow actually\n", - "happens. The implementation of `run()` method in\n", - "[`FedAvgNewtonRaphson`](code/newton_raphson/app/custom/newton_raphson_workflow.py)\n", - "is similar to the classic\n", - "[`FedAvg`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/fedavg.py#L44):\n", - "- Initialize the global model, this is acheived through method `load_model()`\n", - " from base class\n", - " [`ModelController`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/model_controller.py#L292),\n", - " which relies on the\n", - " [`ModelPersistor`](https://nvflare.readthedocs.io/en/main/glossary.html#persistor). A\n", - " custom\n", - " [`NewtonRaphsonModelPersistor`](code/newton_raphson/app/custom/newton_raphson_persistor.py)\n", - " is implemented in this example, which is based on the\n", - " [`NPModelPersistor`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/np/np_model_persistor.py)\n", - " for numpy data, since the _model_ in the case of logistic regression\n", - " is just the parameter vector $\\theta$ that can be represented by a\n", - " numpy array. Only the `__init__` method needs to be re-implemented\n", - " to provide a proper initialization for the global parameter vector\n", - " $\\theta$.\n", - "- During each training round, the global model will be sent to the\n", - " list of participating clients to perform a training task. This is\n", - " done using the\n", - " [`send_model_and_wait()`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/model_controller.py#L41)\n", - " method. Once\n", - " the clients finish their local training, results will be collected\n", - " and sent back to server as\n", - " [`FLModel`](https://nvflare.readthedocs.io/en/main/programming_guide/fl_model.html#flmodel)s.\n", - "- Results sent by clients contain their locally computed gradient and\n", - " Hessian. A [custom aggregation\n", - " function](code/newton_raphson/app/custom/newton_raphson_workflow.py)\n", - " is implemented to get the averaged gradient and Hessian, and compute\n", - " the Newton-Raphson update for the global parameter vector $\\theta$,\n", - " based on the theoretical formula shown above. The averaging of\n", - " gradient and Hessian is based on the\n", - " [`WeightedAggregationHelper`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/aggregators/weighted_aggregation_helper.py#L20),\n", - " which weighs the contribution from each client based on the number\n", - " of local training samples. The aggregated Newton-Raphson update is\n", - " returned as an `FLModel`.\n", - "- After getting the aggregated Newton-Raphson update, an\n", - " [`update_model()`](code/newton_raphson/app/custom/newton_raphson_workflow.py#L172)\n", - " method is implemented to actually apply the Newton-Raphson update to\n", - " the global model.\n", - "- The last step is to save the updated global model, again through\n", - " the `NewtonRaphsonModelPersistor` using `save_model()`.\n", - "\n", - "\n", - "On the client side, the local training logic is implemented\n", - "[here](code/newton_raphson/app/custom/newton_raphson_train.py). The\n", - "implementation is based on the [`Client\n", - "API`](https://nvflare.readthedocs.io/en/main/programming_guide/execution_api_type.html#client-api). This\n", - "allows user to add minimum `nvflare`-specific code to turn a typical\n", - "centralized training script into a federated client side local training\n", - "script.\n", - "- During local training, each client receives a copy of the global\n", - " model, sent by the server, using `flare.receive()` from the Client API.\n", - " The received global model is an instance of `FLModel`.\n", - "- A local validation is first performed, where validation metrics\n", - " (accuracy and precision) are streamed to server using the\n", - " [`SummaryWriter`](https://nvflare.readthedocs.io/en/main/apidocs/nvflare.client.tracking.html#nvflare.client.tracking.SummaryWriter). The\n", - " streamed metrics can be loaded and visualized using tensorboard.\n", - "- Then each client computes it's gradient and Hessian based on local\n", - " training data, using their respective theoretical formula described\n", - " above. This is implemented in the\n", - " [`train_newton_raphson()`](code/newton_raphson/app/custom/newton_raphson_train.py#L82)\n", - " method. Each client then sends the computed results (always in\n", - " `FLModel` format) to server for aggregation, using the Client API call\n", - " `flare.send()`.\n", - "\n", - "Each client site corresponds to a site listed in the data table above.\n", - "\n", - "A [centralized training script](code/train_centralized.py) is also\n", - "provided, which allows for comparing the federated Newton-Raphson\n", - "optimization versus the centralized version. In the centralized\n", - "version, training data samples from all 4 sites were concatenated into\n", - "a single matrix, used to optimize the model parameters. The\n", - "optimized model was then tested separately on testing data samples of\n", - "the 4 sites, using accuracy and precision as metrics.\n", - "\n", - "Comparing the federated [client-side training\n", - "code](code/newton_raphson/app/custom/newton_raphson_train.py) with the\n", - "centralized [training code](code/train_centralized.py), we can see that\n", - "the training logic remains similar: load data, perform training\n", - "(Newton-Raphson updates), and valid trained model. The only added\n", - "differences in the federated code are related to interaction with the\n", - "FL system, such as receiving and send `FLModel`." - ] - }, - { - "cell_type": "markdown", - "id": "c3fc55e0", - "metadata": {}, - "source": [ - "## Install requirements\n", - "First, install the required packages:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04911ca3", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -r code/requirements.txt" - ] - }, - { - "cell_type": "markdown", - "id": "33ea8504", - "metadata": {}, - "source": [ - "## Download and prepare data\n", - "\n", - "Execute the following script\n", - "```\n", - "bash ./code/data/prepare_heart_disease_data.sh\n", - "```\n", - "This will download the heart disease dataset under\n", - "`/tmp/flare/dataset/heart_disease_data/`\n", - "\n", - "Please note that you may need to accept the data terms in order to complete the download." - ] - }, - { - "cell_type": "markdown", - "id": "d548b466", - "metadata": {}, - "source": [ - "## Centralized Logistic Regression\n", - "\n", - "Launch the following script:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8c68fe1a", - "metadata": {}, - "outputs": [], - "source": [ - "! python3 code/train_centralized.py --solver custom" - ] - }, - { - "cell_type": "markdown", - "id": "fa666b79", - "metadata": {}, - "source": [ - "Two implementations of logistic regression are provided in the\n", - "centralized training script, which can be specified by the `--solver`\n", - "argument:\n", - "- One is using `sklearn.LogisticRegression` with the `newton-cholesky`\n", - " solver\n", - "- The other one is manually implemented using the theoretical update\n", - " formulas described above.\n", - "\n", - "Both implementations were tested to converge in 4 iterations and to\n", - "give the same result.\n", - "\n", - "Example output:\n", - "```\n", - "using solver: custom\n", - "loading training data.\n", - "training data X loaded. shape: (486, 13)\n", - "training data y loaded. shape: (486, 1)\n", - "\n", - "site - 1\n", - "validation set n_samples: 104\n", - "accuracy: 0.75\n", - "precision: 0.7115384615384616\n", - "\n", - "site - 2\n", - "validation set n_samples: 89\n", - "accuracy: 0.7528089887640449\n", - "precision: 0.6122448979591837\n", - "\n", - "site - 3\n", - "validation set n_samples: 16\n", - "accuracy: 0.75\n", - "precision: 1.0\n", - "\n", - "site - 4\n", - "validation set n_samples: 45\n", - "accuracy: 0.6\n", - "precision: 0.9047619047619048\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "0b72ef2b", - "metadata": {}, - "source": [ - "## Federated Logistic Regression\n", - "\n", - "Execute the following command to launch federated logistic\n", - "regression. This will run in `nvflare`'s simulator mode.\n", - "```\n", - "nvflare simulator -w ./workspace -n 4 -t 4 job/newton_raphson/\n", - "```\n", - "\n", - "Accuracy and precision for each site can be viewed in Tensorboard:\n", - "```\n", - "tensorboard --logdir=./workspace/server/simulate_job/tb_events\n", - "```\n", - "As can be seen from the figure below, per-site evaluation metrics in\n", - "federated logistic regression are on-par with the centralized version.\n", - "\n", - "\"Tensorboard\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/convert_ml_to_fl.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/convert_ml_to_fl.ipynb deleted file mode 100644 index efef1032b9..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/convert_ml_to_fl.ipynb +++ /dev/null @@ -1,34 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Simple ML/DL to FL transition with NVFlare\n", - "\n", - "Converting Deep Learning (DL) models to Federated Learning (FL) entails several key steps:\n", - "\n", - " - Formulating the algorithm: This involves determining how to adapt a DL model into an FL framework, including specifying the information exchange protocol between the server and clients.\n", - "\n", - " - Code conversion: Adapting existing standalone DL code into FL-compatible code. This typically involves minimal changes, often just a few lines of code, thanks to tools like NVFlare.\n", - "\n", - " - Workflow configuration: Once the code is modified, configuring the workflow to integrate the newly adapted FL code seamlessly.\n", - "\n", - "NVFlare simplifies the process of transitioning from traditional Machine Learning (ML) or DL algorithms to FL. With NVFlare, the conversion process requires only minor code adjustments.\n", - "\n", - "In this section, we have the following three examples for converting traditional ML to FL:\n", - "\n", - " * [Convert Logistics Regression to federated learning](02.3.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb)\n", - " * [Convert KMeans to federated learning](02.3.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb)\n", - " * [Convert Survival Analysis to federated learning](02.3.3_convert_survival_analysis_to_federated_learning/convert_survival_analysis_to_fl.ipynb)" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/lightning_fl_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/code/lightning_fl_job.py similarity index 93% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/lightning_fl_job.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/code/lightning_fl_job.py index b51724a962..aee42bb95b 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/lightning_fl_job.py +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/code/lightning_fl_job.py @@ -41,4 +41,4 @@ job.to(runner, f"site-{i + 1}") job.export_job("/tmp/nvflare/jobs/job_config") - job.simulator_run("/tmp/nvflare/jobs/workdir", gpu="0", log_config="./log_config.json") + job.simulator_run("/tmp/nvflare/jobs/workdir", gpu="0") diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/requirements.txt b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/code/requirements.txt similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/requirements.txt rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/code/requirements.txt diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/src/cifar10_lightning_fl.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/code/src/cifar10_lightning_fl.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/src/cifar10_lightning_fl.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/code/src/cifar10_lightning_fl.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/src/lit_net.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/code/src/lit_net.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/code/src/lit_net.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/code/src/lit_net.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/controller_worker_flow.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/controller_worker_flow.png new file mode 100644 index 0000000000..5deac70f5a Binary files /dev/null and b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/controller_worker_flow.png differ diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb similarity index 88% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb index 71f9a334c6..b09b44d940 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.2_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb @@ -7,7 +7,7 @@ "source": [ "# Converting PyTorch Lightning to FL\n", "\n", - "In this notebook, we use FedAvg and CIFAR10 PyTorch Lightning for client training code to create and run a federated learning job with NVFlare." + "In chapter 1, we have learned how to convert the PyTorch code to a federated learning job with NVFlare. In this section and next section, we will learn how to convert the PyTorch Lightning code to a federated learning job with NVFlare.\n" ] }, { @@ -16,12 +16,16 @@ "metadata": {}, "source": [ "## Basic Concepts\n", + "\n", "At the heart of NVFlare lies the concept of collaboration through\n", "\"tasks.\" An FL controller assigns tasks (e.g., training on local data) to one or more FL clients, processes returned\n", "results (e.g., model weight updates), and may assign additional\n", "tasks based on these results and other factors (e.g., a pre-configured\n", "number of training rounds). The clients run executors which can listen for tasks and perform the necessary computations locally, such as model training. This task-based interaction repeats\n", - "until the experiment’s objectives are met. " + "until the experiment's objectives are met.\n", + "\n", + "\n", + "\"Controller-Executor" ] }, { @@ -30,6 +34,7 @@ "metadata": {}, "source": [ "## Federated Averaging with NVFlare\n", + "\n", "Given the flexible controller and executor concepts, it is easy to implement different computing & communication patterns with NVFlare, such as [FedAvg](https://proceedings.mlr.press/v54/mcmahan17a?ref=https://githubhelp.com) and [cyclic weight transfer](https://academic.oup.com/jamia/article/25/8/945/4956468). \n", "\n", "The controller's `run()` routine is responsible for assigning tasks and processing task results from the Executors. " @@ -114,8 +119,9 @@ "metadata": {}, "source": [ "Using NVFlare's Client Lightning API, we can easily adapt machine learning code that was written for centralized training and apply it in a federated scenario.\n", + "\n", "For general use cases, we can use the Client Lightning API patch function:\n", - "- `flare.patch(trainer)`: Patch the lightning trainer. After flare.patch, functions such as `trainer.fit()` and `trainer.validate()` will get the global model internally and automatically send the result model to the FL server." + "- `flare.patch(trainer)`: Patch the lightning trainer with callbacks. After flare.patch, functions such as `trainer.fit()` and `trainer.validate()` will get the global model internally and automatically send the result model to the FL server." ] }, { @@ -163,7 +169,8 @@ "id": "5da34414-bac4-4352-8077-ab7ade998eec", "metadata": {}, "source": [ - "## Run an NVFlare Job\n", + "## Run an FedAvg PyTorch Lightning Training Job\n", + "\n", "Now that we have defined the FedAvg controller to run our federated compute workflow on the FL server, and our client training script to receive the global models, run local training, and send the results back to the FL server, we can put everything together using NVFlare's Job API." ] }, @@ -183,7 +190,7 @@ "metadata": {}, "outputs": [], "source": [ - "% pip install -r code/requirements.txt" + "! pip install -r code/requirements.txt" ] }, { @@ -196,11 +203,12 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 1, "id": "93889e62-b725-427c-8839-2771ca81d24c", "metadata": {}, + "outputs": [], "source": [ - "```python\n", "from typing import Any\n", "\n", "import torch\n", @@ -278,8 +286,7 @@ "\n", " def configure_optimizers(self):\n", " optimizer = optim.SGD(self.parameters(), lr=0.001, momentum=0.9)\n", - " return {\"optimizer\": optimizer}\n", - "```" + " return {\"optimizer\": optimizer}\n" ] }, { @@ -296,13 +303,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "aaa2b6f4", "metadata": {}, "outputs": [], "source": [ - "from code.src.lit_net import LitNet\n", - "\n", "from nvflare.app_common.workflows.fedavg import FedAvg\n", "from nvflare.app_opt.pt.job_config.base_fed_job import BaseFedJob\n", "from nvflare.job_config.script_runner import ScriptRunner\n", @@ -324,11 +329,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "6962e6cc-995e-4356-8156-3ceba2c7a249", "metadata": {}, "outputs": [], "source": [ + "from nvflare.app_common.workflows.fedavg import FedAvg\n", + "from nvflare.app_opt.pt.job_config.base_fed_job import BaseFedJob\n", + "from nvflare.job_config.script_runner import ScriptRunner\n", + "\n", "n_clients = 2\n", "\n", "controller = FedAvg(\n", @@ -354,21 +363,27 @@ "#### 4. Add clients\n", "Next, we can use the `ScriptRunner` and send it to each of the clients to run our training script.\n", "\n", - "Note that our script could have additional input arguments, such as batch size or data path, but we don't use them here for simplicity." + "Note that our script could have additional input arguments, such as batch size or data path, but we don't use them here for simplicity.\n", + "```python\n", + "\n", + "for i in range(n_clients):\n", + " runner = ScriptRunner(\n", + " script=\"src/cifar10_lightning_fl.py\", script_args=\"\" # f\"--batch_size 32 --data_path /tmp/data/site-{i}\"\n", + " )\n", + " job.to(runner, f\"site-{i+1}\")\n", + "\n", + "\n", + "```" ] }, { "cell_type": "code", "execution_count": null, - "id": "ad5d36fe-9ae5-43c3-80bc-2cdc66bf7a7e", + "id": "9d391acd", "metadata": {}, "outputs": [], "source": [ - "for i in range(n_clients):\n", - " runner = ScriptRunner(\n", - " script=\"src/cifar10_lightning_fl.py\", script_args=\"\" # f\"--batch_size 32 --data_path /tmp/data/site-{i}\"\n", - " )\n", - " job.to(runner, f\"site-{i+1}\")" + "!cat code/src/cifar10_lightning_fl.py" ] }, { @@ -379,17 +394,12 @@ "That's it!\n", "\n", "#### 5. Optionally export the job\n", - "Now, we could export the job and submit it to a real NVFlare deployment using the [Admin client](https://nvflare.readthedocs.io/en/main/real_world_fl/operation.html) or [FLARE API](https://nvflare.readthedocs.io/en/main/real_world_fl/flare_api.html). " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "99a270bf-c906-425b-b999-2306cb76eb62", - "metadata": {}, - "outputs": [], - "source": [ - "job.export_job(\"/tmp/nvflare/jobs/job_config\")" + "Now, we could export the job and submit it to a real NVFlare deployment using the [Admin client](https://nvflare.readthedocs.io/en/main/real_world_fl/operation.html) or [FLARE API](https://nvflare.readthedocs.io/en/main/real_world_fl/flare_api.html). \n", + "\n", + "```python\n", + "\n", + "job.export_job(\"/tmp/nvflare/jobs/job_config\")\n", + "```" ] }, { @@ -410,7 +420,10 @@ }, "outputs": [], "source": [ - "job.simulator_run(\"/tmp/nvflare/jobs/workdir\", gpu=\"0\")" + "%cd code/\n", + "! python3 lightning_fl_job.py\n", + "%cd -\n", + "\n" ] }, { @@ -418,15 +431,23 @@ "id": "fb2e1266", "metadata": {}, "source": [ - "You can see the full code for this job in [lightning_fl_job](code/lightning_fl_job.py)." + "You can see the full code for this job in [lightning_fl_job](code/lightning_fl_job.py).\n", + "\n", + "Now that you see how easy it is to convert a PyTorch Lightning training job to a federated learning job with NVFlare, we will show you how to [convert machine learning training jobs to federated learning](../02.4_convert_machine_learning_to_federated_learning/convert_ml_to_fl.ipynb) in the next section." ] + }, + { + "cell_type": "markdown", + "id": "9370e34a", + "metadata": {}, + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "nvflare_env", "language": "python", - "name": "python3" + "name": "nvflare_env" }, "language_info": { "codemirror_mode": { diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/client_api.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/client_api.ipynb deleted file mode 100644 index 3a0c46da26..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/client_api.ipynb +++ /dev/null @@ -1,346 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "58149c32", - "metadata": {}, - "source": [ - "# Transform Existing Code to FL Easily with the FLARE Client API" - ] - }, - { - "cell_type": "markdown", - "id": "06203527", - "metadata": {}, - "source": [ - "The FLARE Client API provides an easy way to convert centralized, local training code into federated learning code with just a few lines of code changes.\n", - "\n", - "Most of the previous examples up this point have already been using the Client API, but in this section we focus on the core concepts of the Client API and explain some of the ways it can be configured to help you use the Client API more effectively.\n", - "\n", - "You can see the detailed examples with actual integration with deep learing platforms including PyTorch and TensorFlow here: https://github.com/NVIDIA/NVFlare/tree/main/examples/hello-world/ml-to-fl" - ] - }, - { - "cell_type": "markdown", - "id": "be7efa36", - "metadata": {}, - "source": [ - "## Core Concept" - ] - }, - { - "cell_type": "markdown", - "id": "76102eac", - "metadata": {}, - "source": [ - "The general structure of the popular federated learning (FL) workflow, \"FedAvg\" is as follows:\n", - "\n", - "1. **FL server initializes an initial model**\n", - "2. **For each round (global iteration):**\n", - " 1. FL server sends the global model to clients\n", - " 2. Each FL client starts with this global model and trains on their own data\n", - " 3. Each FL client sends back their trained model\n", - " 4. FL server aggregates all the models and produces a new global model\n", - "\n", - "On the client side, the training workflow is as follows:\n", - "\n", - "1. Receive the model from the FL server\n", - "2. Perform local training on the received global model and/or evaluate the received global model for model selection\n", - "3. Send the new model back to the FL server" - ] - }, - { - "cell_type": "markdown", - "id": "50e2b7dd", - "metadata": {}, - "source": [ - "To convert a centralized training code to federated learning, we need to\n", - "adapt the code to do the following steps:\n", - "\n", - "1. Obtain the required information from the received `fl_model`\n", - "2. Run local training\n", - "3. Put the results in a new `fl_model` to be sent back\n", - "\n", - "For a general use case, there are three essential methods for the Client API:\n", - "\n", - "* ``init()``: Initializes NVFlare Client API environment.\n", - "* ``receive()``: Receives model from NVFlare side.\n", - "* ``send()``: Sends the model to NVFlare side.\n", - "\n", - "You can use the Client API to change centralized training code to\n", - "federated learning, for example:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9f21ee16", - "metadata": {}, - "outputs": [], - "source": [ - "import nvflare.client as flare\n", - "\n", - "flare.init() # 1. Initializes NVFlare Client API environment.\n", - "input_model = flare.receive() # 2. Receives model from NVFlare side.\n", - "params = input_model.params # 3. Obtain the required information from received FLModel\n", - "\n", - "# original local training code begins\n", - "new_params = local_train(params)\n", - "# original local training code ends\n", - "\n", - "output_model = flare.FLModel(params=new_params) # 4. Put the results in a new FLModel\n", - "flare.send(output_model) # 5. Sends the model to NVFlare side." - ] - }, - { - "cell_type": "markdown", - "id": "494e4079", - "metadata": {}, - "source": [ - "With 5 lines of code changes, we convert the centralized training code to work in a\n", - "federated learning setting.\n", - "\n", - "After this, we can use the job templates and the Job CLI\n", - "to generate a job and export it to run on a deployed NVFlare system or directly run the job using FL Simulator.\n", - "\n", - "To see a table of the key Client APIs, see the [Client API documentation in the programming guide](https://nvflare.readthedocs.io/en/main/programming_guide/execution_api_type/client_api.html#id2).\n", - "\n", - "Please consult the [Client API Module](https://nvflare.readthedocs.io/en/main/apidocs/nvflare.client.api.html) for more in-depth information about all of the Client API functions.\n", - "\n", - "If you are using PyTorch Lightning in your training code, you can check the [Lightning API Module](https://nvflare.readthedocs.io/en/main/apidocs/nvflare.app_opt.lightning.api.html). Also, be sure to look through the [Convert Torch Lightning to FL notebook](../02.2_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb) and related code." - ] - }, - { - "cell_type": "markdown", - "id": "4a09d80e", - "metadata": {}, - "source": [ - "## Advanced User Options: Client API with Different Implementations\n", - "\n", - "Within the Client API, we offer multiple implementations tailored to diverse requirements:\n", - "\n", - "* In-process Client API: In this setup, the client training script operates within the same process as the NVFlare Client job.\n", - "This configuration, utilizing the ```InProcessClientAPIExecutor```, offers shared memory usage and is efficient with simple configuration. \n", - "This is the default for `ScriptRunner` since by default `launch_external_process=False`. Use this configuration for development or single GPU training.\n", - "\n", - "* Sub-process Client API: Here, the client training script runs in a separate subprocess.\n", - "Utilizing the ```ClientAPILauncherExecutor```, this option offers flexibility in communication mechanisms:\n", - " * Communication via CellPipe (default)\n", - " * Communication via FilePipe (no capability to stream metrics for experiment tracking) \n", - "This configuration is ideal for scenarios requiring multi-GPU or distributed PyTorch training.\n", - "\n", - "Choose the option best suited to your specific requirements and workflow preferences.\n", - "\n", - "These implementations can be easily configured using the JobAPI's `ScriptRunner`.\n", - "By default, the ```InProcessClientAPIExecutor``` is used, however setting `launch_external_process=True` uses the ```ClientAPILauncherExecutor```\n", - "with pre-configured CellPipes for communication and metrics streaming." - ] - }, - { - "cell_type": "markdown", - "id": "b3ac92dd", - "metadata": {}, - "source": [ - "## NVFlare Client API Job with NumPy" - ] - }, - { - "cell_type": "markdown", - "id": "832a4b34", - "metadata": {}, - "source": [ - "In this example we use simple NumPy scripts to showcase the Client API with the `ScriptRunner` for both in-process and sub-process settings. With NumPy, only nvflare is needed so you do not have to install any additional dependencies.\n", - "\n", - "The default mode of the `ScriptRunner` uses `InProcessClientAPIExecutor` with the client training script operating within the same process as the NVFlare Client job. Below, we show a script that sends back full model parameters and then one that sends back model parameters differences before explaining metrics streaming and then showing how to launch those same scripts with the Sub-process Client API." - ] - }, - { - "cell_type": "markdown", - "id": "c1af7da6", - "metadata": {}, - "source": [ - "### Send model parameters back to the NVFlare server\n", - "\n", - "We use the mock training script in [train_full.py](code/src/train_full.py)\n", - "and send back the FLModel with `params_type=\"FULL\"`.\n", - "\n", - "After we modify our training script, we can create a job using the ScriptRunner: [np_client_api_job.py](code/np_client_api_job.py).\n", - "\n", - "The script will run the job using the simulator with the Job API by default:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7952cab8", - "metadata": {}, - "outputs": [], - "source": [ - "! python3 code/np_client_api_job.py --script code/src/train_full.py" - ] - }, - { - "cell_type": "markdown", - "id": "5076b478", - "metadata": {}, - "source": [ - "To instead export the job configuration to use in other modes, run the script with the flag `--export_config`." - ] - }, - { - "cell_type": "markdown", - "id": "efdceecd", - "metadata": {}, - "source": [ - "### Send model parameters differences back to the NVFlare server\n", - "\n", - "We can send model parameter differences back to the NVFlare server by calculating the parameters differences and sending it back: [train_diff.py](code/src/train_diff.py)\n", - "\n", - "Note that we set the `params_type` to `DIFF` when creating `flare.FLModel`.\n", - "\n", - "Then we can run it using the NVFlare Simulator:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "737d8b7c", - "metadata": {}, - "outputs": [], - "source": [ - "! python3 code/np_client_api_job.py --script code/src/train_diff.py" - ] - }, - { - "cell_type": "markdown", - "id": "550479f2", - "metadata": {}, - "source": [ - "### Metrics streaming\n", - "\n", - "We already showed an example with metrics streaming in section 01.5 of Chapter 1 in Part 1, but this is a simple example with the Client API for streaming the training progress to the server with `MLflowWriter`.\n", - "\n", - "NVFlare supports the following writers:\n", - "\n", - " - `SummaryWriter` mimics Tensorboard `SummaryWriter`'s `add_scalar`, `add_scalars` method\n", - " - `WandBWriter` mimics Weights And Biases's `log` method\n", - " - `MLflowWriter` mimics MLflow's tracking api\n", - "\n", - "In this example we use `MLflowWriter` in [train_metrics.py](code/src/train_metrics.py) and configure a corresponding `MLflowReceiver` in the job script [np_client_api_job.py](code/np_client_api_job.py)\n", - "\n", - "Then we can run it using the NVFlare Simulator:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d27f9b3a", - "metadata": {}, - "outputs": [], - "source": [ - "! python3 code/np_client_api_job.py --script code/src/train_metrics.py" - ] - }, - { - "cell_type": "markdown", - "id": "3774d943", - "metadata": {}, - "source": [ - "After the experiment is finished, you can view the results by running the the mlflow command: `mlflow ui --port 5000` inside the directory `/tmp/nvflare/jobs/workdir/server/simulate_job/`." - ] - }, - { - "cell_type": "markdown", - "id": "eadee3dd", - "metadata": {}, - "source": [ - "## Sub-process Client API\n", - "\n", - "The `ScriptRunner` with `launch_external_process=True` uses the `ClientAPILauncherExecutor` for external process script execution.\n", - "This configuration is ideal for scenarios requiring third-party integrations, multi-GPU or distributed PyTorch training, or if additional processes are needed for training." - ] - }, - { - "cell_type": "markdown", - "id": "ade375cd", - "metadata": {}, - "source": [ - "### Launching the script\n", - "\n", - "When launching a script in an external process, it is launched once for the entire job.\n", - "We must ensure our training script [train_full.py](code/src/train_full.py) is in a loop to support this.\n", - "\n", - "Then we can run it using the NVFlare Simulator:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0b3ee641", - "metadata": {}, - "outputs": [], - "source": [ - "! python3 code/np_client_api_job.py --script code/src/train_full.py --launch_process" - ] - }, - { - "cell_type": "markdown", - "id": "5006b87a", - "metadata": {}, - "source": [ - "### Metrics streaming\n", - "\n", - "In this example we use `MLflowWriter` in [train_metrics.py](code/src/train_metrics.py) and configure a corresponding `MLflowReceiver` in the job script [np_client_api_job.py](code/np_client_api_job.py)\n", - "\n", - "Then we can run it using the NVFlare Simulator:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "071834d0", - "metadata": {}, - "outputs": [], - "source": [ - "! python3 np_client_api_job.py --script src/train_metrics.py --launch_process" - ] - }, - { - "cell_type": "markdown", - "id": "95504253", - "metadata": {}, - "source": [ - "If you want to see example code with actual integration with PyTorch and TensorFlow, you can find it in the [Hello World ML to FL](https://github.com/NVIDIA/NVFlare/tree/main/examples/hello-world/ml-to-fl) section of the examples." - ] - }, - { - "cell_type": "markdown", - "id": "c56e633e", - "metadata": {}, - "source": [ - "With this, we are at the end of Chapter 2. The [next notebook](../02.5_recap/recap.ipynb) is a reacap of this chapter." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/np_client_api_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/np_client_api_job.py deleted file mode 100644 index b2a09c5968..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/np_client_api_job.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse - -from nvflare import FedJob -from nvflare.app_common.np.np_model_persistor import NPModelPersistor -from nvflare.app_common.workflows.fedavg import FedAvg -from nvflare.app_opt.tracking.mlflow.mlflow_receiver import MLflowReceiver -from nvflare.job_config.script_runner import FrameworkType, ScriptRunner - - -def define_parser(): - parser = argparse.ArgumentParser() - parser.add_argument("--n_clients", type=int, default=2) - parser.add_argument("--num_rounds", type=int, default=5) - parser.add_argument("--script", type=str, default="src/train_full.py") - parser.add_argument("--launch_process", action=argparse.BooleanOptionalAction, default=False) - parser.add_argument("--export_config", action=argparse.BooleanOptionalAction, default=False) - - return parser.parse_args() - - -def main(): - # define local parameters - args = define_parser() - - n_clients = args.n_clients - num_rounds = args.num_rounds - script = args.script - launch_process = args.launch_process - export_config = args.export_config - - job = FedJob(name="np_client_api") - - persistor_id = job.to_server(NPModelPersistor(), "persistor") - - # Define the controller workflow and send to server - controller = FedAvg(num_clients=n_clients, num_rounds=num_rounds, persistor_id=persistor_id) - job.to_server(controller) - - # Add MLflow Receiver for metrics streaming - if script == "src/train_metrics.py": - receiver = MLflowReceiver( - tracking_uri="file:///tmp/nvflare/jobs/workdir/server/simulate_job/mlruns", - kw_args={ - "experiment_name": "nvflare-fedavg-np-experiment", - "run_name": "nvflare-fedavg-np-with-mlflow", - "experiment_tags": {"mlflow.note.content": "## **NVFlare FedAvg Numpy experiment with MLflow**"}, - "run_tags": {"mlflow.note.content": "## Federated Experiment tracking with MLflow.\n"}, - }, - artifact_location="artifacts", - events=["fed.analytix_log_stats"], - ) - job.to_server(receiver) - - executor = ScriptRunner( - script=script, - launch_external_process=launch_process, - framework=FrameworkType.NUMPY, - ) - job.to_clients(executor) - - if export_config: - job.export_job("/tmp/nvflare/jobs/job_config") - else: - job.simulator_run("/tmp/nvflare/jobs/workdir", n_clients=n_clients, gpu="0") - - -if __name__ == "__main__": - main() diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/src/train_diff.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/src/train_diff.py deleted file mode 100755 index d7dc02ee45..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/src/train_diff.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy - -import nvflare.client as flare - - -def train(input_arr): - output_arr = copy.deepcopy(input_arr) - # mock training with plus 1 - return output_arr + 1 - - -def evaluate(input_arr): - # mock evaluation metrics - return 100 - - -def main(): - # initializes NVFlare interface - flare.init() - - # get system information - sys_info = flare.system_info() - print(f"system info is: {sys_info}") - - while flare.is_running(): - - # get model from NVFlare - input_model = flare.receive() - print(f"received weights is: {input_model.params}") - - input_numpy_array = input_model.params["numpy_key"] - - # training - output_numpy_array = train(input_numpy_array) - - # evaluation - metrics = evaluate(input_numpy_array) - - print(f"finish round: {input_model.current_round}") - - # calculate difference here - diff = output_numpy_array - input_numpy_array - - # send back the model difference - print(f"send back: {diff}") - flare.send( - flare.FLModel( - params={"numpy_key": diff}, - params_type="DIFF", - metrics={"accuracy": metrics}, - current_round=input_model.current_round, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/src/train_full.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/src/train_full.py deleted file mode 100755 index e1598275b5..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/src/train_full.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy - -import nvflare.client as flare - - -def train(input_arr): - output_arr = copy.deepcopy(input_arr) - # mock training with plus 1 - return output_arr + 1 - - -def evaluate(input_arr): - # mock evaluation metrics - return 100 - - -def main(): - # initializes NVFlare interface - flare.init() - - # get system information - sys_info = flare.system_info() - print(f"system info is: {sys_info}") - - while flare.is_running(): - - # get model from NVFlare - input_model = flare.receive() - print(f"received weights is: {input_model.params}", flush=True) - - input_numpy_array = input_model.params["numpy_key"] - - # training - output_numpy_array = train(input_numpy_array) - - # evaluation - metrics = evaluate(input_numpy_array) - - print(f"finish round: {input_model.current_round}", flush=True) - - # send back the model - print(f"send back: {output_numpy_array}", flush=True) - flare.send( - flare.FLModel( - params={"numpy_key": output_numpy_array}, - params_type="FULL", - metrics={"accuracy": metrics}, - current_round=input_model.current_round, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/src/train_metrics.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/src/train_metrics.py deleted file mode 100755 index c6b24e877d..0000000000 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_client_api/code/src/train_metrics.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import time - -import nvflare.client as flare -from nvflare.client.tracking import MLflowWriter - - -def train(input_arr, current_round, epochs=3): - writer = MLflowWriter() - output_arr = copy.deepcopy(input_arr) - num_of_data = 2000 - batch_size = 16 - num_of_batches = num_of_data // batch_size - for i in range(epochs): - for j in range(num_of_batches): - global_step = current_round * num_of_batches * epochs + i * num_of_batches + j - writer.log_metric( - key="global_step", - value=global_step, - step=global_step, - ) - print(f"logged records from epoch: {i}") - # mock training with plus 1 - output_arr += 1 - # assume each epoch takes 1 seconds - time.sleep(1.0) - return output_arr - - -def evaluate(input_arr): - # mock evaluation metrics - return 100 - - -def main(): - # initializes NVFlare interface - flare.init() - - # get system information - sys_info = flare.system_info() - print(f"system info is: {sys_info}") - - while flare.is_running(): - input_model = flare.receive() - print(f"received weights is: {input_model.params}") - - input_numpy_array = input_model.params["numpy_key"] - - # training - output_numpy_array = train(input_numpy_array, current_round=input_model.current_round, epochs=3) - - # evaluation - metrics = evaluate(input_numpy_array) - - print(f"finish round: {input_model.current_round}") - - # send back the model - print(f"send back: {output_numpy_array}") - flare.send( - flare.FLModel( - params={"numpy_key": output_numpy_array}, - params_type="FULL", - metrics={"accuracy": metrics}, - current_round=input_model.current_round, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/data/prepare_heart_disease_data.sh b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/data/prepare_heart_disease_data.sh similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/data/prepare_heart_disease_data.sh rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/data/prepare_heart_disease_data.sh diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/data/utils/convert_data_to_np.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/data/utils/convert_data_to_np.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/data/utils/convert_data_to_np.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/data/utils/convert_data_to_np.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/figs/tb-metrics.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/figs/tb-metrics.png similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/figs/tb-metrics.png rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/figs/tb-metrics.png diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/config/config_fed_client.json b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/config/config_fed_client.json similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/config/config_fed_client.json rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/config/config_fed_client.json diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/config/config_fed_server.json b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/config/config_fed_server.json similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/config/config_fed_server.json rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/config/config_fed_server.json diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_persistor.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_persistor.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_persistor.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_persistor.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/src/newton_raphson_train.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_train.py similarity index 97% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/src/newton_raphson_train.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_train.py index 419b9ed70b..1f426e4a77 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/src/newton_raphson_train.py +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_train.py @@ -20,7 +20,6 @@ from sklearn.metrics import accuracy_score, precision_score import nvflare.client as flare -from nvflare.apis.fl_constant import FLMetaKey from nvflare.app_common.abstract.fl_model import FLModel, ParamsType from nvflare.app_common.np.constants import NPConstants from nvflare.client.tracking import SummaryWriter @@ -169,7 +168,7 @@ def main(): # Send result to server for aggregation. result_model = FLModel(params=result_dict, params_type=ParamsType.FULL) - result_model.meta[FLMetaKey.NUM_STEPS_CURRENT_ROUND] = data["train_X"].shape[0] + result_model.meta["sample_size"] = data["train_X"].shape[0] print( ( diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/src/newton_raphson_workflow.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_workflow.py similarity index 98% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/src/newton_raphson_workflow.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_workflow.py index a4094cb7f6..40a01d6063 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/src/newton_raphson_workflow.py +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/app/custom/newton_raphson_workflow.py @@ -17,7 +17,6 @@ import numpy as np -from nvflare.apis.fl_constant import FLMetaKey from nvflare.app_common.abstract.fl_model import FLModel from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper from nvflare.app_common.app_constant import AppConstants @@ -54,7 +53,6 @@ def run(self) -> None: # converted `FLModel` by `ModelController`. # model = self.load_model() - model.start_round = self.start_round model.total_rounds = self.num_rounds @@ -119,7 +117,7 @@ def newton_raphson_aggregator_fn(self, results: List[FLModel]): for curr_result in results: self.aggregator.add( data=curr_result.params, - weight=curr_result.meta.get(FLMetaKey.NUM_STEPS_CURRENT_ROUND, 1.0), + weight=curr_result.meta.get("sample_size", 1.0), contributor_name=curr_result.meta.get("client_name", AppConstants.CLIENT_UNKNOWN), contribution_round=curr_result.current_round, ) diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/meta.json b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/meta.json similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/meta.json rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/newton_raphson/meta.json diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/requirements.txt b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/requirements.txt similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/requirements.txt rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/requirements.txt diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/train_centralized.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/train_centralized.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/code/train_centralized.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/code/train_centralized.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb new file mode 100644 index 0000000000..96fef946b5 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb @@ -0,0 +1,687 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e8c19632", + "metadata": {}, + "source": [ + "# Converting Logistic Regression to Federated Learning\n", + "\n", + "\n", + "Logistic regression is a fundamental classification algorithm that models the probability of a binary outcome. Despite its name, it's used for classification rather than regression. The model uses the logistic (sigmoid) function to transform a linear combination of features into a probability between 0 and 1.\n", + "\n", + "The Newton-Raphson method is a powerful second-order optimization technique that uses both first-order (gradient) and second-order (Hessian) information to find the optimal model parameters. Unlike first-order methods like gradient descent, Newton's method incorporates curvature information through the Hessian matrix, often leading to faster convergence, especially near the optimum.\n", + "\n", + "In this section, we will convert logistics regression with the 2nd order Newton-Raphson optimization to Federated Learning\n" + ] + }, + { + "cell_type": "markdown", + "id": "7f9d96ed", + "metadata": {}, + "source": [ + "## Federated Logistic Regression with Second-Order Newton-Raphson optimization\n", + "This example shows how to implement a federated binary classification via logistic regression with second-order Newton-Raphson optimization.\n", + "\n", + "The [UCI Heart Disease dataset](https://archive.ics.uci.edu/dataset/45/heart+disease) is\n", + "used in this example. Scripts are provided to download and process the\n", + "dataset as described\n", + "[here](https://github.com/owkin/FLamby/tree/main/flamby/datasets/fed_heart_disease).\n", + "\n", + "This dataset contains samples from 4 sites, splitted into training and\n", + "testing sets as described below:\n", + "|site | sample split |\n", + "|-------------|---------------------------------------|\n", + "|Cleveland | train: 199 samples, test: 104 samples |\n", + "|Hungary | train: 172 samples, test: 89 samples |\n", + "|Switzerland | train: 30 samples, test: 16 samples |\n", + "|Long Beach V | train: 85 samples, test: 45 samples |\n", + "\n", + "The number of features in each sample is 13." + ] + }, + { + "cell_type": "markdown", + "id": "e54f0dcc", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "The [Newton-Raphson\n", + "optimization](https://en.wikipedia.org/wiki/Newton%27s_method) problem\n", + "can be described as follows.\n", + "\n", + "In a binary classification task with logistic regression, the\n", + "probability of a data sample $x$ classified as positive is formulated\n", + "as:\n", + "$$p(x) = \\sigma(\\beta \\cdot x + \\beta_{0})$$\n", + "where $\\sigma(.)$ denotes the sigmoid function. We can incorporate\n", + "$\\beta_{0}$ and $\\beta$ into a single parameter vector $\\theta =\n", + "( \\beta_{0}, \\beta)$. Let $d$ be the number\n", + "of features for each data sample $x$ and let $N$ be the number of data\n", + "samples. We then have the matrix version of the above probability\n", + "equation:\n", + "$$p(X) = \\sigma( X \\theta )$$\n", + "Here $X$ is the matrix of all samples, with shape $N \\times (d+1)$,\n", + "having it's first column filled with value 1 to account for the\n", + "intercept $\\theta_{0}$.\n", + "\n", + "The goal is to compute parameter vector $\\theta$ that maximizes the\n", + "below likelihood function:\n", + "$$L_{\\theta} = \\prod_{i=1}^{N} p(x_i)^{y_i} (1 - p(x_i)^{1-y_i})$$\n", + "\n", + "The Newton-Raphson method optimizes the likelihood function via\n", + "quadratic approximation. Omitting the maths, the theoretical update\n", + "formula for parameter vector $\\theta$ is:\n", + "$$\\theta^{n+1} = \\theta^{n} - H_{\\theta^{n}}^{-1} \\nabla L_{\\theta^{n}}$$\n", + "where\n", + "$$\\nabla L_{\\theta^{n}} = X^{T}(y - p(X))$$\n", + "is the gradient of the likelihood function, with $y$ being the vector\n", + "of ground truth for sample data matrix $X$, and\n", + "$$H_{\\theta^{n}} = -X^{T} D X$$\n", + "is the Hessian of the likelihood function, with $D$ a diagonal matrix\n", + "where diagonal value at $(i,i)$ is $D(i,i) = p(x_i) (1 - p(x_i))$.\n", + "\n", + "In federated Newton-Raphson optimization, each client will compute its\n", + "own gradient $\\nabla L_{\\theta^{n}}$ and Hessian $H_{\\theta^{n}}$\n", + "based on local training samples. A server will aggregate the gradients\n", + "and Hessians computed from all clients, and perform the update of\n", + "parameter $\\theta$ based on the theoretical update formula described\n", + "above." + ] + }, + { + "cell_type": "markdown", + "id": "32003ba9", + "metadata": {}, + "source": [ + "## Implementation\n", + "\n", + "Using `nvflare`, The federated logistic regression with Newton-Raphson\n", + "optimization is implemented as follows.\n", + "\n", + "On the server side, all workflow logics are implemented in\n", + "class `FedAvgNewtonRaphson`, which can be found\n", + "[here](code/newton_raphson/app/custom/newton_raphson_workflow.py). The\n", + "`FedAvgNewtonRaphson` class inherits from the\n", + "[`BaseFedAvg`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/base_fedavg.py)\n", + "class, which itself inherits from the **ModelController**\n", + "([`ModelController`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/model_controller.py))\n", + "class. This is the preferrable approach to implement a custom\n", + "workflow, since `ModelController` decouples communication logic from\n", + "actual workflow (training & validation) logic. The mandatory\n", + "method to override in `ModelController` is the\n", + "[`run()`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/model_controller.py#L37)\n", + "method, where the orchestration of server-side workflow actually\n", + "happens. The implementation of `run()` method in\n", + "[`FedAvgNewtonRaphson`](code/newton_raphson/app/custom/newton_raphson_workflow.py)\n", + "is similar to the classic\n", + "[`FedAvg`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/fedavg.py#L44):\n", + "- Initialize the global model, this is acheived through method `load_model()`\n", + " from base class\n", + " [`ModelController`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/model_controller.py#L292),\n", + " which relies on the\n", + " [`ModelPersistor`](https://nvflare.readthedocs.io/en/main/glossary.html#persistor). A\n", + " custom\n", + " [`NewtonRaphsonModelPersistor`](code/newton_raphson/app/custom/newton_raphson_persistor.py)\n", + " is implemented in this example, which is based on the\n", + " [`NPModelPersistor`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/np/np_model_persistor.py)\n", + " for numpy data, since the _model_ in the case of logistic regression\n", + " is just the parameter vector $\\theta$ that can be represented by a\n", + " numpy array. Only the `__init__` method needs to be re-implemented\n", + " to provide a proper initialization for the global parameter vector\n", + " $\\theta$.\n", + "- During each training round, the global model will be sent to the\n", + " list of participating clients to perform a training task. This is\n", + " done using the\n", + " [`send_model_and_wait()`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/workflows/model_controller.py#L41)\n", + " method. Once\n", + " the clients finish their local training, results will be collected\n", + " and sent back to server as\n", + " [`FLModel`](https://nvflare.readthedocs.io/en/main/programming_guide/fl_model.html#flmodel)s.\n", + "- Results sent by clients contain their locally computed gradient and\n", + " Hessian. A [custom aggregation\n", + " function](code/newton_raphson/app/custom/newton_raphson_workflow.py)\n", + " is implemented to get the averaged gradient and Hessian, and compute\n", + " the Newton-Raphson update for the global parameter vector $\\theta$,\n", + " based on the theoretical formula shown above. The averaging of\n", + " gradient and Hessian is based on the\n", + " [`WeightedAggregationHelper`](https://github.com/NVIDIA/NVFlare/blob/main/nvflare/app_common/aggregators/weighted_aggregation_helper.py#L20),\n", + " which weighs the contribution from each client based on the number\n", + " of local training samples. The aggregated Newton-Raphson update is\n", + " returned as an `FLModel`.\n", + "- After getting the aggregated Newton-Raphson update, an\n", + " [`update_model()`](code/newton_raphson/app/custom/newton_raphson_workflow.py#L172)\n", + " method is implemented to actually apply the Newton-Raphson update to\n", + " the global model.\n", + "- The last step is to save the updated global model, again through\n", + " the `NewtonRaphsonModelPersistor` using `save_model()`.\n", + "\n", + "\n", + "On the client side, the local training logic is implemented\n", + "[here](code/newton_raphson/app/custom/newton_raphson_train.py). The\n", + "implementation is based on the [`Client\n", + "API`](https://nvflare.readthedocs.io/en/main/programming_guide/execution_api_type.html#client-api). This\n", + "allows user to add minimum `nvflare`-specific code to turn a typical\n", + "centralized training script into a federated client side local training\n", + "script.\n", + "- During local training, each client receives a copy of the global\n", + " model, sent by the server, using `flare.receive()` from the Client API.\n", + " The received global model is an instance of `FLModel`.\n", + "- A local validation is first performed, where validation metrics\n", + " (accuracy and precision) are streamed to server using the\n", + " [`SummaryWriter`](https://nvflare.readthedocs.io/en/main/apidocs/nvflare.client.tracking.html#nvflare.client.tracking.SummaryWriter). The\n", + " streamed metrics can be loaded and visualized using tensorboard.\n", + "- Then each client computes it's gradient and Hessian based on local\n", + " training data, using their respective theoretical formula described\n", + " above. This is implemented in the\n", + " [`train_newton_raphson()`](code/newton_raphson/app/custom/newton_raphson_train.py#L82)\n", + " method. Each client then sends the computed results (always in\n", + " `FLModel` format) to server for aggregation, using the Client API call\n", + " `flare.send()`.\n", + "\n", + "Each client site corresponds to a site listed in the data table above.\n", + "\n", + "A [centralized training script](code/train_centralized.py) is also\n", + "provided, which allows for comparing the federated Newton-Raphson\n", + "optimization versus the centralized version. In the centralized\n", + "version, training data samples from all 4 sites were concatenated into\n", + "a single matrix, used to optimize the model parameters. The\n", + "optimized model was then tested separately on testing data samples of\n", + "the 4 sites, using accuracy and precision as metrics.\n", + "\n", + "Comparing the federated [client-side training\n", + "code](code/newton_raphson/app/custom/newton_raphson_train.py) with the\n", + "centralized [training code](code/train_centralized.py), we can see that\n", + "the training logic remains similar: load data, perform training\n", + "(Newton-Raphson updates), and valid trained model. The only added\n", + "differences in the federated code are related to interaction with the\n", + "FL system, such as receiving and send `FLModel`." + ] + }, + { + "cell_type": "markdown", + "id": "c3fc55e0", + "metadata": {}, + "source": [ + "## Install requirements\n", + "First, install the required packages:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04911ca3", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r code/requirements.txt" + ] + }, + { + "cell_type": "markdown", + "id": "33ea8504", + "metadata": {}, + "source": [ + "## Download and prepare data\n", + "\n", + "Execute the following script\n", + "```\n", + "bash ./code/data/prepare_heart_disease_data.sh\n", + "```\n", + "This will download the heart disease dataset under\n", + "`/tmp/flare/dataset/heart_disease_data/`\n", + "\n", + "Please note that you may need to accept the data terms in order to complete the download." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b395c0d9", + "metadata": {}, + "outputs": [], + "source": [ + "# Note: the the download site remember your download history and abort the 2nd download attempt. \n", + "\n", + "! echo y | bash ./code/data/prepare_heart_disease_data.sh\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "61e13343", + "metadata": {}, + "outputs": [], + "source": [ + "! ls -al /tmp/flare/dataset/heart_disease_data/" + ] + }, + { + "cell_type": "markdown", + "id": "d548b466", + "metadata": {}, + "source": [ + "## Centralized Logistic Regression\n", + "\n", + "Two implementations of logistic regression are provided in the\n", + "centralized training script, which can be specified by the `--solver`\n", + "argument:\n", + "- One is using `sklearn.LogisticRegression` with the `newton-cholesky`\n", + " solver\n", + "- The other one is manually implemented using the theoretical update\n", + " formulas described above.\n", + "\n", + "Both implementations were tested to converge in 4 iterations and to\n", + "give the same result.\n", + "\n", + "Launch the following script:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c68fe1a", + "metadata": {}, + "outputs": [], + "source": [ + "%cd code\n", + "! python3 train_centralized.py --solver custom\n", + "\n", + "%cd -" + ] + }, + { + "cell_type": "markdown", + "id": "a1c8d278", + "metadata": {}, + "source": [ + "## Federated Logistic Regression\n", + "\n", + "\n", + "To convert the centralized logistic regression to federated learning, we need to do the following:\n", + "\n", + "1. Decide what model parameters will be transmitted between the server and clients\n", + "2. Define the workflow that orchestrates the federated learning process\n", + "3. Define how to load the initial model on the server side\n", + "4. Modify the client-side training logic to handle models received from the server\n", + "5. Implement the aggregation logic for the gradients and Hessians computed by the clients\n", + "6. Configure the job via FLARE job API\n", + "\n", + "Let's examine each step.\n", + "\n", + "### Model Parameters\n", + "\n", + "We decided to simply capture the model parameters in the FLModel:\n", + "\n", + "\n", + "```python\n", + "\n", + "model = FLModel(params={\"gradient\": gradient, \"hessian\": hessian})\n", + "```\n", + "\n", + "We could optionally use FLModel.optimizer_params to store the Hessian, but either approach works.\n", + "\n", + "We add a few metadata fields to help with the training process. We use the training sample size as the weight, storing this information in the metadata:\n", + "\n", + "```python\n", + "\n", + "model = FLModel(params=result_dict, params_type=ParamsType.FULL)\n", + "model.meta[\"sample_size\"] = data[\"train_X\"].shape[0]\n", + "```\n", + "\n", + "### Workflow\n", + "\n", + "We decided to choose the FedAvg type of scatter and gather workflow. So we can based the class using the `BaseFedAvg` class. \n", + "\n", + "```python\n", + "\n", + "class FedAvgNewtonRaphson(BaseFedAvg):\n", + "\n", + " def __init__(self, damping_factor, epsilon=1.0, *args, **kwargs):\n", + " super().__init__(*args, **kwargs)\n", + " \"\"\"\n", + " Args:\n", + " damping_factor: damping factor for Newton Raphson updates.\n", + " epsilon: a regularization factor to avoid empty hessian for\n", + " matrix inversion\n", + " \"\"\"\n", + " self.damping_factor = damping_factor\n", + " self.epsilon = epsilon\n", + " self.aggregator = WeightedAggregationHelper()\n", + "\n", + " def run(self) -> None:\n", + " \n", + " # First load the model and set up some training params.\n", + " # A `persisitor` (NewtonRaphsonModelPersistor) will load\n", + " # the model in `ModelLearnable` format, then will be\n", + " # converted `FLModel` by `ModelController`.\n", + " #\n", + " model = self.load_model()\n", + "\n", + " model.start_round = self.start_round\n", + " model.total_rounds = self.num_rounds\n", + "\n", + " \n", + " for self.current_round in range(self.start_round, self.start_round + self.num_rounds):\n", + "\n", + " # Get the list of clients.\n", + " clients = self.sample_clients(self.num_clients)\n", + "\n", + " model.current_round = self.current_round\n", + "\n", + " results = self.send_model_and_wait(targets=clients, data=model)\n", + "\n", + " # Aggregate results receieved from clients.\n", + " aggregate_results = self.aggregate(results, aggregate_fn=self.newton_raphson_aggregator_fn)\n", + "\n", + " # Update global model based on the following formula:\n", + " # weights = weights + updates, where\n", + " # updates = -damping_factor * Hessian^{-1} . Gradient\n", + " self.update_model(model, aggregate_results)\n", + "\n", + " # Save global model.\n", + " self.save_model(model)\n", + "\n", + " self.info(\"Finished FedAvg.\")\n", + "\n", + "```\n", + "As you can see the `run()` method is the only method we need to implement. Its nothing but a for loop that sends the model to the clients and aggregate the results. \n", + "\n", + "### Model Loader\n", + "\n", + "we need to decide how to load the initial model on the server side. We decide to implement a custom persistor that loads the model from a numpy file. \n", + "\n", + "```python\n", + "\n", + "class NewtonRaphsonModelPersistor(NPModelPersistor):\n", + " \"\"\"\n", + " This class defines the persistor for Newton Raphson model.\n", + "\n", + " A persistor controls the logic behind initializing, loading\n", + " and saving of the model / parameters for each round of a\n", + " federated learning process.\n", + "\n", + " In the 2nd order Newton Raphson case, a model is just a\n", + " 1-D numpy vector containing the parameters for logistic\n", + " regression. The length of the parameter vector is defined\n", + " by the number of features in the dataset.\n", + "\n", + " \"\"\"\n", + "\n", + " def __init__(self, model_dir=\"models\", model_name=\"weights.npy\", n_features=13):\n", + " super().__init__()\n", + "\n", + " self.model_dir = model_dir\n", + " self.model_name = model_name\n", + " self.n_features = n_features\n", + "\n", + " # A default model is loaded when no local model is available.\n", + " # This happen when training starts.\n", + " #\n", + " # A `model` for a binary logistic regression is just a matrix,\n", + " # with shape (n_features + 1, 1).\n", + " # For the UCI ML Heart Disease dataset, the n_features = 13.\n", + " #\n", + " # A default matrix with value 0s is created.\n", + " #\n", + " self.default_data = np.zeros((self.n_features + 1, 1), dtype=np.float32)\n", + "\n", + "```\n", + "\n", + "\n", + "### Client Training Logic \n", + "\n", + "Now, we need to convert the centralized training logic to the federated training logic with Client API.\n", + "\n", + "```python\n", + "\n", + "\n", + "def main():\n", + " \n", + " args = parse_arguments()\n", + "\n", + " flare.init()\n", + "\n", + " site_name = flare.get_site_name()\n", + " \n", + " # Load client site data.\n", + " data = load_data(args.data_root, site_name)\n", + "\n", + "\n", + " # keep running until the job is terminated or end of training round\n", + " while flare.is_running():\n", + "\n", + " # Receive global model (FLModel) from server.\n", + " global_model = flare.receive()\n", + "\n", + " # Get the weights, aka parameter theta for logistic regression.\n", + " global_weights = global_model.params[\"weights\"]\n", + "\n", + " # Local validation before training\n", + " validation_scores = validate(data, global_weights)\n", + "\n", + " # Local training\n", + " result_dict = train_newton_raphson(data, theta=global_weights)\n", + "\n", + " # Send result to server for aggregation.\n", + " local_model = FLModel(params=result_dict, params_type=ParamsType.FULL)\n", + " local_model.meta[\"sample_size\"] = data[\"train_X\"].shape[0]\n", + "\n", + " flare.send(local_model)\n", + "\n", + "```\n", + "\n", + "This is pretty straight forward. We receive the global model, perform the local training and send the result to the server. The code structure is the same to the centralized training with additional loop for the federated training. \n", + "\n", + "We added the sample size to the meta data so we can use it in weighted aggregation as the aggregation weight.\n", + "\n", + "\n", + "### Aggregation Logic\n", + "\n", + "Now, lets loop at the aggregation logic. \n", + "\n", + "```python\n", + "\n", + " def newton_raphson_aggregator_fn(self, results: List[FLModel]):\n", + " \"\"\"\n", + " This uses the default thread-safe WeightedAggregationHelper,\n", + " which implement a weighted average of all values received from\n", + " a `result` dictionary.\n", + "\n", + " Args:\n", + " results: a list of `FLModel`s. Each `FLModel` is received\n", + " from a client. The field `params` is a dictionary that\n", + " contains values to be aggregated: the gradient and hessian.\n", + " \"\"\"\n", + " \n", + " # On client side the `sample_size` key is used to track the number of samples for each client.\n", + " for curr_result in results:\n", + " self.aggregator.add(\n", + " data=curr_result.params,\n", + " weight=curr_result.meta.get(\"sample_size\", 1.0),\n", + " contributor_name=curr_result.meta.get(\"client_name\", AppConstants.CLIENT_UNKNOWN),\n", + " contribution_round=curr_result.current_round,\n", + " )\n", + "\n", + " aggregated_dict = self.aggregator.get_result()\n", + " \n", + " # Compute global model update:\n", + " # update = - damping_factor * Hessian^{-1} . Gradient\n", + " # A regularization is added to avoid empty hessian.\n", + " #\n", + " reg = self.epsilon * np.eye(aggregated_dict[\"hessian\"].shape[0])\n", + "\n", + " newton_raphson_updates = self.damping_factor * np.linalg.solve(\n", + " aggregated_dict[\"hessian\"] + reg, aggregated_dict[\"gradient\"]\n", + " )\n", + " \n", + " # Convert the aggregated result to `FLModel`, this `FLModel`\n", + " # will then be used by `update_model` method from the base class,\n", + " # to update the global model weights.\n", + " #\n", + " aggr_result = FLModel(\n", + " params={\"newton_raphson_updates\": newton_raphson_updates},\n", + " params_type=results[0].params_type,\n", + " meta={\n", + " \"nr_aggregated\": len(results),\n", + " AppConstants.CURRENT_ROUND: results[0].current_round,\n", + " AppConstants.NUM_ROUNDS: self.num_rounds,\n", + " },\n", + " )\n", + " return aggr_result\n", + "\n", + " def update_model(self, model, model_update, replace_meta=True) -> FLModel:\n", + " \"\"\"\n", + " Update logistic regression parameters based on\n", + " aggregated gradient and hessian.\n", + "\n", + " \"\"\"\n", + " if replace_meta:\n", + " model.meta = model_update.meta\n", + " else:\n", + " model.meta.update(model_update.meta)\n", + "\n", + " model.metrics = model_update.metrics\n", + " model.params[NPConstants.NUMPY_KEY] += model_update.params[\"newton_raphson_updates\"]\n", + "\n", + "```\n", + "Again, we just need to use FLModel to store the result and update the model. \n", + "\n", + "\n", + "### Job Configuration\n", + "\n", + "With the above steps, we have converted the centralized training to the federated training. \n", + "\n", + "Now, lets connect the pieces together and define the job configuration and run with simulator. \n", + "\n", + "In this example, we decided to sub-process instead of in-process training. \n", + "\n", + "We manually define the job configuration and run with simulator. \n", + "\n", + "#### server job configuration\n", + "\n", + "The key is defined a workflow ```FedAvgNewtonRaphson``` and corresponding arguments: number round, clients and damping factor. \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c5767f4", + "metadata": {}, + "outputs": [], + "source": [ + "! cat code/newton_raphson/app/config/config_fed_server.json" + ] + }, + { + "cell_type": "markdown", + "id": "f13fdf5d", + "metadata": {}, + "source": [ + "#### client job configuration\n", + "\n", + "Notice that we used the ClientAPILauncherExecutor with a Cell Pipe, we also need a separate pipe for metrics relay" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e9b8eef", + "metadata": {}, + "outputs": [], + "source": [ + "! cat code/newton_raphson/app/config/config_fed_client.json" + ] + }, + { + "cell_type": "markdown", + "id": "9a3fe4de", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "id": "0b72ef2b", + "metadata": {}, + "source": [ + "## Running Federated Logistic Regression Job\n", + "\n", + "Execute the following command to launch federated logistic\n", + "regression. This will run in `nvflare`'s simulator mode.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2faea343", + "metadata": {}, + "outputs": [], + "source": [ + "! nvflare simulator -w /tmp/nvflare/job/lr/workspace -n 4 -t 4 code/newton_raphson/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "Accuracy and precision for each site can be viewed in Tensorboard:\n", + "```\n", + "tensorboard --logdir=/tmp/nvflare/job/lr/workspace/server/simulate_job/tb_events\n", + "```\n", + "As can be seen from the figure below, per-site evaluation metrics in\n", + "federated logistic regression are on-par with the centralized version.\n", + "\n", + "\"Tensorboard\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0858459", + "metadata": {}, + "outputs": [], + "source": [ + "! tensorboard --logdir=/tmp/nvflare/job/lr/workspace/server/simulate_job/tb_events" + ] + }, + { + "cell_type": "markdown", + "id": "d8383681", + "metadata": {}, + "source": [ + "Now that we have converted the centralized logistic regression to federated learning, let's move on to the next example." + ] + }, + { + "cell_type": "markdown", + "id": "d990f546", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nvflare_env", + "language": "python", + "name": "nvflare_env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/figs/minibatch.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/figs/minibatch.png similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/figs/minibatch.png rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/figs/minibatch.png diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/kmeans_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/kmeans_job.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/kmeans_job.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/kmeans_job.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/requirements.txt b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/requirements.txt similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/requirements.txt rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/requirements.txt diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/src/kmeans_assembler.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/src/kmeans_assembler.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/src/kmeans_assembler.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/src/kmeans_assembler.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/src/kmeans_fl.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/src/kmeans_fl.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/src/kmeans_fl.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/src/kmeans_fl.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/utils/prepare_data.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/utils/prepare_data.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/utils/prepare_data.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/utils/prepare_data.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/utils/split_data.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/utils/split_data.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/code/utils/split_data.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/code/utils/split_data.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb similarity index 99% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb index 9a8cb7a548..20376a251f 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb @@ -14,6 +14,7 @@ "metadata": {}, "source": [ "## Introduction to Scikit-learn, tabular data, and federated k-Means\n", + "\n", "### Scikit-learn\n", "This example shows how to use [NVIDIA FLARE](https://nvflare.readthedocs.io/en/main/index.html) on tabular data.\n", "It uses [Scikit-learn](https://scikit-learn.org/),\n", diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_baseline.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_baseline.png similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_baseline.png rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_baseline.png diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_fl.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_fl.png similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_fl.png rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_fl.png diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_fl_he.png b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_fl_he.png similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_fl_he.png rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/figs/km_curve_fl_he.png diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/km_job.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/km_job.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/km_job.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/km_job.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/requirements.txt b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/requirements.txt similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/requirements.txt rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/requirements.txt diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_train.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_train.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_train.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_train.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_train_he.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_train_he.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_train_he.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_train_he.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_wf.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_wf.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_wf.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_wf.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_wf_he.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_wf_he.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_wf_he.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/src/kaplan_meier_wf_he.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/utils/baseline_kaplan_meier.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/utils/baseline_kaplan_meier.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/utils/baseline_kaplan_meier.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/utils/baseline_kaplan_meier.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/utils/prepare_data.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/utils/prepare_data.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/utils/prepare_data.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/utils/prepare_data.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/utils/prepare_he_context.py b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/utils/prepare_he_context.py similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/code/utils/prepare_he_context.py rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/code/utils/prepare_he_context.py diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/convert_survival_analysis_to_fl.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/convert_survival_analysis_to_fl.ipynb similarity index 100% rename from examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/convert_survival_analysis_to_fl.ipynb rename to examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/02.4.3_convert_survival_analysis_to_federated_learning/convert_survival_analysis_to_fl.ipynb diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/convert_ml_to_fl.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/convert_ml_to_fl.ipynb new file mode 100644 index 0000000000..0efe327f83 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.4_convert_machine_learning_to_federated_learning/convert_ml_to_fl.ipynb @@ -0,0 +1,36 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to Convert Centralized Machine Learning to Federated Learning\n", + "\n", + "Converting different deep learning algorithms to federated learning is very similar, but for traditional machine learning, we have to deal with them case by case. However, they all essentially still follow the same process using the Client API:\n", + "\n", + "* Formulating the algorithm: see how to structure the model exchange to be represented by `FLModel`\n", + "* Receive model from global\n", + "* Update and train local model \n", + "* Send the global model back to aggregator\n", + " \n", + "In this section, we will study three different commonly used machine learning algorithms:\n", + "\n", + "* [Convert Logistic Regression to federated learning](./02.4.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb)\n", + "* [Convert KMeans to federated learning](./02.4.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb)\n", + "* [Convert Survival Analysis to federated learning](./02.4.3_convert_survival_analysis_to_federated_learning/convert_survival_analysis_to_fl.ipynb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.5_recap/recap.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.5_recap/recap.ipynb index 77f82177a4..a6898b699f 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.5_recap/recap.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/chapter-2_develop_federated_learning_applications/02.5_recap/recap.ipynb @@ -5,40 +5,32 @@ "id": "7b152728-3366-4432-adb1-29aa3051dc22", "metadata": {}, "source": [ - "# Summary of Chapter 2\n", + "# Recap: Chapter 2 Summary\n", "\n", - "We covered developing federated learning applications in Chapter 2. Here is an overview:\n", "\n", - "1. **Federated Statistics**\n", - " - **Federated Statistics with Image Data**: How to compute local and global image statistics with the consideration that data is private at each of the client sites.\n", - " - [federated_statistics_with_image_data.ipynb](../02.1_federated_statistics/federated_statistics_with_image_data/federated_statistics_with_image_data.ipynb)\n", - " - **Federated Statistics with Tabular Data**: How to create federated statistics for data that can be represented as Pandas DataFrames.\n", - " - [federated_statistics_with_tabular_data.ipynb](../02.1_federated_statistics/federated_statistics_with_tabular_data/federated_statistics_with_tabular_data.ipynb)\n", - "\n", - "2. **Converting PyTorch Lightning to FL**\n", - " - **PyTorch Lightning to FL**: Guide on converting PyTorch Lightning scripts to federated learning.\n", - " - [convert_torch_lightning_to_fl.ipynb](../02.2_convert_torch_lightning_to_federated_learning/convert_torch_lightning_to_fl.ipynb)\n", + "We covered a lot of ground in developing federated learning applications in this Chapter. We focused on how to compute federated statistics and how to leverage the NVFLARE Client API to convert machine learning models to federated learning.\n", "\n", - "3. **Simple ML/DL to FL transition with NVFlare**\n", - " - **Converting Logistic Regression to FL**: How to implement a federated binary classification via logistic regression with second-order Newton-Raphson optimization. \n", - " - [convert_logistic_regression_to_fl.ipynb](../02.3_convert_machine_learning_to_federated_learning/02.3.1_convert_logistic_regression_to_federated_learning/convert_logistic_regression_to_fl.ipynb)\n", - " - **Converting KMeans to FL**: ADD CONTENT HERE. \n", - " - [convert_kmeans_to_fl.ipynb](../02.3_convert_machine_learning_to_federated_learning/02.3.2_convert_kmeans_to_federated_learning/convert_kmeans_to_fl.ipynb)\n", - " - **Secure Federated Kaplan-Meier Analysis via Time-Binning and Homomorphic Encryption**: ADD CONTENT HERE. \n", - " - [convert_survival_analysis_to_fl.ipynb](../02.3_convert_machine_learning_to_federated_learning/02.3.3_convert_survival_analysis_to_federated_learning/convert_survival_analysis_to_fl.ipynb)\n", - "\n", - "4. **Client API**\n", - " - **Client API**: Here we focus on the core concepts of the Client API and explain how to configure it to run within the same process or in a separate subprocess. \n", - " - [client_api.ipynb](../02.4_client_api/client_api.ipynb)\n", + "1. **Federated Statistics**\n", + " Users can leverage NVFLARE's statistics feature to aggregate and visualize global statistics as well as site-specific statistics.\n", "\n", - "5. **Recap of Covered Topics**\n", - " - **Summary and Recap**: A recap of the topics covered in the previous sections.\n", + " This process is fairly straightforward.\n", + " \n", + "2. **Client API**\n", + " We discussed the Client API in detail, including the types of implementations and how to configure it to run within the same process or in a separate subprocess.\n", "\n", - "Each section is designed to provide comprehensive guidance and practical examples to help you implement and customize federated learning in your applications. For detailed instructions and examples, refer to the respective notebooks linked in each section.\n", + " Then we moved on to PyTorch Lightning and several traditional machine learning models and how to convert them to federated learning with the Client API.\n", + " \n", + " We hope that by now, you should be very confident in converting most deep learning and traditional machine learning models to federated learning with NVIDIA FLARE.\n", "\n", "\n", - "Now let's move on to the [Chapter 3](../../../part-2_federated_learning_system/chapter-3_federated_computing_platform/03.0_introduction/introduction.ipynb)." + "Now let's explore the [Federated Computing Platform](../../../part-2_federated_learning_system/chapter-3_federated_computing_platform/03.0_introduction/introduction.ipynb)." ] + }, + { + "cell_type": "markdown", + "id": "bbcd8e9d", + "metadata": {}, + "source": [] } ], "metadata": { diff --git a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/part_1_introduction.ipynb b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/part_1_introduction.ipynb index bf2e5b6589..dc895cf38f 100644 --- a/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/part_1_introduction.ipynb +++ b/examples/tutorials/self-paced-training/part-1_federated_learning_introduction/part_1_introduction.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In part 1, we will use two chapters illustate how to run and develop federated learning applications. " + "we will use two chapters to illustrate how to run and develop federated learning applications. " ] }, { @@ -27,18 +27,18 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In Part 1, we explored the fundamentals of federated learning. We will cover:\n", + "In Part 1, we will explore the fundamentals of federated learning. We will cover:\n", "\n", "#### Chapter 1: \n", "* How to train an image classification model with PyTorch\n", - "* How to convert a standard PyTorch training code to federated learning code\n", - "* How to customize client and server side logics\n", - "* Understanding the federated job structure and configurations\n", + "* How to convert standard PyTorch training code to federated learning code\n", + "* How to customize client and server side logic\n", + "* Understanding federated job structure and configurations\n", "\n", "#### Chapter 2: \n", - "* federated statistics for both image and tabular data.\n", - "* convert porch lightning to federated learning\n", - "* convert traditional ML training code to federated learning code,\n", + "* Federated statistics for both image and tabular data\n", + "* Converting PyTorch Lightning to federated learning\n", + "* Converting traditional ML training code to federated learning code\n", "* FLARE Client API" ] }, diff --git a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.0_introduction/introduction.ipynb b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.0_introduction/introduction.ipynb index 82509ddce4..0a43ccbc6a 100644 --- a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.0_introduction/introduction.ipynb +++ b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.0_introduction/introduction.ipynb @@ -103,7 +103,7 @@ "The core property of FLComponent is event support. FLComponent is able to fire and receive events, enabling the FLARE system to be an event-driven, pluggable system.\n", "\n", "### FLContext\n", - "One of the most important features of NVIDIA FLARE is ```nvflare.apis.fl_context``` to pass data between the FL components. FLContext is available to every method of all FLComponent types (Controller, Aggregator, Filter, Executor).\n", + "One of the most important features of NVIDIA FLARE is nvflare.apis.fl_context, which is used to pass data between the FL components. FLContext is available to every method of all FLComponent types (Controller, Aggregator, Filter, Executor).\n", "\n", "\n", "Through the FL Context, the component developer can:\n", diff --git a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.1_federated_computing_architecture/system_architecture.ipynb b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.1_federated_computing_architecture/system_architecture.ipynb index e6ebe72a7f..adbebcb9a5 100644 --- a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.1_federated_computing_architecture/system_architecture.ipynb +++ b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.1_federated_computing_architecture/system_architecture.ipynb @@ -103,7 +103,7 @@ "\n", "## Event-Based System\n", "\n", - "ALL NVIDIA FLARE's components (FLComponent) has event handling and event firing via the runtine engine. As result, user can write a FLComponent as plugin and listen to event and write any customized logics at any layers. \n", + "ALL NVIDIA FLARE's components (FLComponent) has event handling and event firing via the runtime engine. As a result, users can write an FLComponent as a plugin and listen to events and write any customized logic at any layer.\n", "\n", "\n", "## Federated Learning Framework\n", @@ -126,7 +126,7 @@ "\n", "## Different type of FLARE APIs\n", "\n", - "At its Core, Flare uses controller and executor assign tasks and execute tasks for each job. There we have the \n", + "At its Core, Flare uses controller and executor to assign tasks and execute tasks for each job. There we have the:\n", "\n", "### Python APIs\n", "\n", @@ -150,17 +150,17 @@ " ...\n", "\n", " ```\n", - "This data structure essentially capture the model ( parameter type (Full, Diff), model paramaters (weights), optmizer parameters), metrics, metadata. This kind data structure is understandable by most data scientists. \n", + "This data structure essentially captures the model (parameter type (Full, Diff), model parameters (weights), optimizer parameters), metrics, metadata. This kind of data structure is understandable by most data scientists.\n", "\n", - "The Server side, we have ModelController -- Controller use and consume FLModel, on the client side we have Client API that receive and send model update via FLModel. You already seen this in previous chapters. \n", + "On the Server side, we have ModelController -- Controller uses and consumes FLModel, on the client side we have Client API that receives and sends model updates via FLModel. You have already seen this in previous chapters.\n", "\n", "\n", "* **Job API** -- FLARE Job API is a way to generate job configuration. Although once can direct edit configuration files, one can also use the Job API to construct the needed components and generate the job configuration. The job API can also call job.simulate_run() -- which is combined step of export job configuration and call simulator run. \n", "\n", - "* **Simulator API** -- one can directly invokve simulator_run() method start simulation in python\n", + "* **Simulator API** -- one can directly invoke simulator_run() method to start simulation in python\n", "\n", "\n", - "* **FLARE API** -- FLARE python API is equivallent FLARE Console command API. Instead of interact with FL system via Console command, we can perform most of the command functions via FLARE API. These includes connect to the server, checking status, monitoring jobs, submit job etc. \n", + "* **FLARE API** -- FLARE python API is equivalent to FLARE Console command API. Instead of interacting with FL system via Console command, we can perform most of the command functions via FLARE API. These include connecting to the server, checking status, monitoring jobs, submitting jobs etc.\n", "\n", "\n", "### Command Line Interface\n", @@ -194,7 +194,7 @@ "\n", "# Job Template\n", "\n", - "Job templates are set of existing job configurations with specified structure \n", + "Job templates are a set of existing job configurations with specified structure\n", "\n", "For example \n", "```\n", @@ -207,9 +207,9 @@ "\n", "```\n", "\n", - "Each job template consits \"information card\", info.conf, display card \"info.md\" and job configuration files. \n", + "Each job template consists of an \"information card\", info.conf, display card \"info.md\" and job configuration files.\n", "\n", - "The configuraiton is defined in pyhocon format so we can add comments and explain the details \n", + "The configuration is defined in pyhocon format so we can add comments and explain the details.\n", "\n", "we can take a look at one example \n", "\n", diff --git a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.2_deployment_simulation/simulate_real_world_deployment.ipynb b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.2_deployment_simulation/simulate_real_world_deployment.ipynb index f1dc6c6e40..f16635a56e 100644 --- a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.2_deployment_simulation/simulate_real_world_deployment.ipynb +++ b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.2_deployment_simulation/simulate_real_world_deployment.ipynb @@ -29,7 +29,7 @@ "\n", "[POC command](https://nvflare.readthedocs.io/en/main/user_guide/nvflare_cli/poc_command.html) provides a set of command to create different software packages and simulate client and server. You can also find the tutorials in [here](../../../../setup_poc.ipynb) on how to setup POC. \n", "\n", - "> With POC mode, it is ideally use terminal instead of notebook to setup, as there are some non-consistent behavior when running scripts.\n", + "> With POC mode, it is ideally to use terminal instead of notebook for setup, as there are some inconsistent behaviors when running scripts.\n", "\n", "### POC Prepare\n", "\n", @@ -118,20 +118,21 @@ "```\n", "\n", "\n", - "Notice the command create a working directory at ```/tmp/nvflare/poc/example_project/prod_00```\n", + "Notice the command creates a working directory at ```/tmp/nvflare/poc/example_project/prod_00```\n", "\n", - "based on the default values: \n", "\n", - "POC workspace = \"/tmp/nvflare/poc\"\n", - "project_name = \"example_project\"\n", - "and default folder \"prod_00\"\n", + "based on the default values:\n", "\n", - "All sites are with the default name with \"site-\". \n", + "* POC workspace = \"/tmp/nvflare/poc\"\n", + "* project_name = \"example_project\"\n", + "* and default folder \"prod_00\"\n", "\n", - "The actual process of generate such software package (startup kit) is called **\"provision\"**, the POC is special mode of provision, where both client and server are at localhost\n", "\n", + "All sites are with default names starting with \"site-\".\n", "\n", - "Each site ( site-N and server) representing one location in the federated learning site and running federated learning client. \n", + "The actual process of generating such software packages (startup kit) is called **\"provision\"**. The POC is a special mode of provision, where both client and server are at localhost.\n", + "\n", + "Each site (site-N and server) represents one location in the federated learning system and runs a federated learning client.\n", "\n", "\n", "#### Simulating the real-world deployment\n", @@ -243,8 +244,7 @@ "source": [ "#### Prepare with Named Clients\n", "\n", - "If you just want to have default deployment, but specifiy the client site names ((instead of use default site-1,2 etc.) and not writing a project.yaml file, you do the following\n", - "\n", + "If you just want to have a default deployment but specify the client site names (instead of using default site-1,2 etc.) without writing a project.yaml file, you can do the following:\n", "\n", "\n", "nvflare poc prepare -c [CLIENTS ...]" @@ -331,7 +331,7 @@ "\n", "> Note: Using %%bash -bg to run the above command in a code cell may not always work\n", "\n", - "**Homework**: run the nvflare poc start command with or without -ex option\n", + "**Homework**: run the nvflare poc start command with or without the -ex option\n", "\n", "\n", "#### POC start individial site Only \n", diff --git a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.3_interact_with_federated_computing_system/ ways_to_interact_with_fl_system.ipynb b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.3_interact_with_federated_computing_system/ ways_to_interact_with_fl_system.ipynb index e89c853bf8..f3d01cf3b9 100644 --- a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.3_interact_with_federated_computing_system/ ways_to_interact_with_fl_system.ipynb +++ b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.3_interact_with_federated_computing_system/ ways_to_interact_with_fl_system.ipynb @@ -81,7 +81,7 @@ "source": [ "Now start a FLARE system in POC mode\n", "\n", - "And use a terminal to start the POC withut admin console.\n", + "And use a terminal to start the POC without admin console.\n", "\n", "```nvflare poc start -ex admin@nvidia.com```\n", "\n", @@ -111,7 +111,7 @@ "metadata": {}, "source": [ "\n", - "You can submit job, list jobs and check results, check status of sites, list jobs, abort jobs\n" + "You can submit jobs, list jobs and check results, check status of sites, and abort jobs " ] }, { @@ -127,7 +127,8 @@ "id": "95bce19f", "metadata": {}, "source": [ - "Another way to interact with FLARE system is using FLARE python APIs. These APIs have the equivallent functions of the Admin Commands. And they can be issued directly from notebooks. \n", + "Another way to interact with FLARE system is using FLARE python APIs. These APIs have the equivalent functions of the Admin Commands. And they can be issued directly from notebooks. \n", + "\n", "\n", "Let's take a look how this can be done. \n", "\n", @@ -178,7 +179,7 @@ "id": "22612078", "metadata": {}, "source": [ - "In the terminal, you should see the training output, but here we like to use API to monitoring the job \n", + "In the terminal, you should see the training output, but here we would like to use API to monitor the job \n", "\n", "#### Monitor job\n", "The command ```monitor_job()``` allows you to follow a job until the job is done.\n", @@ -397,9 +398,9 @@ "id": "a216ce5d", "metadata": {}, "source": [ - "So far, we have learnt three different ways to interact with FLARE system. Although we used POC model to simulate the real deployment. In production, the same interaction commands can be used in production setup\n", + "So far, we have learned three different ways to interact with FLARE system. Although we used POC mode to simulate the real deployment, in production, the same interaction commands can be used in production setup.\n", "\n", - "Next, lets see how do we [monitoring FLARE system](../03.4_system_monitoring/system_monitorinig.ipynb)\n", + "Next, lets see how do we [monitor FLARE system](../03.4_system_monitoring/system_monitorinig.ipynb)\n", "\n" ] }, diff --git a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.4_system_monitoring/job_example.ipynb b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.4_system_monitoring/job_example.ipynb index e2fc01868f..0469518e11 100644 --- a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.4_system_monitoring/job_example.ipynb +++ b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.4_system_monitoring/job_example.ipynb @@ -5,8 +5,10 @@ "metadata": {}, "source": [ "# FLARE Monitoring\n", - "FLARE Monitoring provides a initial solution for tracking system metrics of your federated learning jobs.\n", - "Different from Machine learning experiment tracking, where it focused on the training metrics, the monitoring here focused on the FL system: i.e. job and system lifecycle metrics. \n", + "\n", + "FLARE Monitoring provides an initial solution for tracking system metrics of your federated learning jobs.\n", + "Unlike machine learning experiment tracking, which focuses on training metrics, this monitoring focuses on the FL system: i.e., job and system lifecycle metrics.\n", + "\n", "\n", "This guide will walk you through the steps to set up and use the monitoring system effectively.\n", "\n", @@ -97,7 +99,7 @@ "\n", "![setup-1](./figures/setup-1.png)\n", "\n", - "As described in the [system monitorinig introduction](./system_monitorinig.ipynb), we will make different component configurations depending on the setups.\n", + "As described in the [system monitoring introduction](./system_monitorinig.ipynb), we will create different component configurations depending on the setups.\n", "\n", "In this setup, all sites (server and clients) will share the same monitoring system with the same host and port.\n", "\n", @@ -133,27 +135,25 @@ "\n", "#### System Metrics Monitoring Configuration\n", "\n", - "We need to manually edit the configuration files for System Metrics collections.\n", + "We need to manually edit the configuration files for System Metrics collection.\n", "\n", - "For example, we need to add server to include \n", + "For example, we need to add to the server:\n", "\n", "* system metrics collector \n", "* statsd reporter\n", "\n", - "In the default POC setup, these components are added to \n", + "In the default POC setup, these components are added to:\n", "\"/tmp/nvflare/poc/example_project/prod_00/server/local/resources.json\"\n", "\n", - "For Client sides, we need to add \n", + "For client sides, we need to add:\n", "\n", "* system metrics collector \n", "* statsd reporter\n", "\n", - "\"/tmp/nvflare/poc/example_project/prod_00//local/resources.json\"\n", - "\n", + "to \"/tmp/nvflare/poc/example_project/prod_00//local/resources.json\"\n", "for the default POC setup.\n", "\n", - "\n", - "Instead of manually, go through each file, we wrote a small python program to do this: \n", + "Instead of manually going through each file, we wrote a small Python program to do this:\n", "\n", "```bash\n", "cd setup-1\n", @@ -202,11 +202,10 @@ "nvflare job submit -j /tmp/nvflare/jobs/job_config/fedavg\n", "```\n", "\n", - "\n", "## Monitoring View\n", "\n", - "Once you setup the system, you can view from the followingt website\n", - "for statsd-exporter, you can look at \n", + "Once you set up the system, you can view from the following websites.\n", + "For statsd-exporter, you can look at:\n", "\n", "### Statsd-exporter metrics view\n", "\n", @@ -241,15 +240,13 @@ "\n", "\n", "\n", - "## Complete steps\n", - "\n", - "Now, lets go to terminal and following all the steps to do the excersize\n", + "Now, let's go to the terminal and follow all the steps to do the exercise:\n", "\n", - "* install dependencies \n", + "* Install dependencies:\n", "\n", - " ```\n", - " pip install -r jobs/requirements.txt\n", - " \n", + " ```bash\n", + " \n", + " pip install -r jobs/requirements.txt\n", " ```\n", "\n", "* start monitoring systems (statsD, prometheus and grafana)\n", diff --git a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.4_system_monitoring/system_monitorinig.ipynb b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.4_system_monitoring/system_monitorinig.ipynb index 13f8b85f0b..b9dc31430c 100644 --- a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.4_system_monitoring/system_monitorinig.ipynb +++ b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.4_system_monitoring/system_monitorinig.ipynb @@ -15,8 +15,9 @@ "source": [ "# NVFLARE System Metrics Monitoring\n", "\n", - "FLARE Monitoring provides a initial solution for tracking system metrics of your federated learning jobs.\n", - "Different from Machine learning experiment tracking, where it focused on the training metrics, the monitoring here focused on the FL system: i.e. job and system lifecycle metrics.\n", + "FLARE Monitoring provides an initial solution for tracking system metrics of your federated learning jobs.\n", + "Different from Machine learning experiment tracking, where it focuses on the training metrics, the monitoring here focuses on the FL system: i.e. job and system lifecycle metrics.\n", + "\n", "\n", "This guide describes how to set up NVFLARE metrics publishing to StatsD Exporter, which then can be scraped by Prometheus and visualized with Grafana.\n", "\n", @@ -35,7 +36,7 @@ "4. **Set up Grafana** to visualize the metrics from Prometheus.\n", "\n", "> side notes: \n", - " don't confuse statsd-exporter and statsd-reporter. statsd-exporter is a libary (such as from datadog) used to receive metrics and for prometheus to scrape from; statsd-reporter is the FLARE component which export metrics to statsd-exporter. \n", + "Don't confuse statsd-exporter and statsd-reporter. statsd-exporter is a library (such as from datadog) used to receive metrics and for prometheus to scrape from; statsd-reporter is the FLARE component which exports metrics to statsd-exporter. \n", "\n", "\n", "\n", @@ -148,7 +149,6 @@ "2. **JobMetricsCollector**: This component collects job-level metrics and publishes them to the databus. It can be added to the workflow components on both client and server sites.\n", "3. **SysMetricsCollector**: This component collects system-level metrics running in the parent process of the server and clients. The metrics will be published to the databus.\n", "4. **RemoteMetricsReceiver**: This component receives the federated metrics streamed from client sides and publish the metriics. \n", - "4. **RemoteMetricsReceiver**: This component receives the federated metrics streamed from client sides and publishes the metrics.\n", "\n", "\n", "### Components Configuration\n", @@ -222,7 +222,6 @@ "}\n", "``` \n", "\n", - "tags can be key, value pair, they are used for group metrics in the report. Here we used \"site\" to indicate origin of the metrics, the \"dev\" env. to indicating the dev environment. \n", "Tags can be key-value pairs used for grouping metrics in the report. Here we used \"site\" to indicate the origin of the metrics and \"env\" to indicate the development environment.\n", "\n", "\n", @@ -234,7 +233,6 @@ "\n", "```//local/resources.json```\n", "\n", - "by rename ```resources.json.default``` to ```resources.json```\n", "by renaming ```resources.json.default``` to ```resources.json```\n", "in ```//local/resources.json```\n", "\n", @@ -266,7 +264,9 @@ "\n", "\n", "#### 2. Clients Forward Metrics to Server Site\n", + "\n", "In this setup, all client-side metrics will not directly post to the StatsD Exporter. Instead, the metrics are streamed to the server site. Therefore, the client side will need the following components:\n", + "\n", "In this setup, all client-side metrics will not be directly posted to the StatsD Exporter. Instead, the metrics are streamed to the server site. Therefore, the client side will need the following components:\n", "- **JobMetricsCollector**\n", "- **SysMetricsCollector**\n", diff --git a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.5_recap/recap.ipynb b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.5_recap/recap.ipynb index 08c8ad0df8..b264d9cf81 100644 --- a/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.5_recap/recap.ipynb +++ b/examples/tutorials/self-paced-training/part-2_federated_learning_system/chapter-3_federated_computing_platform/03.5_recap/recap.ipynb @@ -5,15 +5,25 @@ "id": "7c3c89fa-6355-4d34-8d85-a9447152996f", "metadata": {}, "source": [ - "# Chapter 3 Summary\n", + "# Recap Chapter 3: Federated Computing Platform\n", "\n", + "In this chapter, we explored key components and features of FLARE's federated computing platform:\n", "\n", - "In this chapter, we explored:\n", + "1. System Architecture\n", + " * Core components and their interactions\n", + " * high level system architecture design\n", "\n", - "* FLARE's overall system architecture\n", - "* POC mode to simulate deployment locally\n", - "* Different ways to interact with the FLARE system\n", - "* Monitoring the system with StatsD, Prometheus, and Grafana\n" + "2. POC (Proof of Concept) Mode\n", + " * Local deployment simulation\n", + " \n", + "3. System Interaction Methods\n", + " * Command-line interface (CLI)\n", + " * Python API\n", + " * FLARE Console\n", + "\n", + "4. System Monitoring and Observability\n", + " * System event metrics with StatsD, prometheus and Grafana dashboards\n", + " " ] }, { diff --git a/examples/tutorials/self-paced-training/part-3_security_and_privacy/chapter-6_Security_in_federated_compute_system/06.1_identity_security/identity_security.ipynb b/examples/tutorials/self-paced-training/part-3_security_and_privacy/chapter-6_Security_in_federated_compute_system/06.1_identity_security/identity_security.ipynb index b6e2936d42..9db2e404a4 100644 --- a/examples/tutorials/self-paced-training/part-3_security_and_privacy/chapter-6_Security_in_federated_compute_system/06.1_identity_security/identity_security.ipynb +++ b/examples/tutorials/self-paced-training/part-3_security_and_privacy/chapter-6_Security_in_federated_compute_system/06.1_identity_security/identity_security.ipynb @@ -182,7 +182,7 @@ "\n", "For more details please refer [documentation](https://nvflare.readthedocs.io/en/main/user_guide/security/identity_security.html)\n", "\n", - " \n" + "Now, lets move on to [site_security](../06.2_site_security_privacy_policy/site_policy.ipynb)\n" ] }, { diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.0_introduction/hori_vert.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.0_introduction/hori_vert.png new file mode 100644 index 0000000000..3274ce26ab Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.0_introduction/hori_vert.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.0_introduction/introduction.ipynb b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.0_introduction/introduction.ipynb index 93567ed2aa..e915ba4b8d 100644 --- a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.0_introduction/introduction.ipynb +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.0_introduction/introduction.ipynb @@ -1,9 +1,67 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "cae3fb2d-8949-4e6d-a2af-0282ee973285", + "metadata": {}, + "source": [ + "# Federated Learning for XGBoost \n", + "This chapter demonstrates how to use NVFlare to train an XGBoost model in a federated learning setting. \n", + "Several potential variations of federated XGBoost are illustrated, including:\n", + "- non-secure horizontal collaboration with histogram-based and tree-based mechanisms.\n", + "- non-secure vertical collaboration with histogram-based mechanism.\n", + "- secure horizontal and vertical collaboration with histogram-based mechanism and homomorphic encryption.\n", + "\n", + "Let's first visit the basics of XGBoost and the collaboration modes." + ] + }, + { + "cell_type": "markdown", + "id": "6da8f899-2804-4d6e-ab03-a02ce115d32f", + "metadata": {}, + "source": [ + "## XGBoost \n", + "XGBoost is a machine learning algorithm that uses decision/regression trees to perform classification and regression tasks, \n", + "mapping a vector of feature values to its label prediction. It is especially powerful for tabular data, so even in the age of LLM, \n", + "it is still widely used for many tabular data use cases. It is also preferred for its explainability and efficiency.\n", + "\n", + "In these examples, we use [DMLC XGBoost](https://github.com/dmlc/xgboost), which is an optimized distributed gradient boosting library. \n", + "It offers advanced features like GPU accelerated capabilities, and distributed/federated learning support.\n", + "\n", + "## Collaboration Modes and Data Split\n", + "Essentially there are two collaboration modes: horizontal and vertical:\n", + "![hori_vert](./hori_vert.png)" + ] + }, + { + "cell_type": "markdown", + "id": "64466b4a-6f69-4663-b6f8-14a050a21634", + "metadata": {}, + "source": [ + "- In horizontal case, each participant has access to the same features (columns - \"x_1 x_2\") and label (\"y\") of different data samples (rows - 1/2/3 for Client A v.s. 4/5/6 for Client B). \n", + "In this case, everyone holds equal status as \"label owner\"\n", + "- In vertical case, each client has access to different features (columns - \"x_1 x_2 x_3\" for Client A v.s. \"x_4 x_5\" for Client B) of the same data samples (rows - 1/2/3).\n", + "We assume that only one is the \"label owner\" (or we call it as the \"active party\") - Client B owns label \"y\" \n", + "\n", + "To simulate the above two collaboration modes, we split the dataset both horizontally and vertically, and \n", + "we give site-1 the label column for simplicity.\n", + "\n", + "## Federated Training of XGBoost\n", + "Continue with this chapter for two scenarios:\n", + "### [Federated XGBoost without Encryption](../10.1_fed_xgboost/fed_xgboost.ipynb)\n", + "This section provides instructions for running federated XGBoost without homomorphic encryption, covering both histogram-based and tree-based horizontal collaboration, as well as histogram-based vertical collaboration.\n", + "\n", + "### [Secure Federated XGBoost with Homomorphic Encryption](../10.2_secure_fed_xgboost/secure_fed_xgboost.ipynb)\n", + "This section includes instructions on running secure federated XGBoost with homomorphic encryption under \n", + "histogram-based horizontal and vertical collaboration. Note that as tree-based methods exchange the local trained models (trees), rather than intermediate gradients / histograms, considering that the final model will be made available to all parties at the end of the federated learning, they do not have the same security concerns as histogram-based methods. Therefore under our current setting, we do not consider Homomorphic Encryption for tree-based methods.\n", + "\n", + "We will then finish this chapter with a [recap](../10.3_recap/recap.ipynb)" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "159f19a6-8dc7-4b35-a1ec-7fa327e4239f", + "id": "58cd3833-4dc5-4a53-b188-3db7104b356b", "metadata": {}, "outputs": [], "source": [] @@ -11,9 +69,9 @@ ], "metadata": { "kernelspec": { - "display_name": "nvflare_example", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "nvflare_example" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -25,7 +83,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.2" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/Running_fed_xgboost_applications.ipynb b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/Running_fed_xgboost_applications.ipynb deleted file mode 100644 index f9529bb0bb..0000000000 --- a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/Running_fed_xgboost_applications.ipynb +++ /dev/null @@ -1,33 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "e74b79f7-0541-47ee-bdf7-b9f3423b0094", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nvflare_example", - "language": "python", - "name": "nvflare_example" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/fed_xgboost.ipynb b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/fed_xgboost.ipynb new file mode 100644 index 0000000000..e7ccbd7fda --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/fed_xgboost.ipynb @@ -0,0 +1,456 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d14f9dec-9990-46a8-b4ce-2ff81275454e", + "metadata": {}, + "source": [ + "# Federated XGBoost\n", + "Several mechanisms have been proposed for training an XGBoost model in a federated learning setting.\n", + "In this section, we illustrate the use of NVFlare to carry out *horizontal* federated learning using two approaches: histogram-based collaboration and tree-based collaboration.\n", + "And *vertical* federated learning using histogram-based collaboration." + ] + }, + { + "cell_type": "markdown", + "id": "2fce92a0-79a3-4cf2-adee-bb9c4125613c", + "metadata": {}, + "source": [ + "## Horizontal Federated XGBoost\n", + "Under horizontal setting, each participant joining the federated learning will have part of \n", + "the whole data samples / instances / records, while each sample has all the features.\n", + "\n", + "### Histogram-based Collaboration\n", + "The histogram-based collaboration federated XGBoost approach leverages NVFlare integration of [federated learning support](https://github.com/dmlc/xgboost/issues/7778) in the XGBoost open-source library,\n", + "which allows the existing *distributed* XGBoost training algorithm to operate in a federated manner,\n", + "with the federated clients acting as the distinct workers in the distributed XGBoost algorithm.\n", + "\n", + "In distributed XGBoost, individual workers share and aggregate gradient information about their respective portions of the training data,\n", + "as required to optimize tree node splitting when building the successive boosted trees.\n", + "\n", + "![hori_hist](./figs/hori_hist.png)\n", + "\n", + "The shared information is in the form of quantile sketches of feature values as well as corresponding sample gradient and sample Hessian histograms (\"Local G/H\") , based on which the global information can be computed (\"Global G/H\").\n", + "\n", + "Under federated histogram-based collaboration, information of precisely the same structure is exchanged among the clients.\n", + "The main differences are that the data is partitioned across the workers according to client data ownership, rather than being arbitrarily partionable, and all communication is via an aggregating federated [gRPC](https://grpc.io) server instead of direct client-to-client communication.\n", + "Histograms from different clients, in particular, are aggregated in the server and then communicated back to the clients.\n", + "\n", + "### Tree-based Collaboration\n", + "Under tree-based collaboration, individual trees are independently trained on each client's local data without aggregating the global sample gradient histogram information. \n", + "Trained trees are collected and passed to the server / other clients for aggregation and / or further boosting rounds.\n", + "\n", + "Comparing with histogram-based collaboration, the major difference is that the histogram-based methods exchange the intermediate results for tree-boosting, while tree-based methods exchange the final tree model.\n", + "\n", + "Under this setting, we can further distinguish between two types of tree-based collaboration: cyclic and bagging.\n", + "\n", + "#### Cyclic Training\n", + "\"Cyclic XGBoost\" is one way of performing tree-based federated boosting with \n", + "multiple sites: \n", + "\n", + "![hori_cyclic](./figs/cyclic.png)\n", + "\n", + "At each round of tree boosting, instead of relying on the whole \n", + "data statistics collected from all clients, the boosting relies on only one client's \n", + "local data. The resulting tree sequence is then forwarded to the next client for \n", + "next round's boosting. One full \"cycle\" will be complete when all clients have been covered.\n", + "\n", + "#### Bagging Aggregation\n", + "\n", + "\"Bagging XGBoost\" is another way of performing tree-based federated boosting with multiple sites: \n", + "\n", + "![hori_cyclic](./figs/tree.png)\n", + "\n", + "At each round of tree boosting, all sites start from the same \"global model\", and boost a number of trees (in current example, 1 tree) based on their local data. The resulting trees are then send to server. A bagging aggregation scheme is applied to all the submitted trees to update the global model, which is further distributed to all clients for next round's boosting. \n", + "\n", + "This scheme bears certain similarity to the [Random Forest mode](https://xgboost.readthedocs.io/en/stable/tutorials/rf.html) of XGBoost, where a `num_parallel_tree` is boosted based on random row/col splits, rather than a single tree. Under federated learning setting, such split is fixed to clients rather than random and without column subsampling. \n", + "\n", + "In addition to basic uniform shrinkage setting where all clients have the same learning rate, based on our research, we enabled scaled shrinkage across clients for weighted aggregation according to each client's data size, which is shown to significantly improve the model's performance on non-uniform quantity splits.\n", + "\n", + "Specifically, the global model is updated by aggregating the trees from all clients as a forest, and the global model is then broadcasted back to all clients for local prediction and further training.\n", + "\n", + "The XGBoost Booster API is leveraged to create in-memory Booster objects that persist across rounds to cache predictions from trees added in previous rounds and retain other data structures needed for training." + ] + }, + { + "cell_type": "markdown", + "id": "851e4197-db50-4054-84da-4faf4129e22c", + "metadata": {}, + "source": [ + "## Vertical Federated XGBoost\n", + "Under vertical setting, each participant joining the federated learning will \n", + "have part of the whole features, while each site has all the overlapping instances.\n", + "\n", + "### Private Set Intersection (PSI)\n", + "In this tutorial, we assume that all parties hold the same population but different features. \n", + "\n", + "In reality, however, not every site will have the same set of data samples (rows), ad we shall use PSI to first compare encrypted versions of the sites' datasets in order to jointly compute the intersection based on common IDs. To learn more about our PSI protocol implementation, see our [psi example](https://github.com/NVIDIA/NVFlare/tree/main/examples/advanced/psi/README.md).\n", + "\n", + "### Histogram-based Collaboration\n", + "Similar to its horizontal counterpart, under vertical collaboration, the gradients for each sample will be first computed with label information by the active party; then the gradients will be broadcasted to all passive parties, where they will be used to compute local feature histograms, and find the local best splits with their corresponding gain values; at the last stage, all local best splits will be synced to find the global best split, with which the next split of the tree can be determined. \n", + "\n", + "By exchanging gradient and split information among all sites and update the global model accordingly, vertical histogram-based method can result in the exact same model as the centralized training. \n", + "\n", + "![vert_hist](./figs/vert_hist.png)\n", + "\n", + "We leverage the [vertical federated learning support](https://github.com/dmlc/xgboost/issues/8424) in the XGBoost open-source library. This allows for the distributed XGBoost algorithm to operate in a federated manner on vertically split data." + ] + }, + { + "cell_type": "markdown", + "id": "f942f5b9-bd00-4536-8c9c-ae566e303001", + "metadata": {}, + "source": [ + "## Setup\n", + "Install required packages for data download and training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85d420de-1a0d-4964-bb1d-a79531c660ac", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r requirements.txt" + ] + }, + { + "cell_type": "markdown", + "id": "a8718f40-04e8-42e9-85f1-e256d00f4a5c", + "metadata": {}, + "source": [ + "## Data Preparation\n", + "Download and Store Data\n", + "To run the examples, we first download the dataset and stored in /tmp/nvflare/dataset/creditcard.csv with the following:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0e134c6-26b0-4535-bdc0-0be1801efd4e", + "metadata": {}, + "outputs": [], + "source": [ + "import kagglehub\n", + "path = kagglehub.dataset_download(\"mlg-ulb/creditcardfraud\")\n", + "! mkdir -p /tmp/nvflare/dataset/\n", + "! cp {path}/creditcard.csv /tmp/nvflare/dataset/" + ] + }, + { + "cell_type": "markdown", + "id": "14be5301-ba4c-4139-9749-01064b45fd0d", + "metadata": {}, + "source": [ + "### Data Split\n", + "To prepare data for further experiments, we perform the following steps:\n", + "1. Split the dataset into training/validation and testing sets. \n", + "2. Split the training/validation set: \n", + " * Into \"train\" and \"valid\" for baseline centralized training.\n", + " * Into \"train\" and \"valid\" for each client under horizontal setting. \n", + " * Into \"train\" and \"valid\" for each client under vertical setting.\n", + "\n", + "Data splits used in this example can be generated with" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10141e9b-b060-4ae6-9c43-303ea3f5ef3f", + "metadata": {}, + "outputs": [], + "source": [ + "! bash prepare_data.sh" + ] + }, + { + "cell_type": "markdown", + "id": "317fab1c-f229-4ad3-a263-5ca0974d9245", + "metadata": {}, + "source": [ + "This will generate data splits for 3 clients under all experimental settings.\n", + "\n", + "From the prints, we can see we have in total `182276` rows (data samples) for training, each with `31` columns (30 features + 1 label) \n", + "\n", + "For vertical splits, site-wise column assignments are: \n", + "- site-1 split cols [0:12]\n", + "- site-2 split cols [12:21]\n", + "- site-3 split cols [21:31]\n", + "\n", + "For horizontal splits, site-wise row assignments are:\n", + "- site-1 split rows [0:60758]\n", + "- site-2 split rows [60758:121516]\n", + "- site-3 split rows [121516:182276]\n", + "\n", + "> **_NOTE:_** In this section, we have divided the dataset into separate columns for each site,\n", + "> assuming that the datasets from different sites have already been joined using Private Set\n", + "> Intersection (PSI). In practice, each site initially has its own separate dataset. To\n", + "> combine these datasets accurately, PSI is needed to match records with the same ID across\n", + "> different sites. \n", + "\n", + "> **_NOTE:_** The generated data files will be stored in the folder `/tmp/nvflare/dataset/xgb_dataset/`" + ] + }, + { + "cell_type": "markdown", + "id": "7fd0b0d8-2e38-498a-becc-d34ce4b57229", + "metadata": {}, + "source": [ + "## Experiments\n", + "We first run the centralized trainings to get the baseline performance, then run the federated XGBoost training using NVFlare Simulator via [JobAPI](https://nvflare.readthedocs.io/en/main/programming_guide/fed_job_api.html).\n", + "\n", + "### Centralized Baseline\n", + "For centralize training, we train the XGBoost model on the whole dataset.\n", + "\n", + "Let's first examining the data used for centralized baseline:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a56fee6-1659-4cb8-8273-4b1e63dcf491", + "metadata": {}, + "outputs": [], + "source": [ + "!tree /tmp/nvflare/dataset/xgb_dataset/base_xgb_data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c21892e7-b481-41ba-9470-001c86f0b4ab", + "metadata": {}, + "outputs": [], + "source": [ + "import csv\n", + "\n", + "def print_first_rows_csv(file_path, num_rows=1):\n", + " with open(file_path, 'r') as file:\n", + " csv_reader = csv.reader(file)\n", + " for i, row in enumerate(csv_reader):\n", + " if i >= num_rows:\n", + " break\n", + " print(','.join(row))\n", + "\n", + "file_path = '/tmp/nvflare/dataset/xgb_dataset/base_xgb_data/train.csv'\n", + "print_first_rows_csv(file_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b64a449-45f2-45af-8f7f-161a767a2152", + "metadata": {}, + "outputs": [], + "source": [ + "! python train_base.py " + ] + }, + { + "cell_type": "markdown", + "id": "2ffb6344-3bed-4089-a546-1818588f5751", + "metadata": {}, + "source": [ + "The results by default will be stored in the folder `/tmp/nvflare/workspace/fedxgb/train_base/`." + ] + }, + { + "cell_type": "markdown", + "id": "6a0c9c8a-c90f-4fcb-b3ed-601e79f7f1f9", + "metadata": {}, + "source": [ + "### Horizontal Experiments\n", + "Let's take a look at the dataset for horizontal experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5645c9d6-5711-4a3c-bf1b-f777b0460831", + "metadata": {}, + "outputs": [], + "source": [ + "!tree /tmp/nvflare/dataset/xgb_dataset/horizontal_xgb_data/" + ] + }, + { + "cell_type": "markdown", + "id": "739b0cb0-2760-4d3f-85e9-382470ffa04c", + "metadata": {}, + "source": [ + "First row of site-1 data, should be identical to the first row of baseline data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c07cdf1-fda6-47c3-aa31-e963a20a9a72", + "metadata": {}, + "outputs": [], + "source": [ + "file_path = '/tmp/nvflare/dataset/xgb_dataset/horizontal_xgb_data/site-1/train.csv'\n", + "print_first_rows_csv(file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "7400fd5e-c094-4ae3-9f00-7c10aee71d78", + "metadata": {}, + "source": [ + "The following cases will be covered:\n", + "- Histogram-based collaboration\n", + "- Tree-based collaboration with cyclic training \n", + "- Tree-based collaboration with bagging training \n", + "\n", + "The experiments can be run with:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a6189d1-285d-4473-9f9f-71b40c794c19", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "! python xgb_fl_job.py --training_algo histogram --data_split_mode horizontal\n", + "! python xgb_fl_job.py --training_algo cyclic --data_split_mode horizontal\n", + "! python xgb_fl_job.py --training_algo bagging --data_split_mode horizontal" + ] + }, + { + "cell_type": "markdown", + "id": "e13d0737-80fd-4a7e-8349-223fcd7badf8", + "metadata": {}, + "source": [ + "### Vertical Experiment\n", + "Let's take a look at the dataset for vertical experiments:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "400c3500-98e1-44f1-9d17-72daeb213499", + "metadata": {}, + "outputs": [], + "source": [ + "!tree /tmp/nvflare/dataset/xgb_dataset/vertical_xgb_data/" + ] + }, + { + "cell_type": "markdown", + "id": "f5243e40-6fe1-483b-b6dd-3104bd03ba2d", + "metadata": {}, + "source": [ + "First row of site-1/2/3 data combined together, should be identical to the first row of baseline data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6553a564-4fe9-413b-b8e9-07f15dc806e7", + "metadata": {}, + "outputs": [], + "source": [ + "file_path = '/tmp/nvflare/dataset/xgb_dataset/vertical_xgb_data/site-1/train.csv'\n", + "print_first_rows_csv(file_path)\n", + "file_path = '/tmp/nvflare/dataset/xgb_dataset/vertical_xgb_data/site-2/train.csv'\n", + "print_first_rows_csv(file_path)\n", + "file_path = '/tmp/nvflare/dataset/xgb_dataset/vertical_xgb_data/site-3/train.csv'\n", + "print_first_rows_csv(file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "818d9f5d-e0f1-4d6c-881a-ea44d9712f14", + "metadata": {}, + "source": [ + "Histogram-based collaboration will be performed for vertical setting:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c38ad75c-dccf-44c9-889b-758225a26d08", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "! python xgb_fl_job.py --training_algo histogram --data_split_mode vertical" + ] + }, + { + "cell_type": "markdown", + "id": "8017a2c8-5e4b-4d3b-995d-4226b6845f59", + "metadata": {}, + "source": [ + "## Results\n", + "We can visualize the results via tensorboard records:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6611189-245c-4a1a-b182-0d080b8e094a", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext tensorboard\n", + "%tensorboard --logdir /tmp/nvflare/workspace/fedxgb/works" + ] + }, + { + "cell_type": "markdown", + "id": "45684795-ce6c-466f-ad2a-86146f8247ea", + "metadata": {}, + "source": [ + "For reference, the training curves for the four settings are below:\n", + "\n", + "![training_curves](./figs/training_curves.png)\n", + "\n", + "As shown, for this task, histogram-based methods, both vertical and horizontal, result in almost identical curves, and achieve better results as compared with bagging / cyclic.\n", + "Bagging and cyclic also converge to same training accuracy at the end of training. \n", + "\n", + "Also as expected, vertical histogram-based method achieves identical performance as baseline training." + ] + }, + { + "cell_type": "markdown", + "id": "8f38ab06-2fdc-4dfe-947c-83397b6db8e2", + "metadata": {}, + "source": [ + "Now let's move on to next section [Secure Federated XGBoost with Homomorphic Encryption](../10.2_secure_fed_xgboost/secure_fed_xgboost.ipynb) to see how to protect data privacy during histogram-based collaborations with federated learning and encryption" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8e5d474-c4be-4d99-8881-3fac805ae82c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/fed_xgboost_introduction.ipynb b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/fed_xgboost_introduction.ipynb deleted file mode 100644 index 2ad53ed423..0000000000 --- a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/fed_xgboost_introduction.ipynb +++ /dev/null @@ -1,33 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "39b5b0cf-ea5d-4eaa-b028-a5e6110a217e", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nvflare_example", - "language": "python", - "name": "nvflare_example" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/cyclic.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/cyclic.png new file mode 100644 index 0000000000..53db525101 Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/cyclic.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/hori_hist.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/hori_hist.png new file mode 100644 index 0000000000..28763ebc45 Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/hori_hist.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/training_curves.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/training_curves.png new file mode 100644 index 0000000000..1ead6f97ce Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/training_curves.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/tree.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/tree.png new file mode 100644 index 0000000000..cab87a94dc Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/tree.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/vert_hist.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/vert_hist.png new file mode 100644 index 0000000000..430e243be4 Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/figs/vert_hist.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/prepare_data.sh b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/prepare_data.sh new file mode 100644 index 0000000000..29e6ed5962 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/prepare_data.sh @@ -0,0 +1,35 @@ +DATASET_PATH="/tmp/nvflare/dataset/creditcard.csv" +SPLIT_PATH="/tmp/nvflare/dataset/xgb_dataset/" + +if [ ! -f "${DATASET_PATH}" ] +then + echo "Please check if you saved CreditCard dataset in ${DATASET_PATH}" +fi + +echo "Generating CreditCard data splits, reading from ${DATASET_PATH}" + +echo "Split data to training/validation v.s. testing" +python3 utils/prepare_data_traintest_split.py \ +--data_path "${DATASET_PATH}" \ +--test_ratio 0.2 \ +--out_folder "${SPLIT_PATH}" + +echo "Split training/validation data" +OUTPUT_PATH="${SPLIT_PATH}/base_xgb_data" +python3 utils/prepare_data_base.py \ +--data_path "${SPLIT_PATH}/train.csv" \ +--out_path "${OUTPUT_PATH}" + +echo "Split training/validation data vertically" +OUTPUT_PATH="${SPLIT_PATH}/vertical_xgb_data" +python3 utils/prepare_data_vertical.py \ +--data_path "${SPLIT_PATH}/train.csv" \ +--site_num 3 \ +--out_path "${OUTPUT_PATH}" + +echo "Split training/validation data horizontally" +OUTPUT_PATH="${SPLIT_PATH}/horizontal_xgb_data" +python3 utils/prepare_data_horizontal.py \ +--data_path "${SPLIT_PATH}/train.csv" \ +--site_num 3 \ +--out_path "${OUTPUT_PATH}" diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/requirements.txt b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/requirements.txt new file mode 100644 index 0000000000..38ed8c98d9 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/requirements.txt @@ -0,0 +1,9 @@ +pandas +torch +scikit-learn +tensorboard +kagglehub +shap +matplotlib +# require xgboost 2.2 version, for now need to install a nightly build +https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/federated-secure/xgboost-2.2.0.dev0%2B4601688195708f7c31fcceeb0e0ac735e7311e61-py3-none-manylinux_2_28_x86_64.whl \ No newline at end of file diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/train_base.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/train_base.py new file mode 100644 index 0000000000..cd15b31bdc --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/train_base.py @@ -0,0 +1,153 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os + +import matplotlib.pyplot as plt +import pandas as pd +import shap +import xgboost as xgb + +PRINT_SAMPLE = False + + +def train_base_args_parser(): + parser = argparse.ArgumentParser(description="Train baseline XGBoost model") + parser.add_argument("--gpu", type=int, default=0, help="Whether to use gpu for training, 0 for cpu, 1 for gpu") + parser.add_argument( + "--data_train_root", + type=str, + default="/tmp/nvflare/dataset/xgb_dataset/base_xgb_data", + help="Path to training data folder", + ) + parser.add_argument( + "--data_test_file", + type=str, + default="/tmp/nvflare/dataset/xgb_dataset/test.csv", + help="Path to testing data file", + ) + parser.add_argument( + "--out_path", + type=str, + default="/tmp/nvflare/workspace/fedxgb/train_base", + help="Output path for the data split file", + ) + return parser + + +def load_test_data(data_path: str): + df = pd.read_csv(data_path) + # Split to feature and label + X = df.iloc[:, 1:] + y = df.iloc[:, 0] + return X, y + + +def main(): + parser = train_base_args_parser() + args = parser.parse_args() + if not os.path.exists(args.out_path): + os.makedirs(args.out_path) + + # Specify file path, rank 0 as the label owner, others as the feature owner + train_path = f"{args.data_train_root}/train.csv" + valid_path = f"{args.data_train_root}/valid.csv" + + # Load file directly to tell the match from loading with DMatrix + df_train = pd.read_csv(train_path, header=None) + if PRINT_SAMPLE: + # print number of rows and columns for each worker + print(f"Direct load: nrow={df_train.shape[0]}, ncol={df_train.shape[1]}") + # print one sample row of the data + print(f"Direct load: one sample row of the data: \n {df_train.iloc[0]}") + + # Load file, file will not be sharded in federated mode. + label = "&label_column=0" + # for Vertical XGBoost, read from csv with label_column and set data_split_mode to 1 for column mode + dtrain = xgb.DMatrix(train_path + f"?format=csv{label}") + dvalid = xgb.DMatrix(valid_path + f"?format=csv{label}") + + if PRINT_SAMPLE: + # print number of rows and columns for each worker + print(f"DMatrix: nrow={dtrain.num_row()}, ncol={dtrain.num_col()}") + # print one sample row of the data + data_sample = dtrain.get_data()[0] + print(f"DMatrix: one sample row of the data: \n {data_sample}") + + # Specify parameters via map, definition are same as c++ version + if args.gpu: + device = "cuda:0" + else: + device = "cpu" + param = { + "max_depth": 3, + "eta": 0.1, + "objective": "binary:logistic", + "eval_metric": "auc", + "tree_method": "hist", + "device": device, + "nthread": 1, + } + + # Specify validations set to watch performance + watchlist = [(dvalid, "eval"), (dtrain, "train")] + num_round = 30 + + # Run training, all the features in training API is available. + bst = xgb.train(param, dtrain, num_round, evals=watchlist) + + # Save the model + bst.save_model(f"{args.out_path}/model.base.json") + xgb.collective.communicator_print("Finished training\n") + + # save feature importance score to file + score = bst.get_score(importance_type="gain") + with open(f"{args.out_path}/feat_importance.base.txt", "w") as f: + for key in score: + f.write(f"{key}: {score[key]}\n") + + # Load test data + X_test, y_test = load_test_data(args.data_test_file) + # construct xgboost DMatrix + dmat_test = xgb.DMatrix(X_test, label=y_test) + + # Explain the model + explainer = shap.TreeExplainer(bst) + explanation = explainer(dmat_test) + + # save the beeswarm plot to png file + shap.plots.beeswarm(explanation, show=False) + img = plt.gcf() + img.savefig(f"{args.out_path}/shap.base.png") + + # dump tree and save to text file + dump = bst.get_dump() + with open(f"{args.out_path}/tree_dump.base.txt", "w") as f: + for tree in dump: + f.write(tree) + + # plot tree and save to png file + xgb.plot_tree(bst, num_trees=0, rankdir="LR") + fig = plt.gcf() + fig.set_size_inches(18, 5) + plt.savefig(f"{args.out_path}/tree.base.png", dpi=100) + + # export tree to dataframe + tree_df = bst.trees_to_dataframe() + tree_df.to_csv(f"{args.out_path}/tree_df.base.csv") + + +if __name__ == "__main__": + main() diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_base.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_base.py new file mode 100644 index 0000000000..a0f452b57f --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_base.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import shutil + +import numpy as np +import pandas as pd + + +def data_split_args_parser(): + parser = argparse.ArgumentParser(description="Generate data split for dataset") + parser.add_argument("--data_path", type=str, help="Path to data file") + parser.add_argument( + "--out_path", + type=str, + default="./dataset", + help="Output path for the data split file", + ) + return parser + + +def split_num_proportion(n, site_num): + split = [] + ratio_vec = np.ones(site_num) + total = sum(ratio_vec) + left = n + for site in range(site_num - 1): + x = int(n * ratio_vec[site] / total) + left = left - x + split.append(x) + split.append(left) + return split + + +def main(): + parser = data_split_args_parser() + args = parser.parse_args() + + df = pd.read_csv(args.data_path, header=None) + + rows_total, cols_total = df.shape[0], df.shape[1] + + print(f"rows_total: {rows_total}, cols_total: {cols_total}") + + if os.path.exists(args.out_path): + shutil.rmtree(args.out_path) + + os.makedirs(args.out_path, exist_ok=True) + + # assign first 80% rows to train + df_train = df.iloc[: int(0.8 * df.shape[0]), :] + # assign last 20% rows to valid + df_valid = df.iloc[int(0.8 * df.shape[0]) :, :] + # save train and valid data + df_train.to_csv(path_or_buf=os.path.join(args.out_path, "train.csv"), index=False, header=False) + df_valid.to_csv(path_or_buf=os.path.join(args.out_path, "valid.csv"), index=False, header=False) + + +if __name__ == "__main__": + main() diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_horizontal.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_horizontal.py new file mode 100644 index 0000000000..92a2c88615 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_horizontal.py @@ -0,0 +1,89 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import shutil + +import numpy as np +import pandas as pd + + +def data_split_args_parser(): + parser = argparse.ArgumentParser(description="Generate data split for dataset") + parser.add_argument("--data_path", type=str, help="Path to data file") + parser.add_argument("--site_num", type=int, default=3, help="Total number of sites") + parser.add_argument( + "--out_path", + type=str, + default="./dataset", + help="Output path for the data split file", + ) + return parser + + +def split_num_proportion(n, site_num): + split = [] + ratio_vec = np.ones(site_num) + total = sum(ratio_vec) + left = n + for site in range(site_num - 1): + x = int(n * ratio_vec[site] / total) + left = left - x + split.append(x) + split.append(left) + return split + + +def main(): + parser = data_split_args_parser() + args = parser.parse_args() + + df = pd.read_csv(args.data_path, header=None) + + rows_total, cols_total = df.shape[0], df.shape[1] + + print(f"site_num: {args.site_num}") + print(f"rows_total: {rows_total}, cols_total: {cols_total}") + + # split row + site_row_size = split_num_proportion(int(0.8 * rows_total), args.site_num) + print(f"site_row_size: {site_row_size}") + + if os.path.exists(args.out_path): + shutil.rmtree(args.out_path) + + # assign first 80% rows to train + df_train = df.iloc[: int(0.8 * rows_total), :] + # assign last 20% rows to valid + df_valid = df.iloc[int(0.8 * rows_total) :, :] + + for site in range(args.site_num): + row_start = sum(site_row_size[:site]) + row_end = sum(site_row_size[: site + 1]) + + df_split = df_train.iloc[row_start:row_end, :] + print(f"site-{site + 1} split rows [{row_start}:{row_end}]") + + data_path = os.path.join(args.out_path, f"site-{site + 1}") + if not os.path.exists(data_path): + os.makedirs(data_path, exist_ok=True) + + # save train and valid data + df_split.to_csv(path_or_buf=os.path.join(data_path, "train.csv"), index=False, header=False) + df_valid.to_csv(path_or_buf=os.path.join(data_path, "valid.csv"), index=False, header=False) + + +if __name__ == "__main__": + main() diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_traintest_split.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_traintest_split.py new file mode 100644 index 0000000000..c913d52dd7 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_traintest_split.py @@ -0,0 +1,71 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os + +import pandas as pd +from sklearn.model_selection import train_test_split + + +def data_split_args_parser(): + parser = argparse.ArgumentParser(description="Generate training/testing split for dataset") + parser.add_argument("--data_path", type=str, help="Path to data file") + parser.add_argument("--test_ratio", type=float, help="Ratio of testing set") + parser.add_argument( + "--out_folder", + type=str, + default="~/dataset", + help="Output folder for training/testing data", + ) + return parser + + +def main(): + parser = data_split_args_parser() + args = parser.parse_args() + + df = pd.read_csv(args.data_path) + df_pos = df[df.Class == 1] + df_neg = df[df.Class == 0] + # print the number of positive and negative samples + print("Number of positive samples: ", len(df_pos)) + print("Number of negative samples: ", len(df_neg)) + + # Split data into training and testing sets + X_pos = df_pos.drop(["Class"], axis=1) + y_pos = df_pos.Class + X_neg = df_neg.drop(["Class"], axis=1) + y_neg = df_neg.Class + X = pd.concat([X_pos, X_neg]) + y = pd.concat([y_pos, y_neg]) + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=args.test_ratio, random_state=77) + df_train = pd.concat([y_train, X_train], axis=1) + df_test = pd.concat([y_test, X_test], axis=1) + + # print the number of positive and negative samples in training and testing sets + print("Number of positive samples in training set: ", len(df_train[df_train.Class == 1])) + print("Number of negative samples in training set: ", len(df_train[df_train.Class == 0])) + print("Number of positive samples in testing set: ", len(df_test[df_test.Class == 1])) + print("Number of negative samples in testing set: ", len(df_test[df_test.Class == 0])) + + # Save training and testing sets + if not os.path.exists(args.out_folder): + os.makedirs(args.out_folder, exist_ok=True) + df_train.to_csv(path_or_buf=os.path.join(args.out_folder, "train.csv"), index=False, header=False) + df_test.to_csv(path_or_buf=os.path.join(args.out_folder, "test.csv"), index=False, header=False) + + +if __name__ == "__main__": + main() diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_vertical.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_vertical.py new file mode 100644 index 0000000000..dcf131a829 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/utils/prepare_data_vertical.py @@ -0,0 +1,93 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import shutil + +import numpy as np +import pandas as pd + + +def data_split_args_parser(): + parser = argparse.ArgumentParser(description="Generate data split for dataset") + parser.add_argument("--data_path", type=str, help="Path to data file") + parser.add_argument("--site_num", type=int, default=3, help="Total number of sites") + parser.add_argument( + "--out_path", + type=str, + default="./dataset", + help="Output path for the data split file", + ) + return parser + + +def split_num_proportion(n, site_num): + split = [] + ratio_vec = np.ones(site_num) + total = sum(ratio_vec) + left = n + for site in range(site_num - 1): + x = int(n * ratio_vec[site] / total) + left = left - x + split.append(x) + split.append(left) + return split + + +def main(): + parser = data_split_args_parser() + args = parser.parse_args() + + df = pd.read_csv(args.data_path, header=None) + + rows_total, cols_total = df.shape[0], df.shape[1] + + print(f"site_num: {args.site_num}") + print(f"rows_total: {rows_total}, cols_total: {cols_total}") + + # split col + cols_labelowner = int(cols_total * 0.4) + site_col_size = split_num_proportion(cols_total - cols_labelowner, args.site_num - 1) + site_col_size.insert(0, cols_labelowner) + print(f"site_col_size: {site_col_size}") + + if os.path.exists(args.out_path): + shutil.rmtree(args.out_path) + + for site in range(args.site_num): + col_start = sum(site_col_size[:site]) + col_end = sum(site_col_size[: site + 1]) + + df_split = df.iloc[:, col_start:col_end] + print(f"site-{site + 1} split cols [{col_start}:{col_end}]") + + data_path = os.path.join(args.out_path, f"site-{site + 1}") + if not os.path.exists(data_path): + os.makedirs(data_path, exist_ok=True) + + # assign first 80% rows to train + df_train = df_split.iloc[: int(0.8 * df_split.shape[0]), :] + # assign last 20% rows to valid + df_valid = df_split.iloc[int(0.8 * df_split.shape[0]) :, :] + + print(f"rows_training: {int(0.8 * df_split.shape[0])}") + + # save train and valid data + df_train.to_csv(path_or_buf=os.path.join(data_path, "train.csv"), index=False, header=False) + df_valid.to_csv(path_or_buf=os.path.join(data_path, "valid.csv"), index=False, header=False) + + +if __name__ == "__main__": + main() diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/xgb_fl_job.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/xgb_fl_job.py new file mode 100644 index 0000000000..093ee1c0ca --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.1_fed_xgboost/xgb_fl_job.py @@ -0,0 +1,203 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from nvflare.app_common.widgets.convert_to_fed_event import ConvertToFedEvent +from nvflare.app_opt.tracking.tb.tb_receiver import TBAnalyticsReceiver +from nvflare.app_opt.tracking.tb.tb_writer import TBWriter +from nvflare.app_opt.xgboost.histogram_based_v2.csv_data_loader import CSVDataLoader +from nvflare.job_config.api import FedJob + +ALGO_DIR_MAP = { + "bagging": "tree-based", + "cyclic": "tree-based", + "histogram": "histogram-based", +} + + +def define_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--data_root", + type=str, + default="/tmp/nvflare/dataset/xgb_dataset", + help="Path to dataset files for each site", + ) + parser.add_argument("--site_num", type=int, default=3, help="Total number of sites") + parser.add_argument("--round_num", type=int, default=30, help="Total number of training rounds") + parser.add_argument( + "--training_algo", type=str, default="histogram", choices=list(ALGO_DIR_MAP.keys()), help="Training algorithm" + ) + parser.add_argument("--nthread", type=int, default=16, help="nthread for xgboost") + parser.add_argument( + "--tree_method", type=str, default="hist", help="tree_method for xgboost - use hist for best perf" + ) + parser.add_argument( + "--data_split_mode", + type=str, + default="horizontal", + choices=["horizontal", "vertical"], + help="dataset split mode, horizontal or vertical", + ) + return parser.parse_args() + + +def _get_job_name(args) -> str: + return f"fedxgb_{args.site_num}_sites_{args.data_split_mode}_{args.training_algo}" + + +def _get_data_path(args) -> str: + return f"{args.data_root}/{args.data_split_mode}_xgb_data" + + +def main(): + args = define_parser() + job_name = _get_job_name(args) + dataset_path = _get_data_path(args) + site_num = args.site_num + job = FedJob(name=job_name, min_clients=site_num) + if args.data_split_mode == "horizontal": + data_split_mode = 0 + else: + data_split_mode = 1 + # Define the controller workflow and send to server + if args.training_algo == "histogram": + from nvflare.app_opt.xgboost.histogram_based_v2.fed_controller import XGBFedController + + controller = XGBFedController( + num_rounds=args.round_num, + data_split_mode=data_split_mode, + secure_training=False, + xgb_options={"early_stopping_rounds": 3, "use_gpus": False}, + xgb_params={ + "max_depth": 3, + "eta": 0.1, + "objective": "binary:logistic", + "eval_metric": "auc", + "tree_method": "hist", + "nthread": 1, + }, + ) + + from nvflare.app_opt.xgboost.histogram_based_v2.fed_executor import FedXGBHistogramExecutor + + executor = FedXGBHistogramExecutor( + data_loader_id="dataloader", + metrics_writer_id="metrics_writer", + ) + # Add tensorboard receiver to server + tb_receiver = TBAnalyticsReceiver( + tb_folder="tb_events", + ) + job.to_server(tb_receiver, id="tb_receiver") + elif args.training_algo == "bagging": + from nvflare.app_common.workflows.scatter_and_gather import ScatterAndGather + + controller = ScatterAndGather( + min_clients=args.site_num, + num_rounds=args.round_num, + start_round=0, + aggregator_id="aggregator", + persistor_id="persistor", + shareable_generator_id="shareable_generator", + wait_time_after_min_received=0, + train_timeout=0, + allow_empty_global_weights=True, + task_check_period=0.01, + persist_every_n_rounds=0, + snapshot_every_n_rounds=0, + ) + from nvflare.app_opt.xgboost.tree_based.model_persistor import XGBModelPersistor + + persistor = XGBModelPersistor(save_name="xgboost_model.json") + from nvflare.app_opt.xgboost.tree_based.shareable_generator import XGBModelShareableGenerator + + shareable_generator = XGBModelShareableGenerator() + from nvflare.app_opt.xgboost.tree_based.bagging_aggregator import XGBBaggingAggregator + + aggregator = XGBBaggingAggregator() + job.to_server(persistor, id="persistor") + job.to_server(shareable_generator, id="shareable_generator") + job.to_server(aggregator, id="aggregator") + elif args.training_algo == "cyclic": + from nvflare.app_common.workflows.cyclic_ctl import CyclicController + + controller = CyclicController( + num_rounds=int(args.round_num / args.site_num), + task_assignment_timeout=60, + persistor_id="persistor", + shareable_generator_id="shareable_generator", + task_check_period=0.01, + persist_every_n_rounds=0, + snapshot_every_n_rounds=0, + ) + from nvflare.app_opt.xgboost.tree_based.model_persistor import XGBModelPersistor + + persistor = XGBModelPersistor(save_name="xgboost_model.json", load_as_dict=False) + from nvflare.app_opt.xgboost.tree_based.shareable_generator import XGBModelShareableGenerator + + shareable_generator = XGBModelShareableGenerator() + job.to_server(persistor, id="persistor") + job.to_server(shareable_generator, id="shareable_generator") + # send controller to server + job.to_server(controller, id="xgb_controller") + + # Add executor and other components to clients + for site_id in range(1, site_num + 1): + if args.training_algo in ["bagging", "cyclic"]: + num_client_bagging = 1 + if args.training_algo == "bagging": + num_client_bagging = args.site_num + + from nvflare.app_opt.xgboost.tree_based.executor import FedXGBTreeExecutor + + executor = FedXGBTreeExecutor( + data_loader_id="dataloader", + training_mode=args.training_algo, + num_client_bagging=num_client_bagging, + num_local_parallel_tree=1, + local_subsample=1, + local_model_path="model.json", + global_model_path="model_global.json", + learning_rate=0.1, + objective="binary:logistic", + max_depth=3, + lr_scale=1, + eval_metric="auc", + tree_method="hist", + nthread=1, + ) + job.to(executor, f"site-{site_id}") + + dataloader = CSVDataLoader(folder=dataset_path) + job.to(dataloader, f"site-{site_id}", id="dataloader") + + if args.training_algo in ["histogram"]: + metrics_writer = TBWriter(event_type="analytix_log_stats") + job.to(metrics_writer, f"site-{site_id}", id="metrics_writer") + + event_to_fed = ConvertToFedEvent( + events_to_convert=["analytix_log_stats"], + fed_event_prefix="fed.", + ) + job.to(event_to_fed, f"site-{site_id}", id="event_to_fed") + + # Export job config and run the job + job.export_job("/tmp/nvflare/workspace/fedxgb/jobs/") + job.simulator_run(f"/tmp/nvflare/workspace/fedxgb/works/{job_name}") + + +if __name__ == "__main__": + main() diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/enhancing_fed_xgboost_security.ipynb b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/enhancing_fed_xgboost_security.ipynb deleted file mode 100644 index 758b75c807..0000000000 --- a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/enhancing_fed_xgboost_security.ipynb +++ /dev/null @@ -1,33 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "4793c97e-7897-4e70-860e-0cbce1c33b95", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nvflare_example", - "language": "python", - "name": "nvflare_example" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/secure_hori.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/secure_hori.png new file mode 100644 index 0000000000..6c4ba4d270 Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/secure_hori.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/secure_vert.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/secure_vert.png new file mode 100644 index 0000000000..0ed982476b Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/secure_vert.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.base.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.base.png new file mode 100644 index 0000000000..c72d9c2ab2 Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.base.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.vert.secure.0.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.vert.secure.0.png new file mode 100644 index 0000000000..34b0dc7e8e Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.vert.secure.0.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.vert.secure.1.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.vert.secure.1.png new file mode 100644 index 0000000000..a1d2e7f339 Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.vert.secure.1.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.vert.secure.2.png b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.vert.secure.2.png new file mode 100644 index 0000000000..baea53f63b Binary files /dev/null and b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/figs/tree.vert.secure.2.png differ diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/horizontal_fed_xgboost_with_he.ipynb b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/horizontal_fed_xgboost_with_he.ipynb deleted file mode 100644 index 4a263b78da..0000000000 --- a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/horizontal_fed_xgboost_with_he.ipynb +++ /dev/null @@ -1,33 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "4fdffd32-8f84-4db6-8bde-56484d7fa9c0", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nvflare_example", - "language": "python", - "name": "nvflare_example" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/prepare_data.sh b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/prepare_data.sh new file mode 100644 index 0000000000..29e6ed5962 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/prepare_data.sh @@ -0,0 +1,35 @@ +DATASET_PATH="/tmp/nvflare/dataset/creditcard.csv" +SPLIT_PATH="/tmp/nvflare/dataset/xgb_dataset/" + +if [ ! -f "${DATASET_PATH}" ] +then + echo "Please check if you saved CreditCard dataset in ${DATASET_PATH}" +fi + +echo "Generating CreditCard data splits, reading from ${DATASET_PATH}" + +echo "Split data to training/validation v.s. testing" +python3 utils/prepare_data_traintest_split.py \ +--data_path "${DATASET_PATH}" \ +--test_ratio 0.2 \ +--out_folder "${SPLIT_PATH}" + +echo "Split training/validation data" +OUTPUT_PATH="${SPLIT_PATH}/base_xgb_data" +python3 utils/prepare_data_base.py \ +--data_path "${SPLIT_PATH}/train.csv" \ +--out_path "${OUTPUT_PATH}" + +echo "Split training/validation data vertically" +OUTPUT_PATH="${SPLIT_PATH}/vertical_xgb_data" +python3 utils/prepare_data_vertical.py \ +--data_path "${SPLIT_PATH}/train.csv" \ +--site_num 3 \ +--out_path "${OUTPUT_PATH}" + +echo "Split training/validation data horizontally" +OUTPUT_PATH="${SPLIT_PATH}/horizontal_xgb_data" +python3 utils/prepare_data_horizontal.py \ +--data_path "${SPLIT_PATH}/train.csv" \ +--site_num 3 \ +--out_path "${OUTPUT_PATH}" diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/project.yml b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/project.yml new file mode 100644 index 0000000000..8d7072ec46 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/project.yml @@ -0,0 +1,75 @@ +api_version: 3 +name: example_project +description: NVIDIA FLARE sample project yaml file + +participants: + # change overseer.example.com to the FQDN of the overseer + - name: overseer + type: overseer + org: nvidia + protocol: https + api_root: /api/v1 + port: 8443 + # change example.com to the FQDN of the server + - name: server1 + type: server + org: nvidia + fed_learn_port: 8002 + admin_port: 8003 + - name: site-1 + type: client + org: nvidia + # listening_host will enable creating one pair of cert/private key for this client + # so it can behave like a server for client api. The value must be a hostname that + # client api can reach via network. + # listening_host: site-1-lh + - name: site-2 + type: client + org: nvidia + - name: admin@nvidia.com + type: admin + org: nvidia + role: project_admin + +# The same methods in all builders are called in their order defined in builders section +builders: + - path: nvflare.lighter.impl.workspace.WorkspaceBuilder + args: + template_file: master_template.yml + - path: nvflare.lighter.impl.template.TemplateBuilder + - path: nvflare.lighter.impl.static_file.StaticFileBuilder + args: + # config_folder can be set to inform NVIDIA FLARE where to get configuration + config_folder: config + + # scheme for communication driver (currently supporting the default, grpc, only). + # scheme: grpc + + # app_validator is used to verify if uploaded app has proper structures + # if not set, no app_validator is included in fed_server.json + # app_validator: PATH_TO_YOUR_OWN_APP_VALIDATOR + + # when docker_image is set to a docker image name, docker.sh will be generated on server/client/admin + # docker_image: + + # download_job_url is set to http://download.server.com/ as default in fed_server.json. You can override this + # to different url. + # download_job_url: http://download.server.com/ + + overseer_agent: + path: nvflare.ha.overseer_agent.HttpOverseerAgent + # if overseer_exists is true, args here are ignored. Provisioning + # tool will fill role, name and other local parameters automatically. + # if overseer_exists is false, args in this section will be used. + overseer_exists: true + # args: + # sp_end_point: example1.com.8002:8003 + + - path: nvflare.lighter.impl.cert.CertBuilder + - path: nvflare.lighter.impl.he.HEBuilder + args: + poly_modulus_degree: 8192 + coeff_mod_bit_sizes: [60, 40, 40] + scale_bits: 40 + scheme: CKKS + - path: nvflare.lighter.impl.signature.SignatureBuilder diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/requirements.txt b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/requirements.txt new file mode 100644 index 0000000000..32af1e00dc --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/requirements.txt @@ -0,0 +1,11 @@ +pandas +torch +scikit-learn +tensorboard +kagglehub +shap +matplotlib +tenseal +# require xgboost 2.2 version, for now need to install a nightly build +https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/federated-secure/xgboost-2.2.0.dev0%2B4601688195708f7c31fcceeb0e0ac735e7311e61-py3-none-manylinux_2_28_x86_64.whl +ipcl_python @ git+https://github.com/intel/pailliercryptolib_python.git@development \ No newline at end of file diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/run_training_standalone.sh b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/run_training_standalone.sh new file mode 100644 index 0000000000..164638f252 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/run_training_standalone.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +workspace_root="/tmp/nvflare/workspace/fedxgb_secure/train_standalone/" +if [ ! -e "$workspace_root" ]; then + mkdir -p "$workspace_root" + echo "Directory created: $workspace_root" +else + echo "Directory already exists: $workspace_root" +fi + +dataset_path="/tmp/nvflare/dataset/xgb_dataset/" + +echo "Training baseline CPU" +python3 ./train_standalone/train_base.py --out_path "$workspace_root/base_cpu" --gpu 0 +echo "Training baseline GPU" +python3 ./train_standalone/train_base.py --out_path "$workspace_root/base_gpu" --gpu 1 +echo "Training horizontal CPU non-encrypted" +python3 ./train_standalone/train_federated.py --data_train_root "$dataset_path/horizontal_xgb_data" --out_path "$workspace_root/hori_cpu_non_enc" --vert 0 --gpu 0 --enc 0 +echo "Training horizontal CPU encrypted" +python3 ./train_standalone/train_federated.py --data_train_root "$dataset_path/horizontal_xgb_data" --out_path "$workspace_root/hori_cpu_enc" --vert 0 --gpu 0 --enc 1 +echo "Training horizontal GPU non-encrypted" +python3 ./train_standalone/train_federated.py --data_train_root "$dataset_path/horizontal_xgb_data" --out_path "$workspace_root/hori_gpu_non_enc" --vert 0 --gpu 1 --enc 0 +echo "Training horizontal GPU encrypted" +python3 ./train_standalone/train_federated.py --data_train_root "$dataset_path/horizontal_xgb_data" --out_path "$workspace_root/hori_gpu_enc" --vert 0 --gpu 1 --enc 1 +echo "Training vertical CPU non-encrypted" +python3 ./train_standalone/train_federated.py --data_train_root "$dataset_path/vertical_xgb_data" --out_path "$workspace_root/vert_cpu_non_enc" --vert 1 --gpu 0 --enc 0 +echo "Training vertical CPU encrypted" +python3 ./train_standalone/train_federated.py --data_train_root "$dataset_path/vertical_xgb_data" --out_path "$workspace_root/vert_cpu_enc" --vert 1 --gpu 0 --enc 1 +echo "Training vertical GPU non-encrypted" +python3 ./train_standalone/train_federated.py --data_train_root "$dataset_path/vertical_xgb_data" --out_path "$workspace_root/vert_gpu_non_enc" --vert 1 --gpu 1 --enc 0 +echo "Training vertical GPU encrypted" +python3 ./train_standalone/train_federated.py --data_train_root "$dataset_path/vertical_xgb_data" --out_path "$workspace_root/vert_gpu_enc" --vert 1 --gpu 1 --enc 1 \ No newline at end of file diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/secure_fed_xgboost.ipynb b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/secure_fed_xgboost.ipynb new file mode 100644 index 0000000000..2acbf89b6a --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/secure_fed_xgboost.ipynb @@ -0,0 +1,315 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d8a97688-34ef-4425-9b25-d4c9fb086ad5", + "metadata": {}, + "source": [ + "# Secure Federated XGBoost with Homomorphic Encryption\n", + "This section illustrates the use of [NVIDIA FLARE](https://nvflare.readthedocs.io/en/main/index.html) enabling secure federated [XGBoost](https://github.com/dmlc/xgboost) under both horizontal and vertical collaborations.\n", + "The examples are based on a [finance dataset](https://www.kaggle.com/datasets/mlg-ulb/creditcardfraud) to perform fraud detection.\n", + "\n", + "## Secure Federated Training of XGBoost\n", + "In last section, we visited several mechanisms for training an XGBoost model in a federated learning setting, including histogram-based vertical, histogram-based horizontal, and tree-based horizontal methods. \n", + "\n", + "In this example, we further extend the existing histogram-based horizontal and vertical federated learning approaches to support secure federated learning using homomorphic encryption. Depending on the characteristics of the data to be encrypted, we can choose between [CKKS](https://github.com/OpenMined/TenSEAL) and [Paillier](https://github.com/intel/pailliercryptolib_python).\n", + "\n", + "In the following, we illustrate both *histogram-based* *horizontal* and *vertical* federated XGBoost, *with* homomorphic encryption. We leverage the [vertical federated learning with secure features support](https://github.com/dmlc/xgboost/issues/9987) and [horizontal federated learning with secure features support](https://github.com/dmlc/xgboost/issues/10170) in the XGBoost open-source library.\n", + "\n", + "### Secure Vertical Federated Training of XGBoost\n", + "For vertical XGBoost, the active party holds the label, which can be considered “the most valuable asset” for the whole process, and should not be accessed by passive parties. Therefore, the active party in this case is the “major contributor” from a model training perspective, with a concern of leaking this information to passive clients. In this case, the security protection is mainly against passive clients over the label information. \n", + "\n", + "To protect label information for vertical collaboration, at every round of XGBoost after the active party computes the gradients for each sample, the gradients will be encrypted before sending to passive parties (Figure 1). Upon receiving the encrypted gradients (ciphertext), they will be accumulated according to the specific feature distribution at each passive party. The resulting cumulative histograms will be returned to the active party, decrypted, and further used for tree building by the active party.\n", + "\n", + "![secure_vert_hist](./figs/secure_vert.png)\n", + "\n", + "### Secure Horizontal Federated Training of XGBoost\n", + "For horizontal XGBoost, each party holds “equal status” (whole feature and label for partial population), while the federated server performs aggregation, without owning any data. Hence in this case, clients have a concern of leaking information to the server, and to each other. Hence, the information to be protected is each clients’ local histograms.\n", + "\n", + "To protect the local histograms for horizontal collaboration, the histograms will be encrypted before sending to the federated server for aggregation. The aggregation will then be performed over ciphertexts and the encrypted global histograms will be returned to clients, where they will be decrypted and used for tree building. In this way, the server will have no access to the plaintext histograms, while each client will only learn the global histogram after aggregation, rather than individual local histograms.\n", + "\n", + "![secure_hori_hist](./figs/secure_hori.png)\n", + "\n", + "### Encryption with proper HE schemes\n", + "With multiple libraries covering various HE schemes both with and without GPU support, it is important to properly choose the most efficient scheme for the specific needs of a particular federated XGBoost setting. Let’s look at one example, assume N=5 number of participants, M=200K total number of data samples, J=30 total number of features, and each feature histogram has K=256 slots. Depending on the type of federated learning applications: (Vertical or Horizontal application, we will need different algorithms. \n", + "\n", + "For vertical application, the encryption target is the individual g/h numbers, and the computation is to add the encrypted numbers according to which histogram slots they fall into. As the number of g/h is the same as the sample number, for each boosting round in theory:\n", + "\n", + "The total encryption needed will be M * 2 = 400k (g and h), and each time encrypts a single number\n", + "The total encrypted addition needed will be (M – K) * 2 * J ≈ 12m\n", + "In this case, an optimal scheme choice would be Paillier because the encryption needs to be performed over a single number. Using schemes targeting vectors like CKKS would be a significant waste of space. \n", + "\n", + "For horizontal application, on the other hand, the encryption target is the local histograms G/H, and the computation is to add local histograms together to form the global histogram. For each boosting round:\n", + "\n", + "The total encryption needed will be N * 2 = 10 (G and H), and each time encrypts a vector of length J * K = 7680\n", + "The total encrypted addition needed will be (N – 1) * 2 = 18\n", + "In this case, an optimal scheme choice would be CKKS because it is able to handle a histogram vector (with length 7680, for example) in one shot.\n", + "\n", + "We provide encryption solutions both with CPU-only, and with efficient GPU acceleration. " + ] + }, + { + "cell_type": "markdown", + "id": "39f5d29c-0f4b-484e-99d7-75adf79a996c", + "metadata": {}, + "source": [ + "## Setup\n", + "Install required packages for data download and training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "360512ae-bf5b-4868-893f-1e8df392bc47", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r requirements.txt" + ] + }, + { + "cell_type": "markdown", + "id": "c66f4223-63a2-438c-97cd-c0bfbdd6c442", + "metadata": {}, + "source": [ + "## Encryption Plugins\n", + "The secure XGBoost requires encryption plugins to work. The plugins are distributed with NVFlare package. If you build NVFlare from source, you need\n", + "to build the plugins following the instructions in this [README](https://github.com/NVIDIA/NVFlare/blob/main/integration/xgboost/encryption_plugins/README.md)\n", + "\n", + "The build process will generate 2 .so files: libcuda_paillier.so and libnvflare.so. Configure the path accordingly following the instructions in \n", + "[XGBoost User Guide](https://nvflare.readthedocs.io/en/main/user_guide/federated_xgboost/secure_xgboost_user_guide.html)" + ] + }, + { + "cell_type": "markdown", + "id": "6de13fdf-da09-4672-a1fe-e5653ceb47bc", + "metadata": {}, + "source": [ + "## Data Preparation\n", + "We follow the same data preparation process as regular federated without secure features. Download and Store Data To run the examples, we use the same data as the last section. We download the dataset and stored in /tmp/nvflare/dataset/creditcard.csv with the following:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51e4ed08-fcf2-4774-a802-260ffc74870d", + "metadata": {}, + "outputs": [], + "source": [ + "import kagglehub\n", + "path = kagglehub.dataset_download(\"mlg-ulb/creditcardfraud\")\n", + "! mkdir -p /tmp/nvflare/dataset/\n", + "! cp {path}/creditcard.csv /tmp/nvflare/dataset/" + ] + }, + { + "cell_type": "markdown", + "id": "37c92744-b97b-4745-9efb-d61230c6cacb", + "metadata": {}, + "source": [ + "### Data Split\n", + "To prepare data for further experiments, we perform the following steps:\n", + "1. Split the dataset into training/validation and testing sets. \n", + "2. Split the training/validation set: \n", + " * Into \"train\" and \"valid\" for baseline centralized training.\n", + " * Into \"train\" and \"valid\" for each client under horizontal setting. \n", + " * Into \"train\" and \"valid\" for each client under vertical setting.\n", + "\n", + "Data splits used in this example can be generated with" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62c65eb5-aa3f-4d6f-bb63-97a9653115cd", + "metadata": {}, + "outputs": [], + "source": [ + "! bash prepare_data.sh" + ] + }, + { + "cell_type": "markdown", + "id": "39e955ae-d064-4e69-b8fd-c3e685fd83f8", + "metadata": {}, + "source": [ + "This will generate data splits for 3 clients under all experimental settings.\n", + "\n", + "> **_NOTE:_** In this section, we have divided the dataset into separate columns for each site,\n", + "> assuming that the datasets from different sites have already been joined using Private Set\n", + "> Intersection (PSI). However, in practice, each site initially has its own separate dataset. To\n", + "> combine these datasets accurately, you need to use PSI to match records with the same ID across\n", + "> different sites. \n", + "\n", + "> **_NOTE:_** The generated data files will be stored in the folder `/tmp/nvflare/dataset/xgb_dataset/`" + ] + }, + { + "cell_type": "markdown", + "id": "45a8e32b-22e2-44d0-ae02-e1611eea0c0c", + "metadata": {}, + "source": [ + "## Run Baseline and Standalone Experiments\n", + "First, we run the baseline centralized training and standalone federated XGBoost training for comparison.\n", + "In this case, we utilized the `mock` plugin to simulate the homomorphic encryption process. \n", + "For more details regarding federated XGBoost and the interface-plugin design,\n", + "please refer to our [documentation](https://nvflare.readthedocs.io/en/main/user_guide/federated_xgboost/secure_xgboost_user_guide.html).\n", + "\n", + "To run all experiments, we provide a script for all settings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "be18fcc9-73fc-4c9d-8083-20bb2ea94fbd", + "metadata": {}, + "outputs": [], + "source": [ + "! bash run_training_standalone.sh" + ] + }, + { + "cell_type": "markdown", + "id": "2be6ddce-bf4b-499e-b135-5592c5328967", + "metadata": {}, + "source": [ + "This will cover baseline centralized training, federated xgboost run in the same machine\n", + "(server and clients are running in different processes) with and without secure feature." + ] + }, + { + "cell_type": "markdown", + "id": "feb613fa-045a-4fc5-aa6b-f7a2779619f7", + "metadata": {}, + "source": [ + "## Run Federated Experiments with NVFlare\n", + "We then run the federated XGBoost training using NVFlare Simulator via [JobAPI](https://nvflare.readthedocs.io/en/main/programming_guide/fed_job_api.html), without and with homomorphic encryption. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "297293bf-bf99-4a93-ab91-1757c96e61bb", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "! python xgb_fl_job.py --data_root /tmp/nvflare/dataset/xgb_dataset/horizontal_xgb_data --data_split_mode horizontal\n", + "! python xgb_fl_job.py --data_root /tmp/nvflare/dataset/xgb_dataset/horizontal_xgb_data --data_split_mode horizontal --secure True\n", + "! python xgb_fl_job.py --data_root /tmp/nvflare/dataset/xgb_dataset/vertical_xgb_data --data_split_mode vertical\n", + "! python xgb_fl_job.py --data_root /tmp/nvflare/dataset/xgb_dataset/vertical_xgb_data --data_split_mode vertical --secure True" + ] + }, + { + "cell_type": "markdown", + "id": "701ad13a-0ca5-47bf-846a-c02ba88ef68a", + "metadata": {}, + "source": [ + "Secure horizontal needs additional tenseal context provisioning:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5cf9590-89a6-454d-bdd8-076af767efaf", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "! nvflare provision -p project.yml -w /tmp/nvflare/workspace/fedxgb_secure/train_fl/works/horizontal_secure\n", + "! nvflare simulator /tmp/nvflare/workspace/fedxgb_secure/train_fl/jobs/horizontal_secure -w /tmp/nvflare/workspace/fedxgb_secure/train_fl/works/horizontal_secure/example_project/prod_00/site-1 -n 3 -t 3" + ] + }, + { + "cell_type": "markdown", + "id": "ead8cbd7-58da-49d6-ae94-9ad013521829", + "metadata": {}, + "source": [ + "## Results\n", + "Comparing the AUC results with centralized baseline, we have four observations:\n", + "1. The performance of the model trained with homomorphic encryption is identical to its counterpart without encryption.\n", + "2. Vertical federated learning (both secure and non-secure) have identical performance as the centralized baseline.\n", + "3. Horizontal federated learning (both secure and non-secure) have performance slightly different from the centralized baseline. This is because under horizontal FL, the local histogram quantiles are based on the local data distribution, which may not be the same as the global distribution.\n", + "4. GPU leads to different results compared to CPU, which is expected as the GPU involves some data conversions.\n", + "\n", + "Below are sample results for CPU training:\n", + "\n", + "The AUC of vertical learning (both secure and non-secure):\n", + "```\n", + "[0]\teval-auc:0.90515\ttrain-auc:0.92747\n", + "[1]\teval-auc:0.90516\ttrain-auc:0.92748\n", + "[2]\teval-auc:0.90518\ttrain-auc:0.92749\n", + "```\n", + "The AUC of horizontal learning (both secure and non-secure):\n", + "```\n", + "[0]\teval-auc:0.89789\ttrain-auc:0.92732\n", + "[1]\teval-auc:0.89791\ttrain-auc:0.92733\n", + "[2]\teval-auc:0.89791\ttrain-auc:0.92733\n", + "```\n", + "\n", + "Comparing the tree models with centralized baseline, we have the following observations:\n", + "1. Vertical federated learning (non-secure) has exactly the same tree model as the centralized baseline.\n", + "2. Vertical federated learning (secure) has the same tree structures as the centralized baseline, however, it produces different tree records at different parties - because each party holds different feature subsets, as illustrated below.\n", + "3. Horizontal federated learning (both secure and non-secure) have different tree models from the centralized baseline.\n", + "\n", + "| ![Tree Structures](./figs/tree.base.png) |\n", + "|:-------------------------------------------------:|\n", + "| *Baseline Model* |\n", + "| ![Tree Structures](./figs/tree.vert.secure.0.png) |\n", + "| *Secure Vertical Model at Party 0* |\n", + "| ![Tree Structures](./figs/tree.vert.secure.1.png) |\n", + "| *Secure Vertical Model at Party 1* |\n", + "| ![Tree Structures](./figs/tree.vert.secure.2.png) |\n", + "| *Secure Vertical Model at Party 2* |\n", + "\n", + "In this case we can notice that Party 0 holds Feature 7 and 10, Party 1 holds Feature 14, 17, and 12, and Party 2 holds none of the effective features for this tree - parties who do not hold the feature will and should not know the split value if it.\n", + "\n", + "By combining the feature splits at all parties, the tree structures will be identical to the centralized baseline model.\n", + "\n", + "When comparing the training and validation accuracy as well as the model outputs,\n", + "experiments conducted with NVFlare produce results that are identical\n", + "to those obtained from standalone scripts.\n", + "\n", + "For more information on the secure xgboost user guide please refer to\n", + "https://nvflare.readthedocs.io/en/main/user_guide/federated_xgboost/secure_xgboost_user_guide.html" + ] + }, + { + "cell_type": "markdown", + "id": "8fa00500-6ae1-49fa-82ca-1c5ab13b1339", + "metadata": {}, + "source": [ + "Now that we covered federated XGBoost under various settings: histogram-based and tree-based, horizontal and vertical, regular and secured. Let's have a [recap](../10.3_recap/recap.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4da46255-f85d-4ecb-9832-177725afdc62", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/train_standalone/train_base.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/train_standalone/train_base.py new file mode 100644 index 0000000000..c3b2fedac1 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/train_standalone/train_base.py @@ -0,0 +1,153 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os + +import matplotlib.pyplot as plt +import pandas as pd +import shap +import xgboost as xgb + +PRINT_SAMPLE = False + + +def train_base_args_parser(): + parser = argparse.ArgumentParser(description="Train baseline XGBoost model") + parser.add_argument("--gpu", type=int, default=0, help="Whether to use gpu for training, 0 for cpu, 1 for gpu") + parser.add_argument( + "--data_train_root", + type=str, + default="/tmp/nvflare/dataset/xgb_dataset/base_xgb_data", + help="Path to training data folder", + ) + parser.add_argument( + "--data_test_file", + type=str, + default="/tmp/nvflare/dataset/xgb_dataset/test.csv", + help="Path to testing data file", + ) + parser.add_argument( + "--out_path", + type=str, + default="/tmp/nvflare/workspace/fedxgb_secure/train_standalone/base", + help="Output path for the data split file", + ) + return parser + + +def load_test_data(data_path: str): + df = pd.read_csv(data_path) + # Split to feature and label + X = df.iloc[:, 1:] + y = df.iloc[:, 0] + return X, y + + +def main(): + parser = train_base_args_parser() + args = parser.parse_args() + if not os.path.exists(args.out_path): + os.makedirs(args.out_path) + + # Specify file path, rank 0 as the label owner, others as the feature owner + train_path = f"{args.data_train_root}/train.csv" + valid_path = f"{args.data_train_root}/valid.csv" + + # Load file directly to tell the match from loading with DMatrix + df_train = pd.read_csv(train_path, header=None) + if PRINT_SAMPLE: + # print number of rows and columns for each worker + print(f"Direct load: nrow={df_train.shape[0]}, ncol={df_train.shape[1]}") + # print one sample row of the data + print(f"Direct load: one sample row of the data: \n {df_train.iloc[0]}") + + # Load file, file will not be sharded in federated mode. + label = "&label_column=0" + # for Vertical XGBoost, read from csv with label_column and set data_split_mode to 1 for column mode + dtrain = xgb.DMatrix(train_path + f"?format=csv{label}") + dvalid = xgb.DMatrix(valid_path + f"?format=csv{label}") + + if PRINT_SAMPLE: + # print number of rows and columns for each worker + print(f"DMatrix: nrow={dtrain.num_row()}, ncol={dtrain.num_col()}") + # print one sample row of the data + data_sample = dtrain.get_data()[0] + print(f"DMatrix: one sample row of the data: \n {data_sample}") + + # Specify parameters via map, definition are same as c++ version + if args.gpu: + device = "cuda:0" + else: + device = "cpu" + param = { + "max_depth": 3, + "eta": 0.1, + "objective": "binary:logistic", + "eval_metric": "auc", + "tree_method": "hist", + "device": device, + "nthread": 1, + } + + # Specify validations set to watch performance + watchlist = [(dvalid, "eval"), (dtrain, "train")] + num_round = 3 + + # Run training, all the features in training API is available. + bst = xgb.train(param, dtrain, num_round, evals=watchlist) + + # Save the model + bst.save_model(f"{args.out_path}/model.base.json") + xgb.collective.communicator_print("Finished training\n") + + # save feature importance score to file + score = bst.get_score(importance_type="gain") + with open(f"{args.out_path}/feat_importance.base.txt", "w") as f: + for key in score: + f.write(f"{key}: {score[key]}\n") + + # Load test data + X_test, y_test = load_test_data(args.data_test_file) + # construct xgboost DMatrix + dmat_test = xgb.DMatrix(X_test, label=y_test) + + # Explain the model + explainer = shap.TreeExplainer(bst) + explanation = explainer(dmat_test) + + # save the beeswarm plot to png file + shap.plots.beeswarm(explanation, show=False) + img = plt.gcf() + img.savefig(f"{args.out_path}/shap.base.png") + + # dump tree and save to text file + dump = bst.get_dump() + with open(f"{args.out_path}/tree_dump.base.txt", "w") as f: + for tree in dump: + f.write(tree) + + # plot tree and save to png file + xgb.plot_tree(bst, num_trees=0, rankdir="LR") + fig = plt.gcf() + fig.set_size_inches(18, 5) + plt.savefig(f"{args.out_path}/tree.base.png", dpi=100) + + # export tree to dataframe + tree_df = bst.trees_to_dataframe() + tree_df.to_csv(f"{args.out_path}/tree_df.base.csv") + + +if __name__ == "__main__": + main() diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/train_standalone/train_federated.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/train_standalone/train_federated.py new file mode 100644 index 0000000000..1b57aa067f --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/train_standalone/train_federated.py @@ -0,0 +1,209 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import multiprocessing +import os +import time + +import matplotlib.pyplot as plt +import pandas as pd +import shap +import xgboost as xgb +import xgboost.federated + +PRINT_SAMPLE = False + + +def train_federated_args_parser(): + parser = argparse.ArgumentParser(description="Train federated XGBoost model") + parser.add_argument("--world_size", type=int, default=3, help="Total number of clients") + parser.add_argument("--gpu", type=int, default=0, help="Whether to use gpu for training, 0 for cpu, 1 for gpu") + parser.add_argument( + "--vert", type=int, default=0, help="Horizontal or vertical training, 0 for horizontal, 1 for vertical" + ) + parser.add_argument( + "--enc", type=int, default=0, help="Whether to use encryption plugin, 0 for non-encrypted, 1 for encrypted" + ) + parser.add_argument( + "--data_train_root", + type=str, + default="/tmp/nvflare/dataset/xgb_dataset/base_xgb_data", + help="Path to training data folder", + ) + parser.add_argument( + "--data_test_file", + type=str, + default="/tmp/nvflare/dataset/xgb_dataset/test.csv", + help="Path to testing data file", + ) + parser.add_argument( + "--out_path", + type=str, + default="/tmp/nvflare/workspace/fedxgb_secure/train_standalone/federated", + help="Output path for the data split file", + ) + return parser + + +def load_test_data(data_path: str): + df = pd.read_csv(data_path) + # Split to feature and label + X = df.iloc[:, 1:] + y = df.iloc[:, 0] + return X, y + + +def run_server(port: int, world_size: int) -> None: + xgboost.federated.run_federated_server(port=port, n_workers=world_size) + + +def run_worker(port: int, world_size: int, rank: int, args) -> None: + if args.enc: + plugin = {"name": "mock"} + else: + plugin = {} + communicator_env = { + "dmlc_communicator": "federated", + "federated_server_address": f"localhost:{port}", + "federated_world_size": world_size, + "federated_rank": rank, + "federated_plugin": plugin, + } + + # Always call this before using distributed module + with xgb.collective.CommunicatorContext(**communicator_env): + # Specify file path, rank 0 as the label owner, others as the feature owner + train_path = f"{args.data_train_root}/site-{rank + 1}/train.csv" + valid_path = f"{args.data_train_root}/site-{rank + 1}/valid.csv" + + # Load file directly to tell the match from loading with DMatrix + df_train = pd.read_csv(train_path, header=None) + if PRINT_SAMPLE: + # print number of rows and columns for each worker + print(f"Direct load: rank={rank}, nrow={df_train.shape[0]}, ncol={df_train.shape[1]}") + # print one sample row of the data + print(f"Direct load: rank={rank}, one sample row of the data: \n {df_train.iloc[0]}") + + # Load file, file will not be sharded in federated mode. + if args.vert: + data_split_mode = 1 + if rank == 0: + label = "&label_column=0" + else: + label = "" + else: + data_split_mode = 0 + label = "&label_column=0" + dtrain = xgb.DMatrix(train_path + f"?format=csv{label}", data_split_mode=data_split_mode) + dvalid = xgb.DMatrix(valid_path + f"?format=csv{label}", data_split_mode=data_split_mode) + + if PRINT_SAMPLE: + # print number of rows and columns for each worker + print(f"DMatrix: rank={rank}, nrow={dtrain.num_row()}, ncol={dtrain.num_col()}") + # print one sample row of the data + data_sample = dtrain.get_data()[0] + print(f"DMatrix: rank={rank}, one sample row of the data: \n {data_sample}") + + # Specify parameters via map, definition are same as c++ version + if args.gpu: + device = "cuda:0" + else: + device = "cpu" + param = { + "max_depth": 3, + "eta": 0.1, + "objective": "binary:logistic", + "eval_metric": "auc", + "tree_method": "hist", + "device": device, + "nthread": 1, + } + + # Specify validations set to watch performance + watchlist = [(dvalid, "eval"), (dtrain, "train")] + num_round = 3 + + # Run training, all the features in training API is available. + bst = xgb.train(param, dtrain, num_round, evals=watchlist) + + # Save the model + rank = xgb.collective.get_rank() + bst.save_model(f"{args.out_path}/model.{rank}.json") + xgb.collective.communicator_print("Finished training\n") + + # save feature importance score to file + score = bst.get_score(importance_type="gain") + with open(f"{args.out_path}/feat_importance.{rank}.txt", "w") as f: + for key in score: + f.write(f"{key}: {score[key]}\n") + + # Load test data + X_test, y_test = load_test_data(args.data_test_file) + # construct xgboost DMatrix + dmat_test = xgb.DMatrix(X_test, label=y_test) + + # Explain the model + explainer = shap.TreeExplainer(bst) + explanation = explainer(dmat_test) + + # save the beeswarm plot to png file + shap.plots.beeswarm(explanation, show=False) + img = plt.gcf() + img.savefig(f"{args.out_path}/shap.{rank}.png") + + # dump tree and save to text file + dump = bst.get_dump() + with open(f"{args.out_path}/tree_dump.{rank}.txt", "w") as f: + for tree in dump: + f.write(tree) + + # plot tree and save to png file + xgb.plot_tree(bst, num_trees=0, rankdir="LR") + fig = plt.gcf() + fig.set_size_inches(18, 5) + plt.savefig(f"{args.out_path}/tree.{rank}.png", dpi=100) + + # export tree to dataframe + tree_df = bst.trees_to_dataframe() + tree_df.to_csv(f"{args.out_path}/tree_df.{rank}.csv") + + +def main(): + parser = train_federated_args_parser() + args = parser.parse_args() + if not os.path.exists(args.out_path): + os.makedirs(args.out_path) + + port = 1111 + world_size = args.world_size + + server = multiprocessing.Process(target=run_server, args=(port, world_size)) + server.start() + time.sleep(1) + if not server.is_alive(): + raise Exception("Error starting Federated Learning server") + + workers = [] + for rank in range(world_size): + worker = multiprocessing.Process(target=run_worker, args=(port, world_size, rank, args)) + workers.append(worker) + worker.start() + for worker in workers: + worker.join() + server.terminate() + + +if __name__ == "__main__": + main() diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/vertical_fed_xgboost_with_he.ipynb b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/vertical_fed_xgboost_with_he.ipynb deleted file mode 100644 index 3c7b730b81..0000000000 --- a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/vertical_fed_xgboost_with_he.ipynb +++ /dev/null @@ -1,33 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "a30f033f-279e-4111-9e70-bce717231f14", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nvflare_example", - "language": "python", - "name": "nvflare_example" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/xgb_fl_job.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/xgb_fl_job.py new file mode 100644 index 0000000000..ffe21af304 --- /dev/null +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.2_secure_fed_xgboost/xgb_fl_job.py @@ -0,0 +1,130 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from nvflare.app_common.widgets.convert_to_fed_event import ConvertToFedEvent +from nvflare.app_opt.tracking.tb.tb_receiver import TBAnalyticsReceiver +from nvflare.app_opt.tracking.tb.tb_writer import TBWriter +from nvflare.app_opt.xgboost.histogram_based_v2.csv_data_loader import CSVDataLoader +from nvflare.app_opt.xgboost.histogram_based_v2.fed_controller import XGBFedController +from nvflare.app_opt.xgboost.histogram_based_v2.fed_executor import FedXGBHistogramExecutor +from nvflare.job_config.api import FedJob + + +def define_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--data_root", + type=str, + help="Path to dataset files for each site", + ) + parser.add_argument("--site_num", type=int, default=3, help="Total number of sites") + parser.add_argument("--round_num", type=int, default=3, help="Total number of training rounds") + parser.add_argument("--nthread", type=int, default=16, help="nthread for xgboost") + parser.add_argument( + "--tree_method", type=str, default="hist", help="tree_method for xgboost - use hist for best perf" + ) + parser.add_argument( + "--data_split_mode", + type=str, + default="horizontal", + choices=["horizontal", "vertical"], + help="dataset split mode, horizontal or vertical", + ) + parser.add_argument( + "--secure", + type=bool, + default=False, + help="Whether to use secure training", + ) + return parser.parse_args() + + +def _get_job_name(args) -> str: + if args.secure: + return f"{args.data_split_mode}_secure" + else: + return f"{args.data_split_mode}" + + +def main(): + args = define_parser() + job_name = _get_job_name(args) + dataset_path = args.data_root + site_num = args.site_num + job = FedJob(name=job_name, min_clients=site_num) + if args.data_split_mode == "horizontal": + data_split_mode = 0 + else: + data_split_mode = 1 + # Define the controller workflow and send to server + controller = XGBFedController( + num_rounds=args.round_num, + data_split_mode=data_split_mode, + secure_training=args.secure, + xgb_options={"early_stopping_rounds": 3, "use_gpus": False}, + xgb_params={ + "max_depth": 3, + "eta": 0.1, + "objective": "binary:logistic", + "eval_metric": "auc", + "tree_method": "hist", + "nthread": 1, + }, + client_ranks={"site-1": 0, "site-2": 1, "site-3": 2}, + in_process=True, + ) + job.to_server(controller, id="xgb_controller") + + # Add tensorboard receiver to server + tb_receiver = TBAnalyticsReceiver( + tb_folder="tb_events", + ) + job.to_server(tb_receiver, id="tb_receiver") + + # Add executor and other components to clients + for site_id in range(1, site_num + 1): + # Define the executor for clients + executor = FedXGBHistogramExecutor( + data_loader_id="dataloader", + metrics_writer_id="metrics_writer", + in_process=True, + ) + job.to(executor, f"site-{site_id}", id="executor") + + dataloader = CSVDataLoader(folder=dataset_path) + job.to(dataloader, f"site-{site_id}", id="dataloader") + + metrics_writer = TBWriter(event_type="analytix_log_stats") + job.to(metrics_writer, f"site-{site_id}", id="metrics_writer") + + event_to_fed = ConvertToFedEvent( + events_to_convert=["analytix_log_stats"], + fed_event_prefix="fed.", + ) + job.to(event_to_fed, f"site-{site_id}", id="event_to_fed") + + # Export job config and run the job + job.export_job("/tmp/nvflare/workspace/fedxgb_secure/train_fl/jobs/") + + # Run the job except for secure horizontal + if args.data_split_mode == "horizontal" and args.secure: + print("Secure horizontal is not supported in this version") + else: + job.simulator_run(f"/tmp/nvflare/workspace/fedxgb_secure/train_fl/works/{job_name}") + + +if __name__ == "__main__": + main() diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.3_recap/recap.ipynb b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.3_recap/recap.ipynb index 9206f8351b..844ede5574 100644 --- a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.3_recap/recap.ipynb +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-10_federated_XGBoost/10.3_recap/recap.ipynb @@ -1,9 +1,39 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "d9df414c-f4d6-4527-b201-a978ed73a3b7", + "metadata": {}, + "source": [ + "# Summary of Chapter 10" + ] + }, + { + "cell_type": "markdown", + "id": "47959d8f-e03c-4a46-a469-213dfe1909b6", + "metadata": {}, + "source": [ + "In this chapter, we visited Federated XGBoost enabled by functionalities from both XGBoost and NVFlare.\n", + "\n", + "Specifically, the following items have been covered:\n", + "\n", + "1. **[Federated XGBoost](../10.1_fed_xgboost/fed_xgboost.ipynb)**: histogram-based and tree-based horizontal collaborations, and histogram-based vertical collaboration, are covered\n", + "2. **[Secure Federated XGBoost](../10.2_secure_fed_xgboost/secure_fed_xgboost.ipynb)**: secure federated XGBoost with homomorphic encryption under \n", + "histogram-based horizontal and vertical collaboration.\n", + "\n", + "Key takeaways of this section are:\n", + "\n", + "1. NVFlare enables federated training of XGBoost models, under both horizontal and vertical collaboration modes.\n", + "2. For horizontal collaboration, federated XGBoost can be performed with tree-based and histogram-based methods, while for vertical collaboration, federated XGBoost relies on histogram-based method.\n", + "3. To protect key information, homomorphic encryption can be added to histogram-based pipelines to ensure data confidentiality under different configurations and needs.\n", + "\n", + "With NVFlare providing full support of federated XGBoost with secure features, users can benefit from efficient and safe federated pipelines of tree-based machine learning algorithms for their applications.\n" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "001060e8-93da-46c9-a848-2ae40e46e1d3", + "id": "b3b7bd34-89df-486b-9ee3-34ae4427e2a3", "metadata": {}, "outputs": [], "source": [] @@ -11,9 +41,9 @@ ], "metadata": { "kernelspec": { - "display_name": "nvflare_example", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "nvflare_example" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -25,7 +55,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.2" + "version": "3.13.2" } }, "nbformat": 4, diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-9_flare_low_level_apis/09.2_tasks_and_data_share/modules.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-9_flare_low_level_apis/09.2_tasks_and_data_share/modules.py index 62ccfb06f2..f7c0a44985 100644 --- a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-9_flare_low_level_apis/09.2_tasks_and_data_share/modules.py +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-9_flare_low_level_apis/09.2_tasks_and_data_share/modules.py @@ -22,7 +22,6 @@ class HelloController(Controller): - def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): # Create the task with name "hello" task = Task(name="hello", data=Shareable()) @@ -43,7 +42,6 @@ def stop_controller(self, fl_ctx: FLContext): class HelloExecutor(Executor): - def execute( self, task_name: str, @@ -57,7 +55,6 @@ def execute( class HelloDataController(Controller): - def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): # Prepare any extra parameters to send to the clients data = DXO( @@ -84,7 +81,6 @@ def stop_controller(self, fl_ctx: FLContext): class HelloDataExecutor(Executor): - def execute( self, task_name: str, @@ -100,7 +96,6 @@ def execute( class HelloResponseController(Controller): - def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): # Prepare any extra parameters to send to the clients dxo = DXO( @@ -136,7 +131,6 @@ def _process_client_response(self, client_task, fl_ctx: FLContext) -> None: class HelloResponseExecutor(Executor): - def execute( self, task_name: str, diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-9_flare_low_level_apis/09.3_p2p_communication/modules.py b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-9_flare_low_level_apis/09.3_p2p_communication/modules.py index 6efc10c997..211a1fcfd4 100644 --- a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-9_flare_low_level_apis/09.3_p2p_communication/modules.py +++ b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-9_flare_low_level_apis/09.3_p2p_communication/modules.py @@ -23,7 +23,6 @@ class BasicController(Controller): - def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.broadcast_and_wait( task=Task(name="talk", data=Shareable()), @@ -40,7 +39,6 @@ def stop_controller(self, fl_ctx: FLContext): class P2PExecutor(Executor): - def execute( self, task_name: str, diff --git a/nvflare/app_common/abstract/statistics_spec.py b/nvflare/app_common/abstract/statistics_spec.py index b34dd93f68..2d5a056398 100644 --- a/nvflare/app_common/abstract/statistics_spec.py +++ b/nvflare/app_common/abstract/statistics_spec.py @@ -318,7 +318,7 @@ def failure_count(self, dataset_name: str, feature_name: str) -> int: """ return 0 - def percentiles(self, dataset_name: str, feature_name: str, percentiles: List) -> Dict: + def quantiles(self, dataset_name: str, feature_name: str, percentiles: List) -> Dict: """Return failed count for given dataset and feature. To perform data privacy min_count check, failure_count is always required. diff --git a/nvflare/app_common/app_constant.py b/nvflare/app_common/app_constant.py index a7c34b8444..fa9fcd2718 100644 --- a/nvflare/app_common/app_constant.py +++ b/nvflare/app_common/app_constant.py @@ -163,7 +163,7 @@ class StatisticsConstants(AppConstants): STATS_VAR = "var" STATS_STDDEV = "stddev" STATS_HISTOGRAM = "histogram" - STATS_PERCENTILE = "percentile" + STATS_QUANTILE = "quantile" STATS_MAX = "max" STATS_MIN = "min" STATS_FEATURES = "stats_features" @@ -174,8 +174,7 @@ class StatisticsConstants(AppConstants): STATS_BIN_RANGE = "range" STATS_TARGET_STATISTICS = "statistics" - STATS_PERCENTILES_KEY = "percentiles" - STATS_CENTROIDS_KEY = "centroids" + STATS_DIGEST_COORD = "digest_coord" FED_STATS_PRE_RUN = "fed_stats_pre_run" FED_STATS_TASK = "fed_stats" @@ -196,7 +195,7 @@ class StatisticsConstants(AppConstants): STATS_MEAN, STATS_MIN, STATS_MAX, - STATS_PERCENTILE, + STATS_QUANTILE, ], STATS_2nd_STATISTICS: [STATS_HISTOGRAM, STATS_VAR, STATS_STDDEV], } diff --git a/nvflare/app_common/executors/statistics/statistics_task_handler.py b/nvflare/app_common/executors/statistics/statistics_task_handler.py index e3b1bb45a7..560b9a464d 100644 --- a/nvflare/app_common/executors/statistics/statistics_task_handler.py +++ b/nvflare/app_common/executors/statistics/statistics_task_handler.py @@ -23,7 +23,7 @@ from nvflare.app_common.app_constant import StatisticsConstants as StC from nvflare.app_common.statistics.numeric_stats import filter_numeric_features from nvflare.app_common.statistics.statisitcs_objects_decomposer import fobs_registration -from nvflare.app_common.statistics.statistics_config_utils import get_feature_bin_range, get_target_percents +from nvflare.app_common.statistics.statistics_config_utils import get_feature_bin_range, get_target_quantiles from nvflare.fuel.utils import fobs from nvflare.security.logging import secure_format_exception @@ -96,7 +96,7 @@ def statistic_functions(self) -> dict: StC.STATS_HISTOGRAM: self.get_histogram, StC.STATS_MAX: self.get_max_value, StC.STATS_MIN: self.get_min_value, - StC.STATS_PERCENTILE: self.get_percentiles_and_centroids, + StC.STATS_QUANTILE: self.get_quantiles_and_centroids, } def _populate_result_statistics(self, statistics_result, ds_features, tm: StatisticConfig, shareable, fl_ctx, fn): @@ -319,7 +319,7 @@ def get_bin_range( return bin_range - def get_percentiles_and_centroids( + def get_quantiles_and_centroids( self, dataset_name: str, feature_name: str, @@ -328,8 +328,8 @@ def get_percentiles_and_centroids( fl_ctx: FLContext, ) -> dict: percentile_config = statistic_configs.config - target_percents = get_target_percents(percentile_config, feature_name) - result = self.stats_generator.percentiles(dataset_name, feature_name, target_percents) + target_percents = get_target_quantiles(percentile_config, feature_name) + result = self.stats_generator.quantiles(dataset_name, feature_name, target_percents) return result def _get_global_value_from_input(self, statistic_key: str, dataset_name: str, feature_name: str, inputs): diff --git a/nvflare/app_common/filters/svt_privacy.py b/nvflare/app_common/filters/svt_privacy.py index 5d168153a6..8381445072 100644 --- a/nvflare/app_common/filters/svt_privacy.py +++ b/nvflare/app_common/filters/svt_privacy.py @@ -45,10 +45,10 @@ def __init__( super().__init__(supported_data_kinds=[DataKind.WEIGHTS, DataKind.WEIGHT_DIFF], data_kinds_to_filter=data_kinds) - self.frac = fraction # fraction of the model to upload - self.eps_1 = epsilon + self.fraction = fraction # fraction of the model to upload + self.epsilon = epsilon self.eps_2 = None # to be derived from eps_1 - self.eps_3 = noise_var + self.noise_var = noise_var self.gamma = gamma self.tau = tau self.replace = replace @@ -76,21 +76,21 @@ def process_dxo(self, dxo: DXO, shareable: Shareable, fl_ctx: FLContext) -> Unio ) # precompute thresholds - n_upload = np.minimum(np.ceil(np.float64(delta_w.size) * self.frac), np.float64(delta_w.size)) + n_upload = np.minimum(np.ceil(np.float64(delta_w.size) * self.fraction), np.float64(delta_w.size)) # eps_1: threshold with noise - lambda_rho = self.gamma * 2.0 / self.eps_1 + lambda_rho = self.gamma * 2.0 / self.epsilon threshold = self.tau + np.random.laplace(scale=lambda_rho) # eps_2: query with noise - self.eps_2 = self.eps_1 * (2.0 * n_upload) ** (2.0 / 3.0) + self.eps_2 = self.epsilon * (2.0 * n_upload) ** (2.0 / 3.0) lambda_nu = self.gamma * 4.0 * n_upload / self.eps_2 self.logger.info( "total params: %s, epsilon: %s, " "perparam budget %s, threshold tau: %s + f(eps_1) = %s, " "clip gamma: %s", delta_w.size, - self.eps_1, - self.eps_1 / n_upload, + self.epsilon, + self.epsilon / n_upload, self.tau, threshold, self.gamma, @@ -107,7 +107,7 @@ def process_dxo(self, dxo: DXO, shareable: Shareable, fl_ctx: FLContext) -> Unio self.log_info(fl_ctx, "selected {} responses, requested {}".format(len(accepted), n_upload)) accepted = np.random.choice(accepted, size=np.int64(n_upload), replace=self.replace) # eps_3 return with noise - noise = np.random.laplace(scale=self.gamma * 2.0 / self.eps_3, size=accepted.shape) + noise = np.random.laplace(scale=self.gamma * 2.0 / self.noise_var, size=accepted.shape) self.log_info(fl_ctx, "noise max: {}, median {}".format(np.max(np.abs(noise)), np.median(np.abs(noise)))) delta_w[accepted] = np.clip(delta_w[accepted] + noise, a_min=-self.gamma, a_max=self.gamma) candidate_idx = list(set(np.arange(delta_w.size)) - set(accepted)) diff --git a/nvflare/app_common/statistics/numeric_stats.py b/nvflare/app_common/statistics/numeric_stats.py index 046f0ce0b4..3c6273f75d 100644 --- a/nvflare/app_common/statistics/numeric_stats.py +++ b/nvflare/app_common/statistics/numeric_stats.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,17 +15,18 @@ from math import sqrt from typing import Dict, List, TypeVar -from tdigest import TDigest - from nvflare.app_common.abstract.statistics_spec import Bin, BinRange, DataType, Feature, Histogram, HistogramType from nvflare.app_common.app_constant import StatisticsConstants as StC -from nvflare.app_common.statistics.statistics_config_utils import get_target_percents +from nvflare.app_opt.statistics.quantile_stats import get_quantiles +from nvflare.fuel.utils.log_utils import get_module_logger T = TypeVar("T") +logger = get_module_logger(name=__name__) + def get_global_feature_data_types( - client_feature_dts: Dict[str, Dict[str, List[Feature]]] + client_feature_dts: Dict[str, Dict[str, List[Feature]]], ) -> Dict[str, Dict[str, DataType]]: global_feature_data_types = {} for client_name in client_feature_dts: @@ -85,14 +86,8 @@ def get_global_stats( ds_stddev[ds_name][feature] = round(sqrt(feature_vars[feature]), precision) global_metrics[StC.STATS_STDDEV] = ds_stddev - elif metric == StC.STATS_PERCENTILE: - global_digest = {} - for client_name in stats: - - global_digest = aggregate_centroids(stats[client_name], global_digest) - - percent_config = statistic_configs.get(StC.STATS_PERCENTILE) - global_metrics[metric] = compute_percentiles(global_digest, percent_config, precision) + elif metric == StC.STATS_QUANTILE: + global_metrics[metric] = get_quantiles(stats, statistic_configs, precision) return global_metrics @@ -231,42 +226,3 @@ def filter_numeric_features(ds_features: Dict[str, List[Feature]]) -> Dict[str, numeric_ds_features[ds_name] = n_features return numeric_ds_features - - -def aggregate_centroids(metrics: Dict[str, Dict[str, Dict]], g_digest: dict) -> dict: - for ds_name in metrics: - if ds_name not in g_digest: - g_digest[ds_name] = {} - - feature_metrics = metrics[ds_name] - for feature_name in feature_metrics: - if feature_metrics[feature_name] is not None: - centroids: List = feature_metrics[feature_name].get(StC.STATS_CENTROIDS_KEY) - if feature_name not in g_digest[ds_name]: - g_digest[ds_name][feature_name] = TDigest() - - for centroid in centroids: - mean = centroid.get("m") - count = centroid.get("c") - g_digest[ds_name][feature_name].update(mean, count) - - return g_digest - - -def compute_percentiles(g_digest: Dict[str, Dict[str, TDigest]], quantile_config: Dict, precision: int = 4) -> dict: - g_ds_metrics = {} - for ds_name in g_digest: - if ds_name not in g_ds_metrics: - g_ds_metrics[ds_name] = {} - - feature_metrics = g_digest[ds_name] - for feature_name in feature_metrics: - digest = feature_metrics[feature_name] - percentiles = get_target_percents(quantile_config, feature_name) - percentile_values = {} - for percentile in percentiles: - percentile_values[percentile] = round(digest.percentile(percentile), precision) - - g_ds_metrics[ds_name][feature_name] = percentile_values - - return g_ds_metrics diff --git a/nvflare/app_common/statistics/statistics_config_utils.py b/nvflare/app_common/statistics/statistics_config_utils.py index b128df2394..76b208c2fc 100644 --- a/nvflare/app_common/statistics/statistics_config_utils.py +++ b/nvflare/app_common/statistics/statistics_config_utils.py @@ -30,7 +30,7 @@ def get_feature_bin_range(feature_name: str, hist_config: dict) -> Optional[List return bin_range -def get_target_percents(percentile_config: dict, feature_name: str): +def get_target_quantiles(percentile_config: dict, feature_name: str): if feature_name in percentile_config: percents = percentile_config.get(feature_name) elif "*" in percentile_config: diff --git a/nvflare/app_common/workflows/statistics_controller.py b/nvflare/app_common/workflows/statistics_controller.py index a835c95900..a43b8a531a 100644 --- a/nvflare/app_common/workflows/statistics_controller.py +++ b/nvflare/app_common/workflows/statistics_controller.py @@ -70,7 +70,7 @@ def __init__( "*": {"bins": 20}, "Age": {"bins": 10, "range": [0, 120]} }, - percentile: { + quantile: { "*": [25, 50, 75, 90], "Age": [50, 75, 95] } @@ -211,7 +211,7 @@ def _get_all_statistic_configs(self) -> List[StatisticConfig]: StC.STATS_MEAN: StatisticConfig(StC.STATS_MEAN, {}), StC.STATS_VAR: StatisticConfig(StC.STATS_VAR, {}), StC.STATS_STDDEV: StatisticConfig(StC.STATS_STDDEV, {}), - StC.STATS_PERCENTILE: StatisticConfig(StC.STATS_PERCENTILE, {}), + StC.STATS_QUANTILE: StatisticConfig(StC.STATS_QUANTILE, {}), } if StC.STATS_HISTOGRAM in self.statistic_configs: @@ -409,14 +409,12 @@ def _combine_all_statistics(self): hist: Histogram = self.client_statistics[statistic][client][ds][feature_name] buckets = StatisticsController._apply_histogram_precision(hist.bins, self.precision) result[feature_name][statistic][client][ds] = buckets - elif statistic == StC.STATS_PERCENTILE: - percentiles = self.client_statistics[statistic][client][ds][feature_name][ - StC.STATS_PERCENTILES_KEY - ] - formatted_percentiles = {} - for p in percentiles: - formatted_percentiles[p] = round(percentiles.get(p), self.precision) - result[feature_name][statistic][client][ds] = formatted_percentiles + elif statistic == StC.STATS_QUANTILE: + quantiles = self.client_statistics[statistic][client][ds][feature_name][StC.STATS_QUANTILE] + formatted_quantiles = {} + for p in quantiles: + formatted_quantiles[p] = round(quantiles.get(p), self.precision) + result[feature_name][statistic][client][ds] = formatted_quantiles else: result[feature_name][statistic][client][ds] = round( self.client_statistics[statistic][client][ds][feature_name], self.precision @@ -434,9 +432,9 @@ def _combine_all_statistics(self): if statistic == StC.STATS_HISTOGRAM: hist: Histogram = self.global_statistics[statistic][ds][feature_name] result[feature_name][statistic][StC.GLOBAL][ds] = hist.bins - elif statistic == StC.STATS_PERCENTILE: - percentiles = self.global_statistics[statistic][ds][feature_name] - result[feature_name][statistic][StC.GLOBAL][ds] = percentiles + elif statistic == StC.STATS_QUANTILE: + quantiles = self.global_statistics[statistic][ds][feature_name] + result[feature_name][statistic][StC.GLOBAL][ds] = quantiles else: result[feature_name][statistic][StC.GLOBAL].update( {ds: self.global_statistics[statistic][ds][feature_name]} diff --git a/nvflare/app_opt/pt/job_config/base_fed_job.py b/nvflare/app_opt/pt/job_config/base_fed_job.py index 0225064967..952e47244f 100644 --- a/nvflare/app_opt/pt/job_config/base_fed_job.py +++ b/nvflare/app_opt/pt/job_config/base_fed_job.py @@ -16,9 +16,9 @@ from torch import nn as nn +from nvflare.apis.analytix import ANALYTIC_EVENT_TYPE from nvflare.app_common.abstract.model_locator import ModelLocator from nvflare.app_common.abstract.model_persistor import ModelPersistor -from nvflare.app_common.tracking.tracker_types import ANALYTIC_EVENT_TYPE from nvflare.app_common.widgets.convert_to_fed_event import ConvertToFedEvent from nvflare.app_common.widgets.intime_model_selector import IntimeModelSelector from nvflare.app_common.widgets.streaming import AnalyticsReceiver diff --git a/nvflare/app_opt/statistics/df/df_core_statistics.py b/nvflare/app_opt/statistics/df/df_core_statistics.py index 5404889e8e..ad0c0ae4d0 100644 --- a/nvflare/app_opt/statistics/df/df_core_statistics.py +++ b/nvflare/app_opt/statistics/df/df_core_statistics.py @@ -12,23 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. from abc import ABC +from math import sqrt from typing import Dict, List, Optional import numpy as np import pandas as pd from pandas.core.series import Series -from tdigest import TDigest from nvflare.app_common.abstract.statistics_spec import BinRange, Feature, Histogram, HistogramType, Statistics from nvflare.app_common.app_constant import StatisticsConstants from nvflare.app_common.statistics.numpy_utils import dtype_to_data_type, get_std_histogram_buckets +from nvflare.fuel.utils.import_utils import optional_import class DFStatisticsCore(Statistics, ABC): - def __init__(self): + def __init__(self, max_bin=None): # assumption: the data can be loaded and cached in the memory self.data: Optional[Dict[str, pd.DataFrame]] = None super(DFStatisticsCore, self).__init__() + self.max_bin = max_bin def features(self) -> Dict[str, List[Feature]]: results: Dict[str, List[Feature]] = {} @@ -92,24 +94,25 @@ def min_value(self, dataset_name: str, feature_name: str) -> float: df = self.data[dataset_name] return df[feature_name].min() - def percentiles(self, dataset_name: str, feature_name: str, percents: List) -> Dict: - digest = self._prepare_t_digest(dataset_name, feature_name) + def quantiles(self, dataset_name: str, feature_name: str, percents: List) -> Dict: + TDigest, flag = optional_import("fastdigest", name="TDigest") results = {} + if not flag: + results[StatisticsConstants.STATS_QUANTILE] = {} + return results + + df = self.data[dataset_name] + data = df[feature_name] + max_bin = self.max_bin if self.max_bin else round(sqrt(len(data))) + digest = TDigest(data) + digest.compress(max_bin) + p_results = {} for p in percents: - v = round(digest.percentile(p), 4) + v = round(digest.quantile(p), 4) p_results[p] = v - results[StatisticsConstants.STATS_PERCENTILES_KEY] = p_results + results[StatisticsConstants.STATS_QUANTILE] = p_results - # Extract centroids (mean, count) from the digest to used for merge for the global - x = digest.centroids_to_list() - results[StatisticsConstants.STATS_CENTROIDS_KEY] = x + # Extract the Q-Digest into a dictionary + results[StatisticsConstants.STATS_DIGEST_COORD] = digest.to_dict() return results - - def _prepare_t_digest(self, dataset_name: str, feature_name: str) -> TDigest: - df = self.data[dataset_name] - data = df[feature_name] - digest = TDigest() - for value in data: - digest.update(value) - return digest diff --git a/nvflare/app_opt/statistics/quantile_stats.py b/nvflare/app_opt/statistics/quantile_stats.py new file mode 100644 index 0000000000..1daef0e6b7 --- /dev/null +++ b/nvflare/app_opt/statistics/quantile_stats.py @@ -0,0 +1,98 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +from nvflare.app_common.app_constant import StatisticsConstants as StC +from nvflare.fuel.utils.log_utils import get_module_logger + +try: + from fastdigest import TDigest + + TDIGEST_AVAILABLE = True +except ImportError: + TDIGEST_AVAILABLE = False + + +logger = get_module_logger(name="quantile_stats") + + +def get_quantiles(stats: Dict, statistic_configs: Dict, precision: int): + + logger.info(f"get_quantiles: stats: {TDIGEST_AVAILABLE=}") + + if not TDIGEST_AVAILABLE: + return {} + + global_digest = {} + for client_name in stats: + global_digest = merge_quantiles(stats[client_name], global_digest) + + quantile_config = statistic_configs.get(StC.STATS_QUANTILE) + return compute_quantiles(global_digest, quantile_config, precision) + + +def get_target_quantiles(quantile_config: dict, feature_name: str): + if feature_name in quantile_config: + percents = quantile_config.get(feature_name) + elif "*" in quantile_config: + percents = quantile_config.get("*") + else: + raise ValueError(f"feature: {feature_name} target percents are not defined.") + + return percents + + +def merge_quantiles(metrics: Dict[str, Dict[str, Dict]], g_digest: dict) -> dict: + + if not TDIGEST_AVAILABLE: + return g_digest + + for ds_name in metrics: + if ds_name not in g_digest: + g_digest[ds_name] = {} + + feature_metrics = metrics[ds_name] + for feature_name in feature_metrics: + if feature_metrics[feature_name] is not None: + digest_dict: Dict = feature_metrics[feature_name].get(StC.STATS_DIGEST_COORD) + feature_digest = TDigest.from_dict(digest_dict) + if feature_name not in g_digest[ds_name]: + g_digest[ds_name][feature_name] = feature_digest + else: + g_digest[ds_name][feature_name] = g_digest[ds_name][feature_name].merge(feature_digest) + + return g_digest + + +def compute_quantiles(g_digest: dict, quantile_config: Dict, precision: int) -> Dict: + g_ds_metrics = {} + if not TDIGEST_AVAILABLE: + return g_digest + + for ds_name in g_digest: + if ds_name not in g_ds_metrics: + g_ds_metrics[ds_name] = {} + + feature_metrics = g_digest[ds_name] + for feature_name in feature_metrics: + digest = feature_metrics[feature_name] + percentiles = get_target_quantiles(quantile_config, feature_name) + quantile_values = {} + for percentile in percentiles: + quantile_values[percentile] = round(digest.quantile(percentile), precision) + + g_ds_metrics[ds_name][feature_name] = quantile_values + + return g_ds_metrics diff --git a/nvflare/app_opt/tf/job_config/base_fed_job.py b/nvflare/app_opt/tf/job_config/base_fed_job.py index bf77cd1092..4140bf7e56 100644 --- a/nvflare/app_opt/tf/job_config/base_fed_job.py +++ b/nvflare/app_opt/tf/job_config/base_fed_job.py @@ -16,8 +16,8 @@ import tensorflow as tf +from nvflare.apis.analytix import ANALYTIC_EVENT_TYPE from nvflare.app_common.abstract.model_persistor import ModelPersistor -from nvflare.app_common.tracking.tracker_types import ANALYTIC_EVENT_TYPE from nvflare.app_common.widgets.convert_to_fed_event import ConvertToFedEvent from nvflare.app_common.widgets.intime_model_selector import IntimeModelSelector from nvflare.app_common.widgets.streaming import AnalyticsReceiver diff --git a/nvflare/app_opt/xgboost/tree_based/executor.py b/nvflare/app_opt/xgboost/tree_based/executor.py index f4ff475c64..2e1bb45101 100644 --- a/nvflare/app_opt/xgboost/tree_based/executor.py +++ b/nvflare/app_opt/xgboost/tree_based/executor.py @@ -174,7 +174,9 @@ def _local_boost_bagging(self, fl_ctx: FLContext): if self.writer: # note: writing auc before current training step, for passed in global model self.writer.add_scalar( - "AUC", auc, int((self.bst.num_boosted_rounds() - self.num_local_round - 1) / self.num_client_bagging) + "train_metrics", + auc, + int((self.bst.num_boosted_rounds() - self.num_local_round - 1) / self.num_client_bagging), ) return bst @@ -193,7 +195,7 @@ def _local_boost_cyclic(self, fl_ctx: FLContext): f"Client {self.client_id} AUC after training: {auc}", ) if self.writer: - self.writer.add_scalar("AUC", auc, self.bst.num_boosted_rounds() - 1) + self.writer.add_scalar("train_metrics", auc, self.bst.num_boosted_rounds() - 1) return self.bst def train( diff --git a/nvflare/dashboard/application/__init__.py b/nvflare/dashboard/application/__init__.py index a100099f7b..5e548fb0f3 100644 --- a/nvflare/dashboard/application/__init__.py +++ b/nvflare/dashboard/application/__init__.py @@ -40,9 +40,13 @@ def init_app(): if credential is None: print("Please set env var NVFL_CREDENTIAL") exit(1) - email = credential.split(":")[0] - pwd = credential.split(":")[1] - Store.seed_user(email, pwd) + parts = credential.split(":") + if len(parts) != 3: + print(f"Invalid value '{credential}' for env var NVFL_CREDENTIAL: it must be email:password:org") + email = parts[0] + pwd = parts[1] + org = parts[2] + Store.seed_user(email, pwd, org) with open(os.path.join(web_root, ".db_init_done"), "ab") as f: f.write(bytes()) return app diff --git a/nvflare/dashboard/application/blob.py b/nvflare/dashboard/application/blob.py index 3f2275b75a..cf0b34b49d 100644 --- a/nvflare/dashboard/application/blob.py +++ b/nvflare/dashboard/application/blob.py @@ -13,301 +13,143 @@ # limitations under the License. import io -import json import os +import re import subprocess import tempfile -from nvflare.lighter import tplt_utils, utils - -from .cert import CertPair, Entity, deserialize_ca_key, make_cert +from nvflare.lighter.constants import PropKey +from nvflare.lighter.entity import Project as ProvProject +from nvflare.lighter.impl.aws import AWSBuilder +from nvflare.lighter.impl.azure import AzureBuilder +from nvflare.lighter.impl.cert import CertBuilder +from nvflare.lighter.impl.signature import SignatureBuilder +from nvflare.lighter.impl.static_file import StaticFileBuilder +from nvflare.lighter.impl.workspace import WorkspaceBuilder +from nvflare.lighter.provisioner import Provisioner + +from .cert import deserialize_ca_key from .models import Client, Project, User - -lighter_folder = os.path.dirname(utils.__file__) -template = utils.load_yaml(os.path.join(lighter_folder, "templates", "master_template.yml")) -supported_csps = ["aws", "azure"] -for csp in supported_csps: - csp_template_file = os.path.join(lighter_folder, "templates", f"{csp}_template.yml") - if os.path.exists(csp_template_file): - template.update(utils.load_yaml(csp_template_file)) +from .store import Store, inc_dl -def get_csp_start_script_name(csp): - return f"{csp}_start.sh" +class DummyLogger: + """This dummy logger is used to suppress all log messages generated by the Provisioner, except for errors. + We print error messages to stdout. + """ + def info(self, msg: str): + pass -def gen_overseer(key): - project = Project.query.first() - entity = Entity(project.overseer) - issuer = Entity(project.short_name) - signing_cert_pair = CertPair(issuer, project.root_key, project.root_cert) - cert_pair = make_cert(entity, signing_cert_pair) - with tempfile.TemporaryDirectory() as tmp_dir: - overseer_dir = os.path.join(tmp_dir, entity.name) - dest_dir = os.path.join(overseer_dir, "startup") - os.mkdir(overseer_dir) - os.mkdir(dest_dir) - utils._write( - os.path.join(dest_dir, "start.sh"), - template["start_ovsr_sh"], - "t", - exe=True, - ) - utils._write( - os.path.join(dest_dir, "gunicorn.conf.py"), - utils.sh_replace(template["gunicorn_conf_py"], {"port": "8443"}), - "t", - exe=False, - ) - utils._write_pki(type="overseer", dest_dir=dest_dir, cert_pair=cert_pair, root_cert=project.root_cert) - run_args = ["zip", "-rq", "-P", key, "tmp.zip", "."] - subprocess.run(run_args, cwd=tmp_dir) - fileobj = io.BytesIO() - with open(os.path.join(tmp_dir, "tmp.zip"), "rb") as fo: - fileobj.write(fo.read()) - fileobj.seek(0) - return fileobj, f"{entity.name}.zip" + def error(self, msg: str): + print(f"ERROR: {msg}") + def debug(self, msg: str): + pass -def gen_server(key, first_server=True): - project = Project.query.first() - if first_server: - entity = Entity(project.server1) - fl_port = 8002 - admin_port = 8003 - else: - entity = Entity(project.server2) - fl_port = 8102 - admin_port = 8103 - issuer = Entity(project.short_name) - signing_cert_pair = CertPair(issuer, project.root_key, project.root_cert) - cert_pair = make_cert(entity, signing_cert_pair) + def warning(self, msg: str): + pass - config = json.loads(template["fed_server"]) - server_0 = config["servers"][0] - server_0["name"] = project.short_name - server_0["service"]["target"] = f"{entity.name}:{fl_port}" - server_0["service"]["scheme"] = project.scheme if hasattr(project, "scheme") else "grpc" - server_0["admin_host"] = entity.name - server_0["admin_port"] = admin_port - if project.ha_mode: - overseer_agent = {"path": "nvflare.ha.overseer_agent.HttpOverseerAgent"} - overseer_agent["args"] = { - "role": "server", - "overseer_end_point": f"https://{project.overseer}:8443/api/v1", - "project": project.short_name, - "name": entity.name, - "fl_port": str(fl_port), - "admin_port": str(admin_port), - } - else: - overseer_agent = {"path": "nvflare.ha.dummy_overseer_agent.DummyOverseerAgent"} - overseer_agent["args"] = {"sp_end_point": f"{project.server1}:8002:8003"} - config["overseer_agent"] = overseer_agent - replacement_dict = { - "admin_port": admin_port, - "fed_learn_port": fl_port, - "config_folder": "config", - "ha_mode": "true" if project.ha_mode else "false", - "docker_image": project.app_location.split(" ")[-1] if project.app_location else "nvflare/nvflare", - "org_name": "", - "type": "server", - "cln_uid": "", +def _get_provisioner(root_dir: str, scheme, docker_image=None): + overseer_agent = { + "path": "nvflare.ha.dummy_overseer_agent.DummyOverseerAgent", + "overseer_exists": False, + "args": {"sp_end_point": "server:8002:8003"}, } - tplt = tplt_utils.Template(template) + builders = [ + WorkspaceBuilder(), + StaticFileBuilder( + config_folder="config", + scheme=scheme, + docker_image=docker_image, + overseer_agent=overseer_agent, + ), + AWSBuilder(), + AzureBuilder(), + CertBuilder(), + SignatureBuilder(), + ] + return Provisioner(root_dir, builders) + + +def gen_server_blob(key): + return _gen_kit(key) + + +def _gen_kit(download_key, prepare_target_cb=None, **cb_kwargs): + # validate download_key + allowed_pattern = r"^[A-Za-z0-9]+$" + if not re.match(allowed_pattern, download_key): + raise RuntimeError(f"ERROR: detected unsafe download key: {download_key}") + + u = Store.get_user(1) + super_user = u.get("user") + fl_port = 8002 + admin_port = 8003 with tempfile.TemporaryDirectory() as tmp_dir: - server_dir = os.path.join(tmp_dir, entity.name) - dest_dir = os.path.join(server_dir, "startup") - os.mkdir(server_dir) - os.mkdir(dest_dir) - utils._write_common( - type="server", - dest_dir=dest_dir, - template=template, - tplt=tplt, - replacement_dict=replacement_dict, - config=config, + project = Project.query.first() + scheme = project.scheme if hasattr(project, "scheme") else "grpc" + docker_image = project.app_location.split(" ")[-1] if project.app_location else "nvflare/nvflare" + provisioner = _get_provisioner(tmp_dir, scheme, docker_image) + + # the root key is protected by password + root_pri_key = deserialize_ca_key(project.root_key) + + prov_project = ProvProject( + project.short_name, + project.description, + props={ + "api_version": 3, + }, + root_private_key=root_pri_key, + serialized_root_cert=project.root_cert, ) - utils._write_pki(type="server", dest_dir=dest_dir, cert_pair=cert_pair, root_cert=project.root_cert) - if not project.ha_mode: - for csp in supported_csps: - utils._write( - os.path.join(dest_dir, get_csp_start_script_name(csp)), - tplt.get_start_sh(csp=csp, type="server", entity=entity), - "t", - exe=True, - ) - signatures = utils.sign_all(dest_dir, deserialize_ca_key(project.root_key)) - json.dump(signatures, open(os.path.join(dest_dir, "signature.json"), "wt")) - - # local folder creation - dest_dir = os.path.join(server_dir, "local") - os.mkdir(dest_dir) - utils._write_local(type="server", dest_dir=dest_dir, template=template) - - # workspace folder file - utils._write( - os.path.join(server_dir, "readme.txt"), - template["readme_fs"], - "t", - ) - run_args = ["zip", "-rq", "-P", key, "tmp.zip", "."] - subprocess.run(run_args, cwd=tmp_dir) - fileobj = io.BytesIO() - with open(os.path.join(tmp_dir, "tmp.zip"), "rb") as fo: - fileobj.write(fo.read()) - fileobj.seek(0) - return fileobj, f"{entity.name}.zip" - - -def gen_client(key, id): - project = Project.query.first() - client = Client.query.get(id) - entity = Entity(client.name, client.organization.name) - issuer = Entity(project.short_name) - signing_cert_pair = CertPair(issuer, project.root_key, project.root_cert) - cert_pair = make_cert(entity, signing_cert_pair) - config = json.loads(template["fed_client"]) - config["servers"][0]["name"] = project.short_name - config["servers"][0]["service"]["scheme"] = project.scheme if hasattr(project, "scheme") else "grpc" - replacement_dict = { - "client_name": entity.name, - "config_folder": "config", - "docker_image": project.app_location.split(" ")[-1] if project.app_location else "nvflare/nvflare", - "org_name": entity.org, - "type": "client", - "cln_uid": f"uid={entity.name}", - } - for k in ["client_name", "org_name", "cln_uid"]: - value = replacement_dict[k] - escaped_value = value.replace("'", "\\'") - replacement_dict[k] = escaped_value - - if project.ha_mode: - overseer_agent = {"path": "nvflare.ha.overseer_agent.HttpOverseerAgent"} - overseer_agent["args"] = { - "role": "client", - "overseer_end_point": f"https://{project.overseer}:8443/api/v1", - "project": project.short_name, - "name": entity.name, - } - else: - overseer_agent = {"path": "nvflare.ha.dummy_overseer_agent.DummyOverseerAgent"} - overseer_agent["args"] = {"sp_end_point": f"{project.server1}:8002:8003"} - config["overseer_agent"] = overseer_agent - - tplt = tplt_utils.Template(template) - with tempfile.TemporaryDirectory() as tmp_dir: - client_dir = os.path.join(tmp_dir, entity.name) - dest_dir = os.path.join(client_dir, "startup") - os.mkdir(client_dir) - os.mkdir(dest_dir) - - utils._write_pki(type="client", dest_dir=dest_dir, cert_pair=cert_pair, root_cert=project.root_cert) - utils._write_common( - type="client", - dest_dir=dest_dir, - template=template, - tplt=tplt, - replacement_dict=replacement_dict, - config=config, + # use org of superuser + org = super_user.get("organization", "nvflare") + server_name = project.server1 + server = prov_project.set_server( + name=server_name, + org=org, + props={ + "fed_learn_port": fl_port, + "admin_port": admin_port, + "default_host": server_name, + }, ) - for csp in supported_csps: - utils._write( - os.path.join(dest_dir, get_csp_start_script_name(csp)), - tplt.get_start_sh(csp=csp, type="client", entity=entity), - "t", - exe=True, - ) - - signatures = utils.sign_all(dest_dir, deserialize_ca_key(project.root_key)) - json.dump(signatures, open(os.path.join(dest_dir, "signature.json"), "wt")) - - # local folder creation - dest_dir = os.path.join(client_dir, "local") - os.mkdir(dest_dir) - utils._write_local(type="client", dest_dir=dest_dir, template=template, capacity=client.capacity.capacity) - - # workspace folder file - utils._write( - os.path.join(client_dir, "readme.txt"), - template["readme_fc"], - "t", - ) + target = server + if prepare_target_cb is not None: + target = prepare_target_cb(prov_project, **cb_kwargs) - run_args = ["zip", "-rq", "-P", key, "tmp.zip", "."] - subprocess.run(run_args, cwd=tmp_dir) + ctx = provisioner.provision(prov_project, logger=DummyLogger()) + result_dir = ctx.get_result_location() + ent_dir = os.path.join(result_dir, target.name) + subprocess.run(["zip", "-rq", "-P", download_key, "tmp.zip", "."], cwd=ent_dir) fileobj = io.BytesIO() - with open(os.path.join(tmp_dir, "tmp.zip"), "rb") as fo: + with open(os.path.join(ent_dir, "tmp.zip"), "rb") as fo: fileobj.write(fo.read()) fileobj.seek(0) - return fileobj, f"{entity.name}.zip" + return fileobj, f"{target.name}.zip" -def gen_user(key, id): - project = Project.query.first() - server_name = project.server1 - user = User.query.get(id) - entity = Entity(user.email, user.organization.name, user.role.name) - issuer = Entity(project.short_name) - signing_cert_pair = CertPair(issuer, project.root_key, project.root_cert) - cert_pair = make_cert(entity, signing_cert_pair) +def gen_client_blob(key, id): + return _gen_kit(key, _prepare_client, client_id=id) - config = json.loads(template["fed_admin"]) - replacement_dict = {"admin_name": entity.name, "cn": server_name, "admin_port": "8003", "docker_image": ""} - if project.ha_mode: - overseer_agent = {"path": "nvflare.ha.overseer_agent.HttpOverseerAgent"} - overseer_agent["args"] = { - "role": "admin", - "overseer_end_point": f"https://{project.overseer}:8443/api/v1", - "project": project.short_name, - "name": entity.name, - } - else: - overseer_agent = {"path": "nvflare.ha.dummy_overseer_agent.DummyOverseerAgent"} - overseer_agent["args"] = {"sp_end_point": f"{project.server1}:8002:8003"} - config["admin"].update({"overseer_agent": overseer_agent}) +def _prepare_client(prov_project: ProvProject, client_id): + client = Client.query.get(client_id) + inc_dl(Client, client_id) + return prov_project.add_client(name=client.name, org=client.organization.name, props={}) - with tempfile.TemporaryDirectory() as tmp_dir: - user_dir = os.path.join(tmp_dir, entity.name) - dest_dir = os.path.join(user_dir, "startup") - os.mkdir(user_dir) - os.mkdir(dest_dir) - utils._write(os.path.join(dest_dir, "fed_admin.json"), json.dumps(config, indent=2), "t") - utils._write( - os.path.join(dest_dir, "fl_admin.sh"), - utils.sh_replace(template["fl_admin_sh"], replacement_dict), - "t", - exe=True, - ) - utils._write_pki(type="client", dest_dir=dest_dir, cert_pair=cert_pair, root_cert=project.root_cert) - signatures = utils.sign_all(dest_dir, deserialize_ca_key(project.root_key)) - json.dump(signatures, open(os.path.join(dest_dir, "signature.json"), "wt")) +def gen_user_blob(key, id): + return _gen_kit(key, _prepare_user, user_id=id) - # local folder creation - dest_dir = os.path.join(user_dir, "local") - os.mkdir(dest_dir) - # workspace folder file - utils._write( - os.path.join(user_dir, "readme.txt"), - template["readme_am"], - "t", - ) - utils._write( - os.path.join(user_dir, "system_info.ipynb"), - utils.sh_replace(template["adm_notebook"], replacement_dict), - "t", - ) - run_args = ["zip", "-rq", "-P", key, "tmp.zip", "."] - subprocess.run(run_args, cwd=tmp_dir) - fileobj = io.BytesIO() - with open(os.path.join(tmp_dir, "tmp.zip"), "rb") as fo: - fileobj.write(fo.read()) - fileobj.seek(0) - return fileobj, f"{entity.name}.zip" +def _prepare_user(prov_project: ProvProject, user_id): + user = User.query.get(user_id) + inc_dl(User, user_id) + admin = prov_project.add_admin(name=user.email, org=user.organization.name, props={PropKey.ROLE: user.role.name}) + return admin diff --git a/nvflare/dashboard/application/cert.py b/nvflare/dashboard/application/cert.py index d04b4d27ad..c8027ff55b 100644 --- a/nvflare/dashboard/application/cert.py +++ b/nvflare/dashboard/application/cert.py @@ -12,15 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime import os from dataclasses import dataclass -from cryptography import x509 from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import serialization + +from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert, serialize_pri_key dashboard_pp = os.environ.get("NVFL_DASHBOARD_PP") if dashboard_pp is not None: @@ -45,73 +43,6 @@ class CertPair: ser_cert: str = None -def serialize_pri_key(pri_key, passphrase=None): - if passphrase is None or not isinstance(passphrase, bytes): - return pri_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ) - else: - return pri_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.BestAvailableEncryption(password=passphrase), - ) - - -def serialize_cert(cert): - return cert.public_bytes(serialization.Encoding.PEM) - - -def generate_keys(): - pri_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) - pub_key = pri_key.public_key() - return pri_key, pub_key - - -def x509_name(cn_name, org_name=None, role=None): - name = [x509.NameAttribute(NameOID.COMMON_NAME, cn_name)] - if org_name is not None: - name.append(x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)) - if role: - name.append(x509.NameAttribute(NameOID.UNSTRUCTURED_NAME, role)) - return x509.Name(name) - - -def generate_cert(subject, issuer, signing_pri_key, subject_pub_key, valid_days=360, ca=False): - x509_subject = x509_name(subject.name, subject.org, subject.role) - x509_issuer = x509_name(issuer.name, issuer.org, issuer.role) - builder = ( - x509.CertificateBuilder() - .subject_name(x509_subject) - .issuer_name(x509_issuer) - .public_key(subject_pub_key) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.datetime.utcnow()) - .not_valid_after( - # Our certificate will be valid for 360 days - datetime.datetime.utcnow() - + datetime.timedelta(days=valid_days) - # Sign our certificate with our private key - ) - .add_extension(x509.SubjectAlternativeName([x509.DNSName(subject.name)]), critical=False) - ) - if ca: - builder = ( - builder.add_extension( - x509.SubjectKeyIdentifier.from_public_key(subject_pub_key), - critical=False, - ) - .add_extension( - x509.AuthorityKeyIdentifier.from_issuer_public_key(subject_pub_key), - critical=False, - ) - .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=False) - ) - return builder.sign(signing_pri_key, hashes.SHA256(), default_backend()) - - def _pack(entity, pri_key, cert, passphrase=None): ser_pri_key = serialize_pri_key(pri_key, passphrase) ser_cert = serialize_cert(cert) @@ -121,17 +52,16 @@ def _pack(entity, pri_key, cert, passphrase=None): def make_root_cert(subject: Entity): pri_key, pub_key = generate_keys() - cert = generate_cert(subject=subject, issuer=subject, signing_pri_key=pri_key, subject_pub_key=pub_key, ca=True) + cert = generate_cert( + subject=Identity(subject.name, subject.org, subject.role), + issuer=Identity(subject.name, subject.org, subject.role), + signing_pri_key=pri_key, + subject_pub_key=pub_key, + ca=True, + ) return _pack(subject, pri_key, cert, passphrase=dashboard_pp) -def make_cert(subject: Entity, issuer_cert_pair: CertPair): - pri_key, pub_key = generate_keys() - issuer_pri_key = deserialize_ca_key(issuer_cert_pair.ser_pri_key) - cert = generate_cert(subject, issuer_cert_pair.owner, issuer_pri_key, pub_key, valid_days=360, ca=False) - return _pack(subject, pri_key, cert, passphrase=None) - - def deserialize_ca_key(ser_pri_key): pri_key = serialization.load_pem_private_key(ser_pri_key, password=dashboard_pp, backend=default_backend()) return pri_key diff --git a/nvflare/dashboard/application/clients.py b/nvflare/dashboard/application/clients.py index cf940bfa21..50c215e6b0 100644 --- a/nvflare/dashboard/application/clients.py +++ b/nvflare/dashboard/application/clients.py @@ -18,6 +18,7 @@ from nvflare.dashboard.application.constants import FLARE_DASHBOARD_NAMESPACE +from .blob import gen_client_blob from .store import Store, check_role @@ -82,7 +83,7 @@ def client_blob(id): c, p = check_role(creator_id, get_jwt(), get_jwt_identity()) if p or c: pin = request.json.get("pin") - fileobj, filename = Store.get_client_blob(pin, id) + fileobj, filename = gen_client_blob(pin, id) response = make_response(fileobj.read()) response.headers.set("Content-Type", "zip") response.headers.set("Content-Disposition", f'attachment; filename="{filename}"') diff --git a/nvflare/dashboard/application/project.py b/nvflare/dashboard/application/project.py index 09c007731b..076c2aa966 100644 --- a/nvflare/dashboard/application/project.py +++ b/nvflare/dashboard/application/project.py @@ -20,6 +20,7 @@ from nvflare.dashboard.application.constants import FLARE_DASHBOARD_NAMESPACE from . import jwt +from .blob import gen_server_blob from .store import Store @@ -108,12 +109,7 @@ def login(): def overseer_blob(): claims = get_jwt() if claims.get("role") == "project_admin": - pin = request.json.get("pin") - fileobj, filename = Store.get_overseer_blob(pin) - response = make_response(fileobj.read()) - response.headers.set("Content-Type", "zip") - response.headers.set("Content-Disposition", f'attachment; filename="{filename}"') - return response + return jsonify({"status": "unauthorized"}), 403 else: return jsonify({"status": "unauthorized"}), 403 @@ -124,7 +120,7 @@ def server_blob(id): claims = get_jwt() if claims.get("role") == "project_admin": pin = request.json.get("pin") - fileobj, filename = Store.get_server_blob(pin, id == 1) + fileobj, filename = gen_server_blob(pin) response = make_response(fileobj.read()) response.headers.set("Content-Type", "zip") response.headers.set("Content-Disposition", f'attachment; filename="{filename}"') diff --git a/nvflare/dashboard/application/store.py b/nvflare/dashboard/application/store.py index ea62d8a2a6..80deaf70b0 100644 --- a/nvflare/dashboard/application/store.py +++ b/nvflare/dashboard/application/store.py @@ -17,7 +17,6 @@ from werkzeug.security import check_password_hash, generate_password_hash -from .blob import gen_client, gen_overseer, gen_server, gen_user from .cert import Entity, make_root_cert from .models import Capacity, Client, Organization, Project, Role, User, db @@ -64,12 +63,12 @@ def ready(cls): return user.approval_state >= 100 if user else False @classmethod - def seed_user(cls, email, pwd): + def seed_user(cls, email, pwd, org): seed_user = { "name": "super_name", "email": email, "password": pwd, - "organization": "", + "organization": org, "role": "project_admin", "approval_state": 200, } @@ -134,16 +133,6 @@ def get_project(cls): project_dict = cls._add_registered_info(project_dict) return add_ok({"project": project_dict}) - @classmethod - def get_overseer_blob(cls, key): - fileobj, filename = gen_overseer(key) - return fileobj, filename - - @classmethod - def get_server_blob(cls, key, first_server=True): - fileobj, filename = gen_server(key, first_server) - return fileobj, filename - @classmethod def get_orgs(cls): all_orgs = Organization.query.all() @@ -256,12 +245,6 @@ def delete_client(cls, id): db.session.commit() return add_ok({}) - @classmethod - def get_client_blob(cls, key, id): - fileobj, filename = gen_client(key, id) - inc_dl(Client, id) - return fileobj, filename - @classmethod def create_user(cls, req): name = req.get("name", "") @@ -374,9 +357,3 @@ def delete_user(cls, id): db.session.commit() return add_ok({}) - - @classmethod - def get_user_blob(cls, key, id): - fileobj, filename = gen_user(key, id) - inc_dl(User, id) - return fileobj, filename diff --git a/nvflare/dashboard/application/users.py b/nvflare/dashboard/application/users.py index 3728c128ec..21afc3c559 100644 --- a/nvflare/dashboard/application/users.py +++ b/nvflare/dashboard/application/users.py @@ -18,6 +18,7 @@ from nvflare.dashboard.application.constants import FLARE_DASHBOARD_NAMESPACE +from .blob import gen_user_blob from .store import Store, check_role @@ -83,7 +84,7 @@ def user_blob(id): if not c and not p: return jsonify({"status": "unauthorized"}), 403 pin = request.json.get("pin") - fileobj, filename = Store.get_user_blob(pin, id) + fileobj, filename = gen_user_blob(pin, id) response = make_response(fileobj.read()) response.headers.set("Content-Type", "zip") response.headers.set("Content-Disposition", f'attachment; filename="{filename}"') diff --git a/nvflare/fuel/flare_api/api_spec.py b/nvflare/fuel/flare_api/api_spec.py index d11e526c3b..35190e9840 100644 --- a/nvflare/fuel/flare_api/api_spec.py +++ b/nvflare/fuel/flare_api/api_spec.py @@ -197,6 +197,29 @@ def download_job_result(self, job_id: str) -> str: """ pass + @abstractmethod + def list_job_components(self, job_id: str) -> List[str]: + """Get the list of additional job components for the specified job. + + Args: + job_id (str): ID of the job + + Returns: a list of the additional job components + + """ + pass + + def download_job_components(self, job_id: str) -> str: + """Download additional job components (e.g., ERRORLOG_site-1) for a specified job. + + Args: + job_id (str): ID of the job + + Returns: folder path to the location of the downloaded additional job components + + """ + pass + @abstractmethod def abort_job(self, job_id: str): """Abort the specified job diff --git a/nvflare/fuel/flare_api/flare_api.py b/nvflare/fuel/flare_api/flare_api.py index fbe73cae0b..d2ecbc2ca2 100644 --- a/nvflare/fuel/flare_api/flare_api.py +++ b/nvflare/fuel/flare_api/flare_api.py @@ -370,6 +370,36 @@ def download_job_result(self, job_id: str) -> str: location = meta.get(MetaKey.LOCATION) return location + def list_job_components(self, job_id: str) -> List[str]: + """Get the list of additional job components for the specified job. + + Args: + job_id (str): ID of the job + + Returns: a list of the additional job components + + """ + self._validate_job_id(job_id) + result = self._do_command(AdminCommandNames.LIST_JOB + " " + job_id) + meta = result[ResultKey.META] + job_components_list = meta.get(MetaKey.JOB_COMPONENTS, []) + return job_components_list + + def download_job_components(self, job_id: str) -> str: + """Download additional job components (e.g., ERRORLOG_site-1) for a specified job. + + Args: + job_id (str): ID of the job + + Returns: folder path to the location of the downloaded additional job components + + """ + self._validate_job_id(job_id) + result = self._do_command(AdminCommandNames.DOWNLOAD_JOB_COMPONENTS + " " + job_id) + meta = result[ResultKey.META] + location = meta.get(MetaKey.LOCATION) + return location + def abort_job(self, job_id: str): """Abort the specified job. diff --git a/nvflare/lighter/constants.py b/nvflare/lighter/constants.py index 594d23c2b7..434d58b33b 100644 --- a/nvflare/lighter/constants.py +++ b/nvflare/lighter/constants.py @@ -18,7 +18,6 @@ class WorkDir: WORKSPACE = "workspace" WIP = "wip_dir" STATE = "state_dir" - RESOURCES = "resources_dir" CURRENT_PROD_DIR = "current_prod_dir" @@ -63,10 +62,10 @@ class PropKey: class CtxKey(WorkDir, PropKey): PROJECT = "__project__" TEMPLATE = "__template__" + TEMP_FILES_LOADED = "__temp_files_loaded__" PROVISION_MODE = "__provision_model__" LOGGER = "__logger__" LAST_PROD_STAGE = "last_prod_stage" - TEMPLATE_FILES = "template_files" SERVER_NAME = "server_name" ROOT_CERT = "root_cert" ROOT_PRI_KEY = "root_pri_key" @@ -135,6 +134,11 @@ class TemplateSectionKey: HELM_CHART_DEPLOYMENT_SERVER = "helm_chart_deployment_server" RELAY_RESOURCES_JSON = "relay_resources_json" FED_RELAY = "fed_relay" + CLOUD_SCRIPT_HEADER = "cloud_script_header" + AZURE_START_SVR_HEADER_SH = "azure_start_svr_header_sh" + AZURE_START_COMMON_SH = "azure_start_common_sh" + AZURE_START_CLN_HEADER_SH = "azure_start_cln_header_sh" + AWS_START_SH = "aws_start_sh" class ProvFileName: @@ -174,6 +178,8 @@ class ProvFileName: CUSTOM_CA_CERT_FILE_NAME = "customRootCA.pem" RELAY_RESOURCES_JSON = "relay__resources.json" FED_RELAY_JSON = "fed_relay.json" + AZURE_START_SH = "azure_start.sh" + AWS_START_SH = "aws_start.sh" class CertFileBasename: diff --git a/nvflare/lighter/ctx.py b/nvflare/lighter/ctx.py index 6576f0b243..c0eba3fca1 100644 --- a/nvflare/lighter/ctx.py +++ b/nvflare/lighter/ctx.py @@ -13,11 +13,13 @@ # limitations under the License. import json import os -from typing import Optional +from typing import List, Optional, Union import yaml +import nvflare.lighter as prov from nvflare.lighter import utils +from nvflare.lighter.utils import load_yaml from .constants import CtxKey, PropKey, ProvisionMode from .entity import Entity, Project @@ -30,9 +32,8 @@ def __init__(self, workspace_root_dir: str, project: Project): wip_dir = os.path.join(workspace_root_dir, "wip") state_dir = os.path.join(workspace_root_dir, "state") - resources_dir = os.path.join(workspace_root_dir, "resources") - self.update({CtxKey.WIP: wip_dir, CtxKey.STATE: state_dir, CtxKey.RESOURCES: resources_dir}) - dirs = [workspace_root_dir, resources_dir, wip_dir, state_dir] + self.update({CtxKey.WIP: wip_dir, CtxKey.STATE: state_dir}) + dirs = [workspace_root_dir, wip_dir, state_dir] utils.make_dirs(dirs) # set commonly used data into ctx @@ -44,18 +45,30 @@ def __init__(self, workspace_root_dir: str, project: Project): fed_learn_port = server.get_prop(PropKey.FED_LEARN_PORT, 8002) self[CtxKey.FED_LEARN_PORT] = fed_learn_port self[CtxKey.SERVER_NAME] = server.name + self[CtxKey.TEMP_FILES_LOADED] = [] + self[CtxKey.TEMPLATE] = {} def get_project(self) -> Project: return self.get(CtxKey.PROJECT) - def set_template(self, template: dict): - self[CtxKey.TEMPLATE] = template + def load_templates(self, temp_files: Union[str, List[str]]): + if isinstance(temp_files, str): + temp_files = [temp_files] + elif not isinstance(temp_files, list): + raise ValueError(f"temp_files must be str or List[str] but got {type(temp_files)}") - def get_template(self): - return self.get(CtxKey.TEMPLATE) + prov_folder = os.path.dirname(prov.__file__) + temp_folder = os.path.join(prov_folder, "templates") + + loaded = self[CtxKey.TEMP_FILES_LOADED] + template = self[CtxKey.TEMPLATE] + for f in temp_files: + if f not in loaded: + template.update(load_yaml(os.path.join(temp_folder, f))) + loaded.append(f) def get_template_section(self, section_key: str): - template = self.get_template() + template = self.get(CtxKey.TEMPLATE) if not template: raise RuntimeError("template is not available") @@ -98,9 +111,6 @@ def get_local_dir(self, entity: Entity): def get_state_dir(self): return self.get(CtxKey.STATE) - def get_resources_dir(self): - return self.get(CtxKey.RESOURCES) - def get_workspace(self): return self.get(CtxKey.WORKSPACE) @@ -113,7 +123,7 @@ def json_load_template_section(self, section_key: str): def build_from_template( self, dest_dir: str, - temp_section: str, + temp_section: Union[str, List[str]], file_name, replacement=None, mode="t", @@ -140,12 +150,20 @@ def build_from_template( def build_section_from_template( self, - temp_section: str, + temp_section: Union[str, List[str]], replacement=None, content_modify_cb=None, **cb_kwargs, ): - section = self.get_template_section(temp_section) + if isinstance(temp_section, str): + temp_section = [temp_section] + elif not isinstance(temp_section, list): + raise ValueError(f"temp_section must be str or List[str] but got {type(temp_section)}") + + section = "" + for s in temp_section: + section += self.get_template_section(s) + if replacement: section = utils.sh_replace(section, replacement) if content_modify_cb: diff --git a/nvflare/lighter/entity.py b/nvflare/lighter/entity.py index 3175063ff3..a2bec1bcfa 100644 --- a/nvflare/lighter/entity.py +++ b/nvflare/lighter/entity.py @@ -315,7 +315,7 @@ def __init__( participants=None, props: dict = None, serialized_root_cert=None, - serialized_root_private_key=None, + root_private_key=None, ): """A container class to hold information about this FL project. @@ -327,7 +327,7 @@ def __init__( participants: if provided, list of participants of the project props: properties of the project serialized_root_cert: if provided, the root cert to be used for the project - serialized_root_private_key: if provided, the root private key for signing certs of sites and admins + root_private_key: if provided, the root private key for signing certs of sites and admins Raises: ValueError: when participant criteria is violated @@ -335,12 +335,12 @@ def __init__( Entity.__init__(self, "project", name, props) if serialized_root_cert: - if not serialized_root_private_key: - raise ValueError("missing serialized_root_private_key while serialized_root_cert is provided") + if not root_private_key: + raise ValueError("missing root_private_key while serialized_root_cert is provided") self.description = description self.serialized_root_cert = serialized_root_cert - self.serialized_root_private_key = serialized_root_private_key + self.root_private_key = root_private_key self.server = None self.overseer = None self.clients = [] @@ -377,6 +377,7 @@ def set_server(self, name: str, org: str, props: dict): self._check_unique_name(name) self.server = Participant(ParticipantType.SERVER, name, org, props, self) self.all_names[name] = True + return self.server def get_server(self): """Get the server definition. Only one server is supported! @@ -424,8 +425,10 @@ def add_participant(self, participant: Participant): def add_client(self, name: str, org: str, props: dict): self._check_unique_name(name) - self.clients.append(Participant(ParticipantType.CLIENT, name, org, props, self)) + client = Participant(ParticipantType.CLIENT, name, org, props, self) + self.clients.append(client) self.all_names[name] = True + return client def get_clients(self): return self.clients @@ -446,6 +449,7 @@ def add_admin(self, name: str, org: str, props: dict): raise ValueError(f"missing role in admin '{name}'") self.admins.append(admin) self.all_names[name] = True + return admin def get_admins(self): return self.admins diff --git a/nvflare/lighter/impl/aws.py b/nvflare/lighter/impl/aws.py new file mode 100644 index 0000000000..f1f0ba3036 --- /dev/null +++ b/nvflare/lighter/impl/aws.py @@ -0,0 +1,65 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.lighter.constants import ProvFileName, TemplateSectionKey +from nvflare.lighter.spec import Builder, Project, ProvisionContext + + +class AWSBuilder(Builder): + def __init__(self): + Builder.__init__(self) + + def initialize(self, project: Project, ctx: ProvisionContext): + ctx.load_templates(["master_template.yml", "aws_template.yml"]) + + def build(self, project: Project, ctx: ProvisionContext): + # build server + server = project.get_server() + dest_dir = ctx.get_kit_dir(server) + replacement = { + "type": "server", + "inbound_rule": "aws ec2 authorize-security-group-ingress --region ${REGION} --group-id $sg_id --protocol tcp --port 8002-8003 --cidr 0.0.0.0/0 >> ${LOGFILE}.sec_grp.log", + "cln_uid": "", + "server_name": server.name, + "ORG": server.org, + } + ctx.build_from_template( + dest_dir=dest_dir, + file_name=ProvFileName.AWS_START_SH, + temp_section=[ + TemplateSectionKey.CLOUD_SCRIPT_HEADER, + TemplateSectionKey.AWS_START_SH, + ], + replacement=replacement, + exe=True, + ) + + for client in project.get_clients(): + dest_dir = ctx.get_kit_dir(client) + replacement = { + "type": "client", + "inbound_rule": "", + "cln_uid": f"uid={client.name}", + "ORG": client.org, + } + + ctx.build_from_template( + dest_dir=dest_dir, + file_name=ProvFileName.AWS_START_SH, + temp_section=[ + TemplateSectionKey.CLOUD_SCRIPT_HEADER, + TemplateSectionKey.AWS_START_SH, + ], + replacement=replacement, + exe=True, + ) diff --git a/nvflare/lighter/impl/azure.py b/nvflare/lighter/impl/azure.py new file mode 100644 index 0000000000..7973bc2212 --- /dev/null +++ b/nvflare/lighter/impl/azure.py @@ -0,0 +1,51 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.lighter.constants import ProvFileName, TemplateSectionKey +from nvflare.lighter.spec import Builder, Project, ProvisionContext + + +class AzureBuilder(Builder): + def __init__(self): + Builder.__init__(self) + + def initialize(self, project: Project, ctx: ProvisionContext): + ctx.load_templates(["master_template.yml", "azure_template.yml"]) + + def build(self, project: Project, ctx: ProvisionContext): + # build server + server = project.get_server() + dest_dir = ctx.get_kit_dir(server) + ctx.build_from_template( + dest_dir=dest_dir, + file_name=ProvFileName.AZURE_START_SH, + temp_section=[ + TemplateSectionKey.CLOUD_SCRIPT_HEADER, + TemplateSectionKey.AZURE_START_SVR_HEADER_SH, + TemplateSectionKey.AZURE_START_COMMON_SH, + ], + exe=True, + ) + + for participant in project.get_clients(): + dest_dir = ctx.get_kit_dir(participant) + ctx.build_from_template( + dest_dir=dest_dir, + file_name=ProvFileName.AZURE_START_SH, + temp_section=[ + TemplateSectionKey.CLOUD_SCRIPT_HEADER, + TemplateSectionKey.AZURE_START_CLN_HEADER_SH, + TemplateSectionKey.AZURE_START_COMMON_SH, + ], + exe=True, + ) diff --git a/nvflare/lighter/impl/cert.py b/nvflare/lighter/impl/cert.py index 9157ff2038..d380baaf86 100644 --- a/nvflare/lighter/impl/cert.py +++ b/nvflare/lighter/impl/cert.py @@ -12,21 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime import json import os from cryptography import x509 from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization from cryptography.x509.oid import NameOID from nvflare.lighter.constants import CertFileBasename, CtxKey, ParticipantType, PropKey from nvflare.lighter.ctx import ProvisionContext from nvflare.lighter.entity import Participant, Project from nvflare.lighter.spec import Builder -from nvflare.lighter.utils import serialize_cert, serialize_pri_key +from nvflare.lighter.utils import Identity, generate_cert, generate_keys, serialize_cert, serialize_pri_key class _CertState: @@ -115,7 +113,16 @@ def initialize(self, project: Project, ctx: ProvisionContext): self.persistent_state = _CertState(state_dir) state = self.persistent_state - if state.is_available: + if project.root_private_key: + # using project provided credentials + self.serialized_cert = project.serialized_root_cert + self.root_cert = x509.load_pem_x509_certificate(self.serialized_cert, default_backend()) + self.pri_key = project.root_private_key + self.pub_key = self.pri_key.public_key() + self.subject = self.root_cert.subject + self.issuer = self.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value + state.is_available = True + elif state.is_available: state_root_cert = state.get_root_cert() self.serialized_cert = state_root_cert.encode("ascii") self.root_cert = x509.load_pem_x509_certificate(self.serialized_cert, default_backend()) @@ -132,7 +139,7 @@ def initialize(self, project: Project, ctx: ProvisionContext): def _build_root(self, subject, subject_org): assert isinstance(self.persistent_state, _CertState) if not self.persistent_state.is_available: - pri_key, pub_key = self._generate_keys() + pri_key, pub_key = generate_keys() self.issuer = subject self.root_cert = self._generate_cert(subject, subject_org, self.issuer, pri_key, pub_key, ca=True) self.pri_key = pri_key @@ -248,7 +255,7 @@ def build(self, project: Project, ctx: ProvisionContext): self._build_write_cert_pair(admin, CertFileBasename.CLIENT, ctx) def get_pri_key_cert(self, participant: Participant): - pri_key, pub_key = self._generate_keys() + pri_key, pub_key = generate_keys() subject = participant.subject subject_org = participant.org if participant.type == ParticipantType.ADMIN: @@ -269,13 +276,7 @@ def get_pri_key_cert(self, participant: Participant): return pri_key, cert @staticmethod - def _generate_keys(): - pri_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) - pub_key = pri_key.public_key() - return pri_key, pub_key - def _generate_cert( - self, subject, subject_org, issuer, @@ -286,58 +287,25 @@ def _generate_cert( role=None, server: Participant = None, ): - x509_subject = self._x509_name(subject, subject_org, role) - x509_issuer = self._x509_name(issuer) - - builder = ( - x509.CertificateBuilder() - .subject_name(x509_subject) - .issuer_name(x509_issuer) - .public_key(subject_pub_key) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.datetime.utcnow()) - .not_valid_after( - # Our certificate will be valid for 360 days - datetime.datetime.utcnow() - + datetime.timedelta(days=valid_days) - # Sign our certificate with our private key - ) - ) - if ca: - builder = ( - builder.add_extension( - x509.SubjectKeyIdentifier.from_public_key(subject_pub_key), - critical=False, - ) - .add_extension( - x509.AuthorityKeyIdentifier.from_issuer_public_key(subject_pub_key), - critical=False, - ) - .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=False) - ) + server_default_host = None + server_additional_hosts = None if server: # This is to generate a server cert. # Use SubjectAlternativeName for all host names - default_host = server.get_default_host() - host_names = server.get_prop(PropKey.HOST_NAMES) - sans = [x509.DNSName(default_host)] - if host_names: - for h in host_names: - if h != default_host: - sans.append(x509.DNSName(h)) - builder = builder.add_extension(x509.SubjectAlternativeName(sans), critical=False) - else: - builder = builder.add_extension(x509.SubjectAlternativeName([x509.DNSName(subject)]), critical=False) - return builder.sign(signing_pri_key, hashes.SHA256(), default_backend()) - - def _x509_name(self, cn_name, org_name=None, role=None): - name = [x509.NameAttribute(NameOID.COMMON_NAME, cn_name)] - if org_name is not None: - name.append(x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)) - if role: - name.append(x509.NameAttribute(NameOID.UNSTRUCTURED_NAME, role)) - return x509.Name(name) + server_default_host = server.get_default_host() + server_additional_hosts = server.get_prop(PropKey.HOST_NAMES) + + return generate_cert( + subject=Identity(subject, subject_org, role), + issuer=Identity(issuer), + signing_pri_key=signing_pri_key, + subject_pub_key=subject_pub_key, + valid_days=valid_days, + ca=ca, + server_default_host=server_default_host, + server_additional_hosts=server_additional_hosts, + ) def finalize(self, project: Project, ctx: ProvisionContext): assert isinstance(self.persistent_state, _CertState) diff --git a/nvflare/lighter/impl/docker_launcher.py b/nvflare/lighter/impl/docker_launcher.py index b876628041..aa47865881 100644 --- a/nvflare/lighter/impl/docker_launcher.py +++ b/nvflare/lighter/impl/docker_launcher.py @@ -168,6 +168,9 @@ def _build_client(self, client: Participant, ctx: ProvisionContext): exe=True, ) + def initialize(self, project: Project, ctx: ProvisionContext): + ctx.load_templates("docker_launcher_template.yml") + def build(self, project: Project, ctx: ProvisionContext): compose = ctx.yaml_load_template_section(TemplateSectionKey.COMPOSE_YAML) self.services = compose.get("services") diff --git a/nvflare/lighter/impl/helm_chart.py b/nvflare/lighter/impl/helm_chart.py index 570e1113eb..2e236b11c7 100644 --- a/nvflare/lighter/impl/helm_chart.py +++ b/nvflare/lighter/impl/helm_chart.py @@ -33,6 +33,7 @@ def __init__(self, docker_image): self.helm_chart_templates_directory = None def initialize(self, project: Project, ctx: ProvisionContext): + ctx.load_templates("master_template.yml") self.helm_chart_directory = os.path.join(ctx.get_wip_dir(), ProvFileName.HELM_CHART_DIR) os.mkdir(self.helm_chart_directory) diff --git a/nvflare/lighter/impl/static_file.py b/nvflare/lighter/impl/static_file.py index 08458e085b..ad410265bf 100644 --- a/nvflare/lighter/impl/static_file.py +++ b/nvflare/lighter/impl/static_file.py @@ -43,7 +43,6 @@ def __init__( download_job_url="", docker_image="", overseer_agent: dict = None, - components="", ): """Build all static files from template. @@ -67,7 +66,6 @@ def __init__( self.download_job_url = download_job_url self.app_validator = app_validator self.overseer_agent = overseer_agent - self.components = components def _build_overseer(self, overseer: Participant, ctx: ProvisionContext): dest_dir = ctx.get_kit_dir(overseer) @@ -829,6 +827,7 @@ def _determine_client_hierarchy(project: Project, ctx: ProvisionContext): ctx[CtxKey.CLIENT_MAP] = client_map def initialize(self, project: Project, ctx: ProvisionContext): + ctx.load_templates("master_template.yml") self._determine_relay_hierarchy(project, ctx) self._determine_client_hierarchy(project, ctx) diff --git a/nvflare/lighter/impl/workspace.py b/nvflare/lighter/impl/workspace.py index a868956861..c1ebff15b3 100644 --- a/nvflare/lighter/impl/workspace.py +++ b/nvflare/lighter/impl/workspace.py @@ -15,10 +15,9 @@ import os import shutil -import nvflare.lighter as prov from nvflare.lighter.constants import CtxKey from nvflare.lighter.spec import Builder, Project, ProvisionContext -from nvflare.lighter.utils import load_yaml, make_dirs +from nvflare.lighter.utils import make_dirs class WorkspaceBuilder(Builder): @@ -47,23 +46,7 @@ def __init__(self, template_file=None): Args: template_file: one or more template file names """ - self.template_files = template_file - - def _build_template(self, ctx: ProvisionContext): - prov_folder = os.path.dirname(prov.__file__) - temp_folder = os.path.join(prov_folder, "templates") - - temp_files_to_load = self.template_files - if not temp_files_to_load: - # load everything - temp_files_to_load = [f for f in os.listdir(temp_folder) if os.path.isfile(f)] - elif isinstance(temp_files_to_load, str): - temp_files_to_load = [temp_files_to_load] - - template = dict() - for f in temp_files_to_load: - template.update(load_yaml(os.path.join(temp_folder, f))) - ctx.set_template(template) + self.template_files = template_file # obsolete def initialize(self, project: Project, ctx: ProvisionContext): workspace_dir = ctx.get_workspace() @@ -74,7 +57,6 @@ def initialize(self, project: Project, ctx: ProvisionContext): if stage > last: last = stage ctx[CtxKey.LAST_PROD_STAGE] = last - self._build_template(ctx) def build(self, project: Project, ctx: ProvisionContext): participants = project.get_all_participants() diff --git a/nvflare/lighter/provisioner.py b/nvflare/lighter/provisioner.py index 94894b54fe..057347001c 100644 --- a/nvflare/lighter/provisioner.py +++ b/nvflare/lighter/provisioner.py @@ -89,8 +89,6 @@ def provision(self, project: Project, mode=None, logger=None) -> ProvisionContex workspace_root_dir = os.path.join(self.root_dir, project.name) ctx = ProvisionContext(workspace_root_dir, project) - if self.template: - ctx.set_template(self.template) if not mode: mode = ProvisionMode.NORMAL diff --git a/nvflare/lighter/utils.py b/nvflare/lighter/utils.py index 8cee919cca..909009cb51 100644 --- a/nvflare/lighter/utils.py +++ b/nvflare/lighter/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import json import os import random @@ -22,23 +23,105 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.primitives.asymmetric import padding, rsa +from cryptography.x509.oid import NameOID from nvflare.lighter.tool_consts import NVFLARE_SIG_FILE, NVFLARE_SUBMITTER_CRT_FILE -def serialize_pri_key(pri_key): - return pri_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), +class Identity: + def __init__(self, name: str, org: str = None, role: str = None): + self.name = name + self.org = org + self.role = role + + +def generate_cert( + subject: Identity, + issuer: Identity, + signing_pri_key, + subject_pub_key, + valid_days=360, + ca=False, + server_default_host=None, + server_additional_hosts=None, +): + if isinstance(server_additional_hosts, str): + server_additional_hosts = [server_additional_hosts] + + x509_subject = x509_name(subject.name, subject.org, subject.role) + x509_issuer = x509_name(issuer.name, issuer.org, issuer.role) + + builder = ( + x509.CertificateBuilder() + .subject_name(x509_subject) + .issuer_name(x509_issuer) + .public_key(subject_pub_key) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after(datetime.datetime.utcnow() + datetime.timedelta(days=valid_days)) ) + if ca: + builder = ( + builder.add_extension( + x509.SubjectKeyIdentifier.from_public_key(subject_pub_key), + critical=False, + ) + .add_extension( + x509.AuthorityKeyIdentifier.from_issuer_public_key(subject_pub_key), + critical=False, + ) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=False) + ) + + if server_default_host: + # This is to generate a server cert. + # Use SubjectAlternativeName for all host names + sans = [x509.DNSName(server_default_host)] + if server_additional_hosts: + for h in server_additional_hosts: + if h != server_default_host: + sans.append(x509.DNSName(h)) + builder = builder.add_extension(x509.SubjectAlternativeName(sans), critical=False) + else: + builder = builder.add_extension(x509.SubjectAlternativeName([x509.DNSName(subject.name)]), critical=False) + return builder.sign(signing_pri_key, hashes.SHA256(), default_backend()) + + +def serialize_pri_key(pri_key, passphrase=None): + if passphrase is None or not isinstance(passphrase, bytes): + return pri_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + else: + return pri_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.BestAvailableEncryption(password=passphrase), + ) def serialize_cert(cert): return cert.public_bytes(serialization.Encoding.PEM) +def generate_keys(): + pri_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) + pub_key = pri_key.public_key() + return pri_key, pub_key + + +def x509_name(cn_name, org_name=None, role=None): + name = [x509.NameAttribute(NameOID.COMMON_NAME, cn_name)] + if org_name is not None: + name.append(x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)) + if role: + name.append(x509.NameAttribute(NameOID.UNSTRUCTURED_NAME, role)) + return x509.Name(name) + + def load_crt(path): with open(path, "rb") as f: return load_crt_bytes(f.read()) @@ -326,69 +409,3 @@ def _write(file_full_path, content, mode, exe=False): def write(file_full_path, content, mode, exe=False): _write(file_full_path, content, mode, exe) - - -def _write_common(type, dest_dir, template, tplt, replacement_dict, config): - mapping = {"server": "svr", "client": "cln"} - write(os.path.join(dest_dir, f"fed_{type}.json"), json.dumps(config, indent=2), "t") - write( - os.path.join(dest_dir, "docker.sh"), - sh_replace(template[f"docker_{mapping[type]}_sh"], replacement_dict), - "t", - exe=True, - ) - write( - os.path.join(dest_dir, "start.sh"), - sh_replace(template[f"start_{mapping[type]}_sh"], replacement_dict), - "t", - exe=True, - ) - write( - os.path.join(dest_dir, "sub_start.sh"), - sh_replace(tplt.get_sub_start_sh(), replacement_dict), - "t", - exe=True, - ) - write( - os.path.join(dest_dir, "stop_fl.sh"), - template["stop_fl_sh"], - "t", - exe=True, - ) - - -def _write_local(type, dest_dir, template, capacity=""): - write( - os.path.join(dest_dir, "log_config.json.default"), - template["log_config"], - "t", - ) - write( - os.path.join(dest_dir, "privacy.json.sample"), - template["sample_privacy"], - "t", - ) - write( - os.path.join(dest_dir, "authorization.json.default"), - template["default_authz"], - "t", - ) - if type == "server": - resources = json.loads(template["local_server_resources"]) - elif type == "client": - resources = json.loads(template["local_client_resources"]) - for component in resources["components"]: - if "nvflare.app_common.resource_managers.gpu_resource_manager.GPUResourceManager" == component["path"]: - component["args"] = json.loads(capacity) - break - write( - os.path.join(dest_dir, "resources.json.default"), - json.dumps(resources, indent=2), - "t", - ) - - -def _write_pki(type, dest_dir, cert_pair, root_cert): - write(os.path.join(dest_dir, f"{type}.crt"), cert_pair.ser_cert, "b", exe=False) - write(os.path.join(dest_dir, f"{type}.key"), cert_pair.ser_pri_key, "b", exe=False) - write(os.path.join(dest_dir, "rootCA.pem"), root_cert, "b", exe=False) diff --git a/setup.cfg b/setup.cfg index b772acd3b7..386c4e42e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,7 +36,6 @@ install_requires = docker>=6.0 websockets>=10.4 pyhocon - tdigest [options.extras_require] HE = diff --git a/tests/unit_test/app_common/statistics/quantile_test.py b/tests/unit_test/app_common/statistics/quantile_test.py new file mode 100644 index 0000000000..38a85fe964 --- /dev/null +++ b/tests/unit_test/app_common/statistics/quantile_test.py @@ -0,0 +1,266 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from typing import List + +import numpy as np +import pandas as pd +import pytest + +from nvflare.apis.fl_context import FLContext +from nvflare.app_common.app_constant import StatisticsConstants +from nvflare.app_opt.statistics.df.df_core_statistics import DFStatisticsCore +from nvflare.app_opt.statistics.quantile_stats import compute_quantiles, merge_quantiles + +try: + from fastdigest import TDigest + + TDIGEST_AVAILABLE = True +except ImportError: + TDIGEST_AVAILABLE = False + + +class MockDFStats(DFStatisticsCore): + def __init__(self, given_median: int): + super().__init__() + self.median = given_median + self.data = {"train": None} + + def initialize(self, fl_ctx: FLContext): + self.load_data() + + def load_data(self): + data = np.concatenate( + (np.arange(0, self.median), [self.median], np.arange(self.median + 1, self.median * 2 + 1)) + ) + + # Shuffle the data to make it unordered + np.random.shuffle(data) + + # Create the DataFrame + df = pd.DataFrame(data, columns=["Feature"]) + self.data = {"train": df} + + +class MockDFStats2(DFStatisticsCore): + def __init__(self, data_array: List[int]): + super().__init__() + self.raw_data = data_array + self.data = {"train": None} + + def initialize(self, fl_ctx: FLContext): + self.load_data() + + def load_data(self): + # Create the DataFrame + df = pd.DataFrame(self.raw_data, columns=["Feature"]) + self.data = {"train": df} + + +class TestQuantile: + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_tdigest1(self): + # Small dataset + data = [1, 2, 3, 4, 5] + fd = TDigest(data) + + # Insert values + np_data = pd.Series(data) + + assert fd.quantile(0.25) == np_data.quantile(0.25) + assert fd.quantile(0.5) == np_data.quantile(0.5) + assert fd.quantile(0.75) == np_data.quantile(0.75) + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_tdigest2(self): + # Small dataset + data = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5] + fd = TDigest(data) + # Insert values + np_data = pd.Series(data) + + assert fd.quantile(0.25) == np_data.quantile(0.25) + assert fd.quantile(0.5) == np_data.quantile(0.5) + assert fd.quantile(0.75) == np_data.quantile(0.75) + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_tdigest3(self): + # Small dataset + data = [-50.0, -40.4, -30.3, -20.3, -10.1, 0, 1.1, 2.2, 3.3, 4.4, 5.5] + fd = TDigest(data) + + np_data = pd.Series(data) + + assert round(fd.quantile(0.25), 2) == np_data.quantile(0.25) + assert round(fd.quantile(0.5), 2) == np_data.quantile(0.5) + assert round(fd.quantile(0.75), 2) == np_data.quantile(0.75) + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_tdigest4(self): + # Small dataset + data = [-5, -4, -3, -2, -1, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0] + fd = TDigest(data) + + np_data = pd.Series(data) + + assert round(fd.quantile(0.25), 2) == np_data.quantile(0.25) + assert round(fd.quantile(0.5), 2) == np_data.quantile(0.5) + assert round(fd.quantile(0.75), 2) == np_data.quantile(0.75) + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_tdigest5(self): + # Small dataset + data1 = [x for x in range(-5, 0)] + data2 = [x * 1.0 for x in range(0, 6)] + data = data1 + data2 + fd = TDigest(data) + + assert fd.quantile(0.5) == 0 + assert fd.quantile(0.1) == -4 + assert fd.quantile(0.9) == 4 + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_tdigest6(self): + # Small dataset + data1 = [x for x in range(-10000, 0)] + data2 = [x * 1.0 for x in range(0, 10000 + 1)] + fd = TDigest(data1) + merged_fd = fd.merge(TDigest(data2)) + + fdx = TDigest(data1 + data2) + + np_data = pd.Series(data1 + data2) + + assert fdx.quantile(0.5) == np_data.quantile(0.5) + assert merged_fd.quantile(0.5) == np_data.quantile(0.5) + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_tdigest7(self): + median = 10 + data = np.concatenate((np.arange(0, median), [median], np.arange(median + 1, median * 2 + 1))) + # Shuffle the data to make it unordered + np.random.shuffle(data) + + fd = TDigest(data) + + v = fd.quantile(0.5) + + print(sorted(data), v, median) + + assert v == median + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_tdigest8(self): + + median = 10 + data = np.concatenate((np.arange(0, median), [median], np.arange(median + 1, median * 2 + 1))) + # Shuffle the data to make it unordered + np.random.shuffle(data) + data1 = [0, 1, 2, 3, 4, 5] + data2 = [100, 110, 120, 130, 140, 150] + + fd1 = TDigest(data1) + fd2 = TDigest(data2) + + fd = TDigest(data) + + fd.merge(fd1) + fd.merge(fd2) + + v = fd.quantile(0.5) + + print(sorted(data), v, median) + + assert v == median + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_tdigest_merge_serde(self): + + median = 10 + data = np.concatenate((np.arange(0, median), [median], np.arange(median + 1, median * 2 + 1))) + # Shuffle the data to make it unordered + np.random.shuffle(data) + data1 = [0, 1, 2, 3, 4, 5] + data2 = [100, 110, 120, 130, 140, 150] + + fd1 = TDigest(data1) + fd2 = TDigest(data2) + + fd = TDigest(data) + + fd.merge(fd1.from_dict(fd1.to_dict())) + fd.merge(fd2.from_dict(fd2.to_dict())) + + v = fd.quantile(0.5) + + print(sorted(data), v, median) + + assert v == median + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_tdigest_compress(self): + + digest = TDigest(range(101)) + print(f"Before: {len(digest)} centroids") + + before_median = digest.quantile(0.5) + before_25 = digest.quantile(0.25) + before_75 = digest.quantile(0.75) + + digest.compress(10) # compress to 10 (or fewer) centroids + + print(f" After: {len(digest)} centroids") + + print(json.dumps(digest.to_dict(), indent=2)) + + after_median = digest.quantile(0.5) + after_25 = digest.quantile(0.25) + after_75 = digest.quantile(0.75) + + assert before_median == after_median + assert before_25 == after_25 + assert before_75 == after_75 + assert len(digest) == 10 + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest is not installed") + def test_percentile_metrics(self): + stats_generator = MockDFStats(given_median=100) + stats_generator.load_data() + percentiles = stats_generator.quantiles("train", "Feature", percents=[0.5]) + result = percentiles.get(StatisticsConstants.STATS_QUANTILE) + digest_dict = percentiles.get(StatisticsConstants.STATS_DIGEST_COORD) + assert digest_dict is not None + assert result is not None + print(sorted(stats_generator.data["train"]["Feature"])) + + assert result.get(0.5) == stats_generator.median + + @pytest.mark.skipif(not TDIGEST_AVAILABLE, reason="fastdigest package not installed") + def test_percentile_metrics_aggregation(self): + stats_generators = [ + MockDFStats2(data_array=[0, 1, 2, 3, 4, 5, 6]), + MockDFStats(given_median=10), + MockDFStats2(data_array=[100, 110, 120, 130, 140, 150, 160]), + ] + global_digest = {} + for g in stats_generators: # each site/client + g.load_data() + local_quantiles = g.quantiles("train", "Feature", percents=[0.5]) + local_metrics = {"train": {"Feature": local_quantiles}} + global_digest = merge_quantiles(local_metrics, global_digest) + + result = compute_quantiles(global_digest, {"Feature": [0.5]}, 2) + + expected_median = 10 + assert result["train"]["Feature"].get(0.5) == expected_median diff --git a/tests/unit_test/app_opt/statistics/__init__.py b/tests/unit_test/app_opt/statistics/__init__.py deleted file mode 100644 index d9155f923f..0000000000 --- a/tests/unit_test/app_opt/statistics/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit_test/app_opt/statistics/percentiles_test.py b/tests/unit_test/app_opt/statistics/percentiles_test.py deleted file mode 100644 index 8a4aa6a428..0000000000 --- a/tests/unit_test/app_opt/statistics/percentiles_test.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -import numpy as np -import pandas as pd - -from nvflare.apis.fl_context import FLContext -from nvflare.app_common.app_constant import StatisticsConstants -from nvflare.app_common.statistics.numeric_stats import aggregate_centroids, compute_percentiles -from nvflare.app_opt.statistics.df.df_core_statistics import DFStatisticsCore - - -class MockDFStats(DFStatisticsCore): - def __init__(self, given_median: int): - super().__init__() - self.median = given_median - self.data = {"train": None} - - def initialize(self, fl_ctx: FLContext): - self.load_data() - - def load_data(self): - data = np.concatenate( - (np.arange(0, self.median), [self.median], np.arange(self.median + 1, self.median * 2 + 1)) - ) - - # Shuffle the data to make it unordered - np.random.shuffle(data) - - # Create the DataFrame - df = pd.DataFrame(data, columns=["Feature"]) - self.data = {"train": df} - - -class MockDFStats2(DFStatisticsCore): - def __init__(self, data_array: List[int]): - super().__init__() - self.raw_data = data_array - self.data = {"train": None} - - def initialize(self, fl_ctx: FLContext): - self.load_data() - - def load_data(self): - # Create the DataFrame - df = pd.DataFrame(self.raw_data, columns=["Feature"]) - self.data = {"train": df} - - -class TestPercentiles: - def test_percentile_metrics(self): - stats_generator = MockDFStats(given_median=100) - stats_generator.load_data() - percentiles = stats_generator.percentiles("train", "Feature", percents=[50]) - result = percentiles.get(StatisticsConstants.STATS_PERCENTILES_KEY) - print(f"{percentiles=}") - assert result is not None - assert result.get(50) == stats_generator.median - - def test_percentile_metrics_aggregation(self): - stats_generators = [ - MockDFStats2(data_array=[0, 1, 2, 3, 4, 5]), - MockDFStats(given_median=10), - MockDFStats2(data_array=[100, 110, 120, 130, 140, 150]), - ] - global_digest = {} - result = {} - for g in stats_generators: # each site/client - g.load_data() - local_percentiles = g.percentiles("train", "Feature", percents=[50]) - local_metrics = {"train": {"Feature": local_percentiles}} - aggregate_centroids(local_metrics, global_digest) - result = compute_percentiles(global_digest, {"Feature": [50]}, 2) - - expected_median = 10 - assert result["train"]["Feature"].get(50) == expected_median diff --git a/tests/unit_test/dashboard/conftest.py b/tests/unit_test/dashboard/conftest.py index 26b110674a..d4f07f6dee 100644 --- a/tests/unit_test/dashboard/conftest.py +++ b/tests/unit_test/dashboard/conftest.py @@ -32,7 +32,7 @@ def app(): if os.path.exists(sqlite_file): os.remove(sqlite_file) os.environ["DATABASE_URL"] = f"sqlite:///{sqlite_file}" - os.environ["NVFL_CREDENTIAL"] = f"{TEST_USER}:{TEST_PW}" + os.environ["NVFL_CREDENTIAL"] = f"{TEST_USER}:{TEST_PW}:nvidia" app = init_app() app.config.update( {