diff --git a/examples/advanced/flower/fedprox/flower_fedprox.ipynb b/examples/advanced/flower/fedprox/flower_fedprox.ipynb index d8b092c5f2..8411618cca 100644 --- a/examples/advanced/flower/fedprox/flower_fedprox.ipynb +++ b/examples/advanced/flower/fedprox/flower_fedprox.ipynb @@ -10,136 +10,96 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "c8ee32ae-cee9-454e-9b68-3427a2b7e991", + "execution_count": null, + "id": "88e86dad-ba2d-454a-9315-631691673ea4", "metadata": {}, "outputs": [], "source": [ - "!nvflare config -jt $PWD/../../../../job_templates" + "#!pip install nvflare flwr flwr_datasets" + ] + }, + { + "cell_type": "markdown", + "id": "53fc6251-8141-4c2d-902b-0892e50f1184", + "metadata": {}, + "source": [ + "## Running Flower's PyTorch-Quickstart\n", + "Run 3 Flower clients and Flower Server in parallel using NVFlare's simulator." ] }, { "cell_type": "code", - "execution_count": 2, - "id": "3479da05-40df-4d47-b942-825c10e03a8a", + "execution_count": null, + "id": "9e23fe35-0a67-4a37-b22e-314e6898cd59", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "The following are the variables you can change in the template\n", - "\n", - "---------------------------------------------------------------------------------------------------------------------------------------\n", - " \n", - " job folder: ./jobs/_flwr_cifar10 \n", - " \n", - "---------------------------------------------------------------------------------------------------------------------------------------\n", - " file_name var_name value component \n", - "---------------------------------------------------------------------------------------------------------------------------------------\n", - " meta.conf app ['@ALL'] \n", - " meta.conf mandatory_clients [] \n", - " meta.conf min_clients 2 \n", - "\n", - " config_fed_client.conf app_config \n", - " config_fed_client.conf app_script client.py \n", - " config_fed_client.conf component_ids ['metric_relay'] ExternalConfigurator \n", - " config_fed_client.conf config_file_name client_api_config.json ExternalConfigurator \n", - " config_fed_client.conf evaluate_task_name evaluate \n", - " config_fed_client.conf event_type fed.analytix_log_stats MetricRelay \n", - " config_fed_client.conf external_execution_wait 5.0 \n", - " config_fed_client.conf fed_event True MetricRelay \n", - " config_fed_client.conf heartbeat_interval 5.0 MetricRelay \n", - " config_fed_client.conf heartbeat_timeout 30.0 MetricRelay \n", - " config_fed_client.conf last_result_transfer_timeout 300.0 \n", - " config_fed_client.conf launch_once True SubprocessLauncher \n", - " config_fed_client.conf mode PASSIVE CellPipe \n", - " config_fed_client.conf monitor_interval 0.01 \n", - " config_fed_client.conf params_exchange_format pytorch \n", - " config_fed_client.conf params_transfer_type DIFF \n", - " config_fed_client.conf pipe_channel_name metric MetricRelay \n", - " config_fed_client.conf read_interval 0.5 \n", - " config_fed_client.conf root_url {ROOT_URL} CellPipe \n", - " config_fed_client.conf script python3 custom/{app_script} {app_c SubprocessLauncher \n", - " config_fed_client.conf secure_mode {SECURE_MODE} CellPipe \n", - " config_fed_client.conf site_name {SITE_NAME} CellPipe \n", - " config_fed_client.conf token {JOB_ID} CellPipe \n", - " config_fed_client.conf train_with_evaluation True \n", - " config_fed_client.conf workers 4 \n", - " config_fed_client.conf workspace_dir {WORKSPACE} CellPipe \n", - "\n", - " config_fed_server.conf allow_empty_global_weights False ScatterAndGather \n", - " config_fed_server.conf best_global_model_file_name best_FL_global_model.pt PTFileModelPersistor \n", - " config_fed_server.conf events ['fed.analytix_log_stats'] TBAnalyticsReceiver \n", - " config_fed_server.conf expected_data_kind WEIGHT_DIFF InTimeAccumulateWeightedAggregator \n", - " config_fed_server.conf global_model_file_name FL_global_model.pt PTFileModelPersistor \n", - " config_fed_server.conf ignore_result_error False ScatterAndGather \n", - " config_fed_server.conf key_metric accuracy IntimeModelSelector \n", - " config_fed_server.conf min_clients 2 ScatterAndGather \n", - " config_fed_server.conf model_class_path net.Net \n", - " config_fed_server.conf negate_key_metric False IntimeModelSelector \n", - " config_fed_server.conf num_rounds 2 ScatterAndGather \n", - " config_fed_server.conf persist_every_n_rounds 1 ScatterAndGather \n", - " config_fed_server.conf snapshot_every_n_rounds 1 ScatterAndGather \n", - " config_fed_server.conf start_round 0 ScatterAndGather \n", - " config_fed_server.conf task_check_period 0.5 ScatterAndGather \n", - " config_fed_server.conf train_timeout 0 ScatterAndGather \n", - " config_fed_server.conf validation_metric_name initial_metrics IntimeModelSelector \n", - " config_fed_server.conf wait_time_after_min_received 0 ScatterAndGather \n", - " config_fed_server.conf weigh_by_local_iter False IntimeModelSelector \n", - "\n", - "---------------------------------------------------------------------------------------------------------------------------------------\n" - ] - } - ], + "outputs": [], "source": [ - "app_script=\"client.py\"\n", + "from nvflare import SimulatorRunner \n", "\n", - "!nvflare job create -force -j \"./jobs/_flwr_cifar10\" -w \"sag_pt\" -sd \"flwr_scripts\" -f app/config_fed_client.conf app_script={app_script}" + "n_clients = 3\n", + "\n", + "simulator = SimulatorRunner(\n", + " job_folder=f\"./jobs/flwr_cifar10\",\n", + " workspace=f\"/tmp/nvflare/flwr_cifar10\",\n", + " n_clients=n_clients,\n", + " threads=n_clients\n", + ")\n", + "run_status = simulator.run()\n", + "print(\"Simulator finished with run_status\", run_status)" + ] + }, + { + "cell_type": "markdown", + "id": "b452ff94-cf23-4788-9f55-a0b675849910", + "metadata": {}, + "source": [ + "## Running Flower's PyTorch-Quickstart + TensorBoard streaming\n", + "Run 3 Flower clients and Flower Server in parallel using NVFlare's simulator.\n", + "\n", + "Stream the TensorBoard metrics to the server at each iteration using NVFlare's metric streaming." ] }, { "cell_type": "code", - "execution_count": 3, - "id": "88e86dad-ba2d-454a-9315-631691673ea4", + "execution_count": null, + "id": "9a23a0d6-1e6e-4bbf-8a0c-97e0d14cc432", "metadata": {}, "outputs": [], "source": [ - "#!pip install flwr flwr_datasets" + "#!pip install tensorboard" ] }, { "cell_type": "code", - "execution_count": 4, - "id": "9e23fe35-0a67-4a37-b22e-314e6898cd59", + "execution_count": null, + "id": "c90df20b-f3b3-4d4b-885b-830874a49552", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2024-03-06 15:59:17,528 - SimulatorRunner - INFO - Create the Simulator Server.\n", - "2024-03-06 15:59:17,532 - CoreCell - INFO - server: creating listener on tcp://0:36825\n", - "2024-03-06 15:59:17,550 - CoreCell - INFO - server: created backbone external listener for tcp://0:36825\n", - "2024-03-06 15:59:17,551 - ConnectorManager - INFO - 13991: Try start_listener Listener resources: {'secure': False, 'host': 'localhost'}\n", - "2024-03-06 15:59:17,553 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00002 PASSIVE tcp://0:61614] is starting\n", - "2024-03-06 15:59:18,056 - CoreCell - INFO - server: created backbone internal listener for tcp://localhost:61614\n", - "2024-03-06 15:59:18,057 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 PASSIVE tcp://0:36825] is starting\n", - "2024-03-06 15:59:18,133 - nvflare.fuel.hci.server.hci - INFO - Starting Admin Server localhost on Port 40275\n", - "2024-03-06 15:59:18,134 - SimulatorRunner - INFO - Deploy the Apps.\n", - "2024-03-06 15:59:18,141 - SimulatorRunner - INFO - Create the simulate clients.\n", - "2024-03-06 15:59:18,146 - ClientManager - INFO - Client: New client site-1@192.168.1.203 joined. Sent token: a429000a-ef61-4ea2-b750-05044df590b9. Total clients: 1\n", - "2024-03-06 15:59:18,147 - FederatedClient - INFO - Successfully registered client:site-1 for project simulator_server. Token:a429000a-ef61-4ea2-b750-05044df590b9 SSID:\n", - "2024-03-06 15:59:18,149 - ClientManager - INFO - Client: New client site-2@192.168.1.203 joined. Sent token: 8bb59863-a3bc-4518-88d1-f72ba9197f25. Total clients: 2\n", - "2024-03-06 15:59:18,150 - FederatedClient - INFO - Successfully registered client:site-2 for project simulator_server. Token:8bb59863-a3bc-4518-88d1-f72ba9197f25 SSID:\n", - "2024-03-06 15:59:18,152 - ClientManager - INFO - Client: New client site-3@192.168.1.203 joined. Sent token: e8cbe70f-195f-4d9d-b5a9-e7d4b2c1d736. Total clients: 3\n", - "2024-03-06 15:59:18,153 - FederatedClient - INFO - Successfully registered client:site-3 for project simulator_server. Token:e8cbe70f-195f-4d9d-b5a9-e7d4b2c1d736 SSID:\n", - "2024-03-06 15:59:18,154 - SimulatorRunner - INFO - Set the client status ready.\n", - "2024-03-06 15:59:18,155 - SimulatorRunner - INFO - Deploy and start the Server App.\n", - "2024-03-06 15:59:18,156 - Cell - INFO - Register blob CB for channel='server_command', topic='*'\n", - "2024-03-06 15:59:18,158 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", - "2024-03-06 15:59:18,158 - ServerCommandAgent - INFO - ServerCommandAgent cell register_request_cb: server.simulate_job\n" + "2024-03-08 17:29:25,752 - SimulatorRunner - INFO - Create the Simulator Server.\n", + "2024-03-08 17:29:25,757 - CoreCell - INFO - server: creating listener on tcp://0:41531\n", + "2024-03-08 17:29:25,774 - CoreCell - INFO - server: created backbone external listener for tcp://0:41531\n", + "2024-03-08 17:29:25,775 - ConnectorManager - INFO - 122617: Try start_listener Listener resources: {'secure': False, 'host': 'localhost'}\n", + "2024-03-08 17:29:25,777 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00002 PASSIVE tcp://0:50955] is starting\n", + "2024-03-08 17:29:26,279 - CoreCell - INFO - server: created backbone internal listener for tcp://localhost:50955\n", + "2024-03-08 17:29:26,282 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 PASSIVE tcp://0:41531] is starting\n", + "2024-03-08 17:29:26,357 - nvflare.fuel.hci.server.hci - INFO - Starting Admin Server localhost on Port 54963\n", + "2024-03-08 17:29:26,358 - SimulatorRunner - INFO - Deploy the Apps.\n", + "2024-03-08 17:29:26,365 - SimulatorRunner - INFO - Create the simulate clients.\n", + "2024-03-08 17:29:26,370 - ClientManager - INFO - Client: New client site-1@192.168.1.203 joined. Sent token: 3425755c-586c-4d3b-a276-33a9338d19d2. Total clients: 1\n", + "2024-03-08 17:29:26,371 - FederatedClient - INFO - Successfully registered client:site-1 for project simulator_server. Token:3425755c-586c-4d3b-a276-33a9338d19d2 SSID:\n", + "2024-03-08 17:29:26,373 - ClientManager - INFO - Client: New client site-2@192.168.1.203 joined. Sent token: 01f7fa8c-7623-4457-83ce-cc0aca388840. Total clients: 2\n", + "2024-03-08 17:29:26,374 - FederatedClient - INFO - Successfully registered client:site-2 for project simulator_server. Token:01f7fa8c-7623-4457-83ce-cc0aca388840 SSID:\n", + "2024-03-08 17:29:26,376 - ClientManager - INFO - Client: New client site-3@192.168.1.203 joined. Sent token: ead1968f-6c20-45f0-bed1-5db70c6626d5. Total clients: 3\n", + "2024-03-08 17:29:26,377 - FederatedClient - INFO - Successfully registered client:site-3 for project simulator_server. Token:ead1968f-6c20-45f0-bed1-5db70c6626d5 SSID:\n", + "2024-03-08 17:29:26,378 - SimulatorRunner - INFO - Set the client status ready.\n", + "2024-03-08 17:29:26,378 - SimulatorRunner - INFO - Deploy and start the Server App.\n", + "2024-03-08 17:29:26,380 - Cell - INFO - Register blob CB for channel='server_command', topic='*'\n", + "2024-03-08 17:29:26,381 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", + "2024-03-08 17:29:26,382 - ServerCommandAgent - INFO - ServerCommandAgent cell register_request_cb: server.simulate_job\n" ] }, { @@ -154,825 +114,439 @@ "name": "stdout", "output_type": "stream", "text": [ - "2024-03-06 15:59:19,577 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job]: Server runner starting ...\n", - "2024-03-06 15:59:19,579 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job]: starting workflow controller_launcher () ...\n", - "2024-03-06 15:59:19,581 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Initializing ModelController workflow.\n", - "2024-03-06 15:59:19,583 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Workflow controller_launcher () started\n", - "2024-03-06 15:59:19,585 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Beginning model controller run.\n", - "2024-03-06 15:59:19,586 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Start Controller Launcher.\n", - "2024-03-06 15:59:20,158 - SimulatorClientRunner - INFO - Start the clients run simulation.\n", - "2024-03-06 15:59:20,334 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:20,333 | app.py:163 | Starting Flower server, config: ServerConfig(num_rounds=30, round_timeout=None)\n", - "2024-03-06 15:59:20,340 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:20,339 | app.py:176 | Flower ECE: gRPC server running (30 rounds), SSL is disabled\n", - "2024-03-06 15:59:20,341 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:20,339 | server.py:89 | Initializing global parameters\n", - "2024-03-06 15:59:20,342 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:20,339 | server.py:276 | Requesting initial parameters from one random client\n", - "2024-03-06 15:59:21,163 - SimulatorClientRunner - INFO - Simulate Run client: site-1 on GPU group: None\n", - "2024-03-06 15:59:21,166 - SimulatorClientRunner - INFO - Simulate Run client: site-2 on GPU group: None\n", - "2024-03-06 15:59:21,183 - SimulatorClientRunner - INFO - Simulate Run client: site-3 on GPU group: None\n", - "2024-03-06 15:59:22,267 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00006 127.0.0.1:36825 <= 127.0.0.1:54948] is created: PID: 13991\n", - "2024-03-06 15:59:22,276 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00007 127.0.0.1:36825 <= 127.0.0.1:54964] is created: PID: 13991\n", - "2024-03-06 15:59:22,286 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00008 127.0.0.1:36825 <= 127.0.0.1:54976] is created: PID: 13991\n", - "2024-03-06 15:59:22,201 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", - "2024-03-06 15:59:22,211 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", - "2024-03-06 15:59:22,221 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", - "2024-03-06 15:59:22,265 - CoreCell - INFO - site-1.simulate_job: created backbone external connector to tcp://localhost:36825\n", - "2024-03-06 15:59:22,265 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:36825] is starting\n", - "2024-03-06 15:59:22,266 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:54948 => 127.0.0.1:36825] is created: PID: 14189\n", - "2024-03-06 15:59:22,274 - CoreCell - INFO - site-2.simulate_job: created backbone external connector to tcp://localhost:36825\n", - "2024-03-06 15:59:22,274 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:36825] is starting\n", - "2024-03-06 15:59:22,276 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:54964 => 127.0.0.1:36825] is created: PID: 14192\n", - "2024-03-06 15:59:22,285 - CoreCell - INFO - site-3.simulate_job: created backbone external connector to tcp://localhost:36825\n", - "2024-03-06 15:59:22,285 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:36825] is starting\n", - "2024-03-06 15:59:22,286 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:54976 => 127.0.0.1:36825] is created: PID: 14194\n", - "2024-03-06 15:59:23,816 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", - "2024-03-06 15:59:23,816 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", - "2024-03-06 15:59:23,829 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", - "2024-03-06 15:59:24,321 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", - "2024-03-06 15:59:24,322 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", - "2024-03-06 15:59:24,333 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: synced to Server Runner in 0.513308048248291 seconds\n", - "2024-03-06 15:59:24,333 - ExecutorLauncher - INFO - [identity=site-2, run=simulate_job]: Start Executor Launcher.\n", - "2024-03-06 15:59:24,335 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", - "2024-03-06 15:59:24,336 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: synced to Server Runner in 0.5149381160736084 seconds\n", - "2024-03-06 15:59:24,336 - ExecutorLauncher - INFO - [identity=site-3, run=simulate_job]: Start Executor Launcher.\n", - "2024-03-06 15:59:24,342 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: synced to Server Runner in 0.5080032348632812 seconds\n", - "2024-03-06 15:59:24,342 - ExecutorLauncher - INFO - [identity=site-1, run=simulate_job]: Start Executor Launcher.\n", - "Running ... [python3 custom/server.py]\n", - "2024-03-06 15:59:29,661 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:29,660 | server.py:280 | Received initial parameters from one random client\n", - "2024-03-06 15:59:29,662 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:29,660 | server.py:91 | Evaluating initial parameters\n", - "2024-03-06 15:59:29,663 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:29,661 | server.py:104 | FL starting\n", - "2024-03-06 15:59:29,652 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:29,652 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", - "2024-03-06 15:59:29,654 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,653 | connection.py:55 | ChannelConnectivity.IDLE\n", - "2024-03-06 15:59:29,655 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,655 | connection.py:55 | ChannelConnectivity.CONNECTING\n", - "2024-03-06 15:59:29,656 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,655 | connection.py:55 | ChannelConnectivity.READY\n", - "2024-03-06 15:59:29,967 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,967 | server.py:222 | fit_round 1: strategy sampled 2 clients (out of 2)\n", - "2024-03-06 15:59:29,964 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:29,964 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", - "2024-03-06 15:59:29,965 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,965 | connection.py:55 | ChannelConnectivity.IDLE\n", - "2024-03-06 15:59:29,966 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,966 | connection.py:55 | ChannelConnectivity.READY\n", - "2024-03-06 15:59:30,004 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:30,004 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", - "2024-03-06 15:59:30,005 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:30,005 | connection.py:55 | ChannelConnectivity.IDLE\n", - "2024-03-06 15:59:30,006 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:30,006 | connection.py:55 | ChannelConnectivity.CONNECTING\n", - "2024-03-06 15:59:30,006 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:30,006 | connection.py:55 | ChannelConnectivity.READY\n", - "Training: 100%|██████████| 417/417 [00:04<00:00, 96.12it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:04<00:00, 94.81it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.13it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.96it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.35it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.52it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.40it/s]\n", - "2024-03-06 15:59:44,530 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:44,530 | server.py:236 | fit_round 1 received 2 results and 0 failures\n", - "2024-03-06 15:59:44,533 - SubprocessLauncher - INFO - WARNING flwr 2024-03-06 15:59:44,533 | fedavg.py:250 | No fit_metrics_aggregation_fn provided\n", - "2024-03-06 15:59:44,534 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:44,533 | server.py:173 | evaluate_round 1: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.50it/s]\n", - "2024-03-06 15:59:45,478 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:45,478 | server.py:187 | evaluate_round 1 received 3 results and 0 failures\n", - "2024-03-06 15:59:45,481 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:45,478 | server.py:222 | fit_round 2: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.55it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.50it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 113.72it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.66it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.19it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.30it/s]s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.37it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.13it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.43it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.65it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.25it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.31it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.76it/s]\n", - "2024-03-06 15:59:59,095 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:59,095 | server.py:236 | fit_round 2 received 3 results and 0 failures\n", - "2024-03-06 15:59:59,098 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:59,098 | server.py:173 | evaluate_round 2: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.72it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.11it/s]\n", - "Running ... [python3 custom/server.py]\n", - "2024-03-06 15:59:59,876 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:59,876 | server.py:187 | evaluate_round 2 received 3 results and 0 failures\n", - "2024-03-06 15:59:59,878 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:59,876 | server.py:222 | fit_round 3: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.96it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.97it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.91it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.99it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.24it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.70it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.13it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.00it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.79it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.56it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.74it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.66it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.33it/s]\n", + "2024-03-08 17:29:27,865 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job]: Server runner starting ...\n", + "2024-03-08 17:29:27,867 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job]: starting workflow controller_launcher () ...\n", + "2024-03-08 17:29:27,869 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Initializing ModelController workflow.\n", + "2024-03-08 17:29:27,871 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Workflow controller_launcher () started\n", + "2024-03-08 17:29:27,873 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Beginning model controller run.\n", + "2024-03-08 17:29:27,874 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Start Controller Launcher.\n", + "2024-03-08 17:29:28,409 - SimulatorClientRunner - INFO - Start the clients run simulation.\n", + "2024-03-08 17:29:28,666 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:29:28,665 | app.py:163 | Starting Flower server, config: ServerConfig(num_rounds=30, round_timeout=None)\n", + "2024-03-08 17:29:28,673 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:29:28,673 | app.py:176 | Flower ECE: gRPC server running (30 rounds), SSL is disabled\n", + "2024-03-08 17:29:28,674 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:29:28,673 | server.py:89 | Initializing global parameters\n", + "2024-03-08 17:29:28,675 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:29:28,673 | server.py:276 | Requesting initial parameters from one random client\n", + "2024-03-08 17:29:29,413 - SimulatorClientRunner - INFO - Simulate Run client: site-1 on GPU group: None\n", + "2024-03-08 17:29:29,416 - SimulatorClientRunner - INFO - Simulate Run client: site-2 on GPU group: None\n", + "2024-03-08 17:29:29,429 - SimulatorClientRunner - INFO - Simulate Run client: site-3 on GPU group: None\n", + "2024-03-08 17:29:30,512 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00006 127.0.0.1:41531 <= 127.0.0.1:39446] is created: PID: 122617\n", + "2024-03-08 17:29:30,520 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00007 127.0.0.1:41531 <= 127.0.0.1:39462] is created: PID: 122617\n", + "2024-03-08 17:29:30,521 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00008 127.0.0.1:41531 <= 127.0.0.1:39464] is created: PID: 122617\n", + "2024-03-08 17:29:30,451 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", + "2024-03-08 17:29:30,452 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", + "2024-03-08 17:29:30,460 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", + "2024-03-08 17:29:30,509 - CoreCell - INFO - site-1.simulate_job: created backbone external connector to tcp://localhost:41531\n", + "2024-03-08 17:29:30,509 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:41531] is starting\n", + "2024-03-08 17:29:30,511 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:39446 => 127.0.0.1:41531] is created: PID: 122785\n", + "2024-03-08 17:29:30,518 - CoreCell - INFO - site-2.simulate_job: created backbone external connector to tcp://localhost:41531\n", + "2024-03-08 17:29:30,518 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:41531] is starting\n", + "2024-03-08 17:29:30,520 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:39462 => 127.0.0.1:41531] is created: PID: 122788\n", + "2024-03-08 17:29:30,520 - CoreCell - INFO - site-3.simulate_job: created backbone external connector to tcp://localhost:41531\n", + "2024-03-08 17:29:30,520 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:41531] is starting\n", + "2024-03-08 17:29:30,521 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:39464 => 127.0.0.1:41531] is created: PID: 122790\n", + "2024-03-08 17:29:32,143 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", + "2024-03-08 17:29:32,149 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", + "2024-03-08 17:29:32,154 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", + "2024-03-08 17:29:32,663 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00009 127.0.0.1:41531 <= 127.0.0.1:39472] is created: PID: 122617\n", + "2024-03-08 17:29:32,667 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00010 127.0.0.1:41531 <= 127.0.0.1:39488] is created: PID: 122617\n", + "2024-03-08 17:29:32,671 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00011 127.0.0.1:41531 <= 127.0.0.1:39502] is created: PID: 122617\n", + "2024-03-08 17:29:32,649 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", + "2024-03-08 17:29:32,653 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", + "2024-03-08 17:29:32,659 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", + "2024-03-08 17:29:32,660 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: synced to Server Runner in 0.5111207962036133 seconds\n", + "2024-03-08 17:29:32,661 - CoreCell - INFO - site-3_simulate_job_passive: created backbone external connector to tcp://0:41531\n", + "2024-03-08 17:29:32,661 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00002 ACTIVE tcp://0:41531] is starting\n", + "2024-03-08 17:29:32,662 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: synced to Server Runner in 0.509554386138916 seconds\n", + "2024-03-08 17:29:32,662 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00004 127.0.0.1:39472 => 127.0.0.1:41531] is created: PID: 122790\n", + "2024-03-08 17:29:32,662 - CellPipe - INFO - registered CellPipe request CB for cell_pipe.metric\n", + "2024-03-08 17:29:32,663 - CoreCell - INFO - site-2_simulate_job_passive: created backbone external connector to tcp://0:41531\n", + "2024-03-08 17:29:32,663 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00002 ACTIVE tcp://0:41531] is starting\n", + "2024-03-08 17:29:32,664 - CellPipe - INFO - registered CellPipe request CB for cell_pipe.task\n", + "2024-03-08 17:29:32,665 - CellPipe - INFO - registered CellPipe request CB for cell_pipe.metric\n", + "2024-03-08 17:29:32,665 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00004 127.0.0.1:39488 => 127.0.0.1:41531] is created: PID: 122788\n", + "2024-03-08 17:29:32,667 - CellPipe - INFO - registered CellPipe request CB for cell_pipe.task\n", + "2024-03-08 17:29:32,668 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: client runner started\n", + "2024-03-08 17:29:32,668 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-3\n", + "2024-03-08 17:29:32,668 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: synced to Server Runner in 0.5096893310546875 seconds\n", + "2024-03-08 17:29:32,669 - CoreCell - INFO - site-1_simulate_job_passive: created backbone external connector to tcp://0:41531\n", + "2024-03-08 17:29:32,669 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00002 ACTIVE tcp://0:41531] is starting\n", + "2024-03-08 17:29:32,670 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: client runner started\n", + "2024-03-08 17:29:32,670 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-2\n", + "2024-03-08 17:29:32,671 - CellPipe - INFO - registered CellPipe request CB for cell_pipe.metric\n", + "2024-03-08 17:29:32,671 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00004 127.0.0.1:39502 => 127.0.0.1:41531] is created: PID: 122785\n", + "2024-03-08 17:29:32,672 - CellPipe - INFO - registered CellPipe request CB for cell_pipe.task\n", + "2024-03-08 17:29:32,674 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: client runner started\n", + "2024-03-08 17:29:32,675 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-1\n", + "2024-03-08 17:29:37,292 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00012 127.0.0.1:41531 <= 127.0.0.1:39510] is created: PID: 122617\n", + "2024-03-08 17:29:37,305 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:29:37,305 | server.py:280 | Received initial parameters from one random client\n", + "2024-03-08 17:29:37,306 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:29:37,305 | server.py:91 | Evaluating initial parameters\n", + "2024-03-08 17:29:37,307 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:29:37,305 | server.py:104 | FL starting\n", + "2024-03-08 17:29:37,489 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00013 127.0.0.1:41531 <= 127.0.0.1:39522] is created: PID: 122617\n", + "2024-03-08 17:29:37,499 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:37,499 | server.py:222 | fit_round 1: strategy sampled 2 clients (out of 2)\n", + "2024-03-08 17:29:37,299 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:29:37,299 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", + "2024-03-08 17:29:37,300 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:37,300 | connection.py:55 | ChannelConnectivity.IDLE\n", + "2024-03-08 17:29:37,301 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:37,300 | connection.py:55 | ChannelConnectivity.CONNECTING\n", + "2024-03-08 17:29:37,301 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:37,301 | connection.py:55 | ChannelConnectivity.READY\n", + "2024-03-08 17:29:37,496 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:29:37,496 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", + "2024-03-08 17:29:37,497 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:37,497 | connection.py:55 | ChannelConnectivity.IDLE\n", + "2024-03-08 17:29:37,497 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:37,497 | connection.py:55 | ChannelConnectivity.CONNECTING\n", + "2024-03-08 17:29:37,498 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:37,498 | connection.py:55 | ChannelConnectivity.READY\n", + "2024-03-08 17:29:37,501 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00014 127.0.0.1:41531 <= 127.0.0.1:39532] is created: PID: 122617\n", + "2024-03-08 17:29:37,505 - SubprocessLauncher - INFO - Flare system info is: {'SITE_NAME': 'site-2', 'JOB_ID': 'simulate_job'}\n", + "2024-03-08 17:29:37,507 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:29:37,507 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", + "2024-03-08 17:29:37,508 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:37,508 | connection.py:55 | ChannelConnectivity.IDLE\n", + "2024-03-08 17:29:37,509 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:37,509 | connection.py:55 | ChannelConnectivity.CONNECTING\n", + "2024-03-08 17:29:37,510 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:37,510 | connection.py:55 | ChannelConnectivity.READY\n", + "2024-03-08 17:29:37,510 - SubprocessLauncher - INFO - Flare system info is: {'SITE_NAME': 'site-3', 'JOB_ID': 'simulate_job'}\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:29:41,210 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:41,210 | server.py:236 | fit_round 1 received 2 results and 0 failures\n", + "2024-03-08 17:29:41,212 - SubprocessLauncher - INFO - WARNING flwr 2024-03-08 17:29:41,212 | fedavg.py:250 | No fit_metrics_aggregation_fn provided\n", + "2024-03-08 17:29:41,214 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:41,212 | server.py:173 | evaluate_round 1: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 116.28it/s]s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 112.92it/s]s]\n", + "2024-03-08 17:29:41,229 - SubprocessLauncher - INFO - Flare system info is: {'SITE_NAME': 'site-1', 'JOB_ID': 'simulate_job'}\n", + "2024-03-08 17:29:42,197 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:42,197 | server.py:187 | evaluate_round 1 received 3 results and 0 failures\n", + "2024-03-08 17:29:42,199 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:42,197 | server.py:222 | fit_round 2: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 131.15it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 129.04it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 109.26it/s]\n", + "2024-03-08 17:29:45,937 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:45,937 | server.py:236 | fit_round 2 received 3 results and 0 failures\n", + "2024-03-08 17:29:45,940 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:45,939 | server.py:173 | evaluate_round 2: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 112.58it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 112.50it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 111.99it/s]\n", + "2024-03-08 17:29:46,764 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:46,763 | server.py:187 | evaluate_round 2 received 3 results and 0 failures\n", + "2024-03-08 17:29:46,765 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:46,764 | server.py:222 | fit_round 3: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.20it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.91it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 129.64it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:29:50,404 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:50,404 | server.py:236 | fit_round 3 received 3 results and 0 failures\n", + "2024-03-08 17:29:50,409 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:50,409 | server.py:173 | evaluate_round 3: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.50it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 116.53it/s]s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 114.85it/s]s]\n", + "2024-03-08 17:29:51,210 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:51,209 | server.py:187 | evaluate_round 3 received 3 results and 0 failures\n", + "2024-03-08 17:29:51,211 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:51,210 | server.py:222 | fit_round 4: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.79it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.51it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.15it/s]\n", + "2024-03-08 17:29:54,740 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:54,740 | server.py:236 | fit_round 4 received 3 results and 0 failures\n", + "2024-03-08 17:29:54,746 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:54,746 | server.py:173 | evaluate_round 4: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.16it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.93it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.71it/s]\n", + "2024-03-08 17:29:55,540 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:55,540 | server.py:187 | evaluate_round 4 received 3 results and 0 failures\n", + "2024-03-08 17:29:55,543 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:55,540 | server.py:222 | fit_round 5: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.77it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.95it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.77it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:29:59,063 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:59,062 | server.py:236 | fit_round 5 received 3 results and 0 failures\n", + "2024-03-08 17:29:59,065 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:59,065 | server.py:173 | evaluate_round 5: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.50it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.64it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.73it/s]\n", + "2024-03-08 17:29:59,852 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:59,852 | server.py:187 | evaluate_round 5 received 3 results and 0 failures\n", + "2024-03-08 17:29:59,853 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:29:59,852 | server.py:222 | fit_round 6: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.18it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.40it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.15it/s]\n", + "2024-03-08 17:30:03,371 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:03,371 | server.py:236 | fit_round 6 received 3 results and 0 failures\n", + "2024-03-08 17:30:03,373 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:03,373 | server.py:173 | evaluate_round 6: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.56it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.65it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.22it/s]\n", + "2024-03-08 17:30:04,171 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:04,171 | server.py:187 | evaluate_round 6 received 3 results and 0 failures\n", + "2024-03-08 17:30:04,173 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:04,171 | server.py:222 | fit_round 7: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.48it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.66it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.17it/s]\n", + "2024-03-08 17:30:07,674 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:07,673 | server.py:236 | fit_round 7 received 3 results and 0 failures\n", + "2024-03-08 17:30:07,676 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:07,676 | server.py:173 | evaluate_round 7: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.23it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.60it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.67it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:30:08,457 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:08,457 | server.py:187 | evaluate_round 7 received 3 results and 0 failures\n", + "2024-03-08 17:30:08,459 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:08,458 | server.py:222 | fit_round 8: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.75it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.04it/s]]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.05it/s]\n", + "2024-03-08 17:30:11,885 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:11,885 | server.py:236 | fit_round 8 received 3 results and 0 failures\n", + "2024-03-08 17:30:11,889 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:11,889 | server.py:173 | evaluate_round 8: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.44it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.71it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.28it/s]\n", + "2024-03-08 17:30:12,682 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:12,682 | server.py:187 | evaluate_round 8 received 3 results and 0 failures\n", + "2024-03-08 17:30:12,684 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:12,682 | server.py:222 | fit_round 9: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.01it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.71it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.26it/s]\n", + "2024-03-08 17:30:16,199 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:16,199 | server.py:236 | fit_round 9 received 3 results and 0 failures\n", + "2024-03-08 17:30:16,202 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:16,202 | server.py:173 | evaluate_round 9: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.06it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.62it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.17it/s]\n", + "2024-03-08 17:30:16,997 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:16,996 | server.py:187 | evaluate_round 9 received 3 results and 0 failures\n", + "2024-03-08 17:30:16,999 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:16,997 | server.py:222 | fit_round 10: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.27it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.86it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.59it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:30:20,469 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:20,469 | server.py:236 | fit_round 10 received 3 results and 0 failures\n", + "2024-03-08 17:30:20,473 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:20,473 | server.py:173 | evaluate_round 10: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.48it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.04it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.43it/s]\n", + "2024-03-08 17:30:21,253 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:21,252 | server.py:187 | evaluate_round 10 received 3 results and 0 failures\n", + "2024-03-08 17:30:21,255 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:21,253 | server.py:222 | fit_round 11: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.27it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.96it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.71it/s]\n", + "2024-03-08 17:30:24,710 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:24,709 | server.py:236 | fit_round 11 received 3 results and 0 failures\n", + "2024-03-08 17:30:24,716 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:24,715 | server.py:173 | evaluate_round 11: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.87it/s]\n", "Training: 100%|██████████| 417/417 [00:03<00:00, 122.31it/s]\n", - "2024-03-06 16:00:13,474 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:13,474 | server.py:236 | fit_round 3 received 3 results and 0 failures\n", - "2024-03-06 16:00:13,476 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:13,476 | server.py:173 | evaluate_round 3: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.59it/s]\n", - "2024-03-06 16:00:14,233 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:14,233 | server.py:187 | evaluate_round 3 received 3 results and 0 failures\n", - "2024-03-06 16:00:14,235 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:14,233 | server.py:222 | fit_round 4: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 143.89it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.50it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.71it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.57it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.92it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.89it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.81it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.23it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.97it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.90it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.95it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.38it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.30it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.97it/s]\n", - "2024-03-06 16:00:27,478 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:27,478 | server.py:236 | fit_round 4 received 3 results and 0 failures\n", - "2024-03-06 16:00:27,480 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:27,480 | server.py:173 | evaluate_round 4: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.91it/s]]\n", - "2024-03-06 16:00:28,252 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:28,252 | server.py:187 | evaluate_round 4 received 3 results and 0 failures\n", - "2024-03-06 16:00:28,254 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:28,252 | server.py:222 | fit_round 5: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.01it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.24it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.43it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.91it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.62it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.38it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.73it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.10it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.00it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.99it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.01it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.70it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.12it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.70it/s]\n", - "2024-03-06 16:00:41,818 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:41,818 | server.py:236 | fit_round 5 received 3 results and 0 failures\n", - "2024-03-06 16:00:41,820 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:41,820 | server.py:173 | evaluate_round 5: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.96it/s]\n", - "2024-03-06 16:00:42,596 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:42,596 | server.py:187 | evaluate_round 5 received 3 results and 0 failures\n", - "2024-03-06 16:00:42,599 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:42,596 | server.py:222 | fit_round 6: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.87it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.61it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.00it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.45it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.69it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.85it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.11it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.98it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.72it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.57it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.19it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.93it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.90it/s]\n", - "2024-03-06 16:00:56,098 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:56,097 | server.py:236 | fit_round 6 received 3 results and 0 failures\n", - "2024-03-06 16:00:56,100 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:56,100 | server.py:173 | evaluate_round 6: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.58it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.29it/s]\n", - "2024-03-06 16:00:56,888 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:56,888 | server.py:187 | evaluate_round 6 received 3 results and 0 failures\n", - "2024-03-06 16:00:56,890 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:56,888 | server.py:222 | fit_round 7: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.09it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.65it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.02it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.61it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.78it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.23it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.79it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.10it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.08it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.41it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.43it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.76it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.56it/s]\n", - "2024-03-06 16:01:10,353 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:10,353 | server.py:236 | fit_round 7 received 3 results and 0 failures\n", - "2024-03-06 16:01:10,355 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:10,355 | server.py:173 | evaluate_round 7: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.12it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.07it/s]\n", - "2024-03-06 16:01:11,142 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:11,142 | server.py:187 | evaluate_round 7 received 3 results and 0 failures\n", - "2024-03-06 16:01:11,144 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:11,142 | server.py:222 | fit_round 8: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.13it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.68it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.09it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.56it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.34it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.12it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.46it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.50it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.50it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.14it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.70it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.75it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.31it/s]\n", - "2024-03-06 16:01:24,618 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:24,618 | server.py:236 | fit_round 8 received 3 results and 0 failures\n", - "2024-03-06 16:01:24,624 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:24,624 | server.py:173 | evaluate_round 8: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.18it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.15it/s]\n", - "2024-03-06 16:01:25,394 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:25,394 | server.py:187 | evaluate_round 8 received 3 results and 0 failures\n", - "2024-03-06 16:01:25,396 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:25,394 | server.py:222 | fit_round 9: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 141.40it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.26it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.32it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.11it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.93it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.26it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.98it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.09it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.96it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.29it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.29it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.72it/s]]\n", - "2024-03-06 16:01:39,110 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:39,110 | server.py:236 | fit_round 9 received 3 results and 0 failures\n", - "2024-03-06 16:01:39,116 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:39,116 | server.py:173 | evaluate_round 9: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.76it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.05it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.83it/s]\n", - "Running ... [python3 custom/server.py]\n", - "2024-03-06 16:01:39,887 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:39,886 | server.py:187 | evaluate_round 9 received 3 results and 0 failures\n", - "2024-03-06 16:01:39,889 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:39,887 | server.py:222 | fit_round 10: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.56it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.73it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.02it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.73it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.17it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.22it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.60it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.43it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.62it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.76it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.72it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.48it/s]\n", - "2024-03-06 16:01:53,516 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:53,516 | server.py:236 | fit_round 10 received 3 results and 0 failures\n", - "2024-03-06 16:01:53,522 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:53,522 | server.py:173 | evaluate_round 10: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.55it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.46it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.98it/s]\n", - "2024-03-06 16:01:54,295 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:54,295 | server.py:187 | evaluate_round 10 received 3 results and 0 failures\n", - "2024-03-06 16:01:54,297 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:54,295 | server.py:222 | fit_round 11: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.78it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.71it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.44it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.89it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.79it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.08it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.22it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.08it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.57it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.95it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.22it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.70it/s]\n", - "2024-03-06 16:02:07,696 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:07,695 | server.py:236 | fit_round 11 received 3 results and 0 failures\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.16it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.92it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.88it/s]\n", - "2024-03-06 16:02:07,698 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:07,698 | server.py:173 | evaluate_round 11: strategy sampled 3 clients (out of 3)\n", - "2024-03-06 16:02:08,468 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:08,467 | server.py:187 | evaluate_round 11 received 3 results and 0 failures\n", - "2024-03-06 16:02:08,470 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:08,468 | server.py:222 | fit_round 12: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.42it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.70it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.45it/s]]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.14it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.67it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.44it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.35it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.82it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.44it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.19it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.66it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.81it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.65it/s]\n", - "2024-03-06 16:02:21,975 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:21,975 | server.py:236 | fit_round 12 received 3 results and 0 failures\n", - "2024-03-06 16:02:21,977 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:21,977 | server.py:173 | evaluate_round 12: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.78it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.80it/s]\n", - "2024-03-06 16:02:22,750 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:22,750 | server.py:187 | evaluate_round 12 received 3 results and 0 failures\n", - "2024-03-06 16:02:22,752 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:22,750 | server.py:222 | fit_round 13: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.17it/s]\n", + "2024-03-08 17:30:25,506 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:25,506 | server.py:187 | evaluate_round 11 received 3 results and 0 failures\n", + "2024-03-08 17:30:25,508 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:25,506 | server.py:222 | fit_round 12: strategy sampled 3 clients (out of 3)\n", "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.16it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.94it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.34it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.41it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.51it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.23it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.22it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.99it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.65it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.71it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.44it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.49it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.10it/s]\n", - "2024-03-06 16:02:36,173 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:36,173 | server.py:236 | fit_round 13 received 3 results and 0 failures\n", - "2024-03-06 16:02:36,177 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:36,177 | server.py:173 | evaluate_round 13: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.54it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.21it/s]]\n", - "2024-03-06 16:02:36,949 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:36,948 | server.py:187 | evaluate_round 13 received 3 results and 0 failures\n", - "2024-03-06 16:02:36,951 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:36,949 | server.py:222 | fit_round 14: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 141.12it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.16it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.79it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.00it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.56it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.16it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.85it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.71it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.99it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.34it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.39it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.94it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.52it/s]\n", - "2024-03-06 16:02:50,424 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:50,424 | server.py:236 | fit_round 14 received 3 results and 0 failures\n", - "2024-03-06 16:02:50,430 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:50,430 | server.py:173 | evaluate_round 14: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.58it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.89it/s]\n", - "2024-03-06 16:02:51,193 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:51,193 | server.py:187 | evaluate_round 14 received 3 results and 0 failures\n", - "2024-03-06 16:02:51,195 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:51,193 | server.py:222 | fit_round 15: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.28it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 141.90it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.95it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.22it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.25it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.59it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.58it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.87it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.58it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.60it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.35it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.02it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.32it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.65it/s]s]\n", - "2024-03-06 16:03:04,773 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:04,772 | server.py:236 | fit_round 15 received 3 results and 0 failures\n", - "2024-03-06 16:03:04,778 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:04,778 | server.py:173 | evaluate_round 15: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.93it/s]s]\n", - "2024-03-06 16:03:05,564 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:05,564 | server.py:187 | evaluate_round 15 received 3 results and 0 failures\n", - "2024-03-06 16:03:05,567 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:05,564 | server.py:222 | fit_round 16: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.38it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.05it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.02it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.03it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.43it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.15it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.83it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.53it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.26it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.70it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.93it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.72it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.60it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.72it/s]\n", - "2024-03-06 16:03:19,126 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:19,126 | server.py:236 | fit_round 16 received 3 results and 0 failures\n", - "2024-03-06 16:03:19,128 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:19,128 | server.py:173 | evaluate_round 16: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.12it/s]\n", - "Running ... [python3 custom/server.py]\n", - "2024-03-06 16:03:19,909 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:19,909 | server.py:187 | evaluate_round 16 received 3 results and 0 failures\n", - "2024-03-06 16:03:19,911 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:19,909 | server.py:222 | fit_round 17: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.80it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.01it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.09it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.07it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.73it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.85it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.19it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.99it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.41it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.58it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.76it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.54it/s]\n", - "2024-03-06 16:03:33,354 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:33,354 | server.py:236 | fit_round 17 received 3 results and 0 failures\n", - "2024-03-06 16:03:33,356 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:33,356 | server.py:173 | evaluate_round 17: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.10it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.08it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.96it/s]\n", - "2024-03-06 16:03:34,127 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:34,127 | server.py:187 | evaluate_round 17 received 3 results and 0 failures\n", - "2024-03-06 16:03:34,129 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:34,127 | server.py:222 | fit_round 18: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.25it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.56it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.69it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.95it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.00it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.05it/s]s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.51it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.46it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.49it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.00it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.96it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.67it/s]\n", - "2024-03-06 16:03:47,546 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:47,546 | server.py:236 | fit_round 18 received 3 results and 0 failures\n", - "2024-03-06 16:03:47,548 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:47,548 | server.py:173 | evaluate_round 18: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.02it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.27it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.23it/s]\n", - "2024-03-06 16:03:48,319 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:48,318 | server.py:187 | evaluate_round 18 received 3 results and 0 failures\n", - "2024-03-06 16:03:48,321 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:48,319 | server.py:222 | fit_round 19: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.89it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.08it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.83it/s]]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.49it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.81it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.47it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.70it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.59it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.20it/s]s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.76it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.39it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.46it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.14it/s]\n", - "2024-03-06 16:04:01,895 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:01,894 | server.py:236 | fit_round 19 received 3 results and 0 failures\n", - "2024-03-06 16:04:01,900 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:01,899 | server.py:173 | evaluate_round 19: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.00it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.93it/s]\n", - "2024-03-06 16:04:02,690 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:02,690 | server.py:187 | evaluate_round 19 received 3 results and 0 failures\n", - "2024-03-06 16:04:02,692 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:02,690 | server.py:222 | fit_round 20: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.85it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.13it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.20it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.73it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.67it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.17it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.05it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.09it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.35it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.87it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.97it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.85it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.70it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.43it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.99it/s]\n", - "2024-03-06 16:04:16,288 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:16,287 | server.py:236 | fit_round 20 received 3 results and 0 failures\n", - "2024-03-06 16:04:16,290 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:16,290 | server.py:173 | evaluate_round 20: strategy sampled 3 clients (out of 3)\n", - "2024-03-06 16:04:17,056 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:17,056 | server.py:187 | evaluate_round 20 received 3 results and 0 failures\n", - "2024-03-06 16:04:17,059 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:17,056 | server.py:222 | fit_round 21: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.25it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.03it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.98it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.12it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.52it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.56it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.13it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.61it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.58it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.38it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.45it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.99it/s]]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.21it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.69it/s]\n", - "2024-03-06 16:04:30,694 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:30,694 | server.py:236 | fit_round 21 received 3 results and 0 failures\n", - "2024-03-06 16:04:30,696 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:30,696 | server.py:173 | evaluate_round 21: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.87it/s]\n", - "2024-03-06 16:04:31,454 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:31,453 | server.py:187 | evaluate_round 21 received 3 results and 0 failures\n", - "2024-03-06 16:04:31,456 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:31,454 | server.py:222 | fit_round 22: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 141.09it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.61it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.55it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.62it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.52it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.87it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.20it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.75it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.08it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.23it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.90it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.93it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.30it/s]\n", - "2024-03-06 16:04:45,011 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:45,011 | server.py:236 | fit_round 22 received 3 results and 0 failures\n", - "2024-03-06 16:04:45,014 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:45,014 | server.py:173 | evaluate_round 22: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.94it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.24it/s]\n", - "2024-03-06 16:04:45,799 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:45,799 | server.py:187 | evaluate_round 22 received 3 results and 0 failures\n", - "2024-03-06 16:04:45,801 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:45,799 | server.py:222 | fit_round 23: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.70it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.71it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.12it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.43it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.45it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.47it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.83it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.03it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.22it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.06it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.52it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.83it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.40it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.53it/s]\n", - "2024-03-06 16:04:59,317 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:59,317 | server.py:236 | fit_round 23 received 3 results and 0 failures\n", - "2024-03-06 16:04:59,322 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:59,322 | server.py:173 | evaluate_round 23: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.36it/s]\n", - "Running ... [python3 custom/server.py]\n", - "2024-03-06 16:05:00,101 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:00,100 | server.py:187 | evaluate_round 23 received 3 results and 0 failures\n", - "2024-03-06 16:05:00,103 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:00,101 | server.py:222 | fit_round 24: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.22it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.99it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.29it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.58it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.58it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.59it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.60it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.69it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.00it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.27it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.68it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.67it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.75it/s]\n", - "2024-03-06 16:05:13,713 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:13,713 | server.py:236 | fit_round 24 received 3 results and 0 failures\n", - "2024-03-06 16:05:13,716 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:13,715 | server.py:173 | evaluate_round 24: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.94it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.51it/s]\n", - "2024-03-06 16:05:14,482 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:14,482 | server.py:187 | evaluate_round 24 received 3 results and 0 failures\n", - "2024-03-06 16:05:14,484 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:14,482 | server.py:222 | fit_round 25: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.36it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.65it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.44it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.47it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:30:28,997 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:28,997 | server.py:236 | fit_round 12 received 3 results and 0 failures\n", + "2024-03-08 17:30:28,999 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:28,999 | server.py:173 | evaluate_round 12: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.08it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.36it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.06it/s]\n", + "2024-03-08 17:30:29,788 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:29,788 | server.py:187 | evaluate_round 12 received 3 results and 0 failures\n", + "2024-03-08 17:30:29,790 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:29,788 | server.py:222 | fit_round 13: strategy sampled 3 clients (out of 3)\n", "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.51it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.13it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.24it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.76it/s]s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.74it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.89it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.38it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.67it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.27it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.36it/s]\n", - "2024-03-06 16:05:28,035 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:28,035 | server.py:236 | fit_round 25 received 3 results and 0 failures\n", - "2024-03-06 16:05:28,039 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:28,038 | server.py:173 | evaluate_round 25: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.68it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.41it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.06it/s]\n", - "2024-03-06 16:05:28,822 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:28,822 | server.py:187 | evaluate_round 25 received 3 results and 0 failures\n", - "2024-03-06 16:05:28,823 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:28,822 | server.py:222 | fit_round 26: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.95it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.06it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.23it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.72it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.33it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.71it/s]\n", + "2024-03-08 17:30:33,313 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:33,313 | server.py:236 | fit_round 13 received 3 results and 0 failures\n", + "2024-03-08 17:30:33,317 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:33,317 | server.py:173 | evaluate_round 13: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.96it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.77it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.91it/s]\n", + "2024-03-08 17:30:34,112 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:34,111 | server.py:187 | evaluate_round 13 received 3 results and 0 failures\n", + "2024-03-08 17:30:34,113 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:34,112 | server.py:222 | fit_round 14: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.89it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.89it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.61it/s]\n", + "2024-03-08 17:30:37,589 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:37,589 | server.py:236 | fit_round 14 received 3 results and 0 failures\n", + "2024-03-08 17:30:37,593 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:37,593 | server.py:173 | evaluate_round 14: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.40it/s]\n", "Training: 100%|██████████| 417/417 [00:03<00:00, 122.97it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.64it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.25it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.51it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.25it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 128.94it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.45it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.78it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.14it/s]\n", - "2024-03-06 16:05:42,197 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:42,197 | server.py:236 | fit_round 26 received 3 results and 0 failures\n", - "2024-03-06 16:05:42,201 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:42,201 | server.py:173 | evaluate_round 26: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.42it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.81it/s]\n", - "2024-03-06 16:05:42,972 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:42,972 | server.py:187 | evaluate_round 26 received 3 results and 0 failures\n", - "2024-03-06 16:05:42,974 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:42,972 | server.py:222 | fit_round 27: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.78it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 141.90it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.37it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.65it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.03it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.10it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.53it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.62it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.58it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.26it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.33it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.81it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.20it/s]\n", - "2024-03-06 16:05:56,504 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:56,504 | server.py:236 | fit_round 27 received 3 results and 0 failures\n", - "2024-03-06 16:05:56,508 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:56,508 | server.py:173 | evaluate_round 27: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.20it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.93it/s]\n", - "2024-03-06 16:05:57,308 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:57,308 | server.py:187 | evaluate_round 27 received 3 results and 0 failures\n", - "2024-03-06 16:05:57,310 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:57,308 | server.py:222 | fit_round 28: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.95it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.97it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.80it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.37it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.43it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.84it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.42it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.51it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.07it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 130.11it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.60it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.71it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.51it/s]\n", - "2024-03-06 16:06:10,845 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:10,845 | server.py:236 | fit_round 28 received 3 results and 0 failures\n", - "2024-03-06 16:06:10,851 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:10,851 | server.py:173 | evaluate_round 28: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.40it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.90it/s]\n", - "2024-03-06 16:06:11,629 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:11,628 | server.py:187 | evaluate_round 28 received 3 results and 0 failures\n", - "2024-03-06 16:06:11,631 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:11,629 | server.py:222 | fit_round 29: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.37it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.42it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.00it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.64it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.08it/s]s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.63it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.08it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.54it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.15it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.23it/s]\n", "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.41it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.89it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.72it/s]\n", - "2024-03-06 16:06:25,201 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:25,201 | server.py:236 | fit_round 29 received 3 results and 0 failures\n", - "2024-03-06 16:06:25,203 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:25,203 | server.py:173 | evaluate_round 29: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.43it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.14it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.76it/s]\n", - "2024-03-06 16:06:25,971 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:25,971 | server.py:187 | evaluate_round 29 received 3 results and 0 failures\n", - "2024-03-06 16:06:25,973 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:25,971 | server.py:222 | fit_round 30: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.01it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.47it/s]\n", + "2024-03-08 17:30:38,381 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:38,381 | server.py:187 | evaluate_round 14 received 3 results and 0 failures\n", + "2024-03-08 17:30:38,383 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:38,381 | server.py:222 | fit_round 15: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.87it/s]\n", "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.17it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.33it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.85it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.45it/s]\n", - "Running ... [python3 custom/server.py]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.95it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.07it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.93it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.29it/s]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.18it/s]]\n", - "Running ... [python3 custom/client.py --node-id 0]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.25it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.89it/s]\n", - "2024-03-06 16:06:39,622 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:39,622 | server.py:236 | fit_round 30 received 3 results and 0 failures\n", - "2024-03-06 16:06:39,628 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:39,627 | server.py:173 | evaluate_round 30: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.46it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.01it/s]\n", - "Running ... [python3 custom/server.py]\n", - "2024-03-06 16:06:40,388 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:40,387 | server.py:187 | evaluate_round 30 received 3 results and 0 failures\n", - "2024-03-06 16:06:40,390 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | server.py:153 | FL finished in 430.7270666919994\n", - "2024-03-06 16:06:40,391 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | app.py:226 | app_fit: losses_distributed [(1, 212.5634969075521), (2, 177.07842000325522), (3, 157.1292928059896), (4, 146.80193583170572), (5, 137.0935516357422), (6, 128.4277369181315), (7, 121.84776306152344), (8, 112.52642059326172), (9, 106.32784016927083), (10, 101.43400319417317), (11, 95.19340260823567), (12, 88.39397430419922), (13, 80.34169514973958), (14, 75.0222880045573), (15, 67.75357564290364), (16, 63.75060272216797), (17, 56.23624165852865), (18, 53.15401840209961), (19, 46.236531575520836), (20, 43.1369883219401), (21, 40.2807362874349), (22, 39.57940419514974), (23, 34.96051915486654), (24, 29.61727523803711), (25, 31.82533073425293), (26, 28.117460250854492), (27, 25.476886749267578), (28, 25.248918533325195), (29, 26.424275080362957), (30, 26.067562103271484)]\n", - "2024-03-06 16:06:40,392 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | app.py:227 | app_fit: metrics_distributed_fit {}\n", - "2024-03-06 16:06:40,394 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | app.py:228 | app_fit: metrics_distributed {'accuracy': [(1, 0.25674865026994603), (2, 0.39472105578884226), (3, 0.46010797840431916), (4, 0.4989002199560088), (5, 0.5380923815236952), (6, 0.5699860027994401), (7, 0.593381323735253), (8, 0.6232753449310138), (9, 0.650369926014797), (10, 0.6696660667866426), (11, 0.6860627874425115), (12, 0.7186562687462508), (13, 0.7454509098180364), (14, 0.7661467706458708), (15, 0.7933413317336533), (16, 0.8125374925014996), (17, 0.8355328934213158), (18, 0.8529294141171766), (19, 0.8812237552489502), (20, 0.8912217556488702), (21, 0.9072185562887423), (22, 0.9102179564087183), (23, 0.9269146170765847), (24, 0.9489102179564087), (25, 0.9404119176164767), (26, 0.9543091381723655), (27, 0.9707058588282343), (28, 0.9744051189762047), (29, 0.9681063787242552), (30, 0.9673065386922616)]}\n", - "2024-03-06 16:06:40,395 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | app.py:229 | app_fit: losses_centralized []\n", - "2024-03-06 16:06:40,396 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | app.py:230 | app_fit: metrics_centralized {}\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.20it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.39it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.16it/s]\n", - "2024-03-06 16:06:40,398 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:40,397 | connection.py:220 | gRPC channel closed\n", - "2024-03-06 16:06:40,398 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,398 | app.py:398 | Disconnect and shut down\n", - "2024-03-06 16:06:40,399 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:40,399 | connection.py:220 | gRPC channel closed\n", - "2024-03-06 16:06:40,399 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:40,399 | connection.py:220 | gRPC channel closed\n", - "2024-03-06 16:06:40,399 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,399 | app.py:398 | Disconnect and shut down\n", - "2024-03-06 16:06:40,399 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,399 | app.py:398 | Disconnect and shut down\n", - "run success\n", - "2024-03-06 16:06:44,716 - ExecutorLauncher - INFO - [identity=site-2, run=simulate_job]: Stop Executor Launcher.\n", - "2024-03-06 16:06:44,716 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: client runner started\n", - "2024-03-06 16:06:44,717 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-2\n", - "run success\n", - "2024-03-06 16:06:44,722 - ExecutorLauncher - INFO - [identity=site-1, run=simulate_job]: Stop Executor Launcher.\n", - "2024-03-06 16:06:44,723 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: client runner started\n", - "2024-03-06 16:06:44,723 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-1\n", - "run success\n", - "2024-03-06 16:06:44,760 - ExecutorLauncher - INFO - [identity=site-3, run=simulate_job]: Stop Executor Launcher.\n", - "2024-03-06 16:06:44,761 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: client runner started\n", - "2024-03-06 16:06:44,761 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-3\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.79it/s]\n", + "2024-03-08 17:30:41,816 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:41,815 | server.py:236 | fit_round 15 received 3 results and 0 failures\n", + "2024-03-08 17:30:41,821 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:41,821 | server.py:173 | evaluate_round 15: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.30it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.06it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.70it/s]\n", + "2024-03-08 17:30:42,648 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:42,648 | server.py:187 | evaluate_round 15 received 3 results and 0 failures\n", + "2024-03-08 17:30:42,650 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:42,648 | server.py:222 | fit_round 16: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.28it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.98it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 129.73it/s]\n", + "2024-03-08 17:30:46,130 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:46,129 | server.py:236 | fit_round 16 received 3 results and 0 failures\n", + "2024-03-08 17:30:46,132 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:46,132 | server.py:173 | evaluate_round 16: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.73it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.97it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.19it/s]\n", + "2024-03-08 17:30:46,937 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:46,937 | server.py:187 | evaluate_round 16 received 3 results and 0 failures\n", + "2024-03-08 17:30:46,939 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:46,937 | server.py:222 | fit_round 17: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.38it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.57it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.22it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:30:50,389 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:50,389 | server.py:236 | fit_round 17 received 3 results and 0 failures\n", + "2024-03-08 17:30:50,391 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:50,391 | server.py:173 | evaluate_round 17: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.38it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.24it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.30it/s]\n", + "2024-03-08 17:30:51,189 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:51,189 | server.py:187 | evaluate_round 17 received 3 results and 0 failures\n", + "2024-03-08 17:30:51,191 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:51,189 | server.py:222 | fit_round 18: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.79it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.37it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.25it/s]\n", + "2024-03-08 17:30:54,737 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:54,737 | server.py:236 | fit_round 18 received 3 results and 0 failures\n", + "2024-03-08 17:30:54,740 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:54,740 | server.py:173 | evaluate_round 18: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.45it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.23it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.19it/s]\n", + "2024-03-08 17:30:55,537 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:55,537 | server.py:187 | evaluate_round 18 received 3 results and 0 failures\n", + "2024-03-08 17:30:55,539 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:55,537 | server.py:222 | fit_round 19: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.91it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.70it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.45it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:30:59,121 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:59,121 | server.py:236 | fit_round 19 received 3 results and 0 failures\n", + "2024-03-08 17:30:59,125 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:59,124 | server.py:173 | evaluate_round 19: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.73it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.93it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 116.92it/s]\n", + "2024-03-08 17:30:59,954 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:59,954 | server.py:187 | evaluate_round 19 received 3 results and 0 failures\n", + "2024-03-08 17:30:59,956 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:30:59,954 | server.py:222 | fit_round 20: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 130.02it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 129.62it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 129.20it/s]\n", + "2024-03-08 17:31:03,607 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:03,606 | server.py:236 | fit_round 20 received 3 results and 0 failures\n", + "2024-03-08 17:31:03,609 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:03,609 | server.py:173 | evaluate_round 20: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 116.56it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 114.90it/s]s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 114.64it/s]s]\n", + "2024-03-08 17:31:04,415 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:04,414 | server.py:187 | evaluate_round 20 received 3 results and 0 failures\n", + "2024-03-08 17:31:04,416 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:04,414 | server.py:222 | fit_round 21: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.01it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.15it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.25it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:31:08,037 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:08,036 | server.py:236 | fit_round 21 received 3 results and 0 failures\n", + "2024-03-08 17:31:08,043 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:08,042 | server.py:173 | evaluate_round 21: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.06it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 117.77it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 115.62it/s]s]\n", + "2024-03-08 17:31:08,856 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:08,856 | server.py:187 | evaluate_round 21 received 3 results and 0 failures\n", + "2024-03-08 17:31:08,858 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:08,857 | server.py:222 | fit_round 22: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 131.77it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.69it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 131.84it/s]\n", + "2024-03-08 17:31:12,388 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:12,388 | server.py:236 | fit_round 22 received 3 results and 0 failures\n", + "2024-03-08 17:31:12,392 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:12,392 | server.py:173 | evaluate_round 22: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.34it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.89it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.59it/s]\n", + "2024-03-08 17:31:13,197 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:13,196 | server.py:187 | evaluate_round 22 received 3 results and 0 failures\n", + "2024-03-08 17:31:13,198 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:13,197 | server.py:222 | fit_round 23: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.38it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.02it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.08it/s]\n", + "2024-03-08 17:31:16,698 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:16,698 | server.py:236 | fit_round 23 received 3 results and 0 failures\n", + "2024-03-08 17:31:16,701 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:16,701 | server.py:173 | evaluate_round 23: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.03it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.07it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.75it/s]\n", + "2024-03-08 17:31:17,513 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:17,512 | server.py:187 | evaluate_round 23 received 3 results and 0 failures\n", + "2024-03-08 17:31:17,514 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:17,513 | server.py:222 | fit_round 24: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.53it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.88it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 131.73it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:31:21,109 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:21,109 | server.py:236 | fit_round 24 received 3 results and 0 failures\n", + "2024-03-08 17:31:21,115 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:21,115 | server.py:173 | evaluate_round 24: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.89it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.00it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 116.43it/s]\n", + "2024-03-08 17:31:21,920 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:21,920 | server.py:187 | evaluate_round 24 received 3 results and 0 failures\n", + "2024-03-08 17:31:21,921 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:21,920 | server.py:222 | fit_round 25: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.31it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.95it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.44it/s]\n", + "2024-03-08 17:31:25,484 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:25,483 | server.py:236 | fit_round 25 received 3 results and 0 failures\n", + "2024-03-08 17:31:25,486 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:25,486 | server.py:173 | evaluate_round 25: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.37it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.33it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 117.31it/s]\n", + "2024-03-08 17:31:26,286 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:26,286 | server.py:187 | evaluate_round 25 received 3 results and 0 failures\n", + "2024-03-08 17:31:26,288 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:26,286 | server.py:222 | fit_round 26: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.40it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.92it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.84it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:31:29,810 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:29,809 | server.py:236 | fit_round 26 received 3 results and 0 failures\n", + "2024-03-08 17:31:29,817 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:29,817 | server.py:173 | evaluate_round 26: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.21it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.94it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.84it/s]\n", + "2024-03-08 17:31:30,620 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:30,620 | server.py:187 | evaluate_round 26 received 3 results and 0 failures\n", + "2024-03-08 17:31:30,622 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:30,620 | server.py:222 | fit_round 27: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.06it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.43it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.89it/s]\n", + "2024-03-08 17:31:34,206 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:34,206 | server.py:236 | fit_round 27 received 3 results and 0 failures\n", + "2024-03-08 17:31:34,211 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:34,211 | server.py:173 | evaluate_round 27: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.69it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.09it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 116.81it/s]s]\n", + "2024-03-08 17:31:34,999 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:34,999 | server.py:187 | evaluate_round 27 received 3 results and 0 failures\n", + "2024-03-08 17:31:35,001 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:34,999 | server.py:222 | fit_round 28: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.34it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.92it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.75it/s]]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:31:38,517 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:38,517 | server.py:236 | fit_round 28 received 3 results and 0 failures\n", + "2024-03-08 17:31:38,521 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:38,521 | server.py:173 | evaluate_round 28: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.58it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.37it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.07it/s]\n", + "2024-03-08 17:31:39,358 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:39,358 | server.py:187 | evaluate_round 28 received 3 results and 0 failures\n", + "2024-03-08 17:31:39,361 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:39,358 | server.py:222 | fit_round 29: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.09it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 128.93it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 127.80it/s]\n", + "2024-03-08 17:31:42,950 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:42,950 | server.py:236 | fit_round 29 received 3 results and 0 failures\n", + "2024-03-08 17:31:42,954 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:42,954 | server.py:173 | evaluate_round 29: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.34it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 116.82it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 116.35it/s]\n", + "2024-03-08 17:31:43,747 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:43,747 | server.py:187 | evaluate_round 29 received 3 results and 0 failures\n", + "2024-03-08 17:31:43,749 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:43,747 | server.py:222 | fit_round 30: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.74it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.62it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.77it/s]\n", + "2024-03-08 17:31:47,284 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:47,284 | server.py:236 | fit_round 30 received 3 results and 0 failures\n", + "2024-03-08 17:31:47,290 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:47,290 | server.py:173 | evaluate_round 30: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.83it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.62it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 118.31it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-08 17:31:48,120 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:48,120 | server.py:187 | evaluate_round 30 received 3 results and 0 failures\n", + "2024-03-08 17:31:48,122 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:31:48,120 | server.py:153 | FL finished in 130.81519856399973\n", + "2024-03-08 17:31:48,124 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:31:48,121 | app.py:226 | app_fit: losses_distributed [(1, 241.25020345052084), (2, 237.8002471923828), (3, 220.30952962239584), (4, 204.25214640299478), (5, 193.9594980875651), (6, 182.6523183186849), (7, 174.97066243489584), (8, 169.68782552083334), (9, 164.67220052083334), (10, 161.17755126953125), (11, 157.6415812174479), (12, 153.90101114908853), (13, 150.4893798828125), (14, 148.1608632405599), (15, 145.3747304280599), (16, 142.51950073242188), (17, 139.92201232910156), (18, 137.5089111328125), (19, 135.6521199544271), (20, 132.60039265950522), (21, 130.73209126790366), (22, 128.79528299967447), (23, 126.02289072672527), (24, 123.57845815022786), (25, 122.14443715413411), (26, 119.8595682779948), (27, 117.2993647257487), (28, 115.81072489420573), (29, 114.09830220540364), (30, 113.56394958496094)]\n", + "2024-03-08 17:31:48,125 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:31:48,121 | app.py:227 | app_fit: metrics_distributed_fit {}\n", + "2024-03-08 17:31:48,126 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:31:48,121 | app.py:228 | app_fit: metrics_distributed {'accuracy': [(1, 0.1266746650669866), (2, 0.16076784643071385), (3, 0.23815236952609478), (4, 0.29324135172965404), (5, 0.32843431313737254), (6, 0.366126774645071), (7, 0.39172165566886624), (8, 0.41041791641671665), (9, 0.4238152369526095), (10, 0.4428114377124575), (11, 0.4528094381123775), (12, 0.46850629874025196), (13, 0.48000399920015996), (14, 0.4871025794841032), (15, 0.4998000399920016), (16, 0.5106978604279144), (17, 0.5235952809438112), (18, 0.5307938412317537), (19, 0.5366926614677064), (20, 0.5510897820435913), (21, 0.5560887822435513), (22, 0.5649870025994801), (23, 0.5681863627274545), (24, 0.5787842431513697), (25, 0.5822835432913417), (26, 0.592881423715257), (27, 0.598380323935213), (28, 0.6022795440911818), (29, 0.608878224355129), (30, 0.6139772045590882)]}\n", + "2024-03-08 17:31:48,127 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:31:48,121 | app.py:229 | app_fit: losses_centralized []\n", + "2024-03-08 17:31:48,128 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:31:48,121 | app.py:230 | app_fit: metrics_centralized {}\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.41it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.24it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 127.82it/s]\n", + "2024-03-08 17:31:48,129 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:48,129 | connection.py:220 | gRPC channel closed\n", + "2024-03-08 17:31:48,129 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:48,129 | connection.py:220 | gRPC channel closed\n", + "2024-03-08 17:31:48,129 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:31:48,129 | app.py:398 | Disconnect and shut down\n", + "2024-03-08 17:31:48,129 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-08 17:31:48,129 | connection.py:220 | gRPC channel closed\n", + "2024-03-08 17:31:48,129 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:31:48,129 | app.py:398 | Disconnect and shut down\n", + "2024-03-08 17:31:48,130 - SubprocessLauncher - INFO - INFO flwr 2024-03-08 17:31:48,129 | app.py:398 | Disconnect and shut down\n", + "2024-03-08 17:31:52,420 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,462 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,607 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,637 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,638 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,683 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,794 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,795 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,806 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,808 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,818 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:52,820 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,430 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,473 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,618 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,648 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,649 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,694 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,806 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,806 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,818 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,819 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,829 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:31:57,831 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", "run success\n" ] }, @@ -980,7 +554,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/media/hroth/NVIDIA/home_old/hroth/Code2/flower/baselines/fedprox/.venv_fedprox/lib/python3.10/site-packages/nvflare/app_common/launchers/subprocess_launcher.py:88: ResourceWarning: unclosed file <_io.BufferedReader name=65>\n", + "/media/hroth/NVIDIA/home_old/hroth/Code2/flower/baselines/fedprox/.venv_fedprox/lib/python3.10/site-packages/nvflare/app_common/launchers/subprocess_launcher.py:88: ResourceWarning: unclosed file <_io.BufferedReader name=82>\n", " self._process = None\n", "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" ] @@ -989,82 +563,88 @@ "name": "stdout", "output_type": "stream", "text": [ - "2024-03-06 16:06:50,132 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Stop Controller Launcher.\n", - "2024-03-06 16:06:50,134 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Workflow: controller_launcher finalizing ...\n", - "2024-03-06 16:06:50,414 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: ABOUT_TO_END_RUN fired\n", - "2024-03-06 16:06:50,417 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Firing CHECK_END_RUN_READINESS ...\n", - "2024-03-06 16:06:50,418 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: END_RUN fired\n", - "2024-03-06 16:06:50,419 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Server runner finished.\n", - "2024-03-06 16:06:50,776 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-2, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", - "2024-03-06 16:06:50,779 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-1, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", - "2024-03-06 16:06:50,780 - GetTaskCommand - INFO - return task to client. client_name: site-2 task_name: __end_run__ task_id: sharable_header_task_id: \n", - "2024-03-06 16:06:50,782 - GetTaskCommand - INFO - return task to client. client_name: site-1 task_name: __end_run__ task_id: sharable_header_task_id: \n", - "2024-03-06 16:06:50,814 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-3, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", - "2024-03-06 16:06:50,816 - GetTaskCommand - INFO - return task to client. client_name: site-3 task_name: __end_run__ task_id: sharable_header_task_id: \n", - "2024-03-06 16:06:50,835 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00006 Not Connected] is closed PID: 13991\n", - "2024-03-06 16:06:50,836 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00007 Not Connected] is closed PID: 13991\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/lib/python3.10/subprocess.py:1072: ResourceWarning: subprocess 14189 is still running\n", - " _warn(\"subprocess %s is still running\" % self.pid,\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", - "/usr/lib/python3.10/subprocess.py:1072: ResourceWarning: subprocess 14192 is still running\n", - " _warn(\"subprocess %s is still running\" % self.pid,\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", - "/usr/lib/python3.10/subprocess.py:1072: ResourceWarning: subprocess 14194 is still running\n", - " _warn(\"subprocess %s is still running\" % self.pid,\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2024-03-06 16:06:50,867 - FederatedClient - INFO - Shutting down client run: site-1\n", - "2024-03-06 16:06:50,868 - FederatedClient - INFO - Shutting down client run: site-2\n", - "2024-03-06 16:06:50,868 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00008 Not Connected] is closed PID: 13991\n", - "2024-03-06 16:06:50,868 - FederatedClient - INFO - Shutting down client run: site-3\n", - "2024-03-06 16:06:50,870 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: asked to abort - triggered abort_signal to stop the RUN\n", - "2024-03-06 16:06:50,787 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", - "2024-03-06 16:06:50,787 - ClientRunner - INFO - [identity=site-2, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", - "2024-03-06 16:06:50,787 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: started end-run events sequence\n", - "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: ABOUT_TO_END_RUN fired\n", - "2024-03-06 16:06:50,788 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", - "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: Firing CHECK_END_RUN_READINESS ...\n", - "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-1, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", - "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: started end-run events sequence\n", - "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: END_RUN fired\n", - "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: ABOUT_TO_END_RUN fired\n", - "2024-03-06 16:06:50,788 - ClientTaskWorker - INFO - End the Simulator run.\n", - "2024-03-06 16:06:50,789 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: Firing CHECK_END_RUN_READINESS ...\n", - "2024-03-06 16:06:50,789 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: END_RUN fired\n", - "2024-03-06 16:06:50,789 - ClientTaskWorker - INFO - End the Simulator run.\n", - "2024-03-06 16:06:50,819 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", - "2024-03-06 16:06:50,820 - ClientRunner - INFO - [identity=site-3, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", - "2024-03-06 16:06:50,820 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: started end-run events sequence\n", - "2024-03-06 16:06:50,820 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: ABOUT_TO_END_RUN fired\n", - "2024-03-06 16:06:50,820 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: Firing CHECK_END_RUN_READINESS ...\n", - "2024-03-06 16:06:50,820 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: END_RUN fired\n", - "2024-03-06 16:06:50,820 - ClientTaskWorker - INFO - End the Simulator run.\n", - "2024-03-06 16:06:50,832 - ClientTaskWorker - INFO - Clean up ClientRunner for : site-1 \n", - "2024-03-06 16:06:50,833 - ClientTaskWorker - INFO - Clean up ClientRunner for : site-2 \n", - "2024-03-06 16:06:50,835 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 Not Connected] is closed PID: 14189\n", - "2024-03-06 16:06:50,836 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 Not Connected] is closed PID: 14192\n", - "2024-03-06 16:06:50,864 - ClientTaskWorker - INFO - Clean up ClientRunner for : site-3 \n", - "2024-03-06 16:06:50,867 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 Not Connected] is closed PID: 14194\n", - "2024-03-06 16:06:53,007 - SimulatorServer - INFO - Server app stopped.\n", + "2024-03-08 17:31:58,047 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Stop Controller Launcher.\n", + "2024-03-08 17:31:58,049 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Workflow: controller_launcher finalizing ...\n", + "2024-03-08 17:31:58,104 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: ABOUT_TO_END_RUN fired\n", + "2024-03-08 17:31:58,106 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Firing CHECK_END_RUN_READINESS ...\n", + "2024-03-08 17:31:58,113 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-3, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", + "2024-03-08 17:31:58,114 - GetTaskCommand - INFO - return task to client. client_name: site-3 task_name: __end_run__ task_id: sharable_header_task_id: \n", + "2024-03-08 17:31:58,187 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-2, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", + "2024-03-08 17:31:58,189 - GetTaskCommand - INFO - return task to client. client_name: site-2 task_name: __end_run__ task_id: sharable_header_task_id: \n", + "2024-03-08 17:31:58,117 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", + "2024-03-08 17:31:58,117 - ClientRunner - INFO - [identity=site-3, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", + "2024-03-08 17:31:58,118 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: started end-run events sequence\n", + "2024-03-08 17:31:58,118 - MetricRelay - INFO - [identity=site-3, run=simulate_job]: Stopping pipe handler\n", + "2024-03-08 17:31:58,192 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", + "2024-03-08 17:31:58,192 - ClientRunner - INFO - [identity=site-2, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", + "2024-03-08 17:31:58,192 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: started end-run events sequence\n", + "2024-03-08 17:31:58,192 - MetricRelay - INFO - [identity=site-2, run=simulate_job]: Stopping pipe handler\n", + "2024-03-08 17:31:58,604 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-1, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", + "2024-03-08 17:31:58,606 - GetTaskCommand - INFO - return task to client. client_name: site-1 task_name: __end_run__ task_id: sharable_header_task_id: \n", + "2024-03-08 17:31:58,609 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", + "2024-03-08 17:31:58,609 - ClientRunner - INFO - [identity=site-1, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", + "2024-03-08 17:31:58,609 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: started end-run events sequence\n", + "2024-03-08 17:31:58,609 - MetricRelay - INFO - [identity=site-1, run=simulate_job]: Stopping pipe handler\n", + "2024-03-08 17:32:00,115 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: END_RUN fired\n", + "2024-03-08 17:32:00,118 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Server runner finished.\n", + "2024-03-08 17:32:00,984 - SimulatorServer - INFO - Server app stopped.\n", "\n", "\n", - "2024-03-06 16:06:53,464 - nvflare.fuel.hci.server.hci - INFO - Admin Server localhost on Port 40275 shutdown!\n", - "2024-03-06 16:06:53,467 - SimulatorServer - INFO - shutting down server\n", - "2024-03-06 16:06:53,468 - SimulatorServer - INFO - canceling sync locks\n", - "2024-03-06 16:06:53,470 - SimulatorServer - INFO - server off\n", - "2024-03-06 16:06:56,891 - MPM - INFO - MPM: Good Bye!\n", - "Simulator finished with run_status 0\n" + "2024-03-08 17:32:01,126 - nvflare.fuel.hci.server.hci - INFO - Admin Server localhost on Port 54963 shutdown!\n", + "2024-03-08 17:32:01,129 - SimulatorServer - INFO - shutting down server\n", + "2024-03-08 17:32:01,130 - SimulatorServer - INFO - canceling sync locks\n", + "2024-03-08 17:32:01,131 - SimulatorServer - INFO - server off\n", + "2024-03-08 17:32:02,443 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,485 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,630 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,660 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,662 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,705 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,807 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,808 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,830 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,831 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,831 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:02,843 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:07,454 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:07,497 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:07,641 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:07,672 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:07,673 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:07,716 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:07,841 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:07,843 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:07,854 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:12,467 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:12,509 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:12,654 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:12,683 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:12,685 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:12,727 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:12,853 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:12,854 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:12,866 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:17,479 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:17,520 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:17,666 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:17,699 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:17,700 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:17,737 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:17,864 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:17,866 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:17,878 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:22,519 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:22,696 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:22,737 - SubprocessLauncher - INFO - Error processing frame: RuntimeError: cannot schedule new futures after shutdown\n", + "2024-03-08 17:32:23,019 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00012 Not Connected] is closed PID: 122617\n", + "2024-03-08 17:32:23,196 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00013 Not Connected] is closed PID: 122617\n", + "2024-03-08 17:32:23,238 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00014 Not Connected] is closed PID: 122617\n", + "2024-03-08 17:32:47,610 - MetricRelay - INFO - metric pipe status changed to _PEER_GONE_\n", + "2024-03-08 17:32:47,721 - MetricRelay - INFO - metric pipe status changed to _PEER_GONE_\n", + "2024-03-08 17:32:47,863 - MetricRelay - INFO - metric pipe status changed to _PEER_GONE_\n", + "2024-03-08 17:33:17,988 - ClientAPILauncherExecutor - INFO - pipe status changed to _PEER_GONE_\n", + "2024-03-08 17:33:18,001 - ClientAPILauncherExecutor - INFO - pipe status changed to _PEER_GONE_\n", + "2024-03-08 17:33:18,016 - ClientAPILauncherExecutor - INFO - pipe status changed to _PEER_GONE_\n" ] } ], @@ -1074,8 +654,8 @@ "n_clients = 3\n", "\n", "simulator = SimulatorRunner(\n", - " job_folder=f\"./jobs/flwr_cifar10\",\n", - " workspace=f\"/tmp/nvflare/flwr_cifar10\",\n", + " job_folder=f\"./jobs/flwr_cifar10_tb_streaming\", # or mlflow, wandb\n", + " workspace=f\"/tmp/nvflare/flwr_cifar10_tb_streaming\",\n", " n_clients=n_clients,\n", " threads=n_clients\n", ")\n", @@ -1083,13 +663,23 @@ "print(\"Simulator finished with run_status\", run_status)" ] }, + { + "cell_type": "markdown", + "id": "b5210ed3-6ede-449a-8afa-f5308c23f7c4", + "metadata": {}, + "source": [ + "## Visualize in TensorBoard" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c90df20b-f3b3-4d4b-885b-830874a49552", + "id": "f525b4e1-40ed-4f45-9f49-38d191cd99cf", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "#!tensorboard --logdir /tmp/nvflare/flwr_cifar10_tb_streaming" + ] } ], "metadata": { diff --git a/examples/advanced/flower/fedprox/flwr_scripts/client.py b/examples/advanced/flower/fedprox/flwr_scripts/client.py deleted file mode 100644 index 0591d40581..0000000000 --- a/examples/advanced/flower/fedprox/flwr_scripts/client.py +++ /dev/null @@ -1,97 +0,0 @@ -from collections import OrderedDict -import warnings - -import flwr as fl -import torch -import torch.nn as nn -import torch.nn.functional as F -from torchvision.transforms import Compose, ToTensor, Normalize -from torch.utils.data import DataLoader -from torchvision.datasets import CIFAR10 - -# ############################################################################# -# Regular PyTorch pipeline: nn.Module, train, test, and DataLoader -# ############################################################################# - -warnings.filterwarnings("ignore", category=UserWarning) -DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - -class Net(nn.Module): - """Model (simple CNN adapted from 'PyTorch: A 60 Minute Blitz')""" - - def __init__(self) -> None: - super(Net, self).__init__() - self.conv1 = nn.Conv2d(3, 6, 5) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(6, 16, 5) - self.fc1 = nn.Linear(16 * 5 * 5, 120) - self.fc2 = nn.Linear(120, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = x.view(-1, 16 * 5 * 5) - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - return self.fc3(x) - -def train(net, trainloader, epochs): - """Train the model on the training set.""" - criterion = torch.nn.CrossEntropyLoss() - optimizer = torch.optim.SGD(net.parameters(), lr=0.001, momentum=0.9) - for _ in range(epochs): - for images, labels in trainloader: - optimizer.zero_grad() - criterion(net(images.to(DEVICE)), labels.to(DEVICE)).backward() - optimizer.step() - -def test(net, testloader): - """Validate the model on the test set.""" - criterion = torch.nn.CrossEntropyLoss() - correct, total, loss = 0, 0, 0.0 - with torch.no_grad(): - for images, labels in testloader: - outputs = net(images.to(DEVICE)) - loss += criterion(outputs, labels.to(DEVICE)).item() - total += labels.size(0) - correct += (torch.max(outputs.data, 1)[1] == labels).sum().item() - return loss / len(testloader.dataset), correct / total - -def load_data(): - """Load CIFAR-10 (training and test set).""" - trf = Compose([ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - trainset = CIFAR10("./data", train=True, download=True, transform=trf) - testset = CIFAR10("./data", train=False, download=True, transform=trf) - return DataLoader(trainset, batch_size=32, shuffle=True), DataLoader(testset) - -# ############################################################################# -# Federating the pipeline with Flower -# ############################################################################# - -# Load model and data (simple CNN, CIFAR-10) -net = Net().to(DEVICE) -trainloader, testloader = load_data() - -# Define Flower client -class FlowerClient(fl.client.NumPyClient): - def get_parameters(self, config): - return [val.cpu().numpy() for _, val in net.state_dict().items()] - - def set_parameters(self, parameters): - params_dict = zip(net.state_dict().keys(), parameters) - state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) - net.load_state_dict(state_dict, strict=True) - - def fit(self, parameters, config): - self.set_parameters(parameters) - train(net, trainloader, epochs=1) - return self.get_parameters(config={}), len(trainloader.dataset), {} - - def evaluate(self, parameters, config): - self.set_parameters(parameters) - loss, accuracy = test(net, testloader) - return float(loss), len(testloader.dataset), {"accuracy": float(accuracy)} - -# Start Flower client -fl.client.start_numpy_client(server_address="127.0.0.1:8080", client=FlowerClient(), insecure=True) diff --git a/examples/advanced/flower/fedprox/flwr_scripts/server.py b/examples/advanced/flower/fedprox/flwr_scripts/server.py deleted file mode 100644 index 0a27b6e701..0000000000 --- a/examples/advanced/flower/fedprox/flwr_scripts/server.py +++ /dev/null @@ -1,7 +0,0 @@ -import flwr as fl - -# Start Flower server -fl.server.start_server( - server_address="0.0.0.0:8080", - config=fl.server.ServerConfig(num_rounds=3), -) diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/custom/client.py b/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/custom/client.py index 4898bed85b..2ca3bdd75b 100644 --- a/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/custom/client.py +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/custom/client.py @@ -4,7 +4,6 @@ import flwr as fl from flwr_datasets import FederatedDataset -import numpy as np import torch import torch.nn as nn import torch.nn.functional as F diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/config/config_fed_client.conf b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/config/config_fed_client.conf new file mode 100644 index 0000000000..83614f1716 --- /dev/null +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/config/config_fed_client.conf @@ -0,0 +1,114 @@ +{ + # version of the configuration + format_version = 2 + + # Client Computing Executors. + executors = [ + { + # tasks the executors are defined to handle + tasks = ["train"] + + # This particular executor + executor { + + # Executor name : ClientAPILauncherExecutor + # This is an executor for Client API. The underline data exchange is using Pipe. + path = "nvflare.app_common.executors.client_api_launcher_executor.ClientAPILauncherExecutor" + + args { + + # This executor take an component named "launcher" + launcher_id = "launcher" + + # This executor needs Pipe component + pipe_id = "pipe" + + # Timeout in seconds for waiting for a heartbeat from the training script. Defaults to 30 seconds. + # Please refer to the class docstring for all available arguments + heartbeat_timeout = 60 + + # format of the exchange parameters + params_exchange_format = "numpy" + + # if the transfer_type is FULL, then it will be sent directly + # if the transfer_type is DIFF, then we will calculate the + # difference VS received parameters and send the difference + params_transfer_type = "FULL" + + # if train_with_evaluation is true, the executor will expect + # the custom code need to send back both the trained parameters and the evaluation metric + # otherwise only trained parameters are expected + train_with_evaluation = true + + } + } + } + ], + + # this defined an array of task data filters. If provided, it will control the data from server controller to client executor + task_data_filters = [] + + # this defined an array of task result filters. If provided, it will control the result from client executor to server controller + task_result_filters = [] + + components = [ + { + # component id is "launcher" + id = "launcher" + + # the class path of this component + path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" + + args { + # the launcher will invoke the script + script = "python3 custom/client.py --node-id 0" + # if launch_once is true, the SubprocessLauncher will launch once for the whole job + # if launch_once is false, the SubprocessLauncher will launch a process for each task it receives from server + launch_once = true + } + }, + { + id = "pipe" + path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" + args { + mode = "PASSIVE" + site_name = "{SITE_NAME}" + token = "{JOB_ID}" + root_url = "{ROOT_URL}" + secure_mode = "{SECURE_MODE}" + workspace_dir = "{WORKSPACE}" + } + }, + { + id = "metrics_pipe" + path = "nvflare.fuel.utils.pipe.cell_pipe.CellPipe" + args { + mode = "PASSIVE" + site_name = "{SITE_NAME}" + token = "{JOB_ID}" + root_url = "{ROOT_URL}" + secure_mode = "{SECURE_MODE}" + workspace_dir = "{WORKSPACE}" + } + }, + { + id = "metric_relay" + path = "nvflare.app_common.widgets.metric_relay.MetricRelay" + args { + pipe_id = "metrics_pipe" + event_type = "fed.analytix_log_stats" + # how fast should it read from the peer + read_interval = 0.1 + } + }, + { + # we use this component so the client api `flare.init()` can get required information + id = "client_api_config_preparer" + path = "nvflare.app_common.widgets.external_configurator.ExternalConfigurator" + args { + component_ids = ["metric_relay"] + } + } + ] +} + diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/config/config_fed_server.conf b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/config/config_fed_server.conf new file mode 100644 index 0000000000..86efb629de --- /dev/null +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/config/config_fed_server.conf @@ -0,0 +1,29 @@ +{ + format_version = 2 + task_data_filters = [] + task_result_filters = [] + workflows = [ + { + id = "controller_launcher" + path = "controller_launcher.ControllerLauncher" + args { + launcher_id = "launcher" + } + } + ] + components = [ + { + id = "launcher" + path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" + args { + script = "python3 custom/server.py" + launch_once = true + } + }, + { + id = "tb_analytics_receiver" + path = "nvflare.app_opt.tracking.tb.tb_receiver.TBAnalyticsReceiver" + args.events = ["fed.analytix_log_stats"] + } + ] +} diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/client.py b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/client.py new file mode 100644 index 0000000000..26b7b32269 --- /dev/null +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/client.py @@ -0,0 +1,160 @@ +import argparse +import warnings +from collections import OrderedDict + +import flwr as fl +from flwr_datasets import FederatedDataset +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +from torchvision.transforms import Compose, Normalize, ToTensor +from tqdm import tqdm + +import nvflare.client as flare +from nvflare.client.tracking import SummaryWriter +writer = SummaryWriter() +TRAIN_STEP = 0 +TEST_STEP = 0 + +# ############################################################################# +# 1. Regular PyTorch pipeline: nn.Module, train, test, and DataLoader +# ############################################################################# + +warnings.filterwarnings("ignore", category=UserWarning) +DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + +class Net(nn.Module): + """Model (simple CNN adapted from 'PyTorch: A 60 Minute Blitz')""" + + def __init__(self) -> None: + super(Net, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = x.view(-1, 16 * 5 * 5) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + return self.fc3(x) + +def train(net, trainloader, epochs): + """Train the model on the training set.""" + criterion = torch.nn.CrossEntropyLoss() + optimizer = torch.optim.SGD(net.parameters(), lr=0.001, momentum=0.9) + + global TRAIN_STEP + for _ in range(epochs): + avg_loss = 0.0 + for batch in tqdm(trainloader, "Training"): + images = batch["img"] + labels = batch["label"] + optimizer.zero_grad() + loss = criterion(net(images.to(DEVICE)), labels.to(DEVICE)) + loss.backward() + optimizer.step() + avg_loss += loss.item() + writer.add_scalar("train_loss", avg_loss/len(trainloader), TRAIN_STEP) + TRAIN_STEP += 1 + + +def test(net, testloader): + """Validate the model on the test set.""" + criterion = torch.nn.CrossEntropyLoss() + correct, loss = 0, 0.0 + global TEST_STEP + with torch.no_grad(): + for batch in tqdm(testloader, "Testing"): + images = batch["img"].to(DEVICE) + labels = batch["label"].to(DEVICE) + outputs = net(images) + loss += criterion(outputs, labels).item() + correct += (torch.max(outputs.data, 1)[1] == labels).sum().item() + accuracy = correct / len(testloader.dataset) + writer.add_scalar("test_loss", loss, TEST_STEP) + writer.add_scalar("test_accuracy", accuracy, TEST_STEP) + TEST_STEP += 1 + return loss, accuracy + + +def load_data(node_id): + """Load partition CIFAR10 data.""" + fds = FederatedDataset(dataset="cifar10", partitioners={"train": 3}) + partition = fds.load_partition(node_id) + # Divide data on each node: 80% train, 20% test + partition_train_test = partition.train_test_split(test_size=0.2) + pytorch_transforms = Compose( + [ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] + ) + + def apply_transforms(batch): + """Apply transforms to the partition from FederatedDataset.""" + batch["img"] = [pytorch_transforms(img) for img in batch["img"]] + return batch + + partition_train_test = partition_train_test.with_transform(apply_transforms) + trainloader = DataLoader(partition_train_test["train"], batch_size=32, shuffle=True) + testloader = DataLoader(partition_train_test["test"], batch_size=32) + return trainloader, testloader + + +# ############################################################################# +# 2. Federation of the pipeline with Flower +# ############################################################################# + +# Get node id +parser = argparse.ArgumentParser(description="Flower") +parser.add_argument( + "--node-id", + choices=[0, 1, 2], + required=True, + type=int, + help="Partition of the dataset divided into 3 iid partitions created artificially.", +) +node_id = parser.parse_args().node_id + +# Load model and data (simple CNN, CIFAR-10) +net = Net().to(DEVICE) +trainloader, testloader = load_data(node_id=node_id) + + +# Define Flower client +class FlowerClient(fl.client.NumPyClient): + def get_parameters(self, config): + return [val.cpu().numpy() for _, val in net.state_dict().items()] + + def set_parameters(self, parameters): + params_dict = zip(net.state_dict().keys(), parameters) + state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) + net.load_state_dict(state_dict, strict=True) + + def fit(self, parameters, config): + self.set_parameters(parameters) + train(net, trainloader, epochs=1) + return self.get_parameters(config={}), len(trainloader.dataset), {} + + def evaluate(self, parameters, config): + self.set_parameters(parameters) + loss, accuracy = test(net, testloader) + return loss, len(testloader.dataset), {"accuracy": accuracy} + + +# initializes NVFlare interface +flare.init() + +# get system information +sys_info = flare.system_info() +print(f"Flare system info is: {sys_info}") + +# Start Flower client +fl.client.start_client( + server_address="127.0.0.1:8080", + client=FlowerClient().to_client(), +) diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/controller_launcher.py b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/controller_launcher.py new file mode 100644 index 0000000000..17a775ac20 --- /dev/null +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/controller_launcher.py @@ -0,0 +1,83 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from nvflare.app_common.workflows.model_controller import ModelController +from nvflare.app_common.abstract.launcher import Launcher, LauncherRunStatus +from nvflare.apis.fl_context import FLContext +from nvflare.apis.shareable import Shareable +from nvflare.app_common.app_constant import AppConstants, ValidateType +from nvflare.fuel.utils.validation_utils import check_object_type + + +class ControllerLauncher(ModelController): + """The base controller for FedAvg Workflow. *Note*: This class is based on the experimental `ModelController`. + + Implements [FederatedAveraging](https://arxiv.org/abs/1602.05629). + The model persistor (persistor_id) is used to load the initial global model which is sent to a list of clients. + Each client sends it's updated weights after local training which is aggregated. + Next, the global model is updated. + The model_persistor also saves the model after training. + + Provides the default implementations for the follow routines: + - def sample_clients(self, min_clients) + - def aggregate(self, results: List[FLModel], aggregate_fn=None) -> FLModel + - def update_model(self, aggr_result) + + The `run` routine needs to be implemented by the derived class: + + - def run(self) + """ + + def __init__(self, + launcher_id, + task_name=AppConstants.TASK_TRAIN + ): + super().__init__() + self._launcher_id = launcher_id + self._task_name = task_name + self.is_initialized = False + + def _init_launcher(self, fl_ctx: FLContext): + engine = fl_ctx.get_engine() + launcher: Launcher = engine.get_component(self._launcher_id) + if launcher is None: + raise RuntimeError(f"Launcher can not be found using {self._launcher_id}") + check_object_type(self._launcher_id, launcher, Launcher) + self.launcher = launcher + self.is_initialized = True + + def run(self): + self.info("Start Controller Launcher.") + + if not self.is_initialized: + self._init_launcher(self.fl_ctx) + + self.launcher.initialize(fl_ctx=self.fl_ctx) + + while True: + time.sleep(10.0) + run_status = self.launcher.check_run_status(task_name=self._task_name, fl_ctx=self.fl_ctx) + if run_status == LauncherRunStatus.RUNNING: + print(f"Running ... [{self.launcher._script}]") + elif run_status == LauncherRunStatus.COMPLETE_SUCCESS: + print("run success") + break + else: + print(f"run failed or not start: {run_status}") + break + self.launcher.finalize(fl_ctx=self.fl_ctx) + self.info("Stop Controller Launcher.") + diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/executor_launcher.py b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/executor_launcher.py new file mode 100644 index 0000000000..b208a2446c --- /dev/null +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/executor_launcher.py @@ -0,0 +1,89 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from nvflare.apis.dxo import MetaKey, from_shareable +from nvflare.apis.event_type import EventType +from nvflare.apis.executor import Executor +from nvflare.apis.fl_constant import ReturnCode +from nvflare.apis.fl_context import FLContext +from nvflare.apis.shareable import Shareable, make_reply +from nvflare.apis.signal import Signal +from nvflare.app_common.abstract.learner_spec import Learner +from nvflare.app_common.app_constant import AppConstants, ValidateType +from nvflare.security.logging import secure_format_exception +from nvflare.app_common.workflows.model_controller import ModelController +from nvflare.app_common.abstract.launcher import Launcher, LauncherRunStatus +from nvflare.apis.fl_context import FLContext +from nvflare.apis.shareable import Shareable +from nvflare.fuel.utils.validation_utils import check_object_type + + +class ExecutorLauncher(Executor): + def __init__( + self, + launcher_id="launcher", + task_name=AppConstants.TASK_TRAIN + ): + """Key component to run learner on clients. + + Args: + learner_id (str): id of the learner object + train_task (str, optional): task name for train. Defaults to AppConstants.TASK_TRAIN. + submit_model_task (str, optional): task name for submit model. Defaults to AppConstants.TASK_SUBMIT_MODEL. + validate_task (str, optional): task name for validation. Defaults to AppConstants.TASK_VALIDATION. + """ + super().__init__() + self._launcher_id = launcher_id + self._task_name = task_name + self.is_initialized = False + + def _init_launcher(self, fl_ctx: FLContext): + engine = fl_ctx.get_engine() + launcher: Launcher = engine.get_component(self._launcher_id) + if launcher is None: + raise RuntimeError(f"Launcher can not be found using {self._launcher_id}") + check_object_type(self._launcher_id, launcher, Launcher) + self.launcher = launcher + self.is_initialized = True + + def handle_event(self, event_type: str, fl_ctx: FLContext): + if event_type == EventType.START_RUN: + if not self.is_initialized: + self._init_launcher(fl_ctx) + + self._launch_script(fl_ctx) + + def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable: + pass + + def _launch_script(self, fl_ctx: FLContext): + self.log_info(fl_ctx, "Start Executor Launcher.") + + self.launcher.initialize(fl_ctx=fl_ctx) + + while True: + time.sleep(10.0) + run_status = self.launcher.check_run_status(task_name=self._task_name, fl_ctx=fl_ctx) + if run_status == LauncherRunStatus.RUNNING: + print(f"Running ... [{self.launcher._script}]") + elif run_status == LauncherRunStatus.COMPLETE_SUCCESS: + print("run success") + break + else: + print(f"run failed or not start: {run_status}") + break + self.launcher.finalize(fl_ctx=fl_ctx) + self.log_info(fl_ctx, "Stop Executor Launcher.") diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/server.py b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/server.py new file mode 100644 index 0000000000..551a35814f --- /dev/null +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/server.py @@ -0,0 +1,31 @@ +from typing import List, Tuple + +import utils +import flwr as fl +from flwr.common import Metrics + + +# Define metric aggregation function +def weighted_average(metrics: List[Tuple[int, Metrics]]) -> Metrics: + # Multiply accuracy of each client by number of examples used + accuracies = [num_examples * m["accuracy"] for num_examples, m in metrics] + examples = [num_examples for num_examples, _ in metrics] + + # Aggregate and return custom metric (weighted average) + return {"accuracy": sum(accuracies) / sum(examples)} + + +# Define strategy +strategy = fl.server.strategy.FedAvg(evaluate_metrics_aggregation_fn=weighted_average) + +# Start Flower server +history = fl.server.start_server( + server_address="0.0.0.0:8080", + config=fl.server.ServerConfig(num_rounds=30), + strategy=strategy, +) + +utils.plot_metric_from_history( + hist=history, + save_plot_path=".", +) diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/utils.py b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/utils.py new file mode 100644 index 0000000000..230b1744ce --- /dev/null +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/app/custom/utils.py @@ -0,0 +1,117 @@ +"""Contains utility functions for CNN FL on MNIST.""" + +import pickle +from pathlib import Path +from secrets import token_hex +from typing import Dict, Optional, Union + +import matplotlib.pyplot as plt +import numpy as np +from flwr.server.history import History + + +def plot_metric_from_history( + hist: History, + save_plot_path: str, + suffix: Optional[str] = "", +) -> None: + """Plot from Flower server History. + + Parameters + ---------- + hist : History + Object containing evaluation for all rounds. + save_plot_path : str + Folder to save the plot to. + suffix: Optional[str] + Optional string to add at the end of the filename for the plot. + """ + #metric_type = "centralized" + metric_type = "FL" + metric_dict = ( + hist.metrics_centralized + if metric_type == "centralized" + else hist.metrics_distributed + ) + _, values = zip(*metric_dict["accuracy"]) + + # let's extract centralised loss (main metric reported in FedProx paper) + #rounds_loss, values_loss = zip(*hist.losses_centralized) + rounds_loss, values_loss = zip(*hist.losses_distributed) + + _, axs = plt.subplots(nrows=2, ncols=1, sharex="row") + axs[0].plot(np.asarray(rounds_loss), np.asarray(values_loss)) + axs[1].plot(np.asarray(rounds_loss), np.asarray(values)) + + axs[0].set_ylabel("Loss") + axs[1].set_ylabel("Accuracy") + + # plt.title(f"{metric_type.capitalize()} Validation - MNIST") + plt.xlabel("Rounds") + # plt.legend(loc="lower right") + + plt.savefig(Path(save_plot_path) / Path(f"{metric_type}_metrics{suffix}.png")) + plt.close() + + +def save_results_as_pickle( + history: History, + file_path: Union[str, Path], + extra_results: Optional[Dict] = None, + default_filename: str = "results.pkl", +) -> None: + """Save results from simulation to pickle. + + Parameters + ---------- + history: History + History returned by start_simulation. + file_path: Union[str, Path] + Path to file to create and store both history and extra_results. + If path is a directory, the default_filename will be used. + path doesn't exist, it will be created. If file exists, a + randomly generated suffix will be added to the file name. This + is done to avoid overwritting results. + extra_results : Optional[Dict] + A dictionary containing additional results you would like + to be saved to disk. Default: {} (an empty dictionary) + default_filename: Optional[str] + File used by default if file_path points to a directory instead + to a file. Default: "results.pkl" + """ + path = Path(file_path) + + # ensure path exists + path.mkdir(exist_ok=True, parents=True) + + def _add_random_suffix(path_: Path): + """Add a randomly generated suffix to the file name (so it doesn't. + + overwrite the file). + """ + print(f"File `{path_}` exists! ") + suffix = token_hex(4) + print(f"New results to be saved with suffix: {suffix}") + return path_.parent / (path_.stem + "_" + suffix + ".pkl") + + def _complete_path_with_default_name(path_: Path): + """Append the default file name to the path.""" + print("Using default filename") + return path_ / default_filename + + if path.is_dir(): + path = _complete_path_with_default_name(path) + + if path.is_file(): + # file exists already + path = _add_random_suffix(path) + + print(f"Results will be saved into: {path}") + + data = {"history": history} + if extra_results is not None: + data = {**data, **extra_results} + + # save results to pickle + with open(str(path), "wb") as handle: + pickle.dump(data, handle, protocol=pickle.HIGHEST_PROTOCOL) diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/meta.conf b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/meta.conf new file mode 100644 index 0000000000..692d32a814 --- /dev/null +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10_tb_streaming/meta.conf @@ -0,0 +1,11 @@ +{ + name = "flwr_cifar10" + resource_spec {} + deploy_map { + app = [ + "@ALL" + ] + } + min_clients = 2 + mandatory_clients = [] +}