diff --git a/examples/advanced/flower/fedprox/flower_fedprox.ipynb b/examples/advanced/flower/fedprox/flower_fedprox.ipynb index b4401a3830..d8b092c5f2 100644 --- a/examples/advanced/flower/fedprox/flower_fedprox.ipynb +++ b/examples/advanced/flower/fedprox/flower_fedprox.ipynb @@ -5,12 +5,12 @@ "id": "a6c92407-974d-45b5-b42f-de363b090966", "metadata": {}, "source": [ - "# Flower launcher example with FedProx" + "# NVFlare launcher example with Flower" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "c8ee32ae-cee9-454e-9b68-3427a2b7e991", "metadata": {}, "outputs": [], @@ -20,10 +20,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "3479da05-40df-4d47-b942-825c10e03a8a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "The following are the variables you can change in the template\n", + "\n", + "---------------------------------------------------------------------------------------------------------------------------------------\n", + " \n", + " job folder: ./jobs/_flwr_cifar10 \n", + " \n", + "---------------------------------------------------------------------------------------------------------------------------------------\n", + " file_name var_name value component \n", + "---------------------------------------------------------------------------------------------------------------------------------------\n", + " meta.conf app ['@ALL'] \n", + " meta.conf mandatory_clients [] \n", + " meta.conf min_clients 2 \n", + "\n", + " config_fed_client.conf app_config \n", + " config_fed_client.conf app_script client.py \n", + " config_fed_client.conf component_ids ['metric_relay'] ExternalConfigurator \n", + " config_fed_client.conf config_file_name client_api_config.json ExternalConfigurator \n", + " config_fed_client.conf evaluate_task_name evaluate \n", + " config_fed_client.conf event_type fed.analytix_log_stats MetricRelay \n", + " config_fed_client.conf external_execution_wait 5.0 \n", + " config_fed_client.conf fed_event True MetricRelay \n", + " config_fed_client.conf heartbeat_interval 5.0 MetricRelay \n", + " config_fed_client.conf heartbeat_timeout 30.0 MetricRelay \n", + " config_fed_client.conf last_result_transfer_timeout 300.0 \n", + " config_fed_client.conf launch_once True SubprocessLauncher \n", + " config_fed_client.conf mode PASSIVE CellPipe \n", + " config_fed_client.conf monitor_interval 0.01 \n", + " config_fed_client.conf params_exchange_format pytorch \n", + " config_fed_client.conf params_transfer_type DIFF \n", + " config_fed_client.conf pipe_channel_name metric MetricRelay \n", + " config_fed_client.conf read_interval 0.5 \n", + " config_fed_client.conf root_url {ROOT_URL} CellPipe \n", + " config_fed_client.conf script python3 custom/{app_script} {app_c SubprocessLauncher \n", + " config_fed_client.conf secure_mode {SECURE_MODE} CellPipe \n", + " config_fed_client.conf site_name {SITE_NAME} CellPipe \n", + " config_fed_client.conf token {JOB_ID} CellPipe \n", + " config_fed_client.conf train_with_evaluation True \n", + " config_fed_client.conf workers 4 \n", + " config_fed_client.conf workspace_dir {WORKSPACE} CellPipe \n", + "\n", + " config_fed_server.conf allow_empty_global_weights False ScatterAndGather \n", + " config_fed_server.conf best_global_model_file_name best_FL_global_model.pt PTFileModelPersistor \n", + " config_fed_server.conf events ['fed.analytix_log_stats'] TBAnalyticsReceiver \n", + " config_fed_server.conf expected_data_kind WEIGHT_DIFF InTimeAccumulateWeightedAggregator \n", + " config_fed_server.conf global_model_file_name FL_global_model.pt PTFileModelPersistor \n", + " config_fed_server.conf ignore_result_error False ScatterAndGather \n", + " config_fed_server.conf key_metric accuracy IntimeModelSelector \n", + " config_fed_server.conf min_clients 2 ScatterAndGather \n", + " config_fed_server.conf model_class_path net.Net \n", + " config_fed_server.conf negate_key_metric False IntimeModelSelector \n", + " config_fed_server.conf num_rounds 2 ScatterAndGather \n", + " config_fed_server.conf persist_every_n_rounds 1 ScatterAndGather \n", + " config_fed_server.conf snapshot_every_n_rounds 1 ScatterAndGather \n", + " config_fed_server.conf start_round 0 ScatterAndGather \n", + " config_fed_server.conf task_check_period 0.5 ScatterAndGather \n", + " config_fed_server.conf train_timeout 0 ScatterAndGather \n", + " config_fed_server.conf validation_metric_name initial_metrics IntimeModelSelector \n", + " config_fed_server.conf wait_time_after_min_received 0 ScatterAndGather \n", + " config_fed_server.conf weigh_by_local_iter False IntimeModelSelector \n", + "\n", + "---------------------------------------------------------------------------------------------------------------------------------------\n" + ] + } + ], "source": [ "app_script=\"client.py\"\n", "\n", @@ -32,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "88e86dad-ba2d-454a-9315-631691673ea4", "metadata": {}, "outputs": [], @@ -42,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 4, "id": "9e23fe35-0a67-4a37-b22e-314e6898cd59", "metadata": {}, "outputs": [ @@ -50,27 +119,27 @@ "name": "stdout", "output_type": "stream", "text": [ - "2024-03-04 13:06:27,435 - SimulatorRunner - INFO - Create the Simulator Server.\n", - "2024-03-04 13:06:27,440 - CoreCell - INFO - server: creating listener on tcp://0:36641\n", - "2024-03-04 13:06:27,455 - CoreCell - INFO - server: created backbone external listener for tcp://0:36641\n", - "2024-03-04 13:06:27,456 - ConnectorManager - INFO - 408946: Try start_listener Listener resources: {'secure': False, 'host': 'localhost'}\n", - "2024-03-04 13:06:27,457 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00002 PASSIVE tcp://0:33172] is starting\n", - "2024-03-04 13:06:27,959 - CoreCell - INFO - server: created backbone internal listener for tcp://localhost:33172\n", - "2024-03-04 13:06:27,962 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 PASSIVE tcp://0:36641] is starting\n", - "2024-03-04 13:06:28,038 - nvflare.fuel.hci.server.hci - INFO - Starting Admin Server localhost on Port 49911\n", - "2024-03-04 13:06:28,039 - SimulatorRunner - INFO - Deploy the Apps.\n", - "2024-03-04 13:06:28,047 - SimulatorRunner - INFO - Create the simulate clients.\n", - "2024-03-04 13:06:28,051 - ClientManager - INFO - Client: New client site-1@192.168.1.203 joined. Sent token: f92eb8d4-7837-4963-8989-164ebb7d709c. Total clients: 1\n", - "2024-03-04 13:06:28,053 - FederatedClient - INFO - Successfully registered client:site-1 for project simulator_server. Token:f92eb8d4-7837-4963-8989-164ebb7d709c SSID:\n", - "2024-03-04 13:06:28,055 - ClientManager - INFO - Client: New client site-2@192.168.1.203 joined. Sent token: 44c0aaae-41e2-43ec-8ec3-7c9369fd3f6a. Total clients: 2\n", - "2024-03-04 13:06:28,056 - FederatedClient - INFO - Successfully registered client:site-2 for project simulator_server. Token:44c0aaae-41e2-43ec-8ec3-7c9369fd3f6a SSID:\n", - "2024-03-04 13:06:28,058 - ClientManager - INFO - Client: New client site-3@192.168.1.203 joined. Sent token: 3774cb8b-94dd-4599-9f4f-6df5012b6844. Total clients: 3\n", - "2024-03-04 13:06:28,059 - FederatedClient - INFO - Successfully registered client:site-3 for project simulator_server. Token:3774cb8b-94dd-4599-9f4f-6df5012b6844 SSID:\n", - "2024-03-04 13:06:28,060 - SimulatorRunner - INFO - Set the client status ready.\n", - "2024-03-04 13:06:28,061 - SimulatorRunner - INFO - Deploy and start the Server App.\n", - "2024-03-04 13:06:28,063 - Cell - INFO - Register blob CB for channel='server_command', topic='*'\n", - "2024-03-04 13:06:28,065 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", - "2024-03-04 13:06:28,066 - ServerCommandAgent - INFO - ServerCommandAgent cell register_request_cb: server.simulate_job\n" + "2024-03-06 15:59:17,528 - SimulatorRunner - INFO - Create the Simulator Server.\n", + "2024-03-06 15:59:17,532 - CoreCell - INFO - server: creating listener on tcp://0:36825\n", + "2024-03-06 15:59:17,550 - CoreCell - INFO - server: created backbone external listener for tcp://0:36825\n", + "2024-03-06 15:59:17,551 - ConnectorManager - INFO - 13991: Try start_listener Listener resources: {'secure': False, 'host': 'localhost'}\n", + "2024-03-06 15:59:17,553 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00002 PASSIVE tcp://0:61614] is starting\n", + "2024-03-06 15:59:18,056 - CoreCell - INFO - server: created backbone internal listener for tcp://localhost:61614\n", + "2024-03-06 15:59:18,057 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 PASSIVE tcp://0:36825] is starting\n", + "2024-03-06 15:59:18,133 - nvflare.fuel.hci.server.hci - INFO - Starting Admin Server localhost on Port 40275\n", + "2024-03-06 15:59:18,134 - SimulatorRunner - INFO - Deploy the Apps.\n", + "2024-03-06 15:59:18,141 - SimulatorRunner - INFO - Create the simulate clients.\n", + "2024-03-06 15:59:18,146 - ClientManager - INFO - Client: New client site-1@192.168.1.203 joined. Sent token: a429000a-ef61-4ea2-b750-05044df590b9. Total clients: 1\n", + "2024-03-06 15:59:18,147 - FederatedClient - INFO - Successfully registered client:site-1 for project simulator_server. Token:a429000a-ef61-4ea2-b750-05044df590b9 SSID:\n", + "2024-03-06 15:59:18,149 - ClientManager - INFO - Client: New client site-2@192.168.1.203 joined. Sent token: 8bb59863-a3bc-4518-88d1-f72ba9197f25. Total clients: 2\n", + "2024-03-06 15:59:18,150 - FederatedClient - INFO - Successfully registered client:site-2 for project simulator_server. Token:8bb59863-a3bc-4518-88d1-f72ba9197f25 SSID:\n", + "2024-03-06 15:59:18,152 - ClientManager - INFO - Client: New client site-3@192.168.1.203 joined. Sent token: e8cbe70f-195f-4d9d-b5a9-e7d4b2c1d736. Total clients: 3\n", + "2024-03-06 15:59:18,153 - FederatedClient - INFO - Successfully registered client:site-3 for project simulator_server. Token:e8cbe70f-195f-4d9d-b5a9-e7d4b2c1d736 SSID:\n", + "2024-03-06 15:59:18,154 - SimulatorRunner - INFO - Set the client status ready.\n", + "2024-03-06 15:59:18,155 - SimulatorRunner - INFO - Deploy and start the Server App.\n", + "2024-03-06 15:59:18,156 - Cell - INFO - Register blob CB for channel='server_command', topic='*'\n", + "2024-03-06 15:59:18,158 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", + "2024-03-06 15:59:18,158 - ServerCommandAgent - INFO - ServerCommandAgent cell register_request_cb: server.simulate_job\n" ] }, { @@ -85,825 +154,825 @@ "name": "stdout", "output_type": "stream", "text": [ - "2024-03-04 13:06:29,556 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job]: Server runner starting ...\n", - "2024-03-04 13:06:29,558 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job]: starting workflow controller_launcher () ...\n", - "2024-03-04 13:06:29,560 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Initializing ModelController workflow.\n", - "2024-03-04 13:06:29,562 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Workflow controller_launcher () started\n", - "2024-03-04 13:06:29,564 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Beginning model controller run.\n", - "2024-03-04 13:06:29,565 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Start Controller Launcher.\n", - "2024-03-04 13:06:30,075 - SimulatorClientRunner - INFO - Start the clients run simulation.\n", - "2024-03-04 13:06:30,337 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:06:30,337 | app.py:163 | Starting Flower server, config: ServerConfig(num_rounds=30, round_timeout=None)\n", - "2024-03-04 13:06:30,344 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:06:30,344 | app.py:176 | Flower ECE: gRPC server running (30 rounds), SSL is disabled\n", - "2024-03-04 13:06:30,345 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:06:30,344 | server.py:89 | Initializing global parameters\n", - "2024-03-04 13:06:30,346 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:06:30,344 | server.py:276 | Requesting initial parameters from one random client\n", - "2024-03-04 13:06:31,079 - SimulatorClientRunner - INFO - Simulate Run client: site-1 on GPU group: None\n", - "2024-03-04 13:06:31,095 - SimulatorClientRunner - INFO - Simulate Run client: site-2 on GPU group: None\n", - "2024-03-04 13:06:31,100 - SimulatorClientRunner - INFO - Simulate Run client: site-3 on GPU group: None\n", - "2024-03-04 13:06:32,182 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00006 127.0.0.1:36641 <= 127.0.0.1:60152] is created: PID: 408946\n", - "2024-03-04 13:06:32,197 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00007 127.0.0.1:36641 <= 127.0.0.1:60156] is created: PID: 408946\n", - "2024-03-04 13:06:32,199 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00008 127.0.0.1:36641 <= 127.0.0.1:60166] is created: PID: 408946\n", - "2024-03-04 13:06:32,116 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", - "2024-03-04 13:06:32,127 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", - "2024-03-04 13:06:32,138 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", - "2024-03-04 13:06:32,179 - CoreCell - INFO - site-1.simulate_job: created backbone external connector to tcp://localhost:36641\n", - "2024-03-04 13:06:32,180 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:36641] is starting\n", - "2024-03-04 13:06:32,181 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:60152 => 127.0.0.1:36641] is created: PID: 409112\n", - "2024-03-04 13:06:32,196 - CoreCell - INFO - site-2.simulate_job: created backbone external connector to tcp://localhost:36641\n", - "2024-03-04 13:06:32,196 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:36641] is starting\n", - "2024-03-04 13:06:32,197 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:60156 => 127.0.0.1:36641] is created: PID: 409117\n", - "2024-03-04 13:06:32,198 - CoreCell - INFO - site-3.simulate_job: created backbone external connector to tcp://localhost:36641\n", - "2024-03-04 13:06:32,198 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:36641] is starting\n", - "2024-03-04 13:06:32,199 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:60166 => 127.0.0.1:36641] is created: PID: 409119\n", - "2024-03-04 13:06:33,710 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", - "2024-03-04 13:06:33,716 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", - "2024-03-04 13:06:33,721 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", - "2024-03-04 13:06:34,216 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", - "2024-03-04 13:06:34,222 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", - "2024-03-04 13:06:34,226 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", - "2024-03-04 13:06:34,228 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: synced to Server Runner in 0.5128936767578125 seconds\n", - "2024-03-04 13:06:34,228 - ExecutorLauncher - INFO - [identity=site-1, run=simulate_job]: Start Executor Launcher.\n", - "2024-03-04 13:06:34,232 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: synced to Server Runner in 0.510692834854126 seconds\n", - "2024-03-04 13:06:34,232 - ExecutorLauncher - INFO - [identity=site-2, run=simulate_job]: Start Executor Launcher.\n", - "2024-03-04 13:06:34,233 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: synced to Server Runner in 0.5081589221954346 seconds\n", - "2024-03-04 13:06:34,233 - ExecutorLauncher - INFO - [identity=site-3, run=simulate_job]: Start Executor Launcher.\n", - "Running ... [python3 custom/server.py ]\n", - "2024-03-04 13:06:42,779 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:06:42,779 | server.py:280 | Received initial parameters from one random client\n", - "2024-03-04 13:06:42,781 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:06:42,779 | server.py:91 | Evaluating initial parameters\n", - "2024-03-04 13:06:42,783 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:06:42,779 | server.py:104 | FL starting\n", - "2024-03-04 13:06:42,770 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:06:42,770 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", - "2024-03-04 13:06:42,772 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:42,772 | connection.py:55 | ChannelConnectivity.IDLE\n", - "2024-03-04 13:06:42,773 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:42,773 | connection.py:55 | ChannelConnectivity.CONNECTING\n", - "2024-03-04 13:06:42,775 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:42,775 | connection.py:55 | ChannelConnectivity.READY\n", - "2024-03-04 13:06:43,056 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:43,056 | server.py:222 | fit_round 1: strategy sampled 2 clients (out of 2)\n", - "2024-03-04 13:06:43,052 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:06:43,052 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", - "2024-03-04 13:06:43,054 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:43,054 | connection.py:55 | ChannelConnectivity.IDLE\n", - "2024-03-04 13:06:43,055 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:43,055 | connection.py:55 | ChannelConnectivity.CONNECTING\n", - "2024-03-04 13:06:43,055 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:43,055 | connection.py:55 | ChannelConnectivity.READY\n", - "2024-03-04 13:06:43,065 - SubprocessLauncher - INFO - START FLOWER CLIENT [node_id=2]\n", - "2024-03-04 13:06:43,068 - SubprocessLauncher - INFO - START FLOWER CLIENT [node_id=0]\n", - "2024-03-04 13:06:43,492 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:06:43,492 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", - "2024-03-04 13:06:43,495 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:43,494 | connection.py:55 | ChannelConnectivity.IDLE\n", - "2024-03-04 13:06:43,495 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:43,495 | connection.py:55 | ChannelConnectivity.CONNECTING\n", - "2024-03-04 13:06:43,495 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:43,495 | connection.py:55 | ChannelConnectivity.READY\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.67it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.16it/s]]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.50it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.80it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.45it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.33it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 129.58it/s]s]\n", - "2024-03-04 13:06:57,033 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:57,033 | server.py:236 | fit_round 1 received 2 results and 0 failures\n", - "2024-03-04 13:06:57,038 - SubprocessLauncher - INFO - WARNING flwr 2024-03-04 13:06:57,038 | fedavg.py:250 | No fit_metrics_aggregation_fn provided\n", - "2024-03-04 13:06:57,039 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:57,038 | server.py:173 | evaluate_round 1: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.85it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Running ... [python3 custom/client.py ]\n", - "2024-03-04 13:06:57,062 - SubprocessLauncher - INFO - START FLOWER CLIENT [node_id=1]\n", - "2024-03-04 13:06:58,012 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:58,012 | server.py:187 | evaluate_round 1 received 3 results and 0 failures\n", - "2024-03-04 13:06:58,015 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:06:58,012 | server.py:222 | fit_round 2: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 131.93it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 129.22it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 110.73it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.21it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.19it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.76it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.05it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.28it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.45it/s]s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.49it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.54it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.63it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.03it/s]\n", - "2024-03-04 13:07:11,973 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:11,973 | server.py:236 | fit_round 2 received 3 results and 0 failures\n", - "2024-03-04 13:07:11,975 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:11,975 | server.py:173 | evaluate_round 2: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.87it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.48it/s]\n", - "2024-03-04 13:07:12,823 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:12,823 | server.py:187 | evaluate_round 2 received 3 results and 0 failures\n", - "2024-03-04 13:07:12,824 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:12,823 | server.py:222 | fit_round 3: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.41it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 128.29it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 124.66it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.85it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.79it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.71it/s]s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.98it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.10it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.19it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 112.11it/s]s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 110.14it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 108.58it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 111.10it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 110.27it/s]\n", - "2024-03-04 13:07:27,431 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:27,431 | server.py:236 | fit_round 3 received 3 results and 0 failures\n", - "2024-03-04 13:07:27,434 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:27,433 | server.py:173 | evaluate_round 3: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 110.41it/s]\n", - "2024-03-04 13:07:28,313 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:28,312 | server.py:187 | evaluate_round 3 received 3 results and 0 failures\n", - "2024-03-04 13:07:28,314 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:28,313 | server.py:222 | fit_round 4: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 123.05it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 122.27it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 120.99it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 112.44it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 109.56it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 109.01it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.27it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.59it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.68it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.37it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.13it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.21it/s]]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 108.93it/s]]\n", - "2024-03-04 13:07:43,240 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:43,240 | server.py:236 | fit_round 4 received 3 results and 0 failures\n", - "2024-03-04 13:07:43,243 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:43,243 | server.py:173 | evaluate_round 4: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 109.92it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 107.06it/s]]\n", - "2024-03-04 13:07:44,124 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:44,124 | server.py:187 | evaluate_round 4 received 3 results and 0 failures\n", - "2024-03-04 13:07:44,126 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:44,124 | server.py:222 | fit_round 5: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 124.44it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 124.26it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 120.78it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 111.95it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 109.09it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 108.85it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.43it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 113.39it/s]s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 112.92it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.97it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.47it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.68it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 108.80it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 108.51it/s]s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 107.10it/s]\n", - "2024-03-04 13:07:59,163 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:59,163 | server.py:236 | fit_round 5 received 3 results and 0 failures\n", - "2024-03-04 13:07:59,166 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:07:59,166 | server.py:173 | evaluate_round 5: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/server.py ]\n", - "2024-03-04 13:08:00,021 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:00,021 | server.py:187 | evaluate_round 5 received 3 results and 0 failures\n", - "2024-03-04 13:08:00,023 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:00,021 | server.py:222 | fit_round 6: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 128.70it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 124.95it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 123.53it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.32it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.59it/s]s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.61it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.61it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.43it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.87it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.45it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.70it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.06it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.94it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.90it/s]\n", - "2024-03-04 13:08:14,251 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:14,251 | server.py:236 | fit_round 6 received 3 results and 0 failures\n", - "2024-03-04 13:08:14,254 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:14,253 | server.py:173 | evaluate_round 6: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.94it/s]\n", - "2024-03-04 13:08:15,070 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:15,070 | server.py:187 | evaluate_round 6 received 3 results and 0 failures\n", - "2024-03-04 13:08:15,072 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:15,070 | server.py:222 | fit_round 7: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.71it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 130.43it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 130.85it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.25it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.89it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.79it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.39it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.52it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.64it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.08it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.78it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.98it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.09it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 112.76it/s]\n", - "2024-03-04 13:08:29,561 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:29,561 | server.py:236 | fit_round 7 received 3 results and 0 failures\n", - "2024-03-04 13:08:29,563 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:29,563 | server.py:173 | evaluate_round 7: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 112.71it/s]s]\n", - "2024-03-04 13:08:30,388 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:30,388 | server.py:187 | evaluate_round 7 received 3 results and 0 failures\n", - "2024-03-04 13:08:30,389 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:30,388 | server.py:222 | fit_round 8: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 129.31it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 129.36it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 128.55it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.65it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.87it/s]s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.40it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.21it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.34it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.23it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.76it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.72it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.69it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.86it/s]\n", - "2024-03-04 13:08:44,474 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:44,474 | server.py:236 | fit_round 8 received 3 results and 0 failures\n", - "2024-03-04 13:08:44,480 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:44,480 | server.py:173 | evaluate_round 8: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.89it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.78it/s]\n", - "2024-03-04 13:08:45,267 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:45,266 | server.py:187 | evaluate_round 8 received 3 results and 0 failures\n", - "2024-03-04 13:08:45,269 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:45,267 | server.py:222 | fit_round 9: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.22it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.95it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.61it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.61it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.85it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.13it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.66it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.21it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.80it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.66it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.96it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.15it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.62it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.11it/s]\n", - "2024-03-04 13:08:59,075 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:59,075 | server.py:236 | fit_round 9 received 3 results and 0 failures\n", - "2024-03-04 13:08:59,077 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:59,077 | server.py:173 | evaluate_round 9: strategy sampled 3 clients (out of 3)\n", + "2024-03-06 15:59:19,577 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job]: Server runner starting ...\n", + "2024-03-06 15:59:19,579 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job]: starting workflow controller_launcher () ...\n", + "2024-03-06 15:59:19,581 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Initializing ModelController workflow.\n", + "2024-03-06 15:59:19,583 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Workflow controller_launcher () started\n", + "2024-03-06 15:59:19,585 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Beginning model controller run.\n", + "2024-03-06 15:59:19,586 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Start Controller Launcher.\n", + "2024-03-06 15:59:20,158 - SimulatorClientRunner - INFO - Start the clients run simulation.\n", + "2024-03-06 15:59:20,334 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:20,333 | app.py:163 | Starting Flower server, config: ServerConfig(num_rounds=30, round_timeout=None)\n", + "2024-03-06 15:59:20,340 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:20,339 | app.py:176 | Flower ECE: gRPC server running (30 rounds), SSL is disabled\n", + "2024-03-06 15:59:20,341 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:20,339 | server.py:89 | Initializing global parameters\n", + "2024-03-06 15:59:20,342 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:20,339 | server.py:276 | Requesting initial parameters from one random client\n", + "2024-03-06 15:59:21,163 - SimulatorClientRunner - INFO - Simulate Run client: site-1 on GPU group: None\n", + "2024-03-06 15:59:21,166 - SimulatorClientRunner - INFO - Simulate Run client: site-2 on GPU group: None\n", + "2024-03-06 15:59:21,183 - SimulatorClientRunner - INFO - Simulate Run client: site-3 on GPU group: None\n", + "2024-03-06 15:59:22,267 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00006 127.0.0.1:36825 <= 127.0.0.1:54948] is created: PID: 13991\n", + "2024-03-06 15:59:22,276 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00007 127.0.0.1:36825 <= 127.0.0.1:54964] is created: PID: 13991\n", + "2024-03-06 15:59:22,286 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00008 127.0.0.1:36825 <= 127.0.0.1:54976] is created: PID: 13991\n", + "2024-03-06 15:59:22,201 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", + "2024-03-06 15:59:22,211 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", + "2024-03-06 15:59:22,221 - ClientTaskWorker - INFO - ClientTaskWorker started to run\n", + "2024-03-06 15:59:22,265 - CoreCell - INFO - site-1.simulate_job: created backbone external connector to tcp://localhost:36825\n", + "2024-03-06 15:59:22,265 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:36825] is starting\n", + "2024-03-06 15:59:22,266 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:54948 => 127.0.0.1:36825] is created: PID: 14189\n", + "2024-03-06 15:59:22,274 - CoreCell - INFO - site-2.simulate_job: created backbone external connector to tcp://localhost:36825\n", + "2024-03-06 15:59:22,274 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:36825] is starting\n", + "2024-03-06 15:59:22,276 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:54964 => 127.0.0.1:36825] is created: PID: 14192\n", + "2024-03-06 15:59:22,285 - CoreCell - INFO - site-3.simulate_job: created backbone external connector to tcp://localhost:36825\n", + "2024-03-06 15:59:22,285 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connector [CH00001 ACTIVE tcp://localhost:36825] is starting\n", + "2024-03-06 15:59:22,286 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 127.0.0.1:54976 => 127.0.0.1:36825] is created: PID: 14194\n", + "2024-03-06 15:59:23,816 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", + "2024-03-06 15:59:23,816 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", + "2024-03-06 15:59:23,829 - Cell - INFO - Register blob CB for channel='aux_communication', topic='*'\n", + "2024-03-06 15:59:24,321 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", + "2024-03-06 15:59:24,322 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", + "2024-03-06 15:59:24,333 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: synced to Server Runner in 0.513308048248291 seconds\n", + "2024-03-06 15:59:24,333 - ExecutorLauncher - INFO - [identity=site-2, run=simulate_job]: Start Executor Launcher.\n", + "2024-03-06 15:59:24,335 - Cell - INFO - broadcast: channel='aux_communication', topic='__sync_runner__', targets=['server.simulate_job'], timeout=2.0\n", + "2024-03-06 15:59:24,336 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: synced to Server Runner in 0.5149381160736084 seconds\n", + "2024-03-06 15:59:24,336 - ExecutorLauncher - INFO - [identity=site-3, run=simulate_job]: Start Executor Launcher.\n", + "2024-03-06 15:59:24,342 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: synced to Server Runner in 0.5080032348632812 seconds\n", + "2024-03-06 15:59:24,342 - ExecutorLauncher - INFO - [identity=site-1, run=simulate_job]: Start Executor Launcher.\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-06 15:59:29,661 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:29,660 | server.py:280 | Received initial parameters from one random client\n", + "2024-03-06 15:59:29,662 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:29,660 | server.py:91 | Evaluating initial parameters\n", + "2024-03-06 15:59:29,663 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:29,661 | server.py:104 | FL starting\n", + "2024-03-06 15:59:29,652 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:29,652 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", + "2024-03-06 15:59:29,654 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,653 | connection.py:55 | ChannelConnectivity.IDLE\n", + "2024-03-06 15:59:29,655 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,655 | connection.py:55 | ChannelConnectivity.CONNECTING\n", + "2024-03-06 15:59:29,656 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,655 | connection.py:55 | ChannelConnectivity.READY\n", + "2024-03-06 15:59:29,967 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,967 | server.py:222 | fit_round 1: strategy sampled 2 clients (out of 2)\n", + "2024-03-06 15:59:29,964 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:29,964 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", + "2024-03-06 15:59:29,965 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,965 | connection.py:55 | ChannelConnectivity.IDLE\n", + "2024-03-06 15:59:29,966 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:29,966 | connection.py:55 | ChannelConnectivity.READY\n", + "2024-03-06 15:59:30,004 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 15:59:30,004 | grpc.py:52 | Opened insecure gRPC connection (no certificates were passed)\n", + "2024-03-06 15:59:30,005 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:30,005 | connection.py:55 | ChannelConnectivity.IDLE\n", + "2024-03-06 15:59:30,006 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:30,006 | connection.py:55 | ChannelConnectivity.CONNECTING\n", + "2024-03-06 15:59:30,006 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:30,006 | connection.py:55 | ChannelConnectivity.READY\n", + "Training: 100%|██████████| 417/417 [00:04<00:00, 96.12it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:04<00:00, 94.81it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.13it/s]]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.96it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.35it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.52it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.40it/s]\n", + "2024-03-06 15:59:44,530 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:44,530 | server.py:236 | fit_round 1 received 2 results and 0 failures\n", + "2024-03-06 15:59:44,533 - SubprocessLauncher - INFO - WARNING flwr 2024-03-06 15:59:44,533 | fedavg.py:250 | No fit_metrics_aggregation_fn provided\n", + "2024-03-06 15:59:44,534 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:44,533 | server.py:173 | evaluate_round 1: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.50it/s]\n", + "2024-03-06 15:59:45,478 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:45,478 | server.py:187 | evaluate_round 1 received 3 results and 0 failures\n", + "2024-03-06 15:59:45,481 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:45,478 | server.py:222 | fit_round 2: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.55it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.50it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 113.72it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.66it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.19it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.30it/s]s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.37it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.13it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.43it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.65it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.25it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", "Training: 100%|██████████| 417/417 [00:03<00:00, 121.31it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "2024-03-04 13:08:59,862 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:59,862 | server.py:187 | evaluate_round 9 received 3 results and 0 failures\n", - "2024-03-04 13:08:59,865 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:08:59,862 | server.py:222 | fit_round 10: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.66it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.66it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.23it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.58it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.53it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.21it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.93it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.21it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.67it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.43it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.24it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.21it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.93it/s]\n", - "2024-03-04 13:09:14,170 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:14,170 | server.py:236 | fit_round 10 received 3 results and 0 failures\n", - "2024-03-04 13:09:14,174 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:14,174 | server.py:173 | evaluate_round 10: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 113.46it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.94it/s]\n", - "2024-03-04 13:09:14,989 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:14,989 | server.py:187 | evaluate_round 10 received 3 results and 0 failures\n", - "2024-03-04 13:09:14,991 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:14,989 | server.py:222 | fit_round 11: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 131.89it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.23it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 131.19it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.53it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.32it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.12it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.48it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.28it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.27it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.51it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.28it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.60it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.86it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.16it/s]\n", - "2024-03-04 13:09:28,725 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:28,725 | server.py:236 | fit_round 11 received 3 results and 0 failures\n", - "2024-03-04 13:09:28,729 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:28,729 | server.py:173 | evaluate_round 11: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.11it/s]\n", - "2024-03-04 13:09:29,516 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:29,515 | server.py:187 | evaluate_round 11 received 3 results and 0 failures\n", - "2024-03-04 13:09:29,517 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:29,516 | server.py:222 | fit_round 12: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.88it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.32it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.32it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.04it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.11it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.83it/s]\n", - "Running ... [python3 custom/client.py ]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.76it/s]\n", + "2024-03-06 15:59:59,095 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:59,095 | server.py:236 | fit_round 2 received 3 results and 0 failures\n", + "2024-03-06 15:59:59,098 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:59,098 | server.py:173 | evaluate_round 2: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.72it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.11it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-06 15:59:59,876 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:59,876 | server.py:187 | evaluate_round 2 received 3 results and 0 failures\n", + "2024-03-06 15:59:59,878 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 15:59:59,876 | server.py:222 | fit_round 3: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.96it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.97it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.91it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.99it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.24it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.70it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.13it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", "Training: 100%|██████████| 417/417 [00:03<00:00, 124.00it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.30it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.10it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.35it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.03it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.17it/s]\n", - "2024-03-04 13:09:43,550 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:43,550 | server.py:236 | fit_round 12 received 3 results and 0 failures\n", - "2024-03-04 13:09:43,554 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:43,554 | server.py:173 | evaluate_round 12: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.17it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.96it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.60it/s]\n", - "2024-03-04 13:09:44,361 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:44,361 | server.py:187 | evaluate_round 12 received 3 results and 0 failures\n", - "2024-03-04 13:09:44,363 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:44,361 | server.py:222 | fit_round 13: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.98it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.56it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.76it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.75it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.12it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.25it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.18it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.80it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.07it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.79it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.93it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.63it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.05it/s]\n", - "2024-03-04 13:09:58,318 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:58,318 | server.py:236 | fit_round 13 received 3 results and 0 failures\n", - "2024-03-04 13:09:58,323 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:58,323 | server.py:173 | evaluate_round 13: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.48it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.39it/s]\n", - "2024-03-04 13:09:59,118 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:59,118 | server.py:187 | evaluate_round 13 received 3 results and 0 failures\n", - "2024-03-04 13:09:59,120 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:09:59,118 | server.py:222 | fit_round 14: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.98it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.99it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.86it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.04it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.58it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.25it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.05it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.35it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.11it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.34it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.16it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.91it/s]\n", - "2024-03-04 13:10:13,170 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:13,170 | server.py:236 | fit_round 14 received 3 results and 0 failures\n", - "2024-03-04 13:10:13,173 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:13,173 | server.py:173 | evaluate_round 14: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.22it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.24it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.31it/s]\n", - "2024-03-04 13:10:13,978 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:13,978 | server.py:187 | evaluate_round 14 received 3 results and 0 failures\n", - "2024-03-04 13:10:13,979 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:13,978 | server.py:222 | fit_round 15: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.71it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.93it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.72it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.32it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.10it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.45it/s]]\n", - "Running ... [python3 custom/server.py ]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.79it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.56it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.74it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.66it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.33it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.31it/s]\n", + "2024-03-06 16:00:13,474 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:13,474 | server.py:236 | fit_round 3 received 3 results and 0 failures\n", + "2024-03-06 16:00:13,476 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:13,476 | server.py:173 | evaluate_round 3: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.59it/s]\n", + "2024-03-06 16:00:14,233 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:14,233 | server.py:187 | evaluate_round 3 received 3 results and 0 failures\n", + "2024-03-06 16:00:14,235 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:14,233 | server.py:222 | fit_round 4: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 143.89it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.50it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.71it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.57it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.92it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", "Training: 100%|██████████| 417/417 [00:03<00:00, 124.89it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.52it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.76it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.33it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.01it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.82it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.59it/s]s]\n", - "2024-03-04 13:10:27,943 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:27,943 | server.py:236 | fit_round 15 received 3 results and 0 failures\n", - "2024-03-04 13:10:27,945 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:27,945 | server.py:173 | evaluate_round 15: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.96it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.42it/s]\n", - "2024-03-04 13:10:28,720 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:28,719 | server.py:187 | evaluate_round 15 received 3 results and 0 failures\n", - "2024-03-04 13:10:28,721 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:28,720 | server.py:222 | fit_round 16: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.67it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.72it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.16it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.59it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.12it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.23it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.29it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.13it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.21it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.10it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.39it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 112.30it/s]s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.82it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.91it/s]\n", - "2024-03-04 13:10:43,089 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:43,089 | server.py:236 | fit_round 16 received 3 results and 0 failures\n", - "2024-03-04 13:10:43,094 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:43,094 | server.py:173 | evaluate_round 16: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.50it/s]s]\n", - "2024-03-04 13:10:43,901 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:43,901 | server.py:187 | evaluate_round 16 received 3 results and 0 failures\n", - "2024-03-04 13:10:43,903 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:43,901 | server.py:222 | fit_round 17: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.76it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.48it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 131.59it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.23it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.22it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.23it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.82it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.52it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.20it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.87it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.61it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 113.68it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.59it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.85it/s]s]\n", - "2024-03-04 13:10:58,184 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:58,184 | server.py:236 | fit_round 17 received 3 results and 0 failures\n", - "2024-03-04 13:10:58,188 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:58,188 | server.py:173 | evaluate_round 17: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.75it/s]\n", - "2024-03-04 13:10:59,017 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:59,016 | server.py:187 | evaluate_round 17 received 3 results and 0 failures\n", - "2024-03-04 13:10:59,018 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:10:59,017 | server.py:222 | fit_round 18: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.80it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.13it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 128.35it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.08it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.12it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.39it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.15it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.69it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.60it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.00it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.67it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.74it/s]s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.28it/s]\n", - "2024-03-04 13:11:13,483 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:13,483 | server.py:236 | fit_round 18 received 3 results and 0 failures\n", - "2024-03-04 13:11:13,488 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:13,488 | server.py:173 | evaluate_round 18: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.19it/s]s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 112.34it/s]]\n", - "2024-03-04 13:11:14,271 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:14,271 | server.py:187 | evaluate_round 18 received 3 results and 0 failures\n", - "2024-03-04 13:11:14,273 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:14,271 | server.py:222 | fit_round 19: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.47it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.63it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.47it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.50it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.97it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.34it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.65it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.22it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.03it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.59it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.38it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.78it/s]\n", - "2024-03-04 13:11:28,329 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:28,329 | server.py:236 | fit_round 19 received 3 results and 0 failures\n", - "2024-03-04 13:11:28,334 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:28,334 | server.py:173 | evaluate_round 19: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.13it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.72it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.81it/s]\n", - "2024-03-04 13:11:29,119 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:29,119 | server.py:187 | evaluate_round 19 received 3 results and 0 failures\n", - "2024-03-04 13:11:29,121 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:29,119 | server.py:222 | fit_round 20: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.81it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.23it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.97it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.90it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.95it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.38it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.30it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.97it/s]\n", + "2024-03-06 16:00:27,478 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:27,478 | server.py:236 | fit_round 4 received 3 results and 0 failures\n", + "2024-03-06 16:00:27,480 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:27,480 | server.py:173 | evaluate_round 4: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.91it/s]]\n", + "2024-03-06 16:00:28,252 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:28,252 | server.py:187 | evaluate_round 4 received 3 results and 0 failures\n", + "2024-03-06 16:00:28,254 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:28,252 | server.py:222 | fit_round 5: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.01it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.24it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.43it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.91it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.62it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.38it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.73it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.10it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.00it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.99it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.01it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.70it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.12it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.70it/s]\n", + "2024-03-06 16:00:41,818 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:41,818 | server.py:236 | fit_round 5 received 3 results and 0 failures\n", + "2024-03-06 16:00:41,820 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:41,820 | server.py:173 | evaluate_round 5: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.96it/s]\n", + "2024-03-06 16:00:42,596 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:42,596 | server.py:187 | evaluate_round 5 received 3 results and 0 failures\n", + "2024-03-06 16:00:42,599 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:42,596 | server.py:222 | fit_round 6: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.87it/s]\n", "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.61it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.01it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.70it/s]]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.18it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.73it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.01it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.85it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.28it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.20it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.21it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.61it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.35it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "2024-03-04 13:11:43,078 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:43,078 | server.py:236 | fit_round 20 received 3 results and 0 failures\n", - "2024-03-04 13:11:43,081 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:43,080 | server.py:173 | evaluate_round 20: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.01it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.80it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.72it/s]\n", - "2024-03-04 13:11:43,929 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:43,928 | server.py:187 | evaluate_round 20 received 3 results and 0 failures\n", - "2024-03-04 13:11:43,931 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:43,929 | server.py:222 | fit_round 21: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 128.60it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 125.51it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 125.98it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.82it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.53it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.88it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.80it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.02it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.66it/s]s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.26it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.59it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.48it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.13it/s]\n", - "2024-03-04 13:11:58,058 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:58,057 | server.py:236 | fit_round 21 received 3 results and 0 failures\n", - "2024-03-04 13:11:58,064 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:58,063 | server.py:173 | evaluate_round 21: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.45it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.87it/s]\n", - "2024-03-04 13:11:58,874 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:58,874 | server.py:187 | evaluate_round 21 received 3 results and 0 failures\n", - "2024-03-04 13:11:58,876 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:11:58,874 | server.py:222 | fit_round 22: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.25it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 131.75it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.22it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.03it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.14it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.00it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.05it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.49it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.08it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.59it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.02it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.85it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.62it/s]\n", - "2024-03-04 13:12:13,094 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:13,093 | server.py:236 | fit_round 22 received 3 results and 0 failures\n", - "2024-03-04 13:12:13,096 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:13,096 | server.py:173 | evaluate_round 22: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.66it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.31it/s]\n", - "2024-03-04 13:12:13,894 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:13,894 | server.py:187 | evaluate_round 22 received 3 results and 0 failures\n", - "2024-03-04 13:12:13,896 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:13,894 | server.py:222 | fit_round 23: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.71it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.06it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.05it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.31it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.76it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.47it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.99it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.00it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.45it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.69it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.85it/s]]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.11it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.98it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.72it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.57it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.19it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.93it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.90it/s]\n", + "2024-03-06 16:00:56,098 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:56,097 | server.py:236 | fit_round 6 received 3 results and 0 failures\n", + "2024-03-06 16:00:56,100 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:56,100 | server.py:173 | evaluate_round 6: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.58it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.29it/s]\n", + "2024-03-06 16:00:56,888 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:56,888 | server.py:187 | evaluate_round 6 received 3 results and 0 failures\n", + "2024-03-06 16:00:56,890 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:00:56,888 | server.py:222 | fit_round 7: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.09it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.65it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.02it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.61it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.78it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.23it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.79it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.10it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.08it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.41it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.43it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.76it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.56it/s]\n", + "2024-03-06 16:01:10,353 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:10,353 | server.py:236 | fit_round 7 received 3 results and 0 failures\n", + "2024-03-06 16:01:10,355 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:10,355 | server.py:173 | evaluate_round 7: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.12it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.07it/s]\n", + "2024-03-06 16:01:11,142 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:11,142 | server.py:187 | evaluate_round 7 received 3 results and 0 failures\n", + "2024-03-06 16:01:11,144 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:11,142 | server.py:222 | fit_round 8: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.13it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.68it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.09it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.56it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.34it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.12it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.46it/s]]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.50it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.50it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.14it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.70it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.75it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.31it/s]\n", + "2024-03-06 16:01:24,618 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:24,618 | server.py:236 | fit_round 8 received 3 results and 0 failures\n", + "2024-03-06 16:01:24,624 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:24,624 | server.py:173 | evaluate_round 8: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.18it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.15it/s]\n", + "2024-03-06 16:01:25,394 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:25,394 | server.py:187 | evaluate_round 8 received 3 results and 0 failures\n", + "2024-03-06 16:01:25,396 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:25,394 | server.py:222 | fit_round 9: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 141.40it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.26it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.32it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.11it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.93it/s]\n", "Training: 100%|██████████| 417/417 [00:03<00:00, 121.26it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.93it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.85it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.07it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.13it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.94it/s]\n", - "2024-03-04 13:12:27,958 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:27,958 | server.py:236 | fit_round 23 received 3 results and 0 failures\n", - "2024-03-04 13:12:27,960 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:27,960 | server.py:173 | evaluate_round 23: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.45it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.42it/s]\n", - "2024-03-04 13:12:28,738 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:28,737 | server.py:187 | evaluate_round 23 received 3 results and 0 failures\n", - "2024-03-04 13:12:28,739 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:28,738 | server.py:222 | fit_round 24: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.66it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.38it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.19it/s]\n", - "Running ... [python3 custom/server.py ]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.98it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.09it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.96it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.29it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.29it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.72it/s]]\n", + "2024-03-06 16:01:39,110 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:39,110 | server.py:236 | fit_round 9 received 3 results and 0 failures\n", + "2024-03-06 16:01:39,116 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:39,116 | server.py:173 | evaluate_round 9: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.76it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.05it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.83it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-06 16:01:39,887 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:39,886 | server.py:187 | evaluate_round 9 received 3 results and 0 failures\n", + "2024-03-06 16:01:39,889 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:39,887 | server.py:222 | fit_round 10: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.56it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.73it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.02it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.73it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.17it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.22it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.60it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.43it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.62it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.76it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.72it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.48it/s]\n", + "2024-03-06 16:01:53,516 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:53,516 | server.py:236 | fit_round 10 received 3 results and 0 failures\n", + "2024-03-06 16:01:53,522 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:53,522 | server.py:173 | evaluate_round 10: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.55it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.46it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.98it/s]\n", + "2024-03-06 16:01:54,295 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:54,295 | server.py:187 | evaluate_round 10 received 3 results and 0 failures\n", + "2024-03-06 16:01:54,297 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:01:54,295 | server.py:222 | fit_round 11: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.78it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.71it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.44it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.89it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.79it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.08it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.22it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.08it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.57it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.95it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.22it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.70it/s]\n", + "2024-03-06 16:02:07,696 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:07,695 | server.py:236 | fit_round 11 received 3 results and 0 failures\n", + "Running ... [python3 custom/client.py --node-id 0]\n", "Training: 100%|██████████| 417/417 [00:03<00:00, 122.16it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.06it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.79it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.77it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.45it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.82it/s]]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 127.50it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.93it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.92it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.88it/s]\n", + "2024-03-06 16:02:07,698 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:07,698 | server.py:173 | evaluate_round 11: strategy sampled 3 clients (out of 3)\n", + "2024-03-06 16:02:08,468 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:08,467 | server.py:187 | evaluate_round 11 received 3 results and 0 failures\n", + "2024-03-06 16:02:08,470 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:08,468 | server.py:222 | fit_round 12: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.42it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.70it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.45it/s]]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.14it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.67it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.44it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.35it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.82it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.44it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.19it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.66it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.81it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.65it/s]\n", + "2024-03-06 16:02:21,975 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:21,975 | server.py:236 | fit_round 12 received 3 results and 0 failures\n", + "2024-03-06 16:02:21,977 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:21,977 | server.py:173 | evaluate_round 12: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.78it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.80it/s]\n", + "2024-03-06 16:02:22,750 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:22,750 | server.py:187 | evaluate_round 12 received 3 results and 0 failures\n", + "2024-03-06 16:02:22,752 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:22,750 | server.py:222 | fit_round 13: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.16it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.94it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.34it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.41it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.51it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.23it/s]]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.22it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.99it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.65it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.71it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.44it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.49it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.10it/s]\n", + "2024-03-06 16:02:36,173 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:36,173 | server.py:236 | fit_round 13 received 3 results and 0 failures\n", + "2024-03-06 16:02:36,177 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:36,177 | server.py:173 | evaluate_round 13: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.54it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.21it/s]]\n", + "2024-03-06 16:02:36,949 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:36,948 | server.py:187 | evaluate_round 13 received 3 results and 0 failures\n", + "2024-03-06 16:02:36,951 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:36,949 | server.py:222 | fit_round 14: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 141.12it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.16it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.79it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.00it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.56it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.16it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.85it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.71it/s]\n", "Training: 100%|██████████| 417/417 [00:03<00:00, 123.99it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.66it/s]\n", - "2024-03-04 13:12:42,502 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:42,502 | server.py:236 | fit_round 24 received 3 results and 0 failures\n", - "2024-03-04 13:12:42,508 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:42,508 | server.py:173 | evaluate_round 24: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.98it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.77it/s]\n", - "2024-03-04 13:12:43,313 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:43,312 | server.py:187 | evaluate_round 24 received 3 results and 0 failures\n", - "2024-03-04 13:12:43,314 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:43,313 | server.py:222 | fit_round 25: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.30it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.80it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.13it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.35it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.73it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 113.36it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.98it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.57it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 113.82it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.43it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.91it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.62it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.30it/s]]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.29it/s]\n", - "2024-03-04 13:12:57,636 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:57,635 | server.py:236 | fit_round 25 received 3 results and 0 failures\n", - "2024-03-04 13:12:57,640 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:57,640 | server.py:173 | evaluate_round 25: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.11it/s]\n", - "2024-03-04 13:12:58,416 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:58,416 | server.py:187 | evaluate_round 25 received 3 results and 0 failures\n", - "2024-03-04 13:12:58,418 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:12:58,416 | server.py:222 | fit_round 26: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.61it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.42it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.87it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.32it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.07it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.36it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.39it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 113.85it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 114.01it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.34it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.39it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.94it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.52it/s]\n", + "2024-03-06 16:02:50,424 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:50,424 | server.py:236 | fit_round 14 received 3 results and 0 failures\n", + "2024-03-06 16:02:50,430 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:50,430 | server.py:173 | evaluate_round 14: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.58it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.89it/s]\n", + "2024-03-06 16:02:51,193 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:51,193 | server.py:187 | evaluate_round 14 received 3 results and 0 failures\n", + "2024-03-06 16:02:51,195 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:02:51,193 | server.py:222 | fit_round 15: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.28it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 141.90it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.95it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.22it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.25it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.59it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.58it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.87it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.58it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.60it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.35it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.02it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.32it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.65it/s]s]\n", + "2024-03-06 16:03:04,773 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:04,772 | server.py:236 | fit_round 15 received 3 results and 0 failures\n", + "2024-03-06 16:03:04,778 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:04,778 | server.py:173 | evaluate_round 15: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.93it/s]s]\n", + "2024-03-06 16:03:05,564 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:05,564 | server.py:187 | evaluate_round 15 received 3 results and 0 failures\n", + "2024-03-06 16:03:05,567 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:05,564 | server.py:222 | fit_round 16: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.38it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.05it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.02it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.03it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.43it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.15it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.83it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.53it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.26it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.70it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.93it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.72it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.60it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.72it/s]\n", + "2024-03-06 16:03:19,126 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:19,126 | server.py:236 | fit_round 16 received 3 results and 0 failures\n", + "2024-03-06 16:03:19,128 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:19,128 | server.py:173 | evaluate_round 16: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.12it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-06 16:03:19,909 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:19,909 | server.py:187 | evaluate_round 16 received 3 results and 0 failures\n", + "2024-03-06 16:03:19,911 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:19,909 | server.py:222 | fit_round 17: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.80it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.01it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.09it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.07it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.73it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.85it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.19it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.99it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.41it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.58it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.76it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.54it/s]\n", + "2024-03-06 16:03:33,354 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:33,354 | server.py:236 | fit_round 17 received 3 results and 0 failures\n", + "2024-03-06 16:03:33,356 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:33,356 | server.py:173 | evaluate_round 17: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.10it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.08it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.96it/s]\n", + "2024-03-06 16:03:34,127 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:34,127 | server.py:187 | evaluate_round 17 received 3 results and 0 failures\n", + "2024-03-06 16:03:34,129 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:34,127 | server.py:222 | fit_round 18: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.25it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.56it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.69it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.95it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.00it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.05it/s]s]\n", + "Running ... [python3 custom/server.py]\n", "Training: 100%|██████████| 417/417 [00:03<00:00, 124.51it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.32it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.58it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.98it/s]\n", - "2024-03-04 13:13:12,785 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:12,785 | server.py:236 | fit_round 26 received 3 results and 0 failures\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.72it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.19it/s]\n", - "2024-03-04 13:13:12,791 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:12,791 | server.py:173 | evaluate_round 26: strategy sampled 3 clients (out of 3)\n", - "2024-03-04 13:13:13,599 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:13,599 | server.py:187 | evaluate_round 26 received 3 results and 0 failures\n", - "2024-03-04 13:13:13,601 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:13,599 | server.py:222 | fit_round 27: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.59it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 131.14it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.48it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.32it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.08it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.88it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.70it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.92it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.93it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.52it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.71it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.02it/s]\n", - "2024-03-04 13:13:27,785 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:27,785 | server.py:236 | fit_round 27 received 3 results and 0 failures\n", - "2024-03-04 13:13:27,791 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:27,791 | server.py:173 | evaluate_round 27: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.63it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.98it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 115.88it/s]\n", - "2024-03-04 13:13:28,592 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:28,591 | server.py:187 | evaluate_round 27 received 3 results and 0 failures\n", - "2024-03-04 13:13:28,593 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:28,592 | server.py:222 | fit_round 28: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.83it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.35it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.61it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.10it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.82it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.81it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.21it/s]s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.71it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.10it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.54it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.38it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.49it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.09it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 124.53it/s]\n", - "2024-03-04 13:13:42,595 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:42,594 | server.py:236 | fit_round 28 received 3 results and 0 failures\n", - "2024-03-04 13:13:42,599 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:42,599 | server.py:173 | evaluate_round 28: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.74it/s]\n", - "2024-03-04 13:13:43,370 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:43,370 | server.py:187 | evaluate_round 28 received 3 results and 0 failures\n", - "2024-03-04 13:13:43,372 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:43,370 | server.py:222 | fit_round 29: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.17it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.31it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.09it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 123.57it/s]\n", - "Running ... [python3 custom/client.py ]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.46it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.49it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.00it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.96it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.67it/s]\n", + "2024-03-06 16:03:47,546 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:47,546 | server.py:236 | fit_round 18 received 3 results and 0 failures\n", + "2024-03-06 16:03:47,548 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:47,548 | server.py:173 | evaluate_round 18: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.02it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.27it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.23it/s]\n", + "2024-03-06 16:03:48,319 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:48,318 | server.py:187 | evaluate_round 18 received 3 results and 0 failures\n", + "2024-03-06 16:03:48,321 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:03:48,319 | server.py:222 | fit_round 19: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.89it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.08it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.83it/s]]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.49it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.81it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 120.47it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.70it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.59it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.20it/s]s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.76it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.39it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.46it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.14it/s]\n", + "2024-03-06 16:04:01,895 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:01,894 | server.py:236 | fit_round 19 received 3 results and 0 failures\n", + "2024-03-06 16:04:01,900 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:01,899 | server.py:173 | evaluate_round 19: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.00it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.93it/s]\n", + "2024-03-06 16:04:02,690 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:02,690 | server.py:187 | evaluate_round 19 received 3 results and 0 failures\n", + "2024-03-06 16:04:02,692 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:02,690 | server.py:222 | fit_round 20: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.85it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.13it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.20it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.73it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.67it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.17it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.05it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.09it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.35it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.87it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.97it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.85it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.70it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.43it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.99it/s]\n", + "2024-03-06 16:04:16,288 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:16,287 | server.py:236 | fit_round 20 received 3 results and 0 failures\n", + "2024-03-06 16:04:16,290 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:16,290 | server.py:173 | evaluate_round 20: strategy sampled 3 clients (out of 3)\n", + "2024-03-06 16:04:17,056 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:17,056 | server.py:187 | evaluate_round 20 received 3 results and 0 failures\n", + "2024-03-06 16:04:17,059 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:17,056 | server.py:222 | fit_round 21: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.25it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.03it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.98it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.12it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.52it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.56it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.13it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.61it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.58it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.38it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.45it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.99it/s]]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.21it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.69it/s]\n", + "2024-03-06 16:04:30,694 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:30,694 | server.py:236 | fit_round 21 received 3 results and 0 failures\n", + "2024-03-06 16:04:30,696 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:30,696 | server.py:173 | evaluate_round 21: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.87it/s]\n", + "2024-03-06 16:04:31,454 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:31,453 | server.py:187 | evaluate_round 21 received 3 results and 0 failures\n", + "2024-03-06 16:04:31,456 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:31,454 | server.py:222 | fit_round 22: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 141.09it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.61it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.55it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.62it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.52it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.87it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.20it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.75it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.08it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.23it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.90it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.93it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.30it/s]\n", + "2024-03-06 16:04:45,011 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:45,011 | server.py:236 | fit_round 22 received 3 results and 0 failures\n", + "2024-03-06 16:04:45,014 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:45,014 | server.py:173 | evaluate_round 22: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.94it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.24it/s]\n", + "2024-03-06 16:04:45,799 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:45,799 | server.py:187 | evaluate_round 22 received 3 results and 0 failures\n", + "2024-03-06 16:04:45,801 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:45,799 | server.py:222 | fit_round 23: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.70it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.71it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.12it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.43it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.45it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.47it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.83it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.03it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.22it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.06it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.52it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.83it/s]]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.40it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.53it/s]\n", + "2024-03-06 16:04:59,317 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:59,317 | server.py:236 | fit_round 23 received 3 results and 0 failures\n", + "2024-03-06 16:04:59,322 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:04:59,322 | server.py:173 | evaluate_round 23: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.36it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-06 16:05:00,101 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:00,100 | server.py:187 | evaluate_round 23 received 3 results and 0 failures\n", + "2024-03-06 16:05:00,103 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:00,101 | server.py:222 | fit_round 24: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.22it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.99it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.29it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.58it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.58it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.59it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.60it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.69it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.00it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.27it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.68it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.67it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.75it/s]\n", + "2024-03-06 16:05:13,713 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:13,713 | server.py:236 | fit_round 24 received 3 results and 0 failures\n", + "2024-03-06 16:05:13,716 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:13,715 | server.py:173 | evaluate_round 24: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.94it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.51it/s]\n", + "2024-03-06 16:05:14,482 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:14,482 | server.py:187 | evaluate_round 24 received 3 results and 0 failures\n", + "2024-03-06 16:05:14,484 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:14,482 | server.py:222 | fit_round 25: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.36it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.65it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.51it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.13it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.24it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.76it/s]s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.74it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.89it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.38it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.67it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.27it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.36it/s]\n", + "2024-03-06 16:05:28,035 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:28,035 | server.py:236 | fit_round 25 received 3 results and 0 failures\n", + "2024-03-06 16:05:28,039 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:28,038 | server.py:173 | evaluate_round 25: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.68it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.41it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.06it/s]\n", + "2024-03-06 16:05:28,822 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:28,822 | server.py:187 | evaluate_round 25 received 3 results and 0 failures\n", + "2024-03-06 16:05:28,823 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:28,822 | server.py:222 | fit_round 26: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.95it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.06it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.23it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.72it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.97it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.64it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.25it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.51it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 126.25it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 128.94it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.45it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.78it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.14it/s]\n", + "2024-03-06 16:05:42,197 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:42,197 | server.py:236 | fit_round 26 received 3 results and 0 failures\n", + "2024-03-06 16:05:42,201 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:42,201 | server.py:173 | evaluate_round 26: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.42it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.81it/s]\n", + "2024-03-06 16:05:42,972 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:42,972 | server.py:187 | evaluate_round 26 received 3 results and 0 failures\n", + "2024-03-06 16:05:42,974 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:42,972 | server.py:222 | fit_round 27: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.78it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 141.90it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.37it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.65it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.03it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.10it/s]]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 127.53it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.62it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.58it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.26it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.33it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.81it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.20it/s]\n", + "2024-03-06 16:05:56,504 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:56,504 | server.py:236 | fit_round 27 received 3 results and 0 failures\n", + "2024-03-06 16:05:56,508 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:56,508 | server.py:173 | evaluate_round 27: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.20it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.93it/s]\n", + "2024-03-06 16:05:57,308 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:57,308 | server.py:187 | evaluate_round 27 received 3 results and 0 failures\n", + "2024-03-06 16:05:57,310 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:05:57,308 | server.py:222 | fit_round 28: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.95it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.97it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 133.80it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.37it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.43it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.84it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.42it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.51it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.07it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 130.11it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.60it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.71it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 129.51it/s]\n", + "2024-03-06 16:06:10,845 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:10,845 | server.py:236 | fit_round 28 received 3 results and 0 failures\n", + "2024-03-06 16:06:10,851 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:10,851 | server.py:173 | evaluate_round 28: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.40it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.90it/s]\n", + "2024-03-06 16:06:11,629 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:11,628 | server.py:187 | evaluate_round 28 received 3 results and 0 failures\n", + "2024-03-06 16:06:11,631 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:11,629 | server.py:222 | fit_round 29: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.37it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.42it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.00it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.64it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.08it/s]s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.63it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.08it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.54it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.15it/s]\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.41it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.89it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.72it/s]\n", + "2024-03-06 16:06:25,201 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:25,201 | server.py:236 | fit_round 29 received 3 results and 0 failures\n", + "2024-03-06 16:06:25,203 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:25,203 | server.py:173 | evaluate_round 29: strategy sampled 3 clients (out of 3)\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.43it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.14it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.76it/s]\n", + "2024-03-06 16:06:25,971 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:25,971 | server.py:187 | evaluate_round 29 received 3 results and 0 failures\n", + "2024-03-06 16:06:25,973 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:25,971 | server.py:222 | fit_round 30: strategy sampled 3 clients (out of 3)\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.01it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.47it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 138.17it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.33it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 122.85it/s]\n", "Training: 100%|██████████| 417/417 [00:03<00:00, 122.45it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.86it/s]]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 126.34it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.11it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.93it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 125.96it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.43it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.65it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.45it/s]]\n", - "2024-03-04 13:13:57,434 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:57,433 | server.py:236 | fit_round 29 received 3 results and 0 failures\n", - "2024-03-04 13:13:57,437 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:57,436 | server.py:173 | evaluate_round 29: strategy sampled 3 clients (out of 3)\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.63it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 116.31it/s]\n", - "2024-03-04 13:13:58,237 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:58,237 | server.py:187 | evaluate_round 29 received 3 results and 0 failures\n", - "2024-03-04 13:13:58,239 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:13:58,237 | server.py:222 | fit_round 30: strategy sampled 3 clients (out of 3)\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 137.18it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 136.32it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 132.07it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.33it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.69it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 118.29it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.59it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 121.77it/s]\n", - "Running ... [python3 custom/client.py ]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 117.74it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.17it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 119.39it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.11it/s]\n", - "Running ... [python3 custom/server.py ]\n", - "2024-03-04 13:14:12,192 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:14:12,192 | server.py:236 | fit_round 30 received 3 results and 0 failures\n", - "2024-03-04 13:14:12,194 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:14:12,194 | server.py:173 | evaluate_round 30: strategy sampled 3 clients (out of 3)\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.36it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 120.74it/s]\n", - "Training: 100%|██████████| 417/417 [00:03<00:00, 122.28it/s]\n", - "2024-03-04 13:14:12,984 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:14:12,984 | server.py:187 | evaluate_round 30 received 3 results and 0 failures\n", - "2024-03-04 13:14:12,987 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:14:12,984 | server.py:153 | FL finished in 450.2050577910268\n", - "2024-03-04 13:14:12,988 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:14:12,985 | app.py:226 | app_fit: losses_distributed [(1, 215.32686360677084), (2, 178.74586486816406), (3, 160.28912353515625), (4, 148.65972391764322), (5, 141.98733520507812), (6, 135.06395467122397), (7, 130.1152598063151), (8, 126.45310974121094), (9, 121.86576334635417), (10, 118.93226623535156), (11, 118.69506581624348), (12, 116.39707438151042), (13, 115.23836008707683), (14, 116.06920623779297), (15, 116.73556009928386), (16, 117.1082534790039), (17, 119.54142252604167), (18, 122.28423309326172), (19, 124.51559193929036), (20, 126.64361063639323), (21, 130.27264658610025), (22, 138.25418599446616), (23, 140.67740885416666), (24, 145.6735585530599), (25, 150.50450134277344), (26, 159.2623545328776), (27, 162.1035359700521), (28, 169.5833485921224), (29, 177.41465759277344), (30, 182.8104451497396)]\n", - "2024-03-04 13:14:12,989 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:14:12,985 | app.py:227 | app_fit: metrics_distributed_fit {}\n", - "2024-03-04 13:14:12,990 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:14:12,985 | app.py:228 | app_fit: metrics_distributed {'accuracy': [(1, 0.2489502099580084), (2, 0.3802239552089582), (3, 0.4495100979804039), (4, 0.4933013397320536), (5, 0.5165966806638672), (6, 0.5441911617676466), (7, 0.5624875024995001), (8, 0.5741851629674065), (9, 0.5946810637872425), (10, 0.6057788442311538), (11, 0.6076784643071386), (12, 0.6197760447910418), (13, 0.6232753449310138), (14, 0.626874625074985), (15, 0.6264747050589882), (16, 0.6339732053589282), (17, 0.6291741651669666), (18, 0.6345730853829235), (19, 0.6260747850429914), (20, 0.6345730853829235), (21, 0.6260747850429914), (22, 0.6199760047990401), (23, 0.625874825034993), (24, 0.6255748850229954), (25, 0.6250749850029994), (26, 0.6217756448710258), (27, 0.6187762447510498), (28, 0.617376524695061), (29, 0.6169766046790642), (30, 0.6150769846030794)]}\n", - "2024-03-04 13:14:12,990 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:14:12,985 | app.py:229 | app_fit: losses_centralized []\n", - "2024-03-04 13:14:12,991 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:14:12,985 | app.py:230 | app_fit: metrics_centralized {}\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.26it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 135.03it/s]\n", - "Testing: 100%|██████████| 105/105 [00:00<00:00, 134.62it/s]\n", - "2024-03-04 13:14:12,993 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:14:12,993 | connection.py:220 | gRPC channel closed\n", - "2024-03-04 13:14:12,993 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:14:12,993 | connection.py:220 | gRPC channel closed\n", - "2024-03-04 13:14:12,993 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-04 13:14:12,993 | connection.py:220 | gRPC channel closed\n", - "2024-03-04 13:14:12,993 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:14:12,993 | app.py:398 | Disconnect and shut down\n", - "2024-03-04 13:14:12,993 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:14:12,993 | app.py:398 | Disconnect and shut down\n", - "2024-03-04 13:14:12,993 - SubprocessLauncher - INFO - INFO flwr 2024-03-04 13:14:12,993 | app.py:398 | Disconnect and shut down\n", + "Running ... [python3 custom/server.py]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.95it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.07it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.93it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.29it/s]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 124.18it/s]]\n", + "Running ... [python3 custom/client.py --node-id 0]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 123.25it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 125.89it/s]\n", + "2024-03-06 16:06:39,622 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:39,622 | server.py:236 | fit_round 30 received 3 results and 0 failures\n", + "2024-03-06 16:06:39,628 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:39,627 | server.py:173 | evaluate_round 30: strategy sampled 3 clients (out of 3)\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 119.46it/s]\n", + "Training: 100%|██████████| 417/417 [00:03<00:00, 121.01it/s]\n", + "Running ... [python3 custom/server.py]\n", + "2024-03-06 16:06:40,388 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:40,387 | server.py:187 | evaluate_round 30 received 3 results and 0 failures\n", + "2024-03-06 16:06:40,390 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | server.py:153 | FL finished in 430.7270666919994\n", + "2024-03-06 16:06:40,391 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | app.py:226 | app_fit: losses_distributed [(1, 212.5634969075521), (2, 177.07842000325522), (3, 157.1292928059896), (4, 146.80193583170572), (5, 137.0935516357422), (6, 128.4277369181315), (7, 121.84776306152344), (8, 112.52642059326172), (9, 106.32784016927083), (10, 101.43400319417317), (11, 95.19340260823567), (12, 88.39397430419922), (13, 80.34169514973958), (14, 75.0222880045573), (15, 67.75357564290364), (16, 63.75060272216797), (17, 56.23624165852865), (18, 53.15401840209961), (19, 46.236531575520836), (20, 43.1369883219401), (21, 40.2807362874349), (22, 39.57940419514974), (23, 34.96051915486654), (24, 29.61727523803711), (25, 31.82533073425293), (26, 28.117460250854492), (27, 25.476886749267578), (28, 25.248918533325195), (29, 26.424275080362957), (30, 26.067562103271484)]\n", + "2024-03-06 16:06:40,392 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | app.py:227 | app_fit: metrics_distributed_fit {}\n", + "2024-03-06 16:06:40,394 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | app.py:228 | app_fit: metrics_distributed {'accuracy': [(1, 0.25674865026994603), (2, 0.39472105578884226), (3, 0.46010797840431916), (4, 0.4989002199560088), (5, 0.5380923815236952), (6, 0.5699860027994401), (7, 0.593381323735253), (8, 0.6232753449310138), (9, 0.650369926014797), (10, 0.6696660667866426), (11, 0.6860627874425115), (12, 0.7186562687462508), (13, 0.7454509098180364), (14, 0.7661467706458708), (15, 0.7933413317336533), (16, 0.8125374925014996), (17, 0.8355328934213158), (18, 0.8529294141171766), (19, 0.8812237552489502), (20, 0.8912217556488702), (21, 0.9072185562887423), (22, 0.9102179564087183), (23, 0.9269146170765847), (24, 0.9489102179564087), (25, 0.9404119176164767), (26, 0.9543091381723655), (27, 0.9707058588282343), (28, 0.9744051189762047), (29, 0.9681063787242552), (30, 0.9673065386922616)]}\n", + "2024-03-06 16:06:40,395 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | app.py:229 | app_fit: losses_centralized []\n", + "2024-03-06 16:06:40,396 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,388 | app.py:230 | app_fit: metrics_centralized {}\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 142.20it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 140.39it/s]\n", + "Testing: 100%|██████████| 105/105 [00:00<00:00, 139.16it/s]\n", + "2024-03-06 16:06:40,398 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:40,397 | connection.py:220 | gRPC channel closed\n", + "2024-03-06 16:06:40,398 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,398 | app.py:398 | Disconnect and shut down\n", + "2024-03-06 16:06:40,399 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:40,399 | connection.py:220 | gRPC channel closed\n", + "2024-03-06 16:06:40,399 - SubprocessLauncher - INFO - DEBUG flwr 2024-03-06 16:06:40,399 | connection.py:220 | gRPC channel closed\n", + "2024-03-06 16:06:40,399 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,399 | app.py:398 | Disconnect and shut down\n", + "2024-03-06 16:06:40,399 - SubprocessLauncher - INFO - INFO flwr 2024-03-06 16:06:40,399 | app.py:398 | Disconnect and shut down\n", + "run success\n", + "2024-03-06 16:06:44,716 - ExecutorLauncher - INFO - [identity=site-2, run=simulate_job]: Stop Executor Launcher.\n", + "2024-03-06 16:06:44,716 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: client runner started\n", + "2024-03-06 16:06:44,717 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-2\n", + "run success\n", + "2024-03-06 16:06:44,722 - ExecutorLauncher - INFO - [identity=site-1, run=simulate_job]: Stop Executor Launcher.\n", + "2024-03-06 16:06:44,723 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: client runner started\n", + "2024-03-06 16:06:44,723 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-1\n", + "run success\n", + "2024-03-06 16:06:44,760 - ExecutorLauncher - INFO - [identity=site-3, run=simulate_job]: Stop Executor Launcher.\n", + "2024-03-06 16:06:44,761 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: client runner started\n", + "2024-03-06 16:06:44,761 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-3\n", "run success\n" ] }, @@ -911,7 +980,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/media/hroth/NVIDIA/home_old/hroth/Code2/flower/baselines/fedprox/.venv_fedprox/lib/python3.10/site-packages/nvflare/app_common/launchers/subprocess_launcher.py:88: ResourceWarning: unclosed file <_io.BufferedReader name=66>\n", + "/media/hroth/NVIDIA/home_old/hroth/Code2/flower/baselines/fedprox/.venv_fedprox/lib/python3.10/site-packages/nvflare/app_common/launchers/subprocess_launcher.py:88: ResourceWarning: unclosed file <_io.BufferedReader name=65>\n", " self._process = None\n", "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" ] @@ -920,51 +989,33 @@ "name": "stdout", "output_type": "stream", "text": [ - "2024-03-04 13:14:20,142 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Stop Controller Launcher.\n", - "2024-03-04 13:14:20,144 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Workflow: controller_launcher finalizing ...\n", - "2024-03-04 13:14:20,351 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: ABOUT_TO_END_RUN fired\n", - "2024-03-04 13:14:20,354 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Firing CHECK_END_RUN_READINESS ...\n", - "2024-03-04 13:14:20,356 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: END_RUN fired\n", - "2024-03-04 13:14:20,357 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Server runner finished.\n", - "2024-03-04 13:14:20,914 - SimulatorServer - INFO - Server app stopped.\n", - "\n", - "\n", - "2024-03-04 13:14:21,256 - nvflare.fuel.hci.server.hci - INFO - Admin Server localhost on Port 49911 shutdown!\n", - "2024-03-04 13:14:21,259 - SimulatorServer - INFO - shutting down server\n", - "2024-03-04 13:14:21,260 - SimulatorServer - INFO - canceling sync locks\n", - "2024-03-04 13:14:21,262 - SimulatorServer - INFO - server off\n", - "2024-03-04 13:14:24,640 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-1, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", - "2024-03-04 13:14:24,642 - GetTaskCommand - INFO - return task to client. client_name: site-1 task_name: __end_run__ task_id: sharable_header_task_id: \n", - "2024-03-04 13:14:24,647 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-3, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", - "2024-03-04 13:14:24,649 - GetTaskCommand - INFO - return task to client. client_name: site-3 task_name: __end_run__ task_id: sharable_header_task_id: \n", - "2024-03-04 13:14:24,652 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-2, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", - "2024-03-04 13:14:24,653 - GetTaskCommand - INFO - return task to client. client_name: site-2 task_name: __end_run__ task_id: sharable_header_task_id: \n" + "2024-03-06 16:06:50,132 - ControllerLauncher - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Stop Controller Launcher.\n", + "2024-03-06 16:06:50,134 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Workflow: controller_launcher finalizing ...\n", + "2024-03-06 16:06:50,414 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: ABOUT_TO_END_RUN fired\n", + "2024-03-06 16:06:50,417 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Firing CHECK_END_RUN_READINESS ...\n", + "2024-03-06 16:06:50,418 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: END_RUN fired\n", + "2024-03-06 16:06:50,419 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: Server runner finished.\n", + "2024-03-06 16:06:50,776 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-2, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", + "2024-03-06 16:06:50,779 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-1, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", + "2024-03-06 16:06:50,780 - GetTaskCommand - INFO - return task to client. client_name: site-2 task_name: __end_run__ task_id: sharable_header_task_id: \n", + "2024-03-06 16:06:50,782 - GetTaskCommand - INFO - return task to client. client_name: site-1 task_name: __end_run__ task_id: sharable_header_task_id: \n", + "2024-03-06 16:06:50,814 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher, peer=site-3, peer_run=simulate_job]: server runner is finalizing - asked client to end the run\n", + "2024-03-06 16:06:50,816 - GetTaskCommand - INFO - return task to client. client_name: site-3 task_name: __end_run__ task_id: sharable_header_task_id: \n", + "2024-03-06 16:06:50,835 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00006 Not Connected] is closed PID: 13991\n", + "2024-03-06 16:06:50,836 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00007 Not Connected] is closed PID: 13991\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/usr/lib/python3.10/subprocess.py:1072: ResourceWarning: subprocess 409112 is still running\n", + "/usr/lib/python3.10/subprocess.py:1072: ResourceWarning: subprocess 14189 is still running\n", " _warn(\"subprocess %s is still running\" % self.pid,\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2024-03-04 13:14:24,689 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00006 Not Connected] is closed PID: 408946\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/lib/python3.10/subprocess.py:1072: ResourceWarning: subprocess 409117 is still running\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/usr/lib/python3.10/subprocess.py:1072: ResourceWarning: subprocess 14192 is still running\n", " _warn(\"subprocess %s is still running\" % self.pid,\n", "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", - "/usr/lib/python3.10/subprocess.py:1072: ResourceWarning: subprocess 409119 is still running\n", + "/usr/lib/python3.10/subprocess.py:1072: ResourceWarning: subprocess 14194 is still running\n", " _warn(\"subprocess %s is still running\" % self.pid,\n", "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" ] @@ -973,56 +1024,46 @@ "name": "stdout", "output_type": "stream", "text": [ - "2024-03-04 13:14:24,702 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00007 Not Connected] is closed PID: 408946\n", - "2024-03-04 13:14:24,702 - FederatedClient - INFO - Shutting down client run: site-1\n", - "2024-03-04 13:14:24,703 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00008 Not Connected] is closed PID: 408946\n", - "2024-03-04 13:14:24,703 - FederatedClient - INFO - Shutting down client run: site-2\n", - "2024-03-04 13:14:24,703 - FederatedClient - INFO - Shutting down client run: site-3\n", - "2024-03-04 13:14:24,704 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: asked to abort - triggered abort_signal to stop the RUN\n", - "Running ... [python3 custom/client.py ]\n", - "run success\n", - "2024-03-04 13:14:24,634 - ExecutorLauncher - INFO - [identity=site-1, run=simulate_job]: Stop Executor Launcher.\n", - "2024-03-04 13:14:24,634 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: client runner started\n", - "2024-03-04 13:14:24,635 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-1\n", - "Running ... [python3 custom/client.py ]\n", - "run success\n", - "2024-03-04 13:14:24,643 - ExecutorLauncher - INFO - [identity=site-3, run=simulate_job]: Stop Executor Launcher.\n", - "2024-03-04 13:14:24,643 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: client runner started\n", - "2024-03-04 13:14:24,644 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-3\n", - "Running ... [python3 custom/client.py ]\n", - "run success\n", - "2024-03-04 13:14:24,645 - ExecutorLauncher - INFO - [identity=site-2, run=simulate_job]: Stop Executor Launcher.\n", - "2024-03-04 13:14:24,646 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", - "2024-03-04 13:14:24,646 - ClientRunner - INFO - [identity=site-1, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", - "2024-03-04 13:14:24,646 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: client runner started\n", - "2024-03-04 13:14:24,646 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: started end-run events sequence\n", - "2024-03-04 13:14:24,646 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: ABOUT_TO_END_RUN fired\n", - "2024-03-04 13:14:24,646 - ClientTaskWorker - INFO - Initialize ClientRunner for client: site-2\n", - "2024-03-04 13:14:24,646 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: Firing CHECK_END_RUN_READINESS ...\n", - "2024-03-04 13:14:24,646 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: END_RUN fired\n", - "2024-03-04 13:14:24,646 - ClientTaskWorker - INFO - End the Simulator run.\n", - "2024-03-04 13:14:24,654 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", - "2024-03-04 13:14:24,654 - ClientRunner - INFO - [identity=site-3, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", - "2024-03-04 13:14:24,654 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: started end-run events sequence\n", - "2024-03-04 13:14:24,654 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: ABOUT_TO_END_RUN fired\n", - "2024-03-04 13:14:24,654 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: Firing CHECK_END_RUN_READINESS ...\n", - "2024-03-04 13:14:24,654 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: END_RUN fired\n", - "2024-03-04 13:14:24,655 - ClientTaskWorker - INFO - End the Simulator run.\n", - "2024-03-04 13:14:24,656 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", - "2024-03-04 13:14:24,656 - ClientRunner - INFO - [identity=site-2, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", - "2024-03-04 13:14:24,656 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: started end-run events sequence\n", - "2024-03-04 13:14:24,656 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: ABOUT_TO_END_RUN fired\n", - "2024-03-04 13:14:24,657 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: Firing CHECK_END_RUN_READINESS ...\n", - "2024-03-04 13:14:24,657 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: END_RUN fired\n", - "2024-03-04 13:14:24,657 - ClientTaskWorker - INFO - End the Simulator run.\n", - "2024-03-04 13:14:24,687 - ClientTaskWorker - INFO - Clean up ClientRunner for : site-1 \n", - "2024-03-04 13:14:24,689 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 Not Connected] is closed PID: 409112\n", - "2024-03-04 13:14:24,699 - ClientTaskWorker - INFO - Clean up ClientRunner for : site-2 \n", - "2024-03-04 13:14:24,700 - ClientTaskWorker - INFO - Clean up ClientRunner for : site-3 \n", - "2024-03-04 13:14:24,702 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 Not Connected] is closed PID: 409117\n", - "2024-03-04 13:14:24,702 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 Not Connected] is closed PID: 409119\n", - "2024-03-04 13:14:28,236 - MPM - WARNING - #### MPM: still running thread Thread-13\n", - "2024-03-04 13:14:28,239 - MPM - INFO - MPM: Good Bye!\n", + "2024-03-06 16:06:50,867 - FederatedClient - INFO - Shutting down client run: site-1\n", + "2024-03-06 16:06:50,868 - FederatedClient - INFO - Shutting down client run: site-2\n", + "2024-03-06 16:06:50,868 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00008 Not Connected] is closed PID: 13991\n", + "2024-03-06 16:06:50,868 - FederatedClient - INFO - Shutting down client run: site-3\n", + "2024-03-06 16:06:50,870 - ServerRunner - INFO - [identity=simulator_server, run=simulate_job, wf=controller_launcher]: asked to abort - triggered abort_signal to stop the RUN\n", + "2024-03-06 16:06:50,787 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", + "2024-03-06 16:06:50,787 - ClientRunner - INFO - [identity=site-2, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", + "2024-03-06 16:06:50,787 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: started end-run events sequence\n", + "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: ABOUT_TO_END_RUN fired\n", + "2024-03-06 16:06:50,788 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", + "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: Firing CHECK_END_RUN_READINESS ...\n", + "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-1, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", + "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: started end-run events sequence\n", + "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-2, run=simulate_job]: END_RUN fired\n", + "2024-03-06 16:06:50,788 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: ABOUT_TO_END_RUN fired\n", + "2024-03-06 16:06:50,788 - ClientTaskWorker - INFO - End the Simulator run.\n", + "2024-03-06 16:06:50,789 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: Firing CHECK_END_RUN_READINESS ...\n", + "2024-03-06 16:06:50,789 - ClientRunner - INFO - [identity=site-1, run=simulate_job]: END_RUN fired\n", + "2024-03-06 16:06:50,789 - ClientTaskWorker - INFO - End the Simulator run.\n", + "2024-03-06 16:06:50,819 - FederatedClient - INFO - pull_task completed. Task name:__end_run__ Status:True \n", + "2024-03-06 16:06:50,820 - ClientRunner - INFO - [identity=site-3, run=simulate_job, peer=simulator_server, peer_run=simulate_job]: server asked to end the run\n", + "2024-03-06 16:06:50,820 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: started end-run events sequence\n", + "2024-03-06 16:06:50,820 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: ABOUT_TO_END_RUN fired\n", + "2024-03-06 16:06:50,820 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: Firing CHECK_END_RUN_READINESS ...\n", + "2024-03-06 16:06:50,820 - ClientRunner - INFO - [identity=site-3, run=simulate_job]: END_RUN fired\n", + "2024-03-06 16:06:50,820 - ClientTaskWorker - INFO - End the Simulator run.\n", + "2024-03-06 16:06:50,832 - ClientTaskWorker - INFO - Clean up ClientRunner for : site-1 \n", + "2024-03-06 16:06:50,833 - ClientTaskWorker - INFO - Clean up ClientRunner for : site-2 \n", + "2024-03-06 16:06:50,835 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 Not Connected] is closed PID: 14189\n", + "2024-03-06 16:06:50,836 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 Not Connected] is closed PID: 14192\n", + "2024-03-06 16:06:50,864 - ClientTaskWorker - INFO - Clean up ClientRunner for : site-3 \n", + "2024-03-06 16:06:50,867 - nvflare.fuel.f3.sfm.conn_manager - INFO - Connection [CN00002 Not Connected] is closed PID: 14194\n", + "2024-03-06 16:06:53,007 - SimulatorServer - INFO - Server app stopped.\n", + "\n", + "\n", + "2024-03-06 16:06:53,464 - nvflare.fuel.hci.server.hci - INFO - Admin Server localhost on Port 40275 shutdown!\n", + "2024-03-06 16:06:53,467 - SimulatorServer - INFO - shutting down server\n", + "2024-03-06 16:06:53,468 - SimulatorServer - INFO - canceling sync locks\n", + "2024-03-06 16:06:53,470 - SimulatorServer - INFO - server off\n", + "2024-03-06 16:06:56,891 - MPM - INFO - MPM: Good Bye!\n", "Simulator finished with run_status 0\n" ] } diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/config/config_fed_client.conf b/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/config/config_fed_client.conf index c2d353fc48..460550bb68 100644 --- a/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/config/config_fed_client.conf +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/config/config_fed_client.conf @@ -1,7 +1,5 @@ { format_version = 2 - app_script = "client.py" - app_config = "" executors = [ { tasks = [ @@ -22,7 +20,7 @@ id = "launcher" path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" args { - script = "python3 custom/{app_script} {app_config} " + script = "python3 custom/client.py --node-id 0" launch_once = true } } diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/config/config_fed_server.conf b/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/config/config_fed_server.conf index 29cbe7d583..46ef7eaa41 100644 --- a/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/config/config_fed_server.conf +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/config/config_fed_server.conf @@ -2,8 +2,6 @@ format_version = 2 task_data_filters = [] task_result_filters = [] - app_script = "server.py" - app_config = "" workflows = [ { id = "controller_launcher" @@ -18,7 +16,7 @@ id = "launcher" path = "nvflare.app_common.launchers.subprocess_launcher.SubprocessLauncher" args { - script = "python3 custom/{app_script} {app_config} " + script = "python3 custom/server.py" launch_once = true } } diff --git a/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/custom/client.py b/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/custom/client.py index a9c462ee9d..4898bed85b 100644 --- a/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/custom/client.py +++ b/examples/advanced/flower/fedprox/jobs/flwr_cifar10/app/custom/client.py @@ -96,17 +96,15 @@ def apply_transforms(batch): # ############################################################################# # Get node id -#parser = argparse.ArgumentParser(description="Flower") -#parser.add_argument( -# "--node-id", -# choices=[0, 1, 2], -# required=True, -# type=int, -# help="Partition of the dataset divided into 3 iid partitions created artificially.", -#) -#node_id = parser.parse_args().node_id -node_id = np.random.randint(0,3) -print(f"START FLOWER CLIENT [node_id={node_id}]") +parser = argparse.ArgumentParser(description="Flower") +parser.add_argument( + "--node-id", + choices=[0, 1, 2], + required=True, + type=int, + help="Partition of the dataset divided into 3 iid partitions created artificially.", +) +node_id = parser.parse_args().node_id # Load model and data (simple CNN, CIFAR-10) net = Net().to(DEVICE)