From eddc593203a625ed80ea9b755b0b596204caee35 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 8 Jun 2021 12:06:42 -0700 Subject: [PATCH 01/25] Add .opam and dune files for local engine --- src/app/test_executive/dune | 11 ++++++----- src/integration_test_local_engine.opam | 5 +++++ 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 src/integration_test_local_engine.opam diff --git a/src/app/test_executive/dune b/src/app/test_executive/dune index 71f50f05d9d..8956a08f082 100644 --- a/src/app/test_executive/dune +++ b/src/app/test_executive/dune @@ -1,8 +1,9 @@ (executable (name test_executive) - (libraries - core_kernel yojson cmdliner file_system currency mina_base + (libraries core_kernel yojson cmdliner file_system currency mina_base runtime_config signature_lib secrets integration_test_lib - integration_test_cloud_engine bash_colors) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_coda ppx_jane ppx_deriving_yojson ppx_coda ppx_version))) + integration_test_cloud_engine integration_test_local_engine bash_colors) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_coda ppx_jane ppx_deriving_yojson ppx_coda ppx_version))) diff --git a/src/integration_test_local_engine.opam b/src/integration_test_local_engine.opam new file mode 100644 index 00000000000..a1b56499734 --- /dev/null +++ b/src/integration_test_local_engine.opam @@ -0,0 +1,5 @@ +opam-version: "1.2" +version: "0.1" +build: [ + ["dune" "build" "--only" "src" "--root" "." "-j" jobs "@install"] +] From 6fe2363489121425ed13c1b24c603db86543de7d Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 8 Jun 2021 12:08:09 -0700 Subject: [PATCH 02/25] Add engine interfaces for local engine Import the engine interface that will be implemented for the local engine. Additionally add the local engine as a CLI flag to the test_executive --- src/app/test_executive/test_executive.ml | 3 ++- .../cli_inputs.ml | 19 +++++++++++++++++++ src/lib/integration_test_local_engine/dune | 14 ++++++++++++++ .../integration_test_local_engine.ml | 6 ++++++ .../integration_test_local_engine.mli | 1 + 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 src/lib/integration_test_local_engine/cli_inputs.ml create mode 100644 src/lib/integration_test_local_engine/dune create mode 100644 src/lib/integration_test_local_engine/integration_test_local_engine.ml create mode 100644 src/lib/integration_test_local_engine/integration_test_local_engine.mli diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index b6f9e03ba00..612a228d0b5 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -36,7 +36,8 @@ let validate_inputs {coda_image; _} = failwith "Coda image cannot be an empt string" let engines : engine list = - [("cloud", (module Integration_test_cloud_engine : Intf.Engine.S))] + [ ("cloud", (module Integration_test_cloud_engine : Intf.Engine.S)) + ; ("local", (module Integration_test_local_engine : Intf.Engine.S)) ] let tests : test list = [ ("reliability", (module Reliability_test.Make : Intf.Test.Functor_intf)) diff --git a/src/lib/integration_test_local_engine/cli_inputs.ml b/src/lib/integration_test_local_engine/cli_inputs.ml new file mode 100644 index 00000000000..460cf410b70 --- /dev/null +++ b/src/lib/integration_test_local_engine/cli_inputs.ml @@ -0,0 +1,19 @@ +open Cmdliner + +type t = {coda_automation_location: string} + +let term = + let coda_automation_location = + let doc = + "Location of the coda automation repository to use when deploying the \ + network." + in + let env = Arg.env_var "CODA_AUTOMATION_LOCATION" ~doc in + Arg.( + value & opt string "./automation" + & info + ["coda-automation-location"] + ~env ~docv:"CODA_AUTOMATION_LOCATION" ~doc) + in + let cons_inputs coda_automation_location = {coda_automation_location} in + Term.(const cons_inputs $ coda_automation_location) diff --git a/src/lib/integration_test_local_engine/dune b/src/lib/integration_test_local_engine/dune new file mode 100644 index 00000000000..8b315292313 --- /dev/null +++ b/src/lib/integration_test_local_engine/dune @@ -0,0 +1,14 @@ +(library + (public_name integration_test_local_engine) + (name integration_test_local_engine) + (inline_tests) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_coda ppx_version ppx_optcomp graphql_ppx ppx_let ppx_inline_test + ppx_custom_printf ppx_deriving_yojson lens.ppx_deriving ppx_pipebang + ppx_sexp_conv)) + (libraries core async lens mina_base pipe_lib runtime_config + genesis_constants graphql_lib transition_frontier user_command_input + genesis_ledger_helper integration_test_lib block_time interruptible + exit_handlers transition_router block_producer)) diff --git a/src/lib/integration_test_local_engine/integration_test_local_engine.ml b/src/lib/integration_test_local_engine/integration_test_local_engine.ml new file mode 100644 index 00000000000..0f623f5c010 --- /dev/null +++ b/src/lib/integration_test_local_engine/integration_test_local_engine.ml @@ -0,0 +1,6 @@ +let name = "local" + +module Network = Swarm_network +module Network_config = Mina_automation.Network_config +module Network_manager = Mina_automation.Network_manager +module Log_engine = Docker_pipe_log_engine diff --git a/src/lib/integration_test_local_engine/integration_test_local_engine.mli b/src/lib/integration_test_local_engine/integration_test_local_engine.mli new file mode 100644 index 00000000000..e1ba61b4e77 --- /dev/null +++ b/src/lib/integration_test_local_engine/integration_test_local_engine.mli @@ -0,0 +1 @@ +include Integration_test_lib.Intf.Engine.S From 28f4e20f9790802369345f465369acc7a6a595db Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 8 Jun 2021 12:10:04 -0700 Subject: [PATCH 03/25] Add implementation for network_config and stubbed interfaces This adds the functionality to create a base docker-compose file from the specified test configs in the integration framework. Each node will be assigned to a service that will be running a container for each node in the network. This implementation currently supports creating a docker-compose file, creating a docker stack to run each container, and basic monitoring before continuing the rest of the test_executive execution. Lots of code in this PR has been stubbed and copied to make the compiler happy. Much of this code will be refactored to support the changes needed for the local engine later. --- .../docker_compose.ml | 61 ++ .../docker_pipe_log_engine.ml | 25 + .../docker_pipe_log_engine.mli | 3 + .../mina_automation.ml | 497 ++++++++++++++++ .../swarm_network.ml | 557 ++++++++++++++++++ 5 files changed, 1143 insertions(+) create mode 100644 src/lib/integration_test_local_engine/docker_compose.ml create mode 100644 src/lib/integration_test_local_engine/docker_pipe_log_engine.ml create mode 100644 src/lib/integration_test_local_engine/docker_pipe_log_engine.mli create mode 100644 src/lib/integration_test_local_engine/mina_automation.ml create mode 100644 src/lib/integration_test_local_engine/swarm_network.ml diff --git a/src/lib/integration_test_local_engine/docker_compose.ml b/src/lib/integration_test_local_engine/docker_compose.ml new file mode 100644 index 00000000000..26d1a90dfbf --- /dev/null +++ b/src/lib/integration_test_local_engine/docker_compose.ml @@ -0,0 +1,61 @@ +open Core + +module Compose = struct + module DockerMap = struct + type 'a t = (string, 'a, String.comparator_witness) Map.t + + let empty = Map.empty (module String) + end + + module Service = struct + module Volume = struct + type t = {type_: string; source: string; target: string} + [@@deriving to_yojson] + + let create name = + {type_= "bind"; source= "." ^/ name; target= "/root" ^/ name} + + let to_yojson {type_; source; target} = + let field k v = (k, `String v) in + let fields = + [ type_ |> field "type" + ; source |> field "source" + ; target |> field "target" ] + in + `Assoc fields + end + + module Environment = struct + type t = string DockerMap.t + + let create = + List.fold ~init:DockerMap.empty ~f:(fun accum env -> + let key, data = env in + Map.set accum ~key ~data ) + + let to_yojson m = + `Assoc (m |> Map.map ~f:(fun x -> `String x) |> Map.to_alist) + end + + type replicas = {replicas: int} [@@deriving to_yojson] + + type t = + { image: string + ; volumes: Volume.t list + ; deploy: replicas + ; command: string list + ; environment: Environment.t } + [@@deriving to_yojson] + end + + type service_map = Service.t DockerMap.t + + let service_map_to_yojson m = + `Assoc (m |> Map.map ~f:Service.to_yojson |> Map.to_alist) + + type t = {version: string; services: service_map} [@@deriving to_yojson] +end + +type t = Compose.t [@@deriving to_yojson] + +let to_string = Fn.compose Yojson.Safe.pretty_to_string to_yojson diff --git a/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml b/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml new file mode 100644 index 00000000000..88ffc83008c --- /dev/null +++ b/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml @@ -0,0 +1,25 @@ +open Async +open Core +open Integration_test_lib +module Timeout = Timeout_lib.Core_time +module Node = Swarm_network.Node + +(* TODO: Implement local engine logging *) + +type t = + { logger: Logger.t + ; event_writer: (Node.t * Event_type.event) Pipe.Writer.t + ; event_reader: (Node.t * Event_type.event) Pipe.Reader.t } + +let event_reader {event_reader; _} = event_reader + +let create ~logger ~(network : Swarm_network.t) = + [%log info] "docker_pipe_log_engine: create %s" network.namespace ; + let event_reader, event_writer = Pipe.create () in + Deferred.Or_error.return {logger; event_reader; event_writer} + +let destroy t : unit Deferred.Or_error.t = + let {logger; event_reader= _; event_writer} = t in + Pipe.close event_writer ; + [%log debug] "subscription deleted" ; + Deferred.Or_error.error_string "subscription deleted" diff --git a/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli b/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli new file mode 100644 index 00000000000..fc1ebc65ce4 --- /dev/null +++ b/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli @@ -0,0 +1,3 @@ +include + Integration_test_lib.Intf.Engine.Log_engine_intf + with module Network := Swarm_network diff --git a/src/lib/integration_test_local_engine/mina_automation.ml b/src/lib/integration_test_local_engine/mina_automation.ml new file mode 100644 index 00000000000..86c557bda7b --- /dev/null +++ b/src/lib/integration_test_local_engine/mina_automation.ml @@ -0,0 +1,497 @@ +open Core +open Async +open Currency +open Signature_lib +open Mina_base +open Integration_test_lib + +let version = "3.9" + +module Network_config = struct + module Cli_inputs = Cli_inputs + + type docker_volume_configs = {name: string; data: string} + [@@deriving to_yojson] + + type block_producer_config = + { name: string + ; id: string + ; keypair: Network_keypair.t + ; public_key: string + ; private_key: string + ; keypair_secret: string + ; libp2p_secret: string } + [@@deriving to_yojson] + + type docker_config = + { version: string + ; stack_name: string + ; coda_image: string + ; docker_volume_configs: docker_volume_configs list + ; block_producer_configs: block_producer_config list + ; log_precomputed_blocks: bool + ; archive_node_count: int + ; mina_archive_schema: string + ; snark_worker_replicas: int + ; snark_worker_fee: string + ; snark_worker_public_key: string + ; runtime_config: Yojson.Safe.t + [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] } + [@@deriving to_yojson] + + type t = + { mina_automation_location: string + ; keypairs: Network_keypair.t list + ; debug_arg: bool + ; constants: Test_config.constants + ; docker: docker_config } + [@@deriving to_yojson] + + let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) + ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) + = + let { Test_config.k + ; delta + ; slots_per_epoch + ; slots_per_sub_window + ; proof_level + ; txpool_max_size + ; requires_graphql= _ + ; block_producers + ; num_snark_workers + ; num_archive_nodes + ; log_precomputed_blocks + ; snark_worker_fee + ; snark_worker_public_key } = + test_config + in + let user_from_env = Option.value (Unix.getenv "USER") ~default:"auto" in + let user_sanitized = + Str.global_replace (Str.regexp "\\W|_-") "" user_from_env + in + let user_len = Int.min 5 (String.length user_sanitized) in + let user = String.sub user_sanitized ~pos:0 ~len:user_len in + let git_commit = Mina_version.commit_id_short in + (* see ./src/app/test_executive/README.md for information regarding the namespace name format and length restrictions *) + let stack_name = "it-" ^ user ^ "-" ^ git_commit ^ "-" ^ test_name in + (* GENERATE ACCOUNTS AND KEYPAIRS *) + let num_block_producers = List.length block_producers in + let block_producer_keypairs, runtime_accounts = + (* the first keypair is the genesis winner and is assumed to be untimed. Therefore dropping it, and not assigning it to any block producer *) + let keypairs = + List.drop (Array.to_list (Lazy.force Sample_keypairs.keypairs)) 1 + in + if num_block_producers > List.length keypairs then + failwith + "not enough sample keypairs for specified number of block producers" ; + let f index ({Test_config.Block_producer.balance; timing}, (pk, sk)) = + let runtime_account = + let timing = + match timing with + | Account.Timing.Untimed -> + None + | Timed t -> + Some + { Runtime_config.Accounts.Single.Timed.initial_minimum_balance= + t.initial_minimum_balance + ; cliff_time= t.cliff_time + ; cliff_amount= t.cliff_amount + ; vesting_period= t.vesting_period + ; vesting_increment= t.vesting_increment } + in + let default = Runtime_config.Accounts.Single.default in + { default with + pk= Some (Public_key.Compressed.to_string pk) + ; sk= None + ; balance= + Balance.of_formatted_string balance + (* delegation currently unsupported *) + ; delegate= None + ; timing } + in + let secret_name = "test-keypair-" ^ Int.to_string index in + let keypair = + {Keypair.public_key= Public_key.decompress_exn pk; private_key= sk} + in + ( Network_keypair.create_network_keypair ~keypair ~secret_name + , runtime_account ) + in + List.mapi ~f + (List.zip_exn block_producers + (List.take keypairs (List.length block_producers))) + |> List.unzip + in + (* DAEMON CONFIG *) + let proof_config = + (* TODO: lift configuration of these up Test_config.t *) + { Runtime_config.Proof_keys.level= Some proof_level + ; sub_windows_per_window= None + ; ledger_depth= None + ; work_delay= None + ; block_window_duration_ms= None + ; transaction_capacity= None + ; coinbase_amount= None + ; supercharged_coinbase_factor= None + ; account_creation_fee= None + ; fork= None } + in + let constraint_constants = + Genesis_ledger_helper.make_constraint_constants + ~default:Genesis_constants.Constraint_constants.compiled proof_config + in + let runtime_config = + { Runtime_config.daemon= + Some {txpool_max_size= Some txpool_max_size; peer_list_url= None} + ; genesis= + Some + { k= Some k + ; delta= Some delta + ; slots_per_epoch= Some slots_per_epoch + ; sub_windows_per_window= + Some constraint_constants.supercharged_coinbase_factor + ; slots_per_sub_window= Some slots_per_sub_window + ; genesis_state_timestamp= + Some Core.Time.(to_string_abs ~zone:Zone.utc (now ())) } + ; proof= + None + (* was: Some proof_config; TODO: prebake ledger and only set hash *) + ; ledger= + Some + { base= Accounts runtime_accounts + ; add_genesis_winner= None + ; num_accounts= None + ; balances= [] + ; hash= None + ; name= None } + ; epoch_data= None } + in + let genesis_constants = + Or_error.ok_exn + (Genesis_ledger_helper.make_genesis_constants ~logger + ~default:Genesis_constants.compiled runtime_config) + in + let constants : Test_config.constants = + {constraints= constraint_constants; genesis= genesis_constants} + in + (* BLOCK PRODUCER CONFIG *) + let block_producer_config index keypair = + { name= "test-block-producer-" ^ Int.to_string (index + 1) + ; id= Int.to_string index + ; keypair + ; keypair_secret= keypair.secret_name + ; public_key= keypair.public_key_file + ; private_key= keypair.private_key_file + ; libp2p_secret= "" } + in + let block_producer_configs = + List.mapi block_producer_keypairs ~f:block_producer_config + in + (* Combine configs for block producer configs and runtime config to be a docker bind volume *) + let docker_volume_configs = + List.map block_producer_configs ~f:(fun config -> + {name= "sk_" ^ config.name; data= config.private_key} ) + @ [ { name= "runtime_config" + ; data= + Yojson.Safe.to_string (Runtime_config.to_yojson runtime_config) + } ] + in + let mina_archive_schema = + "https://raw.githubusercontent.com/MinaProtocol/mina/develop/src/app/archive/create_schema.sql" + in + { mina_automation_location= cli_inputs.coda_automation_location + ; debug_arg= debug + ; keypairs= block_producer_keypairs + ; constants + ; docker= + { version + ; stack_name + ; coda_image= images.coda + ; docker_volume_configs + ; runtime_config= Runtime_config.to_yojson runtime_config + ; block_producer_configs + ; log_precomputed_blocks + ; archive_node_count= num_archive_nodes + ; mina_archive_schema + ; snark_worker_replicas= num_snark_workers + ; snark_worker_public_key + ; snark_worker_fee } } + + let to_docker network_config = + let default_seed_envs = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_LIBP2P_PASS", "") ] + in + let default_block_producer_envs = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_PRIVKEY_PASS", "naughty blue worm") + ; ("CODA_LIBP2P_PASS", "") ] + in + let default_seed_command ~runtime_config = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-client-port" + ; "8301" + ; "-generate-genesis-proof" + ; "true" + ; "-peer" + ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-seed" + ; "-config-file" + ; runtime_config ] + in + let default_block_producer_command ~runtime_config ~private_key_config = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-log-txn-pool-gossip" + ; "true" + ; "-enable-peer-exchange" + ; "true" + ; "-enable-flooding" + ; "true" + ; "-peer" + ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-client-port" + ; "8301" + ; "-generate-genesis-proof" + ; "true" + ; "-block-producer-key" + ; private_key_config + ; "-config-file" + ; runtime_config ] + in + let default_snark_coord_command ~runtime_config ~snark_coordinator_key + ~snark_worker_fee = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-log-txn-pool-gossip" + ; "true" + ; "-external-port" + ; "10909" + ; "-rest-port" + ; "3085" + ; "-client-port" + ; "8301" + ; "-work-selection" + ; "seq" + ; "-peer" + ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-run-snark-coordinator" + ; snark_coordinator_key + ; "-snark-worker-fee" + ; snark_worker_fee + ; "-config-file" + ; runtime_config ] + in + let open Docker_compose.Compose in + let runtime_config = Service.Volume.create "runtime_config" in + let service_map = + List.fold network_config.docker.block_producer_configs + ~init:DockerMap.empty ~f:(fun accum config -> + let private_key_config = + Service.Volume.create ("sk_" ^ config.name) + in + Map.set accum ~key:config.name + ~data: + { Service.image= network_config.docker.coda_image + ; volumes= [private_key_config; runtime_config] + ; deploy= {replicas= 1} + ; command= + default_block_producer_command + ~runtime_config:runtime_config.target + ~private_key_config:private_key_config.target + ; environment= + Service.Environment.create default_block_producer_envs } ) + |> Map.set ~key:"seed" + ~data: + { Service.image= network_config.docker.coda_image + ; volumes= [runtime_config] + ; deploy= {replicas= 1} + ; command= + default_seed_command ~runtime_config:runtime_config.target + ; environment= Service.Environment.create default_seed_envs } + in + let service_map = + if network_config.docker.snark_worker_replicas > 0 then + Map.set service_map ~key:"snark_worker" + ~data: + { Service.image= network_config.docker.coda_image + ; volumes= [runtime_config] + ; deploy= {replicas= network_config.docker.snark_worker_replicas} + ; command= + default_snark_coord_command + ~runtime_config:runtime_config.target + ~snark_coordinator_key: + network_config.docker.snark_worker_public_key + ~snark_worker_fee:network_config.docker.snark_worker_fee + ; environment= Service.Environment.create default_seed_envs } + else service_map + in + {version; services= service_map} +end + +module Network_manager = struct + type t = + { stack_name: string + ; logger: Logger.t + ; testnet_dir: string + ; testnet_log_filter: string + ; constants: Test_config.constants + ; seed_nodes: Swarm_network.Node.t list + ; nodes_by_app_id: Swarm_network.Node.t String.Map.t + ; block_producer_nodes: Swarm_network.Node.t list + ; mutable deployed: bool + ; keypairs: Keypair.t list } + + let run_cmd t prog args = Util.run_cmd t.testnet_dir prog args + + let run_cmd_exn t prog args = Util.run_cmd_exn t.testnet_dir prog args + + let create ~logger (network_config : Network_config.t) = + let%bind all_stacks_str = + Util.run_cmd_exn "/" "docker" ["stack"; "ls"; "--format"; "{{.Name}}"] + in + let all_stacks = String.split ~on:'\n' all_stacks_str in + let testnet_dir = + network_config.mina_automation_location + ^/ network_config.docker.stack_name + in + let%bind () = + if + List.mem all_stacks network_config.docker.stack_name + ~equal:String.equal + then + let%bind () = + if network_config.debug_arg then + Util.prompt_continue + "Existing stack name of same name detected, pausing startup. \ + Enter [y/Y] to continue on and remove existing stack name, \ + start clean, and run the test; press Ctrl-C to quit out: " + else + Deferred.return + ([%log info] + "Existing namespace of same name detected; removing to start \ + clean") + in + Util.run_cmd_exn "/" "docker" + ["stack"; "rm"; network_config.docker.stack_name] + >>| Fn.const () + else return () + in + let%bind () = + if%bind File_system.dir_exists testnet_dir then ( + [%log info] "Old docker stack directory found; removing to start clean" ; + File_system.remove_dir testnet_dir ) + else return () + in + let%bind () = Unix.mkdir testnet_dir in + [%log info] "Writing network configuration" ; + Out_channel.with_file ~fail_if_exists:true (testnet_dir ^/ "compose.json") + ~f:(fun ch -> + Network_config.to_docker network_config + |> Docker_compose.to_string + |> Out_channel.output_string ch ) ; + List.iter network_config.docker.docker_volume_configs ~f:(fun config -> + [%log info] "Writing volume config: %s" (testnet_dir ^/ config.name) ; + Out_channel.with_file ~fail_if_exists:false + (testnet_dir ^/ config.name) ~f:(fun ch -> + config.data |> Out_channel.output_string ch ) ; + ignore (Util.run_cmd_exn testnet_dir "chmod" ["600"; config.name]) ) ; + let cons_node swarm_name service_id network_keypair_opt = + { Swarm_network.Node.swarm_name + ; service_id + ; graphql_enabled= true + ; network_keypair= network_keypair_opt } + in + let seed_nodes = + [cons_node network_config.docker.stack_name "seed" None] + in + let block_producer_nodes = + List.map network_config.docker.block_producer_configs + ~f:(fun bp_config -> + cons_node network_config.docker.stack_name + (network_config.docker.stack_name ^ "_" ^ bp_config.name) + (Some bp_config.keypair) ) + in + let nodes_by_app_id = + let all_nodes = seed_nodes @ block_producer_nodes in + all_nodes + |> List.map ~f:(fun node -> (node.service_id, node)) + |> String.Map.of_alist_exn + in + let t = + { stack_name= network_config.docker.stack_name + ; logger + ; testnet_dir + ; constants= network_config.constants + ; seed_nodes + ; block_producer_nodes + ; nodes_by_app_id + ; deployed= false + ; testnet_log_filter= "" + ; keypairs= + List.map network_config.keypairs ~f:(fun {keypair; _} -> keypair) } + in + Deferred.return t + + let deploy t = + if t.deployed then failwith "network already deployed" ; + [%log' info t.logger] "Deploying network" ; + [%log' info t.logger] "Stack_name in deploy: %s" t.stack_name ; + let%map _ = + run_cmd_exn t "docker" + ["stack"; "deploy"; "-c"; "compose.json"; t.stack_name] + in + t.deployed <- true ; + let result = + { Swarm_network.namespace= t.stack_name + ; constants= t.constants + ; seeds= t.seed_nodes + ; block_producers= t.block_producer_nodes + ; snark_coordinators= [] + ; archive_nodes= [] + ; nodes_by_app_id= t.nodes_by_app_id + ; testnet_log_filter= t.testnet_log_filter + ; keypairs= t.keypairs } + in + let nodes_to_string = + Fn.compose (String.concat ~sep:", ") (List.map ~f:Swarm_network.Node.id) + in + [%log' info t.logger] "Network deployed" ; + [%log' info t.logger] "testnet swarm: %s" t.stack_name ; + [%log' info t.logger] "snark coordinators: %s" + (nodes_to_string result.snark_coordinators) ; + [%log' info t.logger] "block producers: %s" + (nodes_to_string result.block_producers) ; + [%log' info t.logger] "archive nodes: %s" + (nodes_to_string result.archive_nodes) ; + result + + let destroy t = + [%log' info t.logger] "Destroying network" ; + if not t.deployed then failwith "network not deployed" ; + let%bind _ = run_cmd_exn t "docker" ["stack"; "rm"; t.stack_name] in + t.deployed <- false ; + Deferred.unit + + let cleanup t = + let%bind () = if t.deployed then destroy t else return () in + [%log' info t.logger] "Cleaning up network configuration" ; + let%bind () = File_system.remove_dir t.testnet_dir in + Deferred.unit +end diff --git a/src/lib/integration_test_local_engine/swarm_network.ml b/src/lib/integration_test_local_engine/swarm_network.ml new file mode 100644 index 00000000000..9f7767a0c16 --- /dev/null +++ b/src/lib/integration_test_local_engine/swarm_network.ml @@ -0,0 +1,557 @@ +open Core +open Async +open Integration_test_lib + +(* Parts of this implementation is still untouched from the Kubernetes implementation. This is done + just to make the compiler happy as work is done on the local engine. +*) + +(* exclude from bisect_ppx to avoid type error on GraphQL modules *) +[@@@coverage exclude_file] + +module Node = struct + type t = + { swarm_name: string + ; service_id: string + ; graphql_enabled: bool + ; network_keypair: Network_keypair.t option } + + let id {service_id; _} = service_id + + let network_keypair {network_keypair; _} = network_keypair + + let base_kube_args t = ["--cluster"; t.swarm_name] + + let get_container_cmd t = + Printf.sprintf "$(docker ps -f name=%s --quiet)" t.service_id + + let run_in_postgresql_container node ~cmd = + let base_args = base_kube_args node in + let base_kube_cmd = "kubectl " ^ String.concat ~sep:" " base_args in + let kubectl_cmd = + Printf.sprintf "%s -c %s exec -i %s-0 -- %s" base_kube_cmd + node.service_id node.service_id cmd + in + let%bind cwd = Unix.getcwd () in + Util.run_cmd_exn cwd "sh" ["-c"; kubectl_cmd] + + (* + Maybe we get rid of this? We are redirecting all logs to stdout which the test executive will pick up + in a directed pipe + *) + let get_logs_in_container node = + let base_args = base_kube_args node in + let base_kube_cmd = "kubectl " ^ String.concat ~sep:" " base_args in + let pod_cmd = + sprintf "%s get pod -l \"app=%s\" -o name" base_kube_cmd node.service_id + in + let%bind cwd = Unix.getcwd () in + let%bind pod = Util.run_cmd_exn cwd "sh" ["-c"; pod_cmd] in + let kubectl_cmd = + Printf.sprintf "%s logs -c %s -n %s %s" base_kube_cmd node.service_id + node.swarm_name pod + in + Util.run_cmd_exn cwd "sh" ["-c"; kubectl_cmd] + + let run_in_container node cmd = + let base_docker_cmd = "docker exec" in + let docker_cmd = + Printf.sprintf "%s %s %s" base_docker_cmd (get_container_cmd node) cmd + in + let%bind.Deferred.Let_syntax cwd = Unix.getcwd () in + Malleable_error.return (Util.run_cmd_exn cwd "sh" ["-c"; docker_cmd]) + + let start ~fresh_state node : unit Malleable_error.t = + let open Malleable_error.Let_syntax in + let%bind _ = + Deferred.bind ~f:Malleable_error.return (run_in_container node "ps aux") + in + let%bind () = + if fresh_state then + let%bind _ = run_in_container node "rm -rf .mina-config/*" in + Malleable_error.return () + else Malleable_error.return () + in + let%bind _ = run_in_container node "./start.sh" in + Malleable_error.return () + + let stop node = + let open Malleable_error.Let_syntax in + let%bind _ = run_in_container node "ps aux" in + let%bind _ = run_in_container node "./stop.sh" in + let%bind _ = run_in_container node "ps aux" in + return () + + module Decoders = Graphql_lib.Decoders + + module Graphql = struct + let ingress_uri node = + let host = Printf.sprintf "%s.graphql.test.o1test.net" node.swarm_name in + let path = Printf.sprintf "/%s/graphql" node.service_id in + Uri.make ~scheme:"http" ~host ~path ~port:80 () + + module Client = Graphql_lib.Client.Make (struct + let preprocess_variables_string = Fn.id + + let headers = String.Map.empty + end) + + module Unlock_account = + [%graphql + {| + mutation ($password: String!, $public_key: PublicKey!) { + unlockAccount(input: {password: $password, publicKey: $public_key }) { + public_key: publicKey @bsDecoder(fn: "Decoders.public_key") + } + } + |}] + + module Send_payment = + [%graphql + {| + mutation ($sender: PublicKey!, + $receiver: PublicKey!, + $amount: UInt64!, + $token: UInt64, + $fee: UInt64!, + $nonce: UInt32, + $memo: String) { + sendPayment(input: + {from: $sender, to: $receiver, amount: $amount, token: $token, fee: $fee, nonce: $nonce, memo: $memo}) { + payment { + id + } + } + } + |}] + + module Get_balance = + [%graphql + {| + query ($public_key: PublicKey, $token: UInt64) { + account(publicKey: $public_key, token: $token) { + balance { + total @bsDecoder(fn: "Decoders.balance") + } + } + } + |}] + + module Query_peer_id = + [%graphql + {| + query { + daemonStatus { + addrsAndPorts { + peer { + peerId + } + } + peers { peerId } + + } + } + |}] + + module Best_chain = + [%graphql + {| + query { + bestChain { + stateHash + } + } + |}] + end + + (* this function will repeatedly attempt to connect to graphql port times before giving up *) + let exec_graphql_request ?(num_tries = 10) ?(retry_delay_sec = 30.0) + ?(initial_delay_sec = 30.0) ~logger ~node ~query_name query_obj = + let open Deferred.Let_syntax in + if not node.graphql_enabled then + Deferred.Or_error.error_string + "graphql is not enabled (hint: set `requires_graphql= true` in the \ + test config)" + else + let uri = Graphql.ingress_uri node in + let metadata = + [("query", `String query_name); ("uri", `String (Uri.to_string uri))] + in + [%log info] "Attempting to send GraphQL request \"$query\" to \"$uri\"" + ~metadata ; + let rec retry n = + if n <= 0 then ( + [%log error] + "GraphQL request \"$query\" to \"$uri\" failed too many times" + ~metadata ; + Deferred.Or_error.errorf + "GraphQL \"%s\" to \"%s\" request failed too many times" query_name + (Uri.to_string uri) ) + else + match%bind Graphql.Client.query query_obj uri with + | Ok result -> + [%log info] "GraphQL request \"$query\" to \"$uri\" succeeded" + ~metadata ; + Deferred.Or_error.return result + | Error (`Failed_request err_string) -> + [%log warn] + "GraphQL request \"$query\" to \"$uri\" failed: \"$error\" \ + ($num_tries attempts left)" + ~metadata: + ( metadata + @ [("error", `String err_string); ("num_tries", `Int (n - 1))] + ) ; + let%bind () = after (Time.Span.of_sec retry_delay_sec) in + retry (n - 1) + | Error (`Graphql_error err_string) -> + [%log error] + "GraphQL request \"$query\" to \"$uri\" returned an error: \ + \"$error\" (this is a graphql error so not retrying)" + ~metadata:(metadata @ [("error", `String err_string)]) ; + Deferred.Or_error.error_string err_string + in + let%bind () = after (Time.Span.of_sec initial_delay_sec) in + retry num_tries + + let get_peer_id ~logger t = + let open Deferred.Or_error.Let_syntax in + [%log info] "Getting node's peer_id, and the peer_ids of node's peers" + ~metadata: + [("namespace", `String t.swarm_name); ("pod_id", `String t.service_id)] ; + let query_obj = Graphql.Query_peer_id.make () in + let%bind query_result_obj = + exec_graphql_request ~logger ~node:t ~query_name:"query_peer_id" + query_obj + in + [%log info] "get_peer_id, finished exec_graphql_request" ; + let self_id_obj = ((query_result_obj#daemonStatus)#addrsAndPorts)#peer in + let%bind self_id = + match self_id_obj with + | None -> + Deferred.Or_error.error_string "Peer not found" + | Some peer -> + return peer#peerId + in + let peers = (query_result_obj#daemonStatus)#peers |> Array.to_list in + let peer_ids = List.map peers ~f:(fun peer -> peer#peerId) in + [%log info] + "get_peer_id, result of graphql query (self_id,[peers]) (%s,%s)" self_id + (String.concat ~sep:" " peer_ids) ; + return (self_id, peer_ids) + + let must_get_peer_id ~logger t = + get_peer_id ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error + + let get_best_chain ~logger t = + let open Deferred.Or_error.Let_syntax in + let query = Graphql.Best_chain.make () in + let%bind result = + exec_graphql_request ~logger ~node:t ~query_name:"best_chain" query + in + match result#bestChain with + | None | Some [||] -> + Deferred.Or_error.error_string "failed to get best chains" + | Some chain -> + return + @@ List.map ~f:(fun block -> block#stateHash) (Array.to_list chain) + + let must_get_best_chain ~logger t = + get_best_chain ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error + + let get_balance ~logger t ~account_id = + let open Deferred.Or_error.Let_syntax in + [%log info] "Getting account balance" + ~metadata: + [ ("namespace", `String t.swarm_name) + ; ("pod_id", `String t.service_id) + ; ("account_id", Mina_base.Account_id.to_yojson account_id) ] ; + let pk = Mina_base.Account_id.public_key account_id in + let token = Mina_base.Account_id.token_id account_id in + let get_balance_obj = + Graphql.Get_balance.make + ~public_key:(Graphql_lib.Encoders.public_key pk) + ~token:(Graphql_lib.Encoders.token token) + () + in + let%bind balance_obj = + exec_graphql_request ~logger ~node:t ~query_name:"get_balance_graphql" + get_balance_obj + in + match balance_obj#account with + | None -> + Deferred.Or_error.errorf + !"Account with %{sexp:Mina_base.Account_id.t} not found" + account_id + | Some acc -> + return (acc#balance)#total + + let must_get_balance ~logger t ~account_id = + get_balance ~logger t ~account_id + |> Deferred.bind ~f:Malleable_error.or_hard_error + + (* if we expect failure, might want retry_on_graphql_error to be false *) + let send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee = + [%log info] "Sending a payment" + ~metadata: + [("namespace", `String t.swarm_name); ("pod_id", `String t.service_id)] ; + let open Deferred.Or_error.Let_syntax in + let sender_pk_str = + Signature_lib.Public_key.Compressed.to_string sender_pub_key + in + [%log info] "send_payment: unlocking account" + ~metadata:[("sender_pk", `String sender_pk_str)] ; + let unlock_sender_account_graphql () = + let unlock_account_obj = + Graphql.Unlock_account.make ~password:"naughty blue worm" + ~public_key:(Graphql_lib.Encoders.public_key sender_pub_key) + () + in + exec_graphql_request ~logger ~node:t + ~query_name:"unlock_sender_account_graphql" unlock_account_obj + in + let%bind _ = unlock_sender_account_graphql () in + let send_payment_graphql () = + let send_payment_obj = + Graphql.Send_payment.make + ~sender:(Graphql_lib.Encoders.public_key sender_pub_key) + ~receiver:(Graphql_lib.Encoders.public_key receiver_pub_key) + ~amount:(Graphql_lib.Encoders.amount amount) + ~fee:(Graphql_lib.Encoders.fee fee) + () + in + exec_graphql_request ~logger ~node:t ~query_name:"send_payment_graphql" + send_payment_obj + in + let%map sent_payment_obj = send_payment_graphql () in + let (`UserCommand id_obj) = (sent_payment_obj#sendPayment)#payment in + let user_cmd_id = id_obj#id in + [%log info] "Sent payment" + ~metadata:[("user_command_id", `String user_cmd_id)] ; + () + + let must_send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount + ~fee = + send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee + |> Deferred.bind ~f:Malleable_error.or_hard_error + + let dump_archive_data ~logger (t : t) ~data_file = + let open Malleable_error.Let_syntax in + [%log info] "Dumping archive data from (node: %s, container: %s)" + t.service_id t.service_id ; + let%map data = + Deferred.bind ~f:Malleable_error.return + (run_in_postgresql_container t + ~cmd: + "pg_dump --create --no-owner \ + postgres://postgres:foobar@localhost:5432/archive") + in + [%log info] "Dumping archive data to file %s" data_file ; + Out_channel.with_file data_file ~f:(fun out_ch -> + Out_channel.output_string out_ch data ) + + let dump_container_logs ~logger (t : t) ~log_file = + let open Malleable_error.Let_syntax in + [%log info] "Dumping container logs from (node: %s, container: %s)" + t.service_id t.service_id ; + let%map logs = + Deferred.bind ~f:Malleable_error.return (get_logs_in_container t) + in + [%log info] "Dumping container log to file %s" log_file ; + Out_channel.with_file log_file ~f:(fun out_ch -> + Out_channel.output_string out_ch logs ) + + let dump_precomputed_blocks ~logger (t : t) = + let open Malleable_error.Let_syntax in + [%log info] + "Dumping precomputed blocks from logs for (node: %s, container: %s)" + t.service_id t.service_id ; + let%bind logs = + Deferred.bind ~f:Malleable_error.return (get_logs_in_container t) + in + (* kubectl logs may include non-log output, like "Using password from environment variable" *) + let log_lines = + String.split logs ~on:'\n' + |> List.filter ~f:(String.is_prefix ~prefix:"{\"timestamp\":") + in + let jsons = List.map log_lines ~f:Yojson.Safe.from_string in + let metadata_jsons = + List.map jsons ~f:(fun json -> + match json with + | `Assoc items -> ( + match List.Assoc.find items ~equal:String.equal "metadata" with + | Some md -> + md + | None -> + failwithf "Log line is missing metadata: %s" + (Yojson.Safe.to_string json) + () ) + | other -> + failwithf "Expected log line to be a JSON record, got: %s" + (Yojson.Safe.to_string other) + () ) + in + let state_hash_and_blocks = + List.fold metadata_jsons ~init:[] ~f:(fun acc json -> + match json with + | `Assoc items -> ( + match + List.Assoc.find items ~equal:String.equal "precomputed_block" + with + | Some block -> ( + match List.Assoc.find items ~equal:String.equal "state_hash" with + | Some state_hash -> + (state_hash, block) :: acc + | None -> + failwith + "Log metadata contains a precomputed block, but no state \ + hash" ) + | None -> + acc ) + | other -> + failwithf "Expected log line to be a JSON record, got: %s" + (Yojson.Safe.to_string other) + () ) + in + let%bind.Deferred.Let_syntax () = + Deferred.List.iter state_hash_and_blocks + ~f:(fun (state_hash_json, block_json) -> + let double_quoted_state_hash = + Yojson.Safe.to_string state_hash_json + in + let state_hash = + String.sub double_quoted_state_hash ~pos:1 + ~len:(String.length double_quoted_state_hash - 2) + in + let block = Yojson.Safe.pretty_to_string block_json in + let filename = state_hash ^ ".json" in + match%map.Deferred.Let_syntax Sys.file_exists filename with + | `Yes -> + [%log info] + "File already exists for precomputed block with state hash %s" + state_hash + | _ -> + [%log info] + "Dumping precomputed block with state hash %s to file %s" + state_hash filename ; + Out_channel.with_file filename ~f:(fun out_ch -> + Out_channel.output_string out_ch block ) ) + in + Malleable_error.return () +end + +type t = + { namespace: string + ; constants: Test_config.constants + ; seeds: Node.t list + ; block_producers: Node.t list + ; snark_coordinators: Node.t list + ; archive_nodes: Node.t list + ; testnet_log_filter: string + ; keypairs: Signature_lib.Keypair.t list + ; nodes_by_app_id: Node.t String.Map.t } + +let constants {constants; _} = constants + +let constraint_constants {constants; _} = constants.constraints + +let genesis_constants {constants; _} = constants.genesis + +let seeds {seeds; _} = seeds + +let block_producers {block_producers; _} = block_producers + +let snark_coordinators {snark_coordinators; _} = snark_coordinators + +let archive_nodes {archive_nodes; _} = archive_nodes + +let keypairs {keypairs; _} = keypairs + +let all_nodes {seeds; block_producers; snark_coordinators; archive_nodes; _} = + List.concat [seeds; block_producers; snark_coordinators; archive_nodes] + +let lookup_node_by_app_id t = Map.find t.nodes_by_app_id + +let initialize ~logger network = + Print.print_endline "initialize" ; + let open Malleable_error.Let_syntax in + let poll_interval = Time.Span.of_sec 15.0 in + let max_polls = 60 (* 15 mins *) in + let all_services = + all_nodes network + |> List.map ~f:(fun {service_id; _} -> service_id) + |> String.Set.of_list + in + let get_service_statuses () = + let%map output = + Deferred.bind ~f:Malleable_error.return + (Util.run_cmd_exn "/" "docker" + ["service"; "ls"; "--format"; "{{.Name}}: {{.Replicas}}"]) + in + output |> String.split_lines + |> List.map ~f:(fun line -> + let parts = String.split line ~on:':' in + assert (List.length parts = 2) ; + (List.nth_exn parts 0, List.nth_exn parts 1) ) + |> List.filter ~f:(fun (service_name, _) -> + String.Set.mem all_services service_name ) + in + let rec poll n = + let%bind pod_statuses = get_service_statuses () in + (* TODO: detect "bad statuses" (eg CrashLoopBackoff) and terminate early *) + let bad_service_statuses = + List.filter pod_statuses ~f:(fun (_, status) -> + let parts = String.split status ~on:'/' in + assert (List.length parts = 2) ; + let num, denom = (List.nth_exn parts 0, List.nth_exn parts 1) in + String.strip num <> String.strip denom ) + in + if List.is_empty bad_service_statuses then return () + else if n < max_polls then + let%bind () = + after poll_interval |> Deferred.bind ~f:Malleable_error.return + in + poll (n + 1) + else + let bad_pod_statuses_json = + `List + (List.map bad_service_statuses ~f:(fun (service_name, status) -> + `Assoc + [ ("service_name", `String service_name) + ; ("status", `String status) ] )) + in + [%log fatal] + "Not all pods were assigned to nodes and ready in time: \ + $bad_pod_statuses" + ~metadata:[("bad_pod_statuses", bad_pod_statuses_json)] ; + Malleable_error.hard_error_format + "Some pods either were not assigned to nodes or did deploy properly \ + (errors: %s)" + (Yojson.Safe.to_string bad_pod_statuses_json) + in + [%log info] "Waiting for pods to be assigned nodes and become ready" ; + Deferred.bind (poll 0) ~f:(fun res -> + if Malleable_error.is_ok res then + let seed_nodes = seeds network in + let seed_pod_ids = + seed_nodes + |> List.map ~f:(fun {Node.service_id; _} -> service_id) + |> String.Set.of_list + in + let non_seed_nodes = + network |> all_nodes + |> List.filter ~f:(fun {Node.service_id; _} -> + not (String.Set.mem seed_pod_ids service_id) ) + in + (* TODO: parallelize (requires accumlative hard errors) *) + let%bind () = + Malleable_error.List.iter seed_nodes + ~f:(Node.start ~fresh_state:false) + in + (* put a short delay before starting other nodes, to help avoid artifact generation races *) + let%bind () = + after (Time.Span.of_sec 30.0) + |> Deferred.bind ~f:Malleable_error.return + in + Malleable_error.List.iter non_seed_nodes + ~f:(Node.start ~fresh_state:false) + else Deferred.return res ) From 9ecf456e39cfd333345ab4e05b5558383ed23180 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Fri, 11 Jun 2021 10:17:13 -0700 Subject: [PATCH 04/25] Implemented support for snark coordinator and workers Adds support for snark coordinator and workers based off the test configurations passed into the test executive. Right now, we only implement 1 snark coordinator for all other snark workers. Additionally added a new file for all the default Mina node commands and environment variables. --- .../docker_compose.ml | 10 +- .../mina_automation.ml | 183 +++++++----------- .../node_constants.ml | 98 ++++++++++ .../swarm_network.ml | 14 +- 4 files changed, 187 insertions(+), 118 deletions(-) create mode 100644 src/lib/integration_test_local_engine/node_constants.ml diff --git a/src/lib/integration_test_local_engine/docker_compose.ml b/src/lib/integration_test_local_engine/docker_compose.ml index 26d1a90dfbf..fe12f239fcc 100644 --- a/src/lib/integration_test_local_engine/docker_compose.ml +++ b/src/lib/integration_test_local_engine/docker_compose.ml @@ -10,7 +10,6 @@ module Compose = struct module Service = struct module Volume = struct type t = {type_: string; source: string; target: string} - [@@deriving to_yojson] let create name = {type_= "bind"; source= "." ^/ name; target= "/root" ^/ name} @@ -37,12 +36,9 @@ module Compose = struct `Assoc (m |> Map.map ~f:(fun x -> `String x) |> Map.to_alist) end - type replicas = {replicas: int} [@@deriving to_yojson] - type t = { image: string ; volumes: Volume.t list - ; deploy: replicas ; command: string list ; environment: Environment.t } [@@deriving to_yojson] @@ -50,6 +46,12 @@ module Compose = struct type service_map = Service.t DockerMap.t + (* Used to combine different type of service maps. There is an assumption that these maps + are disjoint and will not conflict *) + let merge_maps (map_a : service_map) (map_b : service_map) = + Map.fold map_b ~init:map_a ~f:(fun ~key ~data acc -> + Map.update acc key ~f:(function None -> data | Some data' -> data') ) + let service_map_to_yojson m = `Assoc (m |> Map.map ~f:Service.to_yojson |> Map.to_alist) diff --git a/src/lib/integration_test_local_engine/mina_automation.ml b/src/lib/integration_test_local_engine/mina_automation.ml index 86c557bda7b..0b851d36941 100644 --- a/src/lib/integration_test_local_engine/mina_automation.ml +++ b/src/lib/integration_test_local_engine/mina_automation.ml @@ -23,18 +23,20 @@ module Network_config = struct ; libp2p_secret: string } [@@deriving to_yojson] + type snark_coordinator_configs = + {name: string; id: string; public_key: string; snark_worker_fee: string} + [@@deriving to_yojson] + type docker_config = { version: string ; stack_name: string ; coda_image: string ; docker_volume_configs: docker_volume_configs list ; block_producer_configs: block_producer_config list + ; snark_coordinator_configs: snark_coordinator_configs list ; log_precomputed_blocks: bool ; archive_node_count: int ; mina_archive_schema: string - ; snark_worker_replicas: int - ; snark_worker_fee: string - ; snark_worker_public_key: string ; runtime_config: Yojson.Safe.t [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] } [@@deriving to_yojson] @@ -186,6 +188,23 @@ module Network_config = struct let block_producer_configs = List.mapi block_producer_keypairs ~f:block_producer_config in + let snark_coordinator_configs = + if num_snark_workers > 0 then + List.mapi + (List.init num_snark_workers ~f:(const 0)) + ~f:(fun index _ -> + { name= "test-snark-worker-" ^ Int.to_string (index + 1) + ; id= Int.to_string index + ; snark_worker_fee + ; public_key= snark_worker_public_key } ) + (* Add one snark coordinator for all workers *) + |> List.append + [ { name= "test-snark-coordinator" + ; id= "1" + ; snark_worker_fee + ; public_key= snark_worker_public_key } ] + else [] + in (* Combine configs for block producer configs and runtime config to be a docker bind volume *) let docker_volume_configs = List.map block_producer_configs ~f:(fun config -> @@ -209,98 +228,16 @@ module Network_config = struct ; docker_volume_configs ; runtime_config= Runtime_config.to_yojson runtime_config ; block_producer_configs + ; snark_coordinator_configs ; log_precomputed_blocks ; archive_node_count= num_archive_nodes - ; mina_archive_schema - ; snark_worker_replicas= num_snark_workers - ; snark_worker_public_key - ; snark_worker_fee } } + ; mina_archive_schema } } let to_docker network_config = - let default_seed_envs = - [ ("DAEMON_REST_PORT", "3085") - ; ("DAEMON_CLIENT_PORT", "8301") - ; ("DAEMON_METRICS_PORT", "10001") - ; ("CODA_LIBP2P_PASS", "") ] - in - let default_block_producer_envs = - [ ("DAEMON_REST_PORT", "3085") - ; ("DAEMON_CLIENT_PORT", "8301") - ; ("DAEMON_METRICS_PORT", "10001") - ; ("CODA_PRIVKEY_PASS", "naughty blue worm") - ; ("CODA_LIBP2P_PASS", "") ] - in - let default_seed_command ~runtime_config = - [ "daemon" - ; "-log-level" - ; "Debug" - ; "-log-json" - ; "-log-snark-work-gossip" - ; "true" - ; "-client-port" - ; "8301" - ; "-generate-genesis-proof" - ; "true" - ; "-peer" - ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - ; "-seed" - ; "-config-file" - ; runtime_config ] - in - let default_block_producer_command ~runtime_config ~private_key_config = - [ "daemon" - ; "-log-level" - ; "Debug" - ; "-log-json" - ; "-log-snark-work-gossip" - ; "true" - ; "-log-txn-pool-gossip" - ; "true" - ; "-enable-peer-exchange" - ; "true" - ; "-enable-flooding" - ; "true" - ; "-peer" - ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - ; "-client-port" - ; "8301" - ; "-generate-genesis-proof" - ; "true" - ; "-block-producer-key" - ; private_key_config - ; "-config-file" - ; runtime_config ] - in - let default_snark_coord_command ~runtime_config ~snark_coordinator_key - ~snark_worker_fee = - [ "daemon" - ; "-log-level" - ; "Debug" - ; "-log-json" - ; "-log-snark-work-gossip" - ; "true" - ; "-log-txn-pool-gossip" - ; "true" - ; "-external-port" - ; "10909" - ; "-rest-port" - ; "3085" - ; "-client-port" - ; "8301" - ; "-work-selection" - ; "seq" - ; "-peer" - ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - ; "-run-snark-coordinator" - ; snark_coordinator_key - ; "-snark-worker-fee" - ; snark_worker_fee - ; "-config-file" - ; runtime_config ] - in let open Docker_compose.Compose in + let open Node_constants in let runtime_config = Service.Volume.create "runtime_config" in - let service_map = + let blocks_seed_map = List.fold network_config.docker.block_producer_configs ~init:DockerMap.empty ~f:(fun accum config -> let private_key_config = @@ -310,39 +247,57 @@ module Network_config = struct ~data: { Service.image= network_config.docker.coda_image ; volumes= [private_key_config; runtime_config] - ; deploy= {replicas= 1} ; command= default_block_producer_command ~runtime_config:runtime_config.target ~private_key_config:private_key_config.target ; environment= Service.Environment.create default_block_producer_envs } ) + (* Add a seed node to the map as well*) |> Map.set ~key:"seed" ~data: { Service.image= network_config.docker.coda_image ; volumes= [runtime_config] - ; deploy= {replicas= 1} ; command= default_seed_command ~runtime_config:runtime_config.target ; environment= Service.Environment.create default_seed_envs } in - let service_map = - if network_config.docker.snark_worker_replicas > 0 then - Map.set service_map ~key:"snark_worker" - ~data: - { Service.image= network_config.docker.coda_image - ; volumes= [runtime_config] - ; deploy= {replicas= network_config.docker.snark_worker_replicas} - ; command= - default_snark_coord_command - ~runtime_config:runtime_config.target - ~snark_coordinator_key: - network_config.docker.snark_worker_public_key - ~snark_worker_fee:network_config.docker.snark_worker_fee - ; environment= Service.Environment.create default_seed_envs } - else service_map + let snark_worker_map = + List.fold network_config.docker.snark_coordinator_configs + ~init:DockerMap.empty ~f:(fun accum config -> + let command, environment = + match String.substr_index config.name ~pattern:"coordinator" with + | Some _ -> + let coordinator_command = + default_snark_coord_command + ~runtime_config:runtime_config.target + ~snark_coordinator_key:config.public_key + ~snark_worker_fee:config.snark_worker_fee + in + let coordinator_environment = + Service.Environment.create + (default_snark_coord_envs + ~snark_coordinator_key:config.public_key + ~snark_worker_fee:config.snark_worker_fee) + in + (coordinator_command, coordinator_environment) + | None -> + let worker_command = + default_snark_worker_command + ~daemon_address:"test-snark-coordinator" + in + let worker_environment = Service.Environment.create [] in + (worker_command, worker_environment) + in + Map.set accum ~key:config.name + ~data: + { Service.image= network_config.docker.coda_image + ; volumes= [runtime_config] + ; command + ; environment } ) in - {version; services= service_map} + let services = merge_maps blocks_seed_map snark_worker_map in + {version; services} end module Network_manager = struct @@ -355,6 +310,7 @@ module Network_manager = struct ; seed_nodes: Swarm_network.Node.t list ; nodes_by_app_id: Swarm_network.Node.t String.Map.t ; block_producer_nodes: Swarm_network.Node.t list + ; snark_coordinator_configs: Swarm_network.Node.t list ; mutable deployed: bool ; keypairs: Keypair.t list } @@ -428,6 +384,13 @@ module Network_manager = struct (network_config.docker.stack_name ^ "_" ^ bp_config.name) (Some bp_config.keypair) ) in + let snark_coordinator_configs = + List.map network_config.docker.snark_coordinator_configs + ~f:(fun snark_config -> + cons_node network_config.docker.stack_name + (network_config.docker.stack_name ^ "_" ^ snark_config.name) + None ) + in let nodes_by_app_id = let all_nodes = seed_nodes @ block_producer_nodes in all_nodes @@ -441,6 +404,7 @@ module Network_manager = struct ; constants= network_config.constants ; seed_nodes ; block_producer_nodes + ; snark_coordinator_configs ; nodes_by_app_id ; deployed= false ; testnet_log_filter= "" @@ -453,17 +417,18 @@ module Network_manager = struct if t.deployed then failwith "network already deployed" ; [%log' info t.logger] "Deploying network" ; [%log' info t.logger] "Stack_name in deploy: %s" t.stack_name ; - let%map _ = + let%map s = run_cmd_exn t "docker" ["stack"; "deploy"; "-c"; "compose.json"; t.stack_name] in + [%log' info t.logger] "DEPLOYED STRING: %s" s ; t.deployed <- true ; let result = { Swarm_network.namespace= t.stack_name ; constants= t.constants ; seeds= t.seed_nodes ; block_producers= t.block_producer_nodes - ; snark_coordinators= [] + ; snark_coordinators= t.snark_coordinator_configs ; archive_nodes= [] ; nodes_by_app_id= t.nodes_by_app_id ; testnet_log_filter= t.testnet_log_filter diff --git a/src/lib/integration_test_local_engine/node_constants.ml b/src/lib/integration_test_local_engine/node_constants.ml new file mode 100644 index 00000000000..3d6cd18759e --- /dev/null +++ b/src/lib/integration_test_local_engine/node_constants.ml @@ -0,0 +1,98 @@ +let default_seed_envs = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_LIBP2P_PASS", "") ] + +let default_block_producer_envs = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_PRIVKEY_PASS", "naughty blue worm") + ; ("CODA_LIBP2P_PASS", "") ] + +let default_snark_coord_envs ~snark_coordinator_key ~snark_worker_fee = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_SNARK_KEY", snark_coordinator_key) + ; ("CODA_SNARK_FEE", snark_worker_fee) + ; ("WORK_SELECTION", "seq") + ; ("CODA_PRIVKEY_PASS", "naughty blue worm") + ; ("CODA_LIBP2P_PASS", "") ] + +let default_seed_command ~runtime_config = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-client-port" + ; "8301" + ; "-generate-genesis-proof" + ; "true" + ; "-peer" + ; "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-seed" + ; "-config-file" + ; runtime_config ] + +let default_block_producer_command ~runtime_config ~private_key_config = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-log-txn-pool-gossip" + ; "true" + ; "-enable-peer-exchange" + ; "true" + ; "-enable-flooding" + ; "true" + ; "-peer" + ; "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-client-port" + ; "8301" + ; "-generate-genesis-proof" + ; "true" + ; "-block-producer-key" + ; private_key_config + ; "-config-file" + ; runtime_config ] + +let default_snark_coord_command ~runtime_config ~snark_coordinator_key + ~snark_worker_fee = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-log-txn-pool-gossip" + ; "true" + ; "-external-port" + ; "10909" + ; "-rest-port" + ; "3085" + ; "-client-port" + ; "8301" + ; "-work-selection" + ; "seq" + ; "-peer" + ; "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-run-snark-coordinator" + ; snark_coordinator_key + ; "-snark-worker-fee" + ; snark_worker_fee + ; "-config-file" + ; runtime_config ] + +let default_snark_worker_command ~daemon_address = + [ "internal" + ; "snark-worker" + ; "-proof-level" + ; "full" + ; "-daemon-address" + ; daemon_address ^ ":8301" ] diff --git a/src/lib/integration_test_local_engine/swarm_network.ml b/src/lib/integration_test_local_engine/swarm_network.ml index 9f7767a0c16..e7eaf1997e1 100644 --- a/src/lib/integration_test_local_engine/swarm_network.ml +++ b/src/lib/integration_test_local_engine/swarm_network.ml @@ -35,10 +35,6 @@ module Node = struct let%bind cwd = Unix.getcwd () in Util.run_cmd_exn cwd "sh" ["-c"; kubectl_cmd] - (* - Maybe we get rid of this? We are redirecting all logs to stdout which the test executive will pick up - in a directed pipe - *) let get_logs_in_container node = let base_args = base_kube_args node in let base_kube_cmd = "kubectl " ^ String.concat ~sep:" " base_args in @@ -72,7 +68,15 @@ module Node = struct Malleable_error.return () else Malleable_error.return () in - let%bind _ = run_in_container node "./start.sh" in + let cmd = + match String.substr_index node.service_id ~pattern:"snark-worker" with + | Some _ -> + (* Snark-workers should wait for work to be generated so they don't error a 'get_work' RPC call*) + "/bin/bash -c 'sleep 120 && ./start.sh'" + | None -> + "./start.sh" + in + let%bind _ = run_in_container node cmd in Malleable_error.return () let stop node = From 2f89ef8ff17623765232a67349e8d8c9119cc36e Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 8 Jun 2021 12:06:42 -0700 Subject: [PATCH 05/25] Add .opam and dune files for local engine --- src/app/test_executive/dune | 11 ++++++----- src/integration_test_local_engine.opam | 5 +++++ 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 src/integration_test_local_engine.opam diff --git a/src/app/test_executive/dune b/src/app/test_executive/dune index 71f50f05d9d..8956a08f082 100644 --- a/src/app/test_executive/dune +++ b/src/app/test_executive/dune @@ -1,8 +1,9 @@ (executable (name test_executive) - (libraries - core_kernel yojson cmdliner file_system currency mina_base + (libraries core_kernel yojson cmdliner file_system currency mina_base runtime_config signature_lib secrets integration_test_lib - integration_test_cloud_engine bash_colors) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_coda ppx_jane ppx_deriving_yojson ppx_coda ppx_version))) + integration_test_cloud_engine integration_test_local_engine bash_colors) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_coda ppx_jane ppx_deriving_yojson ppx_coda ppx_version))) diff --git a/src/integration_test_local_engine.opam b/src/integration_test_local_engine.opam new file mode 100644 index 00000000000..a1b56499734 --- /dev/null +++ b/src/integration_test_local_engine.opam @@ -0,0 +1,5 @@ +opam-version: "1.2" +version: "0.1" +build: [ + ["dune" "build" "--only" "src" "--root" "." "-j" jobs "@install"] +] From bb3787baf479a758e85aae5a2462a5a6610de000 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 8 Jun 2021 12:08:09 -0700 Subject: [PATCH 06/25] Add engine interfaces for local engine Import the engine interface that will be implemented for the local engine. Additionally add the local engine as a CLI flag to the test_executive --- src/app/test_executive/test_executive.ml | 4 +++- .../cli_inputs.ml | 19 +++++++++++++++++++ src/lib/integration_test_local_engine/dune | 14 ++++++++++++++ .../integration_test_local_engine.ml | 6 ++++++ .../integration_test_local_engine.mli | 1 + 5 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 src/lib/integration_test_local_engine/cli_inputs.ml create mode 100644 src/lib/integration_test_local_engine/dune create mode 100644 src/lib/integration_test_local_engine/integration_test_local_engine.ml create mode 100644 src/lib/integration_test_local_engine/integration_test_local_engine.mli diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 6469f808d05..5f52df00ef1 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -37,7 +37,9 @@ let validate_inputs { coda_image; _ } = failwith "Coda image cannot be an empt string" let engines : engine list = - [ ("cloud", (module Integration_test_cloud_engine : Intf.Engine.S)) ] + [ ("cloud", (module Integration_test_cloud_engine : Intf.Engine.S)) + ; ("local", (module Integration_test_local_engine : Intf.Engine.S)) + ] let tests : test list = [ ("reliability", (module Reliability_test.Make : Intf.Test.Functor_intf)) diff --git a/src/lib/integration_test_local_engine/cli_inputs.ml b/src/lib/integration_test_local_engine/cli_inputs.ml new file mode 100644 index 00000000000..460cf410b70 --- /dev/null +++ b/src/lib/integration_test_local_engine/cli_inputs.ml @@ -0,0 +1,19 @@ +open Cmdliner + +type t = {coda_automation_location: string} + +let term = + let coda_automation_location = + let doc = + "Location of the coda automation repository to use when deploying the \ + network." + in + let env = Arg.env_var "CODA_AUTOMATION_LOCATION" ~doc in + Arg.( + value & opt string "./automation" + & info + ["coda-automation-location"] + ~env ~docv:"CODA_AUTOMATION_LOCATION" ~doc) + in + let cons_inputs coda_automation_location = {coda_automation_location} in + Term.(const cons_inputs $ coda_automation_location) diff --git a/src/lib/integration_test_local_engine/dune b/src/lib/integration_test_local_engine/dune new file mode 100644 index 00000000000..8b315292313 --- /dev/null +++ b/src/lib/integration_test_local_engine/dune @@ -0,0 +1,14 @@ +(library + (public_name integration_test_local_engine) + (name integration_test_local_engine) + (inline_tests) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_coda ppx_version ppx_optcomp graphql_ppx ppx_let ppx_inline_test + ppx_custom_printf ppx_deriving_yojson lens.ppx_deriving ppx_pipebang + ppx_sexp_conv)) + (libraries core async lens mina_base pipe_lib runtime_config + genesis_constants graphql_lib transition_frontier user_command_input + genesis_ledger_helper integration_test_lib block_time interruptible + exit_handlers transition_router block_producer)) diff --git a/src/lib/integration_test_local_engine/integration_test_local_engine.ml b/src/lib/integration_test_local_engine/integration_test_local_engine.ml new file mode 100644 index 00000000000..0f623f5c010 --- /dev/null +++ b/src/lib/integration_test_local_engine/integration_test_local_engine.ml @@ -0,0 +1,6 @@ +let name = "local" + +module Network = Swarm_network +module Network_config = Mina_automation.Network_config +module Network_manager = Mina_automation.Network_manager +module Log_engine = Docker_pipe_log_engine diff --git a/src/lib/integration_test_local_engine/integration_test_local_engine.mli b/src/lib/integration_test_local_engine/integration_test_local_engine.mli new file mode 100644 index 00000000000..e1ba61b4e77 --- /dev/null +++ b/src/lib/integration_test_local_engine/integration_test_local_engine.mli @@ -0,0 +1 @@ +include Integration_test_lib.Intf.Engine.S From bd88d17531b2fc9d766b6f767d3684adbafdef4d Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 8 Jun 2021 12:10:04 -0700 Subject: [PATCH 07/25] Add implementation for network_config and stubbed interfaces This adds the functionality to create a base docker-compose file from the specified test configs in the integration framework. Each node will be assigned to a service that will be running a container for each node in the network. This implementation currently supports creating a docker-compose file, creating a docker stack to run each container, and basic monitoring before continuing the rest of the test_executive execution. Lots of code in this PR has been stubbed and copied to make the compiler happy. Much of this code will be refactored to support the changes needed for the local engine later. --- .../docker_compose.ml | 61 ++ .../docker_pipe_log_engine.ml | 25 + .../docker_pipe_log_engine.mli | 3 + .../mina_automation.ml | 497 ++++++++++++++++ .../swarm_network.ml | 557 ++++++++++++++++++ 5 files changed, 1143 insertions(+) create mode 100644 src/lib/integration_test_local_engine/docker_compose.ml create mode 100644 src/lib/integration_test_local_engine/docker_pipe_log_engine.ml create mode 100644 src/lib/integration_test_local_engine/docker_pipe_log_engine.mli create mode 100644 src/lib/integration_test_local_engine/mina_automation.ml create mode 100644 src/lib/integration_test_local_engine/swarm_network.ml diff --git a/src/lib/integration_test_local_engine/docker_compose.ml b/src/lib/integration_test_local_engine/docker_compose.ml new file mode 100644 index 00000000000..26d1a90dfbf --- /dev/null +++ b/src/lib/integration_test_local_engine/docker_compose.ml @@ -0,0 +1,61 @@ +open Core + +module Compose = struct + module DockerMap = struct + type 'a t = (string, 'a, String.comparator_witness) Map.t + + let empty = Map.empty (module String) + end + + module Service = struct + module Volume = struct + type t = {type_: string; source: string; target: string} + [@@deriving to_yojson] + + let create name = + {type_= "bind"; source= "." ^/ name; target= "/root" ^/ name} + + let to_yojson {type_; source; target} = + let field k v = (k, `String v) in + let fields = + [ type_ |> field "type" + ; source |> field "source" + ; target |> field "target" ] + in + `Assoc fields + end + + module Environment = struct + type t = string DockerMap.t + + let create = + List.fold ~init:DockerMap.empty ~f:(fun accum env -> + let key, data = env in + Map.set accum ~key ~data ) + + let to_yojson m = + `Assoc (m |> Map.map ~f:(fun x -> `String x) |> Map.to_alist) + end + + type replicas = {replicas: int} [@@deriving to_yojson] + + type t = + { image: string + ; volumes: Volume.t list + ; deploy: replicas + ; command: string list + ; environment: Environment.t } + [@@deriving to_yojson] + end + + type service_map = Service.t DockerMap.t + + let service_map_to_yojson m = + `Assoc (m |> Map.map ~f:Service.to_yojson |> Map.to_alist) + + type t = {version: string; services: service_map} [@@deriving to_yojson] +end + +type t = Compose.t [@@deriving to_yojson] + +let to_string = Fn.compose Yojson.Safe.pretty_to_string to_yojson diff --git a/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml b/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml new file mode 100644 index 00000000000..88ffc83008c --- /dev/null +++ b/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml @@ -0,0 +1,25 @@ +open Async +open Core +open Integration_test_lib +module Timeout = Timeout_lib.Core_time +module Node = Swarm_network.Node + +(* TODO: Implement local engine logging *) + +type t = + { logger: Logger.t + ; event_writer: (Node.t * Event_type.event) Pipe.Writer.t + ; event_reader: (Node.t * Event_type.event) Pipe.Reader.t } + +let event_reader {event_reader; _} = event_reader + +let create ~logger ~(network : Swarm_network.t) = + [%log info] "docker_pipe_log_engine: create %s" network.namespace ; + let event_reader, event_writer = Pipe.create () in + Deferred.Or_error.return {logger; event_reader; event_writer} + +let destroy t : unit Deferred.Or_error.t = + let {logger; event_reader= _; event_writer} = t in + Pipe.close event_writer ; + [%log debug] "subscription deleted" ; + Deferred.Or_error.error_string "subscription deleted" diff --git a/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli b/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli new file mode 100644 index 00000000000..fc1ebc65ce4 --- /dev/null +++ b/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli @@ -0,0 +1,3 @@ +include + Integration_test_lib.Intf.Engine.Log_engine_intf + with module Network := Swarm_network diff --git a/src/lib/integration_test_local_engine/mina_automation.ml b/src/lib/integration_test_local_engine/mina_automation.ml new file mode 100644 index 00000000000..86c557bda7b --- /dev/null +++ b/src/lib/integration_test_local_engine/mina_automation.ml @@ -0,0 +1,497 @@ +open Core +open Async +open Currency +open Signature_lib +open Mina_base +open Integration_test_lib + +let version = "3.9" + +module Network_config = struct + module Cli_inputs = Cli_inputs + + type docker_volume_configs = {name: string; data: string} + [@@deriving to_yojson] + + type block_producer_config = + { name: string + ; id: string + ; keypair: Network_keypair.t + ; public_key: string + ; private_key: string + ; keypair_secret: string + ; libp2p_secret: string } + [@@deriving to_yojson] + + type docker_config = + { version: string + ; stack_name: string + ; coda_image: string + ; docker_volume_configs: docker_volume_configs list + ; block_producer_configs: block_producer_config list + ; log_precomputed_blocks: bool + ; archive_node_count: int + ; mina_archive_schema: string + ; snark_worker_replicas: int + ; snark_worker_fee: string + ; snark_worker_public_key: string + ; runtime_config: Yojson.Safe.t + [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] } + [@@deriving to_yojson] + + type t = + { mina_automation_location: string + ; keypairs: Network_keypair.t list + ; debug_arg: bool + ; constants: Test_config.constants + ; docker: docker_config } + [@@deriving to_yojson] + + let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) + ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) + = + let { Test_config.k + ; delta + ; slots_per_epoch + ; slots_per_sub_window + ; proof_level + ; txpool_max_size + ; requires_graphql= _ + ; block_producers + ; num_snark_workers + ; num_archive_nodes + ; log_precomputed_blocks + ; snark_worker_fee + ; snark_worker_public_key } = + test_config + in + let user_from_env = Option.value (Unix.getenv "USER") ~default:"auto" in + let user_sanitized = + Str.global_replace (Str.regexp "\\W|_-") "" user_from_env + in + let user_len = Int.min 5 (String.length user_sanitized) in + let user = String.sub user_sanitized ~pos:0 ~len:user_len in + let git_commit = Mina_version.commit_id_short in + (* see ./src/app/test_executive/README.md for information regarding the namespace name format and length restrictions *) + let stack_name = "it-" ^ user ^ "-" ^ git_commit ^ "-" ^ test_name in + (* GENERATE ACCOUNTS AND KEYPAIRS *) + let num_block_producers = List.length block_producers in + let block_producer_keypairs, runtime_accounts = + (* the first keypair is the genesis winner and is assumed to be untimed. Therefore dropping it, and not assigning it to any block producer *) + let keypairs = + List.drop (Array.to_list (Lazy.force Sample_keypairs.keypairs)) 1 + in + if num_block_producers > List.length keypairs then + failwith + "not enough sample keypairs for specified number of block producers" ; + let f index ({Test_config.Block_producer.balance; timing}, (pk, sk)) = + let runtime_account = + let timing = + match timing with + | Account.Timing.Untimed -> + None + | Timed t -> + Some + { Runtime_config.Accounts.Single.Timed.initial_minimum_balance= + t.initial_minimum_balance + ; cliff_time= t.cliff_time + ; cliff_amount= t.cliff_amount + ; vesting_period= t.vesting_period + ; vesting_increment= t.vesting_increment } + in + let default = Runtime_config.Accounts.Single.default in + { default with + pk= Some (Public_key.Compressed.to_string pk) + ; sk= None + ; balance= + Balance.of_formatted_string balance + (* delegation currently unsupported *) + ; delegate= None + ; timing } + in + let secret_name = "test-keypair-" ^ Int.to_string index in + let keypair = + {Keypair.public_key= Public_key.decompress_exn pk; private_key= sk} + in + ( Network_keypair.create_network_keypair ~keypair ~secret_name + , runtime_account ) + in + List.mapi ~f + (List.zip_exn block_producers + (List.take keypairs (List.length block_producers))) + |> List.unzip + in + (* DAEMON CONFIG *) + let proof_config = + (* TODO: lift configuration of these up Test_config.t *) + { Runtime_config.Proof_keys.level= Some proof_level + ; sub_windows_per_window= None + ; ledger_depth= None + ; work_delay= None + ; block_window_duration_ms= None + ; transaction_capacity= None + ; coinbase_amount= None + ; supercharged_coinbase_factor= None + ; account_creation_fee= None + ; fork= None } + in + let constraint_constants = + Genesis_ledger_helper.make_constraint_constants + ~default:Genesis_constants.Constraint_constants.compiled proof_config + in + let runtime_config = + { Runtime_config.daemon= + Some {txpool_max_size= Some txpool_max_size; peer_list_url= None} + ; genesis= + Some + { k= Some k + ; delta= Some delta + ; slots_per_epoch= Some slots_per_epoch + ; sub_windows_per_window= + Some constraint_constants.supercharged_coinbase_factor + ; slots_per_sub_window= Some slots_per_sub_window + ; genesis_state_timestamp= + Some Core.Time.(to_string_abs ~zone:Zone.utc (now ())) } + ; proof= + None + (* was: Some proof_config; TODO: prebake ledger and only set hash *) + ; ledger= + Some + { base= Accounts runtime_accounts + ; add_genesis_winner= None + ; num_accounts= None + ; balances= [] + ; hash= None + ; name= None } + ; epoch_data= None } + in + let genesis_constants = + Or_error.ok_exn + (Genesis_ledger_helper.make_genesis_constants ~logger + ~default:Genesis_constants.compiled runtime_config) + in + let constants : Test_config.constants = + {constraints= constraint_constants; genesis= genesis_constants} + in + (* BLOCK PRODUCER CONFIG *) + let block_producer_config index keypair = + { name= "test-block-producer-" ^ Int.to_string (index + 1) + ; id= Int.to_string index + ; keypair + ; keypair_secret= keypair.secret_name + ; public_key= keypair.public_key_file + ; private_key= keypair.private_key_file + ; libp2p_secret= "" } + in + let block_producer_configs = + List.mapi block_producer_keypairs ~f:block_producer_config + in + (* Combine configs for block producer configs and runtime config to be a docker bind volume *) + let docker_volume_configs = + List.map block_producer_configs ~f:(fun config -> + {name= "sk_" ^ config.name; data= config.private_key} ) + @ [ { name= "runtime_config" + ; data= + Yojson.Safe.to_string (Runtime_config.to_yojson runtime_config) + } ] + in + let mina_archive_schema = + "https://raw.githubusercontent.com/MinaProtocol/mina/develop/src/app/archive/create_schema.sql" + in + { mina_automation_location= cli_inputs.coda_automation_location + ; debug_arg= debug + ; keypairs= block_producer_keypairs + ; constants + ; docker= + { version + ; stack_name + ; coda_image= images.coda + ; docker_volume_configs + ; runtime_config= Runtime_config.to_yojson runtime_config + ; block_producer_configs + ; log_precomputed_blocks + ; archive_node_count= num_archive_nodes + ; mina_archive_schema + ; snark_worker_replicas= num_snark_workers + ; snark_worker_public_key + ; snark_worker_fee } } + + let to_docker network_config = + let default_seed_envs = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_LIBP2P_PASS", "") ] + in + let default_block_producer_envs = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_PRIVKEY_PASS", "naughty blue worm") + ; ("CODA_LIBP2P_PASS", "") ] + in + let default_seed_command ~runtime_config = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-client-port" + ; "8301" + ; "-generate-genesis-proof" + ; "true" + ; "-peer" + ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-seed" + ; "-config-file" + ; runtime_config ] + in + let default_block_producer_command ~runtime_config ~private_key_config = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-log-txn-pool-gossip" + ; "true" + ; "-enable-peer-exchange" + ; "true" + ; "-enable-flooding" + ; "true" + ; "-peer" + ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-client-port" + ; "8301" + ; "-generate-genesis-proof" + ; "true" + ; "-block-producer-key" + ; private_key_config + ; "-config-file" + ; runtime_config ] + in + let default_snark_coord_command ~runtime_config ~snark_coordinator_key + ~snark_worker_fee = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-log-txn-pool-gossip" + ; "true" + ; "-external-port" + ; "10909" + ; "-rest-port" + ; "3085" + ; "-client-port" + ; "8301" + ; "-work-selection" + ; "seq" + ; "-peer" + ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-run-snark-coordinator" + ; snark_coordinator_key + ; "-snark-worker-fee" + ; snark_worker_fee + ; "-config-file" + ; runtime_config ] + in + let open Docker_compose.Compose in + let runtime_config = Service.Volume.create "runtime_config" in + let service_map = + List.fold network_config.docker.block_producer_configs + ~init:DockerMap.empty ~f:(fun accum config -> + let private_key_config = + Service.Volume.create ("sk_" ^ config.name) + in + Map.set accum ~key:config.name + ~data: + { Service.image= network_config.docker.coda_image + ; volumes= [private_key_config; runtime_config] + ; deploy= {replicas= 1} + ; command= + default_block_producer_command + ~runtime_config:runtime_config.target + ~private_key_config:private_key_config.target + ; environment= + Service.Environment.create default_block_producer_envs } ) + |> Map.set ~key:"seed" + ~data: + { Service.image= network_config.docker.coda_image + ; volumes= [runtime_config] + ; deploy= {replicas= 1} + ; command= + default_seed_command ~runtime_config:runtime_config.target + ; environment= Service.Environment.create default_seed_envs } + in + let service_map = + if network_config.docker.snark_worker_replicas > 0 then + Map.set service_map ~key:"snark_worker" + ~data: + { Service.image= network_config.docker.coda_image + ; volumes= [runtime_config] + ; deploy= {replicas= network_config.docker.snark_worker_replicas} + ; command= + default_snark_coord_command + ~runtime_config:runtime_config.target + ~snark_coordinator_key: + network_config.docker.snark_worker_public_key + ~snark_worker_fee:network_config.docker.snark_worker_fee + ; environment= Service.Environment.create default_seed_envs } + else service_map + in + {version; services= service_map} +end + +module Network_manager = struct + type t = + { stack_name: string + ; logger: Logger.t + ; testnet_dir: string + ; testnet_log_filter: string + ; constants: Test_config.constants + ; seed_nodes: Swarm_network.Node.t list + ; nodes_by_app_id: Swarm_network.Node.t String.Map.t + ; block_producer_nodes: Swarm_network.Node.t list + ; mutable deployed: bool + ; keypairs: Keypair.t list } + + let run_cmd t prog args = Util.run_cmd t.testnet_dir prog args + + let run_cmd_exn t prog args = Util.run_cmd_exn t.testnet_dir prog args + + let create ~logger (network_config : Network_config.t) = + let%bind all_stacks_str = + Util.run_cmd_exn "/" "docker" ["stack"; "ls"; "--format"; "{{.Name}}"] + in + let all_stacks = String.split ~on:'\n' all_stacks_str in + let testnet_dir = + network_config.mina_automation_location + ^/ network_config.docker.stack_name + in + let%bind () = + if + List.mem all_stacks network_config.docker.stack_name + ~equal:String.equal + then + let%bind () = + if network_config.debug_arg then + Util.prompt_continue + "Existing stack name of same name detected, pausing startup. \ + Enter [y/Y] to continue on and remove existing stack name, \ + start clean, and run the test; press Ctrl-C to quit out: " + else + Deferred.return + ([%log info] + "Existing namespace of same name detected; removing to start \ + clean") + in + Util.run_cmd_exn "/" "docker" + ["stack"; "rm"; network_config.docker.stack_name] + >>| Fn.const () + else return () + in + let%bind () = + if%bind File_system.dir_exists testnet_dir then ( + [%log info] "Old docker stack directory found; removing to start clean" ; + File_system.remove_dir testnet_dir ) + else return () + in + let%bind () = Unix.mkdir testnet_dir in + [%log info] "Writing network configuration" ; + Out_channel.with_file ~fail_if_exists:true (testnet_dir ^/ "compose.json") + ~f:(fun ch -> + Network_config.to_docker network_config + |> Docker_compose.to_string + |> Out_channel.output_string ch ) ; + List.iter network_config.docker.docker_volume_configs ~f:(fun config -> + [%log info] "Writing volume config: %s" (testnet_dir ^/ config.name) ; + Out_channel.with_file ~fail_if_exists:false + (testnet_dir ^/ config.name) ~f:(fun ch -> + config.data |> Out_channel.output_string ch ) ; + ignore (Util.run_cmd_exn testnet_dir "chmod" ["600"; config.name]) ) ; + let cons_node swarm_name service_id network_keypair_opt = + { Swarm_network.Node.swarm_name + ; service_id + ; graphql_enabled= true + ; network_keypair= network_keypair_opt } + in + let seed_nodes = + [cons_node network_config.docker.stack_name "seed" None] + in + let block_producer_nodes = + List.map network_config.docker.block_producer_configs + ~f:(fun bp_config -> + cons_node network_config.docker.stack_name + (network_config.docker.stack_name ^ "_" ^ bp_config.name) + (Some bp_config.keypair) ) + in + let nodes_by_app_id = + let all_nodes = seed_nodes @ block_producer_nodes in + all_nodes + |> List.map ~f:(fun node -> (node.service_id, node)) + |> String.Map.of_alist_exn + in + let t = + { stack_name= network_config.docker.stack_name + ; logger + ; testnet_dir + ; constants= network_config.constants + ; seed_nodes + ; block_producer_nodes + ; nodes_by_app_id + ; deployed= false + ; testnet_log_filter= "" + ; keypairs= + List.map network_config.keypairs ~f:(fun {keypair; _} -> keypair) } + in + Deferred.return t + + let deploy t = + if t.deployed then failwith "network already deployed" ; + [%log' info t.logger] "Deploying network" ; + [%log' info t.logger] "Stack_name in deploy: %s" t.stack_name ; + let%map _ = + run_cmd_exn t "docker" + ["stack"; "deploy"; "-c"; "compose.json"; t.stack_name] + in + t.deployed <- true ; + let result = + { Swarm_network.namespace= t.stack_name + ; constants= t.constants + ; seeds= t.seed_nodes + ; block_producers= t.block_producer_nodes + ; snark_coordinators= [] + ; archive_nodes= [] + ; nodes_by_app_id= t.nodes_by_app_id + ; testnet_log_filter= t.testnet_log_filter + ; keypairs= t.keypairs } + in + let nodes_to_string = + Fn.compose (String.concat ~sep:", ") (List.map ~f:Swarm_network.Node.id) + in + [%log' info t.logger] "Network deployed" ; + [%log' info t.logger] "testnet swarm: %s" t.stack_name ; + [%log' info t.logger] "snark coordinators: %s" + (nodes_to_string result.snark_coordinators) ; + [%log' info t.logger] "block producers: %s" + (nodes_to_string result.block_producers) ; + [%log' info t.logger] "archive nodes: %s" + (nodes_to_string result.archive_nodes) ; + result + + let destroy t = + [%log' info t.logger] "Destroying network" ; + if not t.deployed then failwith "network not deployed" ; + let%bind _ = run_cmd_exn t "docker" ["stack"; "rm"; t.stack_name] in + t.deployed <- false ; + Deferred.unit + + let cleanup t = + let%bind () = if t.deployed then destroy t else return () in + [%log' info t.logger] "Cleaning up network configuration" ; + let%bind () = File_system.remove_dir t.testnet_dir in + Deferred.unit +end diff --git a/src/lib/integration_test_local_engine/swarm_network.ml b/src/lib/integration_test_local_engine/swarm_network.ml new file mode 100644 index 00000000000..9f7767a0c16 --- /dev/null +++ b/src/lib/integration_test_local_engine/swarm_network.ml @@ -0,0 +1,557 @@ +open Core +open Async +open Integration_test_lib + +(* Parts of this implementation is still untouched from the Kubernetes implementation. This is done + just to make the compiler happy as work is done on the local engine. +*) + +(* exclude from bisect_ppx to avoid type error on GraphQL modules *) +[@@@coverage exclude_file] + +module Node = struct + type t = + { swarm_name: string + ; service_id: string + ; graphql_enabled: bool + ; network_keypair: Network_keypair.t option } + + let id {service_id; _} = service_id + + let network_keypair {network_keypair; _} = network_keypair + + let base_kube_args t = ["--cluster"; t.swarm_name] + + let get_container_cmd t = + Printf.sprintf "$(docker ps -f name=%s --quiet)" t.service_id + + let run_in_postgresql_container node ~cmd = + let base_args = base_kube_args node in + let base_kube_cmd = "kubectl " ^ String.concat ~sep:" " base_args in + let kubectl_cmd = + Printf.sprintf "%s -c %s exec -i %s-0 -- %s" base_kube_cmd + node.service_id node.service_id cmd + in + let%bind cwd = Unix.getcwd () in + Util.run_cmd_exn cwd "sh" ["-c"; kubectl_cmd] + + (* + Maybe we get rid of this? We are redirecting all logs to stdout which the test executive will pick up + in a directed pipe + *) + let get_logs_in_container node = + let base_args = base_kube_args node in + let base_kube_cmd = "kubectl " ^ String.concat ~sep:" " base_args in + let pod_cmd = + sprintf "%s get pod -l \"app=%s\" -o name" base_kube_cmd node.service_id + in + let%bind cwd = Unix.getcwd () in + let%bind pod = Util.run_cmd_exn cwd "sh" ["-c"; pod_cmd] in + let kubectl_cmd = + Printf.sprintf "%s logs -c %s -n %s %s" base_kube_cmd node.service_id + node.swarm_name pod + in + Util.run_cmd_exn cwd "sh" ["-c"; kubectl_cmd] + + let run_in_container node cmd = + let base_docker_cmd = "docker exec" in + let docker_cmd = + Printf.sprintf "%s %s %s" base_docker_cmd (get_container_cmd node) cmd + in + let%bind.Deferred.Let_syntax cwd = Unix.getcwd () in + Malleable_error.return (Util.run_cmd_exn cwd "sh" ["-c"; docker_cmd]) + + let start ~fresh_state node : unit Malleable_error.t = + let open Malleable_error.Let_syntax in + let%bind _ = + Deferred.bind ~f:Malleable_error.return (run_in_container node "ps aux") + in + let%bind () = + if fresh_state then + let%bind _ = run_in_container node "rm -rf .mina-config/*" in + Malleable_error.return () + else Malleable_error.return () + in + let%bind _ = run_in_container node "./start.sh" in + Malleable_error.return () + + let stop node = + let open Malleable_error.Let_syntax in + let%bind _ = run_in_container node "ps aux" in + let%bind _ = run_in_container node "./stop.sh" in + let%bind _ = run_in_container node "ps aux" in + return () + + module Decoders = Graphql_lib.Decoders + + module Graphql = struct + let ingress_uri node = + let host = Printf.sprintf "%s.graphql.test.o1test.net" node.swarm_name in + let path = Printf.sprintf "/%s/graphql" node.service_id in + Uri.make ~scheme:"http" ~host ~path ~port:80 () + + module Client = Graphql_lib.Client.Make (struct + let preprocess_variables_string = Fn.id + + let headers = String.Map.empty + end) + + module Unlock_account = + [%graphql + {| + mutation ($password: String!, $public_key: PublicKey!) { + unlockAccount(input: {password: $password, publicKey: $public_key }) { + public_key: publicKey @bsDecoder(fn: "Decoders.public_key") + } + } + |}] + + module Send_payment = + [%graphql + {| + mutation ($sender: PublicKey!, + $receiver: PublicKey!, + $amount: UInt64!, + $token: UInt64, + $fee: UInt64!, + $nonce: UInt32, + $memo: String) { + sendPayment(input: + {from: $sender, to: $receiver, amount: $amount, token: $token, fee: $fee, nonce: $nonce, memo: $memo}) { + payment { + id + } + } + } + |}] + + module Get_balance = + [%graphql + {| + query ($public_key: PublicKey, $token: UInt64) { + account(publicKey: $public_key, token: $token) { + balance { + total @bsDecoder(fn: "Decoders.balance") + } + } + } + |}] + + module Query_peer_id = + [%graphql + {| + query { + daemonStatus { + addrsAndPorts { + peer { + peerId + } + } + peers { peerId } + + } + } + |}] + + module Best_chain = + [%graphql + {| + query { + bestChain { + stateHash + } + } + |}] + end + + (* this function will repeatedly attempt to connect to graphql port times before giving up *) + let exec_graphql_request ?(num_tries = 10) ?(retry_delay_sec = 30.0) + ?(initial_delay_sec = 30.0) ~logger ~node ~query_name query_obj = + let open Deferred.Let_syntax in + if not node.graphql_enabled then + Deferred.Or_error.error_string + "graphql is not enabled (hint: set `requires_graphql= true` in the \ + test config)" + else + let uri = Graphql.ingress_uri node in + let metadata = + [("query", `String query_name); ("uri", `String (Uri.to_string uri))] + in + [%log info] "Attempting to send GraphQL request \"$query\" to \"$uri\"" + ~metadata ; + let rec retry n = + if n <= 0 then ( + [%log error] + "GraphQL request \"$query\" to \"$uri\" failed too many times" + ~metadata ; + Deferred.Or_error.errorf + "GraphQL \"%s\" to \"%s\" request failed too many times" query_name + (Uri.to_string uri) ) + else + match%bind Graphql.Client.query query_obj uri with + | Ok result -> + [%log info] "GraphQL request \"$query\" to \"$uri\" succeeded" + ~metadata ; + Deferred.Or_error.return result + | Error (`Failed_request err_string) -> + [%log warn] + "GraphQL request \"$query\" to \"$uri\" failed: \"$error\" \ + ($num_tries attempts left)" + ~metadata: + ( metadata + @ [("error", `String err_string); ("num_tries", `Int (n - 1))] + ) ; + let%bind () = after (Time.Span.of_sec retry_delay_sec) in + retry (n - 1) + | Error (`Graphql_error err_string) -> + [%log error] + "GraphQL request \"$query\" to \"$uri\" returned an error: \ + \"$error\" (this is a graphql error so not retrying)" + ~metadata:(metadata @ [("error", `String err_string)]) ; + Deferred.Or_error.error_string err_string + in + let%bind () = after (Time.Span.of_sec initial_delay_sec) in + retry num_tries + + let get_peer_id ~logger t = + let open Deferred.Or_error.Let_syntax in + [%log info] "Getting node's peer_id, and the peer_ids of node's peers" + ~metadata: + [("namespace", `String t.swarm_name); ("pod_id", `String t.service_id)] ; + let query_obj = Graphql.Query_peer_id.make () in + let%bind query_result_obj = + exec_graphql_request ~logger ~node:t ~query_name:"query_peer_id" + query_obj + in + [%log info] "get_peer_id, finished exec_graphql_request" ; + let self_id_obj = ((query_result_obj#daemonStatus)#addrsAndPorts)#peer in + let%bind self_id = + match self_id_obj with + | None -> + Deferred.Or_error.error_string "Peer not found" + | Some peer -> + return peer#peerId + in + let peers = (query_result_obj#daemonStatus)#peers |> Array.to_list in + let peer_ids = List.map peers ~f:(fun peer -> peer#peerId) in + [%log info] + "get_peer_id, result of graphql query (self_id,[peers]) (%s,%s)" self_id + (String.concat ~sep:" " peer_ids) ; + return (self_id, peer_ids) + + let must_get_peer_id ~logger t = + get_peer_id ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error + + let get_best_chain ~logger t = + let open Deferred.Or_error.Let_syntax in + let query = Graphql.Best_chain.make () in + let%bind result = + exec_graphql_request ~logger ~node:t ~query_name:"best_chain" query + in + match result#bestChain with + | None | Some [||] -> + Deferred.Or_error.error_string "failed to get best chains" + | Some chain -> + return + @@ List.map ~f:(fun block -> block#stateHash) (Array.to_list chain) + + let must_get_best_chain ~logger t = + get_best_chain ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error + + let get_balance ~logger t ~account_id = + let open Deferred.Or_error.Let_syntax in + [%log info] "Getting account balance" + ~metadata: + [ ("namespace", `String t.swarm_name) + ; ("pod_id", `String t.service_id) + ; ("account_id", Mina_base.Account_id.to_yojson account_id) ] ; + let pk = Mina_base.Account_id.public_key account_id in + let token = Mina_base.Account_id.token_id account_id in + let get_balance_obj = + Graphql.Get_balance.make + ~public_key:(Graphql_lib.Encoders.public_key pk) + ~token:(Graphql_lib.Encoders.token token) + () + in + let%bind balance_obj = + exec_graphql_request ~logger ~node:t ~query_name:"get_balance_graphql" + get_balance_obj + in + match balance_obj#account with + | None -> + Deferred.Or_error.errorf + !"Account with %{sexp:Mina_base.Account_id.t} not found" + account_id + | Some acc -> + return (acc#balance)#total + + let must_get_balance ~logger t ~account_id = + get_balance ~logger t ~account_id + |> Deferred.bind ~f:Malleable_error.or_hard_error + + (* if we expect failure, might want retry_on_graphql_error to be false *) + let send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee = + [%log info] "Sending a payment" + ~metadata: + [("namespace", `String t.swarm_name); ("pod_id", `String t.service_id)] ; + let open Deferred.Or_error.Let_syntax in + let sender_pk_str = + Signature_lib.Public_key.Compressed.to_string sender_pub_key + in + [%log info] "send_payment: unlocking account" + ~metadata:[("sender_pk", `String sender_pk_str)] ; + let unlock_sender_account_graphql () = + let unlock_account_obj = + Graphql.Unlock_account.make ~password:"naughty blue worm" + ~public_key:(Graphql_lib.Encoders.public_key sender_pub_key) + () + in + exec_graphql_request ~logger ~node:t + ~query_name:"unlock_sender_account_graphql" unlock_account_obj + in + let%bind _ = unlock_sender_account_graphql () in + let send_payment_graphql () = + let send_payment_obj = + Graphql.Send_payment.make + ~sender:(Graphql_lib.Encoders.public_key sender_pub_key) + ~receiver:(Graphql_lib.Encoders.public_key receiver_pub_key) + ~amount:(Graphql_lib.Encoders.amount amount) + ~fee:(Graphql_lib.Encoders.fee fee) + () + in + exec_graphql_request ~logger ~node:t ~query_name:"send_payment_graphql" + send_payment_obj + in + let%map sent_payment_obj = send_payment_graphql () in + let (`UserCommand id_obj) = (sent_payment_obj#sendPayment)#payment in + let user_cmd_id = id_obj#id in + [%log info] "Sent payment" + ~metadata:[("user_command_id", `String user_cmd_id)] ; + () + + let must_send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount + ~fee = + send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee + |> Deferred.bind ~f:Malleable_error.or_hard_error + + let dump_archive_data ~logger (t : t) ~data_file = + let open Malleable_error.Let_syntax in + [%log info] "Dumping archive data from (node: %s, container: %s)" + t.service_id t.service_id ; + let%map data = + Deferred.bind ~f:Malleable_error.return + (run_in_postgresql_container t + ~cmd: + "pg_dump --create --no-owner \ + postgres://postgres:foobar@localhost:5432/archive") + in + [%log info] "Dumping archive data to file %s" data_file ; + Out_channel.with_file data_file ~f:(fun out_ch -> + Out_channel.output_string out_ch data ) + + let dump_container_logs ~logger (t : t) ~log_file = + let open Malleable_error.Let_syntax in + [%log info] "Dumping container logs from (node: %s, container: %s)" + t.service_id t.service_id ; + let%map logs = + Deferred.bind ~f:Malleable_error.return (get_logs_in_container t) + in + [%log info] "Dumping container log to file %s" log_file ; + Out_channel.with_file log_file ~f:(fun out_ch -> + Out_channel.output_string out_ch logs ) + + let dump_precomputed_blocks ~logger (t : t) = + let open Malleable_error.Let_syntax in + [%log info] + "Dumping precomputed blocks from logs for (node: %s, container: %s)" + t.service_id t.service_id ; + let%bind logs = + Deferred.bind ~f:Malleable_error.return (get_logs_in_container t) + in + (* kubectl logs may include non-log output, like "Using password from environment variable" *) + let log_lines = + String.split logs ~on:'\n' + |> List.filter ~f:(String.is_prefix ~prefix:"{\"timestamp\":") + in + let jsons = List.map log_lines ~f:Yojson.Safe.from_string in + let metadata_jsons = + List.map jsons ~f:(fun json -> + match json with + | `Assoc items -> ( + match List.Assoc.find items ~equal:String.equal "metadata" with + | Some md -> + md + | None -> + failwithf "Log line is missing metadata: %s" + (Yojson.Safe.to_string json) + () ) + | other -> + failwithf "Expected log line to be a JSON record, got: %s" + (Yojson.Safe.to_string other) + () ) + in + let state_hash_and_blocks = + List.fold metadata_jsons ~init:[] ~f:(fun acc json -> + match json with + | `Assoc items -> ( + match + List.Assoc.find items ~equal:String.equal "precomputed_block" + with + | Some block -> ( + match List.Assoc.find items ~equal:String.equal "state_hash" with + | Some state_hash -> + (state_hash, block) :: acc + | None -> + failwith + "Log metadata contains a precomputed block, but no state \ + hash" ) + | None -> + acc ) + | other -> + failwithf "Expected log line to be a JSON record, got: %s" + (Yojson.Safe.to_string other) + () ) + in + let%bind.Deferred.Let_syntax () = + Deferred.List.iter state_hash_and_blocks + ~f:(fun (state_hash_json, block_json) -> + let double_quoted_state_hash = + Yojson.Safe.to_string state_hash_json + in + let state_hash = + String.sub double_quoted_state_hash ~pos:1 + ~len:(String.length double_quoted_state_hash - 2) + in + let block = Yojson.Safe.pretty_to_string block_json in + let filename = state_hash ^ ".json" in + match%map.Deferred.Let_syntax Sys.file_exists filename with + | `Yes -> + [%log info] + "File already exists for precomputed block with state hash %s" + state_hash + | _ -> + [%log info] + "Dumping precomputed block with state hash %s to file %s" + state_hash filename ; + Out_channel.with_file filename ~f:(fun out_ch -> + Out_channel.output_string out_ch block ) ) + in + Malleable_error.return () +end + +type t = + { namespace: string + ; constants: Test_config.constants + ; seeds: Node.t list + ; block_producers: Node.t list + ; snark_coordinators: Node.t list + ; archive_nodes: Node.t list + ; testnet_log_filter: string + ; keypairs: Signature_lib.Keypair.t list + ; nodes_by_app_id: Node.t String.Map.t } + +let constants {constants; _} = constants + +let constraint_constants {constants; _} = constants.constraints + +let genesis_constants {constants; _} = constants.genesis + +let seeds {seeds; _} = seeds + +let block_producers {block_producers; _} = block_producers + +let snark_coordinators {snark_coordinators; _} = snark_coordinators + +let archive_nodes {archive_nodes; _} = archive_nodes + +let keypairs {keypairs; _} = keypairs + +let all_nodes {seeds; block_producers; snark_coordinators; archive_nodes; _} = + List.concat [seeds; block_producers; snark_coordinators; archive_nodes] + +let lookup_node_by_app_id t = Map.find t.nodes_by_app_id + +let initialize ~logger network = + Print.print_endline "initialize" ; + let open Malleable_error.Let_syntax in + let poll_interval = Time.Span.of_sec 15.0 in + let max_polls = 60 (* 15 mins *) in + let all_services = + all_nodes network + |> List.map ~f:(fun {service_id; _} -> service_id) + |> String.Set.of_list + in + let get_service_statuses () = + let%map output = + Deferred.bind ~f:Malleable_error.return + (Util.run_cmd_exn "/" "docker" + ["service"; "ls"; "--format"; "{{.Name}}: {{.Replicas}}"]) + in + output |> String.split_lines + |> List.map ~f:(fun line -> + let parts = String.split line ~on:':' in + assert (List.length parts = 2) ; + (List.nth_exn parts 0, List.nth_exn parts 1) ) + |> List.filter ~f:(fun (service_name, _) -> + String.Set.mem all_services service_name ) + in + let rec poll n = + let%bind pod_statuses = get_service_statuses () in + (* TODO: detect "bad statuses" (eg CrashLoopBackoff) and terminate early *) + let bad_service_statuses = + List.filter pod_statuses ~f:(fun (_, status) -> + let parts = String.split status ~on:'/' in + assert (List.length parts = 2) ; + let num, denom = (List.nth_exn parts 0, List.nth_exn parts 1) in + String.strip num <> String.strip denom ) + in + if List.is_empty bad_service_statuses then return () + else if n < max_polls then + let%bind () = + after poll_interval |> Deferred.bind ~f:Malleable_error.return + in + poll (n + 1) + else + let bad_pod_statuses_json = + `List + (List.map bad_service_statuses ~f:(fun (service_name, status) -> + `Assoc + [ ("service_name", `String service_name) + ; ("status", `String status) ] )) + in + [%log fatal] + "Not all pods were assigned to nodes and ready in time: \ + $bad_pod_statuses" + ~metadata:[("bad_pod_statuses", bad_pod_statuses_json)] ; + Malleable_error.hard_error_format + "Some pods either were not assigned to nodes or did deploy properly \ + (errors: %s)" + (Yojson.Safe.to_string bad_pod_statuses_json) + in + [%log info] "Waiting for pods to be assigned nodes and become ready" ; + Deferred.bind (poll 0) ~f:(fun res -> + if Malleable_error.is_ok res then + let seed_nodes = seeds network in + let seed_pod_ids = + seed_nodes + |> List.map ~f:(fun {Node.service_id; _} -> service_id) + |> String.Set.of_list + in + let non_seed_nodes = + network |> all_nodes + |> List.filter ~f:(fun {Node.service_id; _} -> + not (String.Set.mem seed_pod_ids service_id) ) + in + (* TODO: parallelize (requires accumlative hard errors) *) + let%bind () = + Malleable_error.List.iter seed_nodes + ~f:(Node.start ~fresh_state:false) + in + (* put a short delay before starting other nodes, to help avoid artifact generation races *) + let%bind () = + after (Time.Span.of_sec 30.0) + |> Deferred.bind ~f:Malleable_error.return + in + Malleable_error.List.iter non_seed_nodes + ~f:(Node.start ~fresh_state:false) + else Deferred.return res ) From 5c45ec4213e86e28146855cac787f65b08ccbc91 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Fri, 11 Jun 2021 10:17:13 -0700 Subject: [PATCH 08/25] Implemented support for snark coordinator and workers Adds support for snark coordinator and workers based off the test configurations passed into the test executive. Right now, we only implement 1 snark coordinator for all other snark workers. Additionally added a new file for all the default Mina node commands and environment variables. --- .../docker_compose.ml | 10 +- .../mina_automation.ml | 183 +++++++----------- .../node_constants.ml | 98 ++++++++++ .../swarm_network.ml | 14 +- 4 files changed, 187 insertions(+), 118 deletions(-) create mode 100644 src/lib/integration_test_local_engine/node_constants.ml diff --git a/src/lib/integration_test_local_engine/docker_compose.ml b/src/lib/integration_test_local_engine/docker_compose.ml index 26d1a90dfbf..fe12f239fcc 100644 --- a/src/lib/integration_test_local_engine/docker_compose.ml +++ b/src/lib/integration_test_local_engine/docker_compose.ml @@ -10,7 +10,6 @@ module Compose = struct module Service = struct module Volume = struct type t = {type_: string; source: string; target: string} - [@@deriving to_yojson] let create name = {type_= "bind"; source= "." ^/ name; target= "/root" ^/ name} @@ -37,12 +36,9 @@ module Compose = struct `Assoc (m |> Map.map ~f:(fun x -> `String x) |> Map.to_alist) end - type replicas = {replicas: int} [@@deriving to_yojson] - type t = { image: string ; volumes: Volume.t list - ; deploy: replicas ; command: string list ; environment: Environment.t } [@@deriving to_yojson] @@ -50,6 +46,12 @@ module Compose = struct type service_map = Service.t DockerMap.t + (* Used to combine different type of service maps. There is an assumption that these maps + are disjoint and will not conflict *) + let merge_maps (map_a : service_map) (map_b : service_map) = + Map.fold map_b ~init:map_a ~f:(fun ~key ~data acc -> + Map.update acc key ~f:(function None -> data | Some data' -> data') ) + let service_map_to_yojson m = `Assoc (m |> Map.map ~f:Service.to_yojson |> Map.to_alist) diff --git a/src/lib/integration_test_local_engine/mina_automation.ml b/src/lib/integration_test_local_engine/mina_automation.ml index 86c557bda7b..0b851d36941 100644 --- a/src/lib/integration_test_local_engine/mina_automation.ml +++ b/src/lib/integration_test_local_engine/mina_automation.ml @@ -23,18 +23,20 @@ module Network_config = struct ; libp2p_secret: string } [@@deriving to_yojson] + type snark_coordinator_configs = + {name: string; id: string; public_key: string; snark_worker_fee: string} + [@@deriving to_yojson] + type docker_config = { version: string ; stack_name: string ; coda_image: string ; docker_volume_configs: docker_volume_configs list ; block_producer_configs: block_producer_config list + ; snark_coordinator_configs: snark_coordinator_configs list ; log_precomputed_blocks: bool ; archive_node_count: int ; mina_archive_schema: string - ; snark_worker_replicas: int - ; snark_worker_fee: string - ; snark_worker_public_key: string ; runtime_config: Yojson.Safe.t [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] } [@@deriving to_yojson] @@ -186,6 +188,23 @@ module Network_config = struct let block_producer_configs = List.mapi block_producer_keypairs ~f:block_producer_config in + let snark_coordinator_configs = + if num_snark_workers > 0 then + List.mapi + (List.init num_snark_workers ~f:(const 0)) + ~f:(fun index _ -> + { name= "test-snark-worker-" ^ Int.to_string (index + 1) + ; id= Int.to_string index + ; snark_worker_fee + ; public_key= snark_worker_public_key } ) + (* Add one snark coordinator for all workers *) + |> List.append + [ { name= "test-snark-coordinator" + ; id= "1" + ; snark_worker_fee + ; public_key= snark_worker_public_key } ] + else [] + in (* Combine configs for block producer configs and runtime config to be a docker bind volume *) let docker_volume_configs = List.map block_producer_configs ~f:(fun config -> @@ -209,98 +228,16 @@ module Network_config = struct ; docker_volume_configs ; runtime_config= Runtime_config.to_yojson runtime_config ; block_producer_configs + ; snark_coordinator_configs ; log_precomputed_blocks ; archive_node_count= num_archive_nodes - ; mina_archive_schema - ; snark_worker_replicas= num_snark_workers - ; snark_worker_public_key - ; snark_worker_fee } } + ; mina_archive_schema } } let to_docker network_config = - let default_seed_envs = - [ ("DAEMON_REST_PORT", "3085") - ; ("DAEMON_CLIENT_PORT", "8301") - ; ("DAEMON_METRICS_PORT", "10001") - ; ("CODA_LIBP2P_PASS", "") ] - in - let default_block_producer_envs = - [ ("DAEMON_REST_PORT", "3085") - ; ("DAEMON_CLIENT_PORT", "8301") - ; ("DAEMON_METRICS_PORT", "10001") - ; ("CODA_PRIVKEY_PASS", "naughty blue worm") - ; ("CODA_LIBP2P_PASS", "") ] - in - let default_seed_command ~runtime_config = - [ "daemon" - ; "-log-level" - ; "Debug" - ; "-log-json" - ; "-log-snark-work-gossip" - ; "true" - ; "-client-port" - ; "8301" - ; "-generate-genesis-proof" - ; "true" - ; "-peer" - ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - ; "-seed" - ; "-config-file" - ; runtime_config ] - in - let default_block_producer_command ~runtime_config ~private_key_config = - [ "daemon" - ; "-log-level" - ; "Debug" - ; "-log-json" - ; "-log-snark-work-gossip" - ; "true" - ; "-log-txn-pool-gossip" - ; "true" - ; "-enable-peer-exchange" - ; "true" - ; "-enable-flooding" - ; "true" - ; "-peer" - ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - ; "-client-port" - ; "8301" - ; "-generate-genesis-proof" - ; "true" - ; "-block-producer-key" - ; private_key_config - ; "-config-file" - ; runtime_config ] - in - let default_snark_coord_command ~runtime_config ~snark_coordinator_key - ~snark_worker_fee = - [ "daemon" - ; "-log-level" - ; "Debug" - ; "-log-json" - ; "-log-snark-work-gossip" - ; "true" - ; "-log-txn-pool-gossip" - ; "true" - ; "-external-port" - ; "10909" - ; "-rest-port" - ; "3085" - ; "-client-port" - ; "8301" - ; "-work-selection" - ; "seq" - ; "-peer" - ; "/dns4/mina_seed-node/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - ; "-run-snark-coordinator" - ; snark_coordinator_key - ; "-snark-worker-fee" - ; snark_worker_fee - ; "-config-file" - ; runtime_config ] - in let open Docker_compose.Compose in + let open Node_constants in let runtime_config = Service.Volume.create "runtime_config" in - let service_map = + let blocks_seed_map = List.fold network_config.docker.block_producer_configs ~init:DockerMap.empty ~f:(fun accum config -> let private_key_config = @@ -310,39 +247,57 @@ module Network_config = struct ~data: { Service.image= network_config.docker.coda_image ; volumes= [private_key_config; runtime_config] - ; deploy= {replicas= 1} ; command= default_block_producer_command ~runtime_config:runtime_config.target ~private_key_config:private_key_config.target ; environment= Service.Environment.create default_block_producer_envs } ) + (* Add a seed node to the map as well*) |> Map.set ~key:"seed" ~data: { Service.image= network_config.docker.coda_image ; volumes= [runtime_config] - ; deploy= {replicas= 1} ; command= default_seed_command ~runtime_config:runtime_config.target ; environment= Service.Environment.create default_seed_envs } in - let service_map = - if network_config.docker.snark_worker_replicas > 0 then - Map.set service_map ~key:"snark_worker" - ~data: - { Service.image= network_config.docker.coda_image - ; volumes= [runtime_config] - ; deploy= {replicas= network_config.docker.snark_worker_replicas} - ; command= - default_snark_coord_command - ~runtime_config:runtime_config.target - ~snark_coordinator_key: - network_config.docker.snark_worker_public_key - ~snark_worker_fee:network_config.docker.snark_worker_fee - ; environment= Service.Environment.create default_seed_envs } - else service_map + let snark_worker_map = + List.fold network_config.docker.snark_coordinator_configs + ~init:DockerMap.empty ~f:(fun accum config -> + let command, environment = + match String.substr_index config.name ~pattern:"coordinator" with + | Some _ -> + let coordinator_command = + default_snark_coord_command + ~runtime_config:runtime_config.target + ~snark_coordinator_key:config.public_key + ~snark_worker_fee:config.snark_worker_fee + in + let coordinator_environment = + Service.Environment.create + (default_snark_coord_envs + ~snark_coordinator_key:config.public_key + ~snark_worker_fee:config.snark_worker_fee) + in + (coordinator_command, coordinator_environment) + | None -> + let worker_command = + default_snark_worker_command + ~daemon_address:"test-snark-coordinator" + in + let worker_environment = Service.Environment.create [] in + (worker_command, worker_environment) + in + Map.set accum ~key:config.name + ~data: + { Service.image= network_config.docker.coda_image + ; volumes= [runtime_config] + ; command + ; environment } ) in - {version; services= service_map} + let services = merge_maps blocks_seed_map snark_worker_map in + {version; services} end module Network_manager = struct @@ -355,6 +310,7 @@ module Network_manager = struct ; seed_nodes: Swarm_network.Node.t list ; nodes_by_app_id: Swarm_network.Node.t String.Map.t ; block_producer_nodes: Swarm_network.Node.t list + ; snark_coordinator_configs: Swarm_network.Node.t list ; mutable deployed: bool ; keypairs: Keypair.t list } @@ -428,6 +384,13 @@ module Network_manager = struct (network_config.docker.stack_name ^ "_" ^ bp_config.name) (Some bp_config.keypair) ) in + let snark_coordinator_configs = + List.map network_config.docker.snark_coordinator_configs + ~f:(fun snark_config -> + cons_node network_config.docker.stack_name + (network_config.docker.stack_name ^ "_" ^ snark_config.name) + None ) + in let nodes_by_app_id = let all_nodes = seed_nodes @ block_producer_nodes in all_nodes @@ -441,6 +404,7 @@ module Network_manager = struct ; constants= network_config.constants ; seed_nodes ; block_producer_nodes + ; snark_coordinator_configs ; nodes_by_app_id ; deployed= false ; testnet_log_filter= "" @@ -453,17 +417,18 @@ module Network_manager = struct if t.deployed then failwith "network already deployed" ; [%log' info t.logger] "Deploying network" ; [%log' info t.logger] "Stack_name in deploy: %s" t.stack_name ; - let%map _ = + let%map s = run_cmd_exn t "docker" ["stack"; "deploy"; "-c"; "compose.json"; t.stack_name] in + [%log' info t.logger] "DEPLOYED STRING: %s" s ; t.deployed <- true ; let result = { Swarm_network.namespace= t.stack_name ; constants= t.constants ; seeds= t.seed_nodes ; block_producers= t.block_producer_nodes - ; snark_coordinators= [] + ; snark_coordinators= t.snark_coordinator_configs ; archive_nodes= [] ; nodes_by_app_id= t.nodes_by_app_id ; testnet_log_filter= t.testnet_log_filter diff --git a/src/lib/integration_test_local_engine/node_constants.ml b/src/lib/integration_test_local_engine/node_constants.ml new file mode 100644 index 00000000000..3d6cd18759e --- /dev/null +++ b/src/lib/integration_test_local_engine/node_constants.ml @@ -0,0 +1,98 @@ +let default_seed_envs = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_LIBP2P_PASS", "") ] + +let default_block_producer_envs = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_PRIVKEY_PASS", "naughty blue worm") + ; ("CODA_LIBP2P_PASS", "") ] + +let default_snark_coord_envs ~snark_coordinator_key ~snark_worker_fee = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_SNARK_KEY", snark_coordinator_key) + ; ("CODA_SNARK_FEE", snark_worker_fee) + ; ("WORK_SELECTION", "seq") + ; ("CODA_PRIVKEY_PASS", "naughty blue worm") + ; ("CODA_LIBP2P_PASS", "") ] + +let default_seed_command ~runtime_config = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-client-port" + ; "8301" + ; "-generate-genesis-proof" + ; "true" + ; "-peer" + ; "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-seed" + ; "-config-file" + ; runtime_config ] + +let default_block_producer_command ~runtime_config ~private_key_config = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-log-txn-pool-gossip" + ; "true" + ; "-enable-peer-exchange" + ; "true" + ; "-enable-flooding" + ; "true" + ; "-peer" + ; "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-client-port" + ; "8301" + ; "-generate-genesis-proof" + ; "true" + ; "-block-producer-key" + ; private_key_config + ; "-config-file" + ; runtime_config ] + +let default_snark_coord_command ~runtime_config ~snark_coordinator_key + ~snark_worker_fee = + [ "daemon" + ; "-log-level" + ; "Debug" + ; "-log-json" + ; "-log-snark-work-gossip" + ; "true" + ; "-log-txn-pool-gossip" + ; "true" + ; "-external-port" + ; "10909" + ; "-rest-port" + ; "3085" + ; "-client-port" + ; "8301" + ; "-work-selection" + ; "seq" + ; "-peer" + ; "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + ; "-run-snark-coordinator" + ; snark_coordinator_key + ; "-snark-worker-fee" + ; snark_worker_fee + ; "-config-file" + ; runtime_config ] + +let default_snark_worker_command ~daemon_address = + [ "internal" + ; "snark-worker" + ; "-proof-level" + ; "full" + ; "-daemon-address" + ; daemon_address ^ ":8301" ] diff --git a/src/lib/integration_test_local_engine/swarm_network.ml b/src/lib/integration_test_local_engine/swarm_network.ml index 9f7767a0c16..e7eaf1997e1 100644 --- a/src/lib/integration_test_local_engine/swarm_network.ml +++ b/src/lib/integration_test_local_engine/swarm_network.ml @@ -35,10 +35,6 @@ module Node = struct let%bind cwd = Unix.getcwd () in Util.run_cmd_exn cwd "sh" ["-c"; kubectl_cmd] - (* - Maybe we get rid of this? We are redirecting all logs to stdout which the test executive will pick up - in a directed pipe - *) let get_logs_in_container node = let base_args = base_kube_args node in let base_kube_cmd = "kubectl " ^ String.concat ~sep:" " base_args in @@ -72,7 +68,15 @@ module Node = struct Malleable_error.return () else Malleable_error.return () in - let%bind _ = run_in_container node "./start.sh" in + let cmd = + match String.substr_index node.service_id ~pattern:"snark-worker" with + | Some _ -> + (* Snark-workers should wait for work to be generated so they don't error a 'get_work' RPC call*) + "/bin/bash -c 'sleep 120 && ./start.sh'" + | None -> + "./start.sh" + in + let%bind _ = run_in_container node cmd in Malleable_error.return () let stop node = From 0707643c332717644b55d2328635c1ca823924ba Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Fri, 18 Jun 2021 14:33:24 -0700 Subject: [PATCH 09/25] Local Integration work refactor and feedback Addressed the feedback given in the Network_Config PR. Made the docker_compose.ml file more simple to read with additional refactors to mina_docker.ml. Additionally removed a lot of the copied over code in swarm_network to clean up the file. --- .../cli_inputs.ml | 18 +- .../docker_compose.ml | 48 +- .../docker_pipe_log_engine.ml | 13 +- .../docker_pipe_log_engine.mli | 2 +- .../integration_test_local_engine.ml | 4 +- .../{mina_automation.ml => mina_docker.ml} | 414 ++++++++------- .../node_config.ml | 183 +++++++ .../node_constants.ml | 98 ---- .../swarm_network.ml | 492 +++--------------- 9 files changed, 511 insertions(+), 761 deletions(-) rename src/lib/integration_test_local_engine/{mina_automation.ml => mina_docker.ml} (52%) create mode 100644 src/lib/integration_test_local_engine/node_config.ml delete mode 100644 src/lib/integration_test_local_engine/node_constants.ml diff --git a/src/lib/integration_test_local_engine/cli_inputs.ml b/src/lib/integration_test_local_engine/cli_inputs.ml index 460cf410b70..b0382124fa8 100644 --- a/src/lib/integration_test_local_engine/cli_inputs.ml +++ b/src/lib/integration_test_local_engine/cli_inputs.ml @@ -1,19 +1,5 @@ open Cmdliner -type t = {coda_automation_location: string} +type t = unit -let term = - let coda_automation_location = - let doc = - "Location of the coda automation repository to use when deploying the \ - network." - in - let env = Arg.env_var "CODA_AUTOMATION_LOCATION" ~doc in - Arg.( - value & opt string "./automation" - & info - ["coda-automation-location"] - ~env ~docv:"CODA_AUTOMATION_LOCATION" ~doc) - in - let cons_inputs coda_automation_location = {coda_automation_location} in - Term.(const cons_inputs $ coda_automation_location) +let term = Term.const () diff --git a/src/lib/integration_test_local_engine/docker_compose.ml b/src/lib/integration_test_local_engine/docker_compose.ml index fe12f239fcc..ff0c779196f 100644 --- a/src/lib/integration_test_local_engine/docker_compose.ml +++ b/src/lib/integration_test_local_engine/docker_compose.ml @@ -1,61 +1,45 @@ open Core module Compose = struct - module DockerMap = struct - type 'a t = (string, 'a, String.comparator_witness) Map.t - - let empty = Map.empty (module String) - end + module StringMap = Map.Make (String) module Service = struct module Volume = struct - type t = {type_: string; source: string; target: string} + type t = + { type_ : string [@key "type"]; source : string; target : string } + [@@deriving to_yojson] let create name = - {type_= "bind"; source= "." ^/ name; target= "/root" ^/ name} - - let to_yojson {type_; source; target} = - let field k v = (k, `String v) in - let fields = - [ type_ |> field "type" - ; source |> field "source" - ; target |> field "target" ] - in - `Assoc fields + { type_ = "bind"; source = "." ^/ name; target = "/root" ^/ name } end module Environment = struct - type t = string DockerMap.t + type t = string StringMap.t - let create = - List.fold ~init:DockerMap.empty ~f:(fun accum env -> - let key, data = env in - Map.set accum ~key ~data ) + let create = StringMap.of_alist_exn let to_yojson m = `Assoc (m |> Map.map ~f:(fun x -> `String x) |> Map.to_alist) end type t = - { image: string - ; volumes: Volume.t list - ; command: string list - ; environment: Environment.t } + { image : string + ; volumes : Volume.t list + ; command : string list + ; environment : Environment.t + } [@@deriving to_yojson] end - type service_map = Service.t DockerMap.t + type service_map = Service.t StringMap.t - (* Used to combine different type of service maps. There is an assumption that these maps - are disjoint and will not conflict *) - let merge_maps (map_a : service_map) (map_b : service_map) = - Map.fold map_b ~init:map_a ~f:(fun ~key ~data acc -> - Map.update acc key ~f:(function None -> data | Some data' -> data') ) + let merge (m1 : service_map) (m2 : service_map) = + Base.Map.merge_skewed m1 m2 ~combine:(fun ~key:_ left _ -> left) let service_map_to_yojson m = `Assoc (m |> Map.map ~f:Service.to_yojson |> Map.to_alist) - type t = {version: string; services: service_map} [@@deriving to_yojson] + type t = { version : string; services : service_map } [@@deriving to_yojson] end type t = Compose.t [@@deriving to_yojson] diff --git a/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml b/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml index 88ffc83008c..910eb683b16 100644 --- a/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml +++ b/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml @@ -7,19 +7,20 @@ module Node = Swarm_network.Node (* TODO: Implement local engine logging *) type t = - { logger: Logger.t - ; event_writer: (Node.t * Event_type.event) Pipe.Writer.t - ; event_reader: (Node.t * Event_type.event) Pipe.Reader.t } + { logger : Logger.t + ; event_writer : (Node.t * Event_type.event) Pipe.Writer.t + ; event_reader : (Node.t * Event_type.event) Pipe.Reader.t + } -let event_reader {event_reader; _} = event_reader +let event_reader { event_reader; _ } = event_reader let create ~logger ~(network : Swarm_network.t) = [%log info] "docker_pipe_log_engine: create %s" network.namespace ; let event_reader, event_writer = Pipe.create () in - Deferred.Or_error.return {logger; event_reader; event_writer} + Deferred.Or_error.return { logger; event_reader; event_writer } let destroy t : unit Deferred.Or_error.t = - let {logger; event_reader= _; event_writer} = t in + let { logger; event_reader = _; event_writer } = t in Pipe.close event_writer ; [%log debug] "subscription deleted" ; Deferred.Or_error.error_string "subscription deleted" diff --git a/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli b/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli index fc1ebc65ce4..6398559beff 100644 --- a/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli +++ b/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli @@ -1,3 +1,3 @@ include Integration_test_lib.Intf.Engine.Log_engine_intf - with module Network := Swarm_network + with module Network := Swarm_network diff --git a/src/lib/integration_test_local_engine/integration_test_local_engine.ml b/src/lib/integration_test_local_engine/integration_test_local_engine.ml index 0f623f5c010..6647db9f725 100644 --- a/src/lib/integration_test_local_engine/integration_test_local_engine.ml +++ b/src/lib/integration_test_local_engine/integration_test_local_engine.ml @@ -1,6 +1,6 @@ let name = "local" module Network = Swarm_network -module Network_config = Mina_automation.Network_config -module Network_manager = Mina_automation.Network_manager +module Network_config = Mina_docker.Network_config +module Network_manager = Mina_docker.Network_manager module Log_engine = Docker_pipe_log_engine diff --git a/src/lib/integration_test_local_engine/mina_automation.ml b/src/lib/integration_test_local_engine/mina_docker.ml similarity index 52% rename from src/lib/integration_test_local_engine/mina_automation.ml rename to src/lib/integration_test_local_engine/mina_docker.ml index 0b851d36941..d5f61a62792 100644 --- a/src/lib/integration_test_local_engine/mina_automation.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -5,51 +5,57 @@ open Signature_lib open Mina_base open Integration_test_lib -let version = "3.9" +let docker_swarm_version = "3.9" module Network_config = struct module Cli_inputs = Cli_inputs - type docker_volume_configs = {name: string; data: string} + type docker_volume_configs = { name : string; data : string } [@@deriving to_yojson] type block_producer_config = - { name: string - ; id: string - ; keypair: Network_keypair.t - ; public_key: string - ; private_key: string - ; keypair_secret: string - ; libp2p_secret: string } + { name : string + ; id : string + ; keypair : Network_keypair.t + ; public_key : string + ; private_key : string + ; keypair_secret : string + ; libp2p_secret : string + } [@@deriving to_yojson] type snark_coordinator_configs = - {name: string; id: string; public_key: string; snark_worker_fee: string} + { name : string + ; id : string + ; public_key : string + ; snark_worker_fee : string + } [@@deriving to_yojson] type docker_config = - { version: string - ; stack_name: string - ; coda_image: string - ; docker_volume_configs: docker_volume_configs list - ; block_producer_configs: block_producer_config list - ; snark_coordinator_configs: snark_coordinator_configs list - ; log_precomputed_blocks: bool - ; archive_node_count: int - ; mina_archive_schema: string - ; runtime_config: Yojson.Safe.t - [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] } + { docker_swarm_version : string + ; stack_name : string + ; coda_image : string + ; docker_volume_configs : docker_volume_configs list + ; block_producer_configs : block_producer_config list + ; snark_coordinator_configs : snark_coordinator_configs list + ; log_precomputed_blocks : bool + ; archive_node_count : int + ; mina_archive_schema : string + ; runtime_config : Yojson.Safe.t + [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] + } [@@deriving to_yojson] type t = - { mina_automation_location: string - ; keypairs: Network_keypair.t list - ; debug_arg: bool - ; constants: Test_config.constants - ; docker: docker_config } + { keypairs : Network_keypair.t list + ; debug_arg : bool + ; constants : Test_config.constants + ; docker : docker_config + } [@@deriving to_yojson] - let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) + let expand ~logger ~test_name ~cli_inputs:_ ~(debug : bool) ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = let { Test_config.k @@ -58,13 +64,14 @@ module Network_config = struct ; slots_per_sub_window ; proof_level ; txpool_max_size - ; requires_graphql= _ + ; requires_graphql = _ ; block_producers ; num_snark_workers ; num_archive_nodes ; log_precomputed_blocks ; snark_worker_fee - ; snark_worker_public_key } = + ; snark_worker_public_key + } = test_config in let user_from_env = Option.value (Unix.getenv "USER") ~default:"auto" in @@ -86,7 +93,7 @@ module Network_config = struct if num_block_producers > List.length keypairs then failwith "not enough sample keypairs for specified number of block producers" ; - let f index ({Test_config.Block_producer.balance; timing}, (pk, sk)) = + let f index ({ Test_config.Block_producer.balance; timing }, (pk, sk)) = let runtime_account = let timing = match timing with @@ -94,26 +101,30 @@ module Network_config = struct None | Timed t -> Some - { Runtime_config.Accounts.Single.Timed.initial_minimum_balance= + { Runtime_config.Accounts.Single.Timed.initial_minimum_balance = t.initial_minimum_balance - ; cliff_time= t.cliff_time - ; cliff_amount= t.cliff_amount - ; vesting_period= t.vesting_period - ; vesting_increment= t.vesting_increment } + ; cliff_time = t.cliff_time + ; cliff_amount = t.cliff_amount + ; vesting_period = t.vesting_period + ; vesting_increment = t.vesting_increment + } in let default = Runtime_config.Accounts.Single.default in { default with - pk= Some (Public_key.Compressed.to_string pk) - ; sk= None - ; balance= + pk = Some (Public_key.Compressed.to_string pk) + ; sk = None + ; balance = Balance.of_formatted_string balance (* delegation currently unsupported *) - ; delegate= None - ; timing } + ; delegate = None + ; timing + } in let secret_name = "test-keypair-" ^ Int.to_string index in let keypair = - {Keypair.public_key= Public_key.decompress_exn pk; private_key= sk} + { Keypair.public_key = Public_key.decompress_exn pk + ; private_key = sk + } in ( Network_keypair.create_network_keypair ~keypair ~secret_name , runtime_account ) @@ -126,46 +137,48 @@ module Network_config = struct (* DAEMON CONFIG *) let proof_config = (* TODO: lift configuration of these up Test_config.t *) - { Runtime_config.Proof_keys.level= Some proof_level - ; sub_windows_per_window= None - ; ledger_depth= None - ; work_delay= None - ; block_window_duration_ms= None - ; transaction_capacity= None - ; coinbase_amount= None - ; supercharged_coinbase_factor= None - ; account_creation_fee= None - ; fork= None } + { Runtime_config.Proof_keys.level = Some proof_level + ; sub_windows_per_window = None + ; ledger_depth = None + ; work_delay = None + ; block_window_duration_ms = None + ; transaction_capacity = None + ; coinbase_amount = None + ; supercharged_coinbase_factor = None + ; account_creation_fee = None + ; fork = None + } in let constraint_constants = Genesis_ledger_helper.make_constraint_constants ~default:Genesis_constants.Constraint_constants.compiled proof_config in let runtime_config = - { Runtime_config.daemon= - Some {txpool_max_size= Some txpool_max_size; peer_list_url= None} - ; genesis= + { Runtime_config.daemon = + Some { txpool_max_size = Some txpool_max_size; peer_list_url = None } + ; genesis = Some - { k= Some k - ; delta= Some delta - ; slots_per_epoch= Some slots_per_epoch - ; sub_windows_per_window= - Some constraint_constants.supercharged_coinbase_factor - ; slots_per_sub_window= Some slots_per_sub_window - ; genesis_state_timestamp= - Some Core.Time.(to_string_abs ~zone:Zone.utc (now ())) } - ; proof= + { k = Some k + ; delta = Some delta + ; slots_per_epoch = Some slots_per_epoch + ; slots_per_sub_window = Some slots_per_sub_window + ; genesis_state_timestamp = + Some Core.Time.(to_string_abs ~zone:Zone.utc (now ())) + } + ; proof = None (* was: Some proof_config; TODO: prebake ledger and only set hash *) - ; ledger= + ; ledger = Some - { base= Accounts runtime_accounts - ; add_genesis_winner= None - ; num_accounts= None - ; balances= [] - ; hash= None - ; name= None } - ; epoch_data= None } + { base = Accounts runtime_accounts + ; add_genesis_winner = None + ; num_accounts = None + ; balances = [] + ; hash = None + ; name = None + } + ; epoch_data = None + } in let genesis_constants = Or_error.ok_exn @@ -173,17 +186,18 @@ module Network_config = struct ~default:Genesis_constants.compiled runtime_config) in let constants : Test_config.constants = - {constraints= constraint_constants; genesis= genesis_constants} + { constraints = constraint_constants; genesis = genesis_constants } in (* BLOCK PRODUCER CONFIG *) let block_producer_config index keypair = - { name= "test-block-producer-" ^ Int.to_string (index + 1) - ; id= Int.to_string index + { name = "test-block-producer-" ^ Int.to_string (index + 1) + ; id = Int.to_string index ; keypair - ; keypair_secret= keypair.secret_name - ; public_key= keypair.public_key_file - ; private_key= keypair.private_key_file - ; libp2p_secret= "" } + ; keypair_secret = keypair.secret_name + ; public_key = keypair.public_key_file + ; private_key = keypair.private_key_file + ; libp2p_secret = "" + } in let block_producer_configs = List.mapi block_producer_keypairs ~f:block_producer_config @@ -193,126 +207,145 @@ module Network_config = struct List.mapi (List.init num_snark_workers ~f:(const 0)) ~f:(fun index _ -> - { name= "test-snark-worker-" ^ Int.to_string (index + 1) - ; id= Int.to_string index + { name = "test-snark-worker-" ^ Int.to_string (index + 1) + ; id = Int.to_string index ; snark_worker_fee - ; public_key= snark_worker_public_key } ) + ; public_key = snark_worker_public_key + }) (* Add one snark coordinator for all workers *) |> List.append - [ { name= "test-snark-coordinator" - ; id= "1" + [ { name = "test-snark-coordinator" + ; id = "1" ; snark_worker_fee - ; public_key= snark_worker_public_key } ] + ; public_key = snark_worker_public_key + } + ] else [] in (* Combine configs for block producer configs and runtime config to be a docker bind volume *) let docker_volume_configs = List.map block_producer_configs ~f:(fun config -> - {name= "sk_" ^ config.name; data= config.private_key} ) - @ [ { name= "runtime_config" - ; data= + { name = "sk_" ^ config.name; data = config.private_key }) + @ [ { name = "runtime_config" + ; data = Yojson.Safe.to_string (Runtime_config.to_yojson runtime_config) - } ] + } + ] in let mina_archive_schema = "https://raw.githubusercontent.com/MinaProtocol/mina/develop/src/app/archive/create_schema.sql" in - { mina_automation_location= cli_inputs.coda_automation_location - ; debug_arg= debug - ; keypairs= block_producer_keypairs + { debug_arg = debug + ; keypairs = block_producer_keypairs ; constants - ; docker= - { version + ; docker = + { docker_swarm_version ; stack_name - ; coda_image= images.coda + ; coda_image = images.coda ; docker_volume_configs - ; runtime_config= Runtime_config.to_yojson runtime_config + ; runtime_config = Runtime_config.to_yojson runtime_config ; block_producer_configs ; snark_coordinator_configs ; log_precomputed_blocks - ; archive_node_count= num_archive_nodes - ; mina_archive_schema } } + ; archive_node_count = num_archive_nodes + ; mina_archive_schema + } + } let to_docker network_config = let open Docker_compose.Compose in - let open Node_constants in + let open Node_config in let runtime_config = Service.Volume.create "runtime_config" in let blocks_seed_map = - List.fold network_config.docker.block_producer_configs - ~init:DockerMap.empty ~f:(fun accum config -> + List.map network_config.docker.block_producer_configs ~f:(fun config -> let private_key_config = Service.Volume.create ("sk_" ^ config.name) in - Map.set accum ~key:config.name - ~data: - { Service.image= network_config.docker.coda_image - ; volumes= [private_key_config; runtime_config] - ; command= - default_block_producer_command - ~runtime_config:runtime_config.target - ~private_key_config:private_key_config.target - ; environment= - Service.Environment.create default_block_producer_envs } ) - (* Add a seed node to the map as well*) - |> Map.set ~key:"seed" - ~data: - { Service.image= network_config.docker.coda_image - ; volumes= [runtime_config] - ; command= - default_seed_command ~runtime_config:runtime_config.target - ; environment= Service.Environment.create default_seed_envs } + let cmd = + Cmd.( + Block_producer_command + (Block_producer_command.default + ~private_key_config:private_key_config.target)) + in + ( config.name + , { Service.image = network_config.docker.coda_image + ; volumes = [ private_key_config; runtime_config ] + ; command = Cmd.create_cmd cmd ~config_file:runtime_config.target + ; environment = Service.Environment.create Envs.base_node_envs + } )) + @ [ (* Add a seed node to the map as well *) + ( "seed" + , { Service.image = network_config.docker.coda_image + ; volumes = [ runtime_config ] + ; command = + Cmd.create_cmd Seed_command ~config_file:runtime_config.target + ; environment = Service.Environment.create Envs.base_node_envs + } ) + ] + |> StringMap.of_alist_exn in let snark_worker_map = - List.fold network_config.docker.snark_coordinator_configs - ~init:DockerMap.empty ~f:(fun accum config -> + List.map network_config.docker.snark_coordinator_configs ~f:(fun config -> let command, environment = match String.substr_index config.name ~pattern:"coordinator" with | Some _ -> + let cmd = + Cmd.( + Snark_coordinator_command + (Snark_coordinator_command.default + ~snark_coordinator_key:config.public_key + ~snark_worker_fee:config.snark_worker_fee)) + in let coordinator_command = - default_snark_coord_command - ~runtime_config:runtime_config.target - ~snark_coordinator_key:config.public_key - ~snark_worker_fee:config.snark_worker_fee + Cmd.create_cmd cmd ~config_file:runtime_config.target in let coordinator_environment = Service.Environment.create - (default_snark_coord_envs + (Envs.snark_coord_envs ~snark_coordinator_key:config.public_key ~snark_worker_fee:config.snark_worker_fee) in (coordinator_command, coordinator_environment) | None -> + let cmd = + Cmd.( + Snark_worker_command + (Snark_worker_command.default + ~daemon_address:"test-snark-coordinator" + ~daemon_port:"8301")) + in let worker_command = - default_snark_worker_command - ~daemon_address:"test-snark-coordinator" + Cmd.create_cmd cmd ~config_file:runtime_config.target in let worker_environment = Service.Environment.create [] in (worker_command, worker_environment) in - Map.set accum ~key:config.name - ~data: - { Service.image= network_config.docker.coda_image - ; volumes= [runtime_config] - ; command - ; environment } ) + ( config.name + , { Service.image = network_config.docker.coda_image + ; volumes = [ runtime_config ] + ; command + ; environment + } )) + |> StringMap.of_alist_exn in - let services = merge_maps blocks_seed_map snark_worker_map in - {version; services} + let services = merge blocks_seed_map snark_worker_map in + { version = docker_swarm_version; services } end module Network_manager = struct type t = - { stack_name: string - ; logger: Logger.t - ; testnet_dir: string - ; testnet_log_filter: string - ; constants: Test_config.constants - ; seed_nodes: Swarm_network.Node.t list - ; nodes_by_app_id: Swarm_network.Node.t String.Map.t - ; block_producer_nodes: Swarm_network.Node.t list - ; snark_coordinator_configs: Swarm_network.Node.t list - ; mutable deployed: bool - ; keypairs: Keypair.t list } + { stack_name : string + ; logger : Logger.t + ; testnet_dir : string + ; testnet_log_filter : string + ; constants : Test_config.constants + ; seed_nodes : Swarm_network.Node.t list + ; nodes_by_app_id : Swarm_network.Node.t String.Map.t + ; block_producer_nodes : Swarm_network.Node.t list + ; snark_coordinator_nodes : Swarm_network.Node.t list + ; mutable deployed : bool + ; keypairs : Keypair.t list + } let run_cmd t prog args = Util.run_cmd t.testnet_dir prog args @@ -320,17 +353,13 @@ module Network_manager = struct let create ~logger (network_config : Network_config.t) = let%bind all_stacks_str = - Util.run_cmd_exn "/" "docker" ["stack"; "ls"; "--format"; "{{.Name}}"] + Util.run_cmd_exn "/" "docker" [ "stack"; "ls"; "--format"; "{{.Name}}" ] in let all_stacks = String.split ~on:'\n' all_stacks_str in - let testnet_dir = - network_config.mina_automation_location - ^/ network_config.docker.stack_name - in + let testnet_dir = network_config.docker.stack_name in let%bind () = if - List.mem all_stacks network_config.docker.stack_name - ~equal:String.equal + List.mem all_stacks network_config.docker.stack_name ~equal:String.equal then let%bind () = if network_config.debug_arg then @@ -341,11 +370,10 @@ module Network_manager = struct else Deferred.return ([%log info] - "Existing namespace of same name detected; removing to start \ - clean") + "Existing stack of same name detected; removing to start clean") in Util.run_cmd_exn "/" "docker" - ["stack"; "rm"; network_config.docker.stack_name] + [ "stack"; "rm"; network_config.docker.stack_name ] >>| Fn.const () else return () in @@ -361,55 +389,60 @@ module Network_manager = struct ~f:(fun ch -> Network_config.to_docker network_config |> Docker_compose.to_string - |> Out_channel.output_string ch ) ; + |> Out_channel.output_string ch) ; List.iter network_config.docker.docker_volume_configs ~f:(fun config -> [%log info] "Writing volume config: %s" (testnet_dir ^/ config.name) ; - Out_channel.with_file ~fail_if_exists:false - (testnet_dir ^/ config.name) ~f:(fun ch -> - config.data |> Out_channel.output_string ch ) ; - ignore (Util.run_cmd_exn testnet_dir "chmod" ["600"; config.name]) ) ; + Out_channel.with_file ~fail_if_exists:false (testnet_dir ^/ config.name) + ~f:(fun ch -> config.data |> Out_channel.output_string ch) ; + ignore (Util.run_cmd_exn testnet_dir "chmod" [ "600"; config.name ])) ; let cons_node swarm_name service_id network_keypair_opt = { Swarm_network.Node.swarm_name ; service_id - ; graphql_enabled= true - ; network_keypair= network_keypair_opt } + ; graphql_enabled = true + ; network_keypair = network_keypair_opt + } in let seed_nodes = - [cons_node network_config.docker.stack_name "seed" None] + [ cons_node network_config.docker.stack_name + (network_config.docker.stack_name ^ "_" ^ "seed") + None + ] in let block_producer_nodes = - List.map network_config.docker.block_producer_configs - ~f:(fun bp_config -> + List.map network_config.docker.block_producer_configs ~f:(fun bp_config -> cons_node network_config.docker.stack_name (network_config.docker.stack_name ^ "_" ^ bp_config.name) - (Some bp_config.keypair) ) + (Some bp_config.keypair)) in - let snark_coordinator_configs = + let snark_coordinator_nodes = List.map network_config.docker.snark_coordinator_configs ~f:(fun snark_config -> cons_node network_config.docker.stack_name (network_config.docker.stack_name ^ "_" ^ snark_config.name) - None ) + None) in let nodes_by_app_id = - let all_nodes = seed_nodes @ block_producer_nodes in + let all_nodes = + seed_nodes @ block_producer_nodes @ snark_coordinator_nodes + in all_nodes |> List.map ~f:(fun node -> (node.service_id, node)) |> String.Map.of_alist_exn in let t = - { stack_name= network_config.docker.stack_name + { stack_name = network_config.docker.stack_name ; logger ; testnet_dir - ; constants= network_config.constants + ; constants = network_config.constants ; seed_nodes ; block_producer_nodes - ; snark_coordinator_configs + ; snark_coordinator_nodes ; nodes_by_app_id - ; deployed= false - ; testnet_log_filter= "" - ; keypairs= - List.map network_config.keypairs ~f:(fun {keypair; _} -> keypair) } + ; deployed = false + ; testnet_log_filter = "" + ; keypairs = + List.map network_config.keypairs ~f:(fun { keypair; _ } -> keypair) + } in Deferred.return t @@ -417,28 +450,29 @@ module Network_manager = struct if t.deployed then failwith "network already deployed" ; [%log' info t.logger] "Deploying network" ; [%log' info t.logger] "Stack_name in deploy: %s" t.stack_name ; - let%map s = + let%map _ = run_cmd_exn t "docker" - ["stack"; "deploy"; "-c"; "compose.json"; t.stack_name] + [ "stack"; "deploy"; "-c"; "compose.json"; t.stack_name ] in - [%log' info t.logger] "DEPLOYED STRING: %s" s ; t.deployed <- true ; let result = - { Swarm_network.namespace= t.stack_name - ; constants= t.constants - ; seeds= t.seed_nodes - ; block_producers= t.block_producer_nodes - ; snark_coordinators= t.snark_coordinator_configs - ; archive_nodes= [] - ; nodes_by_app_id= t.nodes_by_app_id - ; testnet_log_filter= t.testnet_log_filter - ; keypairs= t.keypairs } + { Swarm_network.namespace = t.stack_name + ; constants = t.constants + ; seeds = t.seed_nodes + ; block_producers = t.block_producer_nodes + ; snark_coordinators = t.snark_coordinator_nodes + ; archive_nodes = [] + ; nodes_by_app_id = t.nodes_by_app_id + ; testnet_log_filter = t.testnet_log_filter + ; keypairs = t.keypairs + } in let nodes_to_string = Fn.compose (String.concat ~sep:", ") (List.map ~f:Swarm_network.Node.id) in [%log' info t.logger] "Network deployed" ; [%log' info t.logger] "testnet swarm: %s" t.stack_name ; + [%log' info t.logger] "seed nodes: %s" (nodes_to_string result.seeds) ; [%log' info t.logger] "snark coordinators: %s" (nodes_to_string result.snark_coordinators) ; [%log' info t.logger] "block producers: %s" @@ -450,7 +484,7 @@ module Network_manager = struct let destroy t = [%log' info t.logger] "Destroying network" ; if not t.deployed then failwith "network not deployed" ; - let%bind _ = run_cmd_exn t "docker" ["stack"; "rm"; t.stack_name] in + let%bind _ = run_cmd_exn t "docker" [ "stack"; "rm"; t.stack_name ] in t.deployed <- false ; Deferred.unit diff --git a/src/lib/integration_test_local_engine/node_config.ml b/src/lib/integration_test_local_engine/node_config.ml new file mode 100644 index 00000000000..9833bad8286 --- /dev/null +++ b/src/lib/integration_test_local_engine/node_config.ml @@ -0,0 +1,183 @@ +open Base + +module Envs = struct + let base_node_envs = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("CODA_PRIVKEY_PASS", "naughty blue worm") + ; ("CODA_LIBP2P_PASS", "") + ] + + let snark_coord_envs ~snark_coordinator_key ~snark_worker_fee = + [ ("CODA_SNARK_KEY", snark_coordinator_key) + ; ("CODA_SNARK_FEE", snark_worker_fee) + ; ("WORK_SELECTION", "seq") + ] + @ base_node_envs +end + +module Cli_args = struct + module Log_level = struct + type t = Debug + + let to_string t = match t with Debug -> "Debug" + end + + module Peer = struct + type t = string + + let default = + "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + end + + module Proof_level = struct + type t = Full + + let to_string t = match t with Full -> "Full" + end +end + +module Cmd = struct + open Cli_args + + module Base_command = struct + type t = + { peer : Peer.t + ; log_level : Log_level.t + ; log_snark_work_gossip : bool + ; log_txn_pool_gossip : bool + ; generate_genesis_proof : bool + ; client_port : string + ; rest_port : string + ; metrics_port : string + ; config_file : string + } + + let default ~config_file = + { peer = Peer.default + ; log_level = Log_level.Debug + ; log_snark_work_gossip = true + ; log_txn_pool_gossip = true + ; generate_genesis_proof = true + ; client_port = "8301" + ; rest_port = "3085" + ; metrics_port = "10001" + ; config_file + } + + let to_string t = + [ "-config-file" + ; t.config_file + ; "-log-level" + ; Log_level.to_string t.log_level + ; "-log-snark-work-gossip" + ; Bool.to_string t.log_snark_work_gossip + ; "-log-txn-pool-gossip" + ; Bool.to_string t.log_txn_pool_gossip + ; "-generate-genesis-proof" + ; Bool.to_string t.generate_genesis_proof + ; "-client-port" + ; t.client_port + ; "-rest-port" + ; t.rest_port + ; "-metrics-port" + ; t.metrics_port + ; "-peer" + ; Peer.default + ; "-log-json" + ] + + let default_cmd ~config_file = default ~config_file |> to_string + end + + module Seed_command = struct + let cmd ~config_file = + [ "daemon"; "-seed" ] @ Base_command.default_cmd ~config_file + end + + module Block_producer_command = struct + type t = + { block_producer_key : string + ; enable_flooding : bool + ; enable_peer_exchange : bool + } + + let default ~private_key_config = + { block_producer_key = private_key_config + ; enable_flooding = true + ; enable_peer_exchange = true + } + + let cmd t ~config_file = + [ "daemon" + ; "-block-producer-key" + ; t.block_producer_key + ; "-enable-flooding" + ; Bool.to_string t.enable_flooding + ; "-enable-peer-exchange" + ; Bool.to_string t.enable_peer_exchange + ] + @ Base_command.default_cmd ~config_file + end + + module Snark_coordinator_command = struct + type t = + { snark_coordinator_key : string + ; snark_worker_fee : string + ; work_selection : string + } + + let default ~snark_coordinator_key ~snark_worker_fee = + { snark_coordinator_key; snark_worker_fee; work_selection = "seq" } + + let cmd t ~config_file = + [ "daemon" + ; "-run-snark-coordinator" + ; t.snark_coordinator_key + ; "-snark-worker-fee" + ; t.snark_worker_fee + ; "-work-selection" + ; t.work_selection + ] + @ Base_command.default_cmd ~config_file + end + + module Snark_worker_command = struct + type t = + { daemon_address : string + ; daemon_port : string + ; proof_level : Proof_level.t + } + + let default ~daemon_address ~daemon_port = + { daemon_address; daemon_port; proof_level = Proof_level.Full } + + let cmd t ~config_file = + [ "internal" + ; "snark-worker" + ; "-proof-level" + ; Proof_level.to_string t.proof_level + ; "-daemon-address" + ; t.daemon_address ^ ":" ^ t.daemon_port + ] + @ Base_command.default_cmd ~config_file + end + + type t = + | Seed_command + | Block_producer_command of Block_producer_command.t + | Snark_coordinator_command of Snark_coordinator_command.t + | Snark_worker_command of Snark_worker_command.t + + let create_cmd t ~config_file = + match t with + | Seed_command -> + Seed_command.cmd ~config_file + | Block_producer_command args -> + Block_producer_command.cmd args ~config_file + | Snark_coordinator_command args -> + Snark_coordinator_command.cmd args ~config_file + | Snark_worker_command args -> + Snark_worker_command.cmd args ~config_file +end diff --git a/src/lib/integration_test_local_engine/node_constants.ml b/src/lib/integration_test_local_engine/node_constants.ml deleted file mode 100644 index 3d6cd18759e..00000000000 --- a/src/lib/integration_test_local_engine/node_constants.ml +++ /dev/null @@ -1,98 +0,0 @@ -let default_seed_envs = - [ ("DAEMON_REST_PORT", "3085") - ; ("DAEMON_CLIENT_PORT", "8301") - ; ("DAEMON_METRICS_PORT", "10001") - ; ("CODA_LIBP2P_PASS", "") ] - -let default_block_producer_envs = - [ ("DAEMON_REST_PORT", "3085") - ; ("DAEMON_CLIENT_PORT", "8301") - ; ("DAEMON_METRICS_PORT", "10001") - ; ("CODA_PRIVKEY_PASS", "naughty blue worm") - ; ("CODA_LIBP2P_PASS", "") ] - -let default_snark_coord_envs ~snark_coordinator_key ~snark_worker_fee = - [ ("DAEMON_REST_PORT", "3085") - ; ("DAEMON_CLIENT_PORT", "8301") - ; ("DAEMON_METRICS_PORT", "10001") - ; ("CODA_SNARK_KEY", snark_coordinator_key) - ; ("CODA_SNARK_FEE", snark_worker_fee) - ; ("WORK_SELECTION", "seq") - ; ("CODA_PRIVKEY_PASS", "naughty blue worm") - ; ("CODA_LIBP2P_PASS", "") ] - -let default_seed_command ~runtime_config = - [ "daemon" - ; "-log-level" - ; "Debug" - ; "-log-json" - ; "-log-snark-work-gossip" - ; "true" - ; "-client-port" - ; "8301" - ; "-generate-genesis-proof" - ; "true" - ; "-peer" - ; "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - ; "-seed" - ; "-config-file" - ; runtime_config ] - -let default_block_producer_command ~runtime_config ~private_key_config = - [ "daemon" - ; "-log-level" - ; "Debug" - ; "-log-json" - ; "-log-snark-work-gossip" - ; "true" - ; "-log-txn-pool-gossip" - ; "true" - ; "-enable-peer-exchange" - ; "true" - ; "-enable-flooding" - ; "true" - ; "-peer" - ; "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - ; "-client-port" - ; "8301" - ; "-generate-genesis-proof" - ; "true" - ; "-block-producer-key" - ; private_key_config - ; "-config-file" - ; runtime_config ] - -let default_snark_coord_command ~runtime_config ~snark_coordinator_key - ~snark_worker_fee = - [ "daemon" - ; "-log-level" - ; "Debug" - ; "-log-json" - ; "-log-snark-work-gossip" - ; "true" - ; "-log-txn-pool-gossip" - ; "true" - ; "-external-port" - ; "10909" - ; "-rest-port" - ; "3085" - ; "-client-port" - ; "8301" - ; "-work-selection" - ; "seq" - ; "-peer" - ; "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - ; "-run-snark-coordinator" - ; snark_coordinator_key - ; "-snark-worker-fee" - ; snark_worker_fee - ; "-config-file" - ; runtime_config ] - -let default_snark_worker_command ~daemon_address = - [ "internal" - ; "snark-worker" - ; "-proof-level" - ; "full" - ; "-daemon-address" - ; daemon_address ^ ":8301" ] diff --git a/src/lib/integration_test_local_engine/swarm_network.ml b/src/lib/integration_test_local_engine/swarm_network.ml index e7eaf1997e1..8a10ddbd8be 100644 --- a/src/lib/integration_test_local_engine/swarm_network.ml +++ b/src/lib/integration_test_local_engine/swarm_network.ml @@ -2,60 +2,35 @@ open Core open Async open Integration_test_lib -(* Parts of this implementation is still untouched from the Kubernetes implementation. This is done - just to make the compiler happy as work is done on the local engine. -*) - (* exclude from bisect_ppx to avoid type error on GraphQL modules *) [@@@coverage exclude_file] module Node = struct type t = - { swarm_name: string - ; service_id: string - ; graphql_enabled: bool - ; network_keypair: Network_keypair.t option } - - let id {service_id; _} = service_id + { swarm_name : string + ; service_id : string + ; graphql_enabled : bool + ; network_keypair : Network_keypair.t option + } - let network_keypair {network_keypair; _} = network_keypair + let id { service_id; _ } = service_id - let base_kube_args t = ["--cluster"; t.swarm_name] + let network_keypair { network_keypair; _ } = network_keypair let get_container_cmd t = Printf.sprintf "$(docker ps -f name=%s --quiet)" t.service_id - let run_in_postgresql_container node ~cmd = - let base_args = base_kube_args node in - let base_kube_cmd = "kubectl " ^ String.concat ~sep:" " base_args in - let kubectl_cmd = - Printf.sprintf "%s -c %s exec -i %s-0 -- %s" base_kube_cmd - node.service_id node.service_id cmd - in - let%bind cwd = Unix.getcwd () in - Util.run_cmd_exn cwd "sh" ["-c"; kubectl_cmd] - - let get_logs_in_container node = - let base_args = base_kube_args node in - let base_kube_cmd = "kubectl " ^ String.concat ~sep:" " base_args in - let pod_cmd = - sprintf "%s get pod -l \"app=%s\" -o name" base_kube_cmd node.service_id - in - let%bind cwd = Unix.getcwd () in - let%bind pod = Util.run_cmd_exn cwd "sh" ["-c"; pod_cmd] in - let kubectl_cmd = - Printf.sprintf "%s logs -c %s -n %s %s" base_kube_cmd node.service_id - node.swarm_name pod - in - Util.run_cmd_exn cwd "sh" ["-c"; kubectl_cmd] + let run_in_postgresql_container _ _ = failwith "run_in_postgresql_container" + + let get_logs_in_container _ = failwith "get_logs_in_container" let run_in_container node cmd = let base_docker_cmd = "docker exec" in let docker_cmd = Printf.sprintf "%s %s %s" base_docker_cmd (get_container_cmd node) cmd in - let%bind.Deferred.Let_syntax cwd = Unix.getcwd () in - Malleable_error.return (Util.run_cmd_exn cwd "sh" ["-c"; docker_cmd]) + let%bind.Deferred cwd = Unix.getcwd () in + Malleable_error.return (Util.run_cmd_exn cwd "sh" [ "-c"; docker_cmd ]) let start ~fresh_state node : unit Malleable_error.t = let open Malleable_error.Let_syntax in @@ -88,390 +63,73 @@ module Node = struct module Decoders = Graphql_lib.Decoders - module Graphql = struct - let ingress_uri node = - let host = Printf.sprintf "%s.graphql.test.o1test.net" node.swarm_name in - let path = Printf.sprintf "/%s/graphql" node.service_id in - Uri.make ~scheme:"http" ~host ~path ~port:80 () - - module Client = Graphql_lib.Client.Make (struct - let preprocess_variables_string = Fn.id - - let headers = String.Map.empty - end) - - module Unlock_account = - [%graphql - {| - mutation ($password: String!, $public_key: PublicKey!) { - unlockAccount(input: {password: $password, publicKey: $public_key }) { - public_key: publicKey @bsDecoder(fn: "Decoders.public_key") - } - } - |}] - - module Send_payment = - [%graphql - {| - mutation ($sender: PublicKey!, - $receiver: PublicKey!, - $amount: UInt64!, - $token: UInt64, - $fee: UInt64!, - $nonce: UInt32, - $memo: String) { - sendPayment(input: - {from: $sender, to: $receiver, amount: $amount, token: $token, fee: $fee, nonce: $nonce, memo: $memo}) { - payment { - id - } - } - } - |}] - - module Get_balance = - [%graphql - {| - query ($public_key: PublicKey, $token: UInt64) { - account(publicKey: $public_key, token: $token) { - balance { - total @bsDecoder(fn: "Decoders.balance") - } - } - } - |}] - - module Query_peer_id = - [%graphql - {| - query { - daemonStatus { - addrsAndPorts { - peer { - peerId - } - } - peers { peerId } - - } - } - |}] - - module Best_chain = - [%graphql - {| - query { - bestChain { - stateHash - } - } - |}] - end - - (* this function will repeatedly attempt to connect to graphql port times before giving up *) - let exec_graphql_request ?(num_tries = 10) ?(retry_delay_sec = 30.0) - ?(initial_delay_sec = 30.0) ~logger ~node ~query_name query_obj = - let open Deferred.Let_syntax in - if not node.graphql_enabled then - Deferred.Or_error.error_string - "graphql is not enabled (hint: set `requires_graphql= true` in the \ - test config)" - else - let uri = Graphql.ingress_uri node in - let metadata = - [("query", `String query_name); ("uri", `String (Uri.to_string uri))] - in - [%log info] "Attempting to send GraphQL request \"$query\" to \"$uri\"" - ~metadata ; - let rec retry n = - if n <= 0 then ( - [%log error] - "GraphQL request \"$query\" to \"$uri\" failed too many times" - ~metadata ; - Deferred.Or_error.errorf - "GraphQL \"%s\" to \"%s\" request failed too many times" query_name - (Uri.to_string uri) ) - else - match%bind Graphql.Client.query query_obj uri with - | Ok result -> - [%log info] "GraphQL request \"$query\" to \"$uri\" succeeded" - ~metadata ; - Deferred.Or_error.return result - | Error (`Failed_request err_string) -> - [%log warn] - "GraphQL request \"$query\" to \"$uri\" failed: \"$error\" \ - ($num_tries attempts left)" - ~metadata: - ( metadata - @ [("error", `String err_string); ("num_tries", `Int (n - 1))] - ) ; - let%bind () = after (Time.Span.of_sec retry_delay_sec) in - retry (n - 1) - | Error (`Graphql_error err_string) -> - [%log error] - "GraphQL request \"$query\" to \"$uri\" returned an error: \ - \"$error\" (this is a graphql error so not retrying)" - ~metadata:(metadata @ [("error", `String err_string)]) ; - Deferred.Or_error.error_string err_string - in - let%bind () = after (Time.Span.of_sec initial_delay_sec) in - retry num_tries - - let get_peer_id ~logger t = - let open Deferred.Or_error.Let_syntax in - [%log info] "Getting node's peer_id, and the peer_ids of node's peers" - ~metadata: - [("namespace", `String t.swarm_name); ("pod_id", `String t.service_id)] ; - let query_obj = Graphql.Query_peer_id.make () in - let%bind query_result_obj = - exec_graphql_request ~logger ~node:t ~query_name:"query_peer_id" - query_obj - in - [%log info] "get_peer_id, finished exec_graphql_request" ; - let self_id_obj = ((query_result_obj#daemonStatus)#addrsAndPorts)#peer in - let%bind self_id = - match self_id_obj with - | None -> - Deferred.Or_error.error_string "Peer not found" - | Some peer -> - return peer#peerId - in - let peers = (query_result_obj#daemonStatus)#peers |> Array.to_list in - let peer_ids = List.map peers ~f:(fun peer -> peer#peerId) in - [%log info] - "get_peer_id, result of graphql query (self_id,[peers]) (%s,%s)" self_id - (String.concat ~sep:" " peer_ids) ; - return (self_id, peer_ids) - - let must_get_peer_id ~logger t = - get_peer_id ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error - - let get_best_chain ~logger t = - let open Deferred.Or_error.Let_syntax in - let query = Graphql.Best_chain.make () in - let%bind result = - exec_graphql_request ~logger ~node:t ~query_name:"best_chain" query - in - match result#bestChain with - | None | Some [||] -> - Deferred.Or_error.error_string "failed to get best chains" - | Some chain -> - return - @@ List.map ~f:(fun block -> block#stateHash) (Array.to_list chain) - - let must_get_best_chain ~logger t = - get_best_chain ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error - - let get_balance ~logger t ~account_id = - let open Deferred.Or_error.Let_syntax in - [%log info] "Getting account balance" - ~metadata: - [ ("namespace", `String t.swarm_name) - ; ("pod_id", `String t.service_id) - ; ("account_id", Mina_base.Account_id.to_yojson account_id) ] ; - let pk = Mina_base.Account_id.public_key account_id in - let token = Mina_base.Account_id.token_id account_id in - let get_balance_obj = - Graphql.Get_balance.make - ~public_key:(Graphql_lib.Encoders.public_key pk) - ~token:(Graphql_lib.Encoders.token token) - () - in - let%bind balance_obj = - exec_graphql_request ~logger ~node:t ~query_name:"get_balance_graphql" - get_balance_obj - in - match balance_obj#account with - | None -> - Deferred.Or_error.errorf - !"Account with %{sexp:Mina_base.Account_id.t} not found" - account_id - | Some acc -> - return (acc#balance)#total + let exec_graphql_request ~num_tries:_ ~retry_delay_sec:_ ~initial_delay_sec:_ + ~logger:_ ~node:_ ~query_name:_ _ : _ = + failwith "exec_graphql_request" + + let get_peer_id ~logger:_ _ = failwith "get_peer_id" + + let must_get_peer_id ~logger:_ _ = failwith "must_get_peer_id" + + let get_best_chain ~logger:_ _ = failwith "get_best_chain" + + let must_get_best_chain ~logger:_ _ = failwith "must_get_best_chain" + + let get_balance ~logger:_ _ ~account_id:_ = failwith "get_balance" let must_get_balance ~logger t ~account_id = get_balance ~logger t ~account_id |> Deferred.bind ~f:Malleable_error.or_hard_error (* if we expect failure, might want retry_on_graphql_error to be false *) - let send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee = - [%log info] "Sending a payment" - ~metadata: - [("namespace", `String t.swarm_name); ("pod_id", `String t.service_id)] ; - let open Deferred.Or_error.Let_syntax in - let sender_pk_str = - Signature_lib.Public_key.Compressed.to_string sender_pub_key - in - [%log info] "send_payment: unlocking account" - ~metadata:[("sender_pk", `String sender_pk_str)] ; - let unlock_sender_account_graphql () = - let unlock_account_obj = - Graphql.Unlock_account.make ~password:"naughty blue worm" - ~public_key:(Graphql_lib.Encoders.public_key sender_pub_key) - () - in - exec_graphql_request ~logger ~node:t - ~query_name:"unlock_sender_account_graphql" unlock_account_obj - in - let%bind _ = unlock_sender_account_graphql () in - let send_payment_graphql () = - let send_payment_obj = - Graphql.Send_payment.make - ~sender:(Graphql_lib.Encoders.public_key sender_pub_key) - ~receiver:(Graphql_lib.Encoders.public_key receiver_pub_key) - ~amount:(Graphql_lib.Encoders.amount amount) - ~fee:(Graphql_lib.Encoders.fee fee) - () - in - exec_graphql_request ~logger ~node:t ~query_name:"send_payment_graphql" - send_payment_obj - in - let%map sent_payment_obj = send_payment_graphql () in - let (`UserCommand id_obj) = (sent_payment_obj#sendPayment)#payment in - let user_cmd_id = id_obj#id in - [%log info] "Sent payment" - ~metadata:[("user_command_id", `String user_cmd_id)] ; - () - - let must_send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount - ~fee = + let send_payment ~logger:_ _ ~sender_pub_key:_ ~receiver_pub_key:_ ~amount:_ + ~fee:_ = + failwith "send_payment" + + let must_send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee + = send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee |> Deferred.bind ~f:Malleable_error.or_hard_error - let dump_archive_data ~logger (t : t) ~data_file = - let open Malleable_error.Let_syntax in - [%log info] "Dumping archive data from (node: %s, container: %s)" - t.service_id t.service_id ; - let%map data = - Deferred.bind ~f:Malleable_error.return - (run_in_postgresql_container t - ~cmd: - "pg_dump --create --no-owner \ - postgres://postgres:foobar@localhost:5432/archive") - in - [%log info] "Dumping archive data to file %s" data_file ; - Out_channel.with_file data_file ~f:(fun out_ch -> - Out_channel.output_string out_ch data ) - - let dump_container_logs ~logger (t : t) ~log_file = - let open Malleable_error.Let_syntax in - [%log info] "Dumping container logs from (node: %s, container: %s)" - t.service_id t.service_id ; - let%map logs = - Deferred.bind ~f:Malleable_error.return (get_logs_in_container t) - in - [%log info] "Dumping container log to file %s" log_file ; - Out_channel.with_file log_file ~f:(fun out_ch -> - Out_channel.output_string out_ch logs ) + let dump_archive_data ~logger:_ (_ : t) ~data_file:_ = + failwith "dump_archive_data" - let dump_precomputed_blocks ~logger (t : t) = - let open Malleable_error.Let_syntax in - [%log info] - "Dumping precomputed blocks from logs for (node: %s, container: %s)" - t.service_id t.service_id ; - let%bind logs = - Deferred.bind ~f:Malleable_error.return (get_logs_in_container t) - in - (* kubectl logs may include non-log output, like "Using password from environment variable" *) - let log_lines = - String.split logs ~on:'\n' - |> List.filter ~f:(String.is_prefix ~prefix:"{\"timestamp\":") - in - let jsons = List.map log_lines ~f:Yojson.Safe.from_string in - let metadata_jsons = - List.map jsons ~f:(fun json -> - match json with - | `Assoc items -> ( - match List.Assoc.find items ~equal:String.equal "metadata" with - | Some md -> - md - | None -> - failwithf "Log line is missing metadata: %s" - (Yojson.Safe.to_string json) - () ) - | other -> - failwithf "Expected log line to be a JSON record, got: %s" - (Yojson.Safe.to_string other) - () ) - in - let state_hash_and_blocks = - List.fold metadata_jsons ~init:[] ~f:(fun acc json -> - match json with - | `Assoc items -> ( - match - List.Assoc.find items ~equal:String.equal "precomputed_block" - with - | Some block -> ( - match List.Assoc.find items ~equal:String.equal "state_hash" with - | Some state_hash -> - (state_hash, block) :: acc - | None -> - failwith - "Log metadata contains a precomputed block, but no state \ - hash" ) - | None -> - acc ) - | other -> - failwithf "Expected log line to be a JSON record, got: %s" - (Yojson.Safe.to_string other) - () ) - in - let%bind.Deferred.Let_syntax () = - Deferred.List.iter state_hash_and_blocks - ~f:(fun (state_hash_json, block_json) -> - let double_quoted_state_hash = - Yojson.Safe.to_string state_hash_json - in - let state_hash = - String.sub double_quoted_state_hash ~pos:1 - ~len:(String.length double_quoted_state_hash - 2) - in - let block = Yojson.Safe.pretty_to_string block_json in - let filename = state_hash ^ ".json" in - match%map.Deferred.Let_syntax Sys.file_exists filename with - | `Yes -> - [%log info] - "File already exists for precomputed block with state hash %s" - state_hash - | _ -> - [%log info] - "Dumping precomputed block with state hash %s to file %s" - state_hash filename ; - Out_channel.with_file filename ~f:(fun out_ch -> - Out_channel.output_string out_ch block ) ) - in + let dump_container_logs ~logger:_ (_ : t) ~log_file:_ = Malleable_error.return () + + let dump_precomputed_blocks ~logger:_ (_ : t) = Malleable_error.return () end type t = - { namespace: string - ; constants: Test_config.constants - ; seeds: Node.t list - ; block_producers: Node.t list - ; snark_coordinators: Node.t list - ; archive_nodes: Node.t list - ; testnet_log_filter: string - ; keypairs: Signature_lib.Keypair.t list - ; nodes_by_app_id: Node.t String.Map.t } + { namespace : string + ; constants : Test_config.constants + ; seeds : Node.t list + ; block_producers : Node.t list + ; snark_coordinators : Node.t list + ; archive_nodes : Node.t list + ; testnet_log_filter : string + ; keypairs : Signature_lib.Keypair.t list + ; nodes_by_app_id : Node.t String.Map.t + } -let constants {constants; _} = constants +let constants { constants; _ } = constants -let constraint_constants {constants; _} = constants.constraints +let constraint_constants { constants; _ } = constants.constraints -let genesis_constants {constants; _} = constants.genesis +let genesis_constants { constants; _ } = constants.genesis -let seeds {seeds; _} = seeds +let seeds { seeds; _ } = seeds -let block_producers {block_producers; _} = block_producers +let block_producers { block_producers; _ } = block_producers -let snark_coordinators {snark_coordinators; _} = snark_coordinators +let snark_coordinators { snark_coordinators; _ } = snark_coordinators -let archive_nodes {archive_nodes; _} = archive_nodes +let archive_nodes { archive_nodes; _ } = archive_nodes -let keypairs {keypairs; _} = keypairs +let keypairs { keypairs; _ } = keypairs -let all_nodes {seeds; block_producers; snark_coordinators; archive_nodes; _} = - List.concat [seeds; block_producers; snark_coordinators; archive_nodes] +let all_nodes { seeds; block_producers; snark_coordinators; archive_nodes; _ } = + List.concat [ seeds; block_producers; snark_coordinators; archive_nodes ] let lookup_node_by_app_id t = Map.find t.nodes_by_app_id @@ -482,22 +140,22 @@ let initialize ~logger network = let max_polls = 60 (* 15 mins *) in let all_services = all_nodes network - |> List.map ~f:(fun {service_id; _} -> service_id) + |> List.map ~f:(fun { service_id; _ } -> service_id) |> String.Set.of_list in let get_service_statuses () = let%map output = Deferred.bind ~f:Malleable_error.return (Util.run_cmd_exn "/" "docker" - ["service"; "ls"; "--format"; "{{.Name}}: {{.Replicas}}"]) + [ "service"; "ls"; "--format"; "{{.Name}}: {{.Replicas}}" ]) in output |> String.split_lines |> List.map ~f:(fun line -> let parts = String.split line ~on:':' in assert (List.length parts = 2) ; - (List.nth_exn parts 0, List.nth_exn parts 1) ) + (List.nth_exn parts 0, List.nth_exn parts 1)) |> List.filter ~f:(fun (service_name, _) -> - String.Set.mem all_services service_name ) + String.Set.mem all_services service_name) in let rec poll n = let%bind pod_statuses = get_service_statuses () in @@ -506,8 +164,11 @@ let initialize ~logger network = List.filter pod_statuses ~f:(fun (_, status) -> let parts = String.split status ~on:'/' in assert (List.length parts = 2) ; - let num, denom = (List.nth_exn parts 0, List.nth_exn parts 1) in - String.strip num <> String.strip denom ) + let num, denom = + ( String.strip (List.nth_exn parts 0) + , String.strip (List.nth_exn parts 1) ) + in + not (String.equal num denom)) in if List.is_empty bad_service_statuses then return () else if n < max_polls then @@ -516,21 +177,20 @@ let initialize ~logger network = in poll (n + 1) else - let bad_pod_statuses_json = + let bad_service_statuses_json = `List (List.map bad_service_statuses ~f:(fun (service_name, status) -> `Assoc [ ("service_name", `String service_name) - ; ("status", `String status) ] )) + ; ("status", `String status) + ])) in [%log fatal] - "Not all pods were assigned to nodes and ready in time: \ - $bad_pod_statuses" - ~metadata:[("bad_pod_statuses", bad_pod_statuses_json)] ; + "Not all services could be deployed in time: $bad_service_statuses" + ~metadata:[ ("bad_service_statuses", bad_service_statuses_json) ] ; Malleable_error.hard_error_format - "Some pods either were not assigned to nodes or did deploy properly \ - (errors: %s)" - (Yojson.Safe.to_string bad_pod_statuses_json) + "Some services either were not deployed properly (errors: %s)" + (Yojson.Safe.to_string bad_service_statuses_json) in [%log info] "Waiting for pods to be assigned nodes and become ready" ; Deferred.bind (poll 0) ~f:(fun res -> @@ -538,13 +198,13 @@ let initialize ~logger network = let seed_nodes = seeds network in let seed_pod_ids = seed_nodes - |> List.map ~f:(fun {Node.service_id; _} -> service_id) + |> List.map ~f:(fun { Node.service_id; _ } -> service_id) |> String.Set.of_list in let non_seed_nodes = network |> all_nodes - |> List.filter ~f:(fun {Node.service_id; _} -> - not (String.Set.mem seed_pod_ids service_id) ) + |> List.filter ~f:(fun { Node.service_id; _ } -> + not (String.Set.mem seed_pod_ids service_id)) in (* TODO: parallelize (requires accumlative hard errors) *) let%bind () = @@ -558,4 +218,4 @@ let initialize ~logger network = in Malleable_error.List.iter non_seed_nodes ~f:(Node.start ~fresh_state:false) - else Deferred.return res ) + else Deferred.return res) From 4249a7a9d9b49f5716e65d8c15f0800ae5db57e7 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 29 Jun 2021 09:25:03 -0700 Subject: [PATCH 10/25] Added Archive node support to local test engine Added support for archive nodes in the local engine. If archive nodes are specified in the test plan, the test executive will download the archive schema to the working directory to use the schema as a bind mount for the postgres container. The archive node will then connect to the postgres container as usual. --- .../docker_compose.ml | 3 +- .../mina_docker.ml | 149 +++++++++++++++--- .../node_config.ml | 73 +++++++++ .../swarm_network.ml | 15 +- 4 files changed, 211 insertions(+), 29 deletions(-) diff --git a/src/lib/integration_test_local_engine/docker_compose.ml b/src/lib/integration_test_local_engine/docker_compose.ml index ff0c779196f..45b26eb1145 100644 --- a/src/lib/integration_test_local_engine/docker_compose.ml +++ b/src/lib/integration_test_local_engine/docker_compose.ml @@ -9,8 +9,7 @@ module Compose = struct { type_ : string [@key "type"]; source : string; target : string } [@@deriving to_yojson] - let create name = - { type_ = "bind"; source = "." ^/ name; target = "/root" ^/ name } + let create source target = { type_ = "bind"; source; target } end module Environment = struct diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index d5f61a62792..9cb356f7926 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -7,6 +7,13 @@ open Integration_test_lib let docker_swarm_version = "3.9" +let postgres_image = "docker.io/bitnami/postgresql" + +let mina_archive_schema = + "https://raw.githubusercontent.com/MinaProtocol/mina/develop/src/app/archive/create_schema.sql" + +let mina_create_schema = "create_schema.sql" + module Network_config = struct module Cli_inputs = Cli_inputs @@ -32,13 +39,18 @@ module Network_config = struct } [@@deriving to_yojson] + type archive_node_configs = { name : string; id : string; schema : string } + [@@deriving to_yojson] + type docker_config = { docker_swarm_version : string ; stack_name : string - ; coda_image : string + ; mina_image : string + ; mina_archive_image : string ; docker_volume_configs : docker_volume_configs list ; block_producer_configs : block_producer_config list ; snark_coordinator_configs : snark_coordinator_configs list + ; archive_node_configs : archive_node_configs list ; log_precomputed_blocks : bool ; archive_node_count : int ; mina_archive_schema : string @@ -202,6 +214,7 @@ module Network_config = struct let block_producer_configs = List.mapi block_producer_keypairs ~f:block_producer_config in + (* SNARK COORDINATOR CONFIG *) let snark_coordinator_configs = if num_snark_workers > 0 then List.mapi @@ -222,7 +235,21 @@ module Network_config = struct ] else [] in - (* Combine configs for block producer configs and runtime config to be a docker bind volume *) + (* ARCHIVE CONFIG *) + let archive_node_configs = + if num_archive_nodes > 0 then + List.mapi + (List.init num_archive_nodes ~f:(const 0)) + ~f:(fun index _ -> + { name = "archive-" ^ Int.to_string (index + 1) + ; id = Int.to_string index + ; schema = mina_archive_schema + }) + else [] + in + (* DOCKER VOLUME CONFIG: + * Create a docker volume structure for the runtime_config and all block producer keys + *) let docker_volume_configs = List.map block_producer_configs ~f:(fun config -> { name = "sk_" ^ config.name; data = config.private_key }) @@ -232,34 +259,38 @@ module Network_config = struct } ] in - let mina_archive_schema = - "https://raw.githubusercontent.com/MinaProtocol/mina/develop/src/app/archive/create_schema.sql" - in { debug_arg = debug ; keypairs = block_producer_keypairs ; constants ; docker = { docker_swarm_version ; stack_name - ; coda_image = images.coda + ; mina_image = images.coda + ; mina_archive_image = images.archive_node ; docker_volume_configs ; runtime_config = Runtime_config.to_yojson runtime_config ; block_producer_configs ; snark_coordinator_configs + ; archive_node_configs ; log_precomputed_blocks ; archive_node_count = num_archive_nodes ; mina_archive_schema } } + (* Creates a mapping of the network_config services to a docker-compose file. *) let to_docker network_config = let open Docker_compose.Compose in let open Node_config in - let runtime_config = Service.Volume.create "runtime_config" in - let blocks_seed_map = + let runtime_config = + Service.Volume.create "runtime_config" "/root/runtime_config" + in + let block_producer_map = List.map network_config.docker.block_producer_configs ~f:(fun config -> + let private_key_config_name = "sk_" ^ config.name in let private_key_config = - Service.Volume.create ("sk_" ^ config.name) + Service.Volume.create private_key_config_name + ("/root/" ^ private_key_config_name) in let cmd = Cmd.( @@ -268,20 +299,11 @@ module Network_config = struct ~private_key_config:private_key_config.target)) in ( config.name - , { Service.image = network_config.docker.coda_image + , { Service.image = network_config.docker.mina_image ; volumes = [ private_key_config; runtime_config ] ; command = Cmd.create_cmd cmd ~config_file:runtime_config.target ; environment = Service.Environment.create Envs.base_node_envs } )) - @ [ (* Add a seed node to the map as well *) - ( "seed" - , { Service.image = network_config.docker.coda_image - ; volumes = [ runtime_config ] - ; command = - Cmd.create_cmd Seed_command ~config_file:runtime_config.target - ; environment = Service.Environment.create Envs.base_node_envs - } ) - ] |> StringMap.of_alist_exn in let snark_worker_map = @@ -321,14 +343,75 @@ module Network_config = struct (worker_command, worker_environment) in ( config.name - , { Service.image = network_config.docker.coda_image + , { Service.image = network_config.docker.mina_image ; volumes = [ runtime_config ] ; command ; environment } )) |> StringMap.of_alist_exn in - let services = merge blocks_seed_map snark_worker_map in + let archive_node_configs = + List.mapi network_config.docker.archive_node_configs + ~f:(fun index config -> + let postgres_uri = + Cli_args.Postgres_uri.create + ~host: + ( network_config.docker.stack_name ^ "_postgres-" + ^ Int.to_string (index + 1) ) + |> Cli_args.Postgres_uri.to_string + in + let cmd = + Cmd.( + Archive_node_command + { Archive_node_command.postgres_uri; server_port = "3086" }) + in + ( config.name + , { Service.image = network_config.docker.mina_archive_image + ; volumes = [ runtime_config ] + ; command = Cmd.create_cmd cmd ~config_file:runtime_config.target + ; environment = Service.Environment.create Envs.base_node_envs + } )) + (* Add an equal number of postgres containers as archive nodes *) + @ List.mapi network_config.docker.archive_node_configs ~f:(fun index _ -> + let pg_config = Cli_args.Postgres_uri.default in + (* Mount the archive schema on the /docker-entrypoint-initdb.d postgres entrypoint*) + let archive_config = + Service.Volume.create mina_create_schema + ("/docker-entrypoint-initdb.d/" ^ mina_create_schema) + in + ( "postgres-" ^ Int.to_string (index + 1) + , { Service.image = postgres_image + ; volumes = [ archive_config ] + ; command = [] + ; environment = + Service.Environment.create + (Envs.postgres_envs ~username:pg_config.username + ~password:pg_config.password ~database:pg_config.db + ~port:pg_config.port) + } )) + |> StringMap.of_alist_exn + in + let seed_map = + [ (let command = + if List.length network_config.docker.archive_node_configs > 0 then + (* If an archive node is specified in the test plan, use the seed node to connect to the coda-archive process *) + Cmd.create_cmd Seed_command ~config_file:runtime_config.target + @ [ "-archive-address"; "archive-1:3086" ] + else Cmd.create_cmd Seed_command ~config_file:runtime_config.target + in + ( "seed" + , { Service.image = network_config.docker.mina_image + ; volumes = [ runtime_config ] + ; command + ; environment = Service.Environment.create Envs.base_node_envs + } )) + ] + |> StringMap.of_alist_exn + in + let services = + seed_map |> merge block_producer_map |> merge snark_worker_map + |> merge archive_node_configs + in { version = docker_swarm_version; services } end @@ -343,6 +426,7 @@ module Network_manager = struct ; nodes_by_app_id : Swarm_network.Node.t String.Map.t ; block_producer_nodes : Swarm_network.Node.t list ; snark_coordinator_nodes : Swarm_network.Node.t list + ; archive_node_nodes : Swarm_network.Node.t list ; mutable deployed : bool ; keypairs : Keypair.t list } @@ -404,7 +488,7 @@ module Network_manager = struct in let seed_nodes = [ cons_node network_config.docker.stack_name - (network_config.docker.stack_name ^ "_" ^ "seed") + (network_config.docker.stack_name ^ "_seed") None ] in @@ -421,9 +505,27 @@ module Network_manager = struct (network_config.docker.stack_name ^ "_" ^ snark_config.name) None) in + let%bind _ = + if List.length network_config.docker.archive_node_configs > 0 then + let cmd = + Printf.sprintf "curl -Ls %s > create_schema.sql" + network_config.docker.mina_archive_schema + in + (* Write the archive schema to the working directory to use as an entrypoint for postgres*) + Util.run_cmd_exn testnet_dir "bash" [ "-c"; cmd ] + else return "" + in + let archive_node_nodes = + List.map network_config.docker.archive_node_configs + ~f:(fun archive_node -> + cons_node network_config.docker.stack_name + (network_config.docker.stack_name ^ "_" ^ archive_node.name) + None) + in let nodes_by_app_id = let all_nodes = seed_nodes @ block_producer_nodes @ snark_coordinator_nodes + @ archive_node_nodes in all_nodes |> List.map ~f:(fun node -> (node.service_id, node)) @@ -437,6 +539,7 @@ module Network_manager = struct ; seed_nodes ; block_producer_nodes ; snark_coordinator_nodes + ; archive_node_nodes ; nodes_by_app_id ; deployed = false ; testnet_log_filter = "" @@ -461,7 +564,7 @@ module Network_manager = struct ; seeds = t.seed_nodes ; block_producers = t.block_producer_nodes ; snark_coordinators = t.snark_coordinator_nodes - ; archive_nodes = [] + ; archive_nodes = t.archive_node_nodes ; nodes_by_app_id = t.nodes_by_app_id ; testnet_log_filter = t.testnet_log_filter ; keypairs = t.keypairs diff --git a/src/lib/integration_test_local_engine/node_config.ml b/src/lib/integration_test_local_engine/node_config.ml index 9833bad8286..38b8e92c6d7 100644 --- a/src/lib/integration_test_local_engine/node_config.ml +++ b/src/lib/integration_test_local_engine/node_config.ml @@ -15,6 +15,23 @@ module Envs = struct ; ("WORK_SELECTION", "seq") ] @ base_node_envs + + let postgres_envs ~username ~password ~database ~port = + [ ("BITNAMI_DEBUG", "false") + ; ("POSTGRES_USER", username) + ; ("POSTGRES_PASSWORD", password) + ; ("POSTGRES_DB", database) + ; ("POSTGRESQL_PORT_NUMBER", port) + ; ("POSTGRESQL_ENABLE_LDAP", "no") + ; ("POSTGRESQL_ENABLE_TLS", "no") + ; ("POSTGRESQL_LOG_HOSTNAME", "false") + ; ("POSTGRESQL_LOG_CONNECTIONS", "false") + ; ("POSTGRESQL_LOG_DISCONNECTIONS", "false") + ; ("POSTGRESQL_PGAUDIT_LOG_CATALOG", "off") + ; ("POSTGRESQL_CLIENT_MIN_MESSAGES", "error") + ; ("POSTGRESQL_SHARED_PRELOAD_LIBRARIES", "pgaudit") + ; ("POSTGRES_HOST_AUTH_METHOD", "trust") + ] end module Cli_args = struct @@ -31,6 +48,37 @@ module Cli_args = struct "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" end + module Postgres_uri = struct + type t = + { username : string + ; password : string + ; host : string + ; port : string + ; db : string + } + + let create ~host = + { username = "postgres" + ; password = "password" + ; host + ; port = "5432" + ; db = "archive" + } + + (* Hostname should be dynamic based on the container ID in runtime. Ignore this field for default binding *) + let default = + { username = "postgres" + ; password = "password" + ; host = "" + ; port = "5432" + ; db = "archive" + } + + let to_string t = + Printf.sprintf "postgres://%s:%s@%s:%s/%s" t.username t.password t.host + t.port t.db + end + module Proof_level = struct type t = Full @@ -164,11 +212,34 @@ module Cmd = struct @ Base_command.default_cmd ~config_file end + module Archive_node_command = struct + type t = { postgres_uri : string; server_port : string } + + let default = + { postgres_uri = Postgres_uri.(default |> to_string) + ; server_port = "3086" + } + + let create postgres_uri = { postgres_uri; server_port = "3086" } + + let cmd t ~config_file = + [ "coda-archive" + ; "run" + ; "-postgres-uri" + ; t.postgres_uri + ; "-server-port" + ; t.server_port + ; "-config-file" + ; config_file + ] + end + type t = | Seed_command | Block_producer_command of Block_producer_command.t | Snark_coordinator_command of Snark_coordinator_command.t | Snark_worker_command of Snark_worker_command.t + | Archive_node_command of Archive_node_command.t let create_cmd t ~config_file = match t with @@ -180,4 +251,6 @@ module Cmd = struct Snark_coordinator_command.cmd args ~config_file | Snark_worker_command args -> Snark_worker_command.cmd args ~config_file + | Archive_node_command args -> + Archive_node_command.cmd args ~config_file end diff --git a/src/lib/integration_test_local_engine/swarm_network.ml b/src/lib/integration_test_local_engine/swarm_network.ml index 8a10ddbd8be..4d24b5b7b31 100644 --- a/src/lib/integration_test_local_engine/swarm_network.ml +++ b/src/lib/integration_test_local_engine/swarm_network.ml @@ -196,15 +196,22 @@ let initialize ~logger network = Deferred.bind (poll 0) ~f:(fun res -> if Malleable_error.is_ok res then let seed_nodes = seeds network in - let seed_pod_ids = + let seed_service_ids = seed_nodes |> List.map ~f:(fun { Node.service_id; _ } -> service_id) |> String.Set.of_list in - let non_seed_nodes = + let archive_nodes = archive_nodes network in + let archive_service_ids = + archive_nodes + |> List.map ~f:(fun { Node.service_id; _ } -> service_id) + |> String.Set.of_list + in + let non_seed_archive_nodes = network |> all_nodes |> List.filter ~f:(fun { Node.service_id; _ } -> - not (String.Set.mem seed_pod_ids service_id)) + (not (String.Set.mem seed_service_ids service_id)) + && not (String.Set.mem archive_service_ids service_id)) in (* TODO: parallelize (requires accumlative hard errors) *) let%bind () = @@ -216,6 +223,6 @@ let initialize ~logger network = after (Time.Span.of_sec 30.0) |> Deferred.bind ~f:Malleable_error.return in - Malleable_error.List.iter non_seed_nodes + Malleable_error.List.iter non_seed_archive_nodes ~f:(Node.start ~fresh_state:false) else Deferred.return res) From b86c34ada317128ab82cfca9bef36a8d363bf9b7 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Wed, 7 Jul 2021 15:31:47 -0700 Subject: [PATCH 11/25] Cleaned up modules in mina_docker and node_config Renamed node_config to docker_node_config and refactored some of the modules to have more static references baked in. Additionally refactored mina_docker to use these static references instead of string values in many places. --- .../{node_config.ml => docker_node_config.ml} | 211 ++++++++++++------ .../mina_docker.ml | 110 +++++---- 2 files changed, 207 insertions(+), 114 deletions(-) rename src/lib/integration_test_local_engine/{node_config.ml => docker_node_config.ml} (58%) diff --git a/src/lib/integration_test_local_engine/node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml similarity index 58% rename from src/lib/integration_test_local_engine/node_config.ml rename to src/lib/integration_test_local_engine/docker_node_config.ml index 38b8e92c6d7..c756af1b7d2 100644 --- a/src/lib/integration_test_local_engine/node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -34,62 +34,70 @@ module Envs = struct ] end -module Cli_args = struct - module Log_level = struct - type t = Debug +module Volumes = struct + module Runtime_config = struct + let name = "runtime_config" - let to_string t = match t with Debug -> "Debug" - end - - module Peer = struct - type t = string - - let default = - "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - end - - module Postgres_uri = struct - type t = - { username : string - ; password : string - ; host : string - ; port : string - ; db : string - } - - let create ~host = - { username = "postgres" - ; password = "password" - ; host - ; port = "5432" - ; db = "archive" - } - - (* Hostname should be dynamic based on the container ID in runtime. Ignore this field for default binding *) - let default = - { username = "postgres" - ; password = "password" - ; host = "" - ; port = "5432" - ; db = "archive" - } - - let to_string t = - Printf.sprintf "postgres://%s:%s@%s:%s/%s" t.username t.password t.host - t.port t.db - end - - module Proof_level = struct - type t = Full - - let to_string t = match t with Full -> "Full" + let container_mount_target = "/root/" ^ name end end module Cmd = struct + module Cli_args = struct + module Log_level = struct + type t = Debug + + let to_string t = match t with Debug -> "Debug" + end + + module Peer = struct + type t = string + + let default = + "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" + end + + module Postgres_uri = struct + type t = + { username : string + ; password : string + ; host : string + ; port : string + ; db : string + } + + let create ~host = + { username = "postgres" + ; password = "password" + ; host + ; port = "5432" + ; db = "archive" + } + + (* Hostname should be dynamic based on the container ID in runtime. Ignore this field for default binding *) + let default = + { username = "postgres" + ; password = "password" + ; host = "" + ; port = "5432" + ; db = "archive" + } + + let to_string t = + Printf.sprintf "postgres://%s:%s@%s:%s/%s" t.username t.password t.host + t.port t.db + end + + module Proof_level = struct + type t = Full + + let to_string t = match t with Full -> "Full" + end + end + open Cli_args - module Base_command = struct + module Base = struct type t = { peer : Peer.t ; log_level : Log_level.t @@ -139,12 +147,13 @@ module Cmd = struct let default_cmd ~config_file = default ~config_file |> to_string end - module Seed_command = struct - let cmd ~config_file = - [ "daemon"; "-seed" ] @ Base_command.default_cmd ~config_file + module Seed = struct + let cmd ~config_file = [ "daemon"; "-seed" ] @ Base.default_cmd ~config_file + + let connect_to_archive ~archive_node = [ "-archive-address"; archive_node ] end - module Block_producer_command = struct + module Block_producer = struct type t = { block_producer_key : string ; enable_flooding : bool @@ -166,10 +175,10 @@ module Cmd = struct ; "-enable-peer-exchange" ; Bool.to_string t.enable_peer_exchange ] - @ Base_command.default_cmd ~config_file + @ Base.default_cmd ~config_file end - module Snark_coordinator_command = struct + module Snark_coordinator = struct type t = { snark_coordinator_key : string ; snark_worker_fee : string @@ -188,10 +197,12 @@ module Cmd = struct ; "-work-selection" ; t.work_selection ] - @ Base_command.default_cmd ~config_file + @ Base.default_cmd ~config_file end - module Snark_worker_command = struct + module Snark_worker = struct + let name = "snark-worker" + type t = { daemon_address : string ; daemon_port : string @@ -209,10 +220,10 @@ module Cmd = struct ; "-daemon-address" ; t.daemon_address ^ ":" ^ t.daemon_port ] - @ Base_command.default_cmd ~config_file + @ Base.default_cmd ~config_file end - module Archive_node_command = struct + module Archive_node = struct type t = { postgres_uri : string; server_port : string } let default = @@ -235,22 +246,74 @@ module Cmd = struct end type t = - | Seed_command - | Block_producer_command of Block_producer_command.t - | Snark_coordinator_command of Snark_coordinator_command.t - | Snark_worker_command of Snark_worker_command.t - | Archive_node_command of Archive_node_command.t + | Seed + | Block_producer of Block_producer.t + | Snark_coordinator of Snark_coordinator.t + | Snark_worker of Snark_worker.t + | Archive_node of Archive_node.t let create_cmd t ~config_file = match t with - | Seed_command -> - Seed_command.cmd ~config_file - | Block_producer_command args -> - Block_producer_command.cmd args ~config_file - | Snark_coordinator_command args -> - Snark_coordinator_command.cmd args ~config_file - | Snark_worker_command args -> - Snark_worker_command.cmd args ~config_file - | Archive_node_command args -> - Archive_node_command.cmd args ~config_file + | Seed -> + Seed.cmd ~config_file + | Block_producer args -> + Block_producer.cmd args ~config_file + | Snark_coordinator args -> + Snark_coordinator.cmd args ~config_file + | Snark_worker args -> + Snark_worker.cmd args ~config_file + | Archive_node args -> + Archive_node.cmd args ~config_file +end + +module Services = struct + module Seed = struct + let name = "seed" + + let env = Envs.base_node_envs + + let cmd = Cmd.Seed + end + + module Block_producer = struct + let name = "block-producer" + + let secret_name = "keypair" + + let env = Envs.base_node_envs + + let cmd args = Cmd.Block_producer args + end + + module Snark_coordinator = struct + let name = "snark-coordinator" + + let default_port = "8301" + + let env = Envs.snark_coord_envs + + let cmd args = Cmd.Snark_coordinator args + end + + module Snark_worker = struct + let name = "snark-worker" + + let env = [] + + let cmd args = Cmd.Snark_worker args + end + + module Archive_node = struct + let name = "archive" + + let postgres_name = "postgres" + + let server_port = "3086" + + let envs = Envs.base_node_envs + + let cmd args = Cmd.Archive_node args + + let entrypoint_target = "/docker-entrypoint-initdb.d" + end end diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index 9cb356f7926..fc7e157d6a7 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -200,9 +200,10 @@ module Network_config = struct let constants : Test_config.constants = { constraints = constraint_constants; genesis = genesis_constants } in + let open Docker_node_config.Services in (* BLOCK PRODUCER CONFIG *) let block_producer_config index keypair = - { name = "test-block-producer-" ^ Int.to_string (index + 1) + { name = Block_producer.name ^ "-" ^ Int.to_string (index + 1) ; id = Int.to_string index ; keypair ; keypair_secret = keypair.secret_name @@ -220,14 +221,14 @@ module Network_config = struct List.mapi (List.init num_snark_workers ~f:(const 0)) ~f:(fun index _ -> - { name = "test-snark-worker-" ^ Int.to_string (index + 1) + { name = Snark_worker.name ^ "-" ^ Int.to_string (index + 1) ; id = Int.to_string index ; snark_worker_fee ; public_key = snark_worker_public_key }) (* Add one snark coordinator for all workers *) |> List.append - [ { name = "test-snark-coordinator" + [ { name = Snark_coordinator.name ; id = "1" ; snark_worker_fee ; public_key = snark_worker_public_key @@ -241,19 +242,20 @@ module Network_config = struct List.mapi (List.init num_archive_nodes ~f:(const 0)) ~f:(fun index _ -> - { name = "archive-" ^ Int.to_string (index + 1) + { name = Archive_node.name ^ "-" ^ Int.to_string (index + 1) ; id = Int.to_string index ; schema = mina_archive_schema }) else [] in (* DOCKER VOLUME CONFIG: - * Create a docker volume structure for the runtime_config and all block producer keys - *) + Create a docker volume structure for the runtime_config and all block producer keys + to mount in the specified docker services + *) let docker_volume_configs = List.map block_producer_configs ~f:(fun config -> - { name = "sk_" ^ config.name; data = config.private_key }) - @ [ { name = "runtime_config" + { name = "sk-" ^ config.name; data = config.private_key }) + @ [ { name = Docker_node_config.Volumes.Runtime_config.name ; data = Yojson.Safe.to_string (Runtime_config.to_yojson runtime_config) } @@ -278,24 +280,36 @@ module Network_config = struct } } - (* Creates a mapping of the network_config services to a docker-compose file. *) + (* + Composes a docker_compose.json file from the network_config specification and writes to disk. This docker_compose + file contains docker service definitions for each node in the local network. Each node service has different + configurations which are specified as commands, environment variables, and docker bind volumes. + We start by creating a runtime config volume to mount to each node service as a bind volume and then continue to create each + node service. As we create each definition for a service, we specify the docker command, volume, and environment varibles to + be used (which are mostly defaults). + *) let to_docker network_config = let open Docker_compose.Compose in - let open Node_config in + let open Docker_node_config in + let open Docker_node_config.Services in + (* RUNTIME CONFIG DOCKER BIND VOLUME *) let runtime_config = - Service.Volume.create "runtime_config" "/root/runtime_config" + Service.Volume.create Volumes.Runtime_config.name + Volumes.Runtime_config.container_mount_target in + (* BLOCK PRODUCER DOCKER SERVICE *) let block_producer_map = List.map network_config.docker.block_producer_configs ~f:(fun config -> - let private_key_config_name = "sk_" ^ config.name in + (* BP KEYPAIR CONFIG DOCKER BIND VOLUME *) + let private_key_config_name = "sk-" ^ config.name in let private_key_config = Service.Volume.create private_key_config_name ("/root/" ^ private_key_config_name) in let cmd = Cmd.( - Block_producer_command - (Block_producer_command.default + Block_producer + (Block_producer.default ~private_key_config:private_key_config.target)) in ( config.name @@ -306,15 +320,19 @@ module Network_config = struct } )) |> StringMap.of_alist_exn in + (* SNARK COORD/WORKER DOCKER SERVICE *) let snark_worker_map = List.map network_config.docker.snark_coordinator_configs ~f:(fun config -> + (* Assign different command and environment configs depending on coordinator or worker *) let command, environment = - match String.substr_index config.name ~pattern:"coordinator" with + match + String.substr_index config.name ~pattern:Snark_coordinator.name + with | Some _ -> let cmd = Cmd.( - Snark_coordinator_command - (Snark_coordinator_command.default + Snark_coordinator + (Snark_coordinator.default ~snark_coordinator_key:config.public_key ~snark_worker_fee:config.snark_worker_fee)) in @@ -329,12 +347,12 @@ module Network_config = struct in (coordinator_command, coordinator_environment) | None -> + let daemon_address = Snark_coordinator.name in + let daemon_port = Snark_coordinator.default_port in let cmd = Cmd.( - Snark_worker_command - (Snark_worker_command.default - ~daemon_address:"test-snark-coordinator" - ~daemon_port:"8301")) + Snark_worker + (Snark_worker.default ~daemon_address ~daemon_port)) in let worker_command = Cmd.create_cmd cmd ~config_file:runtime_config.target @@ -350,20 +368,21 @@ module Network_config = struct } )) |> StringMap.of_alist_exn in + (* ARCHIVE_NODE SERVICE *) let archive_node_configs = List.mapi network_config.docker.archive_node_configs ~f:(fun index config -> let postgres_uri = - Cli_args.Postgres_uri.create + Cmd.Cli_args.Postgres_uri.create ~host: - ( network_config.docker.stack_name ^ "_postgres-" + ( network_config.docker.stack_name ^ "_" + ^ Archive_node.postgres_name ^ "-" ^ Int.to_string (index + 1) ) - |> Cli_args.Postgres_uri.to_string + |> Cmd.Cli_args.Postgres_uri.to_string in let cmd = - Cmd.( - Archive_node_command - { Archive_node_command.postgres_uri; server_port = "3086" }) + let server_port = Archive_node.server_port in + Cmd.(Archive_node { Archive_node.postgres_uri; server_port }) in ( config.name , { Service.image = network_config.docker.mina_archive_image @@ -373,13 +392,13 @@ module Network_config = struct } )) (* Add an equal number of postgres containers as archive nodes *) @ List.mapi network_config.docker.archive_node_configs ~f:(fun index _ -> - let pg_config = Cli_args.Postgres_uri.default in - (* Mount the archive schema on the /docker-entrypoint-initdb.d postgres entrypoint*) + let pg_config = Cmd.Cli_args.Postgres_uri.default in + (* Mount the archive schema on the /docker-entrypoint-initdb.d postgres entrypoint *) let archive_config = Service.Volume.create mina_create_schema - ("/docker-entrypoint-initdb.d/" ^ mina_create_schema) + (Archive_node.entrypoint_target ^/ mina_create_schema) in - ( "postgres-" ^ Int.to_string (index + 1) + ( Archive_node.postgres_name ^ "-" ^ Int.to_string (index + 1) , { Service.image = postgres_image ; volumes = [ archive_config ] ; command = [] @@ -391,15 +410,17 @@ module Network_config = struct } )) |> StringMap.of_alist_exn in + (* SEED DOCKER SERVICE *) let seed_map = [ (let command = if List.length network_config.docker.archive_node_configs > 0 then - (* If an archive node is specified in the test plan, use the seed node to connect to the coda-archive process *) - Cmd.create_cmd Seed_command ~config_file:runtime_config.target - @ [ "-archive-address"; "archive-1:3086" ] - else Cmd.create_cmd Seed_command ~config_file:runtime_config.target + (* If an archive node is specified in the test plan, use the seed node to connect to the first mina-archive process *) + Cmd.create_cmd Seed ~config_file:runtime_config.target + @ Cmd.Seed.connect_to_archive + ~archive_node:("archive-1:" ^ Services.Archive_node.server_port) + else Cmd.create_cmd Seed ~config_file:runtime_config.target in - ( "seed" + ( Seed.name , { Service.image = network_config.docker.mina_image ; volumes = [ runtime_config ] ; command @@ -435,6 +456,14 @@ module Network_manager = struct let run_cmd_exn t prog args = Util.run_cmd_exn t.testnet_dir prog args + (* + Creates a docker swarm network by creating a local working directory and writing a + docker_compose.json file to disk with all related resources (e.g. runtime_config and secret keys). + + First, check if there is a running swarm network and let the user remove it. Continue by creating a + working directory and writing out all resources to be used as docker volumes. Additionally for each resource, specify + file permissions of 600 so that the daemon can run properly. + *) let create ~logger (network_config : Network_config.t) = let%bind all_stacks_str = Util.run_cmd_exn "/" "docker" [ "stack"; "ls"; "--format"; "{{.Name}}" ] @@ -488,7 +517,8 @@ module Network_manager = struct in let seed_nodes = [ cons_node network_config.docker.stack_name - (network_config.docker.stack_name ^ "_seed") + ( network_config.docker.stack_name ^ "_" + ^ Docker_node_config.Services.Seed.name ) None ] in @@ -506,12 +536,12 @@ module Network_manager = struct None) in let%bind _ = + (* If any archive nodes are specified, write the archive schema to the working directory to use as an entrypoint for postgres *) if List.length network_config.docker.archive_node_configs > 0 then let cmd = - Printf.sprintf "curl -Ls %s > create_schema.sql" - network_config.docker.mina_archive_schema + Printf.sprintf "curl -Ls %s > %s" + network_config.docker.mina_archive_schema mina_create_schema in - (* Write the archive schema to the working directory to use as an entrypoint for postgres*) Util.run_cmd_exn testnet_dir "bash" [ "-c"; cmd ] else return "" in From c6daae09553bcb9434f12993ff95539402db26d0 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Thu, 15 Jul 2021 09:55:37 -0700 Subject: [PATCH 12/25] Added GraphQL networking to docker containers Added functionality for GraphQL networking to the docker containers. This work was mostly just adding the '--insecure-rest-server' flag to the nodes and opening up the corresponding rest port within the docker-compose file. To allow communication to the docker network from localhost, we map ports on the host starting at 7000 to port 3085 to the node which is used for GraphQL communication. --- .../docker_compose.ml | 1 + .../docker_node_config.ml | 1 + .../mina_docker.ml | 133 +++++++-- .../swarm_network.ml | 258 ++++++++++++++++-- 4 files changed, 350 insertions(+), 43 deletions(-) diff --git a/src/lib/integration_test_local_engine/docker_compose.ml b/src/lib/integration_test_local_engine/docker_compose.ml index 45b26eb1145..72586ae4156 100644 --- a/src/lib/integration_test_local_engine/docker_compose.ml +++ b/src/lib/integration_test_local_engine/docker_compose.ml @@ -25,6 +25,7 @@ module Compose = struct { image : string ; volumes : Volume.t list ; command : string list + ; ports : string list ; environment : Environment.t } [@@deriving to_yojson] diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index c756af1b7d2..fc8ebf9ed57 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -142,6 +142,7 @@ module Cmd = struct ; "-peer" ; Peer.default ; "-log-json" + ; "--insecure-rest-server" ] let default_cmd ~config_file = default ~config_file |> to_string diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index fc7e157d6a7..cc3ad36b17f 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -17,9 +17,24 @@ let mina_create_schema = "create_schema.sql" module Network_config = struct module Cli_inputs = Cli_inputs + module Port = struct + type t = string list + + let get_new_port t = + match t with [] -> [ 7000 ] | x :: xs -> [ x + 1 ] @ [ x ] @ xs + + let get_latest_port t = match t with [] -> 7000 | x :: _ -> x + + let create_rest_port t = + match t with [] -> "7000:3085" | x :: _ -> Int.to_string x ^ ":3085" + end + type docker_volume_configs = { name : string; data : string } [@@deriving to_yojson] + type seed_config = { name : string; ports : (string * string) list } + [@@deriving to_yojson] + type block_producer_config = { name : string ; id : string @@ -28,6 +43,7 @@ module Network_config = struct ; private_key : string ; keypair_secret : string ; libp2p_secret : string + ; ports : (string * string) list } [@@deriving to_yojson] @@ -36,10 +52,16 @@ module Network_config = struct ; id : string ; public_key : string ; snark_worker_fee : string + ; ports : (string * string) list } [@@deriving to_yojson] - type archive_node_configs = { name : string; id : string; schema : string } + type archive_node_configs = + { name : string + ; id : string + ; schema : string + ; ports : (string * string) list + } [@@deriving to_yojson] type docker_config = @@ -48,6 +70,7 @@ module Network_config = struct ; mina_image : string ; mina_archive_image : string ; docker_volume_configs : docker_volume_configs list + ; seed_configs : seed_config list ; block_producer_configs : block_producer_config list ; snark_coordinator_configs : snark_coordinator_configs list ; archive_node_configs : archive_node_configs list @@ -201,8 +224,11 @@ module Network_config = struct { constraints = constraint_constants; genesis = genesis_constants } in let open Docker_node_config.Services in + let available_host_ports_ref = ref [] in (* BLOCK PRODUCER CONFIG *) let block_producer_config index keypair = + available_host_ports_ref := Port.get_new_port !available_host_ports_ref ; + let rest_port = Port.create_rest_port !available_host_ports_ref in { name = Block_producer.name ^ "-" ^ Int.to_string (index + 1) ; id = Int.to_string index ; keypair @@ -210,21 +236,36 @@ module Network_config = struct ; public_key = keypair.public_key_file ; private_key = keypair.private_key_file ; libp2p_secret = "" + ; ports = [ ("rest-port", rest_port) ] } in let block_producer_configs = List.mapi block_producer_keypairs ~f:block_producer_config in + (* SEED NODE CONFIG *) + available_host_ports_ref := Port.get_new_port !available_host_ports_ref ; + let seed_rest_port = Port.create_rest_port !available_host_ports_ref in + let seed_configs = + [ { name = Seed.name; ports = [ ("rest-port", seed_rest_port) ] } ] + in (* SNARK COORDINATOR CONFIG *) + available_host_ports_ref := Port.get_new_port !available_host_ports_ref ; + let snark_coord_rest_port = + Port.create_rest_port !available_host_ports_ref + in let snark_coordinator_configs = if num_snark_workers > 0 then List.mapi (List.init num_snark_workers ~f:(const 0)) ~f:(fun index _ -> + available_host_ports_ref := + Port.get_new_port !available_host_ports_ref ; + let rest_port = Port.create_rest_port !available_host_ports_ref in { name = Snark_worker.name ^ "-" ^ Int.to_string (index + 1) ; id = Int.to_string index ; snark_worker_fee ; public_key = snark_worker_public_key + ; ports = [ ("rest_port", rest_port) ] }) (* Add one snark coordinator for all workers *) |> List.append @@ -232,6 +273,7 @@ module Network_config = struct ; id = "1" ; snark_worker_fee ; public_key = snark_worker_public_key + ; ports = [ ("rest-port", snark_coord_rest_port) ] } ] else [] @@ -242,9 +284,13 @@ module Network_config = struct List.mapi (List.init num_archive_nodes ~f:(const 0)) ~f:(fun index _ -> + available_host_ports_ref := + Port.get_new_port !available_host_ports_ref ; + let rest_port = Port.create_rest_port !available_host_ports_ref in { name = Archive_node.name ^ "-" ^ Int.to_string (index + 1) ; id = Int.to_string index ; schema = mina_archive_schema + ; ports = [ ("rest-port", rest_port) ] }) else [] in @@ -271,6 +317,7 @@ module Network_config = struct ; mina_archive_image = images.archive_node ; docker_volume_configs ; runtime_config = Runtime_config.to_yojson runtime_config + ; seed_configs ; block_producer_configs ; snark_coordinator_configs ; archive_node_configs @@ -312,10 +359,17 @@ module Network_config = struct (Block_producer.default ~private_key_config:private_key_config.target)) in + let rest_port = + List.find + ~f:(fun ports -> String.equal (fst ports) "rest-port") + config.ports + |> Option.value_exn |> snd + in ( config.name , { Service.image = network_config.docker.mina_image ; volumes = [ private_key_config; runtime_config ] ; command = Cmd.create_cmd cmd ~config_file:runtime_config.target + ; ports = [ rest_port ] ; environment = Service.Environment.create Envs.base_node_envs } )) |> StringMap.of_alist_exn @@ -360,15 +414,22 @@ module Network_config = struct let worker_environment = Service.Environment.create [] in (worker_command, worker_environment) in + let rest_port = + List.find + ~f:(fun ports -> String.equal (fst ports) "rest-port") + config.ports + |> Option.value_exn |> snd + in ( config.name , { Service.image = network_config.docker.mina_image ; volumes = [ runtime_config ] ; command + ; ports = [ rest_port ] ; environment } )) |> StringMap.of_alist_exn in - (* ARCHIVE_NODE SERVICE *) + (* ARCHIVE NODE SERVICE *) let archive_node_configs = List.mapi network_config.docker.archive_node_configs ~f:(fun index config -> @@ -384,10 +445,17 @@ module Network_config = struct let server_port = Archive_node.server_port in Cmd.(Archive_node { Archive_node.postgres_uri; server_port }) in + let rest_port = + List.find + ~f:(fun ports -> String.equal (fst ports) "rest-port") + config.ports + |> Option.value_exn |> snd + in ( config.name , { Service.image = network_config.docker.mina_archive_image ; volumes = [ runtime_config ] ; command = Cmd.create_cmd cmd ~config_file:runtime_config.target + ; ports = [ rest_port ] ; environment = Service.Environment.create Envs.base_node_envs } )) (* Add an equal number of postgres containers as archive nodes *) @@ -402,6 +470,7 @@ module Network_config = struct , { Service.image = postgres_image ; volumes = [ archive_config ] ; command = [] + ; ports = [] ; environment = Service.Environment.create (Envs.postgres_envs ~username:pg_config.username @@ -410,23 +479,33 @@ module Network_config = struct } )) |> StringMap.of_alist_exn in - (* SEED DOCKER SERVICE *) + (* SEED NODE DOCKER SERVICE *) let seed_map = - [ (let command = - if List.length network_config.docker.archive_node_configs > 0 then - (* If an archive node is specified in the test plan, use the seed node to connect to the first mina-archive process *) - Cmd.create_cmd Seed ~config_file:runtime_config.target - @ Cmd.Seed.connect_to_archive - ~archive_node:("archive-1:" ^ Services.Archive_node.server_port) - else Cmd.create_cmd Seed ~config_file:runtime_config.target - in - ( Seed.name - , { Service.image = network_config.docker.mina_image - ; volumes = [ runtime_config ] - ; command - ; environment = Service.Environment.create Envs.base_node_envs - } )) - ] + List.mapi network_config.docker.seed_configs ~f:(fun index config -> + let command = + if List.length network_config.docker.archive_node_configs > 0 then + (* If an archive node is specified in the test plan, use the seed node to connect to the first mina-archive process *) + Cmd.create_cmd Seed ~config_file:runtime_config.target + @ Cmd.Seed.connect_to_archive + ~archive_node: + ( "archive-" + ^ Int.to_string (index + 1) + ^ ":" ^ Services.Archive_node.server_port ) + else Cmd.create_cmd Seed ~config_file:runtime_config.target + in + let rest_port = + List.find + ~f:(fun ports -> String.equal (fst ports) "rest-port") + config.ports + |> Option.value_exn |> snd + in + ( Seed.name + , { Service.image = network_config.docker.mina_image + ; volumes = [ runtime_config ] + ; command + ; ports = [ rest_port ] + ; environment = Service.Environment.create Envs.base_node_envs + } )) |> StringMap.of_alist_exn in let services = @@ -508,32 +587,32 @@ module Network_manager = struct Out_channel.with_file ~fail_if_exists:false (testnet_dir ^/ config.name) ~f:(fun ch -> config.data |> Out_channel.output_string ch) ; ignore (Util.run_cmd_exn testnet_dir "chmod" [ "600"; config.name ])) ; - let cons_node swarm_name service_id network_keypair_opt = + let cons_node swarm_name service_id ports network_keypair_opt = { Swarm_network.Node.swarm_name ; service_id + ; ports ; graphql_enabled = true ; network_keypair = network_keypair_opt } in let seed_nodes = - [ cons_node network_config.docker.stack_name - ( network_config.docker.stack_name ^ "_" - ^ Docker_node_config.Services.Seed.name ) - None - ] + List.map network_config.docker.seed_configs ~f:(fun seed_config -> + cons_node network_config.docker.stack_name + (network_config.docker.stack_name ^ "_" ^ seed_config.name) + seed_config.ports None) in let block_producer_nodes = List.map network_config.docker.block_producer_configs ~f:(fun bp_config -> cons_node network_config.docker.stack_name (network_config.docker.stack_name ^ "_" ^ bp_config.name) - (Some bp_config.keypair)) + bp_config.ports (Some bp_config.keypair)) in let snark_coordinator_nodes = List.map network_config.docker.snark_coordinator_configs ~f:(fun snark_config -> cons_node network_config.docker.stack_name (network_config.docker.stack_name ^ "_" ^ snark_config.name) - None) + snark_config.ports None) in let%bind _ = (* If any archive nodes are specified, write the archive schema to the working directory to use as an entrypoint for postgres *) @@ -550,7 +629,7 @@ module Network_manager = struct ~f:(fun archive_node -> cons_node network_config.docker.stack_name (network_config.docker.stack_name ^ "_" ^ archive_node.name) - None) + archive_node.ports None) in let nodes_by_app_id = let all_nodes = diff --git a/src/lib/integration_test_local_engine/swarm_network.ml b/src/lib/integration_test_local_engine/swarm_network.ml index 4d24b5b7b31..b38b10ba83a 100644 --- a/src/lib/integration_test_local_engine/swarm_network.ml +++ b/src/lib/integration_test_local_engine/swarm_network.ml @@ -9,6 +9,7 @@ module Node = struct type t = { swarm_name : string ; service_id : string + ; ports : (string * string) list ; graphql_enabled : bool ; network_keypair : Network_keypair.t option } @@ -63,28 +64,253 @@ module Node = struct module Decoders = Graphql_lib.Decoders - let exec_graphql_request ~num_tries:_ ~retry_delay_sec:_ ~initial_delay_sec:_ - ~logger:_ ~node:_ ~query_name:_ _ : _ = - failwith "exec_graphql_request" - - let get_peer_id ~logger:_ _ = failwith "get_peer_id" - - let must_get_peer_id ~logger:_ _ = failwith "must_get_peer_id" - - let get_best_chain ~logger:_ _ = failwith "get_best_chain" - - let must_get_best_chain ~logger:_ _ = failwith "must_get_best_chain" - - let get_balance ~logger:_ _ ~account_id:_ = failwith "get_balance" + module Graphql = struct + let ingress_uri node = + let host = Printf.sprintf "0.0.0.0" in + let path = "/graphql" in + let rest_port = + List.find + ~f:(fun ports -> String.equal (fst ports) "rest-port") + node.ports + |> Option.value_exn |> snd + in + Uri.make ~scheme:"http" ~host ~path ~port:(int_of_string rest_port) () + + module Client = Graphql_lib.Client.Make (struct + let preprocess_variables_string = Fn.id + + let headers = String.Map.empty + end) + + module Unlock_account = + [%graphql + {| + mutation ($password: String!, $public_key: PublicKey!) { + unlockAccount(input: {password: $password, publicKey: $public_key }) { + public_key: publicKey @bsDecoder(fn: "Decoders.public_key") + } + } + |}] + + module Send_payment = + [%graphql + {| + mutation ($sender: PublicKey!, + $receiver: PublicKey!, + $amount: UInt64!, + $token: UInt64, + $fee: UInt64!, + $nonce: UInt32, + $memo: String) { + sendPayment(input: + {from: $sender, to: $receiver, amount: $amount, token: $token, fee: $fee, nonce: $nonce, memo: $memo}) { + payment { + id + } + } + } + |}] + + module Get_balance = + [%graphql + {| + query ($public_key: PublicKey, $token: UInt64) { + account(publicKey: $public_key, token: $token) { + balance { + total @bsDecoder(fn: "Decoders.balance") + } + } + } + |}] + + module Query_peer_id = + [%graphql + {| + query { + daemonStatus { + addrsAndPorts { + peer { + peerId + } + } + peers { peerId } + + } + } + |}] + + module Best_chain = + [%graphql + {| + query { + bestChain { + stateHash + } + } + |}] + end + + let exec_graphql_request ?(num_tries = 10) ?(retry_delay_sec = 30.0) + ?(initial_delay_sec = 30.0) ~logger ~node ~query_name query_obj = + let open Deferred.Let_syntax in + if not node.graphql_enabled then + Deferred.Or_error.error_string + "graphql is not enabled (hint: set `requires_graphql= true` in the \ + test config)" + else + let uri = Graphql.ingress_uri node in + let metadata = + [ ("query", `String query_name); ("uri", `String (Uri.to_string uri)) ] + in + [%log info] "Attempting to send GraphQL request \"$query\" to \"$uri\"" + ~metadata ; + let rec retry n = + if n <= 0 then ( + [%log error] + "GraphQL request \"$query\" to \"$uri\" failed too many times" + ~metadata ; + Deferred.Or_error.errorf + "GraphQL \"%s\" to \"%s\" request failed too many times" query_name + (Uri.to_string uri) ) + else + match%bind Graphql.Client.query query_obj uri with + | Ok result -> + [%log info] "GraphQL request \"$query\" to \"$uri\" succeeded" + ~metadata ; + Deferred.Or_error.return result + | Error (`Failed_request err_string) -> + [%log warn] + "GraphQL request \"$query\" to \"$uri\" failed: \"$error\" \ + ($num_tries attempts left)" + ~metadata: + ( metadata + @ [ ("error", `String err_string) + ; ("num_tries", `Int (n - 1)) + ] ) ; + let%bind () = after (Time.Span.of_sec retry_delay_sec) in + retry (n - 1) + | Error (`Graphql_error err_string) -> + [%log error] + "GraphQL request \"$query\" to \"$uri\" returned an error: \ + \"$error\" (this is a graphql error so not retrying)" + ~metadata:(metadata @ [ ("error", `String err_string) ]) ; + Deferred.Or_error.error_string err_string + in + let%bind () = after (Time.Span.of_sec initial_delay_sec) in + retry num_tries + + let get_peer_id ~logger t = + let open Deferred.Or_error.Let_syntax in + [%log info] "Getting node's peer_id, and the peer_ids of node's peers" + ~metadata:[ ("service_id", `String t.service_id) ] ; + let query_obj = Graphql.Query_peer_id.make () in + let%bind query_result_obj = + exec_graphql_request ~logger ~node:t ~query_name:"query_peer_id" query_obj + in + [%log info] "get_peer_id, finished exec_graphql_request" ; + let self_id_obj = query_result_obj#daemonStatus#addrsAndPorts#peer in + let%bind self_id = + match self_id_obj with + | None -> + Deferred.Or_error.error_string "Peer not found" + | Some peer -> + return peer#peerId + in + let peers = query_result_obj#daemonStatus#peers |> Array.to_list in + let peer_ids = List.map peers ~f:(fun peer -> peer#peerId) in + [%log info] "get_peer_id, result of graphql query (self_id,[peers]) (%s,%s)" + self_id + (String.concat ~sep:" " peer_ids) ; + return (self_id, peer_ids) + + let must_get_peer_id ~logger t = + get_peer_id ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error + + let get_best_chain ~logger t = + let open Deferred.Or_error.Let_syntax in + let query = Graphql.Best_chain.make () in + let%bind result = + exec_graphql_request ~logger ~node:t ~query_name:"best_chain" query + in + match result#bestChain with + | None | Some [||] -> + Deferred.Or_error.error_string "failed to get best chains" + | Some chain -> + return + @@ List.map ~f:(fun block -> block#stateHash) (Array.to_list chain) + + let must_get_best_chain ~logger t = + get_best_chain ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error + + let get_balance ~logger t ~account_id = + let open Deferred.Or_error.Let_syntax in + [%log info] "Getting account balance" + ~metadata: + [ ("service_id", `String t.service_id) + ; ("account_id", Mina_base.Account_id.to_yojson account_id) + ] ; + let pk = Mina_base.Account_id.public_key account_id in + let token = Mina_base.Account_id.token_id account_id in + let get_balance_obj = + Graphql.Get_balance.make + ~public_key:(Graphql_lib.Encoders.public_key pk) + ~token:(Graphql_lib.Encoders.token token) + () + in + let%bind balance_obj = + exec_graphql_request ~logger ~node:t ~query_name:"get_balance_graphql" + get_balance_obj + in + match balance_obj#account with + | None -> + Deferred.Or_error.errorf + !"Account with %{sexp:Mina_base.Account_id.t} not found" + account_id + | Some acc -> + return acc#balance#total let must_get_balance ~logger t ~account_id = get_balance ~logger t ~account_id |> Deferred.bind ~f:Malleable_error.or_hard_error (* if we expect failure, might want retry_on_graphql_error to be false *) - let send_payment ~logger:_ _ ~sender_pub_key:_ ~receiver_pub_key:_ ~amount:_ - ~fee:_ = - failwith "send_payment" + let send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee = + [%log info] "Sending a payment" + ~metadata:[ ("service_id", `String t.service_id) ] ; + let open Deferred.Or_error.Let_syntax in + let sender_pk_str = + Signature_lib.Public_key.Compressed.to_string sender_pub_key + in + [%log info] "send_payment: unlocking account" + ~metadata:[ ("sender_pk", `String sender_pk_str) ] ; + let unlock_sender_account_graphql () = + let unlock_account_obj = + Graphql.Unlock_account.make ~password:"naughty blue worm" + ~public_key:(Graphql_lib.Encoders.public_key sender_pub_key) + () + in + exec_graphql_request ~logger ~node:t + ~query_name:"unlock_sender_account_graphql" unlock_account_obj + in + let%bind _ = unlock_sender_account_graphql () in + let send_payment_graphql () = + let send_payment_obj = + Graphql.Send_payment.make + ~sender:(Graphql_lib.Encoders.public_key sender_pub_key) + ~receiver:(Graphql_lib.Encoders.public_key receiver_pub_key) + ~amount:(Graphql_lib.Encoders.amount amount) + ~fee:(Graphql_lib.Encoders.fee fee) + () + in + exec_graphql_request ~logger ~node:t ~query_name:"send_payment_graphql" + send_payment_obj + in + let%map sent_payment_obj = send_payment_graphql () in + let (`UserCommand id_obj) = sent_payment_obj#sendPayment#payment in + let user_cmd_id = id_obj#id in + [%log info] "Sent payment" + ~metadata:[ ("user_command_id", `String user_cmd_id) ] ; + () let must_send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee = From 71cfc6d816e87396d61ea8093e6d8bcf80ec0302 Mon Sep 17 00:00:00 2001 From: Ilya Polyakovskiy Date: Fri, 12 Nov 2021 18:16:47 +0300 Subject: [PATCH 13/25] Fix compilation --- src/lib/integration_test_local_engine/mina_docker.ml | 2 +- src/lib/integration_test_local_engine/swarm_network.ml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index cc3ad36b17f..d56dc7a8b61 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -313,7 +313,7 @@ module Network_config = struct ; docker = { docker_swarm_version ; stack_name - ; mina_image = images.coda + ; mina_image = images.mina ; mina_archive_image = images.archive_node ; docker_volume_configs ; runtime_config = Runtime_config.to_yojson runtime_config diff --git a/src/lib/integration_test_local_engine/swarm_network.ml b/src/lib/integration_test_local_engine/swarm_network.ml index b38b10ba83a..27974d3282c 100644 --- a/src/lib/integration_test_local_engine/swarm_network.ml +++ b/src/lib/integration_test_local_engine/swarm_network.ml @@ -320,8 +320,7 @@ module Node = struct let dump_archive_data ~logger:_ (_ : t) ~data_file:_ = failwith "dump_archive_data" - let dump_container_logs ~logger:_ (_ : t) ~log_file:_ = - Malleable_error.return () + let dump_mina_logs ~logger:_ (_ : t) ~log_file:_ = Malleable_error.return () let dump_precomputed_blocks ~logger:_ (_ : t) = Malleable_error.return () end From 8613893c594e8ebcca66b008a533554bd682977c Mon Sep 17 00:00:00 2001 From: Ilya Polyakovskiy Date: Tue, 21 Dec 2021 16:12:59 +0300 Subject: [PATCH 14/25] add local version for rebuild-deb script --- scripts/export-local-git-env-vars.sh | 22 ++++++++++++++++++++++ scripts/rebuild-deb.sh | 13 +++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) create mode 100755 scripts/export-local-git-env-vars.sh diff --git a/scripts/export-local-git-env-vars.sh b/scripts/export-local-git-env-vars.sh new file mode 100755 index 00000000000..58c671b1fa9 --- /dev/null +++ b/scripts/export-local-git-env-vars.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -eo pipefail + +set +x +echo "Exporting Variables: " + +export GITHASH=$(git rev-parse --short=7 HEAD) +export GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!g; s!_!-!g' ) +# GITTAG is the closest tagged commit to this commit, while THIS_COMMIT_TAG only has a value when the current commit is tagged +export GITTAG=$(git describe --always --abbrev=0 | sed 's!/!-!g; s!_!-!g') +export PROJECT="mina" + +set +u +export BUILD_NUM=1 +export BUILD_URL="local" +set -u + +export MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=stretch} +export MINA_DEB_VERSION="${GITTAG}-${GITBRANCH}-${GITHASH}" +export MINA_DEB_RELEASE="unstable" +export MINA_DOCKER_TAG="$(echo "${MINA_DEB_VERSION}" | sed 's!/!-!g; s!_!-!g')-${MINA_DEB_CODENAME}" + diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index 5536da18d72..a654f6e7f9e 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -10,13 +10,14 @@ cd "${SCRIPTPATH}/../_build" GITHASH=$(git rev-parse --short=7 HEAD) GITHASH_CONFIG=$(git rev-parse --short=8 --verify HEAD) -set +u -BUILD_NUM=${BUILDKITE_BUILD_NUM} -BUILD_URL=${BUILDKITE_BUILD_URL} -set -u - # Load in env vars for githash/branch/etc. -source "${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh" +if [[ "$1" == "local" ]]; then + source "${SCRIPTPATH}/export-local-git-env-vars.sh" +else + source "${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh" +fi + +MINA_DEB_CODENAME=stretch cd "${SCRIPTPATH}/../_build" From b0ca83b8885cd97aebdeffc72fa63223afe1f2d1 Mon Sep 17 00:00:00 2001 From: Ilya Polyakovskiy Date: Tue, 28 Dec 2021 17:18:56 +0300 Subject: [PATCH 15/25] added local/buildkite modes to rebuild-deb.sh --- scripts/rebuild-deb.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index a654f6e7f9e..a480d0affb6 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -10,12 +10,14 @@ cd "${SCRIPTPATH}/../_build" GITHASH=$(git rev-parse --short=7 HEAD) GITHASH_CONFIG=$(git rev-parse --short=8 --verify HEAD) +mode="${1-buildkite}" + # Load in env vars for githash/branch/etc. -if [[ "$1" == "local" ]]; then - source "${SCRIPTPATH}/export-local-git-env-vars.sh" -else - source "${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh" -fi +case $mode in + local) source "${SCRIPTPATH}/export-local-git-env-vars.sh";; + buildkite)"${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh";; + *) echo "Unknown mode passed: $mode"; exit 1;; +esac MINA_DEB_CODENAME=stretch From 4c3cca318ea9cfdf119336d5a0d2b449a77aaf14 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Mon, 27 Nov 2023 18:51:43 -0800 Subject: [PATCH 16/25] catchup latest develop changes to local integration lib --- .../integration_test_lib/graphql_requests.ml | 4 +- .../docker_compose.ml | 76 +- .../docker_network.ml | 330 ++++ .../docker_node_config.ml | 770 +++++---- .../docker_pipe_log_engine.ml | 137 +- .../docker_pipe_log_engine.mli | 2 +- .../integration_test_local_engine.ml | 2 +- .../mina_docker.ml | 1410 ++++++++++------- .../swarm_network.ml | 453 ------ 9 files changed, 1892 insertions(+), 1292 deletions(-) create mode 100644 src/lib/integration_test_local_engine/docker_network.ml delete mode 100644 src/lib/integration_test_local_engine/swarm_network.ml diff --git a/src/lib/integration_test_lib/graphql_requests.ml b/src/lib/integration_test_lib/graphql_requests.ml index 2cbcefb263d..94a0aa69218 100644 --- a/src/lib/integration_test_lib/graphql_requests.ml +++ b/src/lib/integration_test_lib/graphql_requests.ml @@ -1083,8 +1083,8 @@ let start_filtered_log ~logger ~log_filter node_uri = Graphql.StartFilteredLog.(make @@ makeVariables ~filter:log_filter ()) in let%bind res = - exec_graphql_request ~logger:(Logger.null ()) ~retry_delay_sec:10.0 - ~node_uri ~query_name:"StartFilteredLog" query_obj + exec_graphql_request ~logger:(Logger.null ()) ~retry_delay_sec:0.25 + ~num_tries:100 ~node_uri ~query_name:"StartFilteredLog" query_obj in match res with | Ok query_result_obj -> diff --git a/src/lib/integration_test_local_engine/docker_compose.ml b/src/lib/integration_test_local_engine/docker_compose.ml index 72586ae4156..e374b97f3ed 100644 --- a/src/lib/integration_test_local_engine/docker_compose.ml +++ b/src/lib/integration_test_local_engine/docker_compose.ml @@ -1,8 +1,7 @@ -open Core - -module Compose = struct - module StringMap = Map.Make (String) +open Core_kernel +open Integration_test_lib +module Dockerfile = struct module Service = struct module Volume = struct type t = @@ -10,27 +9,74 @@ module Compose = struct [@@deriving to_yojson] let create source target = { type_ = "bind"; source; target } + + let write_config docker_dir ~filename ~data = + Out_channel.with_file ~fail_if_exists:false + (docker_dir ^ "/" ^ filename) + ~f:(fun ch -> data |> Out_channel.output_string ch) ; + ignore (Util.run_cmd_exn docker_dir "chmod" [ "600"; filename ]) end module Environment = struct - type t = string StringMap.t + type t = (string * string) list - let create = StringMap.of_alist_exn + let default = + [ ("DAEMON_REST_PORT", "3085") + ; ("DAEMON_CLIENT_PORT", "8301") + ; ("DAEMON_METRICS_PORT", "10001") + ; ("DAEMON_EXTERNAL_PORT", "10101") + ; ("MINA_PRIVKEY_PASS", "naughty blue worm") + ; ("MINA_LIBP2P_PASS", "") + ; ("RAYON_NUM_THREADS", "6") + ] - let to_yojson m = - `Assoc (m |> Map.map ~f:(fun x -> `String x) |> Map.to_alist) + let to_yojson env = `Assoc (List.map env ~f:(fun (k, v) -> (k, `String v))) + end + + module Port = struct + type t = { published : int; target : int } [@@deriving to_yojson] + + let create ~published ~target = { published; target } end type t = { image : string - ; volumes : Volume.t list ; command : string list - ; ports : string list + ; entrypoint : string list option + [@to_yojson + fun j -> + match j with + | Some v -> + `List (List.map (fun s -> `String s) v) + | None -> + `Null] + ; ports : Port.t list ; environment : Environment.t + ; volumes : Volume.t list } [@@deriving to_yojson] + + let create ~image ~command ~entrypoint ~ports ~environment ~volumes = + { image; command; entrypoint; ports; environment; volumes } + + let to_yojson { image; command; entrypoint; ports; environment; volumes } = + `Assoc + ( [ ("image", `String image) + ; ("command", `List (List.map ~f:(fun s -> `String s) command)) + ; ("ports", `List (List.map ~f:Port.to_yojson ports)) + ; ("environment", Environment.to_yojson environment) + ; ("volumes", `List (List.map ~f:Volume.to_yojson volumes)) + ] + @ + match entrypoint with + | Some ep -> + [ ("entrypoint", `List (List.map ~f:(fun s -> `String s) ep)) ] + | None -> + [] ) end + module StringMap = Map.Make (String) + type service_map = Service.t StringMap.t let merge (m1 : service_map) (m2 : service_map) = @@ -40,8 +86,12 @@ module Compose = struct `Assoc (m |> Map.map ~f:Service.to_yojson |> Map.to_alist) type t = { version : string; services : service_map } [@@deriving to_yojson] -end -type t = Compose.t [@@deriving to_yojson] + let to_string = Fn.compose Yojson.Safe.pretty_to_string to_yojson -let to_string = Fn.compose Yojson.Safe.pretty_to_string to_yojson + let write_config t ~dir ~filename = + Out_channel.with_file ~fail_if_exists:false + (dir ^ "/" ^ filename) + ~f:(fun ch -> t |> to_string |> Out_channel.output_string ch) ; + Util.run_cmd_exn dir "chmod" [ "600"; filename ] +end diff --git a/src/lib/integration_test_local_engine/docker_network.ml b/src/lib/integration_test_local_engine/docker_network.ml new file mode 100644 index 00000000000..e8a73532278 --- /dev/null +++ b/src/lib/integration_test_local_engine/docker_network.ml @@ -0,0 +1,330 @@ +open Core_kernel +open Async +open Integration_test_lib + +[@@@warning "-27"] + +let get_container_id service_id = + let%bind cwd = Unix.getcwd () in + let open Malleable_error.Let_syntax in + let%bind container_ids = + Deferred.bind ~f:Malleable_error.or_hard_error + (Integration_test_lib.Util.run_cmd_or_error cwd "docker" + [ "ps"; "-f"; sprintf "name=%s" service_id; "--quiet" ] ) + in + let container_id_list = String.split container_ids ~on:'\n' in + match container_id_list with + | [] -> + Malleable_error.hard_error_format "No container id found for service %s" + service_id + | raw_container_id :: _ -> + return (String.strip raw_container_id) + +let run_in_container ?(exit_code = 10) container_id ~cmd = + let%bind.Deferred cwd = Unix.getcwd () in + Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code cwd "docker" + ([ "exec"; container_id ] @ cmd) + +module Node = struct + type config = + { network_keypair : Network_keypair.t option + ; service_id : string + ; postgres_connection_uri : string option + ; graphql_port : int + } + + type t = { config : config; mutable should_be_running : bool } + + let id { config; _ } = config.service_id + + let infra_id { config; _ } = config.service_id + + let should_be_running { should_be_running; _ } = should_be_running + + let network_keypair { config; _ } = config.network_keypair + + let get_ingress_uri node = + Uri.make ~scheme:"http" ~host:"127.0.0.1" ~path:"/graphql" + ~port:node.config.graphql_port () + + let get_container_index_from_service_name service_name = + match String.split_on_chars ~on:[ '_' ] service_name with + | _ :: value :: _ -> + value + | _ -> + failwith "get_container_index_from_service_name: bad service name" + + let dump_archive_data ~logger (t : t) ~data_file = + let service_name = t.config.service_id in + match t.config.postgres_connection_uri with + | None -> + failwith + (sprintf "dump_archive_data: %s not an archive container" service_name) + | Some postgres_uri -> + let open Malleable_error.Let_syntax in + let%bind container_id = get_container_id service_name in + [%log info] "Dumping archive data from (node: %s, container: %s)" + service_name container_id ; + let%map data = + run_in_container container_id + ~cmd:[ "pg_dump"; "--create"; "--no-owner"; postgres_uri ] + in + [%log info] "Dumping archive data to file %s" data_file ; + Out_channel.with_file data_file ~f:(fun out_ch -> + Out_channel.output_string out_ch data ) + + let get_logs_in_container container_id = + let%bind.Deferred cwd = Unix.getcwd () in + Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code:13 cwd "docker" + [ "logs"; container_id ] + + let dump_mina_logs ~logger (t : t) ~log_file = + let open Malleable_error.Let_syntax in + let%bind container_id = get_container_id t.config.service_id in + [%log info] "Dumping mina logs from (node: %s, container: %s)" + t.config.service_id container_id ; + let%map logs = get_logs_in_container container_id in + [%log info] "Dumping mina logs to file %s" log_file ; + Out_channel.with_file log_file ~f:(fun out_ch -> + Out_channel.output_string out_ch logs ) + + let cp_string_to_container_file container_id ~str ~dest = + let tmp_file, oc = + Caml.Filename.open_temp_file ~temp_dir:Filename.temp_dir_name + "integration_test_cp_string" ".tmp" + in + Out_channel.output_string oc str ; + Out_channel.close oc ; + let%bind cwd = Unix.getcwd () in + let dest_file = sprintf "%s:%s" container_id dest in + Integration_test_lib.Util.run_cmd_or_error cwd "docker" + [ "cp"; tmp_file; dest_file ] + + let run_replayer ?(start_slot_since_genesis = 0) ~logger (t : t) = + let open Malleable_error.Let_syntax in + let%bind container_id = get_container_id t.config.service_id in + [%log info] "Running replayer on (node: %s, container: %s)" + t.config.service_id container_id ; + let%bind accounts = + run_in_container container_id + ~cmd:[ "jq"; "-c"; ".ledger.accounts"; "/root/runtime_config.json" ] + in + let replayer_input = + sprintf + {| { "start_slot_since_genesis": %d, + "genesis_ledger": { "accounts": %s, "add_genesis_winner": true }} |} + start_slot_since_genesis accounts + in + let dest = "replayer-input.json" in + let%bind archive_container_id = get_container_id "archive" in + let%bind () = + Deferred.bind ~f:Malleable_error.return + (cp_string_to_container_file archive_container_id ~str:replayer_input + ~dest ) + >>| ignore + in + let postgres_url = Option.value_exn t.config.postgres_connection_uri in + run_in_container container_id + ~cmd: + [ "mina-replayer" + ; "--archive-uri" + ; postgres_url + ; "--input-file" + ; dest + ; "--output-file" + ; "/dev/null" + ; "--continue-on-error" + ] + + let dump_precomputed_blocks ~logger (t : t) = + let open Malleable_error.Let_syntax in + let container_id = t.config.service_id in + [%log info] + "Dumping precomputed blocks from logs for (node: %s, container: %s)" + t.config.service_id container_id ; + let%bind logs = get_logs_in_container container_id in + (* kubectl logs may include non-log output, like "Using password from environment variable" *) + let log_lines = + String.split logs ~on:'\n' + |> List.filter ~f:(String.is_prefix ~prefix:"{\"timestamp\":") + in + let jsons = List.map log_lines ~f:Yojson.Safe.from_string in + let metadata_jsons = + List.map jsons ~f:(fun json -> + match json with + | `Assoc items -> ( + match List.Assoc.find items ~equal:String.equal "metadata" with + | Some md -> + md + | None -> + failwithf "Log line is missing metadata: %s" + (Yojson.Safe.to_string json) + () ) + | other -> + failwithf "Expected log line to be a JSON record, got: %s" + (Yojson.Safe.to_string other) + () ) + in + let state_hash_and_blocks = + List.fold metadata_jsons ~init:[] ~f:(fun acc json -> + match json with + | `Assoc items -> ( + match + List.Assoc.find items ~equal:String.equal "precomputed_block" + with + | Some block -> ( + match + List.Assoc.find items ~equal:String.equal "state_hash" + with + | Some state_hash -> + (state_hash, block) :: acc + | None -> + failwith + "Log metadata contains a precomputed block, but no \ + state hash" ) + | None -> + acc ) + | other -> + failwithf "Expected log line to be a JSON record, got: %s" + (Yojson.Safe.to_string other) + () ) + in + let%bind.Deferred () = + Deferred.List.iter state_hash_and_blocks + ~f:(fun (state_hash_json, block_json) -> + let double_quoted_state_hash = + Yojson.Safe.to_string state_hash_json + in + let state_hash = + String.sub double_quoted_state_hash ~pos:1 + ~len:(String.length double_quoted_state_hash - 2) + in + let block = Yojson.Safe.pretty_to_string block_json in + let filename = state_hash ^ ".json" in + match%map.Deferred Sys.file_exists filename with + | `Yes -> + [%log info] + "File already exists for precomputed block with state hash %s" + state_hash + | _ -> + [%log info] + "Dumping precomputed block with state hash %s to file %s" + state_hash filename ; + Out_channel.with_file filename ~f:(fun out_ch -> + Out_channel.output_string out_ch block ) ) + in + Malleable_error.return () + + let start ~fresh_state node : unit Malleable_error.t = + let open Malleable_error.Let_syntax in + let%bind container_id = get_container_id node.config.service_id in + node.should_be_running <- true ; + let%bind () = + if fresh_state then + run_in_container container_id ~cmd:[ "rm"; "-rf"; ".mina-config/*" ] + >>| ignore + else Malleable_error.return () + in + run_in_container ~exit_code:11 container_id ~cmd:[ "/start.sh" ] >>| ignore + + let stop node = + let open Malleable_error.Let_syntax in + let%bind container_id = get_container_id node.config.service_id in + node.should_be_running <- false ; + run_in_container ~exit_code:12 container_id ~cmd:[ "/stop.sh" ] >>| ignore +end + +module Service_to_deploy = struct + type config = + { network_keypair : Network_keypair.t option + ; postgres_connection_uri : string option + ; graphql_port : int + } + + type t = { stack_name : string; service_name : string; config : config } + + let construct_service stack_name service_name config : t = + { stack_name; service_name; config } + + let init_service_to_deploy_config ?(network_keypair = None) + ?(postgres_connection_uri = None) ~graphql_port = + { network_keypair; postgres_connection_uri; graphql_port } + + let get_node_from_service t = + let%bind cwd = Unix.getcwd () in + let open Malleable_error.Let_syntax in + let service_id = t.stack_name ^ "_" ^ t.service_name in + let%bind container_id = get_container_id service_id in + if String.is_empty container_id then + Malleable_error.hard_error_format "No container id found for service %s" + t.service_name + else + return + { Node.config = + { service_id + ; network_keypair = t.config.network_keypair + ; postgres_connection_uri = t.config.postgres_connection_uri + ; graphql_port = t.config.graphql_port + } + ; should_be_running = false + } +end + +type t = + { namespace : string + ; constants : Test_config.constants + ; seeds : Node.t Core.String.Map.t + ; block_producers : Node.t Core.String.Map.t + ; snark_coordinators : Node.t Core.String.Map.t + ; snark_workers : Node.t Core.String.Map.t + ; archive_nodes : Node.t Core.String.Map.t + ; genesis_keypairs : Network_keypair.t Core.String.Map.t + } + +let constants { constants; _ } = constants + +let constraint_constants { constants; _ } = constants.constraints + +let genesis_constants { constants; _ } = constants.genesis + +let seeds { seeds; _ } = seeds + +let block_producers { block_producers; _ } = block_producers + +let snark_coordinators { snark_coordinators; _ } = snark_coordinators + +let archive_nodes { archive_nodes; _ } = archive_nodes + +let all_mina_nodes { seeds; block_producers; snark_coordinators; _ } = + List.concat + [ Core.String.Map.to_alist seeds + ; Core.String.Map.to_alist block_producers + ; Core.String.Map.to_alist snark_coordinators + ] + |> Core.String.Map.of_alist_exn + +let all_nodes t = + List.concat + [ Core.String.Map.to_alist t.seeds + ; Core.String.Map.to_alist t.block_producers + ; Core.String.Map.to_alist t.snark_coordinators + ; Core.String.Map.to_alist t.snark_workers + ] + |> Core.String.Map.of_alist_exn + +let all_non_seed_nodes t = + List.concat + [ Core.String.Map.to_alist t.block_producers + ; Core.String.Map.to_alist t.snark_coordinators + ; Core.String.Map.to_alist t.snark_workers + ] + |> Core.String.Map.of_alist_exn + +let genesis_keypairs { genesis_keypairs; _ } = genesis_keypairs + +let all_ids t = + let deployments = all_nodes t |> Core.Map.to_alist in + List.fold deployments ~init:[] ~f:(fun acc (_, node) -> + List.cons node.config.service_id acc ) + +let initialize_infra ~logger network = Malleable_error.return () diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index fc8ebf9ed57..aa9a89d8452 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -1,132 +1,145 @@ -open Base - -module Envs = struct - let base_node_envs = - [ ("DAEMON_REST_PORT", "3085") - ; ("DAEMON_CLIENT_PORT", "8301") - ; ("DAEMON_METRICS_PORT", "10001") - ; ("CODA_PRIVKEY_PASS", "naughty blue worm") - ; ("CODA_LIBP2P_PASS", "") - ] +open Core_kernel +open Async +open Integration_test_lib +open Docker_compose - let snark_coord_envs ~snark_coordinator_key ~snark_worker_fee = - [ ("CODA_SNARK_KEY", snark_coordinator_key) - ; ("CODA_SNARK_FEE", snark_worker_fee) - ; ("WORK_SELECTION", "seq") - ] - @ base_node_envs +module PortManager = struct + let mina_internal_rest_port = 3085 - let postgres_envs ~username ~password ~database ~port = - [ ("BITNAMI_DEBUG", "false") - ; ("POSTGRES_USER", username) - ; ("POSTGRES_PASSWORD", password) - ; ("POSTGRES_DB", database) - ; ("POSTGRESQL_PORT_NUMBER", port) - ; ("POSTGRESQL_ENABLE_LDAP", "no") - ; ("POSTGRESQL_ENABLE_TLS", "no") - ; ("POSTGRESQL_LOG_HOSTNAME", "false") - ; ("POSTGRESQL_LOG_CONNECTIONS", "false") - ; ("POSTGRESQL_LOG_DISCONNECTIONS", "false") - ; ("POSTGRESQL_PGAUDIT_LOG_CATALOG", "off") - ; ("POSTGRESQL_CLIENT_MIN_MESSAGES", "error") - ; ("POSTGRESQL_SHARED_PRELOAD_LIBRARIES", "pgaudit") - ; ("POSTGRES_HOST_AUTH_METHOD", "trust") - ] -end + let mina_internal_client_port = 8301 -module Volumes = struct - module Runtime_config = struct - let name = "runtime_config" + let mina_internal_metrics_port = 10001 - let container_mount_target = "/root/" ^ name - end -end + let mina_internal_server_port = 3086 -module Cmd = struct - module Cli_args = struct - module Log_level = struct - type t = Debug - - let to_string t = match t with Debug -> "Debug" - end - - module Peer = struct - type t = string - - let default = - "/dns4/seed/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" - end - - module Postgres_uri = struct - type t = - { username : string - ; password : string - ; host : string - ; port : string - ; db : string - } - - let create ~host = - { username = "postgres" - ; password = "password" - ; host - ; port = "5432" - ; db = "archive" - } - - (* Hostname should be dynamic based on the container ID in runtime. Ignore this field for default binding *) - let default = - { username = "postgres" - ; password = "password" - ; host = "" - ; port = "5432" - ; db = "archive" - } - - let to_string t = - Printf.sprintf "postgres://%s:%s@%s:%s/%s" t.username t.password t.host - t.port t.db - end - - module Proof_level = struct - type t = Full - - let to_string t = match t with Full -> "Full" - end - end - - open Cli_args - - module Base = struct - type t = - { peer : Peer.t - ; log_level : Log_level.t - ; log_snark_work_gossip : bool - ; log_txn_pool_gossip : bool - ; generate_genesis_proof : bool - ; client_port : string - ; rest_port : string - ; metrics_port : string - ; config_file : string - } + let mina_internal_external_port = 10101 + + let postgres_internal_port = 5432 - let default ~config_file = - { peer = Peer.default - ; log_level = Log_level.Debug - ; log_snark_work_gossip = true - ; log_txn_pool_gossip = true - ; generate_genesis_proof = true - ; client_port = "8301" - ; rest_port = "3085" - ; metrics_port = "10001" - ; config_file + type t = + { mutable available_ports : int list + ; mutable used_ports : int list + ; min_port : int + ; max_port : int + } + + let create ~min_port ~max_port = + let available_ports = List.range min_port max_port in + { available_ports; used_ports = []; min_port; max_port } + + let allocate_port t = + match t.available_ports with + | [] -> + failwith "No available ports" + | port :: rest -> + t.available_ports <- rest ; + t.used_ports <- port :: t.used_ports ; + port + + let allocate_ports_for_node t = + let rest_port_source = allocate_port t in + let client_port_source = allocate_port t in + let metrics_port_source = allocate_port t in + [ { Dockerfile.Service.Port.published = rest_port_source + ; target = mina_internal_rest_port } + ; { published = client_port_source; target = mina_internal_client_port } + ; { published = metrics_port_source; target = mina_internal_metrics_port } + ] - let to_string t = - [ "-config-file" - ; t.config_file - ; "-log-level" - ; Log_level.to_string t.log_level + let release_port t port = + t.used_ports <- List.filter t.used_ports ~f:(fun p -> p <> port) ; + t.available_ports <- port :: t.available_ports + + let get_latest_used_port t = + match t.used_ports with [] -> failwith "No used ports" | port :: _ -> port +end + +module Base_node_config = struct + type t = + { peer : string option + ; log_level : string + ; log_snark_work_gossip : bool + ; log_txn_pool_gossip : bool + ; generate_genesis_proof : bool + ; client_port : string + ; rest_port : string + ; external_port : string + ; metrics_port : string + ; runtime_config_path : string option + ; libp2p_key_path : string + ; libp2p_secret : string + } + [@@deriving to_yojson] + + let container_runtime_config_path = "/root/runtime_config.json" + + let container_entrypoint_path = "/root/entrypoint.sh" + + let container_keys_path = "/root/keys" + + let container_libp2p_key_path = container_keys_path ^ "/libp2p_key" + + let entrypoint_script = + ( "entrypoint.sh" + , {|#!/bin/bash + # This file is auto-generated by the local integration test framework. + # Path to the libp2p_key file + LIBP2P_KEY_PATH="|} + ^ container_libp2p_key_path + ^ {|" + # Generate keypair and set permissions if libp2p_key does not exist + if [ ! -f "$LIBP2P_KEY_PATH" ]; then + mina libp2p generate-keypair --privkey-path $LIBP2P_KEY_PATH + fi + /bin/chmod -R 700 |} + ^ container_keys_path ^ {|/ + # Import any compatible keys in |} + ^ container_keys_path ^ {|/*, excluding certain keys + for key_file in |} + ^ container_keys_path + ^ {|/*; do + # Exclude specific keys (e.g., libp2p keys) + if [[ $(basename "$key_file") != "libp2p_key" ]]; then + mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" + fi + done + # Execute the puppeteer script + exec /mina_daemon_puppeteer.py "$@" + |} + ) + + let runtime_config_volume : Docker_compose.Dockerfile.Service.Volume.t = + { type_ = "bind" + ; source = "runtime_config.json" + ; target = container_runtime_config_path + } + + let entrypoint_volume : Docker_compose.Dockerfile.Service.Volume.t = + { type_ = "bind" + ; source = "entrypoint.sh" + ; target = container_entrypoint_path + } + + let default ?(runtime_config_path = None) ?(peer = None) = + { log_snark_work_gossip = true + ; log_txn_pool_gossip = true + ; generate_genesis_proof = true + ; log_level = "Debug" + ; client_port = PortManager.mina_internal_client_port |> Int.to_string + ; rest_port = PortManager.mina_internal_rest_port |> Int.to_string + ; metrics_port = PortManager.mina_internal_metrics_port |> Int.to_string + ; external_port = PortManager.mina_internal_external_port |> Int.to_string + ; runtime_config_path + ; libp2p_key_path = container_libp2p_key_path + ; libp2p_secret = "" + ; peer + } + + let to_list t = + let base_args = + [ "-log-level" + ; t.log_level ; "-log-snark-work-gossip" ; Bool.to_string t.log_snark_work_gossip ; "-log-txn-pool-gossip" @@ -137,184 +150,401 @@ module Cmd = struct ; t.client_port ; "-rest-port" ; t.rest_port + ; "-external-port" + ; t.external_port ; "-metrics-port" ; t.metrics_port - ; "-peer" - ; Peer.default + ; "--libp2p-keypair" + ; t.libp2p_key_path ; "-log-json" ; "--insecure-rest-server" + ; "-external-ip" + ; "0.0.0.0" ] + in + let peer_args = + match t.peer with Some peer -> [ "-peer"; peer ] | None -> [] + in + let runtime_config_path = + match t.runtime_config_path with + | Some path -> + [ "-config-file"; path ] + | None -> + [] + in + List.concat [ base_args; runtime_config_path; peer_args ] +end - let default_cmd ~config_file = default ~config_file |> to_string - end - - module Seed = struct - let cmd ~config_file = [ "daemon"; "-seed" ] @ Base.default_cmd ~config_file - - let connect_to_archive ~archive_node = [ "-archive-address"; archive_node ] - end - - module Block_producer = struct - type t = - { block_producer_key : string - ; enable_flooding : bool - ; enable_peer_exchange : bool - } - - let default ~private_key_config = - { block_producer_key = private_key_config - ; enable_flooding = true - ; enable_peer_exchange = true - } +module Block_producer_config = struct + type config = + { keypair : Network_keypair.t + ; priv_key_path : string + ; enable_flooding : bool + ; enable_peer_exchange : bool + ; peer : string option + ; libp2p_secret : string + ; runtime_config_path : string + } + [@@deriving to_yojson] - let cmd t ~config_file = + type t = + { service_name : string + ; config : config + ; docker_config : Dockerfile.Service.t + } + [@@deriving to_yojson] + + let create_cmd config = + let base_args = + Base_node_config.to_list + (Base_node_config.default + ~runtime_config_path:(Some config.runtime_config_path) + ~peer:config.peer ) + in + let block_producer_args = [ "daemon" ; "-block-producer-key" - ; t.block_producer_key + ; config.priv_key_path ; "-enable-flooding" - ; Bool.to_string t.enable_flooding + ; config.enable_flooding |> Bool.to_string ; "-enable-peer-exchange" - ; Bool.to_string t.enable_peer_exchange + ; config.enable_peer_exchange |> Bool.to_string ] - @ Base.default_cmd ~config_file - end - - module Snark_coordinator = struct - type t = - { snark_coordinator_key : string - ; snark_worker_fee : string - ; work_selection : string - } - - let default ~snark_coordinator_key ~snark_worker_fee = - { snark_coordinator_key; snark_worker_fee; work_selection = "seq" } - - let cmd t ~config_file = - [ "daemon" - ; "-run-snark-coordinator" - ; t.snark_coordinator_key - ; "-snark-worker-fee" - ; t.snark_worker_fee - ; "-work-selection" - ; t.work_selection - ] - @ Base.default_cmd ~config_file - end - - module Snark_worker = struct - let name = "snark-worker" - - type t = - { daemon_address : string - ; daemon_port : string - ; proof_level : Proof_level.t - } - - let default ~daemon_address ~daemon_port = - { daemon_address; daemon_port; proof_level = Proof_level.Full } + in + List.concat [ block_producer_args; base_args ] + + let create_docker_config ~image ~entrypoint ~ports ~volumes ~environment + ~config = + { Dockerfile.Service.image + ; command = create_cmd config + ; entrypoint + ; ports + ; environment + ; volumes + } + + let create ~service_name ~image ~ports ~volumes ~config = + let entrypoint = Some [ "/root/entrypoint.sh" ] in + let docker_config = + create_docker_config ~image ~ports ~volumes + ~environment:Dockerfile.Service.Environment.default ~entrypoint ~config + in + { service_name; config; docker_config } +end - let cmd t ~config_file = - [ "internal" - ; "snark-worker" - ; "-proof-level" - ; Proof_level.to_string t.proof_level - ; "-daemon-address" - ; t.daemon_address ^ ":" ^ t.daemon_port - ] - @ Base.default_cmd ~config_file - end +module Seed_config = struct + let peer_id = "12D3KooWMg66eGtSEx5UZ9EAqEp3W7JaGd6WTxdRFuqhskRN55dT" - module Archive_node = struct - type t = { postgres_uri : string; server_port : string } + let libp2p_keypair = + {|{"box_primitive":"xsalsa20poly1305","pw_primitive":"argon2i","nonce":"7Bbvv2wZ6iCeqVyooU9WR81aygshMrLdXKieaHT","pwsalt":"Bh1WborqSwdzBi7m95iZdrCGspSf","pwdiff":[134217728,6],"ciphertext":"8fgvt4eKSzF5HMr1uEZARVHBoMgDKTx17zV7STVQyhyyEz1SqdH4RrU51MFGMPZJXNznLfz8RnSPsjrVqhc1CenfSLLWP5h7tTn86NbGmzkshCNvUiGEoSb2CrSLsvJsdn13ey9ibbZfdeXyDp9y6mKWYVmefAQLWUC1Kydj4f4yFwCJySEttAhB57647ewBRicTjdpv948MjdAVNf1tTxms4VYg4Jb3pLVeGAPaRtW5QHUkA8LwN5fh3fmaFk1mRudMd67UzGdzrVBeEHAp4zCnN7g2iVdWNmwN3"}|} - let default = - { postgres_uri = Postgres_uri.(default |> to_string) - ; server_port = "3086" - } + let create_libp2p_peer ~peer_name ~external_port = + Printf.sprintf "/dns4/%s/tcp/%d/p2p/%s" peer_name external_port peer_id - let create postgres_uri = { postgres_uri; server_port = "3086" } - - let cmd t ~config_file = - [ "coda-archive" - ; "run" - ; "-postgres-uri" - ; t.postgres_uri - ; "-server-port" - ; t.server_port - ; "-config-file" - ; config_file - ] - end + type config = + { archive_address : string option + ; peer : string option + ; runtime_config_path : string + } + [@@deriving to_yojson] type t = - | Seed - | Block_producer of Block_producer.t - | Snark_coordinator of Snark_coordinator.t - | Snark_worker of Snark_worker.t - | Archive_node of Archive_node.t - - let create_cmd t ~config_file = - match t with - | Seed -> - Seed.cmd ~config_file - | Block_producer args -> - Block_producer.cmd args ~config_file - | Snark_coordinator args -> - Snark_coordinator.cmd args ~config_file - | Snark_worker args -> - Snark_worker.cmd args ~config_file - | Archive_node args -> - Archive_node.cmd args ~config_file + { service_name : string + ; config : config + ; docker_config : Dockerfile.Service.t + } + [@@deriving to_yojson] + + let seed_libp2p_keypair : Docker_compose.Dockerfile.Service.Volume.t = + { type_ = "bind" + ; source = "keys/libp2p_key" + ; target = Base_node_config.container_libp2p_key_path + } + + let create_cmd config = + let base_args = + Base_node_config.to_list + (Base_node_config.default + ~runtime_config_path:(Some config.runtime_config_path) + ~peer:config.peer ) + in + let seed_args = + match config.archive_address with + | Some archive_address -> + [ "daemon"; "-seed"; "-archive-address"; archive_address ] + | None -> + [ "daemon"; "-seed" ] + in + List.concat [ seed_args; base_args ] + + let create_docker_config ~image ~entrypoint ~ports ~volumes ~environment + ~config = + { Dockerfile.Service.image + ; command = create_cmd config + ; entrypoint + ; ports + ; environment + ; volumes + } + + let create ~service_name ~image ~ports ~volumes ~config = + let entrypoint = Some [ "/root/entrypoint.sh" ] in + let docker_config = + create_docker_config ~image ~ports ~volumes + ~environment:Dockerfile.Service.Environment.default ~entrypoint ~config + in + { service_name; config; docker_config } end -module Services = struct - module Seed = struct - let name = "seed" - - let env = Envs.base_node_envs - - let cmd = Cmd.Seed - end - - module Block_producer = struct - let name = "block-producer" - - let secret_name = "keypair" - - let env = Envs.base_node_envs +module Snark_worker_config = struct + type config = + { daemon_address : string; daemon_port : string; proof_level : string } + [@@deriving to_yojson] - let cmd args = Cmd.Block_producer args - end - - module Snark_coordinator = struct - let name = "snark-coordinator" - - let default_port = "8301" - - let env = Envs.snark_coord_envs + type t = + { service_name : string + ; config : config + ; docker_config : Dockerfile.Service.t + } + [@@deriving to_yojson] + + let create_cmd config = + [ "internal" + ; "snark-worker" + ; "-proof-level" + ; config.proof_level + ; "-daemon-address" + ; config.daemon_address ^ ":" ^ config.daemon_port + ; "--shutdown-on-disconnect" + ; "false" + ] - let cmd args = Cmd.Snark_coordinator args - end + let create_docker_config ~image ~entrypoint ~ports ~volumes ~environment + ~config = + { Dockerfile.Service.image + ; command = create_cmd config + ; entrypoint + ; ports + ; environment + ; volumes + } + + let create ~service_name ~image ~ports ~volumes ~config = + let entrypoint = Some [ "/root/entrypoint.sh" ] in + let docker_config = + create_docker_config ~image ~ports ~volumes + ~environment:Dockerfile.Service.Environment.default ~entrypoint ~config + in + { service_name; config; docker_config } +end - module Snark_worker = struct - let name = "snark-worker" +module Snark_coordinator_config = struct + type config = + { snark_coordinator_key : string + ; snark_worker_fee : string + ; work_selection : string + ; worker_nodes : Snark_worker_config.t list + ; peer : string option + ; runtime_config_path : string + } + [@@deriving to_yojson] - let env = [] + type t = + { service_name : string + ; config : config + ; docker_config : Dockerfile.Service.t + } + [@@deriving to_yojson] + + let snark_coordinator_default_env ~snark_coordinator_key ~snark_worker_fee + ~work_selection = + [ ("MINA_SNARK_KEY", snark_coordinator_key) + ; ("MINA_SNARK_FEE", snark_worker_fee) + ; ("WORK_SELECTION", work_selection) + ; ("MINA_CLIENT_TRUSTLIST", "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16") + ] - let cmd args = Cmd.Snark_worker args - end + let create_cmd config = + let base_args = + Base_node_config.to_list + (Base_node_config.default + ~runtime_config_path:(Some config.runtime_config_path) + ~peer:config.peer ) + in + let snark_coordinator_args = + [ "daemon" + ; "-run-snark-coordinator" + ; config.snark_coordinator_key + ; "-snark-worker-fee" + ; config.snark_worker_fee + ; "-work-selection" + ; config.work_selection + ] + in + List.concat [ snark_coordinator_args; base_args ] + + let create_docker_config ~image ~entrypoint ~ports ~volumes ~environment + ~config = + { Dockerfile.Service.image + ; command = create_cmd config + ; entrypoint + ; ports + ; environment + ; volumes + } + + let create ~service_name ~image ~ports ~volumes ~config = + let entrypoint = Some [ "/root/entrypoint.sh" ] in + let environment = + snark_coordinator_default_env + ~snark_coordinator_key:config.snark_coordinator_key + ~snark_worker_fee:config.snark_worker_fee + ~work_selection:config.work_selection + @ Dockerfile.Service.Environment.default + in + let docker_config = + create_docker_config ~image ~ports ~volumes ~environment ~entrypoint + ~config + in + { service_name; config; docker_config } +end - module Archive_node = struct - let name = "archive" +module Postgres_config = struct + type config = + { host : string + ; username : string + ; password : string + ; database : string + ; port : int + } + [@@deriving to_yojson] - let postgres_name = "postgres" + type t = + { service_name : string + ; config : config + ; docker_config : Dockerfile.Service.t + } + [@@deriving to_yojson] + + let postgres_image = "docker.io/bitnami/postgresql" + + let postgres_script = + ( "postgres_entrypoint.sh" + , {|#!/bin/bash +# This file is auto-generated by the local integration test framework. +cd /bitnami +# Create the archive database and import the schema +psql -U postgres -d archive -f ./create_schema.sql +|} + ) + + let postgres_create_schema_volume : Dockerfile.Service.Volume.t = + { type_ = "bind" + ; source = "create_schema.sql" + ; target = "/bitnami/create_schema.sql" + } + + let postgres_zkapp_schema_volume : Dockerfile.Service.Volume.t = + { type_ = "bind" + ; source = "zkapp_tables.sql" + ; target = "/bitnami/zkapp_tables.sql" + } + + let postgres_entrypoint_volume : Dockerfile.Service.Volume.t = + { type_ = "bind" + ; source = "postgres_entrypoint.sh" + ; target = "/docker-entrypoint-initdb.d/postgres_entrypoint.sh" + } + + let postgres_default_envs ~username ~password ~database ~port = + [ ("BITNAMI_DEBUG", "false") + ; ("POSTGRES_USER", username) + ; ("POSTGRES_PASSWORD", password) + ; ("POSTGRES_DB", database) + ; ("PGPASSWORD", password) + ; ("POSTGRESQL_PORT_NUMBER", port) + ; ("POSTGRESQL_ENABLE_LDAP", "no") + ; ("POSTGRESQL_ENABLE_TLS", "no") + ; ("POSTGRESQL_LOG_HOSTNAME", "false") + ; ("POSTGRESQL_LOG_CONNECTIONS", "false") + ; ("POSTGRESQL_LOG_DISCONNECTIONS", "false") + ; ("POSTGRESQL_PGAUDIT_LOG_CATALOG", "off") + ; ("POSTGRESQL_CLIENT_MIN_MESSAGES", "error") + ; ("POSTGRESQL_SHARED_PRELOAD_LIBRARIES", "pgaudit") + ; ("POSTGRES_HOST_AUTH_METHOD", "trust") + ] - let server_port = "3086" + let create_connection_uri ~host ~username ~password ~database ~port = + Printf.sprintf "postgres://%s:%s@%s:%s/%s" username password host + (Int.to_string port) database + + let to_connection_uri t = + create_connection_uri ~host:t.host ~port:t.port ~username:t.username + ~password:t.password ~database:t.database + + let create_docker_config ~image ~entrypoint ~ports ~volumes ~environment = + { Dockerfile.Service.image + ; command = [] + ; entrypoint + ; ports + ; environment + ; volumes + } + + let create ~service_name ~image ~ports ~volumes ~config = + let entrypoint = None in + let environment = + postgres_default_envs ~username:config.username ~password:config.password + ~database:config.database + ~port:(Int.to_string config.port) + in + let docker_config = + create_docker_config ~image ~ports ~volumes ~environment ~entrypoint + in + { service_name; config; docker_config } +end - let envs = Envs.base_node_envs +module Archive_node_config = struct + type config = + { postgres_config : Postgres_config.t + ; server_port : int + ; runtime_config_path : string + } + [@@deriving to_yojson] - let cmd args = Cmd.Archive_node args + type t = + { service_name : string + ; config : config + ; docker_config : Dockerfile.Service.t + } + [@@deriving to_yojson] + + let create_cmd config = + [ "mina-archive" + ; "run" + ; "-postgres-uri" + ; Postgres_config.to_connection_uri config.postgres_config.config + ; "-server-port" + ; Int.to_string config.server_port + ; "-config-file" + ; config.runtime_config_path + ] - let entrypoint_target = "/docker-entrypoint-initdb.d" - end + let create_docker_config ~image ~entrypoint ~ports ~volumes ~environment + ~config = + { Dockerfile.Service.image + ; command = create_cmd config + ; entrypoint + ; ports + ; environment + ; volumes + } + + let create ~service_name ~image ~ports ~volumes ~config = + let entrypoint = None in + let docker_config = + create_docker_config ~image ~ports ~volumes + ~environment:Dockerfile.Service.Environment.default ~entrypoint ~config + in + { service_name; config; docker_config } end diff --git a/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml b/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml index 910eb683b16..1311a14c029 100644 --- a/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml +++ b/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml @@ -2,25 +2,146 @@ open Async open Core open Integration_test_lib module Timeout = Timeout_lib.Core_time -module Node = Swarm_network.Node +module Node = Docker_network.Node -(* TODO: Implement local engine logging *) +(** This implements Log_engine_intf for integration tests, by creating a simple system that polls a mina daemon's graphql endpoint for fetching logs*) + +let log_filter_of_event_type ev_existential = + let open Event_type in + let (Event_type ev_type) = ev_existential in + let (module Ty) = event_type_module ev_type in + match Ty.parse with + | From_error_log _ -> + [] (* TODO: Do we need this? *) + | From_daemon_log (struct_id, _) -> + [ Structured_log_events.string_of_id struct_id ] + | From_puppeteer_log _ -> + [] +(* TODO: Do we need this? *) + +let all_event_types_log_filter = + List.bind ~f:log_filter_of_event_type Event_type.all_event_types type t = { logger : Logger.t ; event_writer : (Node.t * Event_type.event) Pipe.Writer.t ; event_reader : (Node.t * Event_type.event) Pipe.Reader.t + ; background_job : unit Deferred.Or_error.t } let event_reader { event_reader; _ } = event_reader -let create ~logger ~(network : Swarm_network.t) = - [%log info] "docker_pipe_log_engine: create %s" network.namespace ; +let parse_event_from_log_entry ~logger log_entry = + let open Or_error.Let_syntax in + let open Json_parsing in + Or_error.try_with_join (fun () -> + let payload = Yojson.Safe.from_string log_entry in + let%map event = + let%bind msg = + parse (parser_from_of_yojson Logger.Message.of_yojson) payload + in + let event_id = + Option.map ~f:Structured_log_events.string_of_id msg.event_id + in + [%log spam] "parsing daemon structured event, event_id = $event_id" + ~metadata: + [ ("event_id", [%to_yojson: string option] event_id) + ; ("msg", Logger.Message.to_yojson msg) + ; ("metadata", Logger.Metadata.to_yojson msg.metadata) + ; ("payload", payload) + ] ; + match msg.event_id with + | Some _ -> + Event_type.parse_daemon_event msg + | None -> + (* Currently unreachable, but we could include error logs here if + desired. + *) + Event_type.parse_error_log msg + in + event ) + +let rec filtered_log_entries_poll node ~logger ~event_writer + ~last_log_index_seen = + let open Deferred.Let_syntax in + if not (Pipe.is_closed event_writer) then ( + let%bind () = after (Time.Span.of_ms 10000.0) in + match%bind + Integration_test_lib.Graphql_requests.get_filtered_log_entries + (Node.get_ingress_uri node) + ~last_log_index_seen + with + | Ok log_entries -> + Array.iter log_entries ~f:(fun log_entry -> + match parse_event_from_log_entry ~logger log_entry with + | Ok a -> + Pipe.write_without_pushback_if_open event_writer (node, a) + | Error e -> + [%log warn] "Error parsing log $error" + ~metadata:[ ("error", `String (Error.to_string_hum e)) ] ) ; + let last_log_index_seen = + Array.length log_entries + last_log_index_seen + in + filtered_log_entries_poll node ~logger ~event_writer + ~last_log_index_seen + | Error err -> + [%log error] "Encountered an error while polling $node for logs: $err" + ~metadata: + [ ("node", `String (Node.infra_id node)) + ; ("err", Error_json.error_to_yojson err) + ] ; + (* Declare the node to be offline. *) + Pipe.write_without_pushback_if_open event_writer + (node, Event (Node_offline, ())) ; + (* Don't keep looping, the node may be restarting. *) + return (Ok ()) ) + else Deferred.Or_error.error_string "Event writer closed" + +let rec start_filtered_log node ~logger ~log_filter ~event_writer = + let open Deferred.Let_syntax in + if not (Pipe.is_closed event_writer) then + match%bind + Integration_test_lib.Graphql_requests.start_filtered_log ~logger + ~log_filter + (Node.get_ingress_uri node) + with + | Ok () -> + return (Ok ()) + | Error _ -> + start_filtered_log node ~logger ~log_filter ~event_writer + else Deferred.Or_error.error_string "Event writer closed" + +let rec poll_node_for_logs_in_background ~log_filter ~logger ~event_writer + (node : Node.t) = + let open Deferred.Or_error.Let_syntax in + [%log info] "Requesting for $node to start its filtered logs" + ~metadata:[ ("node", `String (Node.infra_id node)) ] ; + let%bind () = start_filtered_log ~logger ~log_filter ~event_writer node in + [%log info] "$node has started its filtered logs. Beginning polling" + ~metadata:[ ("node", `String (Node.infra_id node)) ] ; + let%bind () = + filtered_log_entries_poll node ~last_log_index_seen:0 ~logger ~event_writer + in + poll_node_for_logs_in_background ~log_filter ~logger ~event_writer node + +let poll_for_logs_in_background ~log_filter ~logger ~network ~event_writer = + Docker_network.all_nodes network + |> Core.String.Map.data + |> Deferred.Or_error.List.iter ~how:`Parallel + ~f:(poll_node_for_logs_in_background ~log_filter ~logger ~event_writer) + +let create ~logger ~(network : Docker_network.t) = + let open Deferred.Or_error.Let_syntax in + let log_filter = all_event_types_log_filter in let event_reader, event_writer = Pipe.create () in - Deferred.Or_error.return { logger; event_reader; event_writer } + let background_job = + poll_for_logs_in_background ~log_filter ~logger ~network ~event_writer + in + return { logger; event_reader; event_writer; background_job } let destroy t : unit Deferred.Or_error.t = - let { logger; event_reader = _; event_writer } = t in + let open Deferred.Or_error.Let_syntax in + let { logger; event_reader = _; event_writer; background_job = _ } = t in Pipe.close event_writer ; - [%log debug] "subscription deleted" ; - Deferred.Or_error.error_string "subscription deleted" + [%log debug] "graphql polling log engine destroyed" ; + return () diff --git a/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli b/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli index 6398559beff..8597d5e7efe 100644 --- a/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli +++ b/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli @@ -1,3 +1,3 @@ include Integration_test_lib.Intf.Engine.Log_engine_intf - with module Network := Swarm_network + with module Network := Docker_network diff --git a/src/lib/integration_test_local_engine/integration_test_local_engine.ml b/src/lib/integration_test_local_engine/integration_test_local_engine.ml index 6647db9f725..8869ee80994 100644 --- a/src/lib/integration_test_local_engine/integration_test_local_engine.ml +++ b/src/lib/integration_test_local_engine/integration_test_local_engine.ml @@ -1,6 +1,6 @@ let name = "local" -module Network = Swarm_network +module Network = Docker_network module Network_config = Mina_docker.Network_config module Network_manager = Mina_docker.Network_manager module Log_engine = Docker_pipe_log_engine diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index d56dc7a8b61..a1e090a2c7d 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -5,192 +5,147 @@ open Signature_lib open Mina_base open Integration_test_lib -let docker_swarm_version = "3.9" - -let postgres_image = "docker.io/bitnami/postgresql" - -let mina_archive_schema = - "https://raw.githubusercontent.com/MinaProtocol/mina/develop/src/app/archive/create_schema.sql" - -let mina_create_schema = "create_schema.sql" +let docker_swarm_version = "3.8" module Network_config = struct module Cli_inputs = Cli_inputs - module Port = struct - type t = string list - - let get_new_port t = - match t with [] -> [ 7000 ] | x :: xs -> [ x + 1 ] @ [ x ] @ xs - - let get_latest_port t = match t with [] -> 7000 | x :: _ -> x - - let create_rest_port t = - match t with [] -> "7000:3085" | x :: _ -> Int.to_string x ^ ":3085" - end - - type docker_volume_configs = { name : string; data : string } - [@@deriving to_yojson] - - type seed_config = { name : string; ports : (string * string) list } - [@@deriving to_yojson] - - type block_producer_config = - { name : string - ; id : string - ; keypair : Network_keypair.t - ; public_key : string - ; private_key : string - ; keypair_secret : string - ; libp2p_secret : string - ; ports : (string * string) list - } - [@@deriving to_yojson] - - type snark_coordinator_configs = - { name : string - ; id : string - ; public_key : string - ; snark_worker_fee : string - ; ports : (string * string) list - } - [@@deriving to_yojson] - - type archive_node_configs = - { name : string - ; id : string - ; schema : string - ; ports : (string * string) list - } - [@@deriving to_yojson] - type docker_config = { docker_swarm_version : string ; stack_name : string ; mina_image : string + ; mina_agent_image : string + ; mina_bots_image : string + ; mina_points_image : string ; mina_archive_image : string - ; docker_volume_configs : docker_volume_configs list - ; seed_configs : seed_config list - ; block_producer_configs : block_producer_config list - ; snark_coordinator_configs : snark_coordinator_configs list - ; archive_node_configs : archive_node_configs list - ; log_precomputed_blocks : bool - ; archive_node_count : int - ; mina_archive_schema : string ; runtime_config : Yojson.Safe.t - [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] + ; seed_configs : Docker_node_config.Seed_config.t list + ; block_producer_configs : Docker_node_config.Block_producer_config.t list + ; snark_coordinator_config : + Docker_node_config.Snark_coordinator_config.t option + ; archive_node_configs : Docker_node_config.Archive_node_config.t list + ; mina_archive_schema_aux_files : string list + ; log_precomputed_blocks : bool } [@@deriving to_yojson] type t = - { keypairs : Network_keypair.t list - ; debug_arg : bool + { debug_arg : bool + ; genesis_keypairs : + (Network_keypair.t Core.String.Map.t + [@to_yojson + fun map -> + `Assoc + (Core.Map.fold_right ~init:[] + ~f:(fun ~key:k ~data:v accum -> + (k, Network_keypair.to_yojson v) :: accum ) + map )] ) ; constants : Test_config.constants ; docker : docker_config } [@@deriving to_yojson] - let expand ~logger ~test_name ~cli_inputs:_ ~(debug : bool) + let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = - let { Test_config.k + let _ = cli_inputs in + let { genesis_ledger + ; epoch_data + ; block_producers + ; snark_coordinator + ; snark_worker_fee + ; num_archive_nodes + ; log_precomputed_blocks + ; proof_config + ; Test_config.k ; delta ; slots_per_epoch ; slots_per_sub_window - ; proof_level ; txpool_max_size - ; requires_graphql = _ - ; block_producers - ; num_snark_workers - ; num_archive_nodes - ; log_precomputed_blocks - ; snark_worker_fee - ; snark_worker_public_key + ; _ } = test_config in - let user_from_env = Option.value (Unix.getenv "USER") ~default:"auto" in - let user_sanitized = - Str.global_replace (Str.regexp "\\W|_-") "" user_from_env - in - let user_len = Int.min 5 (String.length user_sanitized) in - let user = String.sub user_sanitized ~pos:0 ~len:user_len in let git_commit = Mina_version.commit_id_short in - (* see ./src/app/test_executive/README.md for information regarding the namespace name format and length restrictions *) - let stack_name = "it-" ^ user ^ "-" ^ git_commit ^ "-" ^ test_name in - (* GENERATE ACCOUNTS AND KEYPAIRS *) - let num_block_producers = List.length block_producers in - let block_producer_keypairs, runtime_accounts = - (* the first keypair is the genesis winner and is assumed to be untimed. Therefore dropping it, and not assigning it to any block producer *) - let keypairs = - List.drop (Array.to_list (Lazy.force Sample_keypairs.keypairs)) 1 - in - if num_block_producers > List.length keypairs then - failwith - "not enough sample keypairs for specified number of block producers" ; - let f index ({ Test_config.Block_producer.balance; timing }, (pk, sk)) = - let runtime_account = - let timing = - match timing with - | Account.Timing.Untimed -> - None - | Timed t -> - Some - { Runtime_config.Accounts.Single.Timed.initial_minimum_balance = - t.initial_minimum_balance - ; cliff_time = t.cliff_time - ; cliff_amount = t.cliff_amount - ; vesting_period = t.vesting_period - ; vesting_increment = t.vesting_increment - } - in + let stack_name = "it-" ^ git_commit ^ "-" ^ test_name in + let key_names_list = + List.map genesis_ledger ~f:(fun acct -> acct.account_name) + in + if List.contains_dup ~compare:String.compare key_names_list then + failwith + "All accounts in genesis ledger must have unique names. Check to make \ + sure you are not using the same account_name more than once" ; + let all_nodes_names_list = + List.map block_producers ~f:(fun acct -> acct.node_name) + @ match snark_coordinator with None -> [] | Some n -> [ n.node_name ] + in + if List.contains_dup ~compare:String.compare all_nodes_names_list then + failwith + "All nodes in testnet must have unique names. Check to make sure you \ + are not using the same node_name more than once" ; + let keypairs = + List.take + (List.tl_exn + (Array.to_list (Lazy.force Key_gen.Sample_keypairs.keypairs)) ) + (List.length genesis_ledger) + in + let runtime_timing_of_timing = function + | Account.Timing.Untimed -> + None + | Timed t -> + Some + { Runtime_config.Accounts.Single.Timed.initial_minimum_balance = + t.initial_minimum_balance + ; cliff_time = t.cliff_time + ; cliff_amount = t.cliff_amount + ; vesting_period = t.vesting_period + ; vesting_increment = t.vesting_increment + } + in + let add_accounts accounts_and_keypairs = + List.map accounts_and_keypairs + ~f:(fun + ( { Test_config.Test_Account.balance; account_name; timing } + , (pk, sk) ) + -> + let timing = runtime_timing_of_timing timing in let default = Runtime_config.Accounts.Single.default in - { default with - pk = Some (Public_key.Compressed.to_string pk) - ; sk = None - ; balance = - Balance.of_formatted_string balance - (* delegation currently unsupported *) - ; delegate = None - ; timing - } - in - let secret_name = "test-keypair-" ^ Int.to_string index in - let keypair = - { Keypair.public_key = Public_key.decompress_exn pk - ; private_key = sk - } - in - ( Network_keypair.create_network_keypair ~keypair ~secret_name - , runtime_account ) - in - List.mapi ~f - (List.zip_exn block_producers - (List.take keypairs (List.length block_producers))) - |> List.unzip - in - (* DAEMON CONFIG *) - let proof_config = - (* TODO: lift configuration of these up Test_config.t *) - { Runtime_config.Proof_keys.level = Some proof_level - ; sub_windows_per_window = None - ; ledger_depth = None - ; work_delay = None - ; block_window_duration_ms = None - ; transaction_capacity = None - ; coinbase_amount = None - ; supercharged_coinbase_factor = None - ; account_creation_fee = None - ; fork = None - } + let account = + { default with + pk = Public_key.Compressed.to_string pk + ; sk = Some (Private_key.to_base58_check sk) + ; balance = Balance.of_mina_string_exn balance + ; delegate = None + ; timing + } + in + (account_name, account) ) in + let genesis_accounts_and_keys = List.zip_exn genesis_ledger keypairs in + let genesis_ledger_accounts = add_accounts genesis_accounts_and_keys in let constraint_constants = Genesis_ledger_helper.make_constraint_constants ~default:Genesis_constants.Constraint_constants.compiled proof_config in + let ledger_is_prefix ledger1 ledger2 = + List.is_prefix ledger2 ~prefix:ledger1 + ~equal:(fun + ({ account_name = name1; _ } : Test_config.Test_Account.t) + ({ account_name = name2; _ } : Test_config.Test_Account.t) + -> String.equal name1 name2 ) + in let runtime_config = { Runtime_config.daemon = - Some { txpool_max_size = Some txpool_max_size; peer_list_url = None } + Some + { txpool_max_size = Some txpool_max_size + ; peer_list_url = None + ; zkapp_proof_update_cost = None + ; zkapp_signed_single_update_cost = None + ; zkapp_signed_pair_update_cost = None + ; zkapp_transaction_cost_limit = None + ; max_event_elements = None + ; max_action_elements = None + } ; genesis = Some { k = Some k @@ -200,130 +155,361 @@ module Network_config = struct ; genesis_state_timestamp = Some Core.Time.(to_string_abs ~zone:Zone.utc (now ())) } - ; proof = - None - (* was: Some proof_config; TODO: prebake ledger and only set hash *) + ; proof = Some proof_config ; ledger = Some - { base = Accounts runtime_accounts + { base = + Accounts + (List.map genesis_ledger_accounts ~f:(fun (_name, acct) -> + acct ) ) ; add_genesis_winner = None ; num_accounts = None ; balances = [] ; hash = None ; name = None } - ; epoch_data = None + ; epoch_data = + Option.map epoch_data ~f:(fun { staking = staking_ledger; next } -> + let genesis_winner_account : Runtime_config.Accounts.single = + Runtime_config.Accounts.Single.of_account + Mina_state.Consensus_state_hooks.genesis_winner_account + |> Result.ok_or_failwith + in + let ledger_of_epoch_accounts + (epoch_accounts : Test_config.Test_Account.t list) = + let epoch_ledger_accounts = + List.map epoch_accounts + ~f:(fun { account_name; balance; timing } -> + let balance = Balance.of_mina_string_exn balance in + let timing = runtime_timing_of_timing timing in + let genesis_account = + match + List.Assoc.find genesis_ledger_accounts account_name + ~equal:String.equal + with + | Some acct -> + acct + | None -> + failwithf + "Epoch ledger account %s not in genesis ledger" + account_name () + in + { genesis_account with balance; timing } ) + in + ( { base = + Accounts (genesis_winner_account :: epoch_ledger_accounts) + ; add_genesis_winner = None (* no effect *) + ; num_accounts = None + ; balances = [] + ; hash = None + ; name = None + } + : Runtime_config.Ledger.t ) + in + let staking = + let ({ epoch_ledger; epoch_seed } + : Test_config.Epoch_data.Data.t ) = + staking_ledger + in + if not (ledger_is_prefix epoch_ledger genesis_ledger) then + failwith "Staking epoch ledger not a prefix of genesis ledger" ; + let ledger = ledger_of_epoch_accounts epoch_ledger in + let seed = epoch_seed in + ({ ledger; seed } : Runtime_config.Epoch_data.Data.t) + in + let next = + Option.map next ~f:(fun { epoch_ledger; epoch_seed } -> + if + not + (ledger_is_prefix staking_ledger.epoch_ledger + epoch_ledger ) + then + failwith + "Staking epoch ledger not a prefix of next epoch ledger" ; + if not (ledger_is_prefix epoch_ledger genesis_ledger) then + failwith + "Next epoch ledger not a prefix of genesis ledger" ; + let ledger = ledger_of_epoch_accounts epoch_ledger in + let seed = epoch_seed in + ({ ledger; seed } : Runtime_config.Epoch_data.Data.t) ) + in + ({ staking; next } : Runtime_config.Epoch_data.t) ) } in let genesis_constants = Or_error.ok_exn (Genesis_ledger_helper.make_genesis_constants ~logger - ~default:Genesis_constants.compiled runtime_config) + ~default:Genesis_constants.compiled runtime_config ) in let constants : Test_config.constants = { constraints = constraint_constants; genesis = genesis_constants } in - let open Docker_node_config.Services in - let available_host_ports_ref = ref [] in - (* BLOCK PRODUCER CONFIG *) - let block_producer_config index keypair = - available_host_ports_ref := Port.get_new_port !available_host_ports_ref ; - let rest_port = Port.create_rest_port !available_host_ports_ref in - { name = Block_producer.name ^ "-" ^ Int.to_string (index + 1) - ; id = Int.to_string index - ; keypair - ; keypair_secret = keypair.secret_name - ; public_key = keypair.public_key_file - ; private_key = keypair.private_key_file - ; libp2p_secret = "" - ; ports = [ ("rest-port", rest_port) ] - } + let mk_net_keypair keypair_name (pk, sk) = + let keypair = + { Keypair.public_key = Public_key.decompress_exn pk; private_key = sk } + in + Network_keypair.create_network_keypair ~keypair_name ~keypair in - let block_producer_configs = - List.mapi block_producer_keypairs ~f:block_producer_config + let long_commit_id = + if String.is_substring Mina_version.commit_id ~substring:"[DIRTY]" then + String.sub Mina_version.commit_id ~pos:7 + ~len:(String.length Mina_version.commit_id - 7) + else Mina_version.commit_id + in + let mina_archive_base_url = + "https://raw.githubusercontent.com/MinaProtocol/mina/" ^ long_commit_id + ^ "/src/app/archive/" + in + let mina_archive_schema_aux_files = + [ sprintf "%screate_schema.sql" mina_archive_base_url + ; sprintf "%szkapp_tables.sql" mina_archive_base_url + ] + in + let genesis_keypairs = + List.fold genesis_accounts_and_keys ~init:String.Map.empty + ~f:(fun map ({ account_name; _ }, (pk, sk)) -> + let keypair = mk_net_keypair account_name (pk, sk) in + String.Map.add_exn map ~key:account_name ~data:keypair ) + in + let open Docker_node_config in + let open Docker_compose.Dockerfile in + let port_manager = PortManager.create ~min_port:10000 ~max_port:11000 in + let docker_volumes = + [ Base_node_config.runtime_config_volume + ; Base_node_config.entrypoint_volume + ] + in + let generate_random_id () = + let rand_char () = + let ascii_a = int_of_char 'a' in + let ascii_z = int_of_char 'z' in + char_of_int (ascii_a + Random.int (ascii_z - ascii_a + 1)) + in + String.init 4 ~f:(fun _ -> rand_char ()) + in + let seed_config = + let config : Seed_config.config = + { archive_address = None + ; peer = None + ; runtime_config_path = Base_node_config.container_runtime_config_path + } + in + Seed_config.create + ~service_name:(sprintf "seed-%s" (generate_random_id ())) + ~image:images.mina + ~ports:(PortManager.allocate_ports_for_node port_manager) + ~volumes:(docker_volumes @ [ Seed_config.seed_libp2p_keypair ]) + ~config in - (* SEED NODE CONFIG *) - available_host_ports_ref := Port.get_new_port !available_host_ports_ref ; - let seed_rest_port = Port.create_rest_port !available_host_ports_ref in + let seed_config_peer = + Some + (Seed_config.create_libp2p_peer ~peer_name:seed_config.service_name + ~external_port:PortManager.mina_internal_external_port ) + in + let archive_node_configs = + List.init num_archive_nodes ~f:(fun index -> + let config = + { Postgres_config.host = + sprintf "postgres-%d-%s" (index + 1) (generate_random_id ()) + ; username = "postgres" + ; password = "password" + ; database = "archive" + ; port = PortManager.postgres_internal_port + } + in + let postgres_port = + Service.Port.create + ~published:(PortManager.allocate_port port_manager) + ~target:PortManager.postgres_internal_port + in + let postgres_config = + Postgres_config.create ~service_name:config.host + ~image:Postgres_config.postgres_image ~ports:[ postgres_port ] + ~volumes: + [ Postgres_config.postgres_create_schema_volume + ; Postgres_config.postgres_zkapp_schema_volume + ; Postgres_config.postgres_entrypoint_volume + ] + ~config + in + let archive_server_port = + Service.Port.create + ~published:(PortManager.allocate_port port_manager) + ~target:PortManager.mina_internal_server_port + in + let config : Archive_node_config.config = + { postgres_config + ; server_port = archive_server_port.target + ; runtime_config_path = + Base_node_config.container_runtime_config_path + } + in + let archive_rest_port = + Service.Port.create + ~published:(PortManager.allocate_port port_manager) + ~target:PortManager.mina_internal_rest_port + in + Archive_node_config.create + ~service_name: + (sprintf "archive-%d-%s" (index + 1) (generate_random_id ())) + ~image:images.archive_node + ~ports:[ archive_server_port; archive_rest_port ] + ~volumes:docker_volumes ~config ) + in + (* Each archive node has it's own seed node *) let seed_configs = - [ { name = Seed.name; ports = [ ("rest-port", seed_rest_port) ] } ] - in - (* SNARK COORDINATOR CONFIG *) - available_host_ports_ref := Port.get_new_port !available_host_ports_ref ; - let snark_coord_rest_port = - Port.create_rest_port !available_host_ports_ref - in - let snark_coordinator_configs = - if num_snark_workers > 0 then - List.mapi - (List.init num_snark_workers ~f:(const 0)) - ~f:(fun index _ -> - available_host_ports_ref := - Port.get_new_port !available_host_ports_ref ; - let rest_port = Port.create_rest_port !available_host_ports_ref in - { name = Snark_worker.name ^ "-" ^ Int.to_string (index + 1) - ; id = Int.to_string index + List.mapi archive_node_configs ~f:(fun index archive_config -> + let config : Seed_config.config = + { archive_address = + Some + (sprintf "%s:%d" archive_config.service_name + PortManager.mina_internal_server_port ) + ; peer = seed_config_peer + ; runtime_config_path = + Base_node_config.container_runtime_config_path + } + in + Seed_config.create + ~service_name: + (sprintf "seed-%d-%s" (index + 1) (generate_random_id ())) + ~image:images.mina + ~ports:(PortManager.allocate_ports_for_node port_manager) + ~volumes:docker_volumes ~config ) + @ [ seed_config ] + in + let block_producer_configs = + List.map block_producers ~f:(fun node -> + let keypair = + match + List.find genesis_accounts_and_keys + ~f:(fun ({ account_name; _ }, _keypair) -> + String.equal account_name node.account_name ) + with + | Some (_acct, keypair) -> + keypair |> mk_net_keypair node.account_name + | None -> + let failstring = + Format.sprintf + "Failing because the account key of all initial block \ + producers must be in the genesis ledger. name of Node: \ + %s. name of Account which does not exist: %s" + node.node_name node.account_name + in + failwith failstring + in + let priv_key_path = + Base_node_config.container_keys_path ^/ node.account_name + in + let volumes = + [ Service.Volume.create ("keys" ^/ node.account_name) priv_key_path + ] + @ docker_volumes + in + let block_producer_config : Block_producer_config.config = + { keypair + ; runtime_config_path = + Base_node_config.container_runtime_config_path + ; peer = seed_config_peer + ; priv_key_path + ; enable_peer_exchange = true + ; enable_flooding = true + ; libp2p_secret = "" + } + in + Block_producer_config.create ~service_name:node.node_name + ~image:images.mina + ~ports:(PortManager.allocate_ports_for_node port_manager) + ~volumes ~config:block_producer_config ) + in + let snark_coordinator_config = + match snark_coordinator with + | None -> + None + | Some snark_coordinator_node -> + let network_kp = + match + String.Map.find genesis_keypairs + snark_coordinator_node.account_name + with + | Some acct -> + acct + | None -> + let failstring = + Format.sprintf + "Failing because the account key of all initial snark \ + coordinators must be in the genesis ledger. name of \ + Node: %s. name of Account which does not exist: %s" + snark_coordinator_node.node_name + snark_coordinator_node.account_name + in + failwith failstring + in + let public_key = + Public_key.Compressed.to_base58_check + (Public_key.compress network_kp.keypair.public_key) + in + let coordinator_ports = + PortManager.allocate_ports_for_node port_manager + in + let daemon_port = + coordinator_ports + |> List.find_exn ~f:(fun p -> + p.target + = Docker_node_config.PortManager.mina_internal_client_port ) + in + let snark_node_service_name = snark_coordinator_node.node_name in + let worker_node_config : Snark_worker_config.config = + { daemon_address = snark_node_service_name + ; daemon_port = Int.to_string daemon_port.target + ; proof_level = "full" + } + in + let worker_nodes = + List.init snark_coordinator_node.worker_nodes ~f:(fun index -> + Docker_node_config.Snark_worker_config.create + ~service_name: + (sprintf "snark-worker-%d-%s" (index + 1) + (generate_random_id ()) ) + ~image:images.mina + ~ports: + (Docker_node_config.PortManager.allocate_ports_for_node + port_manager ) + ~volumes:docker_volumes ~config:worker_node_config ) + in + let snark_coordinator_config : Snark_coordinator_config.config = + { worker_nodes ; snark_worker_fee - ; public_key = snark_worker_public_key - ; ports = [ ("rest_port", rest_port) ] - }) - (* Add one snark coordinator for all workers *) - |> List.append - [ { name = Snark_coordinator.name - ; id = "1" - ; snark_worker_fee - ; public_key = snark_worker_public_key - ; ports = [ ("rest-port", snark_coord_rest_port) ] - } - ] - else [] - in - (* ARCHIVE CONFIG *) - let archive_node_configs = - if num_archive_nodes > 0 then - List.mapi - (List.init num_archive_nodes ~f:(const 0)) - ~f:(fun index _ -> - available_host_ports_ref := - Port.get_new_port !available_host_ports_ref ; - let rest_port = Port.create_rest_port !available_host_ports_ref in - { name = Archive_node.name ^ "-" ^ Int.to_string (index + 1) - ; id = Int.to_string index - ; schema = mina_archive_schema - ; ports = [ ("rest-port", rest_port) ] - }) - else [] - in - (* DOCKER VOLUME CONFIG: - Create a docker volume structure for the runtime_config and all block producer keys - to mount in the specified docker services - *) - let docker_volume_configs = - List.map block_producer_configs ~f:(fun config -> - { name = "sk-" ^ config.name; data = config.private_key }) - @ [ { name = Docker_node_config.Volumes.Runtime_config.name - ; data = - Yojson.Safe.to_string (Runtime_config.to_yojson runtime_config) - } - ] + ; runtime_config_path = + Base_node_config.container_runtime_config_path + ; snark_coordinator_key = public_key + ; peer = seed_config_peer + ; work_selection = "seq" + } + in + Some + (Snark_coordinator_config.create + ~service_name:snark_node_service_name ~image:images.mina + ~ports:coordinator_ports ~volumes:docker_volumes + ~config:snark_coordinator_config ) in { debug_arg = debug - ; keypairs = block_producer_keypairs + ; genesis_keypairs ; constants ; docker = { docker_swarm_version ; stack_name ; mina_image = images.mina + ; mina_agent_image = images.user_agent + ; mina_bots_image = images.bots + ; mina_points_image = images.points ; mina_archive_image = images.archive_node - ; docker_volume_configs ; runtime_config = Runtime_config.to_yojson runtime_config - ; seed_configs + ; log_precomputed_blocks (* Node configs *) ; block_producer_configs - ; snark_coordinator_configs - ; archive_node_configs - ; log_precomputed_blocks - ; archive_node_count = num_archive_nodes - ; mina_archive_schema + ; seed_configs + ; mina_archive_schema_aux_files + ; snark_coordinator_config + ; archive_node_configs (* Resource configs *) } } @@ -336,373 +522,509 @@ module Network_config = struct be used (which are mostly defaults). *) let to_docker network_config = - let open Docker_compose.Compose in - let open Docker_node_config in - let open Docker_node_config.Services in - (* RUNTIME CONFIG DOCKER BIND VOLUME *) - let runtime_config = - Service.Volume.create Volumes.Runtime_config.name - Volumes.Runtime_config.container_mount_target - in - (* BLOCK PRODUCER DOCKER SERVICE *) + let open Docker_compose.Dockerfile in let block_producer_map = List.map network_config.docker.block_producer_configs ~f:(fun config -> - (* BP KEYPAIR CONFIG DOCKER BIND VOLUME *) - let private_key_config_name = "sk-" ^ config.name in - let private_key_config = - Service.Volume.create private_key_config_name - ("/root/" ^ private_key_config_name) - in - let cmd = - Cmd.( - Block_producer - (Block_producer.default - ~private_key_config:private_key_config.target)) - in - let rest_port = - List.find - ~f:(fun ports -> String.equal (fst ports) "rest-port") - config.ports - |> Option.value_exn |> snd - in - ( config.name - , { Service.image = network_config.docker.mina_image - ; volumes = [ private_key_config; runtime_config ] - ; command = Cmd.create_cmd cmd ~config_file:runtime_config.target - ; ports = [ rest_port ] - ; environment = Service.Environment.create Envs.base_node_envs - } )) + (config.service_name, config.docker_config) ) |> StringMap.of_alist_exn in - (* SNARK COORD/WORKER DOCKER SERVICE *) - let snark_worker_map = - List.map network_config.docker.snark_coordinator_configs ~f:(fun config -> - (* Assign different command and environment configs depending on coordinator or worker *) - let command, environment = - match - String.substr_index config.name ~pattern:Snark_coordinator.name - with - | Some _ -> - let cmd = - Cmd.( - Snark_coordinator - (Snark_coordinator.default - ~snark_coordinator_key:config.public_key - ~snark_worker_fee:config.snark_worker_fee)) - in - let coordinator_command = - Cmd.create_cmd cmd ~config_file:runtime_config.target - in - let coordinator_environment = - Service.Environment.create - (Envs.snark_coord_envs - ~snark_coordinator_key:config.public_key - ~snark_worker_fee:config.snark_worker_fee) - in - (coordinator_command, coordinator_environment) - | None -> - let daemon_address = Snark_coordinator.name in - let daemon_port = Snark_coordinator.default_port in - let cmd = - Cmd.( - Snark_worker - (Snark_worker.default ~daemon_address ~daemon_port)) - in - let worker_command = - Cmd.create_cmd cmd ~config_file:runtime_config.target - in - let worker_environment = Service.Environment.create [] in - (worker_command, worker_environment) - in - let rest_port = - List.find - ~f:(fun ports -> String.equal (fst ports) "rest-port") - config.ports - |> Option.value_exn |> snd - in - ( config.name - , { Service.image = network_config.docker.mina_image - ; volumes = [ runtime_config ] - ; command - ; ports = [ rest_port ] - ; environment - } )) + let seed_map = + List.map network_config.docker.seed_configs ~f:(fun config -> + (config.service_name, config.docker_config) ) |> StringMap.of_alist_exn in - (* ARCHIVE NODE SERVICE *) - let archive_node_configs = - List.mapi network_config.docker.archive_node_configs - ~f:(fun index config -> - let postgres_uri = - Cmd.Cli_args.Postgres_uri.create - ~host: - ( network_config.docker.stack_name ^ "_" - ^ Archive_node.postgres_name ^ "-" - ^ Int.to_string (index + 1) ) - |> Cmd.Cli_args.Postgres_uri.to_string - in - let cmd = - let server_port = Archive_node.server_port in - Cmd.(Archive_node { Archive_node.postgres_uri; server_port }) - in - let rest_port = - List.find - ~f:(fun ports -> String.equal (fst ports) "rest-port") - config.ports - |> Option.value_exn |> snd - in - ( config.name - , { Service.image = network_config.docker.mina_archive_image - ; volumes = [ runtime_config ] - ; command = Cmd.create_cmd cmd ~config_file:runtime_config.target - ; ports = [ rest_port ] - ; environment = Service.Environment.create Envs.base_node_envs - } )) - (* Add an equal number of postgres containers as archive nodes *) - @ List.mapi network_config.docker.archive_node_configs ~f:(fun index _ -> - let pg_config = Cmd.Cli_args.Postgres_uri.default in - (* Mount the archive schema on the /docker-entrypoint-initdb.d postgres entrypoint *) - let archive_config = - Service.Volume.create mina_create_schema - (Archive_node.entrypoint_target ^/ mina_create_schema) - in - ( Archive_node.postgres_name ^ "-" ^ Int.to_string (index + 1) - , { Service.image = postgres_image - ; volumes = [ archive_config ] - ; command = [] - ; ports = [] - ; environment = - Service.Environment.create - (Envs.postgres_envs ~username:pg_config.username - ~password:pg_config.password ~database:pg_config.db - ~port:pg_config.port) - } )) + let snark_coordinator_map = + match network_config.docker.snark_coordinator_config with + | Some config -> + StringMap.of_alist_exn [ (config.service_name, config.docker_config) ] + | None -> + StringMap.empty + in + let snark_worker_map = + match network_config.docker.snark_coordinator_config with + | Some snark_coordinator_config -> + List.map snark_coordinator_config.config.worker_nodes + ~f:(fun config -> (config.service_name, config.docker_config)) + |> StringMap.of_alist_exn + | None -> + StringMap.empty + in + let archive_node_map = + List.map network_config.docker.archive_node_configs ~f:(fun config -> + (config.service_name, config.docker_config) ) |> StringMap.of_alist_exn in - (* SEED NODE DOCKER SERVICE *) - let seed_map = - List.mapi network_config.docker.seed_configs ~f:(fun index config -> - let command = - if List.length network_config.docker.archive_node_configs > 0 then - (* If an archive node is specified in the test plan, use the seed node to connect to the first mina-archive process *) - Cmd.create_cmd Seed ~config_file:runtime_config.target - @ Cmd.Seed.connect_to_archive - ~archive_node: - ( "archive-" - ^ Int.to_string (index + 1) - ^ ":" ^ Services.Archive_node.server_port ) - else Cmd.create_cmd Seed ~config_file:runtime_config.target - in - let rest_port = - List.find - ~f:(fun ports -> String.equal (fst ports) "rest-port") - config.ports - |> Option.value_exn |> snd - in - ( Seed.name - , { Service.image = network_config.docker.mina_image - ; volumes = [ runtime_config ] - ; command - ; ports = [ rest_port ] - ; environment = Service.Environment.create Envs.base_node_envs - } )) + let postgres_map = + List.map network_config.docker.archive_node_configs + ~f:(fun archive_config -> + let config = archive_config.config.postgres_config in + (config.service_name, config.docker_config) ) |> StringMap.of_alist_exn in let services = - seed_map |> merge block_producer_map |> merge snark_worker_map - |> merge archive_node_configs + postgres_map |> merge archive_node_map |> merge snark_worker_map + |> merge snark_coordinator_map + |> merge block_producer_map |> merge seed_map in { version = docker_swarm_version; services } end module Network_manager = struct type t = - { stack_name : string - ; logger : Logger.t - ; testnet_dir : string - ; testnet_log_filter : string + { logger : Logger.t + ; stack_name : string + ; graphql_enabled : bool + ; docker_dir : string + ; docker_compose_file_path : string ; constants : Test_config.constants - ; seed_nodes : Swarm_network.Node.t list - ; nodes_by_app_id : Swarm_network.Node.t String.Map.t - ; block_producer_nodes : Swarm_network.Node.t list - ; snark_coordinator_nodes : Swarm_network.Node.t list - ; archive_node_nodes : Swarm_network.Node.t list + ; seed_workloads : Docker_network.Service_to_deploy.t Core.String.Map.t + ; block_producer_workloads : + Docker_network.Service_to_deploy.t Core.String.Map.t + ; snark_coordinator_workloads : + Docker_network.Service_to_deploy.t Core.String.Map.t + ; snark_worker_workloads : + Docker_network.Service_to_deploy.t Core.String.Map.t + ; archive_workloads : Docker_network.Service_to_deploy.t Core.String.Map.t + ; services_by_id : Docker_network.Service_to_deploy.t Core.String.Map.t ; mutable deployed : bool - ; keypairs : Keypair.t list + ; genesis_keypairs : Network_keypair.t Core.String.Map.t } - let run_cmd t prog args = Util.run_cmd t.testnet_dir prog args + let get_current_running_stacks = + let open Malleable_error.Let_syntax in + let%bind all_stacks_str = + Util.run_cmd_or_hard_error "/" "docker" + [ "stack"; "ls"; "--format"; "{{.Name}}" ] + in + return (String.split ~on:'\n' all_stacks_str) - let run_cmd_exn t prog args = Util.run_cmd_exn t.testnet_dir prog args + let remove_stack_if_exists ~logger (network_config : Network_config.t) = + let open Malleable_error.Let_syntax in + let%bind all_stacks = get_current_running_stacks in + if List.mem all_stacks network_config.docker.stack_name ~equal:String.equal + then + let%bind () = + if network_config.debug_arg then + Deferred.bind ~f:Malleable_error.return + (Util.prompt_continue + "Existing stack name of same name detected, pausing startup. \ + Enter [y/Y] to continue on and remove existing stack name, \ + start clean, and run the test; press Ctrl-C to quit out: " ) + else + Malleable_error.return + ([%log info] + "Existing stack of same name detected; removing to start clean" ) + in + Util.run_cmd_or_hard_error "/" "docker" + [ "stack"; "rm"; network_config.docker.stack_name ] + >>| Fn.const () + else return () - (* - Creates a docker swarm network by creating a local working directory and writing a - docker_compose.json file to disk with all related resources (e.g. runtime_config and secret keys). + let generate_docker_stack_file ~logger ~docker_dir ~docker_compose_file_path + ~network_config = + let open Deferred.Let_syntax in + let%bind () = + if%bind File_system.dir_exists docker_dir then ( + [%log info] "Old docker stack directory found; removing to start clean" ; + File_system.remove_dir docker_dir ) + else return () + in + [%log info] "Writing docker configuration %s" docker_dir ; + let%bind () = Unix.mkdir docker_dir in + let%bind _ = + Docker_compose.Dockerfile.write_config ~dir:docker_dir + ~filename:docker_compose_file_path + (Network_config.to_docker network_config) + in + return () - First, check if there is a running swarm network and let the user remove it. Continue by creating a - working directory and writing out all resources to be used as docker volumes. Additionally for each resource, specify - file permissions of 600 so that the daemon can run properly. - *) - let create ~logger (network_config : Network_config.t) = - let%bind all_stacks_str = - Util.run_cmd_exn "/" "docker" [ "stack"; "ls"; "--format"; "{{.Name}}" ] + let write_docker_bind_volumes ~logger ~docker_dir + ~(network_config : Network_config.t) = + let open Deferred.Let_syntax in + [%log info] "Writing runtime_config %s" docker_dir ; + let%bind () = + Yojson.Safe.to_file + (String.concat [ docker_dir; "/runtime_config.json" ]) + network_config.docker.runtime_config + |> Deferred.return in - let all_stacks = String.split ~on:'\n' all_stacks_str in - let testnet_dir = network_config.docker.stack_name in + [%log info] "Writing out the genesis keys to dir %s" docker_dir ; + let kps_base_path = String.concat [ docker_dir; "/keys" ] in + let%bind () = Unix.mkdir kps_base_path in + [%log info] "Writing genesis keys to %s" kps_base_path ; let%bind () = - if - List.mem all_stacks network_config.docker.stack_name ~equal:String.equal - then - let%bind () = - if network_config.debug_arg then - Util.prompt_continue - "Existing stack name of same name detected, pausing startup. \ - Enter [y/Y] to continue on and remove existing stack name, \ - start clean, and run the test; press Ctrl-C to quit out: " - else - Deferred.return - ([%log info] - "Existing stack of same name detected; removing to start clean") - in - Util.run_cmd_exn "/" "docker" - [ "stack"; "rm"; network_config.docker.stack_name ] - >>| Fn.const () - else return () + Core.String.Map.iter network_config.genesis_keypairs ~f:(fun kp -> + let keypath = String.concat [ kps_base_path; "/"; kp.keypair_name ] in + Out_channel.with_file ~fail_if_exists:true keypath ~f:(fun ch -> + kp.private_key |> Out_channel.output_string ch ) ; + Out_channel.with_file ~fail_if_exists:true (keypath ^ ".pub") + ~f:(fun ch -> kp.public_key |> Out_channel.output_string ch) ; + ignore + (Util.run_cmd_exn kps_base_path "chmod" [ "600"; kp.keypair_name ]) ) + |> Deferred.return in + [%log info] "Writing seed libp2p keypair to %s" kps_base_path ; let%bind () = - if%bind File_system.dir_exists testnet_dir then ( - [%log info] "Old docker stack directory found; removing to start clean" ; - File_system.remove_dir testnet_dir ) - else return () + let keypath = String.concat [ kps_base_path; "/"; "libp2p_key" ] in + Out_channel.with_file ~fail_if_exists:true keypath ~f:(fun ch -> + Docker_node_config.Seed_config.libp2p_keypair + |> Out_channel.output_string ch ) ; + ignore (Util.run_cmd_exn kps_base_path "chmod" [ "600"; "libp2p_key" ]) ; + return () in - let%bind () = Unix.mkdir testnet_dir in - [%log info] "Writing network configuration" ; - Out_channel.with_file ~fail_if_exists:true (testnet_dir ^/ "compose.json") - ~f:(fun ch -> - Network_config.to_docker network_config - |> Docker_compose.to_string - |> Out_channel.output_string ch) ; - List.iter network_config.docker.docker_volume_configs ~f:(fun config -> - [%log info] "Writing volume config: %s" (testnet_dir ^/ config.name) ; - Out_channel.with_file ~fail_if_exists:false (testnet_dir ^/ config.name) - ~f:(fun ch -> config.data |> Out_channel.output_string ch) ; - ignore (Util.run_cmd_exn testnet_dir "chmod" [ "600"; config.name ])) ; - let cons_node swarm_name service_id ports network_keypair_opt = - { Swarm_network.Node.swarm_name - ; service_id - ; ports - ; graphql_enabled = true - ; network_keypair = network_keypair_opt - } + let%bind () = + ignore (Util.run_cmd_exn docker_dir "chmod" [ "700"; "keys" ]) + |> Deferred.return + in + [%log info] + "Writing custom entrypoint script (libp2p key generation and puppeteer \ + context)" ; + let entrypoint_filename, entrypoint_script = + Docker_node_config.Base_node_config.entrypoint_script + in + Out_channel.with_file ~fail_if_exists:true + (docker_dir ^/ entrypoint_filename) ~f:(fun ch -> + entrypoint_script |> Out_channel.output_string ch ) ; + let%bind _ = + Deferred.List.iter network_config.docker.mina_archive_schema_aux_files + ~f:(fun schema_url -> + let filename = Filename.basename schema_url in + [%log info] "Downloading %s" schema_url ; + let%bind _ = + Util.run_cmd_or_hard_error docker_dir "curl" + [ "-o"; filename; schema_url ] + in + [%log info] + "Writing custom postgres entrypoint script (import archive node \ + schema)" ; + + Deferred.return () ) + |> Deferred.return + in + ignore (Util.run_cmd_exn docker_dir "chmod" [ "+x"; entrypoint_filename ]) ; + let postgres_entrypoint_filename, postgres_entrypoint_script = + Docker_node_config.Postgres_config.postgres_script + in + Out_channel.with_file ~fail_if_exists:true + (docker_dir ^/ postgres_entrypoint_filename) ~f:(fun ch -> + postgres_entrypoint_script |> Out_channel.output_string ch ) ; + ignore + (Util.run_cmd_exn docker_dir "chmod" + [ "+x"; postgres_entrypoint_filename ] ) ; + return () + + let initialize_workloads ~logger (network_config : Network_config.t) = + let find_rest_port ports = + List.find_map_exn ports ~f:(fun port -> + match port with + | Docker_compose.Dockerfile.Service.Port.{ published; target } -> + if target = Docker_node_config.PortManager.mina_internal_rest_port + then Some published + else None ) in - let seed_nodes = + [%log info] "Initializing seed workloads" ; + let seed_workloads = List.map network_config.docker.seed_configs ~f:(fun seed_config -> - cons_node network_config.docker.stack_name - (network_config.docker.stack_name ^ "_" ^ seed_config.name) - seed_config.ports None) + let graphql_port = find_rest_port seed_config.docker_config.ports in + let node = + Docker_network.Service_to_deploy.construct_service + network_config.docker.stack_name seed_config.service_name + (Docker_network.Service_to_deploy.init_service_to_deploy_config + ~network_keypair:None ~postgres_connection_uri:None + ~graphql_port ) + in + (seed_config.service_name, node) ) + |> Core.String.Map.of_alist_exn in - let block_producer_nodes = + [%log info] "Initializing block producer workloads" ; + let block_producer_workloads = List.map network_config.docker.block_producer_configs ~f:(fun bp_config -> - cons_node network_config.docker.stack_name - (network_config.docker.stack_name ^ "_" ^ bp_config.name) - bp_config.ports (Some bp_config.keypair)) + let graphql_port = find_rest_port bp_config.docker_config.ports in + let node = + Docker_network.Service_to_deploy.construct_service + network_config.docker.stack_name bp_config.service_name + (Docker_network.Service_to_deploy.init_service_to_deploy_config + ~network_keypair:(Some bp_config.config.keypair) + ~postgres_connection_uri:None ~graphql_port ) + in + (bp_config.service_name, node) ) + |> Core.String.Map.of_alist_exn in - let snark_coordinator_nodes = - List.map network_config.docker.snark_coordinator_configs - ~f:(fun snark_config -> - cons_node network_config.docker.stack_name - (network_config.docker.stack_name ^ "_" ^ snark_config.name) - snark_config.ports None) + [%log info] "Initializing snark coordinator and worker workloads" ; + let snark_coordinator_workloads, snark_worker_workloads = + match network_config.docker.snark_coordinator_config with + | Some snark_coordinator_config -> + let snark_coordinator_workloads = + if List.length snark_coordinator_config.config.worker_nodes > 0 then + let graphql_port = + find_rest_port snark_coordinator_config.docker_config.ports + in + let coordinator = + Docker_network.Service_to_deploy.construct_service + network_config.docker.stack_name + snark_coordinator_config.service_name + (Docker_network.Service_to_deploy + .init_service_to_deploy_config ~network_keypair:None + ~postgres_connection_uri:None ~graphql_port ) + in + [ (snark_coordinator_config.service_name, coordinator) ] + |> Core.String.Map.of_alist_exn + else Core.String.Map.empty + in + let snark_worker_workloads = + List.map snark_coordinator_config.config.worker_nodes + ~f:(fun snark_worker_config -> + let graphql_port = + find_rest_port snark_worker_config.docker_config.ports + in + let worker = + Docker_network.Service_to_deploy.construct_service + network_config.docker.stack_name + snark_worker_config.service_name + (Docker_network.Service_to_deploy + .init_service_to_deploy_config ~network_keypair:None + ~postgres_connection_uri:None ~graphql_port ) + in + + (snark_worker_config.service_name, worker) ) + |> Core.String.Map.of_alist_exn + in + (snark_coordinator_workloads, snark_worker_workloads) + | None -> + (Core.String.Map.of_alist_exn [], Core.String.Map.of_alist_exn []) in - let%bind _ = - (* If any archive nodes are specified, write the archive schema to the working directory to use as an entrypoint for postgres *) - if List.length network_config.docker.archive_node_configs > 0 then - let cmd = - Printf.sprintf "curl -Ls %s > %s" - network_config.docker.mina_archive_schema mina_create_schema + [%log info] "Initializing archive node workloads" ; + let archive_workloads = + List.map network_config.docker.archive_node_configs + ~f:(fun archive_config -> + let graphql_port = + find_rest_port archive_config.docker_config.ports + in + let postgres_connection_uri = + Some + (Docker_node_config.Postgres_config.to_connection_uri + archive_config.config.postgres_config.config ) + in + let node = + Docker_network.Service_to_deploy.construct_service + network_config.docker.stack_name archive_config.service_name + (Docker_network.Service_to_deploy.init_service_to_deploy_config + ~network_keypair:None ~postgres_connection_uri ~graphql_port ) + in + (archive_config.service_name, node) ) + |> Core.String.Map.of_alist_exn + in + ( seed_workloads + , block_producer_workloads + , snark_coordinator_workloads + , snark_worker_workloads + , archive_workloads ) + + let poll_until_stack_deployed ~logger = + let poll_interval = Time.Span.of_sec 15.0 in + let max_polls = 20 (* 5 mins *) in + let get_service_statuses () = + let%bind output = + Util.run_cmd_exn "/" "docker" + [ "service"; "ls"; "--format"; "{{.Name}}: {{.Replicas}}" ] + in + return + ( output |> String.split_lines + |> List.map ~f:(fun line -> + match String.split ~on:':' line with + | [ name; replicas ] -> + (String.strip name, String.strip replicas) + | _ -> + failwith "Unexpected format for docker service output" ) ) + in + let rec poll n = + [%log debug] "Checking Docker service statuses, n=%d" n ; + let%bind service_statuses = get_service_statuses () in + let bad_service_statuses = + List.filter service_statuses ~f:(fun (_, status) -> + let parts = String.split ~on:'/' status in + assert (List.length parts = 2) ; + let num, denom = + ( String.strip (List.nth_exn parts 0) + , String.strip (List.nth_exn parts 1) ) + in + not (String.equal num denom) ) + in + let open Malleable_error.Let_syntax in + if List.is_empty bad_service_statuses then return () + else if n > 0 then ( + [%log debug] "Got bad service statuses, polling again ($failed_statuses" + ~metadata: + [ ( "failed_statuses" + , `Assoc + (List.Assoc.map bad_service_statuses ~f:(fun v -> `String v)) + ) + ] ; + let%bind () = + after poll_interval |> Deferred.bind ~f:Malleable_error.return in - Util.run_cmd_exn testnet_dir "bash" [ "-c"; cmd ] - else return "" + poll (n - 1) ) + else + let bad_service_statuses_json = + `List + (List.map bad_service_statuses ~f:(fun (service_name, status) -> + `Assoc + [ ("service_name", `String service_name) + ; ("status", `String status) + ] ) ) + in + [%log fatal] + "Not all services could be deployed in time: $bad_service_statuses" + ~metadata:[ ("bad_service_statuses", bad_service_statuses_json) ] ; + Malleable_error.hard_error_string ~exit_code:4 + (Yojson.Safe.to_string bad_service_statuses_json) in - let archive_node_nodes = - List.map network_config.docker.archive_node_configs - ~f:(fun archive_node -> - cons_node network_config.docker.stack_name - (network_config.docker.stack_name ^ "_" ^ archive_node.name) - archive_node.ports None) - in - let nodes_by_app_id = - let all_nodes = - seed_nodes @ block_producer_nodes @ snark_coordinator_nodes - @ archive_node_nodes + [%log info] "Waiting for Docker services to be deployed" ; + let res = poll max_polls in + match%bind.Deferred res with + | Error _ -> + [%log error] "Not all Docker services were deployed, cannot proceed!" ; + res + | Ok _ -> + [%log info] "Docker services deployed" ; + res + + let create ~logger (network_config : Network_config.t) = + let open Malleable_error.Let_syntax in + let%bind () = remove_stack_if_exists ~logger network_config in + let ( seed_workloads + , block_producer_workloads + , snark_coordinator_workloads + , snark_worker_workloads + , archive_workloads ) = + initialize_workloads ~logger network_config + in + let services_by_id = + let all_workloads = + Core.String.Map.data seed_workloads + @ Core.String.Map.data snark_coordinator_workloads + @ Core.String.Map.data snark_worker_workloads + @ Core.String.Map.data block_producer_workloads + @ Core.String.Map.data archive_workloads in - all_nodes - |> List.map ~f:(fun node -> (node.service_id, node)) + all_workloads + |> List.map ~f:(fun w -> (w.service_name, w)) |> String.Map.of_alist_exn in + let open Deferred.Let_syntax in + let docker_dir = network_config.docker.stack_name in + let docker_compose_file_path = + network_config.docker.stack_name ^ ".compose.json" + in + let%bind () = + generate_docker_stack_file ~logger ~docker_dir ~docker_compose_file_path + ~network_config + in + let%bind () = + write_docker_bind_volumes ~logger ~docker_dir ~network_config + in let t = { stack_name = network_config.docker.stack_name ; logger - ; testnet_dir + ; docker_dir + ; docker_compose_file_path ; constants = network_config.constants - ; seed_nodes - ; block_producer_nodes - ; snark_coordinator_nodes - ; archive_node_nodes - ; nodes_by_app_id + ; graphql_enabled = true + ; seed_workloads + ; block_producer_workloads + ; snark_coordinator_workloads + ; snark_worker_workloads + ; archive_workloads + ; services_by_id ; deployed = false - ; testnet_log_filter = "" - ; keypairs = - List.map network_config.keypairs ~f:(fun { keypair; _ } -> keypair) + ; genesis_keypairs = network_config.genesis_keypairs } in - Deferred.return t + [%log info] "Initializing docker swarm" ; + Malleable_error.return t let deploy t = + let logger = t.logger in if t.deployed then failwith "network already deployed" ; - [%log' info t.logger] "Deploying network" ; - [%log' info t.logger] "Stack_name in deploy: %s" t.stack_name ; - let%map _ = - run_cmd_exn t "docker" - [ "stack"; "deploy"; "-c"; "compose.json"; t.stack_name ] + [%log info] "Deploying stack '%s' from %s" t.stack_name t.docker_dir ; + let open Malleable_error.Let_syntax in + let%bind (_ : string) = + Util.run_cmd_or_hard_error t.docker_dir "docker" + [ "stack"; "deploy"; "-c"; t.docker_compose_file_path; t.stack_name ] in t.deployed <- true ; - let result = - { Swarm_network.namespace = t.stack_name + let%bind () = poll_until_stack_deployed ~logger in + let open Malleable_error.Let_syntax in + let func_for_fold ~(key : string) ~data accum_M = + let%bind mp = accum_M in + let%map node = + Docker_network.Service_to_deploy.get_node_from_service data + in + Core.String.Map.add_exn mp ~key ~data:node + in + let%map seeds = + Core.String.Map.fold t.seed_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold + and block_producers = + Core.String.Map.fold t.block_producer_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold + and snark_coordinators = + Core.String.Map.fold t.snark_coordinator_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold + and snark_workers = + Core.String.Map.fold t.snark_worker_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold + and archive_nodes = + Core.String.Map.fold t.archive_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold + in + let network = + { Docker_network.namespace = t.stack_name ; constants = t.constants - ; seeds = t.seed_nodes - ; block_producers = t.block_producer_nodes - ; snark_coordinators = t.snark_coordinator_nodes - ; archive_nodes = t.archive_node_nodes - ; nodes_by_app_id = t.nodes_by_app_id - ; testnet_log_filter = t.testnet_log_filter - ; keypairs = t.keypairs + ; seeds + ; block_producers + ; snark_coordinators + ; snark_workers + ; archive_nodes + ; genesis_keypairs = t.genesis_keypairs } in let nodes_to_string = - Fn.compose (String.concat ~sep:", ") (List.map ~f:Swarm_network.Node.id) - in - [%log' info t.logger] "Network deployed" ; - [%log' info t.logger] "testnet swarm: %s" t.stack_name ; - [%log' info t.logger] "seed nodes: %s" (nodes_to_string result.seeds) ; - [%log' info t.logger] "snark coordinators: %s" - (nodes_to_string result.snark_coordinators) ; - [%log' info t.logger] "block producers: %s" - (nodes_to_string result.block_producers) ; - [%log' info t.logger] "archive nodes: %s" - (nodes_to_string result.archive_nodes) ; - result + Fn.compose (String.concat ~sep:", ") (List.map ~f:Docker_network.Node.id) + in + [%log info] "Network deployed" ; + [%log info] "testnet namespace: %s" t.stack_name ; + [%log info] "snark coordinators: %s" + (nodes_to_string (Core.String.Map.data network.snark_coordinators)) ; + [%log info] "snark workers: %s" + (nodes_to_string (Core.String.Map.data network.snark_workers)) ; + [%log info] "block producers: %s" + (nodes_to_string (Core.String.Map.data network.block_producers)) ; + [%log info] "archive nodes: %s" + (nodes_to_string (Core.String.Map.data network.archive_nodes)) ; + network let destroy t = [%log' info t.logger] "Destroying network" ; if not t.deployed then failwith "network not deployed" ; - let%bind _ = run_cmd_exn t "docker" [ "stack"; "rm"; t.stack_name ] in + let%bind _ = + Util.run_cmd_exn "/" "docker" [ "stack"; "rm"; t.stack_name ] + in t.deployed <- false ; Deferred.unit let cleanup t = let%bind () = if t.deployed then destroy t else return () in [%log' info t.logger] "Cleaning up network configuration" ; - let%bind () = File_system.remove_dir t.testnet_dir in + let%bind () = File_system.remove_dir t.docker_dir in Deferred.unit + + let destroy t = + Deferred.Or_error.try_with ~here:[%here] (fun () -> destroy t) + |> Deferred.bind ~f:Malleable_error.or_hard_error end diff --git a/src/lib/integration_test_local_engine/swarm_network.ml b/src/lib/integration_test_local_engine/swarm_network.ml deleted file mode 100644 index 27974d3282c..00000000000 --- a/src/lib/integration_test_local_engine/swarm_network.ml +++ /dev/null @@ -1,453 +0,0 @@ -open Core -open Async -open Integration_test_lib - -(* exclude from bisect_ppx to avoid type error on GraphQL modules *) -[@@@coverage exclude_file] - -module Node = struct - type t = - { swarm_name : string - ; service_id : string - ; ports : (string * string) list - ; graphql_enabled : bool - ; network_keypair : Network_keypair.t option - } - - let id { service_id; _ } = service_id - - let network_keypair { network_keypair; _ } = network_keypair - - let get_container_cmd t = - Printf.sprintf "$(docker ps -f name=%s --quiet)" t.service_id - - let run_in_postgresql_container _ _ = failwith "run_in_postgresql_container" - - let get_logs_in_container _ = failwith "get_logs_in_container" - - let run_in_container node cmd = - let base_docker_cmd = "docker exec" in - let docker_cmd = - Printf.sprintf "%s %s %s" base_docker_cmd (get_container_cmd node) cmd - in - let%bind.Deferred cwd = Unix.getcwd () in - Malleable_error.return (Util.run_cmd_exn cwd "sh" [ "-c"; docker_cmd ]) - - let start ~fresh_state node : unit Malleable_error.t = - let open Malleable_error.Let_syntax in - let%bind _ = - Deferred.bind ~f:Malleable_error.return (run_in_container node "ps aux") - in - let%bind () = - if fresh_state then - let%bind _ = run_in_container node "rm -rf .mina-config/*" in - Malleable_error.return () - else Malleable_error.return () - in - let cmd = - match String.substr_index node.service_id ~pattern:"snark-worker" with - | Some _ -> - (* Snark-workers should wait for work to be generated so they don't error a 'get_work' RPC call*) - "/bin/bash -c 'sleep 120 && ./start.sh'" - | None -> - "./start.sh" - in - let%bind _ = run_in_container node cmd in - Malleable_error.return () - - let stop node = - let open Malleable_error.Let_syntax in - let%bind _ = run_in_container node "ps aux" in - let%bind _ = run_in_container node "./stop.sh" in - let%bind _ = run_in_container node "ps aux" in - return () - - module Decoders = Graphql_lib.Decoders - - module Graphql = struct - let ingress_uri node = - let host = Printf.sprintf "0.0.0.0" in - let path = "/graphql" in - let rest_port = - List.find - ~f:(fun ports -> String.equal (fst ports) "rest-port") - node.ports - |> Option.value_exn |> snd - in - Uri.make ~scheme:"http" ~host ~path ~port:(int_of_string rest_port) () - - module Client = Graphql_lib.Client.Make (struct - let preprocess_variables_string = Fn.id - - let headers = String.Map.empty - end) - - module Unlock_account = - [%graphql - {| - mutation ($password: String!, $public_key: PublicKey!) { - unlockAccount(input: {password: $password, publicKey: $public_key }) { - public_key: publicKey @bsDecoder(fn: "Decoders.public_key") - } - } - |}] - - module Send_payment = - [%graphql - {| - mutation ($sender: PublicKey!, - $receiver: PublicKey!, - $amount: UInt64!, - $token: UInt64, - $fee: UInt64!, - $nonce: UInt32, - $memo: String) { - sendPayment(input: - {from: $sender, to: $receiver, amount: $amount, token: $token, fee: $fee, nonce: $nonce, memo: $memo}) { - payment { - id - } - } - } - |}] - - module Get_balance = - [%graphql - {| - query ($public_key: PublicKey, $token: UInt64) { - account(publicKey: $public_key, token: $token) { - balance { - total @bsDecoder(fn: "Decoders.balance") - } - } - } - |}] - - module Query_peer_id = - [%graphql - {| - query { - daemonStatus { - addrsAndPorts { - peer { - peerId - } - } - peers { peerId } - - } - } - |}] - - module Best_chain = - [%graphql - {| - query { - bestChain { - stateHash - } - } - |}] - end - - let exec_graphql_request ?(num_tries = 10) ?(retry_delay_sec = 30.0) - ?(initial_delay_sec = 30.0) ~logger ~node ~query_name query_obj = - let open Deferred.Let_syntax in - if not node.graphql_enabled then - Deferred.Or_error.error_string - "graphql is not enabled (hint: set `requires_graphql= true` in the \ - test config)" - else - let uri = Graphql.ingress_uri node in - let metadata = - [ ("query", `String query_name); ("uri", `String (Uri.to_string uri)) ] - in - [%log info] "Attempting to send GraphQL request \"$query\" to \"$uri\"" - ~metadata ; - let rec retry n = - if n <= 0 then ( - [%log error] - "GraphQL request \"$query\" to \"$uri\" failed too many times" - ~metadata ; - Deferred.Or_error.errorf - "GraphQL \"%s\" to \"%s\" request failed too many times" query_name - (Uri.to_string uri) ) - else - match%bind Graphql.Client.query query_obj uri with - | Ok result -> - [%log info] "GraphQL request \"$query\" to \"$uri\" succeeded" - ~metadata ; - Deferred.Or_error.return result - | Error (`Failed_request err_string) -> - [%log warn] - "GraphQL request \"$query\" to \"$uri\" failed: \"$error\" \ - ($num_tries attempts left)" - ~metadata: - ( metadata - @ [ ("error", `String err_string) - ; ("num_tries", `Int (n - 1)) - ] ) ; - let%bind () = after (Time.Span.of_sec retry_delay_sec) in - retry (n - 1) - | Error (`Graphql_error err_string) -> - [%log error] - "GraphQL request \"$query\" to \"$uri\" returned an error: \ - \"$error\" (this is a graphql error so not retrying)" - ~metadata:(metadata @ [ ("error", `String err_string) ]) ; - Deferred.Or_error.error_string err_string - in - let%bind () = after (Time.Span.of_sec initial_delay_sec) in - retry num_tries - - let get_peer_id ~logger t = - let open Deferred.Or_error.Let_syntax in - [%log info] "Getting node's peer_id, and the peer_ids of node's peers" - ~metadata:[ ("service_id", `String t.service_id) ] ; - let query_obj = Graphql.Query_peer_id.make () in - let%bind query_result_obj = - exec_graphql_request ~logger ~node:t ~query_name:"query_peer_id" query_obj - in - [%log info] "get_peer_id, finished exec_graphql_request" ; - let self_id_obj = query_result_obj#daemonStatus#addrsAndPorts#peer in - let%bind self_id = - match self_id_obj with - | None -> - Deferred.Or_error.error_string "Peer not found" - | Some peer -> - return peer#peerId - in - let peers = query_result_obj#daemonStatus#peers |> Array.to_list in - let peer_ids = List.map peers ~f:(fun peer -> peer#peerId) in - [%log info] "get_peer_id, result of graphql query (self_id,[peers]) (%s,%s)" - self_id - (String.concat ~sep:" " peer_ids) ; - return (self_id, peer_ids) - - let must_get_peer_id ~logger t = - get_peer_id ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error - - let get_best_chain ~logger t = - let open Deferred.Or_error.Let_syntax in - let query = Graphql.Best_chain.make () in - let%bind result = - exec_graphql_request ~logger ~node:t ~query_name:"best_chain" query - in - match result#bestChain with - | None | Some [||] -> - Deferred.Or_error.error_string "failed to get best chains" - | Some chain -> - return - @@ List.map ~f:(fun block -> block#stateHash) (Array.to_list chain) - - let must_get_best_chain ~logger t = - get_best_chain ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error - - let get_balance ~logger t ~account_id = - let open Deferred.Or_error.Let_syntax in - [%log info] "Getting account balance" - ~metadata: - [ ("service_id", `String t.service_id) - ; ("account_id", Mina_base.Account_id.to_yojson account_id) - ] ; - let pk = Mina_base.Account_id.public_key account_id in - let token = Mina_base.Account_id.token_id account_id in - let get_balance_obj = - Graphql.Get_balance.make - ~public_key:(Graphql_lib.Encoders.public_key pk) - ~token:(Graphql_lib.Encoders.token token) - () - in - let%bind balance_obj = - exec_graphql_request ~logger ~node:t ~query_name:"get_balance_graphql" - get_balance_obj - in - match balance_obj#account with - | None -> - Deferred.Or_error.errorf - !"Account with %{sexp:Mina_base.Account_id.t} not found" - account_id - | Some acc -> - return acc#balance#total - - let must_get_balance ~logger t ~account_id = - get_balance ~logger t ~account_id - |> Deferred.bind ~f:Malleable_error.or_hard_error - - (* if we expect failure, might want retry_on_graphql_error to be false *) - let send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee = - [%log info] "Sending a payment" - ~metadata:[ ("service_id", `String t.service_id) ] ; - let open Deferred.Or_error.Let_syntax in - let sender_pk_str = - Signature_lib.Public_key.Compressed.to_string sender_pub_key - in - [%log info] "send_payment: unlocking account" - ~metadata:[ ("sender_pk", `String sender_pk_str) ] ; - let unlock_sender_account_graphql () = - let unlock_account_obj = - Graphql.Unlock_account.make ~password:"naughty blue worm" - ~public_key:(Graphql_lib.Encoders.public_key sender_pub_key) - () - in - exec_graphql_request ~logger ~node:t - ~query_name:"unlock_sender_account_graphql" unlock_account_obj - in - let%bind _ = unlock_sender_account_graphql () in - let send_payment_graphql () = - let send_payment_obj = - Graphql.Send_payment.make - ~sender:(Graphql_lib.Encoders.public_key sender_pub_key) - ~receiver:(Graphql_lib.Encoders.public_key receiver_pub_key) - ~amount:(Graphql_lib.Encoders.amount amount) - ~fee:(Graphql_lib.Encoders.fee fee) - () - in - exec_graphql_request ~logger ~node:t ~query_name:"send_payment_graphql" - send_payment_obj - in - let%map sent_payment_obj = send_payment_graphql () in - let (`UserCommand id_obj) = sent_payment_obj#sendPayment#payment in - let user_cmd_id = id_obj#id in - [%log info] "Sent payment" - ~metadata:[ ("user_command_id", `String user_cmd_id) ] ; - () - - let must_send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee - = - send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee - |> Deferred.bind ~f:Malleable_error.or_hard_error - - let dump_archive_data ~logger:_ (_ : t) ~data_file:_ = - failwith "dump_archive_data" - - let dump_mina_logs ~logger:_ (_ : t) ~log_file:_ = Malleable_error.return () - - let dump_precomputed_blocks ~logger:_ (_ : t) = Malleable_error.return () -end - -type t = - { namespace : string - ; constants : Test_config.constants - ; seeds : Node.t list - ; block_producers : Node.t list - ; snark_coordinators : Node.t list - ; archive_nodes : Node.t list - ; testnet_log_filter : string - ; keypairs : Signature_lib.Keypair.t list - ; nodes_by_app_id : Node.t String.Map.t - } - -let constants { constants; _ } = constants - -let constraint_constants { constants; _ } = constants.constraints - -let genesis_constants { constants; _ } = constants.genesis - -let seeds { seeds; _ } = seeds - -let block_producers { block_producers; _ } = block_producers - -let snark_coordinators { snark_coordinators; _ } = snark_coordinators - -let archive_nodes { archive_nodes; _ } = archive_nodes - -let keypairs { keypairs; _ } = keypairs - -let all_nodes { seeds; block_producers; snark_coordinators; archive_nodes; _ } = - List.concat [ seeds; block_producers; snark_coordinators; archive_nodes ] - -let lookup_node_by_app_id t = Map.find t.nodes_by_app_id - -let initialize ~logger network = - Print.print_endline "initialize" ; - let open Malleable_error.Let_syntax in - let poll_interval = Time.Span.of_sec 15.0 in - let max_polls = 60 (* 15 mins *) in - let all_services = - all_nodes network - |> List.map ~f:(fun { service_id; _ } -> service_id) - |> String.Set.of_list - in - let get_service_statuses () = - let%map output = - Deferred.bind ~f:Malleable_error.return - (Util.run_cmd_exn "/" "docker" - [ "service"; "ls"; "--format"; "{{.Name}}: {{.Replicas}}" ]) - in - output |> String.split_lines - |> List.map ~f:(fun line -> - let parts = String.split line ~on:':' in - assert (List.length parts = 2) ; - (List.nth_exn parts 0, List.nth_exn parts 1)) - |> List.filter ~f:(fun (service_name, _) -> - String.Set.mem all_services service_name) - in - let rec poll n = - let%bind pod_statuses = get_service_statuses () in - (* TODO: detect "bad statuses" (eg CrashLoopBackoff) and terminate early *) - let bad_service_statuses = - List.filter pod_statuses ~f:(fun (_, status) -> - let parts = String.split status ~on:'/' in - assert (List.length parts = 2) ; - let num, denom = - ( String.strip (List.nth_exn parts 0) - , String.strip (List.nth_exn parts 1) ) - in - not (String.equal num denom)) - in - if List.is_empty bad_service_statuses then return () - else if n < max_polls then - let%bind () = - after poll_interval |> Deferred.bind ~f:Malleable_error.return - in - poll (n + 1) - else - let bad_service_statuses_json = - `List - (List.map bad_service_statuses ~f:(fun (service_name, status) -> - `Assoc - [ ("service_name", `String service_name) - ; ("status", `String status) - ])) - in - [%log fatal] - "Not all services could be deployed in time: $bad_service_statuses" - ~metadata:[ ("bad_service_statuses", bad_service_statuses_json) ] ; - Malleable_error.hard_error_format - "Some services either were not deployed properly (errors: %s)" - (Yojson.Safe.to_string bad_service_statuses_json) - in - [%log info] "Waiting for pods to be assigned nodes and become ready" ; - Deferred.bind (poll 0) ~f:(fun res -> - if Malleable_error.is_ok res then - let seed_nodes = seeds network in - let seed_service_ids = - seed_nodes - |> List.map ~f:(fun { Node.service_id; _ } -> service_id) - |> String.Set.of_list - in - let archive_nodes = archive_nodes network in - let archive_service_ids = - archive_nodes - |> List.map ~f:(fun { Node.service_id; _ } -> service_id) - |> String.Set.of_list - in - let non_seed_archive_nodes = - network |> all_nodes - |> List.filter ~f:(fun { Node.service_id; _ } -> - (not (String.Set.mem seed_service_ids service_id)) - && not (String.Set.mem archive_service_ids service_id)) - in - (* TODO: parallelize (requires accumlative hard errors) *) - let%bind () = - Malleable_error.List.iter seed_nodes - ~f:(Node.start ~fresh_state:false) - in - (* put a short delay before starting other nodes, to help avoid artifact generation races *) - let%bind () = - after (Time.Span.of_sec 30.0) - |> Deferred.bind ~f:Malleable_error.return - in - Malleable_error.List.iter non_seed_archive_nodes - ~f:(Node.start ~fresh_state:false) - else Deferred.return res) From ee55b3045f2a9301dbbadcef210d0c304b1ceb0d Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Fri, 22 Dec 2023 21:15:28 -0800 Subject: [PATCH 17/25] refactor graphql log engine to be functor and reuse with cloud and local The changes were done to make the graphql_polling_log_engine more reusable by making it a functor. This allows it to support different networks, such as the docker network. The old files were deleted as they are no longer needed. The error_json was added to the dune file in integration_test_lib for better error handling. --- .../graphql_polling_log_engine.ml | 142 ----------------- .../graphql_polling_log_engine.mli | 3 - .../integration_test_cloud_engine.ml | 6 +- src/lib/integration_test_lib/dune | 1 + .../graphql_polling_log_engine.ml | 144 +++++++++++++++++ .../docker_pipe_log_engine.ml | 147 ------------------ .../docker_pipe_log_engine.mli | 3 - .../integration_test_local_engine.ml | 6 +- 8 files changed, 155 insertions(+), 297 deletions(-) delete mode 100644 src/lib/integration_test_cloud_engine/graphql_polling_log_engine.ml delete mode 100644 src/lib/integration_test_cloud_engine/graphql_polling_log_engine.mli create mode 100644 src/lib/integration_test_lib/graphql_polling_log_engine.ml delete mode 100644 src/lib/integration_test_local_engine/docker_pipe_log_engine.ml delete mode 100644 src/lib/integration_test_local_engine/docker_pipe_log_engine.mli diff --git a/src/lib/integration_test_cloud_engine/graphql_polling_log_engine.ml b/src/lib/integration_test_cloud_engine/graphql_polling_log_engine.ml deleted file mode 100644 index 42d23e91dcd..00000000000 --- a/src/lib/integration_test_cloud_engine/graphql_polling_log_engine.ml +++ /dev/null @@ -1,142 +0,0 @@ -open Async -open Core -open Integration_test_lib -module Timeout = Timeout_lib.Core_time -module Node = Kubernetes_network.Node - -(** This implements Log_engine_intf for integration tests, by creating a simple system that polls a mina daemon's graphql endpoint for fetching logs*) - -let log_filter_of_event_type ev_existential = - let open Event_type in - let (Event_type ev_type) = ev_existential in - let (module Ty) = event_type_module ev_type in - match Ty.parse with - | From_error_log _ -> - [] (* TODO: Do we need this? *) - | From_daemon_log (struct_id, _) -> - [ Structured_log_events.string_of_id struct_id ] - | From_puppeteer_log _ -> - [] -(* TODO: Do we need this? *) - -let all_event_types_log_filter = - List.bind ~f:log_filter_of_event_type Event_type.all_event_types - -type t = - { logger : Logger.t - ; event_writer : (Node.t * Event_type.event) Pipe.Writer.t - ; event_reader : (Node.t * Event_type.event) Pipe.Reader.t - ; background_job : unit Deferred.Or_error.t - } - -let event_reader { event_reader; _ } = event_reader - -let parse_event_from_log_entry ~logger log_entry = - let open Or_error.Let_syntax in - let open Json_parsing in - Or_error.try_with_join (fun () -> - let payload = Yojson.Safe.from_string log_entry in - let%map event = - let%bind msg = - parse (parser_from_of_yojson Logger.Message.of_yojson) payload - in - let event_id = - Option.map ~f:Structured_log_events.string_of_id msg.event_id - in - [%log spam] "parsing daemon structured event, event_id = $event_id" - ~metadata:[ ("event_id", [%to_yojson: string option] event_id) ] ; - match msg.event_id with - | Some _ -> - Event_type.parse_daemon_event msg - | None -> - (* Currently unreachable, but we could include error logs here if - desired. - *) - Event_type.parse_error_log msg - in - event ) - -let rec filtered_log_entries_poll node ~logger ~event_writer - ~last_log_index_seen = - let open Deferred.Let_syntax in - if not (Pipe.is_closed event_writer) then ( - let%bind () = after (Time.Span.of_ms 10000.0) in - match%bind - Integration_test_lib.Graphql_requests.get_filtered_log_entries - (Node.get_ingress_uri node) - ~last_log_index_seen - with - | Ok log_entries -> - Array.iter log_entries ~f:(fun log_entry -> - match parse_event_from_log_entry ~logger log_entry with - | Ok a -> - Pipe.write_without_pushback_if_open event_writer (node, a) - | Error e -> - [%log warn] "Error parsing log $error" - ~metadata:[ ("error", `String (Error.to_string_hum e)) ] ) ; - let last_log_index_seen = - Array.length log_entries + last_log_index_seen - in - filtered_log_entries_poll node ~logger ~event_writer - ~last_log_index_seen - | Error err -> - [%log error] "Encountered an error while polling $node for logs: $err" - ~metadata: - [ ("node", `String (Node.infra_id node)) - ; ("err", Error_json.error_to_yojson err) - ] ; - (* Declare the node to be offline. *) - Pipe.write_without_pushback_if_open event_writer - (node, Event (Node_offline, ())) ; - (* Don't keep looping, the node may be restarting. *) - return (Ok ()) ) - else Deferred.Or_error.error_string "Event writer closed" - -let rec start_filtered_log node ~logger ~log_filter ~event_writer = - let open Deferred.Let_syntax in - if not (Pipe.is_closed event_writer) then - match%bind - Integration_test_lib.Graphql_requests.start_filtered_log ~logger - ~log_filter - (Node.get_ingress_uri node) - with - | Ok () -> - return (Ok ()) - | Error _ -> - start_filtered_log node ~logger ~log_filter ~event_writer - else Deferred.Or_error.error_string "Event writer closed" - -let rec poll_node_for_logs_in_background ~log_filter ~logger ~event_writer - (node : Node.t) = - let open Deferred.Or_error.Let_syntax in - [%log info] "Requesting for $node to start its filtered logs" - ~metadata:[ ("node", `String (Node.infra_id node)) ] ; - let%bind () = start_filtered_log ~logger ~log_filter ~event_writer node in - [%log info] "$node has started its filtered logs. Beginning polling" - ~metadata:[ ("node", `String (Node.infra_id node)) ] ; - let%bind () = - filtered_log_entries_poll node ~last_log_index_seen:0 ~logger ~event_writer - in - poll_node_for_logs_in_background ~log_filter ~logger ~event_writer node - -let poll_for_logs_in_background ~log_filter ~logger ~network ~event_writer = - Kubernetes_network.all_nodes network - |> Core.String.Map.data - |> Deferred.Or_error.List.iter ~how:`Parallel - ~f:(poll_node_for_logs_in_background ~log_filter ~logger ~event_writer) - -let create ~logger ~(network : Kubernetes_network.t) = - let open Deferred.Or_error.Let_syntax in - let log_filter = all_event_types_log_filter in - let event_reader, event_writer = Pipe.create () in - let background_job = - poll_for_logs_in_background ~log_filter ~logger ~network ~event_writer - in - return { logger; event_reader; event_writer; background_job } - -let destroy t : unit Deferred.Or_error.t = - let open Deferred.Or_error.Let_syntax in - let { logger; event_reader = _; event_writer; background_job = _ } = t in - Pipe.close event_writer ; - [%log debug] "graphql polling log engine destroyed" ; - return () diff --git a/src/lib/integration_test_cloud_engine/graphql_polling_log_engine.mli b/src/lib/integration_test_cloud_engine/graphql_polling_log_engine.mli deleted file mode 100644 index 1dbef606c1a..00000000000 --- a/src/lib/integration_test_cloud_engine/graphql_polling_log_engine.mli +++ /dev/null @@ -1,3 +0,0 @@ -include - Integration_test_lib.Intf.Engine.Log_engine_intf - with module Network := Kubernetes_network diff --git a/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml b/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml index 64f71860422..0fce21ade6c 100644 --- a/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml +++ b/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml @@ -3,4 +3,8 @@ let name = "cloud" module Network = Kubernetes_network module Network_config = Mina_automation.Network_config module Network_manager = Mina_automation.Network_manager -module Log_engine = Graphql_polling_log_engine + +module Log_engine = + Integration_test_lib.Graphql_polling_log_engine + .Make_GraphQL_Polling_log_engine + (Kubernetes_network) diff --git a/src/lib/integration_test_lib/dune b/src/lib/integration_test_lib/dune index 54381e3d9ad..2b50d04a6c9 100644 --- a/src/lib/integration_test_lib/dune +++ b/src/lib/integration_test_lib/dune @@ -61,4 +61,5 @@ transition_handler snark_worker one_or_two + error_json )) diff --git a/src/lib/integration_test_lib/graphql_polling_log_engine.ml b/src/lib/integration_test_lib/graphql_polling_log_engine.ml new file mode 100644 index 00000000000..ffc917fb954 --- /dev/null +++ b/src/lib/integration_test_lib/graphql_polling_log_engine.ml @@ -0,0 +1,144 @@ +open Async +open Core +module Timeout = Timeout_lib.Core_time + +(** This implements Log_engine_intf for integration tests, by creating a simple system that polls a mina daemon's graphql endpoint for fetching logs*) + +module Make_GraphQL_Polling_log_engine (Network : Intf.Engine.Network_intf) = +struct + module Node = Network.Node + + let log_filter_of_event_type ev_existential = + let open Event_type in + let (Event_type ev_type) = ev_existential in + let (module Ty) = event_type_module ev_type in + match Ty.parse with + | From_error_log _ -> + [] (* TODO: Do we need this? *) + | From_daemon_log (struct_id, _) -> + [ Structured_log_events.string_of_id struct_id ] + | From_puppeteer_log _ -> + [] + (* TODO: Do we need this? *) + + let all_event_types_log_filter = + List.bind ~f:log_filter_of_event_type Event_type.all_event_types + + type t = + { logger : Logger.t + ; event_writer : (Node.t * Event_type.event) Pipe.Writer.t + ; event_reader : (Node.t * Event_type.event) Pipe.Reader.t + ; background_job : unit Deferred.Or_error.t + } + + let event_reader { event_reader; _ } = event_reader + + let parse_event_from_log_entry ~logger log_entry = + let open Or_error.Let_syntax in + let open Json_parsing in + Or_error.try_with_join (fun () -> + let payload = Yojson.Safe.from_string log_entry in + let%map event = + let%bind msg = + parse (parser_from_of_yojson Logger.Message.of_yojson) payload + in + let event_id = + Option.map ~f:Structured_log_events.string_of_id msg.event_id + in + [%log spam] "parsing daemon structured event, event_id = $event_id" + ~metadata:[ ("event_id", [%to_yojson: string option] event_id) ] ; + match msg.event_id with + | Some _ -> + Event_type.parse_daemon_event msg + | None -> + (* Currently unreachable, but we could include error logs here if + desired. + *) + Event_type.parse_error_log msg + in + event ) + + let rec filtered_log_entries_poll node ~logger ~event_writer + ~last_log_index_seen = + let open Deferred.Let_syntax in + if not (Pipe.is_closed event_writer) then ( + let%bind () = after (Time.Span.of_ms 10000.0) in + match%bind + Graphql_requests.get_filtered_log_entries + (Node.get_ingress_uri node) + ~last_log_index_seen + with + | Ok log_entries -> + Array.iter log_entries ~f:(fun log_entry -> + match parse_event_from_log_entry ~logger log_entry with + | Ok a -> + Pipe.write_without_pushback_if_open event_writer (node, a) + | Error e -> + [%log warn] "Error parsing log $error" + ~metadata:[ ("error", `String (Error.to_string_hum e)) ] ) ; + let last_log_index_seen = + Array.length log_entries + last_log_index_seen + in + filtered_log_entries_poll node ~logger ~event_writer + ~last_log_index_seen + | Error err -> + [%log error] "Encountered an error while polling $node for logs: $err" + ~metadata: + [ ("node", `String (Node.infra_id node)) + ; ("err", Error_json.error_to_yojson err) + ] ; + (* Declare the node to be offline. *) + Pipe.write_without_pushback_if_open event_writer + (node, Event (Node_offline, ())) ; + (* Don't keep looping, the node may be restarting. *) + return (Ok ()) ) + else Deferred.Or_error.error_string "Event writer closed" + + let rec start_filtered_log node ~logger ~log_filter ~event_writer = + let open Deferred.Let_syntax in + if not (Pipe.is_closed event_writer) then + match%bind + Graphql_requests.start_filtered_log ~logger ~log_filter + (Node.get_ingress_uri node) + with + | Ok () -> + return (Ok ()) + | Error _ -> + start_filtered_log node ~logger ~log_filter ~event_writer + else Deferred.Or_error.error_string "Event writer closed" + + let rec poll_node_for_logs_in_background ~log_filter ~logger ~event_writer + (node : Node.t) = + let open Deferred.Or_error.Let_syntax in + [%log info] "Requesting for $node to start its filtered logs" + ~metadata:[ ("node", `String (Node.infra_id node)) ] ; + let%bind () = start_filtered_log ~logger ~log_filter ~event_writer node in + [%log info] "$node has started its filtered logs. Beginning polling" + ~metadata:[ ("node", `String (Node.infra_id node)) ] ; + let%bind () = + filtered_log_entries_poll node ~last_log_index_seen:0 ~logger + ~event_writer + in + poll_node_for_logs_in_background ~log_filter ~logger ~event_writer node + + let poll_for_logs_in_background ~log_filter ~logger ~network ~event_writer = + Network.all_nodes network |> Core.String.Map.data + |> Deferred.Or_error.List.iter ~how:`Parallel + ~f:(poll_node_for_logs_in_background ~log_filter ~logger ~event_writer) + + let create ~logger ~(network : Network.t) = + let open Deferred.Or_error.Let_syntax in + let log_filter = all_event_types_log_filter in + let event_reader, event_writer = Pipe.create () in + let background_job = + poll_for_logs_in_background ~log_filter ~logger ~network ~event_writer + in + return { logger; event_reader; event_writer; background_job } + + let destroy t : unit Deferred.Or_error.t = + let open Deferred.Or_error.Let_syntax in + let { logger; event_reader = _; event_writer; background_job = _ } = t in + Pipe.close event_writer ; + [%log debug] "graphql polling log engine destroyed" ; + return () +end diff --git a/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml b/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml deleted file mode 100644 index 1311a14c029..00000000000 --- a/src/lib/integration_test_local_engine/docker_pipe_log_engine.ml +++ /dev/null @@ -1,147 +0,0 @@ -open Async -open Core -open Integration_test_lib -module Timeout = Timeout_lib.Core_time -module Node = Docker_network.Node - -(** This implements Log_engine_intf for integration tests, by creating a simple system that polls a mina daemon's graphql endpoint for fetching logs*) - -let log_filter_of_event_type ev_existential = - let open Event_type in - let (Event_type ev_type) = ev_existential in - let (module Ty) = event_type_module ev_type in - match Ty.parse with - | From_error_log _ -> - [] (* TODO: Do we need this? *) - | From_daemon_log (struct_id, _) -> - [ Structured_log_events.string_of_id struct_id ] - | From_puppeteer_log _ -> - [] -(* TODO: Do we need this? *) - -let all_event_types_log_filter = - List.bind ~f:log_filter_of_event_type Event_type.all_event_types - -type t = - { logger : Logger.t - ; event_writer : (Node.t * Event_type.event) Pipe.Writer.t - ; event_reader : (Node.t * Event_type.event) Pipe.Reader.t - ; background_job : unit Deferred.Or_error.t - } - -let event_reader { event_reader; _ } = event_reader - -let parse_event_from_log_entry ~logger log_entry = - let open Or_error.Let_syntax in - let open Json_parsing in - Or_error.try_with_join (fun () -> - let payload = Yojson.Safe.from_string log_entry in - let%map event = - let%bind msg = - parse (parser_from_of_yojson Logger.Message.of_yojson) payload - in - let event_id = - Option.map ~f:Structured_log_events.string_of_id msg.event_id - in - [%log spam] "parsing daemon structured event, event_id = $event_id" - ~metadata: - [ ("event_id", [%to_yojson: string option] event_id) - ; ("msg", Logger.Message.to_yojson msg) - ; ("metadata", Logger.Metadata.to_yojson msg.metadata) - ; ("payload", payload) - ] ; - match msg.event_id with - | Some _ -> - Event_type.parse_daemon_event msg - | None -> - (* Currently unreachable, but we could include error logs here if - desired. - *) - Event_type.parse_error_log msg - in - event ) - -let rec filtered_log_entries_poll node ~logger ~event_writer - ~last_log_index_seen = - let open Deferred.Let_syntax in - if not (Pipe.is_closed event_writer) then ( - let%bind () = after (Time.Span.of_ms 10000.0) in - match%bind - Integration_test_lib.Graphql_requests.get_filtered_log_entries - (Node.get_ingress_uri node) - ~last_log_index_seen - with - | Ok log_entries -> - Array.iter log_entries ~f:(fun log_entry -> - match parse_event_from_log_entry ~logger log_entry with - | Ok a -> - Pipe.write_without_pushback_if_open event_writer (node, a) - | Error e -> - [%log warn] "Error parsing log $error" - ~metadata:[ ("error", `String (Error.to_string_hum e)) ] ) ; - let last_log_index_seen = - Array.length log_entries + last_log_index_seen - in - filtered_log_entries_poll node ~logger ~event_writer - ~last_log_index_seen - | Error err -> - [%log error] "Encountered an error while polling $node for logs: $err" - ~metadata: - [ ("node", `String (Node.infra_id node)) - ; ("err", Error_json.error_to_yojson err) - ] ; - (* Declare the node to be offline. *) - Pipe.write_without_pushback_if_open event_writer - (node, Event (Node_offline, ())) ; - (* Don't keep looping, the node may be restarting. *) - return (Ok ()) ) - else Deferred.Or_error.error_string "Event writer closed" - -let rec start_filtered_log node ~logger ~log_filter ~event_writer = - let open Deferred.Let_syntax in - if not (Pipe.is_closed event_writer) then - match%bind - Integration_test_lib.Graphql_requests.start_filtered_log ~logger - ~log_filter - (Node.get_ingress_uri node) - with - | Ok () -> - return (Ok ()) - | Error _ -> - start_filtered_log node ~logger ~log_filter ~event_writer - else Deferred.Or_error.error_string "Event writer closed" - -let rec poll_node_for_logs_in_background ~log_filter ~logger ~event_writer - (node : Node.t) = - let open Deferred.Or_error.Let_syntax in - [%log info] "Requesting for $node to start its filtered logs" - ~metadata:[ ("node", `String (Node.infra_id node)) ] ; - let%bind () = start_filtered_log ~logger ~log_filter ~event_writer node in - [%log info] "$node has started its filtered logs. Beginning polling" - ~metadata:[ ("node", `String (Node.infra_id node)) ] ; - let%bind () = - filtered_log_entries_poll node ~last_log_index_seen:0 ~logger ~event_writer - in - poll_node_for_logs_in_background ~log_filter ~logger ~event_writer node - -let poll_for_logs_in_background ~log_filter ~logger ~network ~event_writer = - Docker_network.all_nodes network - |> Core.String.Map.data - |> Deferred.Or_error.List.iter ~how:`Parallel - ~f:(poll_node_for_logs_in_background ~log_filter ~logger ~event_writer) - -let create ~logger ~(network : Docker_network.t) = - let open Deferred.Or_error.Let_syntax in - let log_filter = all_event_types_log_filter in - let event_reader, event_writer = Pipe.create () in - let background_job = - poll_for_logs_in_background ~log_filter ~logger ~network ~event_writer - in - return { logger; event_reader; event_writer; background_job } - -let destroy t : unit Deferred.Or_error.t = - let open Deferred.Or_error.Let_syntax in - let { logger; event_reader = _; event_writer; background_job = _ } = t in - Pipe.close event_writer ; - [%log debug] "graphql polling log engine destroyed" ; - return () diff --git a/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli b/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli deleted file mode 100644 index 8597d5e7efe..00000000000 --- a/src/lib/integration_test_local_engine/docker_pipe_log_engine.mli +++ /dev/null @@ -1,3 +0,0 @@ -include - Integration_test_lib.Intf.Engine.Log_engine_intf - with module Network := Docker_network diff --git a/src/lib/integration_test_local_engine/integration_test_local_engine.ml b/src/lib/integration_test_local_engine/integration_test_local_engine.ml index 8869ee80994..31d165d8262 100644 --- a/src/lib/integration_test_local_engine/integration_test_local_engine.ml +++ b/src/lib/integration_test_local_engine/integration_test_local_engine.ml @@ -3,4 +3,8 @@ let name = "local" module Network = Docker_network module Network_config = Mina_docker.Network_config module Network_manager = Mina_docker.Network_manager -module Log_engine = Docker_pipe_log_engine + +module Log_engine = + Integration_test_lib.Graphql_polling_log_engine + .Make_GraphQL_Polling_log_engine + (Docker_network) From d56b4a5713b20707c4362d54aa032f6a5775babb Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Fri, 22 Dec 2023 21:31:37 -0800 Subject: [PATCH 18/25] add a polling interval to graphql log functor Changed the Make_GraphQL_Polling_log_engine to accept a polling interval parameter. This is important because inside the start_filtered_log function, we want to call it as soon as the graphql API is available for a node in the docker network. If we wait longer, we have the potential to miss the `Node_initialization` event which will lead to a test failure. Additionally exposed the retry_delay_sec value as a function parameter to start_filtered_log. --- .../integration_test_cloud_engine.ml | 5 +++++ src/lib/integration_test_lib/graphql_polling_log_engine.ml | 6 +++++- src/lib/integration_test_lib/graphql_requests.ml | 6 +++--- .../integration_test_local_engine.ml | 5 +++++ 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml b/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml index 0fce21ade6c..533b9de5862 100644 --- a/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml +++ b/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml @@ -4,7 +4,12 @@ module Network = Kubernetes_network module Network_config = Mina_automation.Network_config module Network_manager = Mina_automation.Network_manager +module Kubernetes_polling_interval = struct + let interval = Core.Time.Span.of_sec 10.0 +end + module Log_engine = Integration_test_lib.Graphql_polling_log_engine .Make_GraphQL_Polling_log_engine (Kubernetes_network) + (Kubernetes_polling_interval) diff --git a/src/lib/integration_test_lib/graphql_polling_log_engine.ml b/src/lib/integration_test_lib/graphql_polling_log_engine.ml index ffc917fb954..c17c753b6a8 100644 --- a/src/lib/integration_test_lib/graphql_polling_log_engine.ml +++ b/src/lib/integration_test_lib/graphql_polling_log_engine.ml @@ -4,7 +4,10 @@ module Timeout = Timeout_lib.Core_time (** This implements Log_engine_intf for integration tests, by creating a simple system that polls a mina daemon's graphql endpoint for fetching logs*) -module Make_GraphQL_Polling_log_engine (Network : Intf.Engine.Network_intf) = +module Make_GraphQL_Polling_log_engine + (Network : Intf.Engine.Network_intf) (Polling_interval : sig + val interval : Time.Span.t + end) = struct module Node = Network.Node @@ -99,6 +102,7 @@ struct if not (Pipe.is_closed event_writer) then match%bind Graphql_requests.start_filtered_log ~logger ~log_filter + ~retry_delay_sec:(Polling_interval.interval |> Time.Span.to_sec) (Node.get_ingress_uri node) with | Ok () -> diff --git a/src/lib/integration_test_lib/graphql_requests.ml b/src/lib/integration_test_lib/graphql_requests.ml index 94a0aa69218..8b1701ff867 100644 --- a/src/lib/integration_test_lib/graphql_requests.ml +++ b/src/lib/integration_test_lib/graphql_requests.ml @@ -1077,14 +1077,14 @@ let get_metrics ~logger node_uri = ; transaction_pool_size } -let start_filtered_log ~logger ~log_filter node_uri = +let start_filtered_log ~logger ~log_filter node_uri ~retry_delay_sec = let open Deferred.Let_syntax in let query_obj = Graphql.StartFilteredLog.(make @@ makeVariables ~filter:log_filter ()) in let%bind res = - exec_graphql_request ~logger:(Logger.null ()) ~retry_delay_sec:0.25 - ~num_tries:100 ~node_uri ~query_name:"StartFilteredLog" query_obj + exec_graphql_request ~logger:(Logger.null ()) ~retry_delay_sec ~node_uri + ~query_name:"StartFilteredLog" query_obj in match res with | Ok query_result_obj -> diff --git a/src/lib/integration_test_local_engine/integration_test_local_engine.ml b/src/lib/integration_test_local_engine/integration_test_local_engine.ml index 31d165d8262..84de86e9253 100644 --- a/src/lib/integration_test_local_engine/integration_test_local_engine.ml +++ b/src/lib/integration_test_local_engine/integration_test_local_engine.ml @@ -4,7 +4,12 @@ module Network = Docker_network module Network_config = Mina_docker.Network_config module Network_manager = Mina_docker.Network_manager +module Docker_polling_interval = struct + let interval = Core.Time.Span.of_sec 0.25 +end + module Log_engine = Integration_test_lib.Graphql_polling_log_engine .Make_GraphQL_Polling_log_engine (Docker_network) + (Docker_polling_interval) From 8cf3062574136ddf26c98bc2dafcad2d3aa91f2b Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Fri, 22 Dec 2023 22:02:33 -0800 Subject: [PATCH 19/25] rename Make_GraphQL_Polling_log_engine to Make_GraphQL_polling_log_engine --- .../integration_test_cloud_engine.ml | 2 +- src/lib/integration_test_lib/graphql_polling_log_engine.ml | 2 +- .../integration_test_local_engine.ml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml b/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml index 533b9de5862..77995460aad 100644 --- a/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml +++ b/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml @@ -10,6 +10,6 @@ end module Log_engine = Integration_test_lib.Graphql_polling_log_engine - .Make_GraphQL_Polling_log_engine + .Make_GraphQL_polling_log_engine (Kubernetes_network) (Kubernetes_polling_interval) diff --git a/src/lib/integration_test_lib/graphql_polling_log_engine.ml b/src/lib/integration_test_lib/graphql_polling_log_engine.ml index c17c753b6a8..8e042345110 100644 --- a/src/lib/integration_test_lib/graphql_polling_log_engine.ml +++ b/src/lib/integration_test_lib/graphql_polling_log_engine.ml @@ -4,7 +4,7 @@ module Timeout = Timeout_lib.Core_time (** This implements Log_engine_intf for integration tests, by creating a simple system that polls a mina daemon's graphql endpoint for fetching logs*) -module Make_GraphQL_Polling_log_engine +module Make_GraphQL_polling_log_engine (Network : Intf.Engine.Network_intf) (Polling_interval : sig val interval : Time.Span.t end) = diff --git a/src/lib/integration_test_local_engine/integration_test_local_engine.ml b/src/lib/integration_test_local_engine/integration_test_local_engine.ml index 84de86e9253..b8296426e07 100644 --- a/src/lib/integration_test_local_engine/integration_test_local_engine.ml +++ b/src/lib/integration_test_local_engine/integration_test_local_engine.ml @@ -10,6 +10,6 @@ end module Log_engine = Integration_test_lib.Graphql_polling_log_engine - .Make_GraphQL_Polling_log_engine + .Make_GraphQL_polling_log_engine (Docker_network) (Docker_polling_interval) From 4c2d4515a55db15f50d3f0fcf0cabf283fcf9c8b Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Fri, 22 Dec 2023 22:04:03 -0800 Subject: [PATCH 20/25] refactor mina nodes to use a base config when generating docker definitions --- .../integration_test_lib/graphql_requests.ml | 2 +- .../docker_compose.ml | 16 --- .../docker_network.ml | 2 +- .../docker_node_config.ml | 113 +++++++++--------- .../mina_docker.ml | 40 ++++--- 5 files changed, 84 insertions(+), 89 deletions(-) diff --git a/src/lib/integration_test_lib/graphql_requests.ml b/src/lib/integration_test_lib/graphql_requests.ml index 8b1701ff867..e64c03a9a8c 100644 --- a/src/lib/integration_test_lib/graphql_requests.ml +++ b/src/lib/integration_test_lib/graphql_requests.ml @@ -1077,7 +1077,7 @@ let get_metrics ~logger node_uri = ; transaction_pool_size } -let start_filtered_log ~logger ~log_filter node_uri ~retry_delay_sec = +let start_filtered_log node_uri ~logger ~log_filter ~retry_delay_sec = let open Deferred.Let_syntax in let query_obj = Graphql.StartFilteredLog.(make @@ makeVariables ~filter:log_filter ()) diff --git a/src/lib/integration_test_local_engine/docker_compose.ml b/src/lib/integration_test_local_engine/docker_compose.ml index e374b97f3ed..9faffd82c99 100644 --- a/src/lib/integration_test_local_engine/docker_compose.ml +++ b/src/lib/integration_test_local_engine/docker_compose.ml @@ -9,27 +9,11 @@ module Dockerfile = struct [@@deriving to_yojson] let create source target = { type_ = "bind"; source; target } - - let write_config docker_dir ~filename ~data = - Out_channel.with_file ~fail_if_exists:false - (docker_dir ^ "/" ^ filename) - ~f:(fun ch -> data |> Out_channel.output_string ch) ; - ignore (Util.run_cmd_exn docker_dir "chmod" [ "600"; filename ]) end module Environment = struct type t = (string * string) list - let default = - [ ("DAEMON_REST_PORT", "3085") - ; ("DAEMON_CLIENT_PORT", "8301") - ; ("DAEMON_METRICS_PORT", "10001") - ; ("DAEMON_EXTERNAL_PORT", "10101") - ; ("MINA_PRIVKEY_PASS", "naughty blue worm") - ; ("MINA_LIBP2P_PASS", "") - ; ("RAYON_NUM_THREADS", "6") - ] - let to_yojson env = `Assoc (List.map env ~f:(fun (k, v) -> (k, `String v))) end diff --git a/src/lib/integration_test_local_engine/docker_network.ml b/src/lib/integration_test_local_engine/docker_network.ml index e8a73532278..e7eeac652d4 100644 --- a/src/lib/integration_test_local_engine/docker_network.ml +++ b/src/lib/integration_test_local_engine/docker_network.ml @@ -253,7 +253,7 @@ module Service_to_deploy = struct let get_node_from_service t = let%bind cwd = Unix.getcwd () in let open Malleable_error.Let_syntax in - let service_id = t.stack_name ^ "_" ^ t.service_name in + let service_id = sprintf "%s_%s" t.stack_name t.service_name in let%bind container_id = get_container_id service_id in if String.is_empty container_id then Malleable_error.hard_error_format "No container id found for service %s" diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index aa9a89d8452..b1a2aef13e2 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -122,7 +122,9 @@ module Base_node_config = struct } let default ?(runtime_config_path = None) ?(peer = None) = - { log_snark_work_gossip = true + { runtime_config_path + ; peer + ; log_snark_work_gossip = true ; log_txn_pool_gossip = true ; generate_genesis_proof = true ; log_level = "Debug" @@ -130,12 +132,20 @@ module Base_node_config = struct ; rest_port = PortManager.mina_internal_rest_port |> Int.to_string ; metrics_port = PortManager.mina_internal_metrics_port |> Int.to_string ; external_port = PortManager.mina_internal_external_port |> Int.to_string - ; runtime_config_path ; libp2p_key_path = container_libp2p_key_path ; libp2p_secret = "" - ; peer } + let to_docker_env_vars t = + [ ("DAEMON_REST_PORT", t.rest_port) + ; ("DAEMON_CLIENT_PORT", t.client_port) + ; ("DAEMON_METRICS_PORT", t.metrics_port) + ; ("DAEMON_EXTERNAL_PORT", t.external_port) + ; ("RAYON_NUM_THREADS", "8") + ; ("MINA_PRIVKEY_PASS", "naughty blue worm") + ; ("MINA_LIBP2P_PASS", "") + ] + let to_list t = let base_args = [ "-log-level" @@ -181,9 +191,7 @@ module Block_producer_config = struct ; priv_key_path : string ; enable_flooding : bool ; enable_peer_exchange : bool - ; peer : string option - ; libp2p_secret : string - ; runtime_config_path : string + ; base_config : Base_node_config.t } [@@deriving to_yojson] @@ -195,12 +203,7 @@ module Block_producer_config = struct [@@deriving to_yojson] let create_cmd config = - let base_args = - Base_node_config.to_list - (Base_node_config.default - ~runtime_config_path:(Some config.runtime_config_path) - ~peer:config.peer ) - in + let base_args = Base_node_config.to_list config.base_config in let block_producer_args = [ "daemon" ; "-block-producer-key" @@ -225,9 +228,10 @@ module Block_producer_config = struct let create ~service_name ~image ~ports ~volumes ~config = let entrypoint = Some [ "/root/entrypoint.sh" ] in + let environment = Base_node_config.to_docker_env_vars config.base_config in let docker_config = - create_docker_config ~image ~ports ~volumes - ~environment:Dockerfile.Service.Environment.default ~entrypoint ~config + create_docker_config ~image ~ports ~volumes ~environment ~entrypoint + ~config in { service_name; config; docker_config } end @@ -242,10 +246,7 @@ module Seed_config = struct Printf.sprintf "/dns4/%s/tcp/%d/p2p/%s" peer_name external_port peer_id type config = - { archive_address : string option - ; peer : string option - ; runtime_config_path : string - } + { archive_address : string option; base_config : Base_node_config.t } [@@deriving to_yojson] type t = @@ -262,12 +263,7 @@ module Seed_config = struct } let create_cmd config = - let base_args = - Base_node_config.to_list - (Base_node_config.default - ~runtime_config_path:(Some config.runtime_config_path) - ~peer:config.peer ) - in + let base_args = Base_node_config.to_list config.base_config in let seed_args = match config.archive_address with | Some archive_address -> @@ -289,16 +285,21 @@ module Seed_config = struct let create ~service_name ~image ~ports ~volumes ~config = let entrypoint = Some [ "/root/entrypoint.sh" ] in + let environment = Base_node_config.to_docker_env_vars config.base_config in let docker_config = - create_docker_config ~image ~ports ~volumes - ~environment:Dockerfile.Service.Environment.default ~entrypoint ~config + create_docker_config ~image ~ports ~volumes ~environment ~entrypoint + ~config in { service_name; config; docker_config } end module Snark_worker_config = struct type config = - { daemon_address : string; daemon_port : string; proof_level : string } + { daemon_address : string + ; daemon_port : string + ; proof_level : string + ; base_config : Base_node_config.t + } [@@deriving to_yojson] type t = @@ -331,9 +332,10 @@ module Snark_worker_config = struct let create ~service_name ~image ~ports ~volumes ~config = let entrypoint = Some [ "/root/entrypoint.sh" ] in + let environment = Base_node_config.to_docker_env_vars config.base_config in let docker_config = - create_docker_config ~image ~ports ~volumes - ~environment:Dockerfile.Service.Environment.default ~entrypoint ~config + create_docker_config ~image ~ports ~volumes ~environment ~entrypoint + ~config in { service_name; config; docker_config } end @@ -344,8 +346,7 @@ module Snark_coordinator_config = struct ; snark_worker_fee : string ; work_selection : string ; worker_nodes : Snark_worker_config.t list - ; peer : string option - ; runtime_config_path : string + ; base_config : Base_node_config.t } [@@deriving to_yojson] @@ -365,12 +366,7 @@ module Snark_coordinator_config = struct ] let create_cmd config = - let base_args = - Base_node_config.to_list - (Base_node_config.default - ~runtime_config_path:(Some config.runtime_config_path) - ~peer:config.peer ) - in + let base_args = Base_node_config.to_list config.base_config in let snark_coordinator_args = [ "daemon" ; "-run-snark-coordinator" @@ -400,7 +396,7 @@ module Snark_coordinator_config = struct ~snark_coordinator_key:config.snark_coordinator_key ~snark_worker_fee:config.snark_worker_fee ~work_selection:config.work_selection - @ Dockerfile.Service.Environment.default + @ Base_node_config.to_docker_env_vars config.base_config in let docker_config = create_docker_config ~image ~ports ~volumes ~environment ~entrypoint @@ -475,8 +471,8 @@ psql -U postgres -d archive -f ./create_schema.sql ] let create_connection_uri ~host ~username ~password ~database ~port = - Printf.sprintf "postgres://%s:%s@%s:%s/%s" username password host - (Int.to_string port) database + Printf.sprintf "postgres://%s:%s@%s:%d/%s" username password host port + database let to_connection_uri t = create_connection_uri ~host:t.host ~port:t.port ~username:t.username @@ -492,14 +488,13 @@ psql -U postgres -d archive -f ./create_schema.sql } let create ~service_name ~image ~ports ~volumes ~config = - let entrypoint = None in let environment = postgres_default_envs ~username:config.username ~password:config.password ~database:config.database ~port:(Int.to_string config.port) in let docker_config = - create_docker_config ~image ~ports ~volumes ~environment ~entrypoint + create_docker_config ~image ~ports ~volumes ~environment ~entrypoint:None in { service_name; config; docker_config } end @@ -508,7 +503,7 @@ module Archive_node_config = struct type config = { postgres_config : Postgres_config.t ; server_port : int - ; runtime_config_path : string + ; base_config : Base_node_config.t } [@@deriving to_yojson] @@ -520,15 +515,23 @@ module Archive_node_config = struct [@@deriving to_yojson] let create_cmd config = - [ "mina-archive" - ; "run" - ; "-postgres-uri" - ; Postgres_config.to_connection_uri config.postgres_config.config - ; "-server-port" - ; Int.to_string config.server_port - ; "-config-file" - ; config.runtime_config_path - ] + let base_args = + [ "mina-archive" + ; "run" + ; "-postgres-uri" + ; Postgres_config.to_connection_uri config.postgres_config.config + ; "-server-port" + ; Int.to_string config.server_port + ] + in + let runtime_config_path = + match config.base_config.runtime_config_path with + | Some path -> + [ "-config-file"; path ] + | None -> + [] + in + List.concat [ base_args; runtime_config_path ] let create_docker_config ~image ~entrypoint ~ports ~volumes ~environment ~config = @@ -541,10 +544,10 @@ module Archive_node_config = struct } let create ~service_name ~image ~ports ~volumes ~config = - let entrypoint = None in + let environment = Base_node_config.to_docker_env_vars config.base_config in let docker_config = - create_docker_config ~image ~ports ~volumes - ~environment:Dockerfile.Service.Environment.default ~entrypoint ~config + create_docker_config ~image ~ports ~volumes ~environment ~entrypoint:None + ~config in { service_name; config; docker_config } end diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index a1e090a2c7d..220f517ea71 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -290,8 +290,10 @@ module Network_config = struct let seed_config = let config : Seed_config.config = { archive_address = None - ; peer = None - ; runtime_config_path = Base_node_config.container_runtime_config_path + ; base_config = + Base_node_config.default ~peer:None + ~runtime_config_path: + (Some Base_node_config.container_runtime_config_path) } in Seed_config.create @@ -340,8 +342,10 @@ module Network_config = struct let config : Archive_node_config.config = { postgres_config ; server_port = archive_server_port.target - ; runtime_config_path = - Base_node_config.container_runtime_config_path + ; base_config = + Base_node_config.default ~peer:None + ~runtime_config_path: + (Some Base_node_config.container_runtime_config_path) } in let archive_rest_port = @@ -364,9 +368,10 @@ module Network_config = struct Some (sprintf "%s:%d" archive_config.service_name PortManager.mina_internal_server_port ) - ; peer = seed_config_peer - ; runtime_config_path = - Base_node_config.container_runtime_config_path + ; base_config = + Base_node_config.default ~peer:seed_config_peer + ~runtime_config_path: + (Some Base_node_config.container_runtime_config_path) } in Seed_config.create @@ -407,13 +412,13 @@ module Network_config = struct in let block_producer_config : Block_producer_config.config = { keypair - ; runtime_config_path = - Base_node_config.container_runtime_config_path - ; peer = seed_config_peer ; priv_key_path ; enable_peer_exchange = true ; enable_flooding = true - ; libp2p_secret = "" + ; base_config = + Base_node_config.default ~peer:seed_config_peer + ~runtime_config_path: + (Some Base_node_config.container_runtime_config_path) } in Block_producer_config.create ~service_name:node.node_name @@ -462,6 +467,8 @@ module Network_config = struct { daemon_address = snark_node_service_name ; daemon_port = Int.to_string daemon_port.target ; proof_level = "full" + ; base_config = + Base_node_config.default ~peer:None ~runtime_config_path:None } in let worker_nodes = @@ -479,11 +486,12 @@ module Network_config = struct let snark_coordinator_config : Snark_coordinator_config.config = { worker_nodes ; snark_worker_fee - ; runtime_config_path = - Base_node_config.container_runtime_config_path ; snark_coordinator_key = public_key - ; peer = seed_config_peer ; work_selection = "seq" + ; base_config = + Base_node_config.default ~peer:seed_config_peer + ~runtime_config_path: + (Some Base_node_config.container_runtime_config_path) } in Some @@ -504,12 +512,12 @@ module Network_config = struct ; mina_points_image = images.points ; mina_archive_image = images.archive_node ; runtime_config = Runtime_config.to_yojson runtime_config - ; log_precomputed_blocks (* Node configs *) + ; log_precomputed_blocks ; block_producer_configs ; seed_configs ; mina_archive_schema_aux_files ; snark_coordinator_config - ; archive_node_configs (* Resource configs *) + ; archive_node_configs } } From 5a6716ed18c4c46f0eb5af650b504e369964002e Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Wed, 27 Dec 2023 12:56:31 -0800 Subject: [PATCH 21/25] Add archive node entrypoint --- .../docker_node_config.ml | 19 ++++++++++++++++++- .../mina_docker.ml | 16 +++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index b1a2aef13e2..bbdd0cb8773 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -514,6 +514,22 @@ module Archive_node_config = struct } [@@deriving to_yojson] + let archive_entrypoint_script = + ( "archive_entrypoint.sh" + , {|#!/bin/bash + # This file is auto-generated by the local integration test framework. + # Sleep for 15 seconds + echo "Sleeping for 15 seconds before starting..." + sleep 15 + exec "$@"|} + ) + + let archive_entrypoint_volume : Docker_compose.Dockerfile.Service.Volume.t = + { type_ = "bind" + ; source = "archive_entrypoint.sh" + ; target = Base_node_config.container_entrypoint_path + } + let create_cmd config = let base_args = [ "mina-archive" @@ -544,9 +560,10 @@ module Archive_node_config = struct } let create ~service_name ~image ~ports ~volumes ~config = + let entrypoint = Some [ "/root/entrypoint.sh" ] in let environment = Base_node_config.to_docker_env_vars config.base_config in let docker_config = - create_docker_config ~image ~ports ~volumes ~environment ~entrypoint:None + create_docker_config ~image ~ports ~volumes ~environment ~entrypoint ~config in { service_name; config; docker_config } diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index 220f517ea71..58af851999d 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -358,7 +358,11 @@ module Network_config = struct (sprintf "archive-%d-%s" (index + 1) (generate_random_id ())) ~image:images.archive_node ~ports:[ archive_server_port; archive_rest_port ] - ~volumes:docker_volumes ~config ) + ~volumes: + [ Base_node_config.runtime_config_volume + ; Archive_node_config.archive_entrypoint_volume + ] + ~config ) in (* Each archive node has it's own seed node *) let seed_configs = @@ -693,6 +697,15 @@ module Network_manager = struct Out_channel.with_file ~fail_if_exists:true (docker_dir ^/ entrypoint_filename) ~f:(fun ch -> entrypoint_script |> Out_channel.output_string ch ) ; + [%log info] + "Writing custom archive entrypoint script (wait for postgres to \ + initialize)" ; + let archive_filename, archive_script = + Docker_node_config.Archive_node_config.archive_entrypoint_script + in + Out_channel.with_file ~fail_if_exists:true (docker_dir ^/ archive_filename) + ~f:(fun ch -> archive_script |> Out_channel.output_string ch) ; + ignore (Util.run_cmd_exn docker_dir "chmod" [ "+x"; archive_filename ]) ; let%bind _ = Deferred.List.iter network_config.docker.mina_archive_schema_aux_files ~f:(fun schema_url -> @@ -710,6 +723,7 @@ module Network_manager = struct |> Deferred.return in ignore (Util.run_cmd_exn docker_dir "chmod" [ "+x"; entrypoint_filename ]) ; + [%log info] "Writing custom postgres entrypoint script (create schema)" ; let postgres_entrypoint_filename, postgres_entrypoint_script = Docker_node_config.Postgres_config.postgres_script in From 126f64df94b17a43fe327532b476474f199eaf70 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 2 Jan 2024 09:48:10 -0800 Subject: [PATCH 22/25] Revert "added local/buildkite modes to rebuild-deb.sh" This reverts commit b0ca83b8885cd97aebdeffc72fa63223afe1f2d1. --- scripts/rebuild-deb.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index 1dbe1d0f954..7b71aadf929 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -12,14 +12,12 @@ cd "${SCRIPTPATH}/../_build" GITHASH=$(git rev-parse --short=7 HEAD) GITHASH_CONFIG=$(git rev-parse --short=8 --verify HEAD) -mode="${1-buildkite}" - # Load in env vars for githash/branch/etc. -case $mode in - local) source "${SCRIPTPATH}/export-local-git-env-vars.sh";; - buildkite)"${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh";; - *) echo "Unknown mode passed: $mode"; exit 1;; -esac +if [[ "$1" == "local" ]]; then + source "${SCRIPTPATH}/export-local-git-env-vars.sh" +else + source "${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh" +fi MINA_DEB_CODENAME=stretch From 9d40215d5d4726b83d3e7ec7eda058fa8856fe39 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 2 Jan 2024 09:49:02 -0800 Subject: [PATCH 23/25] Revert "add local version for rebuild-deb script" This reverts commit 8613893c594e8ebcca66b008a533554bd682977c. --- scripts/export-local-git-env-vars.sh | 22 ---------------------- scripts/rebuild-deb.sh | 13 ++++++------- 2 files changed, 6 insertions(+), 29 deletions(-) delete mode 100755 scripts/export-local-git-env-vars.sh diff --git a/scripts/export-local-git-env-vars.sh b/scripts/export-local-git-env-vars.sh deleted file mode 100755 index 58c671b1fa9..00000000000 --- a/scripts/export-local-git-env-vars.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -eo pipefail - -set +x -echo "Exporting Variables: " - -export GITHASH=$(git rev-parse --short=7 HEAD) -export GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!g; s!_!-!g' ) -# GITTAG is the closest tagged commit to this commit, while THIS_COMMIT_TAG only has a value when the current commit is tagged -export GITTAG=$(git describe --always --abbrev=0 | sed 's!/!-!g; s!_!-!g') -export PROJECT="mina" - -set +u -export BUILD_NUM=1 -export BUILD_URL="local" -set -u - -export MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=stretch} -export MINA_DEB_VERSION="${GITTAG}-${GITBRANCH}-${GITHASH}" -export MINA_DEB_RELEASE="unstable" -export MINA_DOCKER_TAG="$(echo "${MINA_DEB_VERSION}" | sed 's!/!-!g; s!_!-!g')-${MINA_DEB_CODENAME}" - diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index 7b71aadf929..770e1dcbf60 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -12,14 +12,13 @@ cd "${SCRIPTPATH}/../_build" GITHASH=$(git rev-parse --short=7 HEAD) GITHASH_CONFIG=$(git rev-parse --short=8 --verify HEAD) -# Load in env vars for githash/branch/etc. -if [[ "$1" == "local" ]]; then - source "${SCRIPTPATH}/export-local-git-env-vars.sh" -else - source "${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh" -fi +set +u +BUILD_NUM=${BUILDKITE_BUILD_NUM} +BUILD_URL=${BUILDKITE_BUILD_URL} +set -u -MINA_DEB_CODENAME=stretch +# Load in env vars for githash/branch/etc. +source "${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh" cd "${SCRIPTPATH}/../_build" From f7af3ce42f2bc8663213e768ef579a9cebe63472 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Tue, 2 Jan 2024 10:22:36 -0800 Subject: [PATCH 24/25] refactor: rename 'interval' to 'start_filtered_logs_interval' in integration_test_cloud_engine.ml, graphql_polling_log_engine.ml, and integration_test_local_engine.ml for better clarity and understanding of its purpose --- .../integration_test_cloud_engine.ml | 2 +- src/lib/integration_test_lib/graphql_polling_log_engine.ml | 5 +++-- .../integration_test_local_engine.ml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml b/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml index 77995460aad..df72e1e1812 100644 --- a/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml +++ b/src/lib/integration_test_cloud_engine/integration_test_cloud_engine.ml @@ -5,7 +5,7 @@ module Network_config = Mina_automation.Network_config module Network_manager = Mina_automation.Network_manager module Kubernetes_polling_interval = struct - let interval = Core.Time.Span.of_sec 10.0 + let start_filtered_logs_interval = Core.Time.Span.of_sec 10.0 end module Log_engine = diff --git a/src/lib/integration_test_lib/graphql_polling_log_engine.ml b/src/lib/integration_test_lib/graphql_polling_log_engine.ml index 8e042345110..e344df952f9 100644 --- a/src/lib/integration_test_lib/graphql_polling_log_engine.ml +++ b/src/lib/integration_test_lib/graphql_polling_log_engine.ml @@ -6,7 +6,7 @@ module Timeout = Timeout_lib.Core_time module Make_GraphQL_polling_log_engine (Network : Intf.Engine.Network_intf) (Polling_interval : sig - val interval : Time.Span.t + val start_filtered_logs_interval : Time.Span.t end) = struct module Node = Network.Node @@ -102,7 +102,8 @@ struct if not (Pipe.is_closed event_writer) then match%bind Graphql_requests.start_filtered_log ~logger ~log_filter - ~retry_delay_sec:(Polling_interval.interval |> Time.Span.to_sec) + ~retry_delay_sec: + (Polling_interval.start_filtered_logs_interval |> Time.Span.to_sec) (Node.get_ingress_uri node) with | Ok () -> diff --git a/src/lib/integration_test_local_engine/integration_test_local_engine.ml b/src/lib/integration_test_local_engine/integration_test_local_engine.ml index b8296426e07..dc672814be8 100644 --- a/src/lib/integration_test_local_engine/integration_test_local_engine.ml +++ b/src/lib/integration_test_local_engine/integration_test_local_engine.ml @@ -5,7 +5,7 @@ module Network_config = Mina_docker.Network_config module Network_manager = Mina_docker.Network_manager module Docker_polling_interval = struct - let interval = Core.Time.Span.of_sec 0.25 + let start_filtered_logs_interval = Core.Time.Span.of_sec 0.25 end module Log_engine = From e7bb1e34a150ed5c2566df6313420e70136bd93a Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Wed, 3 Jan 2024 11:46:18 -0800 Subject: [PATCH 25/25] refactor(docker_network.ml): remove unused warning directive --- src/lib/integration_test_local_engine/docker_network.ml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/integration_test_local_engine/docker_network.ml b/src/lib/integration_test_local_engine/docker_network.ml index e7eeac652d4..4deb1de6234 100644 --- a/src/lib/integration_test_local_engine/docker_network.ml +++ b/src/lib/integration_test_local_engine/docker_network.ml @@ -2,8 +2,6 @@ open Core_kernel open Async open Integration_test_lib -[@@@warning "-27"] - let get_container_id service_id = let%bind cwd = Unix.getcwd () in let open Malleable_error.Let_syntax in @@ -251,7 +249,6 @@ module Service_to_deploy = struct { network_keypair; postgres_connection_uri; graphql_port } let get_node_from_service t = - let%bind cwd = Unix.getcwd () in let open Malleable_error.Let_syntax in let service_id = sprintf "%s_%s" t.stack_name t.service_name in let%bind container_id = get_container_id service_id in @@ -327,4 +324,7 @@ let all_ids t = List.fold deployments ~init:[] ~f:(fun acc (_, node) -> List.cons node.config.service_id acc ) -let initialize_infra ~logger network = Malleable_error.return () +let initialize_infra ~logger network = + let _ = logger in + let _ = network in + Malleable_error.return ()