From bfe4719f0507bca2246c378b34bafa37c679c485 Mon Sep 17 00:00:00 2001 From: Martin Minkov Date: Fri, 22 Dec 2023 13:25:59 -0800 Subject: [PATCH] centralize globals --- src/lib/integration_test_lib/network_state.ml | 8 +- .../integration_test_lib/wait_condition.ml | 6 - .../docker_compose.ml | 10 +- .../docker_network.ml | 112 +++---- .../docker_node_config.ml | 178 ++++++---- .../mina_docker.ml | 306 ++++++++---------- .../structured_log_events.ml | 1 - 7 files changed, 308 insertions(+), 313 deletions(-) diff --git a/src/lib/integration_test_lib/network_state.ml b/src/lib/integration_test_lib/network_state.ml index 1bfb7e16572c..1e12001e7515 100644 --- a/src/lib/integration_test_lib/network_state.ml +++ b/src/lib/integration_test_lib/network_state.ml @@ -187,13 +187,7 @@ module Make update ~f:(fun state -> [%log debug] "Updating network state with initialization event of $node" - ~metadata: - [ ("node", `String (Node.infra_id node)) - ; ( "event" - , Event_type.event_to_yojson - (Event_type.Event (Event_type.Node_initialization, ())) - ) - ] ; + ~metadata:[ ("node", `String (Node.infra_id node)) ] ; let node_initialization' = String.Map.set state.node_initialization ~key:(Node.id node) ~data:true diff --git a/src/lib/integration_test_lib/wait_condition.ml b/src/lib/integration_test_lib/wait_condition.ml index 95572e9990ce..492f09f0d1fe 100644 --- a/src/lib/integration_test_lib/wait_condition.ml +++ b/src/lib/integration_test_lib/wait_condition.ml @@ -75,12 +75,6 @@ struct ( nodes |> List.map ~f:Node.id |> String.concat ~sep:", " |> sprintf "[%s] to initialize" ) ~f:(fun (state : Network_state.t) -> - List.iter nodes ~f:(fun node -> - print_endline (sprintf "DEBUG: node: %s" (Node.id node)) ) ; - String.Map.iteri state.node_initialization ~f:(fun ~key ~data -> - print_endline - (sprintf "DEBUG: node_initialization: %s -> %b" key data) ) ; - print_endline "------------------------------------------" ; List.for_all nodes ~f:(fun node -> String.Map.find state.node_initialization (Node.id node) |> Option.value ~default:false ) ) diff --git a/src/lib/integration_test_local_engine/docker_compose.ml b/src/lib/integration_test_local_engine/docker_compose.ml index 6f4e4a90f878..e374b97f3ed6 100644 --- a/src/lib/integration_test_local_engine/docker_compose.ml +++ b/src/lib/integration_test_local_engine/docker_compose.ml @@ -88,8 +88,10 @@ module Dockerfile = struct type t = { version : string; services : service_map } [@@deriving to_yojson] let to_string = Fn.compose Yojson.Safe.pretty_to_string to_yojson -end - -type t = Dockerfile.t [@@deriving to_yojson] -let to_string = Fn.compose Yojson.Safe.pretty_to_string to_yojson + let write_config t ~dir ~filename = + Out_channel.with_file ~fail_if_exists:false + (dir ^ "/" ^ filename) + ~f:(fun ch -> t |> to_string |> Out_channel.output_string ch) ; + Util.run_cmd_exn dir "chmod" [ "600"; filename ] +end diff --git a/src/lib/integration_test_local_engine/docker_network.ml b/src/lib/integration_test_local_engine/docker_network.ml index bd9cd46af959..e8a735322789 100644 --- a/src/lib/integration_test_local_engine/docker_network.ml +++ b/src/lib/integration_test_local_engine/docker_network.ml @@ -9,7 +9,6 @@ let get_container_id service_id = let open Malleable_error.Let_syntax in let%bind container_ids = Deferred.bind ~f:Malleable_error.or_hard_error - (* docker ps is a fuzzy search, we assume that the first result is the most accurate result *) (Integration_test_lib.Util.run_cmd_or_error cwd "docker" [ "ps"; "-f"; sprintf "name=%s" service_id; "--quiet" ] ) in @@ -26,33 +25,27 @@ let run_in_container ?(exit_code = 10) container_id ~cmd = Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code cwd "docker" ([ "exec"; container_id ] @ cmd) -type config = { stack_name : string; graphql_enabled : bool } - module Node = struct - type service_info = + type config = { network_keypair : Network_keypair.t option ; service_id : string - ; is_archive_container : bool + ; postgres_connection_uri : string option ; graphql_port : int } - type t = - { service_info : service_info - ; config : config - ; mutable should_be_running : bool - } + type t = { config : config; mutable should_be_running : bool } - let id { service_info; _ } = service_info.service_id + let id { config; _ } = config.service_id - let infra_id { service_info; _ } = service_info.service_id + let infra_id { config; _ } = config.service_id let should_be_running { should_be_running; _ } = should_be_running - let network_keypair { service_info; _ } = service_info.network_keypair + let network_keypair { config; _ } = config.network_keypair let get_ingress_uri node = Uri.make ~scheme:"http" ~host:"127.0.0.1" ~path:"/graphql" - ~port:node.service_info.graphql_port () + ~port:node.config.graphql_port () let get_container_index_from_service_name service_name = match String.split_on_chars ~on:[ '_' ] service_name with @@ -62,28 +55,23 @@ module Node = struct failwith "get_container_index_from_service_name: bad service name" let dump_archive_data ~logger (t : t) ~data_file = - let service_name = t.service_info.service_id in - if not t.service_info.is_archive_container then - failwith - (sprintf "dump_archive_data: %s not an archive container" service_name) - else - let open Malleable_error.Let_syntax in - let%bind container_id = get_container_id service_name in - [%log info] "Dumping archive data from (node: %s, container: %s)" - service_name container_id ; - let postgres_url = - Docker_node_config.Postgres_config.default_connection_uri - ~host: - (sprintf "postgres_%s" - (get_container_index_from_service_name service_name) ) - in - let%map data = - run_in_container container_id - ~cmd:[ "pg_dump"; "--create"; "--no-owner"; postgres_url ] - in - [%log info] "Dumping archive data to file %s" data_file ; - Out_channel.with_file data_file ~f:(fun out_ch -> - Out_channel.output_string out_ch data ) + let service_name = t.config.service_id in + match t.config.postgres_connection_uri with + | None -> + failwith + (sprintf "dump_archive_data: %s not an archive container" service_name) + | Some postgres_uri -> + let open Malleable_error.Let_syntax in + let%bind container_id = get_container_id service_name in + [%log info] "Dumping archive data from (node: %s, container: %s)" + service_name container_id ; + let%map data = + run_in_container container_id + ~cmd:[ "pg_dump"; "--create"; "--no-owner"; postgres_uri ] + in + [%log info] "Dumping archive data to file %s" data_file ; + Out_channel.with_file data_file ~f:(fun out_ch -> + Out_channel.output_string out_ch data ) let get_logs_in_container container_id = let%bind.Deferred cwd = Unix.getcwd () in @@ -92,9 +80,9 @@ module Node = struct let dump_mina_logs ~logger (t : t) ~log_file = let open Malleable_error.Let_syntax in - let%bind container_id = get_container_id t.service_info.service_id in + let%bind container_id = get_container_id t.config.service_id in [%log info] "Dumping mina logs from (node: %s, container: %s)" - t.service_info.service_id container_id ; + t.config.service_id container_id ; let%map logs = get_logs_in_container container_id in [%log info] "Dumping mina logs to file %s" log_file ; Out_channel.with_file log_file ~f:(fun out_ch -> @@ -114,9 +102,9 @@ module Node = struct let run_replayer ?(start_slot_since_genesis = 0) ~logger (t : t) = let open Malleable_error.Let_syntax in - let%bind container_id = get_container_id t.service_info.service_id in + let%bind container_id = get_container_id t.config.service_id in [%log info] "Running replayer on (node: %s, container: %s)" - t.service_info.service_id container_id ; + t.config.service_id container_id ; let%bind accounts = run_in_container container_id ~cmd:[ "jq"; "-c"; ".ledger.accounts"; "/root/runtime_config.json" ] @@ -135,10 +123,7 @@ module Node = struct ~dest ) >>| ignore in - let postgres_url = - Docker_node_config.Postgres_config.default_connection_uri - ~host:"postgres_1" - in + let postgres_url = Option.value_exn t.config.postgres_connection_uri in run_in_container container_id ~cmd: [ "mina-replayer" @@ -153,10 +138,10 @@ module Node = struct let dump_precomputed_blocks ~logger (t : t) = let open Malleable_error.Let_syntax in - let container_id = t.service_info.service_id in + let container_id = t.config.service_id in [%log info] "Dumping precomputed blocks from logs for (node: %s, container: %s)" - t.service_info.service_id container_id ; + t.config.service_id container_id ; let%bind logs = get_logs_in_container container_id in (* kubectl logs may include non-log output, like "Using password from environment variable" *) let log_lines = @@ -232,7 +217,7 @@ module Node = struct let start ~fresh_state node : unit Malleable_error.t = let open Malleable_error.Let_syntax in - let%bind container_id = get_container_id node.service_info.service_id in + let%bind container_id = get_container_id node.config.service_id in node.should_be_running <- true ; let%bind () = if fresh_state then @@ -244,32 +229,28 @@ module Node = struct let stop node = let open Malleable_error.Let_syntax in - let%bind container_id = get_container_id node.service_info.service_id in + let%bind container_id = get_container_id node.config.service_id in node.should_be_running <- false ; run_in_container ~exit_code:12 container_id ~cmd:[ "/stop.sh" ] >>| ignore end module Service_to_deploy = struct - type service_to_deploy_config = + type config = { network_keypair : Network_keypair.t option - ; is_archive_container : bool + ; postgres_connection_uri : string option ; graphql_port : int } - type t = - { stack_name : string - ; service_name : string - ; service_info : service_to_deploy_config - } + type t = { stack_name : string; service_name : string; config : config } - let construct_service stack_name service_name service_info : t = - { stack_name; service_name; service_info } + let construct_service stack_name service_name config : t = + { stack_name; service_name; config } let init_service_to_deploy_config ?(network_keypair = None) - ?(is_archive_container = false) ~graphql_port = - { network_keypair; is_archive_container; graphql_port } + ?(postgres_connection_uri = None) ~graphql_port = + { network_keypair; postgres_connection_uri; graphql_port } - let get_node_from_service t ~config = + let get_node_from_service t = let%bind cwd = Unix.getcwd () in let open Malleable_error.Let_syntax in let service_id = t.stack_name ^ "_" ^ t.service_name in @@ -279,13 +260,12 @@ module Service_to_deploy = struct t.service_name else return - { Node.service_info = + { Node.config = { service_id - ; network_keypair = t.service_info.network_keypair - ; is_archive_container = t.service_info.is_archive_container - ; graphql_port = t.service_info.graphql_port + ; network_keypair = t.config.network_keypair + ; postgres_connection_uri = t.config.postgres_connection_uri + ; graphql_port = t.config.graphql_port } - ; config ; should_be_running = false } end @@ -345,6 +325,6 @@ let genesis_keypairs { genesis_keypairs; _ } = genesis_keypairs let all_ids t = let deployments = all_nodes t |> Core.Map.to_alist in List.fold deployments ~init:[] ~f:(fun acc (_, node) -> - List.cons node.service_info.service_id acc ) + List.cons node.config.service_id acc ) let initialize_infra ~logger network = Malleable_error.return () diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index 2d19e0c1e207..aa9a89d8452b 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -4,6 +4,18 @@ open Integration_test_lib open Docker_compose module PortManager = struct + let mina_internal_rest_port = 3085 + + let mina_internal_client_port = 8301 + + let mina_internal_metrics_port = 10001 + + let mina_internal_server_port = 3086 + + let mina_internal_external_port = 10101 + + let postgres_internal_port = 5432 + type t = { mutable available_ports : int list ; mutable used_ports : int list @@ -28,14 +40,11 @@ module PortManager = struct let rest_port_source = allocate_port t in let client_port_source = allocate_port t in let metrics_port_source = allocate_port t in - let rest_port_target = 3085 in - let client_port_target = 8301 in - let metrics_port_target = 10001 in [ { Dockerfile.Service.Port.published = rest_port_source - ; target = rest_port_target + ; target = mina_internal_rest_port } - ; { published = client_port_source; target = client_port_target } - ; { published = metrics_port_source; target = metrics_port_target } + ; { published = client_port_source; target = mina_internal_client_port } + ; { published = metrics_port_source; target = mina_internal_metrics_port } ] let release_port t port = @@ -50,7 +59,7 @@ module Base_node_config = struct type t = { peer : string option ; log_level : string - ; log_snark_work_gossip : bool option + ; log_snark_work_gossip : bool ; log_txn_pool_gossip : bool ; generate_genesis_proof : bool ; client_port : string @@ -63,22 +72,66 @@ module Base_node_config = struct } [@@deriving to_yojson] - let create_peer ~peer_name = - Printf.sprintf - "/dns4/%s/tcp/10101/p2p/12D3KooWMg66eGtSEx5UZ9EAqEp3W7JaGd6WTxdRFuqhskRN55dT" - peer_name + let container_runtime_config_path = "/root/runtime_config.json" + + let container_entrypoint_path = "/root/entrypoint.sh" + + let container_keys_path = "/root/keys" + + let container_libp2p_key_path = container_keys_path ^ "/libp2p_key" + + let entrypoint_script = + ( "entrypoint.sh" + , {|#!/bin/bash + # This file is auto-generated by the local integration test framework. + # Path to the libp2p_key file + LIBP2P_KEY_PATH="|} + ^ container_libp2p_key_path + ^ {|" + # Generate keypair and set permissions if libp2p_key does not exist + if [ ! -f "$LIBP2P_KEY_PATH" ]; then + mina libp2p generate-keypair --privkey-path $LIBP2P_KEY_PATH + fi + /bin/chmod -R 700 |} + ^ container_keys_path ^ {|/ + # Import any compatible keys in |} + ^ container_keys_path ^ {|/*, excluding certain keys + for key_file in |} + ^ container_keys_path + ^ {|/*; do + # Exclude specific keys (e.g., libp2p keys) + if [[ $(basename "$key_file") != "libp2p_key" ]]; then + mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" + fi + done + # Execute the puppeteer script + exec /mina_daemon_puppeteer.py "$@" + |} + ) + + let runtime_config_volume : Docker_compose.Dockerfile.Service.Volume.t = + { type_ = "bind" + ; source = "runtime_config.json" + ; target = container_runtime_config_path + } + + let entrypoint_volume : Docker_compose.Dockerfile.Service.Volume.t = + { type_ = "bind" + ; source = "entrypoint.sh" + ; target = container_entrypoint_path + } let default ?(runtime_config_path = None) ?(peer = None) = - { log_snark_work_gossip = Some true + { log_snark_work_gossip = true ; log_txn_pool_gossip = true ; generate_genesis_proof = true ; log_level = "Debug" - ; client_port = "8301" - ; rest_port = "3085" - ; metrics_port = "10001" - ; external_port = "10101" + ; client_port = PortManager.mina_internal_client_port |> Int.to_string + ; rest_port = PortManager.mina_internal_rest_port |> Int.to_string + ; metrics_port = PortManager.mina_internal_metrics_port |> Int.to_string + ; external_port = PortManager.mina_internal_external_port |> Int.to_string ; runtime_config_path - ; libp2p_key_path = "/root/keys/libp2p_key" + ; libp2p_key_path = container_libp2p_key_path ; libp2p_secret = "" ; peer } @@ -88,7 +141,7 @@ module Base_node_config = struct [ "-log-level" ; t.log_level ; "-log-snark-work-gossip" - ; Bool.to_string (t.log_snark_work_gossip |> Option.value ~default:true) + ; Bool.to_string t.log_snark_work_gossip ; "-log-txn-pool-gossip" ; Bool.to_string t.log_txn_pool_gossip ; "-generate-genesis-proof" @@ -171,8 +224,6 @@ module Block_producer_config = struct } let create ~service_name ~image ~ports ~volumes ~config = - (* TODO: make this better *) - (* let priv_key_path = "/root/keys/" ^ service_name ^ "-key" in *) let entrypoint = Some [ "/root/entrypoint.sh" ] in let docker_config = create_docker_config ~image ~ports ~volumes @@ -182,6 +233,14 @@ module Block_producer_config = struct end module Seed_config = struct + let peer_id = "12D3KooWMg66eGtSEx5UZ9EAqEp3W7JaGd6WTxdRFuqhskRN55dT" + + let libp2p_keypair = + {|{"box_primitive":"xsalsa20poly1305","pw_primitive":"argon2i","nonce":"7Bbvv2wZ6iCeqVyooU9WR81aygshMrLdXKieaHT","pwsalt":"Bh1WborqSwdzBi7m95iZdrCGspSf","pwdiff":[134217728,6],"ciphertext":"8fgvt4eKSzF5HMr1uEZARVHBoMgDKTx17zV7STVQyhyyEz1SqdH4RrU51MFGMPZJXNznLfz8RnSPsjrVqhc1CenfSLLWP5h7tTn86NbGmzkshCNvUiGEoSb2CrSLsvJsdn13ey9ibbZfdeXyDp9y6mKWYVmefAQLWUC1Kydj4f4yFwCJySEttAhB57647ewBRicTjdpv948MjdAVNf1tTxms4VYg4Jb3pLVeGAPaRtW5QHUkA8LwN5fh3fmaFk1mRudMd67UzGdzrVBeEHAp4zCnN7g2iVdWNmwN3"}|} + + let create_libp2p_peer ~peer_name ~external_port = + Printf.sprintf "/dns4/%s/tcp/%d/p2p/%s" peer_name external_port peer_id + type config = { archive_address : string option ; peer : string option @@ -196,6 +255,12 @@ module Seed_config = struct } [@@deriving to_yojson] + let seed_libp2p_keypair : Docker_compose.Dockerfile.Service.Volume.t = + { type_ = "bind" + ; source = "keys/libp2p_key" + ; target = Base_node_config.container_libp2p_key_path + } + let create_cmd config = let base_args = Base_node_config.to_list @@ -212,11 +277,6 @@ module Seed_config = struct in List.concat [ seed_args; base_args ] - let peer_id = "12D3KooWMg66eGtSEx5UZ9EAqEp3W7JaGd6WTxdRFuqhskRN55dT" - - let libp2p_keypair = - {|{"box_primitive":"xsalsa20poly1305","pw_primitive":"argon2i","nonce":"7Bbvv2wZ6iCeqVyooU9WR81aygshMrLdXKieaHT","pwsalt":"Bh1WborqSwdzBi7m95iZdrCGspSf","pwdiff":[134217728,6],"ciphertext":"8fgvt4eKSzF5HMr1uEZARVHBoMgDKTx17zV7STVQyhyyEz1SqdH4RrU51MFGMPZJXNznLfz8RnSPsjrVqhc1CenfSLLWP5h7tTn86NbGmzkshCNvUiGEoSb2CrSLsvJsdn13ey9ibbZfdeXyDp9y6mKWYVmefAQLWUC1Kydj4f4yFwCJySEttAhB57647ewBRicTjdpv948MjdAVNf1tTxms4VYg4Jb3pLVeGAPaRtW5QHUkA8LwN5fh3fmaFk1mRudMd67UzGdzrVBeEHAp4zCnN7g2iVdWNmwN3"}|} - let create_docker_config ~image ~entrypoint ~ports ~volumes ~environment ~config = { Dockerfile.Service.image @@ -302,7 +362,6 @@ module Snark_coordinator_config = struct ; ("MINA_SNARK_FEE", snark_worker_fee) ; ("WORK_SELECTION", work_selection) ; ("MINA_CLIENT_TRUSTLIST", "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16") - (* Allow all IPs to connect*) ] let create_cmd config = @@ -350,17 +409,6 @@ module Snark_coordinator_config = struct { service_name; config; docker_config } end -module Postgres_config_support = struct - type connection_info = - { host : string - ; username : string - ; password : string - ; database : string - ; port : int - } - [@@deriving to_yojson] -end - module Postgres_config = struct type config = { host : string @@ -378,60 +426,61 @@ module Postgres_config = struct } [@@deriving to_yojson] - let postgres_default_envs ~username ~password ~database ~port = - [ ("BITNAMI_DEBUG", "false") - ; ("POSTGRES_USER", username) - ; ("POSTGRES_PASSWORD", password) - ; ("POSTGRES_DB", database) - ; ("PGPASSWORD", password) - ; ("POSTGRESQL_PORT_NUMBER", port) - ; ("POSTGRESQL_ENABLE_LDAP", "no") - ; ("POSTGRESQL_ENABLE_TLS", "no") - ; ("POSTGRESQL_LOG_HOSTNAME", "false") - ; ("POSTGRESQL_LOG_CONNECTIONS", "false") - ; ("POSTGRESQL_LOG_DISCONNECTIONS", "false") - ; ("POSTGRESQL_PGAUDIT_LOG_CATALOG", "off") - ; ("POSTGRESQL_CLIENT_MIN_MESSAGES", "error") - ; ("POSTGRESQL_SHARED_PRELOAD_LIBRARIES", "pgaudit") - ; ("POSTGRES_HOST_AUTH_METHOD", "trust") - ] + let postgres_image = "docker.io/bitnami/postgresql" let postgres_script = ( "postgres_entrypoint.sh" , {|#!/bin/bash -# This file is auto-generated by docker_compose.ml +# This file is auto-generated by the local integration test framework. cd /bitnami +# Create the archive database and import the schema psql -U postgres -d archive -f ./create_schema.sql |} ) - let archive_create_schema_volume : Dockerfile.Service.Volume.t = + let postgres_create_schema_volume : Dockerfile.Service.Volume.t = { type_ = "bind" ; source = "create_schema.sql" ; target = "/bitnami/create_schema.sql" } - let archive_zkapp_schema_volume : Dockerfile.Service.Volume.t = + let postgres_zkapp_schema_volume : Dockerfile.Service.Volume.t = { type_ = "bind" ; source = "zkapp_tables.sql" ; target = "/bitnami/zkapp_tables.sql" } - let archive_entrypoint_volume : Dockerfile.Service.Volume.t = + let postgres_entrypoint_volume : Dockerfile.Service.Volume.t = { type_ = "bind" ; source = "postgres_entrypoint.sh" ; target = "/docker-entrypoint-initdb.d/postgres_entrypoint.sh" } - let create_connection_info ~host ~username ~password ~database ~port = - { host; username; password; database; port } + let postgres_default_envs ~username ~password ~database ~port = + [ ("BITNAMI_DEBUG", "false") + ; ("POSTGRES_USER", username) + ; ("POSTGRES_PASSWORD", password) + ; ("POSTGRES_DB", database) + ; ("PGPASSWORD", password) + ; ("POSTGRESQL_PORT_NUMBER", port) + ; ("POSTGRESQL_ENABLE_LDAP", "no") + ; ("POSTGRESQL_ENABLE_TLS", "no") + ; ("POSTGRESQL_LOG_HOSTNAME", "false") + ; ("POSTGRESQL_LOG_CONNECTIONS", "false") + ; ("POSTGRESQL_LOG_DISCONNECTIONS", "false") + ; ("POSTGRESQL_PGAUDIT_LOG_CATALOG", "off") + ; ("POSTGRESQL_CLIENT_MIN_MESSAGES", "error") + ; ("POSTGRESQL_SHARED_PRELOAD_LIBRARIES", "pgaudit") + ; ("POSTGRES_HOST_AUTH_METHOD", "trust") + ] - let create_connection_uri { host; username; password; database; port } = + let create_connection_uri ~host ~username ~password ~database ~port = Printf.sprintf "postgres://%s:%s@%s:%s/%s" username password host (Int.to_string port) database - let default_connection_uri ~host = - Printf.sprintf "postgres://postgres:password@%s:5432/archive" host + let to_connection_uri t = + create_connection_uri ~host:t.host ~port:t.port ~username:t.username + ~password:t.password ~database:t.database let create_docker_config ~image ~entrypoint ~ports ~volumes ~environment = { Dockerfile.Service.image @@ -459,7 +508,6 @@ module Archive_node_config = struct type config = { postgres_config : Postgres_config.t ; server_port : int - ; postgres_uri : string ; runtime_config_path : string } [@@deriving to_yojson] @@ -475,7 +523,7 @@ module Archive_node_config = struct [ "mina-archive" ; "run" ; "-postgres-uri" - ; config.postgres_uri + ; Postgres_config.to_connection_uri config.postgres_config.config ; "-server-port" ; Int.to_string config.server_port ; "-config-file" diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index a4b1bd20c228..b381af82cde8 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -5,36 +5,8 @@ open Signature_lib open Mina_base open Integration_test_lib -[@@@warning "-27"] - let docker_swarm_version = "3.8" -let postgres_image = "docker.io/bitnami/postgresql" - -(*TODO: Make this less spagett *) -let entrypoint_script = - ( "entrypoint.sh" - , {|#!/bin/bash -# This file is auto-generated by docker_compose.ml -# Path to the libp2p_key file -LIBP2P_KEY_PATH="/root/keys/libp2p_key" -# Generate keypair and set permissions if libp2p_key does not exist -if [ ! -f "$LIBP2P_KEY_PATH" ]; then - mina libp2p generate-keypair --privkey-path $LIBP2P_KEY_PATH -fi -/bin/chmod -R 700 /root/keys/ -# Import any compatible keys in /root/keys, excluding certain keys -for key_file in /root/keys/*; do - # Exclude specific keys (e.g., libp2p keys) - if [[ $(basename "$key_file") != "libp2p_key" ]]; then - mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" - fi -done -# Execute the puppeteer script -exec /mina_daemon_puppeteer.py "$@" -|} - ) - module Network_config = struct module Cli_inputs = Cli_inputs @@ -54,10 +26,6 @@ module Network_config = struct ; archive_node_configs : Docker_node_config.Archive_node_config.t list ; mina_archive_schema_aux_files : string list ; log_precomputed_blocks : bool - ; cpu_request : int - ; mem_request : string - ; worker_cpu_request : int - ; worker_mem_request : string } [@@deriving to_yojson] @@ -80,13 +48,14 @@ module Network_config = struct let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = + let _ = cli_inputs in let { genesis_ledger ; epoch_data ; block_producers ; snark_coordinator ; snark_worker_fee ; num_archive_nodes - ; log_precomputed_blocks (* ; num_plain_nodes *) + ; log_precomputed_blocks ; proof_config ; Test_config.k ; delta @@ -145,9 +114,7 @@ module Network_config = struct { default with pk = Public_key.Compressed.to_string pk ; sk = Some (Private_key.to_base58_check sk) - ; balance = - Balance.of_mina_string_exn balance - (* delegation currently unsupported *) + ; balance = Balance.of_mina_string_exn balance ; delegate = None ; timing } @@ -188,7 +155,7 @@ module Network_config = struct ; genesis_state_timestamp = Some Core.Time.(to_string_abs ~zone:Zone.utc (now ())) } - ; proof = Some proof_config (* TODO: prebake ledger and only set hash *) + ; proof = Some proof_config ; ledger = Some { base = @@ -202,17 +169,6 @@ module Network_config = struct ; name = None } ; epoch_data = - (* each staking epoch ledger account must also be a genesis ledger account, though - the balance may be different; the converse is not necessarily true, since - an account may have been added after the last epoch ledger was taken - - each staking epoch ledger account must also be in the next epoch ledger, if provided - - if provided, each next_epoch_ledger account must be in the genesis ledger - - in all ledgers, the accounts must be in the same order, so that accounts will - be in the same leaf order - *) Option.map epoch_data ~f:(fun { staking = staking_ledger; next } -> let genesis_winner_account : Runtime_config.Accounts.single = Runtime_config.Accounts.Single.of_account @@ -240,15 +196,6 @@ module Network_config = struct in { genesis_account with balance; timing } ) in - (* because we run integration tests with Proof_level = Full, the winner account - gets added to the genesis ledger - - there isn't a corresponding mechanism to add the winner account to epoch - ledgers, so we add it explicitly here - - `add_genesis_winner` in the record below has no effect, it's ignored in - Runtime_config.Epoch_data.to_yojson, which is used to create the config file - *) ( { base = Accounts (genesis_winner_account :: epoch_ledger_accounts) ; add_genesis_winner = None (* no effect *) @@ -326,100 +273,105 @@ module Network_config = struct in let open Docker_node_config in let open Docker_compose.Dockerfile in - let port_manager = PortManager.create ~min_port:10000 ~max_port:10100 in - let runtime_config_volume : Service.Volume.t = - { type_ = "bind" - ; source = "runtime_config.json" - ; target = "/root/runtime_config.json" - } + let port_manager = PortManager.create ~min_port:10000 ~max_port:11000 in + let docker_volumes = + [ Base_node_config.runtime_config_volume + ; Base_node_config.entrypoint_volume + ] in - let entrypoint_volume : Service.Volume.t = - { type_ = "bind" - ; source = "entrypoint.sh" - ; target = "/root/entrypoint.sh" - } + let generate_random_id () = + let rand_char () = + let ascii_a = int_of_char 'a' in + let ascii_z = int_of_char 'z' in + char_of_int (ascii_a + Random.int (ascii_z - ascii_a + 1)) + in + String.init 4 ~f:(fun _ -> rand_char ()) in - let docker_volumes = [ runtime_config_volume; entrypoint_volume ] in let seed_config = - let seed_config : Seed_config.config = + let config : Seed_config.config = { archive_address = None ; peer = None - ; runtime_config_path = runtime_config_volume.target - } - in - let seed_libp2p_keypair : Service.Volume.t = - { type_ = "bind" - ; source = "keys/libp2p_key" - ; target = "/root/keys/libp2p_key" + ; runtime_config_path = Base_node_config.container_runtime_config_path } in - Seed_config.create ~service_name:"seed" ~image:images.mina + Seed_config.create + ~service_name:(sprintf "seed-%s" (generate_random_id ())) + ~image:images.mina ~ports:(PortManager.allocate_ports_for_node port_manager) - ~volumes:(docker_volumes @ [ seed_libp2p_keypair ]) - ~config:seed_config + ~volumes:(docker_volumes @ [ Seed_config.seed_libp2p_keypair ]) + ~config in let seed_config_peer = - Some (Base_node_config.create_peer ~peer_name:seed_config.service_name) + Some + (Seed_config.create_libp2p_peer ~peer_name:seed_config.service_name + ~external_port:PortManager.mina_internal_external_port ) in let archive_node_configs = List.init num_archive_nodes ~f:(fun index -> - let postgres_target_port = 5432 in - let connection_info = - Postgres_config.create_connection_info - ~host:("postgres" ^ "_" ^ Int.to_string (index + 1)) - ~username:"postgres" ~password:"password" ~database:"archive" - ~port:postgres_target_port + let config = + { Postgres_config.host = + sprintf "postgres-%d-%s" (index + 1) (generate_random_id ()) + ; username = "postgres" + ; password = "password" + ; database = "archive" + ; port = PortManager.postgres_internal_port + } in let postgres_port = Service.Port.create ~published:(PortManager.allocate_port port_manager) - ~target:postgres_target_port + ~target:PortManager.postgres_internal_port in let postgres_config = - Postgres_config.create ~service_name:connection_info.host - ~image:postgres_image ~ports:[ postgres_port ] + Postgres_config.create ~service_name:config.host + ~image:Postgres_config.postgres_image ~ports:[ postgres_port ] ~volumes: - [ Postgres_config.archive_create_schema_volume - ; Postgres_config.archive_zkapp_schema_volume - ; Postgres_config.archive_entrypoint_volume + [ Postgres_config.postgres_create_schema_volume + ; Postgres_config.postgres_zkapp_schema_volume + ; Postgres_config.postgres_entrypoint_volume ] - ~config:connection_info + ~config in let archive_server_port = Service.Port.create ~published:(PortManager.allocate_port port_manager) - ~target:3086 - in - let archive_rest_port = - Service.Port.create - ~published:(PortManager.allocate_port port_manager) - ~target:3085 + ~target:PortManager.mina_internal_server_port in let config : Archive_node_config.config = { postgres_config - ; postgres_uri = - Postgres_config.create_connection_uri connection_info ; server_port = archive_server_port.target - ; runtime_config_path = runtime_config_volume.target + ; runtime_config_path = + Base_node_config.container_runtime_config_path } in + let archive_rest_port = + Service.Port.create + ~published:(PortManager.allocate_port port_manager) + ~target:PortManager.mina_internal_rest_port + in Archive_node_config.create - ~service_name:("archive_" ^ Int.to_string (index + 1)) + ~service_name: + (sprintf "archive-%d-%s" (index + 1) (generate_random_id ())) ~image:images.archive_node ~ports:[ archive_server_port; archive_rest_port ] ~volumes:docker_volumes ~config ) in (* Each archive node has it's own seed node *) let seed_configs = - List.mapi archive_node_configs ~f:(fun idx archive_config -> + List.mapi archive_node_configs ~f:(fun index archive_config -> let config : Seed_config.config = - { archive_address = Some (archive_config.service_name ^ ":3086") + { archive_address = + Some + (sprintf "%s:%d" archive_config.service_name + PortManager.mina_internal_server_port ) ; peer = seed_config_peer - ; runtime_config_path = runtime_config_volume.target + ; runtime_config_path = + Base_node_config.container_runtime_config_path } in Seed_config.create - ~service_name:("seed_" ^ Int.to_string (idx + 1)) + ~service_name: + (sprintf "seed-%d-%s" (index + 1) (generate_random_id ())) ~image:images.mina ~ports:(PortManager.allocate_ports_for_node port_manager) ~volumes:docker_volumes ~config ) @@ -445,24 +397,28 @@ module Network_config = struct in failwith failstring in + let priv_key_path = + Base_node_config.container_keys_path ^/ node.account_name + in let volumes = - [ Service.Volume.create - ("keys" ^/ node.account_name) - ("/root/keys" ^/ node.account_name) + [ Service.Volume.create ("keys" ^/ node.account_name) priv_key_path ] @ docker_volumes in let block_producer_config : Block_producer_config.config = { keypair - ; runtime_config_path = runtime_config_volume.target - ; libp2p_secret = "" + ; runtime_config_path = + Base_node_config.container_runtime_config_path ; peer = seed_config_peer - ; priv_key_path = "/root/keys/" ^ node.node_name ^ "-key" + ; priv_key_path ; enable_peer_exchange = true ; enable_flooding = true + ; libp2p_secret = "" } in - Block_producer_config.create ~service_name:node.node_name + Block_producer_config.create + ~service_name: + (sprintf "%s-%s" node.node_name (generate_random_id ())) ~image:images.mina ~ports:(PortManager.allocate_ports_for_node port_manager) ~volumes ~config:block_producer_config ) @@ -498,10 +454,17 @@ module Network_config = struct PortManager.allocate_ports_for_node port_manager in let daemon_port = - coordinator_ports |> List.find_exn ~f:(fun p -> p.target = 8301) + coordinator_ports + |> List.find_exn ~f:(fun p -> + p.target + = Docker_node_config.PortManager.mina_internal_client_port ) + in + let snark_node_service_name = + sprintf "%s-%s" snark_coordinator_node.node_name + (generate_random_id ()) in let worker_node_config : Snark_worker_config.config = - { daemon_address = snark_coordinator_node.node_name + { daemon_address = snark_node_service_name ; daemon_port = Int.to_string daemon_port.target ; proof_level = "full" } @@ -510,7 +473,8 @@ module Network_config = struct List.init snark_coordinator_node.worker_nodes ~f:(fun index -> Docker_node_config.Snark_worker_config.create ~service_name: - ("snark-worker" ^ "_" ^ Int.to_string (index + 1)) + (sprintf "snark-worker-%d-%s" (index + 1) + (generate_random_id ()) ) ~image:images.mina ~ports: (Docker_node_config.PortManager.allocate_ports_for_node @@ -520,7 +484,8 @@ module Network_config = struct let snark_coordinator_config : Snark_coordinator_config.config = { worker_nodes ; snark_worker_fee - ; runtime_config_path = runtime_config_volume.target + ; runtime_config_path = + Base_node_config.container_runtime_config_path ; snark_coordinator_key = public_key ; peer = seed_config_peer ; work_selection = "seq" @@ -528,7 +493,7 @@ module Network_config = struct in Some (Snark_coordinator_config.create - ~service_name:snark_coordinator_node.node_name ~image:images.mina + ~service_name:snark_node_service_name ~image:images.mina ~ports:coordinator_ports ~volumes:docker_volumes ~config:snark_coordinator_config ) in @@ -550,10 +515,6 @@ module Network_config = struct ; mina_archive_schema_aux_files ; snark_coordinator_config ; archive_node_configs (* Resource configs *) - ; cpu_request = 1 - ; mem_request = "12Gi" - ; worker_cpu_request = 6 - ; worker_mem_request = "8Gi" } } @@ -673,17 +634,26 @@ module Network_manager = struct File_system.remove_dir docker_dir ) else return () in - let%bind () = Unix.mkdir docker_dir in [%log info] "Writing docker configuration %s" docker_dir ; - Out_channel.with_file ~fail_if_exists:true - (docker_dir ^/ docker_compose_file_path) ~f:(fun ch -> - Network_config.to_docker network_config - |> Docker_compose.to_string - |> Out_channel.output_string ch ) ; - [%log info] - "Writing out the genesis keys (in case you want to use them manually) to \ - testnet dir %s" - docker_dir ; + let%bind () = Unix.mkdir docker_dir in + let%bind _ = + Docker_compose.Dockerfile.write_config ~dir:docker_dir + ~filename:docker_compose_file_path + (Network_config.to_docker network_config) + in + return () + + let write_docker_bind_volumes ~logger ~docker_dir + ~(network_config : Network_config.t) = + let open Deferred.Let_syntax in + [%log info] "Writing runtime_config %s" docker_dir ; + let%bind () = + Yojson.Safe.to_file + (String.concat [ docker_dir; "/runtime_config.json" ]) + network_config.docker.runtime_config + |> Deferred.return + in + [%log info] "Writing out the genesis keys to dir %s" docker_dir ; let kps_base_path = String.concat [ docker_dir; "/keys" ] in let%bind () = Unix.mkdir kps_base_path in [%log info] "Writing genesis keys to %s" kps_base_path ; @@ -714,21 +684,12 @@ module Network_manager = struct [%log info] "Writing custom entrypoint script (libp2p key generation and puppeteer \ context)" ; - let entrypoint_filename, entrypoint_script = entrypoint_script in - Out_channel.with_file ~fail_if_exists:true - (docker_dir ^/ entrypoint_filename) ~f:(fun ch -> - entrypoint_script |> Out_channel.output_string ch ) ; - ignore (Util.run_cmd_exn docker_dir "chmod" [ "+x"; entrypoint_filename ]) ; - [%log info] - "Writing custom postgres entrypoint script (libp2p key generation and \ - puppeteer context)" ; let entrypoint_filename, entrypoint_script = - Docker_node_config.Postgres_config.postgres_script + Docker_node_config.Base_node_config.entrypoint_script in Out_channel.with_file ~fail_if_exists:true (docker_dir ^/ entrypoint_filename) ~f:(fun ch -> entrypoint_script |> Out_channel.output_string ch ) ; - ignore (Util.run_cmd_exn docker_dir "chmod" [ "+x"; entrypoint_filename ]) ; let%bind _ = Deferred.List.iter network_config.docker.mina_archive_schema_aux_files ~f:(fun schema_url -> @@ -738,16 +699,23 @@ module Network_manager = struct Util.run_cmd_or_hard_error docker_dir "curl" [ "-o"; filename; schema_url ] in + [%log info] + "Writing custom postgres entrypoint script (import archive node \ + schema)" ; + Deferred.return () ) |> Deferred.return in - [%log info] "Writing runtime_config %s" docker_dir ; - let%bind () = - Yojson.Safe.to_file - (String.concat [ docker_dir; "/runtime_config.json" ]) - network_config.docker.runtime_config - |> Deferred.return + ignore (Util.run_cmd_exn docker_dir "chmod" [ "+x"; entrypoint_filename ]) ; + let postgres_entrypoint_filename, postgres_entrypoint_script = + Docker_node_config.Postgres_config.postgres_script in + Out_channel.with_file ~fail_if_exists:true + (docker_dir ^/ postgres_entrypoint_filename) ~f:(fun ch -> + postgres_entrypoint_script |> Out_channel.output_string ch ) ; + ignore + (Util.run_cmd_exn docker_dir "chmod" + [ "+x"; postgres_entrypoint_filename ] ) ; return () let initialize_workloads ~logger (network_config : Network_config.t) = @@ -755,7 +723,9 @@ module Network_manager = struct List.find_map_exn ports ~f:(fun port -> match port with | Docker_compose.Dockerfile.Service.Port.{ published; target } -> - if target = 3085 then Some published else None ) + if target = Docker_node_config.PortManager.mina_internal_rest_port + then Some published + else None ) in [%log info] "Initializing seed workloads" ; let seed_workloads = @@ -765,7 +735,8 @@ module Network_manager = struct Docker_network.Service_to_deploy.construct_service network_config.docker.stack_name seed_config.service_name (Docker_network.Service_to_deploy.init_service_to_deploy_config - ~network_keypair:None ~is_archive_container:false ~graphql_port ) + ~network_keypair:None ~postgres_connection_uri:None + ~graphql_port ) in (seed_config.service_name, node) ) |> Core.String.Map.of_alist_exn @@ -779,7 +750,7 @@ module Network_manager = struct network_config.docker.stack_name bp_config.service_name (Docker_network.Service_to_deploy.init_service_to_deploy_config ~network_keypair:(Some bp_config.config.keypair) - ~is_archive_container:false ~graphql_port ) + ~postgres_connection_uri:None ~graphql_port ) in (bp_config.service_name, node) ) |> Core.String.Map.of_alist_exn @@ -799,7 +770,7 @@ module Network_manager = struct snark_coordinator_config.service_name (Docker_network.Service_to_deploy .init_service_to_deploy_config ~network_keypair:None - ~is_archive_container:false ~graphql_port ) + ~postgres_connection_uri:None ~graphql_port ) in [ (snark_coordinator_config.service_name, coordinator) ] |> Core.String.Map.of_alist_exn @@ -817,7 +788,7 @@ module Network_manager = struct snark_worker_config.service_name (Docker_network.Service_to_deploy .init_service_to_deploy_config ~network_keypair:None - ~is_archive_container:false ~graphql_port ) + ~postgres_connection_uri:None ~graphql_port ) in (snark_worker_config.service_name, worker) ) @@ -834,11 +805,16 @@ module Network_manager = struct let graphql_port = find_rest_port archive_config.docker_config.ports in + let postgres_connection_uri = + Some + (Docker_node_config.Postgres_config.to_connection_uri + archive_config.config.postgres_config.config ) + in let node = Docker_network.Service_to_deploy.construct_service network_config.docker.stack_name archive_config.service_name (Docker_network.Service_to_deploy.init_service_to_deploy_config - ~network_keypair:None ~is_archive_container:true ~graphql_port ) + ~network_keypair:None ~postgres_connection_uri ~graphql_port ) in (archive_config.service_name, node) ) |> Core.String.Map.of_alist_exn @@ -849,7 +825,7 @@ module Network_manager = struct , snark_worker_workloads , archive_workloads ) - let poll_until_stack_deployed ~logger services = + let poll_until_stack_deployed ~logger = let poll_interval = Time.Span.of_sec 15.0 in let max_polls = 20 (* 5 mins *) in let get_service_statuses () = @@ -882,7 +858,7 @@ module Network_manager = struct let open Malleable_error.Let_syntax in if List.is_empty bad_service_statuses then return () else if n > 0 then ( - [%log debug] "Got bad pod statuses, polling again ($failed_statuses" + [%log debug] "Got bad service statuses, polling again ($failed_statuses" ~metadata: [ ( "failed_statuses" , `Assoc @@ -942,11 +918,16 @@ module Network_manager = struct in let open Deferred.Let_syntax in let docker_dir = network_config.docker.stack_name in - let docker_compose_file_path = docker_dir ^ ".compose.json" in + let docker_compose_file_path = + network_config.docker.stack_name ^ ".compose.json" + in let%bind () = generate_docker_stack_file ~logger ~docker_dir ~docker_compose_file_path ~network_config in + let%bind () = + write_docker_bind_volumes ~logger ~docker_dir ~network_config + in let t = { stack_name = network_config.docker.stack_name ; logger @@ -977,15 +958,12 @@ module Network_manager = struct [ "stack"; "deploy"; "-c"; t.docker_compose_file_path; t.stack_name ] in t.deployed <- true ; - let config : Docker_network.config = - { stack_name = t.stack_name; graphql_enabled = t.graphql_enabled } - in - let%bind () = poll_until_stack_deployed ~logger t.services_by_id in + let%bind () = poll_until_stack_deployed ~logger in let open Malleable_error.Let_syntax in let func_for_fold ~(key : string) ~data accum_M = let%bind mp = accum_M in let%map node = - Docker_network.Service_to_deploy.get_node_from_service data ~config + Docker_network.Service_to_deploy.get_node_from_service data in Core.String.Map.add_exn mp ~key ~data:node in diff --git a/src/lib/structured_log_events/structured_log_events.ml b/src/lib/structured_log_events/structured_log_events.ml index dffe7289ae58..4322c0cf6021 100644 --- a/src/lib/structured_log_events/structured_log_events.ml +++ b/src/lib/structured_log_events/structured_log_events.ml @@ -35,7 +35,6 @@ let parse_exn id json_pairs = List.filter json_pairs ~f:(fun (field_name, _) -> Set.mem repr.arguments field_name ) in - match repr.parse json_pairs with | Some t -> Some t