[tools] refactor how upstream nodes are selected for by txs (0LNetw…

…orkCommunity#952) * move the default init of txparams to a constructor * seprate the txparams type into a new module. place constructors in right place. change default behavior of initializing TX params, to find a good upstream from the list of upstream_nodes. * cargo fix * clear warnings * patch build * patch build * patch get_process * cli args patch * remove default_node from 0L.toml * rename upstream_nodes to rpc_fullnodes * patch tests * app configs documentation * grammar * make upstream_node not an option * build * limit backlog sending when miner has maxed current epoch proofs. * patch issue with swarm not making txs params * make MAX_PROOFS_PER_EPOCH pub * clean * patch test build error * backlog should exit with a TxError if can't evaluate tx status * scaffold tower error * patch build * patch build * change return type * patch build * display trait for towererror * format tower errors * Add tower error * add error code * add tower error * add lower bound mining thresh * patch defaults affecting integration tests
ownyrd · Jan 31, 2022 · 7870613 · 7870613
1 parent 2854a8f
commit 7870613
Show file tree

Hide file tree

Showing 34 changed files with 1,104 additions and 468 deletions.
diff --git a/ol/cli/src/mgmt/management.rs b/ol/cli/src/mgmt/management.rs
@@ -297,7 +297,7 @@ impl Node {
     fn get_process(&self, proc_name: &str) -> Option<HostProcess> {
         match proc_name {
             "node" => self.vitals.node_proc.clone(),
-            "miner" => self.vitals.node_proc.clone(),
+            "tower" => self.vitals.miner_proc.clone(),
             "monitor" => self.vitals.monitor_proc.clone(),
             _ => None,
         }

diff --git a/ol/cli/src/node/client.rs b/ol/cli/src/node/client.rs
@@ -35,84 +35,73 @@ pub fn get_client() -> Option<DiemClient> {
     let waypoint = config
         .get_waypoint(entry_args.swarm_path)
         .expect("could not get waypoint");
-    if let Some(vec_urs) = config.profile.upstream_nodes.as_ref() {
-      for url in vec_urs {
-          match DiemClient::new(url.clone(), waypoint){
-            Ok(client) => { 
-              // TODO: What's the better way to check we can connect to client?
+    // if let Some(vec_urs) = config.profile.upstream_nodes.as_ref() {
+    for url in &config.profile.upstream_nodes {
+        match DiemClient::new(url.clone(), waypoint) {
+            Ok(client) => {
+                // TODO: What's the better way to check we can connect to client?
                 let metadata = client.get_metadata();
                 if metadata.is_ok() {
                     // found a connect-able upstream node
                     return Some(client);
                 }
-            },
-            Err(_) => {},
+            }
+            Err(_) => {}
         };
-      }
+        // }
     }
 
-
     None
 }
 
 /// get client type with defaults from toml for remote node
 pub fn find_a_remote_jsonrpc(config: &AppCfg, waypoint: Waypoint) -> Result<DiemClient, Error> {
     let mut rng = thread_rng();
-    if let Some(list) = &config.profile.upstream_nodes {
-        let len = list.len();
-        let url = list.choose_multiple(&mut rng, len)
-            .into_iter()
-            .find(|&remote_url| {
-                println!("trying upstream url: {}", &remote_url);
-                match make_client(Some(remote_url.to_owned()), waypoint) {
-                    Ok(c) => {
-                      match c.get_metadata(){
-                        Ok(m) => {
-                          if m.version > 0 { true }
-                          else { 
+    let list = &config.profile.upstream_nodes;
+    let len = list.len();
+    let url = list
+        .choose_multiple(&mut rng, len)
+        .into_iter()
+        .find(|&remote_url| {
+            // println!("trying upstream url: {}", &remote_url);
+            match make_client(Some(remote_url.to_owned()), waypoint) {
+                Ok(c) => match c.get_metadata() {
+                    Ok(m) => {
+                        if m.version > 0 {
+                            true
+                        } else {
                             println!("can make client but could not get blockchain height > 0");
                             false
-                          }
-                        },
-                        Err(e) => {
-                          println!("can make client but could not get metadata {:?}", e);
-                          false
-                        },
+                        }
                     }
-                    },
                     Err(e) => {
-                      println!("could not make client {:?}", e);
-                      false
-                    },
+                        println!("can make client but could not get metadata {:?}", e);
+                        false
+                    }
+                },
+                Err(e) => {
+                    println!("could not make client {:?}", e);
+                    false
                 }
-            });
-
-            if let Some(url_clean) = url {
-              return make_client(Some(url_clean.to_owned()), waypoint);
-            }; 
-
-    }
+            }
+        });
+
+    if let Some(url_clean) = url {
+        return make_client(Some(url_clean.to_owned()), waypoint);
+    };
     Err(Error::msg(
         "Cannot connect to any JSON RPC peers in the list of upstream_nodes in 0L.toml",
     ))
 }
 
-/// get client type with defaults from toml for local node
-pub fn default_local_client(config: &AppCfg, waypoint: Waypoint) -> Result<DiemClient, Error> {
-    let local_url = config
-        .profile
-        .default_node
-        .clone()
-        .expect("could not get url from configs");
-
-    make_client(Some(local_url.clone()), waypoint)
+/// the default client will be the first option in the list.
+pub fn default_local_rpc(waypoint: Waypoint) -> Result<DiemClient, Error> {
+    make_client("127.0.0.1".parse().ok(), waypoint)
 }
 
 /// connect a swarm client
-pub fn swarm_test_client(config: &mut AppCfg, swarm_path: PathBuf) -> Result<DiemClient, Error> {
+pub fn swarm_test_client(swarm_path: PathBuf) -> Result<DiemClient, Error> {
     let (url, waypoint) = ol_types::config::get_swarm_rpc_url(swarm_path.clone());
-    config.profile.default_node = Some(url.clone());
-    config.profile.upstream_nodes = Some(vec![url.clone()]);
 
     make_client(Some(url.clone()), waypoint)
 }
@@ -121,20 +110,18 @@ pub fn swarm_test_client(config: &mut AppCfg, swarm_path: PathBuf) -> Result<Die
 pub fn pick_client(swarm_path: Option<PathBuf>, config: &mut AppCfg) -> Result<DiemClient, Error> {
     let is_swarm = *&swarm_path.is_some();
     if let Some(path) = swarm_path {
-        return swarm_test_client(config, path);
+        return swarm_test_client(path);
     };
     let waypoint = config.get_waypoint(swarm_path)?;
 
     // check if is in sync
-    let local_client = default_local_client(config, waypoint.clone())?;
+    let local_client = default_local_rpc(waypoint.clone())?;
 
     let remote_client = find_a_remote_jsonrpc(config, waypoint.clone())?;
     // compares to an upstream random remote client. If it is synced, use the local client as the default
     let mut node = Node::new(local_client, config, is_swarm);
-    match node.check_sync()?
-    .is_synced {
-      true => Ok(node.client),
-      false => Ok(remote_client),
-
+    match node.check_sync()?.is_synced {
+        true => Ok(node.client),
+        false => Ok(remote_client),
     }
 }
diff --git a/ol/documentation/cli-tools/app-configs.md b/ol/documentation/cli-tools/app-configs.md
@@ -0,0 +1,93 @@
+
+# App Configs - 0L.toml
+
+All 0L tools use a configuration file called `0L.toml` located in your 0L home directory. usually this is `$HOME/.0L/0L.toml`.
+
+The Diem services, namely `diem-node`, use their own configuration format (e.g. `$HOME/.0L/validator.node.yaml`).
+
+0L apps including Carpe, and cli tools like `ol`, `tower`, `txs`, `web-monitor`, `onboard` use the 0L.toml for configurations.
+
+# Where is this file
+By default all 0L tools search for this file in `$HOME/.0L/` directory.
+
+If you have placed this file elsewhere, you'll need to explicitly call the config path when starting an app:
+```
+ol --config <path/to/config> ...
+ol -c <path/to/config> ...
+
+```
+
+# workspace
+
+Workspace covers default file and directory paths that the tools may refer to. The most relevant one is the `home` path which is where this file and all other state and config files will be stored.
+
+# profile
+Profile tells the apps the preferences of this node.
+It describes the owner with 
+```
+account = "3dc18d1cf61faac6ac70e3a63f062e4b"
+auth_key = "2bffcbd0e9016013cb8ca78459f69d2b3dc18d1cf61faac6ac70e3a63f062e4b"
+```
+
+A fun personal statement which was used in the first miner proof you submitted.
+```
+statement = "test"
+```
+The IP address of this node (e.g. a validator node), and the optional `vfn_ip` addresss of the validator.
+```
+ip = "127.0.0.1"
+vfn_ip = "0.0.0.0"
+```
+
+## Client connection info.
+
+When using 0L tools you have the option of having one or more upstream nodes for your tools to get data and submit transactions.
+
+`upstream_nodes` contains the endpoint addresses of fullnodes which have the JSON RPC enabled (port `8080` by default).
+
+`upstream_nodes` is a list. The default behavior of the 0L tools is to RANDOMLY pick a URL from the list, and check if it is alive. The list feature exists for Carpe primarily, where random selection is important for balancing load across fullnodes. Testing all URLs at the time of submitting txs increases reliability for end-users.
+
+For Validators and Fullnode operators which want more control, there are other options for configuration.
+
+1. only include a single node in this array
+1. set the URL at runtime
+1. force the first URL from the list
+
+### Use a single RPC node
+Or you'll set a single RPC node in your upstream_nodes.
+
+```
+upstream_nodes = ["http://x.y.z.a:8080/"]
+```
+
+### Set URL at runtime
+1. You have one remote server you would like to use.
+
+You can explicitly set the URL when running the CLI command.
+`txs --url ...`
+
+### Force using the first URL
+1. You usually have a list of URLs you want to submit to. But there is one you use more often, or have more control over (e.g. localhost).
+
+You can explicitly set the URL when running the CLI command.
+`txs --use-first-url ...`
+
+Your configs may look like:
+```
+upstream_nodes = ["http://localhost:8080/", "http://x.y.z.a:8080/"]
+```
+
+# chain_info
+
+This section describes the chain you are connecting to. Importantly the the `base_waypoint` tells the client software what is the trusted state of the chain you are expected. Normally this will be initialized with the genesis waypoint.
+
+Without a valid waypoint client connections will fail. Waypoint can be found on blockexplorers like `http://0lscan.io`.
+
+Note that currently Diem code, you'll need to use a waypoint within the last 100 epochs.
+
+# tx_configs
+
+Here you can set the properties for categories of transaction types. For example the timeout and price you would bid for mining transactions.
+
+The categories are:
+`baseline_cost`, `critical_txs_cost`, `management_txs_cost`, `miner_txs_cost`, `cheap_txs_cost`.
diff --git a/ol/documentation/cli_cheatsheet.md → ol/documentation/cli-tools/cli_cheatsheet.md b/ol/documentation/cli_cheatsheet.md → ol/documentation/cli-tools/cli_cheatsheet.md
diff --git a/...umentation/delay_towers/delay_towers_0.md → ...umentation/delay-towers/delay_towers_0.md b/...umentation/delay_towers/delay_towers_0.md → ...umentation/delay-towers/delay_towers_0.md
diff --git a/...umentation/delay_towers/delay_towers_1.md → ...umentation/delay-towers/delay_towers_1.md b/...umentation/delay_towers/delay_towers_1.md → ...umentation/delay-towers/delay_towers_1.md
diff --git a/...umentation/delay_towers/delay_towers_2.md → ...umentation/delay-towers/delay_towers_2.md b/...umentation/delay_towers/delay_towers_2.md → ...umentation/delay-towers/delay_towers_2.md
diff --git a/...umentation/delay_towers/delay_towers_3.md → ...umentation/delay-towers/delay_towers_3.md b/...umentation/delay_towers/delay_towers_3.md → ...umentation/delay-towers/delay_towers_3.md
diff --git a/ol/onboard/src/commands/wizard_fork_cmd.rs b/ol/onboard/src/commands/wizard_fork_cmd.rs
@@ -17,9 +17,10 @@ use ol_types::block::VDFProof;
 use ol_types::config::IS_TEST;
 use ol_types::{account::ValConfigs, config::TxType, pay_instruction::PayInstruction};
 use reqwest::Url;
+use txs::tx_params::TxParams;
 use std::process::exit;
 use std::{fs::File, io::Write, path::PathBuf};
-use txs::{commands::autopay_batch_cmd, submit_tx};
+use txs::commands::autopay_batch_cmd;
 /// `validator wizard` subcommand
 #[derive(Command, Debug, Default, Options)]
 pub struct ForkCmd {
@@ -228,12 +229,11 @@ pub fn get_autopay_batch(
     )
     .unwrap();
     let script_vec = autopay_batch_cmd::process_instructions(instr_vec.clone());
-    let url = cfg.what_url(false);
-    let mut tx_params = submit_tx::get_tx_params_from_toml(
+    let mut tx_params = TxParams::get_tx_params_from_toml(
         cfg.to_owned(),
         TxType::Miner,
         Some(wallet),
-        url,
+        "0.0.0.0".parse().unwrap(), // this doesn't matter for onboarding autopay signatures.
         None,
         is_swarm,
     )

diff --git a/ol/onboard/src/commands/wizard_val_cmd.rs b/ol/onboard/src/commands/wizard_val_cmd.rs
@@ -16,10 +16,11 @@ use ol_types::config::IS_TEST;
 use ol_types::fixtures;
 use ol_types::{account::ValConfigs, config::TxType, pay_instruction::PayInstruction};
 use reqwest::Url;
+use txs::tx_params::TxParams;
 use std::fs;
 use std::process::exit;
 use std::{fs::File, io::Write, path::PathBuf};
-use txs::{commands::autopay_batch_cmd, submit_tx};
+use txs::commands::autopay_batch_cmd;
 
 /// `validator wizard` subcommand
 #[derive(Command, Debug, Default, Options)]
@@ -227,12 +228,11 @@ pub fn get_autopay_batch(
 
 
     let script_vec = autopay_batch_cmd::process_instructions(instr_vec.clone());
-    let url = cfg.what_url(false);
-    let mut tx_params = submit_tx::get_tx_params_from_toml(
+    let mut tx_params = TxParams::get_tx_params_from_toml(
         cfg.to_owned(),
         TxType::Miner,
         Some(wallet),
-        url,
+        "http://0.0.0.0".parse().unwrap(), // this doesn't matter for onboarding autopay signatures.
         None,
         is_swarm,
     )