Release 5.1.3 (#1125)

* downgrade error msg to info

* cleanup unused prometheus log code

* downgrade state sync error message to info

* makefile for set-waypoint

* patch makefile

* refresh onchain state

* genesis should assign vouches

* set layout for devnet has 4 nodes

* init val in genesis first

* github repo tools can put file in genesis repo

* save files to genesis repo

* reorganize testnet scripts

* cleanup

* documentation

* patch makefile

* makefile

* makefile

* text [skip-ci]

* update yaml configs so that node file uses multiple fullnode networks [skip-ci]

* patch [skip ci]

* patch ports [skip ci]

* validator init on genesis and ol init, will use seed peers

* trim whitespace on genesis_waypoint.txt

* makefile

* reverse state-sync default params

* add helper in init to reset safety data

* bump version

* ol restore has --boundary-only or -b which will not attempt to restore an advanced version. This is the safest restore option.

* ol restore defaults to only fetch the epoch boundary. --highest-version must be explicitly passed if the user wants versions beyond the epoch boundary (this is still experimental).

* ol serve: deprecating the --run-checks or -c option. We now assume the user wants to run checks while running the web-monitor

* improve error messages on safety rules initialize

* validator should panic on trying to startup safety rules

* patch

* makefile

* make testnet start with alice, bob, carol only

* add execution.genesis_file_location to fullnode yaml

* set layout to have 5 addresses for testnet genesis

* alway use cargo release mode even in testnet

* undo

* update fixture files with ip addresses for reference.

* patch validator yaml file generation

* ol start doesn't need to swtich between validator and fullnode mode with the new configs!

* make pubnet config use onchain discovery

* remove setting for fullnode identity in validator file

* rename ol restore option highest_version -> latest_version

Makefile

@@ -36,7 +36,7 @@ endif
 # Registration params
 REMOTE = 'backend=github;repository_owner=${GITHUB_USER};repository=${REPO_NAME};token=${DATA_PATH}/github_token.txt;namespace=${ACC}'
 
-GENESIS_REMOTE = 'backend=github;repository_owner=${REPO_ORG};repository=${REPO_NAME};token=${DATA_PATH}/github_token.txt;namespace=${ACC}'
+GENESIS_REMOTE = 'backend=github;repository_owner=${REPO_ORG};repository=${REPO_NAME};token=${DATA_PATH}/github_token.txt;namespace=${ACC};branch=master'
 
 LOCAL = 'backend=disk;path=${DATA_PATH}/key_store.json;namespace=${ACC}'
 
@@ -290,7 +290,7 @@ verify-gen:
 	--validator-backend ${LOCAL} \
 	--genesis-path ${DATA_PATH}/genesis.blob
 
-genesis:
+genesis: stdlib
 	cargo run -p diem-genesis-tool ${CARGO_ARGS} -- files \
 	--chain-id ${CHAIN_ID} \
 	--validator-backend ${LOCAL} \
@@ -309,10 +309,6 @@ start:
 # run in foreground. Only for testing, use a daemon for net.
 	RUST_LOG=error cargo run -p diem-node -- --config ${DATA_PATH}/validator.node.yaml
 
-# Start a fullnode instead of a validator node
-start-full:
-	cargo run -p diem-node -- --config ${DATA_PATH}/fullnode.node.yaml
-
 daemon:
 	mkdir -p ~/.config/systemd/user/
 	cp ./ol/util/diem-node.service ~/.config/systemd/user/
@@ -401,18 +397,16 @@ ifdef TEST
 	cp ./ol/devnet/set_layout_test.toml ${DATA_PATH}/set_layout.toml
 endif
 
-fix-genesis:
-	cp ./ol/devnet/genesis/${V}/genesis.blob ${DATA_PATH}/
-	cp ./ol/devnet/genesis/${V}/genesis_waypoint ${DATA_PATH}/
-
 
 #### HELPERS ####
 set-waypoint:
 	@if test -f ${DATA_PATH}/key_store.json; then \
 		jq -r '. | with_entries(select(.key|match("-oper/waypoint";"i")))[].value' ${DATA_PATH}/key_store.json > ${DATA_PATH}/client_waypoint; \
-		jq -r '. | with_entries(select(.key|match("-oper/genesis-waypoint";"i")))[].value' ${DATA_PATH}/key_store.json > ${DATA_PATH}/genesis_waypoint; \
+		jq -r '. | with_entries(select(.key|match("-oper/genesis-waypoint";"i")))[].value' ${DATA_PATH}/key_store.json > ${DATA_PATH}/genesis_waypoint.txt; \
 	fi
 
+	cargo r -p ol -- init --update-waypoint --waypoint $(shell cat ${DATA_PATH}/client_waypoint)
+
 	@echo client_waypoint:
 	@cat ${DATA_PATH}/client_waypoint
 
@@ -450,71 +444,65 @@ debug:
 	make smoke-onboard <<< $$'${MNEM}'
 
 
-##### DEVNET TESTS #####
+#### TESTNET #####
+# The testnet is started using the same tools as genesis to have a faithful reproduction of a network from a clean slate.
 
-devnet: clear fix dev-wizard dev-genesis start
-# runs a smoke test from fixtures. 
-# Uses genesis blob from fixtures, assumes 3 validators, and test settings.
-# This will work for validator nodes alice, bob, carol, and any fullnodes; 'eve'
+# 1. The first thing necessary is initializing testnet genesis validators. All genesis nodes need to set up environment variables for their namespace/personas e.g. NS=alice. Also the TEST=y mode must be set, as well as a chain environment e.g. NODE_ENV=test. These settings must be done manually, preferably in .bashrc
 
-dev-join: clear fix fix-genesis dev-wizard
-# REQUIRES MOCK GIT INFRASTRUCTURE: OLSF/dev-genesis OLSF/dev-epoch-archive
-# see `devnet-archive` below 
-# We want to simulate the onboarding/new validator fetching genesis files from the mock archive: dev-genesis-archive
+# 2. Next those validators will register config data to a github repo OLSD/dev-genesis. Note: there could be github http errors, if validators attempt to write the same resource simultaneously
 
-# mock restore backups from dev-epoch-archive
-	rm -rf ~/.0L/restore
-# restore from MOCK archive OLSF/dev-epoch-archive
-	cargo r -p ol -- restore
-# start a node with fullnode.node.yaml configs
-	make start-full
+# THESE STEPS ARE ACHIEVED WITH `make testnet-register`
 
-dev-wizard:
-#  REQUIRES there is a genesis.blob in the fixtures/genesis/<version> you are testing
-	MNEM='${MNEM}' cargo run -p onboard -- val --prebuilt-genesis ${DATA_PATH}/genesis.blob --skip-mining --chain-id 1 --genesis-ceremony
+# 3. Wait. All genesis nodes need to complete registration. Otherwise buidling a genesis.blob (the first block), will fail.
+# 4. Each genesis node builds the genesis file locally, and submits to the github repo. (this remote genesis file is what subsequent non-genesis validators will use to bootstrap their db).
+# 5. Genesis validators can start their nodes.
 
-#### DEVNET RESTART ####
-# usually do this on Alice, which has the dev-epoch-archive repo, and dev-genesis
+# THESE STEPS ARE ACHIEVED WITH  `make testnet`
 
-# Do the ceremony: and also save the genesis fixtures, needs to happen before fix.
-dev-register: clear fix dev-wizard gen-register
-# Do a dev genesis on each node after EVERY NODE COMPLETED registration.
-dev-genesis: genesis dev-save-genesis fix-genesis
 
-#### DEVNET INFRA ####
-# To make reproducible devnet files.
+# 6. Assuming there is progress in the block production, subsequent validators can join.
 
-# Save the files to mock infrastructure i.e. devnet github
-dev-infra: dev-backup-archive dev-commit
+# THIS IS ACHIEVED WITH: testnet-onboard
 
-dev-save-genesis: set-waypoint
-	rsync -a ${DATA_PATH}/genesis* ${SOURCE}/ol/devnet/genesis/${V}/
-	git add ${SOURCE}/ol/devnet/genesis/${V}/
 
-dev-backup-archive:
-	cd ${HOME}/dev-epoch-archive && make devnet-backup
+#### 1. TESTNET SETUP ####
 
-dev-commit:
-	git commit -a -m "save genesis fixtures to ${V}" | true
-	git push | true
+testnet-init: clear fix
+#  REQUIRES there is a genesis.blob in the fixtures/genesis/<version> you are testing
+	MNEM='${MNEM}' cargo run -p onboard -- val --skip-mining --chain-id 1 --genesis-ceremony
 
+# Do the genesis ceremony registration, this includes the step testnet-validator-init-wizard
+testnet-register:  testnet-init gen-register
+# Do a dev genesis on each node after EVERY NODE COMPLETED registration.
+
+# Makes the gensis file on each genesis validator, AND SAVES TO GITHUB so that other validators can be onboarded after genesis.
+testnet-genesis: genesis set-waypoint
+	cargo run -p diem-genesis-tool ${CARGO_ARGS} -- create-repo \
+	--publish-genesis ${DATA_PATH}/genesis.blob \
+	--shared-backend ${GENESIS_REMOTE}
+
+	cargo run -p diem-genesis-tool ${CARGO_ARGS} -- create-repo \
+	--publish-genesis ${DATA_PATH}/genesis_waypoint.txt \
+	--shared-backend ${GENESIS_REMOTE}
+
+#### 2. TESTNET START ####
+
+# Do this to restart the network with new code. Assumes a registration has been completed, and the genesis validators are unchanged. If new IP addresses or number of genesis nodes changed, you must RERUN SETUP below.
+# - builds stdlib from source
+# - clears many of the home files
+# - adds fixtures
+# - initializes node configs
+# - rebuids genesis files and shares to github genesis repo
+# - starts node in validator mode
+testnet: clear fix testnet-init testnet-genesis start
+
+# For subsequent validators joining the testnet. This will fetch the genesis information saved
+testnet-onboard: clear fix
+	MNEM='${MNEM}' cargo run -p onboard -- val --github-org OLSF --repo dev-genesis --chain-id 1
+# start a node with fullnode.node.yaml configs
+	cargo r -p diem-node -- -f ~/.0L/fullnode.node.yaml
 
-TAG=$(shell git tag -l "previous")
-clean-tags:
-	git push origin --delete ${TAG}
-	git tag -d ${TAG}
-
-nuke-testnet:
-	@echo WIPING EVERYTHING but keeping: github_token.txt, autopay_batch.json, set_layout.toml, /vdf_proofs/proof_0.json
 
-	@if test -d ${DATA_PATH}; then \
-		cd ${DATA_PATH} && cp github_token.txt autopay_batch.json set_layout.toml vdf_proofs/proof_0.json ~/; \
-		cd ${DATA_PATH} && rm -rf *; \
-		cd ~ && cp github_token.txt autopay_batch.json set_layout.toml ${DATA_PATH}; \
-		cd ${DATA_PATH} && mkdir vdf_proofs;\
-		cd ~ && cp proof_0.json ${DATA_PATH}/vdf_proofs/; \
-	fi
-
 
 ####### SWARM ########
 
@@ -566,6 +554,13 @@ fork-config:
 	cargo run -p onboard -- fork -u http://167.172.248.37 --prebuilt-genesis ${DATA_PATH}/genesis_from_snapshot.blob
 
 # start node from files
-fork-start: 
+fork-start:
 	rm -rf ~/.0L/db
 	cargo run -p libra-node -- --config ~/.0L/validator.node.yaml
+
+##### UTIL #####
+TAG=$(shell git tag -l "previous")
+clean-tags:
+	git push origin --delete ${TAG}
+	git tag -d ${TAG}
+

config/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "diem-config"
-version = "5.1.2"
+version = "5.1.3"
 authors = ["Diem Association <[email protected]>"]
 description = "Diem diem-config"
 repository = "https://github.com/diem/diem"

config/global-constants/src/lib.rs

@@ -43,7 +43,6 @@ pub const DEFAULT_VAL_PORT: u64 = 6180;
 pub const DEFAULT_VFN_PORT: u64 = 6179;
 pub const DEFAULT_PUB_PORT: u64 = 6178;
 
-
 // TODO: make this lazy static.
 /// Switch settings between production and testing
 pub fn delay_difficulty() -> u64 {

config/management/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "diem-management"
-version = "5.1.2"
+version = "5.1.3"
 authors = ["Diem Association <[email protected]>"]
 description = "Diem Management is a tool used to manage the configuration of a Diem Node"
 repository = "https://github.com/diem/diem"

config/management/genesis/Cargo.toml

@@ -47,6 +47,7 @@ serde_json = "1"
 diem-github-client = { path = "../../../secure/storage/github" }
 ol-types = {path = "../../../ol/types"}
 serde_yaml = "0.8.17"
+base64 = "0.13.0"
 
 [dev-dependencies]
 diem-config = { path = "../..", features = ["fuzzing"]}

config/management/genesis/src/ol_create_repo.rs

@@ -3,8 +3,9 @@
 
 use diem_github_client::Client;
 use diem_management::{config::ConfigPath, error::Error, secure_backend::SharedBackend};
-use std::process::exit;
+use std::{process::exit, path::PathBuf, fs::File};
 use structopt::StructOpt;
+use std::io::prelude::*;
 
 /// Note, it is implicitly expected that the storage supports
 /// a namespace but one has not been set.
@@ -17,13 +18,15 @@ pub struct CreateGenesisRepo {
     #[structopt(flatten)]
     pub backend: SharedBackend,
     #[structopt(long)]
-    pub repo_owner: String,
+    pub repo_owner: Option<String>,
     #[structopt(long)]
-    pub repo_name: String,
+    pub repo_name: Option<String>,
     #[structopt(long)]
     pub pull_request_user: Option<String>,
     #[structopt(long)]
     pub delete_repo_user: Option<String>,
+    #[structopt(long)]
+    pub publish_genesis: Option<PathBuf>,
 }
 
 impl CreateGenesisRepo {
@@ -41,12 +44,29 @@ impl CreateGenesisRepo {
                 let github = Client::new(
                     config.repository_owner.clone(),
                     config.repository.clone(),
-                    config.branch.unwrap_or("main".to_string()),
+                    config.branch.unwrap_or("master".to_string()),
                     config
                         .token
                         .read_token()
                         .expect("could not get github token"),
                 );
+
+                if let Some(p) = self.publish_genesis {
+
+
+                  let mut file = File::open(&p)
+                  .expect("cannot read file");
+
+                  let mut bytes = Vec::new();
+                  file.read_to_end(&mut bytes).expect("could not read file");
+                  let base64_encoded = base64::encode(bytes);
+
+                  let repo_file_path = format!("genesis/{}", p.file_name().unwrap().to_str().unwrap());
+
+                  github.put(&repo_file_path, &base64_encoded).expect("could not put file in github repo");
+
+                  return Ok(format!("published file to genesis repo at {:?}", &repo_file_path));
+                }
                 // Make a pull request of the the forked repo, back to the genesis coordination repository.
                 if let Some(user) = self.pull_request_user {
                     match github.make_genesis_pull_request(&config.repository_owner, &config.repository, &user) {
@@ -68,8 +88,8 @@ impl CreateGenesisRepo {
                     }
                 } else {
                 // Fork the genesis coordination repo into a personal repo
-                    match github.fork_genesis_repo(&self.repo_owner, &self.repo_name) {
-                        Ok(_) => Ok(format!("Created new repo {}", &self.repo_name)),
+                    match github.fork_genesis_repo(&self.repo_owner.unwrap(), &self.repo_name.as_ref().unwrap()) {
+                        Ok(_) => Ok(format!("Created new repo {}", &self.repo_name.unwrap())),
                         Err(e) => Err(Error::StorageWriteError(
                             "github",
                             "fork genesis repo",
@@ -85,3 +105,4 @@ impl CreateGenesisRepo {
         }
     }
 }
+