Literal pubkeys in agent config, v2.0.0 (#77)

* solana.rs: Modify KeyStore config to accept literal pubkeys

This **breaking** change improves on the legacy file-based keystore
appoach to program/mapping public keys. With it, the user can name the
public keys in-line, e.g.:
'key_store.program_key_path = "program_key.json"'
is now expressed as:
'key_store.program_key = "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH"'

This change is accompanied with a small tool to inline the public keys
from an older file-based config, named "agent-migrate-config".

The sample configs are rewritten to use the new format and name the
appropriate pubkeys out-of-the-box.

Tests include a sanity-check for the migration tool.

* README.md: Include key store migration information for v2.0.0

* README.md: add pipe for migrator output

* test_integration.py: Add missing manual agent invocation

This is necessary to provide the agent with an altered migrated
config.

* README.md: typo

* solana.rs: strinigified -> stringified

We don't like our SOL addresses strinigified.

* config.sample.pythtest.toml: The perils of Ctrl-C + Ctrl-V

Cargo.toml

@@ -1,12 +1,16 @@
 [package]
 name = "pyth-agent"
-version = "1.4.0"
+version = "2.0.0"
 edition = "2021"
 
 [[bin]]
 name = "agent"
 path = "src/bin/agent.rs"
 
+[[bin]]
+name = "agent-migrate-config"
+path = "src/bin/agent_migrate_config.rs"
+
 [dependencies]
 anyhow = "1.0.55"
 serde = { version = "1.0.136", features = ["derive"] }
@@ -46,6 +50,7 @@ typed-html = { git = "https://github.com/bodil/typed-html", rev = "4c13ecca" }
 humantime = "2.1.0"
 prometheus-client = "0.19.0"
 lazy_static = "1.4.0"
+toml_edit = "0.19.13"
 
 [dev-dependencies]
 tokio-util = { version = "0.7.0", features = ["full"] }

README.md

@@ -32,24 +32,46 @@ The logging level can be configured at runtime
 through the `RUST_LOG` environment variable using the standard
 `error|warn|info|debug|trace` levels.
 
-### Key Store
-If you already have a key store set up, you can skip this step. If you haven't, you will need to create one before publishing data. A key store contains the cryptographic keys needed to publish data. Once you have a key store set up, please ensure that the configuration file mentioned above contains the correct path to your key store.
+### Key Store Config Migration [v1.x.x LEGACY]
+Pyth agent v2.0.0 introduces a simplified program and mapping key configuration. This breaking change alters how you define program/mapping key options in your agent config:
+```toml
+# Old v1.x.x way
+[primary network]
+key_store.root_path = "/path/to/keystore"
+key_store.publish_keypair_path = "publish_key_pair.json" # Relative path from root_path, "publish_key_pair.json" by default
+key_store.program_key_path = "program_key.json" # Relative path from root_path, "program_key.json" by default
+key_store.mapping_key_path = "mapping_key.json" # Relative path from root_path, "mapping_key.json" by default
+
+# [...]
+
+# New v2.0.0 way
+[primary_network]
+key_store.publish_keypair_path = "/path/to/keypair.json" # The root_path is gone, we specify the full path
+# Not using separate files anymore
+key_store.program_key = "LiteralProgramPubkeyInsideTheConfig" # contents of legacy program_key.json;
+key_store.mapping_key = "LiteralMappingPubkeyInsideTheConfig" # contents of legacy mapping_key.json
+
+# [...]
 
-```bash
-# Install the Solana Tool Suite, needed for creating the key used to sign your transactions.
-sh -c "$(curl -sSfL https://release.solana.com/v1.14.13/install)"
+```
 
-# Create the key store directory. This can be any location that is convenient for you.
-PYTH_KEY_STORE=$HOME/.pythd
+#### Automatic Migration
+If you are upgrading to agent v2.0.0 with an existing config, you can use the provided automatic migrator program:
+```shell
+# Build
+$ cargo build --release
+# Run the migrator, making sure that the key store with previous keys is reachable
+$ target/release/agent-migrate-config -c <existing_config_file>.toml > my_new_config.toml
+```
 
-# Create your keypair (pair of private/public keys) that will be used to sign your transactions.
-# Pyth Network will need to permission this key, so reach out to us once you have created it.
-solana-keygen new --no-bip39-passphrase --outfile $PYTH_KEY_STORE/publish_key_pair.json
+#### `Could not open {mapping|program|...} key file`
+This error can appear if some of your program/mapping/publish key
+files are not reachable under their `key_store.*` setting values.
 
-# Initialize the key store with the public keys of the Pyth Oracle Program on the network you wish to publish to.
-PYTH_KEY_ENV=devnet # Can be devnet, testnet or mainnet
-./scripts/init_key_store.sh $PYTH_KEY_ENV $PYTH_KEY_STORE
-```
+Ensure that your current working directory is correct for reaching the
+key store path inside your config. You may also migrate manually by
+changing `key_store.*_key_path` and `key_store.publish_keypair_path`
+options by hand, as described in the config example above.
 
 ## Run
 `cargo run --release -- --config <your_config.toml>` will build and run the agent in a single step.

config/config.sample.pythnet.toml

@@ -11,8 +11,17 @@ rpc_url = "http://api.pythnet.pyth.network:8899"
 # This can be omitted when oracle.subscriber_enabled is set to false.
 wss_url = "ws://api.pythnet.pyth.network:8900"
 
-# Path to the key store.
-key_store.root_path = "/path/to/keystore"
+# Path to your publishing keypair.
+key_store.publish_keypair_path = "/path/to/keypair.json"
+
+# Oracle program pubkey
+key_store.program_key = "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH"
+
+# Oracle mapping pubkey
+key_store.mapping_key = "AHtgzX45WTKfkPG53L6WYhGEXwQkN1BVknET3sVsLL8J"
+
+# Pythnet accumulator key
+key_store.accumulator_key = "7Vbmv1jt4vyuqBZcpYPpnVhrqVe5e6ZPb6JxDcffRHUM"
 
 # Duration of the interval at which to publish updates
 exporter.publish_interval_duration = "400ms"
@@ -25,8 +34,14 @@ exporter.publish_interval_duration = "400ms"
 rpc_url = "https://api.mainnet-beta.solana.com"
 wss_url = "wss://api.mainnet-beta.solana.com"
 
-# Path to the key store.
-key_store.root_path = "/path/to/keystore"
+# Path to your publishing keypair.
+key_store.publish_keypair_path = "/path/to/keypair.json"
+
+# Oracle program pubkey
+key_store.program_key = "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH"
+
+# Oracle mapping pubkey
+key_store.mapping_key = "AHtgzX45WTKfkPG53L6WYhGEXwQkN1BVknET3sVsLL8J"
 
 # Duration of the interval at which to publish updates. Default interval is 1 seconds.
 # exporter.publish_interval_duration = "1s"

config/config.sample.pythtest.toml

@@ -11,8 +11,19 @@ rpc_url = "https://api.pythtest.pyth.network"
 # This can be omitted when oracle.subscriber_enabled is set to false.
 wss_url = "wss://api.pythtest.pyth.network"
 
-# Path to the key store.
-key_store.root_path = "/path/to/keystore"
+# Path to your publishing keypair.
+key_store.publish_keypair_path = "/path/to/keypair.json"
+
+# Oracle program pubkey
+key_store.program_key = "8tfDNiaEyrV6Q1U4DEXrEigs9DoDtkugzFbybENEbCDz" # conformance
+# key_store.program_key = "gSbePebfvPy7tRqimPoVecS2UsBvYv46ynrzWocc92s" # cross-chain
+
+# Oracle mapping pubkey
+key_store.mapping_key = "AFmdnt9ng1uVxqCmqwQJDAYC5cKTkw8gJKSM5PnzuF6z" # conformance
+# key_store.mapping_key = "BmA9Z6FjioHJPpjT39QazZyhDRUdZy2ezwx4GiDdE2u2" # cross-chain
+
+# Pythtest accumulator key (only for the cross-chain oracle)
+# key_store.accumulator_key = "7Vbmv1jt4vyuqBZcpYPpnVhrqVe5e6ZPb6JxDcffRHUM"
 
 # Duration of the interval at which to publish updates
 exporter.publish_interval_duration = "400ms"
@@ -25,8 +36,14 @@ exporter.publish_interval_duration = "400ms"
 rpc_url = "https://api.testnet.solana.com"
 wss_url = "wss://api.testnet.solana.com"
 
-# Path to the key store.
-key_store.root_path = "/path/to/keystore"
+# Path to your publishing keypair.
+key_store.publish_keypair_path = "/path/to/keypair.json"
+
+# Oracle program pubkey
+key_store.program_key = "8tfDNiaEyrV6Q1U4DEXrEigs9DoDtkugzFbybENEbCDz"
+
+# Oracle mapping pubkey
+key_store.mapping_key = "AFmdnt9ng1uVxqCmqwQJDAYC5cKTkw8gJKSM5PnzuF6z"
 
 # Duration of the interval at which to publish updates. Default interval is 1 seconds.
 # exporter.publish_interval_duration = "1s"

config/config.toml

@@ -1,6 +1,9 @@
 # Configuration for the JRPC API Websocket Server
 [pythd_api_server]
-# The address on which the websocket API server will listen on.
+# The address on which the websocket API server will listen.
+#
+# NOTE: non-loopback addresses must be used carefully, making sure the
+# connection is not exposed for unauthorized access.
 listen_address = "127.0.0.1:8910"
 
 # Configuration for the primary network this agent will publish data to. In most cases this should be a Pythnet endpoint.
@@ -16,19 +19,32 @@ rpc_url = "https://api.pythtest.pyth.network"
 # Note that api.pythtest.pyth.network is a private endpoint: please contact us for access.
 wss_url = "wss://api.pythtest.pyth.network"
 
-# Path to the key store.
-key_store.root_path = "/path/to/keystore"
+# Path to the keypair used to publish price updates. If set to a
+# non-existent file path, the system expects a keypair to be loaded
+# via the remote keypair loader. If the path is valid, the remote
+# keypair loading is disabled.
+key_store.publish_keypair = "/path/to/keypair.json"
+
+# Public key of the oracle program
+key_store.program_key = "RelevantOracleProgramAddress"
+
+# Public key of the root mapping account
+key_store.mapping_key = "RelevantOracleMappingAddress"
+
+# Optional public key of the accumulator program (if provided)
+key_store.accumulator_key = "RelevantOracleAccumulatorAddress"
 
 ### Optional fields ###
 
 # [metrics_server]
 #
 # Where to serve the quick-access dashboard and metrics. Metrics live under "/metrics"
+# NOTE: non-loopback addresses must be used carefully, making sure the
+# connection is not exposed for unauthorized access.
 # bind_address = "127.0.0.1:8888"
 
 # [remote_keypair_loader}
 # Where to serve the remote keypair loading endpoint, under "/primary/load_keypair" and "/secondary/load_keypair"
-#
 # NOTE: non-loopback addresses must be used carefully, making sure the
 # connection is not exposed for unauthorized access.
 # bind_address = "127.0.0.1:9001"

integration-tests/agent_conf.toml

@@ -2,6 +2,5 @@
 bind_address="0.0.0.0:8888"
 
 [primary_network]
-key_store.root_path = "keystore"
 oracle.poll_interval_duration = "1s"
 exporter.transaction_monitor.poll_interval_duration = "1s"