From c362d1a9a943d0c6d23e1de9551e0bcacf558416 Mon Sep 17 00:00:00 2001 From: Bridgerz Date: Thu, 23 May 2024 14:34:55 +0200 Subject: [PATCH] Add config file --- Cargo.lock | 2 + crates/sui-bridge-indexer/Cargo.toml | 2 + crates/sui-bridge-indexer/src/config.rs | 57 +++++++---------------- crates/sui-bridge-indexer/src/config.yaml | 18 +++++++ crates/sui-bridge-indexer/src/main.rs | 34 ++++++++++---- 5 files changed, 66 insertions(+), 47 deletions(-) create mode 100644 crates/sui-bridge-indexer/src/config.yaml diff --git a/Cargo.lock b/Cargo.lock index 9f8060804f3a1..137bdd8153937 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12052,6 +12052,8 @@ dependencies = [ "hex-literal 0.3.4", "mysten-metrics", "prometheus", + "serde", + "serde_yaml 0.8.26", "sui-bridge", "sui-config", "sui-data-ingestion-core", diff --git a/crates/sui-bridge-indexer/Cargo.toml b/crates/sui-bridge-indexer/Cargo.toml index 403313476d567..66833b7044eea 100644 --- a/crates/sui-bridge-indexer/Cargo.toml +++ b/crates/sui-bridge-indexer/Cargo.toml @@ -7,6 +7,7 @@ publish = false edition = "2021" [dependencies] +serde.workspace = true diesel = { version = "2.1.4", features = ["postgres", "r2d2", "serde_json"] } ethers = "2.0" tokio = { workspace = true, features = ["full"] } @@ -21,6 +22,7 @@ bin-version.workspace = true anyhow.workspace = true mysten-metrics.workspace = true bcs.workspace = true +serde_yaml.workspace = true [dev-dependencies] sui-types = { workspace = true, features = ["test-utils"] } diff --git a/crates/sui-bridge-indexer/src/config.rs b/crates/sui-bridge-indexer/src/config.rs index 0f2ab39ac0064..cbfcf3e76a97b 100644 --- a/crates/sui-bridge-indexer/src/config.rs +++ b/crates/sui-bridge-indexer/src/config.rs @@ -1,47 +1,26 @@ +use anyhow::Result; +use std::{fs, path::Path}; + // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use serde::Deserialize; -use clap::*; -use std::path::PathBuf; - -#[derive(Parser, Clone, Debug)] -#[clap( - name = "Bridge Indexer", - about = "Run an indexer for the bridge.\n\ - It uses the data ingestion framework to read sui transactions and listens\n\ - to Ethereum events in order to generate data related to the bridge.\n\ - Data is written to postgres tables and can be used for dashboards and general checks\n\ - on bridge health.", - rename_all = "kebab-case" -)] -pub struct BridgeIndexerConfig { - /// URL of the sui remote store. - #[clap(long, short = 'r', required = true)] - pub remote_store_url: Option, - /// URL for Eth fullnode. - #[clap(long, short = 'e', required = true)] +/// config as loaded from `config.yaml`. +#[derive(Debug, Deserialize)] +pub struct Config { + pub remote_store_url: String, pub eth_rpc_url: String, - /// URL of the DB instance holding indexed bridge data. - #[clap(long, short = 'd', required = true)] pub db_url: String, - /// Path to the file where the progress store is stored. - #[clap( - long, - short = 'p', - default_value = "/tmp/progress_store", - global = true - )] - pub progress_store_file: PathBuf, - /// Path to the directory where the checkpoints are stored. - #[clap(long, short = 'c', default_value = "/tmp", global = true)] - pub checkpoints_path: PathBuf, - /// Number of worker threads to use. - #[clap(long, short = 't', default_value = "1", global = true)] - pub concurrency: usize, - /// Address of the SuiBridge contract - #[clap(long, required = true, short = 'a')] + pub progress_store_file: String, + pub checkpoints_path: String, + pub concurrency: u64, pub eth_sui_bridge_contract_address: String, - /// Block to start indexing from - #[clap(long, required = true, short = 's')] pub start_block: u64, } + +/// Load the config to run. +pub fn load_config(path: &Path) -> Result { + let reader = fs::File::open(path)?; + let config: Config = serde_yaml::from_reader(reader)?; + Ok(config) +} diff --git a/crates/sui-bridge-indexer/src/config.yaml b/crates/sui-bridge-indexer/src/config.yaml new file mode 100644 index 0000000000000..f4644c7a7c71e --- /dev/null +++ b/crates/sui-bridge-indexer/src/config.yaml @@ -0,0 +1,18 @@ +# config.yaml format: + +# URL of the remote store +# remote_store_url: +# URL for Ethereum RPC +# eth_rpc_url: +# Database connection URL +# db_url: +# File to store the progress +# progress_store_file: +# Path to the checkpoints +# checkpoints_path: +# Number of concurrent operations +# concurrency: 1 +# Ethereum to Sui bridge contract address +# eth_sui_bridge_contract_address: +# Starting block number +# start_block: \ No newline at end of file diff --git a/crates/sui-bridge-indexer/src/main.rs b/crates/sui-bridge-indexer/src/main.rs index 09210840ffb56..03377628e08a1 100644 --- a/crates/sui-bridge-indexer/src/main.rs +++ b/crates/sui-bridge-indexer/src/main.rs @@ -2,16 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::Result; -use clap::Parser; +use clap::*; use ethers::types::Address as EthAddress; use mysten_metrics::spawn_logged_monitored_task; use mysten_metrics::start_prometheus_server; use prometheus::Registry; use std::collections::HashMap; use std::collections::HashSet; +use std::env; use std::net::IpAddr; use std::net::Ipv4Addr; use std::net::SocketAddr; +use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; use sui_bridge::{ @@ -21,7 +23,7 @@ use sui_bridge::{ }; use sui_bridge_indexer::postgres_writer::get_connection_pool; use sui_bridge_indexer::{ - config::BridgeIndexerConfig, worker::process_eth_transaction, worker::BridgeWorker, + config::load_config, worker::process_eth_transaction, worker::BridgeWorker, }; use sui_data_ingestion_core::{ DataIngestionMetrics, FileProgressStore, IndexerExecutor, ReaderOptions, WorkerPool, @@ -29,10 +31,26 @@ use sui_data_ingestion_core::{ use tokio::sync::oneshot; use tracing::info; +#[derive(Parser, Clone, Debug)] +#[clap(group(ArgGroup::new("input").required(true).args(&["config_path"])))] +struct Args { + /// Path to a yaml config + #[clap(long, short, default_value = "config.yaml")] + config_path: Option, +} + #[tokio::main] async fn main() -> Result<()> { - let config = BridgeIndexerConfig::parse(); - info!("Parsed config: {:#?}", config); + let args = Args::parse(); + + // load config + let config_path = if let Some(path) = args.config_path { + path.join("config.yaml") + } else { + env::current_dir().unwrap().join("config.yaml") + }; + + let config = load_config(&config_path).unwrap(); // start metrics server let (_exit_sender, exit_receiver) = oneshot::channel(); @@ -88,18 +106,18 @@ async fn main() -> Result<()> { ); // start sui side - let progress_store = FileProgressStore::new(config.progress_store_file); + let progress_store = FileProgressStore::new(config.progress_store_file.into()); let mut executor = IndexerExecutor::new(progress_store, 1 /* workflow types */, metrics); let worker_pool = WorkerPool::new( BridgeWorker::new(vec![], config.db_url.clone()), "bridge worker".into(), - config.concurrency, + config.concurrency as usize, ); executor.register(worker_pool).await?; executor .run( - config.checkpoints_path, - config.remote_store_url, + config.checkpoints_path.into(), + Some(config.remote_store_url), vec![], // optional remote store access options ReaderOptions::default(), exit_receiver,