diff --git a/Cargo.lock b/Cargo.lock index 2a61c35f0a5e..b206a17330af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6781,6 +6781,7 @@ dependencies = [ "proptest", "proptest-arbitrary-interop", "ratatui", + "reqwest", "reth-beacon-consensus", "reth-chainspec", "reth-cli", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index fb86a8ced2b3..8028dbba54f9 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -91,9 +91,9 @@ futures.workspace = true # misc aquamarine.workspace = true -eyre.workspace = true -clap = { workspace = true, features = ["derive", "env"] } backon.workspace = true +clap = { workspace = true, features = ["derive", "env"] } +eyre.workspace = true similar-asserts.workspace = true [dev-dependencies] diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index a146a88b724d..90e9fa37d12d 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -9,7 +9,7 @@ use clap::{value_parser, Parser, Subcommand}; use reth_chainspec::ChainSpec; use reth_cli::chainspec::ChainSpecParser; use reth_cli_commands::{ - config_cmd, db, dump_genesis, import, init_cmd, init_state, + config_cmd, db, download, dump_genesis, import, init_cmd, init_state, node::{self, NoArgs}, p2p, prune, recover, stage, }; @@ -169,6 +169,9 @@ impl, Ext: clap::Args + fmt::Debug> Cl Commands::Db(command) => { runner.run_blocking_until_ctrl_c(command.execute::()) } + Commands::Download(command) => { + runner.run_blocking_until_ctrl_c(command.execute::()) + } Commands::Stage(command) => runner.run_command_until_exit(|ctx| { command.execute::( ctx, @@ -221,6 +224,9 @@ pub enum Commands { /// Database debugging utilities #[command(name = "db")] Db(db::Command), + /// Downloads and optionally decompresses node snapshots from a URL + #[command(name = "download")] + Download(download::Command), /// Manipulate individual stages. #[command(name = "stage")] Stage(stage::Command), diff --git a/book/SUMMARY.md b/book/SUMMARY.md index f93daeaba397..c76002d72955 100644 --- a/book/SUMMARY.md +++ b/book/SUMMARY.md @@ -48,6 +48,7 @@ - [`reth db clear static-file`](./cli/reth/db/clear/static-file.md) - [`reth db version`](./cli/reth/db/version.md) - [`reth db path`](./cli/reth/db/path.md) + - [`reth download`](./cli/reth/download.md) - [`reth stage`](./cli/reth/stage.md) - [`reth stage run`](./cli/reth/stage/run.md) - [`reth stage drop`](./cli/reth/stage/drop.md) diff --git a/book/cli/SUMMARY.md b/book/cli/SUMMARY.md index 5f338a0d1ec7..759b1b8ddb9e 100644 --- a/book/cli/SUMMARY.md +++ b/book/cli/SUMMARY.md @@ -18,6 +18,7 @@ - [`reth db clear static-file`](./reth/db/clear/static-file.md) - [`reth db version`](./reth/db/version.md) - [`reth db path`](./reth/db/path.md) + - [`reth download`](./reth/download.md) - [`reth stage`](./reth/stage.md) - [`reth stage run`](./reth/stage/run.md) - [`reth stage drop`](./reth/stage/drop.md) diff --git a/book/cli/reth.md b/book/cli/reth.md index 70a1dec4dace..60117080c881 100644 --- a/book/cli/reth.md +++ b/book/cli/reth.md @@ -15,6 +15,7 @@ Commands: import This syncs RLP encoded blocks from a file dump-genesis Dumps genesis block JSON configuration to stdout db Database debugging utilities + download Downloads and optionally decompresses node snapshots from a URL stage Manipulate individual stages p2p P2P Debugging utilities config Write config to stdout diff --git a/book/cli/reth/download.md b/book/cli/reth/download.md new file mode 100644 index 000000000000..b39112020c2a --- /dev/null +++ b/book/cli/reth/download.md @@ -0,0 +1,132 @@ +# reth download + +Downloads and optionally decompresses node snapshots from a URL + +```bash +$ reth download --help +``` +```txt +Usage: reth download [OPTIONS] --url + +Options: + --chain + The chain this node is running. + Possible values are either a built-in chain or the path to a chain specification file. + + Built-in chains: + mainnet, sepolia, holesky, dev + + [default: mainnet] + + --instance + Add a new instance of a node. + + Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine. + + Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other. + + Changes to the following port numbers: - `DISCOVERY_PORT`: default + `instance` - 1 - `AUTH_PORT`: default + `instance` * 100 - 100 - `HTTP_RPC_PORT`: default - `instance` + 1 - `WS_RPC_PORT`: default + `instance` * 2 - 2 + + [default: 1] + + -h, --help + Print help (see a summary with '-h') + +Datadir: + --datadir + The path to the data dir for all reth files and subdirectories. + + Defaults to the OS-specific data directory: + + - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` + - Windows: `{FOLDERID_RoamingAppData}/reth/` + - macOS: `$HOME/Library/Application Support/reth/` + + [default: default] + + --datadir.static-files + The absolute path to store static files in. + + -u, --url + Custom URL to download the snapshot from + + -d, --decompress + Whether to automatically decompress the snapshot after downloading + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ] + + --log.file.format + The format to use for logs written to the log file + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled + + [default: 5] + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + [default: always] + + Possible values: + - always: Colors on + - auto: Colors on + - never: Colors off + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output +``` \ No newline at end of file diff --git a/crates/cli/commands/Cargo.toml b/crates/cli/commands/Cargo.toml index 2220efda5c6e..23e0a3e55460 100644 --- a/crates/cli/commands/Cargo.toml +++ b/crates/cli/commands/Cargo.toml @@ -61,18 +61,19 @@ tokio.workspace = true # misc ahash = "0.8" -human_bytes = "0.4.1" -eyre.workspace = true +backon.workspace = true clap = { workspace = true, features = ["derive", "env"] } +eyre.workspace = true +human_bytes = "0.4.1" +reqwest.workspace = true serde.workspace = true serde_json.workspace = true -tracing.workspace = true -backon.workspace = true secp256k1 = { workspace = true, features = [ "global-context", "rand-std", "recovery", ] } +tracing.workspace = true # io fdlimit.workspace = true diff --git a/crates/cli/commands/src/download.rs b/crates/cli/commands/src/download.rs new file mode 100644 index 000000000000..e26cbb041854 --- /dev/null +++ b/crates/cli/commands/src/download.rs @@ -0,0 +1,112 @@ +use std::{io::Write, path::Path, process::Command as ProcessCommand, sync::Arc}; +use tokio::{fs, io::AsyncWriteExt}; + +use clap::Parser; +use eyre::Result; +use reqwest::Client; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; +use reth_cli::chainspec::ChainSpecParser; +use reth_node_core::args::DatadirArgs; + +const SNAPSHOT_FILE: &str = "snapshot.tar.lz4"; + +/// `reth download` command +#[derive(Debug, Parser, Clone)] +pub struct Command { + /// The chain this node is running. + /// + /// Possible values are either a built-in chain or the path to a chain specification file. + #[arg( + long, + value_name = "CHAIN_OR_PATH", + long_help = C::help_message(), + default_value = C::SUPPORTED_CHAINS[0], + value_parser = C::parser() + )] + chain: Arc, + + /// Path where will be stored the snapshot + #[command(flatten)] + datadir: DatadirArgs, + + /// Custom URL to download the snapshot from + #[arg(long, short, required = true)] + url: String, + + /// Whether to automatically decompress the snapshot after downloading + #[arg(long, short)] + decompress: bool, +} + +impl> Command { + /// Downloads and saves the snapshot from the specified URL + pub async fn execute(self) -> Result<()> { + let data_dir = self.datadir.resolve_datadir(self.chain.chain()); + let snapshot_path = data_dir.data_dir().join(SNAPSHOT_FILE); + fs::create_dir_all(&data_dir).await?; + + println!("Starting snapshot download for chain: {:?}", self.chain.chain()); + println!("Target directory: {:?}", data_dir.data_dir()); + println!("Source URL: {}", self.url); + + download_snapshot(&self.url, &snapshot_path).await?; + + println!("Snapshot downloaded successfully to {:?}", snapshot_path); + if self.decompress { + println!("Decompressing snapshot..."); + decompress_snapshot(&snapshot_path, data_dir.data_dir())?; + println!("Snapshot decompressed successfully"); + + // Clean up compressed file + fs::remove_file(&snapshot_path).await?; + } else { + println!( + "Please extract the snapshot using: tar --use-compress-program=lz4 -xf {:?}", + snapshot_path + ); + } + + Ok(()) + } +} + +// Downloads a file from the given URL to the specified path, displaying download progress. +async fn download_snapshot(url: &str, target_path: &Path) -> Result<()> { + let client = Client::new(); + let mut response = client.get(url).send().await?.error_for_status()?; + + let total_size = response.content_length().unwrap_or(0); + let mut file = fs::File::create(&target_path).await?; + let mut downloaded = 0u64; + + while let Some(chunk) = response.chunk().await? { + file.write_all(&chunk).await?; + downloaded += chunk.len() as u64; + + if total_size > 0 { + let progress = (downloaded as f64 / total_size as f64) * 100.0; + print!("\rDownloading... {:.1}%", progress); + std::io::stdout().flush()?; + } + } + println!("\nDownload complete!"); + + Ok(()) +} + +// Helper to decompress snapshot file using lz4 +fn decompress_snapshot(snapshot_path: &Path, target_dir: &Path) -> Result<()> { + let status = ProcessCommand::new("tar") + .arg("--use-compress-program=lz4") + .arg("-xf") + .arg(snapshot_path) + .arg("-C") + .arg(target_dir) + .status()?; + + if !status.success() { + return Err(eyre::eyre!("Failed to decompress snapshot")); + } + + Ok(()) +} diff --git a/crates/cli/commands/src/lib.rs b/crates/cli/commands/src/lib.rs index 166ea438fb97..435301d272f5 100644 --- a/crates/cli/commands/src/lib.rs +++ b/crates/cli/commands/src/lib.rs @@ -11,6 +11,7 @@ pub mod common; pub mod config_cmd; pub mod db; +pub mod download; pub mod dump_genesis; pub mod import; pub mod init_cmd;