feat: add cli download to download public node snapshots

bin/reth/Cargo.toml

@@ -91,9 +91,9 @@ futures.workspace = true
 
 # misc
 aquamarine.workspace = true
-eyre.workspace = true
-clap = { workspace = true, features = ["derive", "env"] }
 backon.workspace = true
+clap = { workspace = true, features = ["derive", "env"] }
+eyre.workspace = true
 similar-asserts.workspace = true
 
 [dev-dependencies]

bin/reth/src/cli/mod.rs

@@ -9,7 +9,7 @@ use clap::{value_parser, Parser, Subcommand};
 use reth_chainspec::ChainSpec;
 use reth_cli::chainspec::ChainSpecParser;
 use reth_cli_commands::{
-    config_cmd, db, dump_genesis, import, init_cmd, init_state,
+    config_cmd, db, download, dump_genesis, import, init_cmd, init_state,
     node::{self, NoArgs},
     p2p, prune, recover, stage,
 };
@@ -169,6 +169,9 @@ impl<C: ChainSpecParser<ChainSpec = ChainSpec>, Ext: clap::Args + fmt::Debug> Cl
             Commands::Db(command) => {
                 runner.run_blocking_until_ctrl_c(command.execute::<EthereumNode>())
             }
+            Commands::Download(command) => {
+                runner.run_blocking_until_ctrl_c(command.execute::<EthereumNode>())
+            }
             Commands::Stage(command) => runner.run_command_until_exit(|ctx| {
                 command.execute::<EthereumNode, _, _, EthNetworkPrimitives>(
                     ctx,
@@ -221,6 +224,9 @@ pub enum Commands<C: ChainSpecParser, Ext: clap::Args + fmt::Debug> {
     /// Database debugging utilities
     #[command(name = "db")]
     Db(db::Command<C>),
+    /// Downloads and optionally decompresses node snapshots from a URL
+    #[command(name = "download")]
+    Download(download::Command<C>),
     /// Manipulate individual stages.
     #[command(name = "stage")]
     Stage(stage::Command<C>),

book/cli/SUMMARY.md

@@ -18,6 +18,7 @@
       - [`reth db clear static-file`](./reth/db/clear/static-file.md)
     - [`reth db version`](./reth/db/version.md)
     - [`reth db path`](./reth/db/path.md)
+  - [`reth download`](./reth/download.md)
   - [`reth stage`](./reth/stage.md)
     - [`reth stage run`](./reth/stage/run.md)
     - [`reth stage drop`](./reth/stage/drop.md)

book/cli/reth.md

@@ -15,6 +15,7 @@ Commands:
   import        This syncs RLP encoded blocks from a file
   dump-genesis  Dumps genesis block JSON configuration to stdout
   db            Database debugging utilities
+  download      Downloads and optionally decompresses node snapshots from a URL
   stage         Manipulate individual stages
   p2p           P2P Debugging utilities
   config        Write config to stdout

book/cli/reth/download.md

@@ -0,0 +1,132 @@
+# reth download
+
+Downloads and optionally decompresses node snapshots from a URL
+
+```bash
+$ reth download --help
+```
+```txt
+Usage: reth download [OPTIONS] --url <URL>
+
+Options:
+      --chain <CHAIN_OR_PATH>
+          The chain this node is running.
+          Possible values are either a built-in chain or the path to a chain specification file.
+
+          Built-in chains:
+              mainnet, sepolia, holesky, dev
+
+          [default: mainnet]
+
+      --instance <INSTANCE>
+          Add a new instance of a node.
+
+          Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine.
+
+          Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other.
+
+          Changes to the following port numbers: - `DISCOVERY_PORT`: default + `instance` - 1 - `AUTH_PORT`: default + `instance` * 100 - 100 - `HTTP_RPC_PORT`: default - `instance` + 1 - `WS_RPC_PORT`: default + `instance` * 2 - 2
+
+          [default: 1]
+
+  -h, --help
+          Print help (see a summary with '-h')
+
+Datadir:
+      --datadir <DATA_DIR>
+          The path to the data dir for all reth files and subdirectories.
+
+          Defaults to the OS-specific data directory:
+
+          - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
+          - Windows: `{FOLDERID_RoamingAppData}/reth/`
+          - macOS: `$HOME/Library/Application Support/reth/`
+
+          [default: default]
+
+      --datadir.static-files <PATH>
+          The absolute path to store static files in.
+
+  -u, --url <URL>
+          Custom URL to download the snapshot from
+
+  -d, --decompress
+          Whether to automatically decompress the snapshot after download
+
+Logging:
+      --log.stdout.format <FORMAT>
+          The format to use for logs written to stdout
+
+          [default: terminal]
+
+          Possible values:
+          - json:     Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
+          - log-fmt:  Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
+          - terminal: Represents terminal-friendly formatting for logs
+
+      --log.stdout.filter <FILTER>
+          The filter to use for logs written to stdout
+
+          [default: ]
+
+      --log.file.format <FORMAT>
+          The format to use for logs written to the log file
+
+          [default: terminal]
+
+          Possible values:
+          - json:     Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
+          - log-fmt:  Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
+          - terminal: Represents terminal-friendly formatting for logs
+
+      --log.file.filter <FILTER>
+          The filter to use for logs written to the log file
+
+          [default: debug]
+
+      --log.file.directory <PATH>
+          The path to put log files in
+
+          [default: /Users/leanarzis/Library/Caches/reth/logs]
+
+      --log.file.max-size <SIZE>
+          The maximum size (in MB) of one log file
+
+          [default: 200]
+
+      --log.file.max-files <COUNT>
+          The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
+
+          [default: 5]
+
+      --log.journald
+          Write logs to journald
+
+      --log.journald.filter <FILTER>
+          The filter to use for logs written to journald
+
+          [default: error]
+
+      --color <COLOR>
+          Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
+
+          [default: always]
+
+          Possible values:
+          - always: Colors on
+          - auto:   Colors on
+          - never:  Colors off
+
+Display:
+  -v, --verbosity...
+          Set the minimum log level.
+
+          -v      Errors
+          -vv     Warnings
+          -vvv    Info
+          -vvvv   Debug
+          -vvvvv  Traces (warning: very verbose!)
+
+  -q, --quiet
+          Silence all log output
+```

crates/cli/commands/Cargo.toml

@@ -61,18 +61,19 @@ tokio.workspace = true
 
 # misc
 ahash = "0.8"
-human_bytes = "0.4.1"
-eyre.workspace = true
+backon.workspace = true
 clap = { workspace = true, features = ["derive", "env"] }
+eyre.workspace = true
+human_bytes = "0.4.1"
+reqwest.workspace = true 
 serde.workspace = true
 serde_json.workspace = true
-tracing.workspace = true
-backon.workspace = true
 secp256k1 = { workspace = true, features = [
     "global-context",
     "rand-std",
     "recovery",
 ] }
+tracing.workspace = true
 
 # io
 fdlimit.workspace = true

crates/cli/commands/src/download.rs

@@ -0,0 +1,112 @@
+use std::{io::Write, path::Path, process::Command as ProcessCommand, sync::Arc};
+use tokio::{fs, io::AsyncWriteExt};
+
+use clap::Parser;
+use eyre::Result;
+use reqwest::Client;
+use reth_chainspec::{EthChainSpec, EthereumHardforks};
+use reth_cli::chainspec::ChainSpecParser;
+use reth_node_core::args::DatadirArgs;
+
+const SNAPSHOT_FILE: &str = "snapshot.tar.lz4";
+
+/// `reth download` command
+#[derive(Debug, Parser, Clone)]
+pub struct Command<C: ChainSpecParser> {
+    /// The chain this node is running.
+    ///
+    /// Possible values are either a built-in chain or the path to a chain specification file.
+    #[arg(
+        long,
+        value_name = "CHAIN_OR_PATH",
+        long_help = C::help_message(),
+        default_value = C::SUPPORTED_CHAINS[0],
+        value_parser = C::parser()
+    )]
+    chain: Arc<C::ChainSpec>,
+
+    /// Path where will be stored the snapshot
+    #[command(flatten)]
+    datadir: DatadirArgs,
+
+    /// Custom URL to download the snapshot from
+    #[arg(long, short, required = true)]
+    url: String,
+
+    /// Whether to automatically decompress the snapshot after downloading
+    #[arg(long, short)]
+    decompress: bool,
+}
+
+impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
+    /// Downloads and saves the snapshot from the specified URL
+    pub async fn execute<N>(self) -> Result<()> {
+        let data_dir = self.datadir.resolve_datadir(self.chain.chain());
+        let snapshot_path = data_dir.data_dir().join(SNAPSHOT_FILE);
+        fs::create_dir_all(&data_dir).await?;
+
+        println!("Starting snapshot download for chain: {:?}", self.chain.chain());
+        println!("Target directory: {:?}", data_dir.data_dir());
+        println!("Source URL: {}", self.url);
+
+        download_snapshot(&self.url, &snapshot_path).await?;
+
+        println!("Snapshot downloaded successfully to {:?}", snapshot_path);
+        if self.decompress {
+            println!("Decompressing snapshot...");
+            decompress_snapshot(&snapshot_path, data_dir.data_dir())?;
+            println!("Snapshot decompressed successfully");
+
+            // Clean up compressed file
+            fs::remove_file(&snapshot_path).await?;
+        } else {
+            println!(
+                "Please extract the snapshot using: tar --use-compress-program=lz4 -xf {:?}",
+                snapshot_path
+            );
+        }
+
+        Ok(())
+    }
+}
+
+// Downloads a file from the given URL to the specified path, displaying download progress.
+async fn download_snapshot(url: &str, target_path: &Path) -> Result<()> {
+    let client = Client::new();
+    let mut response = client.get(url).send().await?.error_for_status()?;
+
+    let total_size = response.content_length().unwrap_or(0);
+    let mut file = fs::File::create(&target_path).await?;
+    let mut downloaded = 0u64;
+
+    while let Some(chunk) = response.chunk().await? {
+        file.write_all(&chunk).await?;
+        downloaded += chunk.len() as u64;
+
+        if total_size > 0 {
+            let progress = (downloaded as f64 / total_size as f64) * 100.0;
+            print!("\rDownloading... {:.1}%", progress);
+            std::io::stdout().flush()?;
+        }
+    }
+    println!("\nDownload complete!");
+
+    Ok(())
+}
+
+// Helper to decompress snapshot file using lz4
+fn decompress_snapshot(snapshot_path: &Path, target_dir: &Path) -> Result<()> {
+    let status = ProcessCommand::new("tar")
+        .arg("--use-compress-program=lz4")
+        .arg("-xf")
+        .arg(snapshot_path)
+        .arg("-C")
+        .arg(target_dir)
+        .status()?;
+
+    if !status.success() {
+        return Err(eyre::eyre!("Failed to decompress snapshot"));
+    }
+
+    Ok(())
+}

crates/cli/commands/src/lib.rs

@@ -11,6 +11,7 @@
 pub mod common;
 pub mod config_cmd;
 pub mod db;
+pub mod download;
 pub mod dump_genesis;
 pub mod import;
 pub mod init_cmd;