Fetch dashboard release in build.rs (#1192)

* Fetch dashboard release in build script

* Update CI and Dockerfile

* Complete descriptions for BuildError variants

* Make build deps optional

* Fmt

* Remove references to npm in documentation, update .dockerignore

* Add better rerun conditions

* Use reqwest::blocking::Client

* Hardcode release URL/checksum

* Fix build when FETCH_DASHBOARD does not exist

* Update Cargo.toml

* Use checksums crate

* Roll back to sha2 crate, stream to hasher through std::io traits

Author: Adam Gleave

.dockerignore

@@ -1,3 +1 @@
-target/
-
-bee-node/src/plugins/dashboard/frontend/node_modules/
+target/

.github/workflows/ci.yml

@@ -95,14 +95,6 @@ jobs:
             ${{ matrix.os }}-${{ matrix.rust }}-
             ${{ matrix.os }}-
 
-      - name: Install dashboard frontend
-        run: |
-          git submodule init
-          git submodule update --recursive --remote
-          cd bee-node/src/plugins/dashboard/frontend/
-          npm install
-          npm run build-bee
-
       - name: Build
         uses: actions-rs/cargo@v1
         with:
@@ -229,14 +221,6 @@ jobs:
             ubuntu-latest-beta-
             ubuntu-latest-
 
-      - name: Install dashboard frontend
-        run: |
-          git submodule init
-          git submodule update --recursive --remote
-          cd bee-node/src/plugins/dashboard/frontend/
-          npm install
-          npm run build-bee
-
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -270,14 +254,6 @@ jobs:
             ubuntu-latest-nightly-
             ubuntu-latest-
 
-      - name: Install dashboard frontend
-        run: |
-          git submodule init
-          git submodule update --recursive --remote
-          cd bee-node/src/plugins/dashboard/frontend/
-          npm install
-          npm run build-bee
-
       - uses: actions-rs/cargo@v1
         with:
           command: install

.github/workflows/coverage.yml

@@ -36,14 +36,6 @@ jobs:
           crate: rustfilt
           version: 0.2.1
 
-      - name: Install dashboard frontend
-        run: |
-          git submodule init
-          git submodule update --recursive --remote
-          cd bee-node/src/plugins/dashboard/frontend/
-          npm install
-          npm run build-bee
-
       - name: Run test coverage
         run: bash .github/workflows/scripts/coverage.sh
 

.gitmodules

@@ -56,6 +56,12 @@ tracing = { version = "0.1.29", default-features = false, optional = true }
 warp = { version = "0.3.1", default-features = false }
 warp-reverse-proxy = { version = "0.4.0", default-features = false, optional = true }
 
+[build-dependencies]
+reqwest = { version = "0.11.5", default-features = false, features = [ "blocking", "default-tls", "json" ], optional = true }
+sha2 = { version = "0.9.6", default-features = false, optional = true }
+tempfile = { version = "3.3.0", default-features = false, optional = true }
+zip = { version = "0.5.13", default-features = false, features = [ "bzip2", "deflate" ], optional = true }
+
 [lib]
 name = "bee_node"
 path = "src/lib.rs"
@@ -67,6 +73,6 @@ path = "src/main.rs"
 [features]
 default = [ "rocksdb" ]
 
-dashboard = [ "cap", "mime_guess", "rust-embed", "serde_repr", "warp-reverse-proxy" ]
+dashboard = [ "cap", "mime_guess", "reqwest", "rust-embed", "serde_repr", "sha2", "tempfile", "warp-reverse-proxy", "zip" ]
 rocksdb = [ "bee-storage-rocksdb" ]
 sled = [ "bee-storage-sled" ]

Cargo.lock

@@ -9,22 +9,22 @@
 ```sh
 apt-get update
 apt-get upgrade
-apt-get install git npm build-essential cmake pkg-config librocksdb-dev llvm clang libclang-dev libssl-dev
+apt-get install git build-essential cmake pkg-config librocksdb-dev llvm clang libclang-dev libssl-dev
 ```
 
 ### MacOS
 
 ```sh
 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-brew install cmake npm
+brew install cmake
 ```
 
 ### Windows
 
 Open Powershell and execute the following commands:
 ```sh
 Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
-choco install git --params '/NoAutoCrlf' nodejs-lts cmake --installargs 'ADD_CMAKE_TO_PATH=System' llvm
+choco install git --params '/NoAutoCrlf' cmake --installargs 'ADD_CMAKE_TO_PATH=System' llvm
 ```
 Restart Powershell
 
@@ -59,11 +59,6 @@ cd bee/bee-node
 With dashboard
 
 ```sh
-git submodule update --init
-cd src/plugins/dashboard/frontend
-npm install
-npm run build-bee
-cd ../../../../
 cargo build --release --features dashboard
 ```
 

bee-node/Cargo.toml

@@ -1,15 +1,68 @@
-// Copyright 2020-2021 IOTA Stiftung
+// Copyright 2020-2022 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
-use std::process::Command;
+use std::{fmt, process::Command};
 
+#[allow(dead_code)]
 #[derive(Debug)]
 enum BuildError {
-    GitCommit,
+    DashboardDownload,
+    DashboardInvalidArchive,
+    DashboardInvalidChecksum,
+    DashboardRequest(Option<u16>, String),
     GitBranch,
+    GitCommit,
+}
+
+impl fmt::Display for BuildError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match &self {
+            Self::DashboardDownload => write!(f, "failed to download latest release archive"),
+            Self::DashboardInvalidArchive => write!(f, "failed to open or extract release archive"),
+            Self::DashboardInvalidChecksum => write!(
+                f,
+                "checksum of downloaded archive did not match that specified in the release"
+            ),
+            Self::DashboardRequest(Some(code), url) => write!(f, "failed request to `{}`: status code {}", url, code),
+            Self::DashboardRequest(_, url) => write!(f, "failed request to `{}`", url),
+            Self::GitBranch => write!(f, "failed to retrieve git branch name"),
+            Self::GitCommit => write!(f, "failed to retrieve git commit"),
+        }
+    }
 }
 
 fn main() -> Result<(), BuildError> {
+    println!("cargo:rerun-if-env-changed=CARGO_FEATURE_DASHBOARD");
+
+    parse_git_information()?;
+
+    #[cfg(feature = "dashboard")]
+    {
+        let dashboard_dir = std::env::var("OUT_DIR").unwrap() + "/dashboard";
+        println!("cargo:rustc-env=DASHBOARD_DIR={}", dashboard_dir);
+        let dashboard_dir = std::path::Path::new(&dashboard_dir);
+
+        // Rebuild if FETCH_DASHBOARD environment variable has changed.
+        println!("cargo:rerun-if-env-changed=FETCH_DASHBOARD");
+        // Rebuild if DASHBOARD_DIR has changed to a different path.
+        println!("cargo:rerun-if-env-changed=DASHBOARD_DIR");
+
+        let should_fetch = std::env::var("FETCH_DASHBOARD").map(|val| val == "1").unwrap_or(false);
+
+        if should_fetch || !dashboard_dir.exists() {
+            if dashboard_dir.exists() {
+                // If the path already exists, we are re-downloading: remove the old files.
+                std::fs::remove_dir_all(dashboard_dir).expect("could not remove existing dashboard");
+            }
+
+            dashboard::fetch(dashboard_dir)?;
+        }
+    }
+
+    Ok(())
+}
+
+fn parse_git_information() -> Result<(), BuildError> {
     match Command::new("git").args(&["rev-parse", "HEAD"]).output() {
         Ok(output) => {
             println!(
@@ -36,3 +89,57 @@ fn main() -> Result<(), BuildError> {
 
     Ok(())
 }
+
+#[cfg(feature = "dashboard")]
+mod dashboard {
+    use super::*;
+
+    use sha2::{Digest, Sha256};
+    use zip::ZipArchive;
+
+    use std::{io, path::Path};
+
+    const RELEASE_URL: &str =
+        "https://github.com/iotaledger/node-dashboard/releases/download/v1.0.0/node-dashboard-bee-1.0.0.zip";
+    const RELEASE_CHECKSUM: &str = "d61bce7d70b51ea88536b22c5341ccdc3c4651bf8e02e093a771312f3f046c30";
+
+    pub(super) fn fetch<P: AsRef<Path>>(dashboard_dir: P) -> Result<(), BuildError> {
+        println!("downloading latest dashboard release from {}", RELEASE_URL);
+
+        let client = reqwest::blocking::Client::builder()
+            .user_agent("bee-fetch-dashboard")
+            .build()
+            .expect("could not create client");
+
+        let mut tmp_file = tempfile::NamedTempFile::new().expect("could not create temp file");
+
+        client
+            .get(RELEASE_URL)
+            .send()
+            .and_then(|resp| resp.error_for_status())
+            .map_err(|e| BuildError::DashboardRequest(e.status().map(|code| code.as_u16()), RELEASE_URL.to_string()))?
+            .copy_to(&mut tmp_file)
+            .expect("copying failed");
+
+        let mut hasher = Sha256::new();
+        io::copy(&mut tmp_file.reopen().expect("could not open temp file"), &mut hasher).expect("io error");
+        let checksum = format!("{:x}", hasher.finalize());
+
+        println!("checksum: {}\noriginal: {}", checksum, RELEASE_CHECKSUM);
+
+        if checksum != RELEASE_CHECKSUM {
+            return Err(BuildError::DashboardInvalidChecksum);
+        }
+
+        println!("checksum ok");
+
+        let mut archive = ZipArchive::new(tmp_file).map_err(|_| BuildError::DashboardInvalidArchive)?;
+
+        println!("extracting release archive to {}", dashboard_dir.as_ref().display());
+        archive
+            .extract(dashboard_dir)
+            .map_err(|_| BuildError::DashboardInvalidArchive)?;
+
+        Ok(())
+    }
+}