Daedalus Rewrite + Code Cleanup (#16)

* [wip] rewrite daedalus, vanilla, fabric, and quilt

* finish forge + neo

* fix docker

* fix neoforge 1.21+

* update concurrency limit

* finish

* remove mac garb

.env

@@ -1,9 +1,15 @@
-RUST_LOG=info
+RUST_LOG=warn,daedalus_client=trace
 
-BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com
+BASE_URL=http://localhost:9000/meta
+
+CONCURRENCY_LIMIT=10
 
 S3_ACCESS_TOKEN=none
 S3_SECRET=none
-S3_URL=none
-S3_REGION=none
-S3_BUCKET_NAME=none
+S3_URL=http://localhost:9000
+S3_REGION=path-style
+S3_BUCKET_NAME=meta
+
+CLOUDFLARE_INTEGRATION=false
+CLOUDFLARE_TOKEN=none
+CLOUDFLARE_ZONE_ID=none

.github/workflows/run.yml

@@ -0,0 +1,49 @@
+name: Run Meta
+
+on:
+ schedule:
+ - cron: '*/5 * * * *'
+
+jobs:
+ run-docker:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Log in to GitHub Container Registry
+ uses: docker/login-action@v2
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+
+ - name: Pull Docker image from GHCR
+ run: docker pull ghcr.io/modrinth/daedalus:latest
+
+ - name: Run Docker container
+ env:
+ BASE_URL: ${{ secrets.BASE_URL }}
+ S3_ACCESS_TOKEN: ${{ secrets.S3_ACCESS_TOKEN }}
+ S3_SECRET: ${{ secrets.S3_SECRET }}
+ S3_URL: ${{ secrets.S3_URL }}
+ S3_REGION: ${{ secrets.S3_REGION }}
+ S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
+ CLOUDFLARE_INTEGRATION: ${{ secrets.CLOUDFLARE_INTEGRATION }}
+ CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }}
+ CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }}
+ run: |
+ docker run -d \
+ --name daedalus \
+ -e BASE_URL=$BASE_URL \
+ -e S3_ACCESS_TOKEN=$S3_ACCESS_TOKEN \
+ -e S3_SECRET=$S3_SECRET \
+ -e S3_URL=$S3_URL \
+ -e S3_REGION=$S3_REGION \
+ -e S3_BUCKET_NAME=$S3_BUCKET_NAME \
+ -e CLOUDFLARE_INTEGRATION=$CLOUDFLARE_INTEGRATION \
+ -e CLOUDFLARE_TOKEN=$CLOUDFLARE_TOKEN \
+ -e CLOUDFLARE_ZONE_ID=$CLOUDFLARE_ZONE_ID \
+ ghcr.io/modrinth/daedalus:latest

.gitignore

@@ -1,6 +1,7 @@
 ### Intellij ###
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+caches/
 
 # User-specific stuff
 .idea/**/workspace.xml

Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.68.2 as build
+FROM rust:1.79.0 as build
 ENV PKG_CONFIG_ALLOW_CROSS=1
 
 WORKDIR /usr/src/daedalus

daedalus/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "daedalus"
-version = "0.1.27"
-authors = ["Jai A <[email protected]>"]
-edition = "2018"
+version = "0.2.0"
+authors = ["Jai A <[email protected]>"]
+edition = "2021"
 license = "MIT"
 description = "Utilities for querying and parsing Minecraft metadata"
 repository = "https://github.com/modrinth/daedalus/"
@@ -14,12 +14,8 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-reqwest = { version = "0.11", features = ["json"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = { version = "0.4", features = ["serde"] }
 bytes = "1"
 thiserror = "1.0"
-tokio = { version = "1", features = ["full"] }
-sha1 = { version = "0.6.1", features = ["std"]}
-bincode = {version = "2.0.0-rc.2", features = ["serde"], optional = true}

daedalus/src/lib.rs

@@ -12,30 +12,6 @@ pub mod modded;
 #[derive(thiserror::Error, Debug)]
 /// An error type representing possible errors when fetching metadata
 pub enum Error {
- #[error("Failed to validate file checksum at url {url} with hash {hash} after {tries} tries")]
- /// A checksum was failed to validate for a file
- ChecksumFailure {
- /// The checksum's hash
- hash: String,
- /// The URL of the file attempted to be downloaded
- url: String,
- /// The amount of tries that the file was downloaded until failure
- tries: u32,
- },
- /// There was an error while deserializing metadata
- #[error("Error while deserializing JSON")]
- SerdeError(#[from] serde_json::Error),
- /// There was a network error when fetching an object
- #[error("Unable to fetch {item}")]
- FetchError {
- /// The internal reqwest error
- inner: reqwest::Error,
- /// The item that was failed to be fetched
- item: String,
- },
- /// There was an error when managing async tasks
- #[error("Error while managing asynchronous tasks")]
- TaskError(#[from] tokio::task::JoinError),
  /// Error while parsing input
  #[error("{0}")]
  ParseError(String),
@@ -124,100 +100,3 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
  ))
  }
 }
-
-/// Downloads a file from specified mirrors
-pub async fn download_file_mirrors(
- base: &str,
- mirrors: &[&str],
- sha1: Option<&str>,
-) -> Result<bytes::Bytes, Error> {
- if mirrors.is_empty() {
- return Err(Error::ParseError("No mirrors provided!".to_string()));
- }
-
- for (index, mirror) in mirrors.iter().enumerate() {
- let result = download_file(&format!("{}{}", mirror, base), sha1).await;
-
- if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) {
- return result;
- }
- }
-
- unreachable!()
-}
-
-/// Downloads a file with retry and checksum functionality
-pub async fn download_file(
- url: &str,
- sha1: Option<&str>,
-) -> Result<bytes::Bytes, Error> {
- let mut headers = reqwest::header::HeaderMap::new();
- if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!(
- "modrinth/daedalus/{} ([email protected])",
- env!("CARGO_PKG_VERSION")
- )) {
- headers.insert(reqwest::header::USER_AGENT, header);
- }
- let client = reqwest::Client::builder()
- .tcp_keepalive(Some(std::time::Duration::from_secs(10)))
- .timeout(std::time::Duration::from_secs(15))
- .default_headers(headers)
- .build()
- .map_err(|err| Error::FetchError {
- inner: err,
- item: url.to_string(),
- })?;
-
- for attempt in 1..=4 {
- let result = client.get(url).send().await;
-
- match result {
- Ok(x) => {
- let bytes = x.bytes().await;
-
- if let Ok(bytes) = bytes {
- if let Some(sha1) = sha1 {
- if &*get_hash(bytes.clone()).await? != sha1 {
- if attempt <= 3 {
- continue;
- } else {
- return Err(Error::ChecksumFailure {
- hash: sha1.to_string(),
- url: url.to_string(),
- tries: attempt,
- });
- }
- }
- }
-
- return Ok(bytes);
- } else if attempt <= 3 {
- continue;
- } else if let Err(err) = bytes {
- return Err(Error::FetchError {
- inner: err,
- item: url.to_string(),
- });
- }
- }
- Err(_) if attempt <= 3 => continue,
- Err(err) => {
- return Err(Error::FetchError {
- inner: err,
- item: url.to_string(),
- })
- }
- }
- }
-
- unreachable!()
-}
-
-/// Computes a checksum of the input bytes
-pub async fn get_hash(bytes: bytes::Bytes) -> Result<String, Error> {
- let hash =
- tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest())
- .await?;
-
- Ok(hash)
-}