Fix forge install issues (#18)

* Fix forge install issues

* remove mac garb

daedalus/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus"
-version = "0.2.0"
+version = "0.2.1"
 authors = ["Jai A <[email protected]>"]
 edition = "2021"
 license = "MIT"

daedalus_client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus_client"
-version = "0.2.0"
+version = "0.2.1"
 authors = ["Jai A <[email protected]>"]
 edition = "2021"
 

daedalus_client/src/fabric.rs

@@ -17,6 +17,7 @@ pub async fn fetch_fabric(
  "fabric",
  "https://meta.fabricmc.net/v2",
  "https://maven.fabricmc.net/",
+ &[],
  semaphore,
  upload_files,
  mirror_artifacts,
@@ -34,7 +35,11 @@ pub async fn fetch_quilt(
  daedalus::modded::CURRENT_QUILT_FORMAT_VERSION,
  "quilt",
  "https://meta.quiltmc.org/v3",
- "https://meta.quiltmc.org/",
+ "https://maven.quiltmc.org/repository/release/",
+ &[
+ // This version is broken as it contains invalid library coordinates
+ "0.17.5-beta.4",
+ ],
  semaphore,
  upload_files,
  mirror_artifacts,
@@ -48,6 +53,7 @@ async fn fetch(
  mod_loader: &str,
  meta_url: &str,
  maven_url: &str,
+ skip_versions: &[&str],
  semaphore: Arc<Semaphore>,
  upload_files: &DashMap<String, UploadFile>,
  mirror_artifacts: &DashMap<String, MirrorArtifact>,
@@ -76,6 +82,7 @@ async fn fetch(
  .game_versions
  .iter()
  .any(|x| x.loaders.iter().any(|x| x.id == version.version))
+ && !skip_versions.contains(&&*version.version)
  {
  fetch_versions.push(version);
  }
@@ -98,7 +105,11 @@ async fn fetch(
  (fetch_versions, fetch_intermediary_versions)
  } else {
  (
- fabric_manifest.loader.iter().collect(),
+ fabric_manifest
+ .loader
+ .iter()
+ .filter(|x| !skip_versions.contains(&&*x.version))
+ .collect(),
  fabric_manifest.intermediary.iter().collect(),
  )
  };
@@ -109,7 +120,9 @@ async fn fetch(
  for x in &fetch_intermediary_versions {
  insert_mirrored_artifact(
  &x.maven,
- maven_url.to_string(),
+ None,
+ vec![maven_url.to_string()],
+ false,
  mirror_artifacts,
  )?;
  }
@@ -142,13 +155,24 @@ async fn fetch(
  let new_name = lib
  .name
  .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
+
+ // Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries
+ if &*lib.name == "net.minecraft:launchwrapper:1.12" {
+ lib.url = Some(
+ "https://libraries.minecraft.net/".to_string(),
+ );
+ }
+
  // If a library is not intermediary, we add it to mirror artifacts to be mirrored
  if lib.name == new_name {
  insert_mirrored_artifact(
  &new_name,
- lib.url
+ None,
+ vec![lib
+ .url
  .clone()
- .unwrap_or_else(|| maven_url.to_string()),
+ .unwrap_or_else(|| maven_url.to_string())],
+ false,
  mirror_artifacts,
  )?;
  } else {

daedalus_client/src/forge.rs

@@ -1,4 +1,6 @@
-use crate::util::{download_file, fetch_json, fetch_xml, format_url};
+use crate::util::{
+ download_file, fetch_json, fetch_xml, format_url, sha1_async,
+};
 use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile};
 use chrono::{DateTime, Utc};
 use daedalus::get_path_from_artifact;
@@ -246,6 +248,7 @@ async fn fetch(
  raw: bytes::Bytes,
  loader: &ForgeVersion,
  maven_url: &str,
+ mod_loader: &str,
  upload_files: &DashMap<String, UploadFile>,
  mirror_artifacts: &DashMap<String, MirrorArtifact>,
  ) -> Result<PartialVersionInfo, Error> {
@@ -399,15 +402,27 @@ async fn fetch(
  .into_iter()
  .map(|mut lib| {
  // For all libraries besides the forge lib extracted, we mirror them from maven servers
- if lib.name != install_profile.install.path {
- // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"
- insert_mirrored_artifact(
- &lib.name,
- lib.url.clone().unwrap_or_else(|| {
- maven_url.to_string()
- }),
- mirror_artifacts,
- )?;
+ // unless the URL is empty/null or available on Minecraft's servers
+ if let Some(url) = lib.url {
+ if lib.name != install_profile.install.path
+ && !url.is_empty()
+ && !url.contains(
+ "https://libraries.minecraft.net/",
+ )
+ {
+ insert_mirrored_artifact(
+ &lib.name,
+ None,
+ vec![
+ url,
+ "https://maven.creeperhost.net/"
+ .to_string(),
+ maven_url.to_string(),
+ ],
+ false,
+ mirror_artifacts,
+ )?;
+ }
  }
 
  lib.url = Some(format_url("maven/"));
@@ -468,6 +483,7 @@ async fn fetch(
  async fn mirror_forge_library(
  mut zip: ZipFileReader,
  mut lib: daedalus::minecraft::Library,
+ maven_url: &str,
  upload_files: &DashMap<String, UploadFile>,
  mirror_artifacts: &DashMap<String, MirrorArtifact>,
  ) -> Result<daedalus::minecraft::Library, Error>
@@ -480,7 +496,9 @@ async fn fetch(
  if !artifact.url.is_empty() {
  insert_mirrored_artifact(
  &lib.name,
- artifact.url.clone(),
+ Some(artifact.sha1.clone()),
+ vec![artifact.url.clone()],
+ true,
  mirror_artifacts,
  )?;
 
@@ -491,10 +509,18 @@ async fn fetch(
  }
  } else if let Some(url) = &lib.url {
  if !url.is_empty() {
- // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"
  insert_mirrored_artifact(
  &lib.name,
- url.clone(),
+ None,
+ vec![
+ url.clone(),
+ "https://libraries.minecraft.net/"
+ .to_string(),
+ "https://maven.creeperhost.net/"
+ .to_string(),
+ maven_url.to_string(),
+ ],
+ false,
  mirror_artifacts,
  )?;
 
@@ -531,6 +557,7 @@ async fn fetch(
  mirror_forge_library(
  zip.clone(),
  lib,
+ maven_url,
  upload_files,
  mirror_artifacts,
  )
@@ -560,7 +587,7 @@ async fn fetch(
  value: &str,
  upload_files: &DashMap<String, UploadFile>,
  libs: &mut Vec<daedalus::minecraft::Library>,
- install_profile_path: Option<&str>,
+ mod_loader: &str,
  version: &ForgeVersion,
  ) -> Result<String, Error> {
  let extract_file =
@@ -595,11 +622,9 @@ async fn fetch(
  })?;
 
  let path = format!(
- "{}:{}@{}",
- install_profile_path.unwrap_or(&*format!(
- "net.minecraftforge:forge:{}",
- version.raw
- )),
+ "com.modrinth.daedalus:{}-installer-extracts:{}:{}@{}",
+ mod_loader,
+ version.raw,
  file_name,
  ext
  );
@@ -634,7 +659,7 @@ async fn fetch(
  &entry.client,
  upload_files,
  &mut version_info.libraries,
- install_profile.path.as_deref(),
+ mod_loader,
  loader,
  )
  .await?
@@ -649,7 +674,7 @@ async fn fetch(
  &entry.server,
  upload_files,
  &mut version_info.libraries,
- install_profile.path.as_deref(),
+ mod_loader,
  loader,
  )
  .await?
@@ -686,6 +711,7 @@ async fn fetch(
  raw,
  loader,
  maven_url,
+ mod_loader,
  upload_files,
  mirror_artifacts,
  )

daedalus_client/src/main.rs

@@ -72,46 +72,63 @@ async fn main() -> Result<()> {
  futures::future::try_join_all(mirror_artifacts.iter().map(|x| {
  upload_url_to_bucket_mirrors(
  format!("maven/{}", x.key()),
- x.value().mirrors.iter().map(|x| x.key().clone()).collect(),
+ x.value()
+ .mirrors
+ .iter()
+ .map(|mirror| {
+ if mirror.entire_url {
+ mirror.path.clone()
+ } else {
+ format!("{}{}", mirror.path, x.key())
+ }
+ })
+ .collect(),
+ x.sha1.clone(),
  &semaphore,
  )
  }))
  .await?;
 
- if let Ok(token) = dotenvy::var("CLOUDFLARE_TOKEN") {
- if let Ok(zone_id) = dotenvy::var("CLOUDFLARE_ZONE_ID") {
- let cache_clears = upload_files
- .into_iter()
- .map(|x| format_url(&x.0))
- .chain(
- mirror_artifacts
- .into_iter()
- .map(|x| format_url(&format!("maven/{}", x.0))),
- )
- .collect::<Vec<_>>();
-
- // Cloudflare ratelimits cache clears to 500 files per request
- for chunk in cache_clears.chunks(500) {
- REQWEST_CLIENT.post(format!("https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache"))
- .bearer_auth(&token)
- .json(&serde_json::json!({
+ if dotenvy::var("CLOUDFLARE_INTEGRATION")
+ .ok()
+ .and_then(|x| x.parse::<bool>().ok())
+ .unwrap_or(false)
+ {
+ if let Ok(token) = dotenvy::var("CLOUDFLARE_TOKEN") {
+ if let Ok(zone_id) = dotenvy::var("CLOUDFLARE_ZONE_ID") {
+ let cache_clears = upload_files
+ .into_iter()
+ .map(|x| format_url(&x.0))
+ .chain(
+ mirror_artifacts
+ .into_iter()
+ .map(|x| format_url(&format!("maven/{}", x.0))),
+ )
+ .collect::<Vec<_>>();
+
+ // Cloudflare ratelimits cache clears to 500 files per request
+ for chunk in cache_clears.chunks(500) {
+ REQWEST_CLIENT.post(format!("https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache"))
+ .bearer_auth(&token)
+ .json(&serde_json::json!({
  "files": chunk
  }))
- .send()
- .await
- .map_err(|err| {
- ErrorKind::Fetch {
- inner: err,
- item: "cloudflare clear cache".to_string(),
- }
- })?
- .error_for_status()
- .map_err(|err| {
- ErrorKind::Fetch {
- inner: err,
- item: "cloudflare clear cache".to_string(),
- }
- })?;
+ .send()
+ .await
+ .map_err(|err| {
+ ErrorKind::Fetch {
+ inner: err,
+ item: "cloudflare clear cache".to_string(),
+ }
+ })?
+ .error_for_status()
+ .map_err(|err| {
+ ErrorKind::Fetch {
+ inner: err,
+ item: "cloudflare clear cache".to_string(),
+ }
+ })?;
+ }
  }
  }
  }
@@ -125,21 +142,37 @@ pub struct UploadFile {
 }
 
 pub struct MirrorArtifact {
- pub mirrors: DashSet<String>,
+ pub sha1: Option<String>,
+ pub mirrors: DashSet<Mirror>,
 }
 
+#[derive(Eq, PartialEq, Hash)]
+pub struct Mirror {
+ path: String,
+ entire_url: bool,
+}
+
+#[tracing::instrument(skip(mirror_artifacts))]
 pub fn insert_mirrored_artifact(
  artifact: &str,
- mirror: String,
+ sha1: Option<String>,
+ mirrors: Vec<String>,
+ entire_url: bool,
  mirror_artifacts: &DashMap<String, MirrorArtifact>,
 ) -> Result<()> {
- mirror_artifacts
+ let mut val = mirror_artifacts
  .entry(get_path_from_artifact(artifact)?)
  .or_insert(MirrorArtifact {
+ sha1,
  mirrors: DashSet::new(),
- })
- .mirrors
- .insert(mirror);
+ });
+
+ for mirror in mirrors {
+ val.mirrors.insert(Mirror {
+ path: mirror,
+ entire_url,
+ });
+ }
 
  Ok(())
 }