cf: verify hash of downloaded mod files and repair (#435)

src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java

@@ -13,6 +13,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Duration;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -57,6 +58,7 @@
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Schedulers;
+import reactor.util.retry.Retry;
 
 @RequiredArgsConstructor
 @Slf4j
@@ -70,6 +72,8 @@ public class CurseForgeInstaller {
  public static final String REPO_SUBDIR_MODPACKS = "modpacks";
  public static final String REPO_SUBDIR_MODS = "mods";
  public static final String REPO_SUBDIR_WORLDS = "worlds";
+ private static final Duration BAD_FILE_DELAY = Duration.ofSeconds(5);
+ public static final int BAD_FILE_ATTEMPTS = 3;
 
  private final Path outputDir;
  private final Path resultsFile;
@@ -734,7 +738,21 @@ else if (category.getSlug().equals("worlds")) {
  );
 
  final Mono<ResolveResult> resolvedFileMono =
- downloadOrResolveFile(context, modInfo, isWorld, outputDir, cfFile);
+ Mono.defer(() ->
+ downloadOrResolveFile(context, modInfo, isWorld, outputDir, cfFile)
+ )
+ // retry the deferred part above if one of the expected failure cases
+ .retryWhen(
+ Retry.fixedDelay(BAD_FILE_ATTEMPTS, BAD_FILE_DELAY)
+ .filter(throwable ->
+ throwable instanceof FileHashInvalidException ||
+ throwable instanceof FailedRequestException
+ )
+ .doBeforeRetry(retrySignal ->
+ log.warn("Retrying to download {} @ {}:{}",
+ cfFile.getFileName(), projectID, fileID)
+ )
+ );
 
  return isWorld ?
  resolvedFileMono
@@ -780,10 +798,12 @@ private Mono<ResolveResult> downloadOrResolveFile(InstallContext context, CurseF
 
  if (locatedFile != null) {
  log.info("Mod file {} already exists", locatedFile);
- return Mono.just(new ResolveResult(locatedFile));
+ return FileHashVerifier.verify(locatedFile, cfFile.getHashes())
+ .map(ResolveResult::new);
  }
  else {
  return context.cfApi.download(cfFile, outputFile, modFileDownloadStatusHandler(this.outputDir, log))
+ .flatMap(path -> FileHashVerifier.verify(path, cfFile.getHashes()))
  .map(ResolveResult::new)
  .onErrorResume(
  e -> e instanceof FailedRequestException

src/main/java/me/itzg/helpers/curseforge/FileHashInvalidException.java

@@ -0,0 +1,8 @@
+package me.itzg.helpers.curseforge;
+
+public class FileHashInvalidException extends RuntimeException {
+
+ public FileHashInvalidException(String message) {
+ super(message);
+ }
+}

src/main/java/me/itzg/helpers/curseforge/FileHashVerifier.java

@@ -0,0 +1,47 @@
+package me.itzg.helpers.curseforge;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
+import me.itzg.helpers.curseforge.model.FileHash;
+import me.itzg.helpers.curseforge.model.HashAlgo;
+import me.itzg.helpers.files.ChecksumAlgo;
+import me.itzg.helpers.files.Checksums;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
+
+@Slf4j
+public class FileHashVerifier {
+
+ private final static Map<HashAlgo, ChecksumAlgo> algos = new EnumMap<>(HashAlgo.class);
+
+ static {
+ algos.put(HashAlgo.Md5, ChecksumAlgo.MD5);
+ algos.put(HashAlgo.Sha1, ChecksumAlgo.SHA1);
+ }
+
+ public static Mono<Path> verify(Path file, List<FileHash> hashes) {
+ for (final FileHash hash : hashes) {
+ final ChecksumAlgo checksumAlgo = algos.get(hash.getAlgo());
+ if (checksumAlgo != null) {
+ return Mono.fromCallable(() -> {
+ log.debug("Verifying hash of {}", file);
+
+ if (!Checksums.valid(file, checksumAlgo, hash.getValue())) {
+ Files.delete(file);
+ throw new FileHashInvalidException("Incorrect checksum: " + file);
+ }
+ else {
+ return file;
+ }
+ })
+ .subscribeOn(Schedulers.boundedElastic());
+ }
+ }
+
+ return Mono.error(new IllegalArgumentException("Unable to find compatible checksum algorithm"));
+ }
+}

src/main/java/me/itzg/helpers/files/ChecksumAlgo.java

@@ -0,0 +1,19 @@
+package me.itzg.helpers.files;
+
+import lombok.Getter;
+
+@Getter
+public enum ChecksumAlgo {
+ MD5("md5", "MD5"),
+ SHA1("sha1", "SHA-1"),
+ SHA256("sha256", "SHA-256"),;
+
+ private final String prefix;
+
+ private final String jdkAlgo;
+
+ ChecksumAlgo(String prefix, String jdkAlgo) {
+ this.prefix = prefix;
+ this.jdkAlgo = jdkAlgo;
+ }
+}

src/main/java/me/itzg/helpers/files/Checksums.java

@@ -6,65 +6,33 @@
 import java.nio.file.Path;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.util.HashMap;
-import java.util.Map;
-import me.itzg.helpers.errors.GenericException;
 import org.apache.commons.codec.binary.Hex;
+import org.jetbrains.annotations.Blocking;
 
 public class Checksums {
 
- public static final String DEFAULT_PREFIX = "sha256";
-
- private static final Map<String/*prefix*/, String/*algo*/> prefixAlgos = new HashMap<>();
- private static final String DELIMITER = ":";
-
- static {
- prefixAlgos.put("md5", "MD5");
- prefixAlgos.put("sha1", "SHA-1");
- prefixAlgos.put("sha256", "SHA-256");
- }
-
  private Checksums() {
  }
 
- /**
- *
- * @param otherChecksum a checksum previously calculated with this method or null to use the default type
- * @param path a file to process
- * @return the file's checksum with a prefix tracking type
- */
- public static String checksumLike(String otherChecksum, Path path) throws IOException {
- final String algo;
- final String prefix;
- if (otherChecksum != null) {
- final String[] parts = otherChecksum.split(DELIMITER, 2);
- algo = prefixAlgos.get(parts[0]);
- if (algo == null) {
- throw new GenericException("Unexpected checksum prefix: " + parts[0]);
- }
- prefix = parts[0];
- }
- else {
- prefix = DEFAULT_PREFIX;
- algo = prefixAlgos.get(DEFAULT_PREFIX);
- }
-
+ @Blocking
+ public static boolean valid(Path file, ChecksumAlgo algo, String expectedCheckum) throws IOException {
  final MessageDigest md;
  try {
- md = MessageDigest.getInstance(algo);
+ md = MessageDigest.getInstance(algo.getJdkAlgo());
  } catch (NoSuchAlgorithmException e) {
  throw new IllegalStateException(e);
  }
 
- try (InputStream inputStream = Files.newInputStream(path)) {
+ try (InputStream inputStream = Files.newInputStream(file)) {
  final byte[] buffer = new byte[1024];
  int len;
  while ((len = inputStream.read(buffer)) >= 0) {
  md.update(buffer, 0, len);
  }
  }
-
  final byte[] digest = md.digest();
- return prefix + DELIMITER + Hex.encodeHexString(digest);
+
+ return expectedCheckum.toLowerCase().equals(Hex.encodeHexString(digest));
  }
+
 }

src/test/java/me/itzg/helpers/curseforge/FileHashVerifierTest.java

@@ -0,0 +1,58 @@
+package me.itzg.helpers.curseforge;
+
+import static java.util.Collections.singletonList;
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import me.itzg.helpers.curseforge.model.FileHash;
+import me.itzg.helpers.curseforge.model.HashAlgo;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+@SuppressWarnings("CodeBlock2Expr")
+class FileHashVerifierTest {
+
+ @Test
+ void validMd5(@TempDir Path tempDir) throws Exception {
+ final Path file = Files.write(tempDir.resolve("hello.txt"), "Hello World".getBytes(StandardCharsets.UTF_8));
+
+ assertThatCode(() -> {
+ FileHashVerifier.verify(file, singletonList(
+ new FileHash().setAlgo(HashAlgo.Md5).setValue("B10A8DB164E0754105B7A99BE72E3FE5")
+ ))
+ .block();
+ })
+ .doesNotThrowAnyException();
+ }
+
+ @Test
+ void validSha1(@TempDir Path tempDir) throws IOException {
+ final Path file = Files.write(tempDir.resolve("hello.txt"), "Hello World".getBytes(StandardCharsets.UTF_8));
+
+ assertThatCode(() -> {
+ FileHashVerifier.verify(file, singletonList(
+ new FileHash().setAlgo(HashAlgo.Sha1).setValue("0A4D55A8D778E5022FAB701977C5D840BBC486D0")
+ ))
+ .block();
+ })
+ .doesNotThrowAnyException();
+
+ }
+
+ @Test
+ void handlesInvalid(@TempDir Path tempDir) throws IOException {
+ final Path file = Files.write(tempDir.resolve("hello.txt"), "Hello World".getBytes(StandardCharsets.UTF_8));
+
+ assertThatThrownBy(() -> {
+ FileHashVerifier.verify(file, singletonList(
+ new FileHash().setAlgo(HashAlgo.Md5).setValue("BAD")
+ ))
+ .block();
+ })
+ .isInstanceOf(FileHashInvalidException.class);
+ }
+}