First steps towards build reproducibility

core/deployment/src/main/java/io/quarkus/deployment/pkg/PackageConfig.java

@@ -1,6 +1,7 @@
 package io.quarkus.deployment.pkg;
 
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -49,6 +50,13 @@ public interface PackageConfig {
      */
     Optional<String> outputName();
 
+    /**
+     * The timestamp used as a reference for generating the packages (e.g. for the creation timestamp of ZIP entries).
+     * <p>
+     * The approach is similar to what is done by the maven-jar-plugin with `project.build.outputTimestamp`.
+     */
+    Optional<Instant> outputTimestamp();
+
     /**
      * Setting this switch to {@code true} will cause Quarkus to write the transformed application bytecode
      * to the build tool's output directory.

core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java

@@ -350,7 +350,7 @@ private void buildUberJar0(CurateOutcomeBuildItem curateOutcomeBuildItem,
             MainClassBuildItem mainClassBuildItem,
             ClassLoadingConfig classLoadingConfig,
             Path runnerJar) throws Exception {
-        try (FileSystem runnerZipFs = createNewZip(runnerJar, packageConfig)) {
+        try (FileSystem runnerZipFs = createNewReproducibleZipFileSystem(runnerJar, packageConfig)) {
 
             log.info("Building uber jar: " + runnerJar);
 
@@ -549,7 +549,7 @@ private JarBuildItem buildLegacyThinJar(CurateOutcomeBuildItem curateOutcomeBuil
         Files.deleteIfExists(runnerJar);
         IoUtils.createOrEmptyDir(libDir);
 
-        try (FileSystem runnerZipFs = createNewZip(runnerJar, packageConfig)) {
+        try (FileSystem runnerZipFs = createNewReproducibleZipFileSystem(runnerJar, packageConfig)) {
 
             log.info("Building thin jar: " + runnerJar);
 
@@ -647,7 +647,7 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
         if (!transformedClasses.getTransformedClassesByJar().isEmpty()) {
             Path transformedZip = quarkus.resolve(TRANSFORMED_BYTECODE_JAR);
             fastJarJarsBuilder.setTransformed(transformedZip);
-            try (FileSystem out = createNewZip(transformedZip, packageConfig)) {
+            try (FileSystem out = createNewReproducibleZipFileSystem(transformedZip, packageConfig)) {
                 for (Set<TransformedClassesBuildItem.TransformedClass> transformedSet : transformedClasses
                         .getTransformedClassesByJar().values()) {
                     for (TransformedClassesBuildItem.TransformedClass transformed : transformedSet) {
@@ -668,7 +668,7 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
         //now generated classes and resources
         Path generatedZip = quarkus.resolve(GENERATED_BYTECODE_JAR);
         fastJarJarsBuilder.setGenerated(generatedZip);
-        try (FileSystem out = createNewZip(generatedZip, packageConfig)) {
+        try (FileSystem out = createNewReproducibleZipFileSystem(generatedZip, packageConfig)) {
             for (GeneratedClassBuildItem i : generatedClasses) {
                 String fileName = fromClassNameToResourceName(i.getName());
                 Path target = out.getPath(fileName);
@@ -703,7 +703,7 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
                     .setResolvedDependency(applicationArchivesBuildItem.getRootArchive().getResolvedDependency())
                     .setPath(runnerJar));
             Predicate<String> ignoredEntriesPredicate = getThinJarIgnoredEntriesPredicate(packageConfig);
-            try (FileSystem runnerZipFs = createNewZip(runnerJar, packageConfig)) {
+            try (FileSystem runnerZipFs = createNewReproducibleZipFileSystem(runnerJar, packageConfig)) {
                 copyFiles(applicationArchivesBuildItem.getRootArchive(), runnerZipFs, null, ignoredEntriesPredicate);
             }
         }
@@ -793,7 +793,7 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
             }
         }
         if (!rebuild) {
-            try (FileSystem runnerZipFs = createNewZip(initJar, packageConfig)) {
+            try (FileSystem runnerZipFs = createNewReproducibleZipFileSystem(initJar, packageConfig)) {
                 ResolvedDependency appArtifact = curateOutcomeBuildItem.getApplicationModel().getAppArtifact();
                 generateManifest(runnerZipFs, classPath.toString(), packageConfig, appArtifact,
                         QuarkusEntryPoint.class.getName(),
@@ -997,7 +997,7 @@ public static String getJarFileName(ResolvedDependency dep, Path resolvedPath) {
     }
 
     private void packageClasses(Path resolvedDep, final Path targetPath, PackageConfig packageConfig) throws IOException {
-        try (FileSystem runnerZipFs = createNewZip(targetPath, packageConfig)) {
+        try (FileSystem runnerZipFs = createNewReproducibleZipFileSystem(targetPath, packageConfig)) {
             Files.walkFileTree(resolvedDep, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
                     new SimpleFileVisitor<Path>() {
                         @Override
@@ -1063,7 +1063,7 @@ private NativeImageSourceJarBuildItem buildNativeImageThinJar(CurateOutcomeBuild
         Path libDir = targetDirectory.resolve(LIB);
         Files.createDirectories(libDir);
 
-        try (FileSystem runnerZipFs = ZipUtils.newZip(runnerJar)) {
+        try (FileSystem runnerZipFs = createNewReproducibleZipFileSystem(runnerJar, packageConfig)) {
 
             log.info("Building native image source jar: " + runnerJar);
 
@@ -1657,12 +1657,13 @@ public boolean decompile(Path jarToDecompile) {
         }
     }
 
-    private static FileSystem createNewZip(Path runnerJar, PackageConfig config) throws IOException {
+    private static FileSystem createNewReproducibleZipFileSystem(Path runnerJar, PackageConfig config) throws IOException {
         boolean useUncompressedJar = !config.jar().compress();
         if (useUncompressedJar) {
-            return ZipUtils.newZip(runnerJar, Map.of("compressionMethod", "STORED"));
+            return ZipUtils.createNewReproducibleZipFileSystem(runnerJar, Map.of("compressionMethod", "STORED"),
+                    config.outputTimestamp().orElse(null));
         }
-        return ZipUtils.newZip(runnerJar);
+        return ZipUtils.createNewReproducibleZipFileSystem(runnerJar, config.outputTimestamp().orElse(null));
     }
 
 }

core/deployment/src/main/java/io/quarkus/deployment/steps/ConfigGenerationBuildStep.java

@@ -22,11 +22,13 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.TreeSet;
 
 import jakarta.annotation.Priority;
 
@@ -268,13 +270,13 @@ void generateBuilders(
         Set<String> configCustomizers = discoverService(SmallRyeConfigBuilderCustomizer.class, reflectiveClass);
 
         // TODO - introduce a way to ignore mappings that are only used for documentation or to prevent warnings
-        Set<ConfigClass> ignoreMappings = new HashSet<>();
+        Set<ConfigClass> ignoreMappings = new LinkedHashSet<>();
         ignoreMappings.add(ConfigClass.configClass(BuildAnalyticsConfig.class, "quarkus.analytics"));
         ignoreMappings.add(ConfigClass.configClass(BuilderConfig.class, "quarkus.builder"));
         ignoreMappings.add(ConfigClass.configClass(CommandLineRuntimeConfig.class, "quarkus"));
         ignoreMappings.add(ConfigClass.configClass(DebugRuntimeConfig.class, "quarkus.debug"));
 
-        Set<ConfigClass> allMappings = new HashSet<>();
+        Set<ConfigClass> allMappings = new LinkedHashSet<>();
         allMappings.addAll(staticSafeConfigMappings(configMappings));
         allMappings.addAll(runtimeConfigMappings(configMappings));
         allMappings.addAll(configItem.getReadResult().getBuildTimeRunTimeMappings());
@@ -285,11 +287,11 @@ void generateBuilders(
         Map<Object, FieldDescriptor> sharedFields = generateSharedConfig(generatedClass, converters, allMappings);
 
         // For Static Init Config
-        Set<ConfigClass> staticMappings = new HashSet<>();
+        Set<ConfigClass> staticMappings = new LinkedHashSet<>();
         staticMappings.addAll(staticSafeConfigMappings(configMappings));
         staticMappings.addAll(configItem.getReadResult().getBuildTimeRunTimeMappings());
         staticMappings.removeAll(ignoreMappings);
-        Set<String> staticCustomizers = new HashSet<>(staticSafeServices(configCustomizers));
+        Set<String> staticCustomizers = new LinkedHashSet<>(staticSafeServices(configCustomizers));
         staticCustomizers.add(StaticInitConfigBuilder.class.getName());
 
         generateConfigBuilder(generatedClass, reflectiveClass, CONFIG_STATIC_NAME,
@@ -312,12 +314,12 @@ void generateBuilders(
         reflectiveClass.produce(ReflectiveClassBuildItem.builder(CONFIG_STATIC_NAME).build());
 
         // For RunTime Config
-        Set<ConfigClass> runTimeMappings = new HashSet<>();
+        Set<ConfigClass> runTimeMappings = new LinkedHashSet<>();
         runTimeMappings.addAll(runtimeConfigMappings(configMappings));
         runTimeMappings.addAll(configItem.getReadResult().getBuildTimeRunTimeMappings());
         runTimeMappings.addAll(configItem.getReadResult().getRunTimeMappings());
         runTimeMappings.removeAll(ignoreMappings);
-        Set<String> runtimeCustomizers = new HashSet<>(configCustomizers);
+        Set<String> runtimeCustomizers = new LinkedHashSet<>(configCustomizers);
         runtimeCustomizers.add(RuntimeConfigBuilder.class.getName());
 
         generateConfigBuilder(generatedClass, reflectiveClass, CONFIG_RUNTIME_NAME,
@@ -841,7 +843,7 @@ private static Set<String> discoverService(
             Class<?> serviceClass,
             BuildProducer<ReflectiveClassBuildItem> reflectiveClass) throws IOException {
         ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
-        Set<String> services = new HashSet<>();
+        Set<String> services = new TreeSet<>();
         for (String service : classNamesNamedIn(classLoader, SERVICES_PREFIX + serviceClass.getName())) {
             // The discovery includes deployment modules, so we only include services available at runtime
             if (QuarkusClassLoader.isClassPresentAtRuntime(service)) {
@@ -854,7 +856,7 @@ private static Set<String> discoverService(
 
     private static Set<String> staticSafeServices(Set<String> services) {
         ClassLoader classloader = Thread.currentThread().getContextClassLoader();
-        Set<String> staticSafe = new HashSet<>();
+        Set<String> staticSafe = new LinkedHashSet<>();
         for (String service : services) {
             // SmallRye Config services are always safe, but they cannot be annotated with @StaticInitSafe
             if (service.startsWith("io.smallrye.config.")) {

core/processor/src/main/java/io/quarkus/annotation/processor/ExtensionAnnotationProcessor.java

@@ -64,8 +64,7 @@ public synchronized void init(ProcessingEnvironment processingEnv) {
         List<ExtensionProcessor> extensionProcessors = new ArrayList<>();
         extensionProcessors.add(new ExtensionBuildProcessor());
 
-        boolean skipDocs = Boolean.getBoolean("skipDocs") || Boolean.getBoolean("quickly");
-        boolean generateDoc = !skipDocs && !"false".equals(processingEnv.getOptions().get(Options.GENERATE_DOC));
+        boolean generateDoc = !"false".equals(processingEnv.getOptions().get(Options.GENERATE_DOC));
 
         // for now, we generate the old config doc by default but we will change this behavior soon
         if (generateDoc) {

core/processor/src/main/java/io/quarkus/annotation/processor/documentation/config/merger/JavadocMerger.java

@@ -4,9 +4,9 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 
 import io.quarkus.annotation.processor.Outputs;
 import io.quarkus.annotation.processor.documentation.config.model.JavadocElements;
@@ -19,7 +19,7 @@ private JavadocMerger() {
     }
 
     public static JavadocRepository mergeJavadocElements(List<Path> buildOutputDirectories) {
-        Map<String, JavadocElement> javadocElementsMap = new HashMap<>();
+        Map<String, JavadocElement> javadocElementsMap = new TreeMap<>();
 
         for (Path buildOutputDirectory : buildOutputDirectories) {
             Path javadocPath = buildOutputDirectory.resolve(Outputs.QUARKUS_CONFIG_DOC_JAVADOC);

core/processor/src/main/java/io/quarkus/annotation/processor/documentation/config/model/ConfigRoot.java

@@ -3,10 +3,10 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.regex.Pattern;
 
 import io.quarkus.annotation.processor.documentation.config.util.Markers;
@@ -25,7 +25,7 @@ public class ConfigRoot implements ConfigItemCollection {
 
     private final String overriddenDocFileName;
     private final List<AbstractConfigItem> items = new ArrayList<>();
-    private final Set<String> qualifiedNames = new HashSet<>();
+    private final Set<String> qualifiedNames = new TreeSet<>();
 
     public ConfigRoot(Extension extension, String prefix, String overriddenDocPrefix, String overriddenDocFileName) {
         this.extension = extension;

core/processor/src/main/java/io/quarkus/annotation/processor/documentation/config/resolver/ConfigResolver.java

@@ -2,11 +2,11 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
+import java.util.TreeMap;
 import java.util.stream.Collectors;
 
 import javax.lang.model.element.TypeElement;
@@ -72,7 +72,7 @@ public ResolvedModel resolveModel() {
         for (DiscoveryConfigRoot discoveryConfigRoot : configCollector.getConfigRoots()) {
             ConfigRoot configRoot = new ConfigRoot(discoveryConfigRoot.getExtension(), discoveryConfigRoot.getPrefix(),
                     discoveryConfigRoot.getOverriddenDocPrefix(), discoveryConfigRoot.getOverriddenDocFileName());
-            Map<String, ConfigSection> existingRootConfigSections = new HashMap<>();
+            Map<String, ConfigSection> existingRootConfigSections = new TreeMap<>();
 
             configRoot.addQualifiedName(discoveryConfigRoot.getQualifiedName());
 

core/processor/src/main/java/io/quarkus/annotation/processor/documentation/config/scanner/ConfigCollector.java

@@ -2,8 +2,8 @@
 
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
+import java.util.TreeMap;
 
 import io.quarkus.annotation.processor.documentation.config.discovery.DiscoveryConfigGroup;
 import io.quarkus.annotation.processor.documentation.config.discovery.DiscoveryConfigRoot;
@@ -15,22 +15,22 @@ public class ConfigCollector {
     /**
      * Key is qualified name of the class + "." + element name (for instance field or method name)
      */
-    private Map<String, JavadocElement> javadocElements = new HashMap<>();
+    private final Map<String, JavadocElement> javadocElements = new TreeMap<>();
 
     /**
      * Key is the qualified name of the class.
      */
-    private Map<String, DiscoveryConfigRoot> configRoots = new HashMap<>();
+    private final Map<String, DiscoveryConfigRoot> configRoots = new TreeMap<>();
 
     /**
      * Key is the qualified name of the class.
      */
-    private Map<String, DiscoveryConfigGroup> resolvedConfigGroups = new HashMap<>();
+    private final Map<String, DiscoveryConfigGroup> resolvedConfigGroups = new TreeMap<>();
 
     /**
      * Key is the qualified name of the class.
      */
-    private Map<String, EnumDefinition> resolvedEnums = new HashMap<>();
+    private final Map<String, EnumDefinition> resolvedEnums = new TreeMap<>();
 
     public void addJavadocElement(String key, JavadocElement element) {
         javadocElements.put(key, element);

core/processor/src/main/java/io/quarkus/annotation/processor/documentation/config/util/JacksonMappers.java

@@ -1,15 +1,24 @@
 package io.quarkus.annotation.processor.documentation.config.util;
 
+import java.util.Locale;
+import java.util.TimeZone;
+
 import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.MapperFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.fasterxml.jackson.module.paramnames.ParameterNamesModule;
 
 public final class JacksonMappers {
 
     private static final ObjectWriter JSON_OBJECT_WRITER = new ObjectMapper()
+            .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true)
+            .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)
+            .setLocale(Locale.US)
+            .setTimeZone(TimeZone.getTimeZone("UTC"))
             .setSerializationInclusion(JsonInclude.Include.NON_DEFAULT).writerWithDefaultPrettyPrinter();
     private static final ObjectWriter YAML_OBJECT_WRITER = new ObjectMapper(new YAMLFactory())
             .setSerializationInclusion(JsonInclude.Include.NON_DEFAULT).writer();

devtools/gradle/gradle-application-plugin/build.gradle.kts

@@ -25,3 +25,9 @@ gradlePlugin {
 tasks.test {
   systemProperty("kotlin_version", libs.versions.kotlin.get())
 }
+
+// to generate reproducible jars
+tasks.withType<Jar>().configureEach {
+    isPreserveFileTimestamps = false 
+    isReproducibleFileOrder = true   
+}

devtools/gradle/gradle-extension-plugin/build.gradle.kts

@@ -22,3 +22,9 @@ gradlePlugin {
 tasks.withType<Test>().configureEach {
     environment("GITHUB_REPOSITORY", "some/repo")
 }
+
+// to generate reproducible jars
+tasks.withType<Jar>().configureEach {
+    isPreserveFileTimestamps = false 
+    isReproducibleFileOrder = true   
+}

devtools/gradle/gradle-model/build.gradle.kts

@@ -14,6 +14,12 @@ java {
     withJavadocJar()
 }
 
+// to generate reproducible jars
+tasks.withType<Jar>().configureEach {
+    isPreserveFileTimestamps = false
+    isReproducibleFileOrder = true
+}
+
 publishing {
     publications.create<MavenPublication>("maven") {
         artifactId = "quarkus-gradle-model"

devtools/maven/src/main/java/io/quarkus/maven/QuarkusBootstrapProvider.java

@@ -348,6 +348,11 @@ public Properties getBuildSystemProperties(QuarkusBootstrapMojo mojo, boolean qu
 
             effectiveProperties.putIfAbsent("quarkus.application.name", mojo.mavenProject().getArtifactId());
             effectiveProperties.putIfAbsent("quarkus.application.version", mojo.mavenProject().getVersion());
+            // pass the project.build.outputTimestamp to Quarkus packaging subsystem
+            if (mojo.mavenProject().getProperties().containsKey("project.build.outputTimestamp")) {
+                effectiveProperties.putIfAbsent("quarkus.package.output-timestamp",
+                        mojo.mavenProject().getProperties().getProperty("project.build.outputTimestamp"));
+            }
 
             for (Map.Entry<String, String> attribute : mojo.manifestEntries().entrySet()) {
                 if (attribute.getValue() == null) {

independent-projects/bootstrap/app-model/src/main/java/io/quarkus/paths/PathTreeVisit.java

@@ -17,7 +17,10 @@ static void walk(Path root, Path rootDir, Path walkDir, PathFilter pathFilter, M
             PathVisitor visitor) {
         final PathTreeVisit visit = new PathTreeVisit(root, rootDir, pathFilter, multiReleaseMapping);
         try (Stream<Path> files = Files.walk(walkDir)) {
-            final Iterator<Path> i = files.iterator();
+            final Iterator<Path> i = files
+                    // we sort the elements to be deterministic, we compare the toString() to make sure the order is the same on Linux and Windows
+                    .sorted((p1, p2) -> p1.toString().compareTo(p2.toString()))
+                    .iterator();
             while (i.hasNext()) {
                 if (!visit.setCurrent(i.next())) {
                     continue;