DO NOT MERGE - PROD tracking - Beta v4.0 prod branch

.github/workflows/manual-release.yml

@@ -1,6 +1,9 @@
 # The purpose of this workflow is to upload the necessary assets when
 # a release is made manually on github.
 name: Manual Release Assets
+permissions:
+  contents: write
+  packages: write
 
 on:
   release:
@@ -13,6 +16,9 @@ on:
         default: "v1.2.3"
 
 jobs:
+  # ==================================================================
+  # Build - for release and for tests
+  # ==================================================================
   build:
     runs-on: ubuntu-latest
     steps:
@@ -45,8 +51,113 @@ jobs:
           jar_asset: ${{ github.workspace }}/plugins/build/libs/linea-tracer-${{ steps.config.outputs.TAG }}.jar
           zip_asset: ${{ github.workspace }}/plugins/build/distributions/linea-tracer-${{ steps.config.outputs.TAG }}.zip
           GH_TOKEN: ${{ github.token }}
+
+      - name: Set up GCC
+        uses: egor-tensin/setup-gcc@eaa888eb19115a521fa72b65cd94fe1f25bbcaac #v1.3
+
+      - name: Build without tests
+        run: ./gradlew build -x test -x spotlessCheck
+        env:
+          JAVA_OPTS: -Xmx2g -Dorg.gradle.daemon=false
+
+      - name: Store distribution artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: distributions
+          path: arithmetization/build/libs
+
+      - name: Run spotless
+        run: ./gradlew --no-daemon --parallel clean spotlessCheck
+
+  # ==================================================================
+  # Unit Tests
+  # ==================================================================
+  unit-tests-london:
+    needs: [ build ]
+    uses: ./.github/workflows/reusable-unit-tests.yml
+    with:
+      zkevm_fork: LONDON
+      tests-with-ssh: false
+
+  unit-tests-paris:
+    needs: [ build ]
+    uses: ./.github/workflows/reusable-unit-tests.yml
+    with:
+      zkevm_fork: PARIS
+      tests-with-ssh: false
+
+  unit-tests-shanghai:
+    needs: [ build ]
+    uses: ./.github/workflows/reusable-unit-tests.yml
+    with:
+      zkevm_fork: SHANGHAI
+      tests-with-ssh: false
+
+  unit-tests-cancun:
+    needs: [ build ]
+    uses: ./.github/workflows/reusable-unit-tests.yml
+    with:
+      zkevm_fork: CANCUN
+      tests-with-ssh: false
+
+  unit-tests-prague:
+    needs: [ build ]
+    uses: ./.github/workflows/reusable-unit-tests.yml
+    with:
+      zkevm_fork: PRAGUE
+      tests-with-ssh: false
+
+  # ==================================================================
+  # Fast Replay Tests
+  # ==================================================================
+  replay-tests:
+    needs: [ build ]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          submodules: false
+
+      - name: Setup Environment
+        uses: ./.github/actions/setup-environment
+        with:
+          enable-ssh: false
+
+      - name: Run replay tests
+        run: GOMEMLIMIT=26GiB ./gradlew :arithmetization:fastReplayTests
+        env:
+          JAVA_OPTS: -Dorg.gradle.daemon=false
+          REPLAY_TESTS_PARALLELISM: 4
+          GOCORSET_FLAGS: -b1024 -v --ansi-escapes=false --report --air
+          ZKEVM_FORK: LONDON
+
+      - name: Upload test report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: replay-tests-report
+          path: arithmetization/build/reports/tests/**/*
+
+      - name: Upload jacoco fast replay tests coverage report
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: jacoco-fast-replay-tests-coverage-report
+          path: arithmetization/build/reports/jacoco/jacocoFastReplayTestsReport/**/*
+
+      - name: Upload jacoco fast replay tests exec file
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: jacoco-fast-replay-tests-exec-file
+          path: arithmetization/build/jacoco/fastReplayTests.exec
+
+  # ==================================================================
+  # Publish release post tests
+  # ==================================================================
   publish:
-    needs: build
+    needs: [ build ]
     if: github.event_name != 'pull_request'
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     env:

.github/workflows/reference-blockchain-tests.yml

@@ -70,12 +70,109 @@ jobs:
       failed_constraint: ${{ inputs.failed_constraint || '' }}
       commit: ${{ inputs.commit || '' }}
 
+  # For Prague, we run blockchain tests coming from the execution spec tests
   blockchain-reference-tests-prague:
-    uses: ./.github/workflows/reusable-blockchain-tests.yml
-    secrets: inherit
-    with:
-      zkevm_fork: PRAGUE
-      test_filter: ${{ inputs.test_filter || 'BlockchainReferenceTest_*' }}
-      failed_module: ${{ inputs.failed_module || '' }}
-      failed_constraint: ${{ inputs.failed_constraint || '' }}
-      commit: ${{ inputs.commit || '' }}
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          submodules: recursive
+      - name: get-branch-name
+        id: extract_branch
+        run: |
+          echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+
+      - name: Setup Test Environment
+        uses: ./.github/actions/setup-environment
+
+      - name: Download execution spec fixtures
+        run: ./gradlew :reference-tests:downloadExecutionSpecFixtures
+        env:
+          JAVA_OPTS: -Dorg.gradle.daemon=false
+
+      - name: Generate execution spec blockchain reference tests
+        run: ./gradlew :reference-tests:generateExecutionSpecBlockchainTests -Dorg.gradle.caching=true
+        env:
+          JAVA_OPTS: -Dorg.gradle.daemon=false
+
+      - name: Download artifact
+        uses: dawidd6/action-download-artifact@ac66b43f0e6a346234dd65d4d0c8fbb31cb316e5 # v11
+        if: ${{ inputs.failed_module != '' }}
+        with:
+          name: failedBlockchainReferenceTests.json
+          path: ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          commit: ${{ inputs.commit || '' }}
+          if_no_artifact_found: ignore
+
+      - name: Rename filter input file
+        if: ${{ inputs.failed_module != '' }}
+        run: mv ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/failedBlockchainReferenceTests.json ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/failedBlockchainReferenceTests-input.json
+
+      - name: Run reference execution spec blockchain tests
+        run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew referenceExecutionSpecBlockchainTests -x spotlessCheck --tests "${{ inputs.test_filter || 'BlockchainReferenceTest_*' }}"
+        timeout-minutes: 360
+        env:
+          REFERENCE_TESTS_PARALLELISM: 3
+          JAVA_OPTS: -Dorg.gradle.daemon=false
+          GOCORSET_FLAGS: -b1024 -v --ansi-escapes=false --report --air
+          FAILED_TEST_JSON_DIRECTORY: ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/
+          FAILED_MODULE: ${{ inputs.failed_module || '' }}
+          FAILED_CONSTRAINT: ${{ inputs.failed_constraint || '' }}
+          REFERENCE_TEST_FILTER_INPUT: failedBlockchainReferenceTests.json
+          ZKEVM_FORK: PRAGUE
+
+      - name: Upload artifact
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: failedBlockchainReferenceTests-PRAGUE.json
+          path: ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/
+
+      - name: Upload test report
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: blockchain-reference-tests-report-PRAGUE
+          path: reference-tests/build/reports/tests/**/*
+
+      - name: Upload jacoco exec spec blockchain tests coverage report
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: jacoco-blockchain-tests-coverage-report-PRAGUE
+          path: reference-tests/build/reports/jacoco/jacocoReferenceExecutionSpecBlockchainTestsReport/**/*
+
+      - name: Upload jacoco blockchain tests exec file
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: jacoco-blockchain-tests-exec-file-PRAGUE
+          path: reference-tests/build/jacoco/referenceExecutionSpecBlockchainTests.exec
+
+      - name: Extract Metrics
+        if: ${{ failure() || cancelled() }}
+        run: |
+          # Gather metrics
+          SUCCESS_COUNTER=$(cat $JSON_REPORT | sed -e "s/.*\"successCounter\":\([0-9]*\).*/\1/")
+          FAILED_COUNTER=$(cat $JSON_REPORT | sed -e "s/.*\"failedCounter\":\([0-9]*\).*/\1/")
+          ABORTED_COUNTER=$(cat $JSON_REPORT | sed -e "s/.*\"abortedCounter\":\([0-9]*\).*/\1/")
+          DISABLED_COUNTER=$(cat $JSON_REPORT | sed -e "s/.*\"disabledCounter\":\([0-9]*\).*/\1/")
+          # Set environment variables
+          echo "SUCCESS=$SUCCESS_COUNTER" >> $GITHUB_ENV
+          echo "FAILED=$FAILED_COUNTER" >> $GITHUB_ENV
+          echo "ABORTED=$ABORTED_COUNTER" >> $GITHUB_ENV
+          echo "DISABLED=$DISABLED_COUNTER" >> $GITHUB_ENV
+        env:
+          JSON_REPORT: ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/BlockchainReferenceTestOutcome.json
+
+      - name: Failure Notification
+        if: github.ref == 'refs/heads/arith-dev' && (failure() || cancelled())
+        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: webhook-trigger
+          payload: |
+            name: "Daily Blockchain"
+            status: "${{ env.SUCCESS }} successful, ${{ env.FAILED }} failed, ${{ env.ABORTED }} aborted, ${{ env.DISABLED }} disabled"

.github/workflows/reusable-blockchain-tests.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Setup Test Environment
         uses: ./.github/actions/setup-environment
 
-      - name: Generate block chain reference tests
+      - name: Generate blockchain reference tests
         run: ./gradlew :reference-tests:generateBlockchainReferenceTests -Dorg.gradle.caching=true
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
@@ -82,7 +82,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: blockchain-refrence-tests-report-${{ inputs.zkevm_fork }}
+          name: blockchain-reference-tests-report-${{ inputs.zkevm_fork }}
           path: reference-tests/build/reports/tests/**/*
 
       - name: Upload jacoco blockchain tests coverage report

arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/AccountSnapshot.java

@@ -281,6 +281,6 @@ public EWord tracedCodeHash() {
   }
 
   public boolean tracedHasCode() {
-    return !this.tracedCodeHash().equals(EWord.of(Hash.EMPTY));
+    return !this.deploymentStatus() && !this.code().isEmpty();
   }
 }

arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/section/skip/TxSkipSection.java

@@ -126,7 +126,8 @@ public void resolveAtEndTransaction(
     final Wei value = (Wei) txMetadata.getBesuTransaction().getValue();
 
     if (txMetadata.senderAddressCollision()) {
-      final BigInteger gasUsed = BigInteger.valueOf(txMetadata.getGasUsed());
+      final BigInteger gasUsed =
+          BigInteger.valueOf(txMetadata.getGasLimit() - txMetadata.getGasRefunded());
       final BigInteger gasPrice = BigInteger.valueOf(txMetadata.getEffectiveGasPrice());
       final BigInteger gasCost = gasUsed.multiply(gasPrice);
       senderNew =

arithmetization/src/main/java/net/consensys/linea/zktracer/module/rlptxn/cancun/CancunRlpTxn.java

@@ -36,6 +36,6 @@ public void traceEndTx(TransactionProcessingMetadata tx) {
   @Override
   protected void traceOperation(RlpTxnOperation op, int userTxNumber, Trace.Rlptxn trace) {
     final CancunRlpTxnOperation cancunOp = (CancunRlpTxnOperation) op;
-    cancunOp.trace(trace);
+    cancunOp.trace(trace, operations.size());
   }
 }

arithmetization/src/main/java/net/consensys/linea/zktracer/module/rlptxn/cancun/CancunRlpTxnOperation.java

@@ -97,7 +97,8 @@ public CancunRlpTxnOperation(RlpUtils rlpUtils, Trm trm, TransactionProcessingMe
     phaseSectionList.add(new IntegerPhaseSection(rlpUtils, S, tx));
   }
 
-  protected void trace(Trace.Rlptxn trace) {
+  protected void trace(Trace.Rlptxn trace, int userTransactionNumberMax) {
+    tracedValues.userTxnNumberMax(userTransactionNumberMax);
     for (PhaseSection section : phaseSectionList) {
       section.trace(trace, tracedValues);
     }

arithmetization/src/main/java/net/consensys/linea/zktracer/module/rlptxn/cancun/GenericTracedValue.java

@@ -34,6 +34,7 @@ public class GenericTracedValue {
   private final boolean type4;
   private int rlpLtByteSize;
   private int rlpLxByteSize;
+  @Setter @Getter private int userTxnNumberMax;
 
   public GenericTracedValue(TransactionProcessingMetadata tx) {
     this.tx = tx;

arithmetization/src/main/java/net/consensys/linea/zktracer/module/rlptxn/cancun/phaseSection/PhaseSection.java

@@ -80,7 +80,8 @@ public void traceTransactionConstantValues(Trace.Rlptxn trace, GenericTracedValu
         .replayProtection(tracedValues.tx().replayProtection())
         .yParity(tracedValues.tx().yParity())
         .requiresEvmExecution(tracedValues.tx().requiresEvmExecution())
-        .isDeployment(tracedValues.tx().isDeployment());
+        .isDeployment(tracedValues.tx().isDeployment())
+        .proverUserTxnNumberMax(tracedValues.userTxnNumberMax());
   }
 
   public void tracePostValues(Trace.Rlptxn trace, GenericTracedValue tracedValues) {