[WIP] Add support for commenting on upstream PRs

.ci/benchmark.py

@@ -1,14 +1,23 @@
 import argparse
 import os
-import sys
+
 from src.benchmark.utils import read_metrics, to_markdown_table
 
 
 def parse_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--path", type=str, required=True, help="Report path.")
-    parser.add_argument("--write-gh-job-summary", action="store_true", help="Write to GitHub job summary.")
-    parser.add_argument("--update-readme", action="store_true", help="Update statistics report in README.md.")
+    parser.add_argument(
+        "--path", type=str, required=True, help="The path of benchmark report."
+    )
+    parser.add_argument(
+        "--output", type=str, required=False, help="The output path of the Markdown report."
+    )
+    parser.add_argument(
+        "--write-gh-job-summary", action="store_true", help="Write to GitHub job summary."
+    )
+    parser.add_argument(
+        "--update-readme", action="store_true", help="Update statistics report in README.md."
+    )
     return parser.parse_args()
 
 
@@ -18,6 +27,11 @@ def generate_report(path: str):
     return html_table
 
 
+def save_output_report(path: str, report):
+    with open(path, "w") as f:
+        f.write(report)
+
+
 def write_job_summary(report):
     summary_path = os.environ["GITHUB_STEP_SUMMARY"]
     with open(summary_path, "a") as f:
@@ -55,6 +69,12 @@ def update_readme(report):
     # Generate statistics report
     report = generate_report(args.path)
 
+    # Output to markdown report
+    if args.output:
+        save_output_report(args.output, report)
+    else:
+        print(report)
+
     # Write to workflow job summary
     if args.write_gh_job_summary:
         write_job_summary(report)

.github/workflows/_ascend_npu_benchmark.yml

@@ -32,7 +32,7 @@ defaults:
     shell: bash -el {0}
 
 jobs:
-  test:
+  benchmark:
     name: run benchmarks for torch_npu
     runs-on: ${{ inputs.runner }}
     env:

.github/workflows/_pytorch_pr_comment.yml

@@ -0,0 +1,70 @@
+name: '_pytorch_pr_comment'
+
+on:
+  workflow_call:
+    inputs:
+      repository:
+        required: false
+        type: string
+        default: 'pytorch/pytorch'
+        description: 'The full name of the repository containing the pull request'
+      pr-number:
+        required: true
+        type: number
+        description: 'The number of the pull request'
+      pr-comment-author:
+        required: true
+        type: string
+        description: 'The author of the pull request comment'
+    secrets:
+      pr-token:
+        required: true
+        description: 'A token used to create or update a pull request comment'
+
+jobs:
+  comment:
+    name: Create or update the pull request comment
+    runs-on: ubuntu-latest
+    steps:
+      # Find the first comment containing the specified string
+      # and by the specified author.
+      # See: https://github.com/peter-evans/find-comment
+      - name: Find comment
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          token: ${{ secrets.pr-token }}
+          repository: ${{ inputs.repository }}
+          issue-number: ${{ inputs.pr-number }}
+          comment-author: ${{ inputs.pr-comment-author }}
+          body-includes: '<!-- ACCELERATOR TESTINFRA START -->'
+
+      - name: Show comment info
+        run: |
+          echo "comment id: ${{ steps.fc.outputs.comment-id }}"
+          echo "comment node id: ${{ steps.fc.outputs.comment-node-id }}"
+          echo "comment body: ${{ steps.fc.outputs.comment-body }}"
+          echo "comment author: ${{ steps.fc.outputs.comment-author }}"
+          echo "comment created at: ${{ steps.fc.outputs.comment-created-at }}"
+
+      # Create or update the comment just found.
+      # See: https://github.com/peter-evans/create-or-update-comment
+      - name: Create or update comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          token: ${{ secrets.pr-token }}
+          repository: ${{ inputs.repository }}
+          issue-number: ${{ inputs.pr-number }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          edit-mode: replace
+          # TODO: use body-path instead
+          body: |
+            <!-- ACCELERATOR TESTINFRA START -->
+
+            ## Report
+
+            - something
+            - something else
+            - other things
+
+            <!-- ACCELERATOR TESTINFRA END -->

.github/workflows/ascend_npu_test.yml

@@ -48,9 +48,10 @@ on:
         required: true
         type: choice
         options:
-          - ascendai/cann:7.1-openeuler2203sp2
-          - ascendai/cann:8.0.rc2.alpha003-910b-ubuntu22.04-py3.9
-          - ascendai/cann:8.0.rc3.alpha002-910b-ubuntu22.04-py3.9
+          - 7.0.1.beta1-910b-ubuntu22.04-py3.8
+          - 8.0.rc1.beta1-910b-ubuntu22.04-py3.8
+          - 8.0.rc2.beta1-910b-ubuntu22.04-py3.9
+          - 8.0.rc3.beta1-910b-ubuntu22.04-py3.10
           - ascendai/cann:latest
         default: 'ascendai/cann:latest'
         description: 'The docker image which will be loaded'
@@ -72,7 +73,7 @@ on:
 # Only cancel the previous runs when triggered by a pull request
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'repository_dispatch' }}
 
 jobs:
   prepare:
@@ -126,3 +127,17 @@ jobs:
       artifact_name: ${{ needs.build.outputs.artifact_name }}
     secrets:
       pr-token: ${{ secrets.COSDT_BOT_TOKEN }}
+
+  # TODO
+  pr-comment:
+    name: Comment report on upstream PRs
+    needs:
+      - benchmark
+    if: ${{ github.event_name == 'repository_dispatch' }}
+    uses: ./.github/workflows/_pytorch_pr_comment.yml
+    with:
+      repository: cosdt/pytorch-upstream
+      pr-number: 1
+      pr-comment-author: cosdt-bot
+    secrets:
+      pr-token: ${{ secrets.COSDT_BOT_TOKEN }}