Enable release workflow for both conda and wheel and enable nightly release (#300)

Summary:
Changes:
- Remove redundant logic to compare commits between release wheels
- Add `version.txt` as the single source of truth for the version of TorchData.
  - Reason: Conda needs the version without running `python setup.py ...`. Without this PR, the version of TorchData is determined by running `python setup.py ...`.
- Fix a bug in `setup.py` that pytorch version is incorrect
- Add `packaging/build_conda.sh` and `packaging/build_wheel.sh` and related utility for TorchData to build packages and wheels. (taking reference from TorchVision and TorchText)
  - Files in `packaging/torchdata/...` are required by conda-build
  - Reason: Conda won't to run `python setup.py ...`
  - Remove --release/--nightly option for `setup.py` as we would use the script above to do so.
- Add conda package build/upload workflow
- Enable nightly build (Example: https://github.com/pytorch/data/runs/5647039889?check_suite_focus=true)
  - Validated the version number for nightly release (wheel/conda)
  - Validated the dependency for nightly release (wheel/conda)

Pull Request resolved: #300

Reviewed By: NivekT

Differential Revision: D35050961

Pulled By: ejguan

fbshipit-source-id: 41a805d271f6862e031ff145419729efba0a01ec

Author: ejguan

.github/workflows/_build_test_upload.yml

@@ -14,13 +14,13 @@ on:
         type: string
     secrets:
       PYTORCH_BINARY_AWS_ACCESS_KEY_ID:
-        required: false
+        required: true
       PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY:
-        required: false
+        required: true
       PYPI_TOKEN:
         required: false
       CONDA_PYTORCHBOT_TOKEN:
-        required: false
+        required: true
 
 jobs:
   get_release_type:
@@ -34,12 +34,10 @@ jobs:
     steps:
       - name: Get Release Type
         run: |
-          if [[ ${{ inputs.branch }} == release/* ]]; then
-            if [[ ${{ inputs.pre_dev_release }} == true ]]; then
-              RELEASE_TYPE=test
-            else
-              RELEASE_TYPE=official
-            fi
+          if [[ ${{ inputs.branch }} == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then
+            RELEASE_TYPE=official
+          elif [[ ${{ inputs.branch }} == release/* ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then
+            RELEASE_TYPE=test
           elif [[ ${{ inputs.branch }} == main ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then
             RELEASE_TYPE=nightly
           else
@@ -52,148 +50,86 @@ jobs:
           echo "::set-output name=type::$RELEASE_TYPE"
         id: get_release_type
 
-  build_test:
+  wheel_build_test:
     if: ${{ needs.get_release_type.outputs.type }}
     needs: get_release_type
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
-      # TODO: Turn the matrix on after cpp landed
       matrix:
         os:
-          # - macos-latest
+          - macos-latest
           - ubuntu-latest
-          # - windows-latest
+          - windows-latest
         python-version:
-          # - 3.7
-          # - 3.8
+          - 3.7
+          - 3.8
           - 3.9
+          - "3.10"
     steps:
       - name: Setup Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install Pre-release/Nightly-release/Official-release PyTorch
-        run: |
-          pip3 install numpy
-          # Add version requirement to PyTorch except nightly release
-          if [[ ${{ inputs.pytorch_version }} ]]; then
-            PYTORCH_VERSION=torch==${{ inputs.pytorch_version }}
-          else
-            PYTORCH_VERSION=torch
-          fi
-
-          PIP_CHANNEL=${{ needs.get_release_type.outputs.type }}
-
-          if [[ $PIP_CHANNEL == 'official' ]]; then
-            pip3 install "$PYTORCH_VERSION"
-          else
-            pip3 install --pre "$PYTORCH_VERSION" -f "https://download.pytorch.org/whl/$PIP_CHANNEL/cpu/torch_$PIP_CHANNEL.html"
-          fi
       - name: Checkout Source Repository
         uses: actions/checkout@v2
         with:
           ref: ${{ inputs.branch }}
-      - name: Get Hash of the Last Commit from Release
-        if: |
-          needs.get_release_type.outputs.type == 'nightly' ||
-          needs.get_release_type.outputs.type == 'test'
-        run: |
-          pip3 install -r requirements.txt
-          # Using --no-deps here to make sure correct version of PyTorch Core
-          pip3 install --pre torchdata -f "https://download.pytorch.org/whl/${{ needs.get_release_type.outputs.type }}/cpu/torch_${{ needs.get_release_type.outputs.type }}.html" --no-deps
-          pushd ~
-          RELEASE_COMMIT=$(python -c 'import torchdata; print(torchdata.version.git_version)')
-          echo "::set-output name=hash::$RELEASE_COMMIT"
-          popd
-          pip3 uninstall -y torchdata
-        id: release_commit
-      - name: Get Hash of the Last Commit from Source
-        if: |
-          needs.get_release_type.outputs.type == 'nightly' ||
-          needs.get_release_type.outputs.type == 'test'
-        run: |
-          SOURCE_COMMIT=$(git rev-parse HEAD)
-          echo "::set-output name=hash::$SOURCE_COMMIT"
-        id: source_commit
-      - name: Check if Build New TorchData Wheel
-        run: |
-          if [[ ${{ needs.get_release_type.outputs.type }} == 'official' || ( ${{ steps.release_commit.outputs.hash }} != ${{ steps.source_commit.outputs.hash }} ) ]]; then
-            echo 1 > torchdata_${{ matrix.os }}_${{ matrix.python-version }}_whl.txt
-            echo "::set-output name=value::true"
-          else
-            echo 0 > torchdata_${{ matrix.os }}_${{ matrix.python-version }}_whl.txt
-            echo "::set-output name=value::false"
-          fi
-        id: trigger_build
-      # The following steps for pre-release will be skipped if the last commit
-      # from branch is the same as the last commit in release wheel
-      - name: Build and Install TorchData Wheel
-        if: steps.trigger_build.outputs.value == 'true'
+      - name: Install PyTorch and Build TorchData Wheel
+        shell: bash
         env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
           PYTORCH_VERSION: ${{ inputs.pytorch_version }}
-        run: |
-          pip3 install wheel
-          if [[ ${{ needs.get_release_type.outputs.type }} == 'nightly' ]]; then
-            python setup.py bdist_wheel --nightly
-          else
-            python setup.py bdist_wheel --release
-          fi
-          pip3 install dist/torchdata*.whl
+        run: packaging/build_wheel.sh
+      - name: Install TorchData Wheel
+        shell: bash
+        run: pip3 install dist/torchdata*.whl
       - name: Validate TorchData Wheel
-        if: steps.trigger_build.outputs.value == 'true'
+        shell: bash
         run: |
           pip3 install pkginfo
           for pkg in dist/torchdata*.whl; do
               echo "PkgInfo of $pkg:"
               pkginfo $pkg
           done
       - name: Install Test Requirements
-        if: steps.trigger_build.outputs.value == 'true'
         run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile
       - name: Run DataPipes Tests with pytest
-        if: steps.trigger_build.outputs.value == 'true'
         run:
           pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py
           --ignore=test/test_audio_examples.py
-      - name: Upload Wheel and Flag File to Github # Flag File is used to prevent no-file error for download-artifact
+      - name: Upload Wheels to Github
         uses: actions/upload-artifact@v2
         with:
           name: torchdata-artifact
-          path: |
-            dist/torchdata*.whl
-            torchdata_*_whl.txt
+          path: dist/torchdata*.whl
 
-  upload_wheel:
-    needs: [get_release_type, build_test]
+  wheel_upload:
+    if: always() && ${{ needs.get_release_type.outputs.type }}
+    needs: [get_release_type, wheel_build_test]
     runs-on: ubuntu-latest
-    container: continuumio/miniconda3
+    outputs:
+      upload: ${{ steps.trigger_upload.outputs.value }}
     steps:
-      - name: Download Wheels from Github
+      - name: Download Artifacts from Github
+        continue-on-error: true
         uses: actions/download-artifact@v2
         with:
           name: torchdata-artifact
-      - name: Determine if Uploading is needed
+      - name: Determine if Wheel Uploading is needed
         run: |
           upload=false
-          for txt in torchdata_*_whl.txt; do
-            v=`cat $txt`
-            if [ $v -eq 1 ]; then
-              upload=true
-              break
-            fi
+          for txt in torchdata*.whl; do
+            upload=true
+            break
           done
           echo "::set-output name=value::$upload"
         id: trigger_upload
       - name: Display All TorchData Wheels
         if: steps.trigger_upload.outputs.value == 'true'
-        run: ls -lh
-        working-directory: dist
-      - name: Upload Wheels to S3 Storage (Nightly or RC)
-        if: |
-          steps.trigger_upload.outputs.value == 'true' &&
-          ( needs.get_release_type.outputs.type == 'nightly' ||
-          needs.get_release_type.outputs.type == 'test' )
+        run: ls -lh torchdata*.whl
+      - name: Upload Wheels to S3 Storage
+        if: steps.trigger_upload.outputs.value == 'true'
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }}
@@ -205,7 +141,7 @@ jobs:
           fi
           pip3 install awscli
           set -x
-          for pkg in dist/torchdata*.whl; do
+          for pkg in torchdata*.whl; do
             aws s3 cp "$pkg" "$S3_PATH" --acl public-read
           done
       - name: Upload Official Wheels to PYPI
@@ -220,25 +156,91 @@ jobs:
             --username __token__ \
             --password "$PYPI_TOKEN" \
             dist/torchdata*.whl
-      - name: Upload Wheels to Conda (Nightly or Official)
-        if: |
-          steps.trigger_upload.outputs.value == 'true' &&
-          ( needs.get_release_type.outputs.type == 'nightly' ||
-          needs.get_release_type.outputs.type == 'official' )
+
+  conda_build_test:
+    if: ${{ needs.get_release_type.outputs.type }}
+    needs: get_release_type
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macos-latest
+          - ubuntu-latest
+          - windows-latest
+        python-version:
+          - 3.7
+          - 3.8
+          - 3.9
+          - "3.10"
+    steps:
+      - name: Create Conda Env
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          activate-environment: conda_build_env
+      - name: Checkout Source Repository
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ inputs.branch }}
+      - name: Build TorchData for Conda
+        shell: bash -l {0}
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          PYTORCH_VERSION: ${{ inputs.pytorch_version }}
+        run: |
+          conda activate conda_build_env
+          conda install -yq conda-build -c conda-forge
+          packaging/build_conda.sh
+      - name: Upload Conda Package to Github
+        uses: actions/upload-artifact@v2
+        with:
+          name: torchdata-artifact
+          path: conda-bld/*/torchdata-*.tar.bz2
+
+  conda_upload:
+    if: always() && ${{ needs.get_release_type.outputs.type }}
+    needs: [get_release_type, conda_build_test]
+    runs-on: ubuntu-latest
+    container: continuumio/miniconda3
+    outputs:
+      upload: ${{ steps.trigger_upload.outputs.value }}
+    steps:
+      - name: Download Artifacts from Github
+        continue-on-error: true
+        uses: actions/download-artifact@v2
+        with:
+          name: torchdata-artifact
+      - name: Determine if Conda Uploading is needed
+        run: |
+          upload=false
+          for pkg in ./*/torchdata-*.tar.bz2; do
+            upload=true
+            break
+          done
+          echo "::set-output name=value::$upload"
+        id: trigger_upload
+      - name: Display All TorchData Conda Package
+        if: steps.trigger_upload.outputs.value == 'true'
+        run: ls -lh ./*/torchdata-*.tar.bz2
+      - name: Build & Upload Wheels to Conda
+        if: steps.trigger_upload.outputs.value == 'true'
+        shell: bash
         env:
           CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
         run: |
           conda install -yq anaconda-client
-          if [ ${{ needs.get_release_type.outputs.type }} == nightly ]; then
-            CONDA_CHANNEL=pytorch-nightly
-          else
+          if [[ ${{ needs.get_release_type.outputs.type }} == 'official' ]]; then
             CONDA_CHANNEL=pytorch
+          else
+            CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }}
           fi
           set -x
-          # anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload dist/*tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force
+          anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ./*/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force
 
   build_docs:
-    needs: [get_release_type, build_test]
+    if: always() && ( needs.wheel_upload.outputs.upload == 'true' || needs.conda_upload.outputs.upload == 'true' )
+    needs: [get_release_type, wheel_upload, conda_upload]
     runs-on: ubuntu-latest
     steps:
       - name: Setup Python 3.8
@@ -257,35 +259,31 @@ jobs:
           python3 -m pip install setuptools
           python3 -m pip install matplotlib
           sudo apt-get install -y yarn
-      - name: Install Pre-release/Nightly-release/Official-release PyTorch
+      - name: Install PyTorch & TorchData
         run: |
           pip3 install numpy
           # Add version requirement to PyTorch except nightly release
-          if [[ ${{ inputs.pytorch_version }} ]]; then
-            PYTORCH_VERSION=torch==${{ inputs.pytorch_version }}
-          else
+          if [[ -z "${{ inputs.pytorch_version }}" ]]; then
             PYTORCH_VERSION=torch
+          else
+            PYTORCH_VERSION=torch==${{ inputs.pytorch_version }}
           fi
 
           PIP_CHANNEL=${{ needs.get_release_type.outputs.type }}
-
           if [[ $PIP_CHANNEL == 'official' ]]; then
-            pip3 install "$PYTORCH_VERSION"
+            pip3 install "$PYTORCH_VERSION" -f https://download.pytorch.org/whl/torch_stable.html
           else
-            pip3 install --pre "$PYTORCH_VERSION" -f "https://download.pytorch.org/whl/$PIP_CHANNEL/cpu/torch_$PIP_CHANNEL.html"
+            pip3 install --pre "$PYTORCH_VERSION" -f "https://download.pytorch.org/whl/$PIP_CHANNEL/torch_$PIP_CHANNEL.html"
           fi
 
-          if [[ ${{ needs.get_release_type.outputs.type }} == 'nightly' ]]; then
-            python setup.py install --nightly
-          else
-            python setup.py install --release
-          fi
+          pip3 install -r requirements.txt
+          python3 setup.py install
       - name: Check env
         run: echo `which spinx-build`
       - name: Build the docset
         run: |
           cd ./docs
-          pip install -r requirements.txt
+          pip3 install -r requirements.txt
           make html
           cd ..
       - name: Export Target Folder

.github/workflows/nightly_release.yml

@@ -1,24 +1,17 @@
 name: Push Nightly Release
 
 on:
-  # [ Note: Manually Trigger the Workflow ]
-  # 1. Go to Actions under pytorch/data repo
-  # 2. In the left sidebar, click the workflow you want to run
-  # 3. Above the list of workflow runs, select Run workflow
-  # 4. Use the Branch dropdown to select the release/* branch
-  # 5. Click Run workflow
-  workflow_dispatch:
-  # TODO: Automatically trigger nightly release
-  # schedule:
-  #   - cron: 30 23 * * *
+  schedule:
+    - cron: 30 23 * * *
 
 jobs:
   build_test_upload:
     uses: ./.github/workflows/_build_test_upload.yml
     with:
-      branch: ""
+      branch: "main"
       pre_dev_release: true
       pytorch_version: ""
     secrets:
       PYTORCH_BINARY_AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }}
       PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }}
+      CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}