[pull] master from Netflix:master

.github/workflows/publish.yml

@@ -3,68 +3,8 @@ on:
   release:
     types: [published]
 jobs:
-  test:
-    name: tests / ${{ matrix.lang }} tests on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        lang: [Python, R]
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Python 2.x
-      if: matrix.lang == 'Python'
-      uses: actions/setup-python@v2
-      with:
-        python-version: '2.x'
-
-    - name: Install Python 2.x dependencies
-      if: matrix.lang == 'Python'
-      run: |
-        python2 -m pip install --upgrade pip
-        python2 -m pip install tox
-
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.7'
-
-    - name: Install Python 3.x dependencies
-      run: |
-        python3 -m pip install --upgrade pip
-        python3 -m pip install tox numpy
-
-    - name: Set up R 4.1
-      if: matrix.lang == 'R'
-      uses: r-lib/actions/setup-r@v1
-      with:
-        r-version: '4.1.1' 
-
-    - name: Install R 4.1 system dependencies
-      if: matrix.lang == 'R' && matrix.os == 'ubuntu-latest'
-      run: sudo apt-get update; sudo apt-get install -y libcurl4-openssl-dev qpdf libgit2-dev
-
-    - name: Install R 4.1 Rlang dependencies
-      if: matrix.lang == 'R'
-      run: |
-        python3 -m pip install . 
-        Rscript -e 'install.packages("devtools", repos="https://cloud.r-project.org", Ncpus=8)'
-        Rscript -e 'devtools::install_deps("R", dependencies=TRUE, repos="https://cloud.r-project.org", upgrade="default")'
-        R CMD INSTALL R
-        Rscript -e 'install.packages(c("data.table", "caret", "glmnet", "Matrix", "rjson"), repos="https://cloud.r-project.org", Ncpus=8)'   
-
-    - name: Execute R tests
-      if: matrix.lang == 'R'
-      run: |
-        cd R/tests
-        Rscript run_tests.R
-
-    - name: Execute Python tests
-      if: matrix.lang == 'python'
-      run: tox
-
+  test: 
+    uses: './.github/workflows/test.yml'
   deploy:
     needs: test
     runs-on: ubuntu-latest

.github/workflows/s3_test.yml

@@ -1,6 +1,9 @@
 name: pr-datastore-tests
 
 on:
+  push:
+    branches:
+    - master
   pull_request_target:
     types:
       - opened
@@ -37,4 +40,4 @@ jobs:
         METAFLOW_S3_TEST_ROOT: ${{ secrets.METAFLOW_S3_TEST_ROOT }}
       run: |
         cd test/data
-        PYTHONPATH=$(pwd)/../../ python3 -m pytest --benchmark-skip -x -s -v
+        PYTHONPATH=$(pwd)/../../ python3 -m pytest --benchmark-skip -x -s -v

.github/workflows/test.yml

@@ -6,58 +6,62 @@ on:
   pull_request:
     branches:
     - master
+  workflow_call:
+
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
     - uses: pre-commit/[email protected]
-  test:
-    name: tests / ${{ matrix.lang }} tests on ${{ matrix.os }}
+
+  Python:
+    name: core / Python ${{ matrix.ver }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        lang: [Python, R]
+        ver: ['3.5', '3.6', '3.7', '3.8', '3.9','3.10',]
 
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 2.x
-      if: matrix.lang == 'Python'
+    - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '2.x'
+        python-version: ${{ matrix.ver }}
 
-    - name: Install Python 2.x dependencies
-      if: matrix.lang == 'Python'
+    - name: Install Python ${{ matrix.ver }} dependencies
       run: |
-        python2 -m pip install --upgrade pip
-        python2 -m pip install tox
+        python3 -m pip install --upgrade pip setuptools
+        python3 -m pip install tox numpy
 
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.7'
+    - name: Execute Python tests
+      run: tox
 
-    - name: Install Python 3.x dependencies
-      run: |
-        python3 -m pip install --upgrade pip
-        python3 -m pip install tox numpy
-
-    - name: Set up R 4.1
-      if: matrix.lang == 'R'
-      uses: r-lib/actions/setup-r@v1
+  R:
+    name: core / R ${{ matrix.ver }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        ver: ['4.0', '4.1']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up ${{ matrix.ver }}
+      uses: r-lib/actions/setup-r@v2
       with:
-        r-version: '4.1.1'
+        r-version: ${{ matrix.ver }}
 
-    - name: Install R 4.1 system dependencies
-      if: matrix.lang == 'R' && matrix.os == 'ubuntu-latest'
-      run: sudo apt-get update; sudo apt-get install -y libcurl4-openssl-dev qpdf libgit2-dev
+    - name: Install R ${{ matrix.ver }} system dependencies
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get update; sudo apt-get install -y libcurl4-openssl-dev qpdf libgit2-dev libharfbuzz-dev libfribidi-dev
 
-    - name: Install R 4.1 Rlang dependencies
-      if: matrix.lang == 'R'
+    - name: Install R ${{ matrix.ver }} Rlang dependencies
       run: |
         python3 -m pip install . 
         Rscript -e 'install.packages("devtools", repos="https://cloud.r-project.org", Ncpus=8)'
@@ -66,11 +70,8 @@ jobs:
         Rscript -e 'install.packages(c("data.table", "caret", "glmnet", "Matrix", "rjson"), repos="https://cloud.r-project.org", Ncpus=8)'   
 
     - name: Execute R tests
-      if: matrix.lang == 'R'
       run: |
         cd R/tests
         Rscript run_tests.R
 
-    - name: Execute Python tests
-      if: matrix.lang == 'python'
-      run: tox
+

.pre-commit-config.yaml

@@ -6,7 +6,9 @@ repos:
       - id: check-yaml
       - id: check-json
   - repo: https://github.com/ambv/black
-    rev: 21.9b0
+    rev: 21.12b0
     hooks:
       - id: black
         language_version: python3
+        exclude: "^metaflow/_vendor/"
+        additional_dependencies: ["click<8.1.0"]

R/README.md

@@ -10,7 +10,7 @@ Getting up and running with Metaflow is easy. Install Metaflow from [github](htt
 
 >```R
 >devtools::install_github("Netflix/metaflow", subdir="R")
->metaflow::install()
+>metaflow::install_metaflow()
 >```
 
 and access tutorials by typing:

README.md

@@ -51,4 +51,5 @@ We welcome contributions to Metaflow. Please see our [contribution guide](https:
 
 ### Code style
 
-We use [black](https://black.readthedocs.io/en/stable/) as a code formatter. The easiest way to ensure your commits are always formatted with the correct version of `black` it is to use [pre-commit](https://pre-commit.com/): install it and then run `pre-commit install` once in your local copy of the repo. 
+We use [black](https://black.readthedocs.io/en/stable/) as a code formatter. The easiest way to ensure your commits are always formatted with the correct version of `black` it is to use [pre-commit](https://pre-commit.com/): install it and then run `pre-commit install` once in your local copy of the repo.
+