Merge branch 'main' into enh-parse-images-from-helm

nebari-dev · Feb 7, 2025 · fe5546c · fe5546c
2 parents 6cfe0d3 + 6aa1bfa
commit fe5546c
Show file tree

Hide file tree

Showing 120 changed files with 2,264 additions and 1,692 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,3 @@
+# The Nebari Maintenance Team will be the default owners for everything in the repo.
+# The Maintenance team will be assigned to review by default when someone opens a pull request.
+*       @nebari-dev/maintenance-team
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -93,7 +93,6 @@ body:
         - "Azure"
         - "GCP"
         - "AWS"
-        - "Digital Ocean"
         - "kind"
     validations:
       required: false

diff --git a/.github/actions/health-check/action.yml b/.github/actions/health-check/action.yml
@@ -0,0 +1,19 @@
+name: health-check
+description: "Check health of Nebari deployment"
+
+inputs:
+  domain:
+    description: Domain name
+    required: true
+
+runs:
+  using: composite
+
+  steps:
+    - name: List kubernetes components
+      shell: bash
+      run: kubectl get --all-namespaces all,cm,secret,pv,pvc,ing
+
+    - name: Check if JupyterHub login page is accessible
+      shell: bash
+      run: curl --insecure --include 'https://${{ inputs.domain }}/hub/home'
diff --git a/.github/actions/init-local/action.yml b/.github/actions/init-local/action.yml
@@ -0,0 +1,81 @@
+name: init-local
+description: "Initialize Nebari config for local deployment"
+
+inputs:
+  directory:
+    description: "Path to directory to initialize in"
+    required: false
+    default: './local-deployment'
+
+outputs:
+  directory:
+    description: "Path to config directory"
+    value: ${{ steps.metadata.outputs.directory }}
+  config:
+    description: "Path to Nebari config"
+    value: ${{ steps.metadata.outputs.config }}
+  project:
+    description: "Project name"
+    value: ${{ steps.metadata.outputs.project }}
+  domain:
+    description: "Domain name"
+    value: ${{ steps.metadata.outputs.domain }}
+
+runs:
+  using: composite
+
+  steps:
+    - shell: bash
+      id: metadata
+      run: |
+        # Setup metadata
+        DIRECTORY=$(realpath '${{ inputs.directory }}')
+        mkdir --parents "${DIRECTORY}"
+        echo "directory=${DIRECTORY}" | tee --append "${GITHUB_OUTPUT}"
+
+        CONFIG="${DIRECTORY}/nebari-config.yaml"
+        echo "config=${CONFIG}" | tee --append "${GITHUB_OUTPUT}"
+
+        PROJECT='github-actions'
+        echo "project=${PROJECT}" | tee --append "${GITHUB_OUTPUT}"
+
+        DOMAIN='github-actions.nebari.dev'
+        nslookup "${DOMAIN}"
+        echo "domain=${DOMAIN}" | tee --append "${GITHUB_OUTPUT}"
+
+    - shell: bash -l {0}
+      id: init
+      working-directory: ${{ steps.metadata.outputs.directory }}
+      run: |
+        nebari init local \
+          --project-name '${{ steps.metadata.outputs.project }}' \
+          --domain-name '${{ steps.metadata.outputs.domain }}' \
+          --auth-provider password \
+          --output '${{ steps.metadata.outputs.config }}'
+
+    - shell: bash
+      run: |
+        # Update nebari config for CI
+
+        # Change default JupyterLab theme
+        cat >> '${{ steps.metadata.outputs.config }}' <<- EOM
+        jupyterlab:
+          default_settings:
+            "@jupyterlab/apputils-extension:themes":
+              theme: JupyterLab Dark
+        EOM
+
+        # Change default value for minio persistence size
+        cat >> '${{ steps.metadata.outputs.config }}' <<- EOM
+        monitoring:
+          enabled: true
+          overrides:
+            minio:
+              persistence:
+                size: 1Gi
+        EOM
+
+    - shell: bash
+      run: |
+        # Display Nebari config
+        cat '${{ steps.metadata.outputs.config }}'
diff --git a/.github/actions/setup-local/action.yml b/.github/actions/setup-local/action.yml
@@ -0,0 +1,30 @@
+name: setup-local
+description: "Setup runner for local deployment"
+
+inputs:
+  kubectl-version:
+    description: "Version of kubectl to install"
+    required: false
+    default: "1.19.16"
+
+runs:
+  using: composite
+
+  steps:
+    - uses: azure/setup-kubectl@v4
+      with:
+        version: v${{ inputs.kubectl-version }}
+
+    - shell: bash
+      run: |
+        # Enable docker permissions for user
+        sudo docker ps
+        sudo usermod -aG docker $USER && newgrp docker
+
+        docker info
+        docker ps
+
+    - shell: bash
+      run: |
+        # Get routing table for docker pods
+        ip route
diff --git a/.github/workflows/test-provider.yaml b/.github/workflows/test-provider.yaml
@@ -29,6 +29,12 @@ on:
         required: true
         type: string
 
+env:
+  ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+  ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+  ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+  PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+
 jobs:
   test-render-providers:
     # Prevents the execution of this test under the following conditions:
@@ -49,7 +55,6 @@ jobs:
         provider:
           - aws
           - azure
-          - do
           - gcp
           - local
           - existing
@@ -73,33 +78,14 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Retrieve secret from Vault
-        uses: hashicorp/[email protected]
-        with:
-          method: jwt
-          url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200"
-          namespace: "admin/quansight"
-          role: "repository-nebari-dev-nebari-role"
-          secrets: |
-            kv/data/repository/nebari-dev/nebari/amazon_web_services/nebari-dev-ci role_name | AWS_ROLE_ARN;
-            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci project_id | PROJECT_ID;
-            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci workload_identity_provider | GCP_WORKFLOW_PROVIDER;
-            kv/data/repository/nebari-dev/nebari/google_cloud_platform/nebari-dev-ci/github-nebari-dev-repo-ci service_account_name | GCP_SERVICE_ACCOUNT;
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci client_id | ARM_CLIENT_ID;
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci tenant_id | ARM_TENANT_ID;
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci subscription_id | ARM_SUBSCRIPTION_ID;
-            kv/data/repository/nebari-dev/nebari/shared_secrets DIGITALOCEAN_TOKEN | DIGITALOCEAN_TOKEN;
-            kv/data/repository/nebari-dev/nebari/shared_secrets SPACES_ACCESS_KEY_ID | SPACES_ACCESS_KEY_ID;
-            kv/data/repository/nebari-dev/nebari/shared_secrets SPACES_SECRET_ACCESS_KEY | SPACES_SECRET_ACCESS_KEY;
-
       - name: 'Authenticate to GCP'
         if: ${{ matrix.provider == 'gcp' }}
         uses: 'google-github-actions/auth@v1'
         with:
           token_format: access_token
           create_credentials_file: 'true'
-          workload_identity_provider: ${{ env.GCP_WORKFLOW_PROVIDER }}
-          service_account: ${{ env.GCP_SERVICE_ACCOUNT }}
+          workload_identity_provider: ${{ secrets.GCP_WORKFLOW_PROVIDER }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
 
       - name: Set required environment variables
         if: ${{ matrix.provider == 'gcp' }}
@@ -110,17 +96,17 @@ jobs:
         if: ${{ matrix.provider == 'aws' }}
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
           role-session-name: github-action
           aws-region: us-east-1
 
       - name: 'Azure login'
         if: ${{ matrix.provider == 'azure' }}
         uses: azure/login@v1
         with:
-          client-id: ${{ env.ARM_CLIENT_ID }}
-          tenant-id: ${{ env.ARM_TENANT_ID }}
-          subscription-id: ${{ env.ARM_SUBSCRIPTION_ID }}
+          client-id: ${{ secrets.ARM_CLIENT_ID }}
+          tenant-id: ${{ secrets.ARM_TENANT_ID }}
+          subscription-id: ${{ secrets.ARM_SUBSCRIPTION_ID }}
 
       - name: Install Nebari
         run: |

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -34,6 +34,7 @@ jobs:
           - "3.10"
           - "3.11"
           - "3.12"
+          - "3.13"
       fail-fast: false
     concurrency:
       group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ matrix.python-version }}

diff --git a/.github/workflows/test_aws_integration.yaml b/.github/workflows/test_aws_integration.yaml
@@ -5,11 +5,6 @@ on:
     - cron: "0 0 * * MON"
   workflow_dispatch:
     inputs:
-      branch:
-        description: 'Nebari branch to deploy, test, destroy'
-        required: true
-        default: main
-        type: string
       image-tag:
         description: 'Nebari image tag created by the nebari-docker-images repo'
         required: true
@@ -30,9 +25,8 @@ on:
 
 env:
   AWS_DEFAULT_REGION: "us-west-2"
-  NEBARI_GH_BRANCH: ${{ github.event.inputs.branch || 'main' }}
   NEBARI_IMAGE_TAG: ${{ github.event.inputs.image-tag || 'main' }}
-  TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}∏
+  TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}
 
 jobs:
   test-aws-integration:
@@ -45,7 +39,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          ref: ${{ env.NEBARI_GH_BRANCH }}
           fetch-depth: 0
 
       - name: Set up Python
@@ -58,21 +51,10 @@ jobs:
           pip install .[dev]
           playwright install
 
-      - name: Retrieve secret from Vault
-        uses: hashicorp/[email protected]
-        with:
-          method: jwt
-          url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200"
-          namespace: "admin/quansight"
-          role: "repository-nebari-dev-nebari-role"
-          secrets: |
-            kv/data/repository/nebari-dev/nebari/amazon_web_services/nebari-dev-ci role_name | AWS_ROLE_ARN;
-            kv/data/repository/nebari-dev/nebari/cloudflare/[email protected]/nebari-dev-ci token | CLOUDFLARE_TOKEN;
-
       - name: Authenticate to AWS
         uses: aws-actions/configure-aws-credentials@v1
         with:
-          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
           role-session-name: github-action
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
 
@@ -84,3 +66,4 @@ jobs:
           NEBARI_SECRET__default_images__jupyterhub: "quay.io/nebari/nebari-jupyterhub:${{ env.NEBARI_IMAGE_TAG }}"
           NEBARI_SECRET__default_images__jupyterlab: "quay.io/nebari/nebari-jupyterlab:${{ env.NEBARI_IMAGE_TAG }}"
           NEBARI_SECRET__default_images__dask_worker: "quay.io/nebari/nebari-dask-worker:${{ env.NEBARI_IMAGE_TAG }}"
+          CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }}
diff --git a/.github/workflows/test_azure_integration.yaml b/.github/workflows/test_azure_integration.yaml
@@ -5,11 +5,6 @@ on:
     - cron: "0 0 * * MON"
   workflow_dispatch:
     inputs:
-      branch:
-        description: 'Nebari branch to deploy, test, destroy'
-        required: true
-        default: main
-        type: string
       image-tag:
         description: 'Nebari image tag created by the nebari-docker-images repo'
         required: true
@@ -28,7 +23,6 @@ on:
         - error
 
 env:
-  NEBARI_GH_BRANCH: ${{ github.event.inputs.branch || 'main' }}
   NEBARI_IMAGE_TAG: ${{ github.event.inputs.image-tag || 'main' }}
   TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}
 
@@ -43,7 +37,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          ref: ${{ env.NEBARI_GH_BRANCH }}
           fetch-depth: 0
 
       - name: Set up Python
@@ -57,25 +50,12 @@ jobs:
           conda install --quiet --yes conda-build
           playwright install
 
-      - name: Retrieve secret from Vault
-        uses: hashicorp/[email protected]
-        with:
-          method: jwt
-          url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200"
-          namespace: "admin/quansight"
-          role: "repository-nebari-dev-nebari-role"
-          secrets: |
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci client_id | ARM_CLIENT_ID;
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci tenant_id | ARM_TENANT_ID;
-            kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci subscription_id | ARM_SUBSCRIPTION_ID;
-            kv/data/repository/nebari-dev/nebari/cloudflare/[email protected]/nebari-dev-ci token | CLOUDFLARE_TOKEN;
-
       - name: 'Azure login'
         uses: azure/login@v2
         with:
-          client-id: ${{ env.ARM_CLIENT_ID }}
-          tenant-id: ${{ env.ARM_TENANT_ID }}
-          subscription-id: ${{ env.ARM_SUBSCRIPTION_ID }}
+          client-id: ${{ secrets.ARM_CLIENT_ID }}
+          tenant-id: ${{ secrets.ARM_TENANT_ID }}
+          subscription-id: ${{ secrets.ARM_SUBSCRIPTION_ID }}
 
       - name: Integration Tests
         run: |
@@ -85,8 +65,8 @@ jobs:
           NEBARI_SECRET__default_images__jupyterhub: "quay.io/nebari/nebari-jupyterhub:${{ env.NEBARI_IMAGE_TAG }}"
           NEBARI_SECRET__default_images__jupyterlab: "quay.io/nebari/nebari-jupyterlab:${{ env.NEBARI_IMAGE_TAG }}"
           NEBARI_SECRET__default_images__dask_worker: "quay.io/nebari/nebari-dask-worker:${{ env.NEBARI_IMAGE_TAG }}"
-          ARM_CLIENT_ID: ${{ env.ARM_CLIENT_ID }}
-          ARM_TENANT_ID: ${{ env.ARM_TENANT_ID }}
-          ARM_SUBSCRIPTION_ID: ${{ env.ARM_SUBSCRIPTION_ID }}
+          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
           ARM_USE_OIDC: "true"
-          CLOUDFLARE_TOKEN: ${{ env.CLOUDFLARE_TOKEN }}
+          CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }}
-Original file line number
+Diff line change
@@ Expand Up / @@ -93,7 +93,6 @@ body: @@
             - "Azure"
             - "GCP"
             - "AWS"
-            - "Digital Ocean"
             - "kind"
         validations:
           required: false
@@ Expand Down @@