Merge pull request #1 from octomike/feature_release_crawler

[WIP] Add update.py semi-automatic release crawler

.circleci/config.yml

@@ -1,53 +1,66 @@
 ---
+tag_filter: &tag_filter
+  filters:
+    tags:
+      only: /.*/
+    branches:
+      ignore: /.*/
+
 version: 2.1
 
 jobs:
+
   build:
     machine:
-      # https://circleci.com/developer/machine/image/ubuntu-2204
       image: ubuntu-2204:2022.10.2
     steps:
     - checkout
-    - restore_cache:
-        keys:
-        - my_cache
     - run:
         name: Build Docker image
-        command: |
-          wget https://raw.githubusercontent.com/bids-apps/maintenance-tools/main/circleci/build_docker.sh
-          bash build_docker.sh  
-    - save_cache:
-        key: my_cache
-        paths:
-        - ~/docker
+        command: bash build_docker.sh
+        no_output_timeout: 30m   # MCR is a large download
     - persist_to_workspace:
         root: /home/circleci
         paths:
         - docker/image.tar
 
-  deploy:
+  test:
     machine:
       image: ubuntu-2204:2022.10.2
     steps:
     - attach_workspace:
         at: /tmp/workspace
-    - run: docker load -i /tmp/workspace/image.tar
     - run:
-        name: push to dockerhub
+        name: Test Docker image
         command: |
-          wget https://raw.githubusercontent.com/bids-apps/maintenance-tools/main/circleci/push_docker.sh
-          bash push_docker.sh  
+          docker load -i /tmp/workspace/docker/image.tar
+          # figure out a better test
+          docker run -ti --rm --read-only \
+            --entrypoint /bin/sh bids/${CIRCLE_PROJECT_REPONAME,,} \
+            -c 'test -d ${MCR_HOME}/runtime/glnxa64'
 
+  deploy:
+    docker:
+    - image: circleci/buildpack-deps:stretch
+    steps:
+    - attach_workspace:
+        at: /tmp/workspace
+    - setup_remote_docker
+    - run: docker load -i /tmp/workspace/docker/image.tar
+    - run:
+        name: Publish Docker image
+        command: push_docker.sh
 
 workflows:
   build-test-deploy:
     jobs:
-    - build
-    - deploy:
+    - build:
+        <<: *tag_filter
+    - test:
         requires:
         - build
-        filters:
-          tags:
-            only: /.*/
-
-# VS Code Extension Version: 1.5.1
+        <<: *tag_filter
+    - deploy:
+        requires:
+        - test
+        <<: *tag_filter

.hadolint.yaml

@@ -1,4 +1,6 @@
 ---
 ignored:
+- DL3003
 - DL3006
 - DL3008
+- SC2086

.pre-commit-config.yaml

@@ -32,5 +32,27 @@ repos:
     types: [dockerfile]
     entry: ghcr.io/hadolint/hadolint hadolint
 
+- repo: https://github.com/psf/black
+  rev: 22.12.0
+  hooks:
+  - id: black
+
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.3.1
+  hooks:
+  - id: pyupgrade
+    args: [--py38-plus]
+
+- repo: https://github.com/pycqa/flake8
+  rev: 6.0.0
+  hooks:
+  - id: flake8
+
+- repo: https://github.com/asottile/reorder_python_imports
+  rev: v3.9.0
+  hooks:
+  - id: reorder-python-imports
+    args: [--py38-plus]
+
 ci:
   skip: [hadolint-docker]

build_docker.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+
+# used to build the Docker image for a project in circle CI
+
+git describe --tags --always > version
+docker build -t "bids/${CIRCLE_PROJECT_REPONAME,,}" .
+mkdir -p ${HOME}/docker
+docker save "bids/${CIRCLE_PROJECT_REPONAME,,}" > ~/docker/image.tar
+# persist guessed branch so we can use it in deploy/tag
+BRANCH=$(git branch --contains tags/${CIRCLE_TAG})
+echo -n "${BRANCH}" > ~/docker/branch

push_docker.sh

@@ -0,0 +1,25 @@
+#! /bin/bash
+
+# used to push the Docker image for a project in circle CI
+
+if [[ -n "${CIRCLE_TAG}" ]]; then
+
+    echo "${DOCKER_PASS}" | docker login --username "${DOCKER_USER}" --password-stdin
+
+    # tag should always be X.Y.Z[-variant]
+    docker tag "bids/${CIRCLE_PROJECT_REPONAME,,}" "bids/${CIRCLE_PROJECT_REPONAME,,}:${CIRCLE_TAG}"
+    docker push "bids/${CIRCLE_PROJECT_REPONAME,,}:${CIRCLE_TAG}"
+
+    # also publish tag for the corresponding matlab release version, which is the name of the current branch
+    docker "tag bids/${CIRCLE_PROJECT_REPONAME,,}" "bids/${CIRCLE_PROJECT_REPONAME,,}:${BRANCH}"
+    docker push "bids/${CIRCLE_PROJECT_REPONAME,,}:${BRANCH}"
+    BRANCH=$(cat /tmp/workspace/docker/branch)
+
+    # update major tag X.Y[-variant] to the latest in this branch
+    MAJOR_TAG=$(echo "${CIRCLE_TAG}" | sed -rn 's#([[:digit:]]+).([[:digit:]]+).([[:digit:]]+)(.*)#\1.\2\4#p')
+    if [[ -n "${MAJOR_TAG}" ]] ; then
+    docker tag "bids/${CIRCLE_PROJECT_REPONAME,,}" "bids/${CIRCLE_PROJECT_REPONAME,,}:${MAJOR_TAG}"
+    docker push "bids/${CIRCLE_PROJECT_REPONAME,,}:${MAJOR_TAG}"
+    fi
+
+fi

requirements.txt

@@ -0,0 +1,4 @@
+rich
+beautifulsoup4
+packaging
+chevron

setup.cfg

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 100

template.mustache

@@ -0,0 +1,48 @@
+FROM bids/base_validator
+
+# Update system
+RUN apt-get -qq update -qq && \
+    apt-get -qq install -qq -y --no-install-recommends \
+        unzip \
+        xorg \
+        wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install MATLAB MCR
+ENV MATLAB_VERSION {{ MATLAB_VERSION }}
+RUN mkdir /opt/mcr_install && \
+    mkdir /opt/mcr && \
+    wget --quiet -P /opt/mcr_install {{ MCR_LINK }} && \
+    unzip -q /opt/mcr_install/*${MATLAB_VERSION}*.zip -d /opt/mcr_install && \
+    cd /opt/mcr_install && mkdir save && \
+{{#core_only}}
+    for f in $(grep -E '(xml|enc)$' productdata/1000.txt) ; do  cp --parents archives/$f save/ ; done && \
+    for f in $(grep -E '(xml|enc)$' productdata/35000.txt) ; do  cp --parents archives/$f save/ ; done && \
+    for f in $(grep -E '(xml|enc)$' productdata/35010.txt) ; do  cp --parents archives/$f save/ ; done && \
+    rm -rf archives && mv save/archives . && rmdir save && \
+{{/core_only}}
+    /opt/mcr_install/install -destinationFolder /opt/mcr -agreeToLicense yes -mode silent && \
+{{#core_only}}
+    rm -rf /opt/mcr/*/cefclient && \
+    rm -rf /opt/mcr/*/mcr/toolbox/matlab/maps && \
+    rm -rf /opt/mcr/*/java/jarext && \
+    rm -rf /opt/mcr/*/toolbox/matlab/system/editor && \
+    rm -rf /opt/mcr/*/toolbox/matlab/codetools && \
+    rm -rf /opt/mcr/*/toolbox/matlab/datatools && \
+    rm -rf /opt/mcr/*/toolbox/matlab/codeanalysis && \
+    rm -rf /opt/mcr/*/toolbox/shared/dastudio && \
+    rm -rf /opt/mcr/*/toolbox/shared/mlreportgen && \
+    rm -rf /opt/mcr/*/sys/java/jre/glnxa64/jre/lib/ext/jfxrt.jar && \
+    rm -rf /opt/mcr/*/sys/java/jre/glnxa64/jre/lib/amd64/libjfxwebkit.so && \
+    rm -rf /opt/mcr/*/bin/glnxa64/libQt* && \
+    rm -rf /opt/mcr/*/bin/glnxa64/qtwebengine && \
+    rm -rf /opt/mcr/*/bin/glnxa64/cef_resources && \
+{{/core_only}}
+    rm -rf /opt/mcr_install /tmp/*
+
+# Configure environment
+ENV MCR_VERSION {{ MCR_VERSION }}
+ENV LD_LIBRARY_PATH /opt/mcr/${MCR_VERSION}/runtime/glnxa64:/opt/mcr/${MCR_VERSION}/bin/glnxa64:/opt/mcr/${MCR_VERSION}/sys/os/glnxa64:/opt/mcr/${MCR_VERSION}/sys/opengl/lib/glnxa64
+ENV MCR_INHIBIT_CTF_LOCK 1
+ENV MCR_HOME /opt/mcr/${MCR_VERSION}