diff --git a/.circleci/config.yml b/.circleci/config.yml index 798f43e..62a8c9c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,53 +1,66 @@ --- +tag_filter: &tag_filter + filters: + tags: + only: /.*/ + branches: + ignore: /.*/ + version: 2.1 jobs: + build: machine: - # https://circleci.com/developer/machine/image/ubuntu-2204 image: ubuntu-2204:2022.10.2 steps: - checkout - - restore_cache: - keys: - - my_cache - run: name: Build Docker image - command: | - wget https://raw.githubusercontent.com/bids-apps/maintenance-tools/main/circleci/build_docker.sh - bash build_docker.sh - - save_cache: - key: my_cache - paths: - - ~/docker + command: bash build_docker.sh + no_output_timeout: 30m # MCR is a large download - persist_to_workspace: root: /home/circleci paths: - docker/image.tar - deploy: + test: machine: image: ubuntu-2204:2022.10.2 steps: - attach_workspace: at: /tmp/workspace - - run: docker load -i /tmp/workspace/image.tar - run: - name: push to dockerhub + name: Test Docker image command: | - wget https://raw.githubusercontent.com/bids-apps/maintenance-tools/main/circleci/push_docker.sh - bash push_docker.sh + docker load -i /tmp/workspace/docker/image.tar + # figure out a better test + docker run -ti --rm --read-only \ + --entrypoint /bin/sh bids/${CIRCLE_PROJECT_REPONAME,,} \ + -c 'test -d ${MCR_HOME}/runtime/glnxa64' + deploy: + docker: + - image: circleci/buildpack-deps:stretch + steps: + - attach_workspace: + at: /tmp/workspace + - setup_remote_docker + - run: docker load -i /tmp/workspace/docker/image.tar + - run: + name: Publish Docker image + command: push_docker.sh workflows: build-test-deploy: jobs: - - build - - deploy: + - build: + <<: *tag_filter + - test: requires: - build - filters: - tags: - only: /.*/ - -# VS Code Extension Version: 1.5.1 + <<: *tag_filter + - deploy: + requires: + - test + <<: *tag_filter diff --git a/.hadolint.yaml b/.hadolint.yaml index 0524f4f..e5aa160 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -1,4 +1,6 @@ --- ignored: +- DL3003 - DL3006 - DL3008 +- SC2086 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c010886..3a180e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,5 +32,27 @@ repos: types: [dockerfile] entry: ghcr.io/hadolint/hadolint hadolint +- repo: https://github.com/psf/black + rev: 22.12.0 + hooks: + - id: black + +- repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: [--py38-plus] + +- repo: https://github.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + +- repo: https://github.com/asottile/reorder_python_imports + rev: v3.9.0 + hooks: + - id: reorder-python-imports + args: [--py38-plus] + ci: skip: [hadolint-docker] diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 003f659..0000000 --- a/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM bids/base_validator - -# Update system -RUN apt-get -qq update -qq && \ - apt-get -qq install -qq -y --no-install-recommends \ - unzip \ - xorg \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Install MATLAB MCR -ENV MATLAB_VERSION R2016b -RUN mkdir /opt/mcr_install && \ - mkdir /opt/mcr && \ - wget --quiet -P /opt/mcr_install http://www.mathworks.com/supportfiles/downloads/${MATLAB_VERSION}/deployment_files/${MATLAB_VERSION}/installers/glnxa64/MCR_${MATLAB_VERSION}_glnxa64_installer.zip && \ - unzip -q /opt/mcr_install/MCR_${MATLAB_VERSION}_glnxa64_installer.zip -d /opt/mcr_install && \ - /opt/mcr_install/install -destinationFolder /opt/mcr -agreeToLicense yes -mode silent && \ - rm -rf /opt/mcr_install /tmp/* - -# Configure environment -ENV MCR_VERSION v91 -ENV LD_LIBRARY_PATH /opt/mcr/${MCR_VERSION}/runtime/glnxa64:/opt/mcr/${MCR_VERSION}/bin/glnxa64:/opt/mcr/${MCR_VERSION}/sys/os/glnxa64:/opt/mcr/${MCR_VERSION}/sys/opengl/lib/glnxa64 -ENV MCR_INHIBIT_CTF_LOCK 1 -ENV MCRPath /opt/mcr/${MCR_VERSION} diff --git a/build_docker.sh b/build_docker.sh new file mode 100644 index 0000000..300786c --- /dev/null +++ b/build_docker.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +# used to build the Docker image for a project in circle CI + +git describe --tags --always > version +docker build -t "bids/${CIRCLE_PROJECT_REPONAME,,}" . +mkdir -p ${HOME}/docker +docker save "bids/${CIRCLE_PROJECT_REPONAME,,}" > ~/docker/image.tar +# persist guessed branch so we can use it in deploy/tag +BRANCH=$(git branch --contains tags/${CIRCLE_TAG}) +echo -n "${BRANCH}" > ~/docker/branch diff --git a/push_docker.sh b/push_docker.sh new file mode 100644 index 0000000..f9b5f8c --- /dev/null +++ b/push_docker.sh @@ -0,0 +1,25 @@ +#! /bin/bash + +# used to push the Docker image for a project in circle CI + +if [[ -n "${CIRCLE_TAG}" ]]; then + + echo "${DOCKER_PASS}" | docker login --username "${DOCKER_USER}" --password-stdin + + # tag should always be X.Y.Z[-variant] + docker tag "bids/${CIRCLE_PROJECT_REPONAME,,}" "bids/${CIRCLE_PROJECT_REPONAME,,}:${CIRCLE_TAG}" + docker push "bids/${CIRCLE_PROJECT_REPONAME,,}:${CIRCLE_TAG}" + + # also publish tag for the corresponding matlab release version, which is the name of the current branch + docker "tag bids/${CIRCLE_PROJECT_REPONAME,,}" "bids/${CIRCLE_PROJECT_REPONAME,,}:${BRANCH}" + docker push "bids/${CIRCLE_PROJECT_REPONAME,,}:${BRANCH}" + BRANCH=$(cat /tmp/workspace/docker/branch) + + # update major tag X.Y[-variant] to the latest in this branch + MAJOR_TAG=$(echo "${CIRCLE_TAG}" | sed -rn 's#([[:digit:]]+).([[:digit:]]+).([[:digit:]]+)(.*)#\1.\2\4#p') + if [[ -n "${MAJOR_TAG}" ]] ; then + docker tag "bids/${CIRCLE_PROJECT_REPONAME,,}" "bids/${CIRCLE_PROJECT_REPONAME,,}:${MAJOR_TAG}" + docker push "bids/${CIRCLE_PROJECT_REPONAME,,}:${MAJOR_TAG}" + fi + +fi diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2bcdad4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +rich +beautifulsoup4 +packaging +chevron diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..7da1f96 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 100 diff --git a/template.mustache b/template.mustache new file mode 100644 index 0000000..e8cee13 --- /dev/null +++ b/template.mustache @@ -0,0 +1,48 @@ +FROM bids/base_validator + +# Update system +RUN apt-get -qq update -qq && \ + apt-get -qq install -qq -y --no-install-recommends \ + unzip \ + xorg \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Install MATLAB MCR +ENV MATLAB_VERSION {{ MATLAB_VERSION }} +RUN mkdir /opt/mcr_install && \ + mkdir /opt/mcr && \ + wget --quiet -P /opt/mcr_install {{ MCR_LINK }} && \ + unzip -q /opt/mcr_install/*${MATLAB_VERSION}*.zip -d /opt/mcr_install && \ + cd /opt/mcr_install && mkdir save && \ +{{#core_only}} + for f in $(grep -E '(xml|enc)$' productdata/1000.txt) ; do cp --parents archives/$f save/ ; done && \ + for f in $(grep -E '(xml|enc)$' productdata/35000.txt) ; do cp --parents archives/$f save/ ; done && \ + for f in $(grep -E '(xml|enc)$' productdata/35010.txt) ; do cp --parents archives/$f save/ ; done && \ + rm -rf archives && mv save/archives . && rmdir save && \ +{{/core_only}} + /opt/mcr_install/install -destinationFolder /opt/mcr -agreeToLicense yes -mode silent && \ +{{#core_only}} + rm -rf /opt/mcr/*/cefclient && \ + rm -rf /opt/mcr/*/mcr/toolbox/matlab/maps && \ + rm -rf /opt/mcr/*/java/jarext && \ + rm -rf /opt/mcr/*/toolbox/matlab/system/editor && \ + rm -rf /opt/mcr/*/toolbox/matlab/codetools && \ + rm -rf /opt/mcr/*/toolbox/matlab/datatools && \ + rm -rf /opt/mcr/*/toolbox/matlab/codeanalysis && \ + rm -rf /opt/mcr/*/toolbox/shared/dastudio && \ + rm -rf /opt/mcr/*/toolbox/shared/mlreportgen && \ + rm -rf /opt/mcr/*/sys/java/jre/glnxa64/jre/lib/ext/jfxrt.jar && \ + rm -rf /opt/mcr/*/sys/java/jre/glnxa64/jre/lib/amd64/libjfxwebkit.so && \ + rm -rf /opt/mcr/*/bin/glnxa64/libQt* && \ + rm -rf /opt/mcr/*/bin/glnxa64/qtwebengine && \ + rm -rf /opt/mcr/*/bin/glnxa64/cef_resources && \ +{{/core_only}} + rm -rf /opt/mcr_install /tmp/* + +# Configure environment +ENV MCR_VERSION {{ MCR_VERSION }} +ENV LD_LIBRARY_PATH /opt/mcr/${MCR_VERSION}/runtime/glnxa64:/opt/mcr/${MCR_VERSION}/bin/glnxa64:/opt/mcr/${MCR_VERSION}/sys/os/glnxa64:/opt/mcr/${MCR_VERSION}/sys/opengl/lib/glnxa64 +ENV MCR_INHIBIT_CTF_LOCK 1 +ENV MCR_HOME /opt/mcr/${MCR_VERSION} diff --git a/update.py b/update.py new file mode 100755 index 0000000..9a64c9d --- /dev/null +++ b/update.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +Dirty website parser to generate Dockerfiles for Matlab's MCR releases. + +Each Matlab release name (R2020a etc) gets a branch that contains a single +Dockerfile and each release version (9.8.0 or 9.8.5 for 9.8 Update 5) becomes +a tag in that branch. Each variant gets a new commit as well. +So a sample history for R2020a could look something like this: + + + [R2020a] <9.8.1-core> Auto-Update + + <9.8.1> Auto-Update + + Merged master + | \ + | + Update in templates + | / + + <9.8.0-core> Auto-Update + + <9.8.0> Auto-Update + + Import + +(Circle)CI should then fire a docker build and push for every tag only. Shared +tags for the major version (e.g. 9.8 always pointing to the latest 9.8 tag are +done in Circle CI to avoid duplicate builds). + +Update.py should be run often enough to catch individual Matlab release updates. +""" +import re +from subprocess import DEVNULL +from subprocess import run +from urllib import request + +import chevron +from bs4 import BeautifulSoup +from packaging import version +from rich import print + + +REL_URL = "https://www.mathworks.com/products/compiler/matlab-runtime.html" +VER_LIMIT = "9.3" # release URLs get weird before that.. +DRY_RUN = True + +template = "template.mustache" +variants = ["", "-core"] + + +def call(cmd, split=True): + if split: + cmd = cmd.split() + print(f"[green]{' '.join(cmd)}[/green]") + if DRY_RUN: + return True + process = run(cmd, stdout=DEVNULL, stderr=DEVNULL) + return process.returncode == 0 + + +def add_dockerfile_to_branch(new_tags, docker): + + mcr_name, mcr_ver, link = docker + + if len(mcr_ver.split(".")) == 2: + mcr_ver = f"{mcr_ver}.0" + mcr_ver_maj = ".".join(mcr_ver.split(".")[:2]) + mcr_ver_dir = f'v{mcr_ver_maj.replace(".", "")}' + + print(f"\n[blue]{mcr_name}[/blue]") + + if not call(f"git checkout {mcr_name}"): + call(f"git checkout -b {mcr_name}") + + for suffix in variants: + + tag = f"{mcr_ver}{suffix}" + + if not DRY_RUN and call(f"git rev-parse --verify {tag}"): + print(f"[red]Skipping {mcr_name}-{tag}, already present[/red]") + continue + print(f"\n[blue]Adding {mcr_name}-{tag}[/blue]") + + if not call("git merge master"): + raise RuntimeError("Merging master failed, will not continue") + + with open(template) as f: + content = chevron.render( + f, + { + "MATLAB_VERSION": mcr_name, + "MCR_VERSION": mcr_ver_dir, + "MCR_LINK": link, + "core_only": suffix == "-core", + }, + ) + if not DRY_RUN: + with open("Dockerfile", "w+") as f2: + f2.write(content) + + call("git add Dockerfile") + # Tag X.Y.Z[-variant] - see circle CI for shared tag X.Y[-variant] + call(["git", "commit", "-m", "Auto-Update"], split=False) + + call(f"git tag {tag}") + + new_tags.append(tag) + + call("git checkout master") + + return new_tags + + +def list_mcr(soup): + + ver_re = re.compile(r"(R2\d{3}.) \((\d\.\d+)\)") + rel_re = re.compile(r"Release/(\d+)/") + + dockers = [] + for row in soup.find_all("table")[0].find_all("tr"): + + tds = row.find_all("td") + + if len(tds) >= 4: + name = tds[0].text + match = ver_re.match(name) + if not match: + continue + mcr_name, mcr_ver = match.groups() + + if version.parse(mcr_ver) <= version.parse(VER_LIMIT): + continue + try: + link = tds[2].a.get("href") + except (KeyError, ValueError) as e: + raise RuntimeError("Error parsing matlab release page") from e + + if "glnxa64" not in link: + raise RuntimeError("Error parsing matlab release page link") + + if match := rel_re.search(link): + mcr_ver = f"{mcr_ver}.{match.groups()[0]}" + + dockers.append((mcr_name, mcr_ver, link)) + + return dockers + + +def main(): + + with request.urlopen(REL_URL) as res: + if res.status != 200: + raise RuntimeError("Could not open matlab release URL") + html = res.read() + + soup = BeautifulSoup(html, "html.parser") + + dockers = list_mcr(soup) + + new_tags = [] + for docker in dockers: + new_tags = add_dockerfile_to_branch(new_tags, docker) + + if new_tags: + print("\nNew tags have been added, verify and update to git with:") + print("git push --all") + for tag in reversed(new_tags): + print(f"git push origin {tag}") + + +if __name__ == "__main__": + main()