.gitlab-ci.yml

# Autogenerated by ~/code/xcookie/xcookie/builders/gitlab_ci.py

stages:
# TEMPLATES
- lint
- build
- gpgsign
- test
- deploy
.common_template: &common_template
  tags:
    # Tags define which runners will accept which jobs
  - docker
  - linux-x86_64
  - build

  - kitware-python-stack
  variables:
    # Change pip's cache directory to be inside the project directory
    # since we can only cache local items.
    PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip
  except:
    # Don't run the pipeline for new tags
  - tags

  cache:
    paths:
    - .cache/pip
.build_sdist_template: &build_sdist_template
  <<: *common_template
  stage: build
  script:
  - python -m pip install setuptools>=0.8 wheel build twine
  - python -m build --sdist --outdir dist
  - python -m twine check ./dist/geowatch*.tar.gz
  artifacts:
    paths:
    - ./dist/geowatch*.tar.gz
.build_wheel_template: &build_wheel_template
  <<: *common_template
  stage: build
  before_script:
  - 'python -V  # Print out python version for debugging'
  - df -h
  script:
  - python -m pip install setuptools>=0.8 wheel build twine
  - python -m build --wheel --outdir dist
  - python -m twine check ./dist/geowatch*.whl
  artifacts:
    paths:
    - ./dist/geowatch*.whl
.common_test_template: &common_test_template
  <<: *common_template
  stage: test
  coverage: /TOTAL.+ ([0-9]{1,3}%)/
.test_full-loose_template: &test_full-loose_template
  <<: *common_test_template
  before_script:
  - |-
    # Setup the correct version of python (which should be the same as this instance)
    python --version  # Print out python version for debugging
    export PYVER=$(python -c "import sys; print('{}{}'.format(*sys.version_info[0:2]))")
    python -m pip install virtualenv
    python -m virtualenv venv$PYVER
    source venv$PYVER/bin/activate
    pip install pip -U
    pip install pip setuptools -U
    pip install pygments
    python --version  # Print out python version for debugging
  script:
  - export INSTALL_EXTRAS="tests,optional,headless"
  - echo "Finding the path to the wheel"
  - ls dist || echo "dist does not exist"
  - echo "Installing helpers"
  - pip install setuptools>=0.8 setuptools_scm wheel build -U
  - pip install tomli pkginfo
  - "export WHEEL_FPATH=$(python -c \"if 1:\n    import pathlib\n    dist_dpath =
    pathlib.Path('dist')\n    candidates = list(dist_dpath.glob('geowatch*.whl'))\n\
    \    candidates += list(dist_dpath.glob('geowatch*.tar.gz'))\n    fpath = sorted(candidates)[-1]\n\
    \    print(str(fpath).replace(chr(92), chr(47)))\n\")"
  - "export MOD_VERSION=$(python -c \"if 1:\n    from pkginfo import Wheel, SDist\n\
    \    fpath = '$WHEEL_FPATH'\n    cls = Wheel if fpath.endswith('.whl') else SDist\n\
    \    print(cls(fpath).version)\n\")"
  - pip install -r requirements/gdal.txt
  - echo "WHEEL_FPATH=$WHEEL_FPATH"
  - echo "INSTALL_EXTRAS=$INSTALL_EXTRAS"
  - echo "MOD_VERSION=$MOD_VERSION"
  - pip install --prefer-binary "geowatch[$INSTALL_EXTRAS]==$MOD_VERSION" -f dist
  - echo "Install finished."
  - echo "Creating test sandbox directory"
  - export WORKSPACE_DNAME="sandbox"
  - echo "WORKSPACE_DNAME=$WORKSPACE_DNAME"
  - mkdir -p $WORKSPACE_DNAME
  - echo "cd-ing into the workspace"
  - cd $WORKSPACE_DNAME
  - pwd
  - ls -altr
  - '# Get the path to the installed package and run the tests'
  - export MOD_DPATH=$(python -c "import geowatch, os; print(os.path.dirname(geowatch.__file__))")
  - export MOD_NAME=geowatch
  - |-
    echo "
    ---
    MOD_DPATH = $MOD_DPATH
    ---
    running the pytest command inside the workspace
    ---
    "
  - |-
    python -m pytest --verbose -p pytester -p no:doctest --xdoctest --cov-config ../pyproject.toml --cov-report term --cov="$MOD_NAME" "$MOD_DPATH" ../tests --durations=100
.test_full-strict_template: &test_full-strict_template
  <<: *common_test_template
  before_script:
  - |-
    # Setup the correct version of python (which should be the same as this instance)
    python --version  # Print out python version for debugging
    export PYVER=$(python -c "import sys; print('{}{}'.format(*sys.version_info[0:2]))")
    python -m pip install virtualenv
    python -m virtualenv venv$PYVER
    source venv$PYVER/bin/activate
    pip install pip -U
    pip install pip setuptools -U
    pip install pygments
    python --version  # Print out python version for debugging
  script:
  - export INSTALL_EXTRAS="tests-strict,runtime-strict,optional-strict,headless-strict"
  - echo "Finding the path to the wheel"
  - ls dist || echo "dist does not exist"
  - echo "Installing helpers"
  - pip install setuptools>=0.8 setuptools_scm wheel build -U
  - pip install tomli pkginfo
  - "export WHEEL_FPATH=$(python -c \"if 1:\n    import pathlib\n    dist_dpath =
    pathlib.Path('dist')\n    candidates = list(dist_dpath.glob('geowatch*.whl'))\n\
    \    candidates += list(dist_dpath.glob('geowatch*.tar.gz'))\n    fpath = sorted(candidates)[-1]\n\
    \    print(str(fpath).replace(chr(92), chr(47)))\n\")"
  - "export MOD_VERSION=$(python -c \"if 1:\n    from pkginfo import Wheel, SDist\n\
    \    fpath = '$WHEEL_FPATH'\n    cls = Wheel if fpath.endswith('.whl') else SDist\n\
    \    print(cls(fpath).version)\n\")"
  - pip install -r requirements/gdal.txt
  - echo "WHEEL_FPATH=$WHEEL_FPATH"
  - echo "INSTALL_EXTRAS=$INSTALL_EXTRAS"
  - echo "MOD_VERSION=$MOD_VERSION"
  - pip install --prefer-binary "geowatch[$INSTALL_EXTRAS]==$MOD_VERSION" -f dist
  - echo "Install finished."
  - echo "Creating test sandbox directory"
  - export WORKSPACE_DNAME="sandbox"
  - echo "WORKSPACE_DNAME=$WORKSPACE_DNAME"
  - mkdir -p $WORKSPACE_DNAME
  - echo "cd-ing into the workspace"
  - cd $WORKSPACE_DNAME
  - pwd
  - ls -altr
  - '# Get the path to the installed package and run the tests'
  - export MOD_DPATH=$(python -c "import geowatch, os; print(os.path.dirname(geowatch.__file__))")
  - export MOD_NAME=geowatch
  - |-
    echo "
    ---
    MOD_DPATH = $MOD_DPATH
    ---
    running the pytest command inside the workspace
    ---
    "
  - |-
    python -m pytest --verbose -p pytester -p no:doctest --xdoctest --cov-config ../pyproject.toml --cov-report term --cov="$MOD_NAME" "$MOD_DPATH" ../tests --durations=100
build/sdist:
  <<: *build_sdist_template
  image: python:3.11
test/sdist/full-loose/cp311-linux-x86_64:
  <<: *test_full-loose_template
  image: python:3.11
  needs:
  - build/sdist
build/cp310-linux-x86_64:
  <<: *build_wheel_template
  image: python:3.10
test/full-loose/cp310-linux-x86_64:
  <<: *test_full-loose_template
  image: python:3.10
  needs:
  - build/cp310-linux-x86_64
test/full-strict/cp310-linux-x86_64:
  <<: *test_full-strict_template
  image: python:3.10
  needs:
  - build/cp310-linux-x86_64
build/cp311-linux-x86_64:
  <<: *build_wheel_template
  image: python:3.11
test/full-loose/cp311-linux-x86_64:
  <<: *test_full-loose_template
  image: python:3.11
  needs:
  - build/cp311-linux-x86_64
test/full-strict/cp311-linux-x86_64:
  <<: *test_full-strict_template
  image: python:3.11
  needs:
  - build/cp311-linux-x86_64
lint:
  <<: *common_template
  image: python:3.11
  stage: lint
  before_script:
  - df -h
  script:
  - pip install -r requirements/linting.txt
  - ./run_linter.sh
  allow_failure: true
gpgsign/wheels:
  <<: *common_template
  image: python:3.11
  stage: gpgsign
  artifacts:
    paths:
    - dist/*.asc
    - dist/*.tar.gz
    - dist/*.whl
    - dist/*.ots
  only:
    refs:
        # Gitlab will only expose protected variables on protected branches
        # (which I've set to be main and release), so only run this stage
        # there.
    - master
    - main
    - release
  script:
  - ls dist
  - export GPG_EXECUTABLE=gpg
  - export GPG_KEYID=$(cat dev/public_gpg_key)
  - echo "GPG_KEYID = $GPG_KEYID"
    # Decrypt and import GPG Keys / trust
    # note the variable pointed to by VARNAME_CI_SECRET is a protected variables only available on main and release branch
  - source dev/secrets_configuration.sh
  - CI_SECRET=${!VARNAME_CI_SECRET}
  - $GPG_EXECUTABLE --version
  - openssl version
  - $GPG_EXECUTABLE --list-keys
    # note CI_KITWARE_SECRET is a protected variables only available on main and release branch
  - CIS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:CIS -d -a
    -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import
  - CIS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:CIS -d -a
    -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust
  - CIS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:CIS -d -a
    -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import
  - GPG_SIGN_CMD="$GPG_EXECUTABLE --batch --yes --detach-sign --armor --local-user
    $GPG_KEYID"
  - |-
    WHEEL_PATHS=(dist/*.whl dist/*.tar.gz)
    WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_PATHS[@]}")
    echo "$WHEEL_PATHS_STR"
    for WHEEL_PATH in "${WHEEL_PATHS[@]}"
    do
        echo "------"
        echo "WHEEL_PATH = $WHEEL_PATH"
        $GPG_SIGN_CMD --output $WHEEL_PATH.asc $WHEEL_PATH
        $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH  || echo "hack, the first run of gpg very fails"
        $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH
    done
  - ls dist
  - python -m pip install opentimestamps-client
  - ots stamp dist/*.tar.gz dist/*.whl dist/*.asc
  needs:
  - job: build/sdist
    artifacts: true
  - job: build/cp310-linux-x86_64
    artifacts: true
  - job: build/cp311-linux-x86_64
    artifacts: true
deploy/wheels:
  <<: *common_template
  image: python:3.11
  stage: deploy
  only:
    refs:
    - release
  script:
  - pip install pyopenssl ndg-httpsclient pyasn1 requests[security] twine -U
  - ls dist
  - |-
    WHEEL_PATHS=(dist/*.whl dist/*.tar.gz)
    WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_PATHS[@]}")
    source dev/secrets_configuration.sh
    TWINE_PASSWORD=${!VARNAME_TWINE_PASSWORD}
    TWINE_USERNAME=${!VARNAME_TWINE_USERNAME}
    echo "$WHEEL_PATHS_STR"
    for WHEEL_PATH in "${WHEEL_PATHS[@]}"
    do
        twine check $WHEEL_PATH.asc $WHEEL_PATH
        twine upload --username $TWINE_USERNAME --password $TWINE_PASSWORD $WHEEL_PATH || echo "upload already exists"
    done
  - |-
    # Have the server git-tag the release and push the tags
    export PROJECT_VERSION=$(python -c "import setup; print(setup.VERSION)")
    # do sed twice to handle the case of https clone with and without a read token
    URL_HOST=$(git remote get-url origin | sed -e 's|https\?://.*@||g' | sed -e 's|https\?://||g' | sed -e 's|git@||g' | sed -e 's|:|/|g')
    source dev/secrets_configuration.sh
    CI_SECRET=${!VARNAME_CI_SECRET}
    PUSH_TOKEN=${!VARNAME_PUSH_TOKEN}
    echo "URL_HOST = $URL_HOST"
    # A git config user name and email is required. Set if needed.
    if [[ "$(git config user.email)" == "" ]]; then
        git config user.email "ci@gitlab.org.com"
        git config user.name "Gitlab-CI"
    fi
    TAG_NAME="v${PROJECT_VERSION}"
    echo "TAG_NAME = $TAG_NAME"
    if [ $(git tag -l "$TAG_NAME") ]; then
        echo "Tag already exists"
    else
        # if we messed up we can delete the tag
        # git push origin :refs/tags/$TAG_NAME
        # and then tag with -f
        git tag $TAG_NAME -m "tarball tag $PROJECT_VERSION"
        git push --tags "https://git-push-token:${PUSH_TOKEN}@${URL_HOST}"
    fi
  - |-
    # https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
    echo "CI_PROJECT_ID=$CI_PROJECT_ID"
    echo "CI_PROJECT_NAME=$CI_PROJECT_NAME"
    echo "CI_API_V4_URL=$CI_API_V4_URL"

    export PROJECT_VERSION=$(python -c "import setup; print(setup.VERSION)")
    echo "PROJECT_VERSION=$PROJECT_VERSION"

    # If running on CI use CI authentication, otherwise
    # assume we developer authentication available.
    if [[ -z "$CI_JOB_TOKEN" ]]; then
        AUTH_HEADER="PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN"
    else
        AUTH_HEADER="JOB-TOKEN: $CI_JOB_TOKEN"
    fi

    # Loop over all of the assets in the wheelhouse (i.e.  dist)
    # and upload them to a package registery. We also store the
    # links to the artifacts so we can attach them to a release
    # page.
    PACKAGE_ARTIFACT_ARRAY=()
    for FPATH in "dist"/*; do
        FNAME=$(basename $FPATH)
        echo "Upload artifact: $FNAME"
        PACKAGE_URL="$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/$CI_PROJECT_NAME/$PROJECT_VERSION/$FNAME"
        curl \
            --header "$AUTH_HEADER" \
            --upload-file $FPATH \
            "$PACKAGE_URL"
        PACKAGE_ARTIFACT_ARRAY+=("$PACKAGE_URL")
    done
  - |-
    export PROJECT_VERSION=$(python -c "import setup; print(setup.VERSION)")
    echo "PROJECT_VERSION=$PROJECT_VERSION"
    TAG_NAME="v$PROJECT_VERSION"

    # Construct the JSON for assets to attach to the release
    RELEASE_ASSET_JSON_LINKS=()
    for ASSET_URL in "${PACKAGE_ARTIFACT_ARRAY[@]}"; do
        ASSET_FNAME=$(basename $ASSET_URL)
        RELEASE_ASSET_JSON_LINKS+=("{\"name\": \"$ASSET_FNAME\", \"url\": \"$ASSET_URL\"},")
    done
    _ASSET_LINK_JSON="${RELEASE_ASSET_JSON_LINKS[@]}"
    # remove the trailing comma
    ASSET_LINK_JSON=${_ASSET_LINK_JSON::-1}
    echo "ASSET_LINK_JSON=$ASSET_LINK_JSON"

    # Build json describing the release
    RELEASE_DATA_JSON="{
        \"name\": \"Version $PROJECT_VERSION\",
        \"description\": \"Automated release of $CI_PROJECT_NAME version $PROJECT_VERSION\",
        \"tag_name\": \"$TAG_NAME\",
        \"assets\": {\"links\": [$ASSET_LINK_JSON]}
    }"
    echo "$RELEASE_DATA_JSON"

    # If running on CI use CI authentication, otherwise
    # assume we developer authentication available.
    if [[ -z "${CI_JOB_TOKEN}" ]]; then
        AUTH_HEADER="PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN"
    else
        AUTH_HEADER="JOB-TOKEN: $CI_JOB_TOKEN"
    fi

    curl \
        --header 'Content-Type: application/json' \
        --header "$AUTH_HEADER" \
        --data "$RELEASE_DATA_JSON" \
        --request POST \
        "$CI_API_V4_URL/projects/$CI_PROJECT_ID/releases"


# end