diff --git a/.github/actions/is_callee/action.yml b/.github/actions/is_callee/action.yml new file mode 100644 index 000000000..b425f3df6 --- /dev/null +++ b/.github/actions/is_callee/action.yml @@ -0,0 +1,21 @@ +name: Check if being used as reusable workflow +description: Determines if running as part of a reusable workflow by checking if the specified current workflow file is the same as the caller +inputs: + current_workflow_file: + required: true + description: The name of the workflow file this action is called from +outputs: + is_callee: + value: ${{ steps.workflows_match.outputs.is_callee }} + description: Returns 'true' (string) if called as a reusable workflow +runs: + using: "composite" + steps: + - id: workflows_match + run: | + if echo ${{ github.workflow_ref }} | grep ${{ inputs.current_workflow_file }}; then + echo "is_callee=false" >> "$GITHUB_OUTPUT" + else + echo "is_callee=true" >> "$GITHUB_OUTPUT" + fi + shell: bash diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml index c3b91fefa..c4c06c5b9 100644 --- a/.github/workflows/fatimage.yml +++ b/.github/workflows/fatimage.yml @@ -10,6 +10,21 @@ on: - LEAFCLOUD - SMS - ARCUS + workflow_call: + inputs: + ci_cloud_override: + type: string + default: LEAFCLOUD + target_branch: + type: string + default: ${{ github.ref }} + outputs: + openhpc-RL8-image: + description: "RL8 image" + value: "${{ jobs.openstack.outputs.openhpc-RL8-image }}" + openhpc-RL9-image: + description: "RL9 image" + value: "${{ jobs.openstack.outputs.openhpc-RL9-image }}" jobs: openstack: @@ -31,12 +46,22 @@ jobs: env: ANSIBLE_FORCE_COLOR: True OS_CLOUD: openstack - CI_CLOUD: ${{ github.event.inputs.ci_cloud }} + CI_CLOUD: ${{ github.event.inputs.ci_cloud || inputs.ci_cloud_override }} ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }} LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }} + outputs: + openhpc-RL8-image: "${{ steps.manifest.outputs.openhpc-RL8-image }}" + openhpc-RL9-image: "${{ steps.manifest.outputs.openhpc-RL9-image }}" steps: - - uses: actions/checkout@v2 + - uses: stackhpc/ansible-slurm-appliance/.github/actions/is_callee@feat/auto-bump-timestamps # todo: change to main once merges + id: callee_check + with: + current_workflow_file: fatimage.yml + + - uses: actions/checkout@v3 + with: + ref: ${{ steps.callee_check.outputs.is_callee == 'true' && inputs.target_branch || github.ref }} - name: Record settings for CI cloud run: | @@ -96,6 +121,7 @@ jobs: IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID) echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT" echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT" + echo "${{ matrix.build.image_name }}-image=${IMAGE_NAME}" >> "$GITHUB_OUTPUT" echo $IMAGE_ID > image-id.txt echo $IMAGE_NAME > image-name.txt diff --git a/.github/workflows/stackhpc.yml b/.github/workflows/stackhpc.yml index eaca3a3ae..600f6a09a 100644 --- a/.github/workflows/stackhpc.yml +++ b/.github/workflows/stackhpc.yml @@ -2,6 +2,11 @@ name: Test deployment and reimage on OpenStack on: workflow_dispatch: + workflow_call: + inputs: + target_branch: + type: string + default: ${{ github.ref }} push: branches: - main @@ -44,7 +49,14 @@ jobs: CI_CLOUD: ${{ vars.CI_CLOUD }} # default from repo settings TF_VAR_os_version: ${{ matrix.os_version }} steps: - - uses: actions/checkout@v2 + - uses: stackhpc/ansible-slurm-appliance/.github/actions/is_callee@feat/auto-bump-timestamps # todo: change to main once merges + id: callee_check + with: + current_workflow_file: stackhpc.yml + + - uses: actions/checkout@v3 + with: + ref: ${{ steps.callee_check.outputs.is_callee == 'true' && inputs.target_branch || github.ref }} - name: Override CI_CLOUD if PR label is present if: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/trivyscan.yml b/.github/workflows/trivyscan.yml index 5b65baca1..d7302ab2b 100644 --- a/.github/workflows/trivyscan.yml +++ b/.github/workflows/trivyscan.yml @@ -12,7 +12,7 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }} # to branch/PR + build cancel-in-progress: true - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/update-timestamps.yml b/.github/workflows/update-timestamps.yml new file mode 100644 index 000000000..763d21355 --- /dev/null +++ b/.github/workflows/update-timestamps.yml @@ -0,0 +1,170 @@ +name: Check for new Release Train snapshots +on: + workflow_dispatch: # temporary + pull_request: #temporary + schedule: + - cron: '0 7 * * *' # Run at 7am on default branch + +jobs: + upstream_check: + outputs: + new_fatimage: "${{ steps.fatimage_check.outputs.new_fatimage }}" + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + + - name: Check automation branch exists + id: auto-branch-check + run: | + git fetch + branches=$(git branch -r) + set +e + echo $branches | grep auto/bump-timestamps + branch_exists="$?" + echo "branch_exists=$branch_exists" >> "$GITHUB_OUTPUT" + + - name: Create automation branch + if: steps.auto-branch-check.outputs.branch_exists == '1' + run: | + git checkout -b auto/bump-timestamps + git config --global --add --bool push.autoSetupRemote true + git push + + - uses: actions/checkout@v2 + with: + ref: auto/bump-timestamps + + - name: Check for updated Ark timestamps and replace in defaults.yml + run: | + dev/setup-env.sh + . venv/bin/activate + . environments/.stackhpc/activate + ansible-playbook ansible/ci/update_timestamps.yml -v + + - name: Check if timestamps were changed + id: timestamp_check + run: | + set +e + git diff --quiet + echo "timestamps_changed=$?" >> "$GITHUB_OUTPUT" + + # TODO: find way to stop CI running if pushing to existing PR + - name: Push new timestamps + if: steps.timestamp_check.outputs.timestamps_changed == '1' + run: | + git fetch origin refs/notes/*:refs/notes/* + git add environments/common/inventory/group_vars/all/defaults.yml + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + git commit -m "Bumped repo timestamps" + git notes add --force -m "timestamp_bump_commit" + git config --global --add --bool push.autoSetupRemote true + git push + git push origin refs/notes/* + + - name: Check if new fatimage needed + id: fatimage_check + run: | + git fetch origin refs/notes/*:refs/notes/* + NEED_NEW_IMAGE="false" + set +e + if git notes show ; then + HEAD_NOTES=$(git notes show) + if [[ $HEAD_NOTES == "timestamp_bump_commit" ]] ; then + NEED_NEW_IMAGE="true" + fi + fi + set -e + echo $NEED_NEW_IMAGE + echo "new_fatimage=$NEED_NEW_IMAGE" >> "$GITHUB_OUTPUT" + + build_fatimage: + if: needs.upstream_check.outputs.new_fatimage == 'true' + needs: upstream_check + secrets: inherit + uses: ./.github/workflows/fatimage.yml + with: + ci_cloud_override: 'LEAFCLOUD' + target_branch: auto/bump-timestamps + + bump_images: + if: needs.upstream_check.outputs.new_fatimage == 'true' + needs: build_fatimage + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + with: + ref: auto/bump-timestamps + - name: Bump CI with new images + run: | + git checkout auto/bump-timestamps + sed -i 's/"RL8".*$/"RL8": "${{ needs.build_fatimage.outputs.openhpc-RL8-image }}",/' environments/.stackhpc/terraform/cluster_image.auto.tfvars.json + sed -i 's/"RL9".*$/"RL9": "${{ needs.build_fatimage.outputs.openhpc-RL9-image }}"/' environments/.stackhpc/terraform/cluster_image.auto.tfvars.json + + - name: Push new images + run: | + git add environments/.stackhpc/terraform/cluster_image.auto.tfvars.json + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + git commit -m "Bumped images" + git push + + create_pr: + needs: bump_images + if: always() && (needs.bump_images.result == 'skipped' || needs.bump_images.result == 'success') + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + with: + ref: auto/bump-timestamps + + - name: Check if PR exists + id: pr-check + run: | + set +e + gh pr list --json headRefName --jq '.[].headRefName' | grep auto/bump-timestamps + PR_EXISTS=$? + echo "pr_exists=$PR_EXISTS" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} + + - name: Create PR + if: steps.pr-check.outputs.pr_exists == '1' + run: gh pr create --title "[Auto] Bump repo timestamps to latest" --base main --head auto/bump-timestamps --body "Updated Release Train timestamps in defaults.yml with latest from Ark" + env: + GH_TOKEN: ${{ github.token }} + + run_ci: + needs: + - bump_images + - upstream_check + if: always() && (needs.bump_images.result == 'success' || needs.upstream_check.outputs.new_fatimage == 'false') # should always run only on image bump commits + uses: ./.github/workflows/stackhpc.yml + secrets: inherit + with: + target_branch: auto/bump-timestamps + + comment_result: + if: always() && (needs.run_ci.result == 'failure' || needs.run_ci.result == 'success') && (needs.create_pr.result == 'skipped' || needs.create_pr.result == 'success') + needs: + - run_ci + - create_pr + runs-on: ubuntu-22.04 + steps: + - name: Checkout branch + uses: actions/checkout@v3 + + - name: Get created PR number + id: number_check + run: | + PR_NUMBER=$(gh pr list --head auto/bump-timestamps --state open --json number --jq .[0].number) + echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} + + - name: Comment CI status + uses: thollander/actions-comment-pull-request@v1 + with: + message: 'CI ${{ needs.run_ci.result }}: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + pr_number: ${{ steps.number_check.outputs.pr_number }} diff --git a/ansible/ci/library/latest_timestamps.py b/ansible/ci/library/latest_timestamps.py new file mode 100644 index 000000000..aae153d2e --- /dev/null +++ b/ansible/ci/library/latest_timestamps.py @@ -0,0 +1,95 @@ +#!/usr/bin/python + +# Copyright: (c) 2018, Terry Jones +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = r''' +--- +module: latest_timestamps + +short_description: Gets the latest set of snapshots from Pulp and overwrites + +version_added: "1.0.0" + +description: Gets the latest set of snapshots from given source URLs and returns dictionary in to replace 'appliances_repo_timestamps' with + +author: + - William Tripp +''' + +EXAMPLES = r''' +# Pass in a message +- name: Get latest timestamps + latest_timestamps: + repos_dict: "{{ appliances_repo_timestamp_sources }}" + content_url: "https://ark.stackhpc.com/pulp/content" + register: result + +''' + +RETURN = r''' +# These are examples of possible return values, and in general should use other names for return values. +latest_dict: + description: Dictionary with updated timestamps + type: dict + returned: always +changed_timestamps: + description: List of repos that've been updated + type: str[] + returned: always +''' + +from ansible.module_utils.basic import AnsibleModule +import requests +from bs4 import BeautifulSoup +from copy import deepcopy + +def run_module(): + module_args = dict( + repos_dict=dict(type='dict', required=True), + content_url=dict(type='str', required=True) + ) + + result = dict( + changed=False, + original_message='', + message='' + ) + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) + + original_timestamps = dict(module.params['repos_dict']) + latest_timestamps = deepcopy(original_timestamps) + changed_timestamps = [] + + for repo in original_timestamps: + for version in original_timestamps[repo]: + + html_txt = requests.get( + url= module.params['content_url'] + '/' + original_timestamps[repo][version]['path'] + ).text + timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html + timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags + latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed + + latest_timestamps[repo][version]['timestamp'] = latest_timestamp + if original_timestamps[repo][version]['timestamp'] != latest_timestamp: + changed_timestamps.append(repo+' '+version+': '+original_timestamps[repo][version]['timestamp']+' -> '+latest_timestamp) + + result['latest_dict'] = latest_timestamps + result['changed_timestamps'] = changed_timestamps + + module.exit_json(**result) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/ansible/ci/update_timestamps.yml b/ansible/ci/update_timestamps.yml new file mode 100644 index 000000000..c9ea78b1a --- /dev/null +++ b/ansible/ci/update_timestamps.yml @@ -0,0 +1,22 @@ +- hosts: localhost + tasks: + - name: Get latest timestamps from sources + latest_timestamps: + repos_dict: "{{ appliances_pulp_repos }}" + content_url: "https://ark.stackhpc.com/pulp/content" + register: _result + + - name: Print updated timestamps + ansible.builtin.debug: + var: _result.changed_timestamps + + - name: Overwrite repo timestamps with latest + ansible.builtin.blockinfile: + path: "{{ playbook_dir }}/../../environments/common/inventory/group_vars/all/defaults.yml" + marker: "# {mark} Marker for ansible/ci/update_timestamps.yml (GH workflow managed)" + block: | + {{ yaml_template | to_nice_yaml(indent=2) }} + vars: + yaml_template: + appliances_pulp_repos: "{{ _result.latest_dict }}" + when: (_result.changed_timestamps | count) > 0 diff --git a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json index 3c1e19058..73af32f80 100644 --- a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json +++ b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json @@ -1,6 +1,6 @@ { "cluster_image": { - "RL8": "openhpc-RL8-250109-1444-ecea8219", - "RL9": "openhpc-RL9-250109-1444-ecea8219" + "RL8": "openhpc-RL8-250113-1126-a2eaffc2", + "RL9": "openhpc-RL9-250113-1127-a2eaffc2" } } diff --git a/environments/common/inventory/group_vars/all/defaults.yml b/environments/common/inventory/group_vars/all/defaults.yml index e26bc3018..bdcd564d2 100644 --- a/environments/common/inventory/group_vars/all/defaults.yml +++ b/environments/common/inventory/group_vars/all/defaults.yml @@ -103,54 +103,62 @@ appliances_extra_packages: "{{ appliances_extra_packages_default + appliances_ex ###################### ark repo timestamps ################################################### +# BEGIN Marker for ansible/ci/update_timestamps.yml (GH workflow managed) appliances_pulp_repos: - baseos: - '9.4': - timestamp: 20241115T011711 - path: rocky/9.4/BaseOS/x86_64/os - '9.5': - timestamp: 20241216T013503 - path: rocky/9.5/BaseOS/x86_64/os - '8.10': - timestamp: 20241217T123729 - path: rocky/8.10/BaseOS/x86_64/os appstream: + '8.10': + path: rocky/8.10/AppStream/x86_64/os + timestamp: 20250113T025821 '9.4': - timestamp: 20241112T003151 path: rocky/9.4/AppStream/x86_64/os + timestamp: 20241112T003151 '9.5': - timestamp: 20241217T005008 path: rocky/9.5/AppStream/x86_64/os + timestamp: 20241228T031409 + baseos: '8.10': - timestamp: 20241217T123729 - path: rocky/8.10/AppStream/x86_64/os + path: rocky/8.10/BaseOS/x86_64/os + timestamp: 20250113T025821 + '9.4': + path: rocky/9.4/BaseOS/x86_64/os + timestamp: 20241115T011711 + '9.5': + path: rocky/9.5/BaseOS/x86_64/os + timestamp: 20250113T040158 + ceph: + '8': + path: centos/8-stream/storage/x86_64/ceph-quincy + timestamp: 20231104T015751 + '9': + path: centos/9-stream/storage/x86_64/ceph-reef + timestamp: 20240923T233036 crb: + '8.10': + path: rocky/8.10/PowerTools/x86_64/os + timestamp: 20250113T025821 '9.4': - timestamp: 20241115T003133 path: rocky/9.4/CRB/x86_64/os + timestamp: 20241115T003133 '9.5': - timestamp: 20241217T005008 path: rocky/9.5/CRB/x86_64/os - '8.10': - timestamp: 20241217T123729 - path: rocky/8.10/PowerTools/x86_64/os + timestamp: 20250113T030824 + epel: + '8': + path: epel/8/Everything/x86_64 + timestamp: 20250112T235732 + '9': + path: epel/9/Everything/x86_64 + timestamp: 20250112T235732 extras: + '8.10': + path: rocky/8.10/extras/x86_64/os + timestamp: 20250113T025821 '9.4': - timestamp: 20241118T002802 path: rocky/9.4/extras/x86_64/os + timestamp: 20241118T002802 '9.5': - timestamp: 20241218T004632 path: rocky/9.5/extras/x86_64/os - '8.10': - timestamp: 20241217T123729 - path: rocky/8.10/extras/x86_64/os - epel: - '9': - timestamp: 20241213T010218 - path: epel/9/Everything/x86_64 - '8': - timestamp: 20241216T235733 - path: epel/8/Everything/x86_64 + timestamp: 20250113T030824 openhpc_base: '8': path: OpenHPC/2/EL_8 @@ -165,10 +173,4 @@ appliances_pulp_repos: '9': path: OpenHPC/3/updates/EL_9 timestamp: 20241218T154614 - ceph: - '8': - timestamp: 20231104T015751 - path: centos/8-stream/storage/x86_64/ceph-quincy - '9': - timestamp: 20240923T233036 - path: centos/9-stream/storage/x86_64/ceph-reef +# END Marker for ansible/ci/update_timestamps.yml (GH workflow managed) diff --git a/requirements.txt b/requirements.txt index 872ee9516..a0cf4db8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ selinux # this is a shim to avoid having to use --system-site-packages, you stil netaddr matplotlib pulp-cli==0.23.2 +beautifulsoup4==4.12.3