diff --git a/.github/workflows/post-jobs-slack.yaml b/.github/workflows/post-jobs-slack.yaml index 2bef252..7ae4178 100644 --- a/.github/workflows/post-jobs-slack.yaml +++ b/.github/workflows/post-jobs-slack.yaml @@ -4,7 +4,7 @@ on: jobs: slack-poster: runs-on: ubuntu-latest - name: Run Jobs Slack Poster + name: Run Jobs Test Poster steps: - uses: actions/checkout@v3 with: @@ -13,14 +13,11 @@ jobs: - id: updater name: Job Updater uses: ./ - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} with: filename: "example/jobs.yaml" previous_filename: "example/jobs-previous.yaml" keys: "url" unique: "url" - deploy: false test: true @@ -28,14 +25,11 @@ jobs: - id: multifield_updater name: Job Updater uses: ./ - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} with: filename: "example/jobs.yaml" previous_filename: "example/jobs-previous.yaml" keys: "name,location,url" unique: "url" - deploy: false test: true diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..400c742 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020-2023 Vanessa Sochat, HPC Social Community + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 5a6e549..fd92d3b 100644 --- a/README.md +++ b/README.md @@ -9,22 +9,28 @@ and a yaml file with a list of jobs (or other links): url: https://my-job.org/12345 ``` -The action will inspect the file to determine lines that are newly added (compared to the parent commit) -for a field of interest (e.g., the "url" attribute in a list of jobs), extract this field, and then post to a Slack channel. +The action will inspect the file to determine lines that are newly added (compared to a parent commit or second file) +for a field of interest (e.g., the "url" attribute in a list of jobs), extract this field, and then post to a Slack channel, +a Discord Channel, Twitter, or Mastodon. ![img/example.png](img/example.png) -This is custom made to help the [US-RSE](https://github.com/US-RSE/usrse.github.io) site -to have job updates posted to slack! +This is custom made to help the [hpc.social](https://hpc.social/jobs) and [US-RSE](https://github.com/US-RSE/usrse.github.io) site +to have job updates posted to slack! If you'd like help setting this up for your group, please +ping [@vsoch](https://github.com/vsoch). -## Quickstart -1. Create a [webhook app](https://api.slack.com/messaging/webhooks#getting_started) and grab the URL and save to `SLACK_WEBHOOK` in your repository secrets. +## Usage + +You'll generally want to: + +1. Generate needed credentials for your apps of choice. 2. Add a GitHub workflow file, as shown below, with your desired triggers. -For more details on the above, keep reading. +More specifically, add a GitHub workflow file in `.github/workflows` to specify the following. Note that +the workflow below will do the check and update on any push to main (e.g., a merged pull request). -## 1. Slack Setup +### Deploy to Slack You'll want to [follow the instructions here](https://api.slack.com/messaging/webhooks#getting_started) to create a webhook for your slack community and channel of interest. This usually means first creating an application and selecting your slack @@ -39,21 +45,13 @@ curl -X POST -H 'Content-type: application/json' --data '{"text":"Hello, World!" ``` Click on "Add new webhook to workspace" and then test the provided url with the bot. Copy the webhook URL -and put it in a safe place. We will want to keep this URL as a secret in our eventual GitHub workflow. - - -## 2. Usage - -Add a GitHub workflow file in `.github/workflows` to specify the following. Note that -the workflow below will do the check and update on any push to main (e.g., a merged pull request). - -### Deploy to Slack +and save this to `SLACK_WEBHOOK` in your repository secrets. ```yaml on: push: paths: - - '_data/jobs.yml' + - '_data/jobs.yaml' branches: - main @@ -69,12 +67,12 @@ jobs: - id: updater name: Job Updater uses: rseng/jobs-updater@main - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} with: - filename: "_data/jobs.yml" + filename: "_data/jobs.yaml" keys: "url,name" unique: "url" + slack_webhook: ${{ secrets.SLACK_WEBHOOK }} + slack_deploy: true - run: echo ${{ steps.updater.outputs.fields }} name: Show New Jobs @@ -82,20 +80,20 @@ jobs: ``` In the above, we will include the url and name fields, and use the url field to determine uniqueness (default). -By default, given that you have the slack webhook in the environment, deployment will -happen because deploy is true. If you just want to test, then do: +Given that you have the slack webhook as a secret provided to the action and `deploy_slack` is true, the default `deploy` +variable (to indicate all services) is true and deployment will happen. If you just want to test, then do: ```yaml ... - id: updater name: Job Updater uses: rseng/jobs-updater@main - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} with: - filename: "_data/jobs.yml" + filename: "_data/jobs.yaml" keys: "url" deploy: false + slack_webhook: ${{ secrets.SLACK_WEBHOOK }} + slack_deploy: true ``` If you want to run a test run (meaning a random number of jobs will be selected that @@ -106,12 +104,11 @@ aren't necessarily new) then add test: - id: updater name: Job Updater uses: rseng/jobs-updater@main - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} with: - filename: "_data/jobs.yml" + filename: "_data/jobs.yaml" keys: "url" test: true + slack_deploy: true ``` If test is true, deploy will always be set to false. @@ -123,17 +120,15 @@ to true, and also define all the needed environment variables in your repository secrets. ```yaml +... - id: updater name: Job Updater uses: rseng/jobs-updater@add/deploy-arg - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} with: - filename: "_data/jobs.yml" + filename: "_data/jobs.yaml" keys: "url,name" - - deploy: true test: false + slack_deploy: true # Also deploy to Twitter (all secrets required in repository secrets) twitter_deploy: true @@ -153,13 +148,9 @@ secrets. - id: updater name: Job Updater uses: rseng/jobs-updater@main - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} with: - filename: "_data/jobs.yml" + filename: "_data/jobs.yaml" key: "url" - - deploy: true test: false # Also deploy to Mastodon (all secrets required in repository secrets) @@ -175,3 +166,56 @@ secrets. mastodon_access_token: ${{ secrets.MASTODON_ACCESS_TOKEN }} mastodon_api_base_url: ${{ secrets.MASTODON_API_BASE_URL }} ``` + +### Deploy to Discord + +To deploy to Discord you will need to [create a webhook](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks) +and then set `deploy_discord` to true, along with adding the webhook to your repository secrets as `DISCORD_WEBHOOK`. + +```yaml + - id: updater + name: Job Updater + uses: rseng/jobs-updater@main + with: + filename: "_data/jobs.yaml" + key: "url" + test: false + discord_deploy: true + discord_webhook: ${{ secrets.DISCORD_WEBHOOK }} +``` + +## Variables + +### Inputs + +The following variables are available. You can also look at the [action.yml](action.yml). + +| Name | Description | Required | Default | +|------|-------------|----------|---------| +| filename | The filename for the jobs | true | unset | +| previous_filename | The previous filename (for manual tesing or running alongside update) | false | unset | +| keys | Comma separated list of keys to post (defaults to url) | false | url | +| unique | Field to use to determine uniqueness | true | url | +| hashtag | A hashtag to use (defaults to `#Rseng`) | false | #RSEng | +| test | Test the updater (ensure there are jobs) | true | false | +| deploy | Global deploy across any service set to true? | true | true | +| slack_deploy | Deploy to Slack? | true | false | +| slack_webhook | Slack webhook to deploy to. | false | unset | +| discord_deploy | Deploy to Discord? | true | false | +| discord_webhook | Discord webhook to deploy to. | false | unset | +| twitter_deploy | Deploy to Twitter? | false | unset | +| twitter_api_key | API key generated for the user account to tweet | false | unset | +| twitter_api_secret |API secret generated for the user account to tweet | false | unset | +| twitter_consumer_key | Consumer key generated for the entire app | false | unset | +| twitter_consumer_secret | Consumer secret generated for the entire app | false | unset | +| mastodon_deploy | Boolean to deploy to Mastodon | false | unset | +| mastodon_access_token | API key generated for the user account to tweet | false | unset | +| mastodon_api_base_url | Base URL of the Mastodon instance to post to, e.g., https://fosstodon.org/ | false | unset | + +### Outputs + +| Name | Description | +|------|-------------| +| Fields (keys) parsed | The fields that are parsed in the jobs | +| Matrix | Matrix (list of lists) with value (index 1), icon (index 2) and full message (index 3) | +| Empty Matrix | true if empty, false otherwise | diff --git a/action.yml b/action.yml index b02a923..ecf53c3 100644 --- a/action.yml +++ b/action.yml @@ -1,5 +1,5 @@ name: 'Jobs Updater' -description: "The jobs updater will respond on some trigger, and them parse a jobs file for changes, posting a field of interest to slack." +description: "The jobs updater will respond on some trigger, and them parse a jobs file for changes, posting to one or more services." inputs: filename: description: the filename for the jobs @@ -15,19 +15,39 @@ inputs: description: Field to use to determine uniqueness required: true default: url + hashtag: + description: A hashtag to use (defaults to Rseng) + required: false + default: "#RSEng" test: description: Test the updater (ensure there are jobs) required: true default: false deploy: - description: Deploy to slack? + description: Global deploy across any service set to true? required: true default: true + + slack_deploy: + description: Deploy to Slack? + required: true + default: false + slack_webhook: + description: Slack webhook to deploy to. + required: false + discord_deploy: + description: Deploy to Discord? + required: true + default: false + discord_webhook: + description: Discord webhook to deploy to. + required: false + # If you want to post to Twitter, all of these credentials are required for a specific user # API keys are typically generated on behalf of a user (at the bottom of the interface) twitter_deploy: - description: Boolean to deploy to Twitter + description: Deploy to Twitter? required: false twitter_api_key: description: API key generated for the user account to tweet @@ -86,12 +106,17 @@ runs: ACTION_DIR: ${{ github.action_path }} INPUT_REPO: ${{ github.repository }} INPUT_TEST: ${{ inputs.test }} + INPUT_HASHTAG: ${{ inputs.hashtag }} INPUT_DEPLOY: ${{ inputs.deploy }} + SLACK_DEPLOY: ${{ inputs.slack_deploy }} + SLACK_WEBHOOK: ${{ inputs.slack_webhook }} TWITTER_DEPLOY: ${{ inputs.twitter_deploy }} TWITTER_API_KEY: ${{ inputs.twitter_api_key }} TWITTER_API_SECRET: ${{ inputs.twitter_api_secret }} TWITTER_CONSUMER_KEY: ${{ inputs.twitter_consumer_key }} TWITTER_CONSUMER_SECRET: ${{ inputs.twitter_consumer_secret }} + DISCORD_DEPLOY: ${{ inputs.discord_deploy }} + DISCORD_WEBHOOK: ${{ inputs.discord_webhook }} MASTODON_DEPLOY: ${{ inputs.mastodon_deploy }} MASTODON_ACCESS_TOKEN: ${{ inputs.mastodon_access_token }} MASTODON_API_BASE_URL: ${{ inputs.mastodon_api_base_url }} diff --git a/entrypoint.sh b/entrypoint.sh index 1664ee8..a630946 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -35,12 +35,7 @@ else fi fi -# Required to have slack webhook in environment -if [ -z ${SLACK_WEBHOOK+x} ]; then - printf "Warning, SLACK_WEBHOOK not found, will not deploy to slack.\n" -fi - -# Are we deploying? +# Are we globally deploying? DEPLOY=true if [[ "${INPUT_DEPLOY}" == "false" ]]; then DEPLOY=false @@ -58,28 +53,62 @@ if [ ! -z ${TWITTER_API_KEY+x} ] && [ ! -z ${TWITTER_API_SECRET+x} ] && [ ! -z $ DEPLOY_TWITTER=true fi -# likewise for mastodon +# Likewise for mastodon DEPLOY_MASTODON=false if [ ! -z ${MASTODON_ACCESS_TOKEN+x} ] && [ ! -z ${MASTODON_API_BASE_URL+x} ] && [[ "${MASTODON_DEPLOY}" == "true" ]]; then DEPLOY_MASTODON=true fi -COMMAND="python ${ACTION_DIR}/find-updates.py update --keys ${INPUT_KEYS} --unique ${INPUT_UNIQUE} --original ${JOBFILE} --updated ${INPUT_FILENAME}" +# And Slack +DEPLOY_SLACK=false +if [ ! -z ${SLACK_WEBHOOK+x} ] && [[ "${SLACK_DEPLOY}" == "true" ]]; then + DEPLOY_SLACK=true +fi + +# And Discord +DEPLOY_DISCORD=false +if [ ! -z ${DISCORD_WEBHOOK+x} ] && [[ "${DISCORD_DEPLOY}" == "true" ]]; then + DEPLOY_DISCORD=true +fi + +# Alert the user everything that will happen +printf " Global Deploy: ${DEPLOY}\n" +printf "Deploy Mastodon: ${DEPLOY_MASTODON}\n" +printf " Deploy Twitter: ${DEPLOY_TWITTER}\n" +printf " Deploy Discord: ${DEPLOY_DISCORD}\n" +printf " Deploy Slack: ${DEPLOY_SLACK}\n" +printf " Original: ${JOBFILE}\n" +printf " Updated: ${INPUT_FILENAME}\n" +printf " Hashtag: ${INPUT_HASHTAG}\n" +printf " Unique: ${INPUT_UNIQUE}\n" +printf " Keys: ${INPUT_KEYS}\n" +printf " Test: ${INPUT_TEST}\n" + + +COMMAND="python ${ACTION_DIR}/find-updates.py update --keys ${INPUT_KEYS} --unique ${INPUT_UNIQUE} --original ${JOBFILE} --updated ${INPUT_FILENAME} --hashtag ${INPUT_HASHTAG}" if [[ "${DEPLOY}" == "true" ]]; then - COMMAND="${COMMAND} --deploy" + COMMAND="${COMMAND} --deploy" fi if [[ "${INPUT_TEST}" == "true" ]]; then - COMMAND="${COMMAND} --test" + COMMAND="${COMMAND} --test" fi if [[ "${DEPLOY_TWITTER}" == "true" ]]; then - COMMAND="${COMMAND} --deploy-twitter" + COMMAND="${COMMAND} --deploy-twitter" fi if [[ "${DEPLOY_MASTODON}" == "true" ]]; then - COMMAND="${COMMAND} --deploy-mastodon" + COMMAND="${COMMAND} --deploy-mastodon" +fi + +if [[ "${DEPLOY_SLACK}" == "true" ]]; then + COMMAND="${COMMAND} --deploy-slack" +fi + +if [[ "${DEPLOY_DISCORD}" == "true" ]]; then + COMMAND="${COMMAND} --deploy-discord" fi echo "${COMMAND}" diff --git a/find-updates.py b/find-updates.py old mode 100644 new mode 100755 index 081fae5..376652d --- a/find-updates.py +++ b/find-updates.py @@ -17,16 +17,28 @@ import tweepy from mastodon import Mastodon +# Shared headers for slack / discord +headers = {"Content-type": "application/json"} +success_codes = [200, 201, 204] + + def read_yaml(filename): + """ + Read yaml from file. + """ with open(filename, "r") as stream: content = yaml.load(stream, Loader=yaml.FullLoader) return content def write_file(content, filename): + """ + Write yaml to file. + """ with open(filename, "w") as fd: fd.write(content) + def set_env_and_output(name, value): """ helper function to echo a key/value pair to the environment file @@ -71,6 +83,13 @@ def get_parser(): help="deploy to Slack", ) + update.add_argument( + "--hashtag", + dest="hashtag", + default="#RSEng", + help="A hashtag (starting with #) to include in the post, defaults to #RSEng", + ) + update.add_argument( "--deploy-twitter", dest="deploy_twitter", @@ -79,6 +98,22 @@ def get_parser(): help="deploy to Twitter (required api token/secret, and consumer token/secret)", ) + update.add_argument( + "--deploy-slack", + dest="deploy_slack", + action="store_true", + default=False, + help="deploy to Slack (required webhook URL in environment)", + ) + + update.add_argument( + "--deploy-discord", + dest="deploy_discord", + action="store_true", + default=False, + help="deploy to Discord (required webhook URL in environment)", + ) + update.add_argument( "--deploy-mastodon", dest="deploy_mastodon", @@ -119,19 +154,32 @@ def get_parser(): return parser -def get_twitter_client(): +def get_required_envars(required, client_name): + """ + Get and return a set of required environment variables. + """ envars = {} - for envar in [ - "TWITTER_API_KEY", - "TWITTER_API_SECRET", - "TWITTER_CONSUMER_KEY", - "TWITTER_CONSUMER_SECRET", - ]: + for envar in required: value = os.environ.get(envar) if not value: - sys.exit("%s is not set, and required when twitter deploy is true!" % envar) + sys.exit( + f"{envar} is not set, and required when {client_name} deploy is true!" + ) envars[envar] = value + return envars + +def get_twitter_client(): + """ + Get a Twitter client, also ensure all needed envars are provided. + """ + required = [ + "TWITTER_API_KEY", + "TWITTER_API_SECRET", + "TWITTER_CONSUMER_KEY", + "TWITTER_CONSUMER_SECRET", + ] + envars = get_required_envars(required, "twitter") return tweepy.Client( consumer_key=envars["TWITTER_CONSUMER_KEY"], consumer_secret=envars["TWITTER_CONSUMER_SECRET"], @@ -139,25 +187,27 @@ def get_twitter_client(): access_token_secret=envars["TWITTER_API_SECRET"], ) + def get_mastodon_client(): - envars = {} - for envar in [ + """ + Get a Mastodon client, requiring a token and base URL. + """ + required = [ "MASTODON_ACCESS_TOKEN", "MASTODON_API_BASE_URL", - ]: - value = os.environ.get(envar) - if not value: - sys.exit("%s is not set, and required when mastodon deploy is true!" % envar) - envars[envar] = value - + ] + envars = get_required_envars(required, "mastodon") return Mastodon( access_token=envars["MASTODON_ACCESS_TOKEN"], api_base_url=envars["MASTODON_API_BASE_URL"], ) + def prepare_post(entry, keys): - """Prepare the slack or tweet. There should be a descriptor for - all fields except for url. + """ + Prepare the post. + + There should be a descriptor for all fields except for url. """ post = "" for key in keys: @@ -169,6 +219,46 @@ def prepare_post(entry, keys): return post +def deploy_slack(webhook, message): + """ + Deploy a post to slack + """ + data = {"text": message, "unfurl_links": True} + response = requests.post(webhook, headers=headers, data=json.dumps(data)) + if response.status_code not in success_codes: + print(response) + sys.exit( + "Issue with making Slack POST request: %s, %s" + % (response.reason, response.status_code) + ) + + +def deploy_discord(webhook, message): + """ + Deploy a post to Discord + """ + data = {"content": message} + response = requests.post(webhook, headers=headers, data=json.dumps(data)) + if response.status_code not in success_codes: + print(response) + sys.exit( + "Issue with making Discord POST request: %s, %s" + % (response.reason, response.status_code) + ) + + +def deploy_twitter(client, message): + """ + Deploy to Twitter. + + Twitter supports emojis, so we add them back. + """ + try: + client.create_tweet(text=message) + except Exception as e: + print(f"Issue posting tweet: {e}, and length is {len(message)}") + + def main(): parser = get_parser() @@ -185,15 +275,20 @@ def help(return_code=0): if not os.path.exists(filename): sys.exit(f"{filename} does not exist.") - # Cut out early if we are deploying to twitter or mastodon but missing envars - client = None + # Deploying to Twitter? + twitter_client = None if args.deploy_twitter: - client = get_twitter_client() + twitter_client = get_twitter_client() mastodon_client = None if args.deploy_mastodon: mastodon_client = get_mastodon_client() + # Prepare webhooks for slack and mastodon + slack_webhook = os.environ.get("SLACK_WEBHOOK") + discord_webhook = os.environ.get("DISCORD_WEBHOOK") + + # Get original and updated jobs original = read_yaml(args.original) updated = read_yaml(args.updated) @@ -202,9 +297,16 @@ def help(return_code=0): # Find new posts in updated previous = set() + missing_count = 0 for item in original: if args.unique in item: previous.add(item[args.unique]) + else: + missing_count += 1 + + # Warn the user if some are missing the unique key + if missing_count: + print(f"Warning: key {args.unique} is missing in {missing_count} items.") # Create a lookup by the unique id new = [] @@ -227,12 +329,6 @@ def help(return_code=0): set_env_and_output("empty_matrix", "true") sys.exit(0) - # Prepare the data - webhook = os.environ.get("SLACK_WEBHOOK") - if not webhook and args.deploy: - sys.exit("Cannot find SLACK_WEBHOOK in environment.") - - headers = {"Content-type": "application/json"} matrix = [] # Format into slack messages @@ -243,6 +339,7 @@ def help(return_code=0): "👀ī¸", "✨ī¸", "🤖ī¸", + "😎ī¸", "đŸ’ŧī¸", "🤩ī¸", "😸ī¸", @@ -257,9 +354,9 @@ def help(return_code=0): # Prepare the post post = prepare_post(entry, keys) - choice = random.choice(icons) - message = "New Job! %s\n%s" % (choice, post) + message = f"New {args.hashtag} Job! {choice}: {post}" + newline_message = f"New {args.hashtag} Job! {choice}\n{post}" print(message) # Convert dates, etc. back to string @@ -271,35 +368,28 @@ def help(return_code=0): continue # Add the job name to the matrix - # IMPORTANT: emojis in output mess up the action + # IMPORTANT: emojis in output can mess up some of the services matrix.append(filtered) - data = {"text": message, "unfurl_links": True} + + # Don't continue if testing or global deploy is false + if not args.deploy or args.test is True: + continue # If we are instructed to deploy to twitter and have a client - if args.deploy_twitter and client: - message = "New #RSEng Job! %s\n%s" % (choice, post) - print(message) - try: - client.create_tweet(text=message) - except Exception as e: - print("Issue posting tweet: %s, and length is %s" % (e, len(message))) + if args.deploy_twitter and twitter_client: + deploy_twitter(twitter_client, newline_message) # If we are instructed to deploy to mastodon and have a client if args.deploy_mastodon and mastodon_client: - message = "New #RSEng Job! %s: %s" % (choice, post) mastodon_client.toot(status=message) - # Don't continue if testing - if not args.deploy or args.test: - continue + # Deploy to Slack + if slack_webhook is not None and args.deploy_slack: + deploy_slack(slack_webhook, message) - response = requests.post(webhook, headers=headers, data=json.dumps(data)) - if response.status_code not in [200, 201]: - print(response) - sys.exit( - "Issue with making POST request: %s, %s" - % (response.reason, response.status_code) - ) + # Deploy to Discord + if discord_webhook is not None and args.deploy_discord: + deploy_discord(discord_webhook, message) set_env_and_output("fields", json.dumps(keys)) set_env_and_output("matrix", json.dumps(matrix))