From 322a1235c167d920ab1254a4b4a25c5892d275c3 Mon Sep 17 00:00:00 2001 From: Vanessasaurus <814322+vsoch@users.noreply.github.com> Date: Sun, 22 Oct 2023 03:06:20 -0600 Subject: [PATCH] updated doctrings, black, and version bump (#85) * updated doctrings, black, and version bump Signed-off-by: vsoch * update container base to bookworm Signed-off-by: vsoch --------- Signed-off-by: vsoch Co-authored-by: vsoch --- .github/workflows/main.yml | 8 ++++---- CHANGELOG.md | 1 + Dockerfile | 3 ++- rse/__init__.py | 10 ---------- rse/app/api.py | 2 +- rse/app/config.py | 2 +- rse/app/export.py | 3 +-- rse/app/server.py | 2 +- rse/app/views/main.py | 4 +--- rse/app/views/repositories.py | 10 +--------- rse/client/__init__.py | 2 +- rse/client/add.py | 3 +-- rse/client/annotate.py | 3 +-- rse/client/clear.py | 3 +-- rse/client/config.py | 3 +-- rse/client/exists.py | 3 +-- rse/client/export.py | 4 +--- rse/client/generate.py | 3 +-- rse/client/get.py | 3 +-- rse/client/imp.py | 4 +--- rse/client/init.py | 3 +-- rse/client/label.py | 3 +-- rse/client/listing.py | 3 +-- rse/client/metrics.py | 2 +- rse/client/scrape.py | 3 +-- rse/client/search.py | 3 +-- rse/client/shell.py | 3 +-- rse/client/start.py | 3 +-- rse/client/topics.py | 3 +-- rse/client/update.py | 3 +-- rse/defaults.py | 2 +- rse/exceptions.py | 2 +- rse/logger/message.py | 3 +-- rse/main/__init__.py | 5 +---- rse/main/config/__init__.py | 2 +- rse/main/criteria.py | 2 +- rse/main/database/__init__.py | 2 +- rse/main/database/base.py | 2 +- rse/main/database/filesystem.py | 4 +--- rse/main/database/models.py | 2 +- rse/main/database/relational.py | 3 +-- rse/main/database/sqlite.py | 2 +- rse/main/export/__init__.py | 3 +-- rse/main/parsers/__init__.py | 2 +- rse/main/parsers/base.py | 2 +- rse/main/parsers/custom.py | 3 +-- rse/main/parsers/github.py | 3 +-- rse/main/parsers/gitlab.py | 3 +-- rse/main/parsers/zenodo.py | 3 +-- rse/main/scrapers/__init__.py | 2 +- rse/main/scrapers/ascl.py | 5 +---- rse/main/scrapers/base.py | 2 +- rse/main/scrapers/biogrids.py | 4 +--- rse/main/scrapers/biotools.py | 5 +---- rse/main/scrapers/csv.py | 3 +-- rse/main/scrapers/debian.py | 4 +--- rse/main/scrapers/googlesheet.py | 3 +-- rse/main/scrapers/hal.py | 1 - rse/main/scrapers/imperial.py | 3 +-- rse/main/scrapers/joss.py | 4 +--- rse/main/scrapers/molssi.py | 4 +--- rse/main/scrapers/ropensci.py | 3 --- rse/main/scrapers/rsnl.py | 3 +-- rse/main/taxonomy.py | 2 +- rse/utils/command.py | 1 - rse/version.py | 4 ++-- 66 files changed, 67 insertions(+), 141 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 69178e1..edf6b81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: formatting: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup black environment run: conda create --quiet --name black pyflakes @@ -21,7 +21,7 @@ jobs: run: | export PATH="/usr/share/miniconda/bin:$PATH" source activate black - pip install black + pip install black==23.3.0 black --check rse - name: Check imports with pyflakes @@ -38,7 +38,7 @@ jobs: env: RSE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup testing environment run: conda create --quiet --name testing pytest @@ -56,7 +56,7 @@ jobs: env: CONTAINER: quay.io/vanessa/rse steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build Docker Image run: docker build -t "${CONTAINER}" . - name: Tag and Preview Container diff --git a/CHANGELOG.md b/CHANGELOG.md index 05b7a09..b1a0728 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are: The versions coincide with releases on pip. ## [0.0.x](https://github.com/rseng/rse/tree/master) (0.0.x) + - allow an erroneous response for GitHub org get repos (0.0.48) - allow custom import for csv and google-sheet (0.0.47) - support for csv import (0.0.46) - ensure Google scraper skips malformed rows, etc (0.0.45) diff --git a/Dockerfile b/Dockerfile index b828d41..c92e2f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM bitnami/minideb:stretch +FROM bitnami/minideb:bookworm + # docker build -t quay.io/vanessa/rse . LABEL MAINTAINER @vsoch ENV PATH /opt/conda/bin:${PATH} diff --git a/rse/__init__.py b/rse/__init__.py index 2050c7f..0a042b9 100644 --- a/rse/__init__.py +++ b/rse/__init__.py @@ -1,13 +1,3 @@ -""" - -Copyright (C) 2020-2022 Vanessa Sochat. - -This Source Code Form is subject to the terms of the -Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed -with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -""" - from .version import __version__ assert __version__ diff --git a/rse/app/api.py b/rse/app/api.py index eb8f739..ebceb9c 100644 --- a/rse/app/api.py +++ b/rse/app/api.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/app/config.py b/rse/app/config.py index 092984d..2f75b81 100644 --- a/rse/app/config.py +++ b/rse/app/config.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/app/export.py b/rse/app/export.py index ea7b6ff..e4af224 100644 --- a/rse/app/export.py +++ b/rse/app/export.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -157,7 +157,6 @@ def kill(): ) for url, outfile in urls.items(): - # Skip if we've already created it if os.path.exists(outfile): continue diff --git a/rse/app/server.py b/rse/app/server.py index b959804..945146c 100644 --- a/rse/app/server.py +++ b/rse/app/server.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/app/views/main.py b/rse/app/views/main.py index 10e22c3..d5be3ca 100644 --- a/rse/app/views/main.py +++ b/rse/app/views/main.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -33,11 +33,9 @@ def index(): @app.route("/") def topics_view(): - topics = app.client.topics() repos = [] for name in app.client.list(): - # Obtain the repository and load the data. repo = app.client.get(name[0]) repo.parser.load(repo.data) diff --git a/rse/app/views/repositories.py b/rse/app/views/repositories.py index 15a57d6..c759f0b 100644 --- a/rse/app/views/repositories.py +++ b/rse/app/views/repositories.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -18,7 +18,6 @@ @app.route("%srepository/" % RSE_URL_PREFIX) def repository_view(uid): - # Obtain the repository and load the data. repo = app.client.get(uid) repo.parser.load(repo.data) @@ -55,7 +54,6 @@ def annotate_repos(message=""): @app.route("%sannotate-criteria" % RSE_URL_PREFIX, methods=["POST", "GET"]) def annotate_criteria(): - # If it's a post, update the annotation username = None if request.method == "POST": @@ -96,7 +94,6 @@ def annotate_criteria(): @app.route("%srepository//annotate-criteria" % RSE_URL_PREFIX) def annotate_static_criteria(uid): - # Get criteria / annotation set for specific repository username = request.args.get("username") repo = app.client.get(uid) @@ -115,7 +112,6 @@ def annotate_static_criteria(uid): @app.route("%srepository//annotate-taxonomy" % RSE_URL_PREFIX) def annotate_static_taxonomy(uid): - # Get criteria / annotation set for specific repository username = request.args.get("username") repo = app.client.get(uid) @@ -137,7 +133,6 @@ def annotate_static_taxonomy(uid): @app.route("%sannotate-taxonomy" % RSE_URL_PREFIX, methods=["GET", "POST"]) def annotate_taxonomy(): - # If we don't have a color lookup, make one if not hasattr(app, "taxonomy"): app.taxonomy = generate_taxonomy(app) @@ -181,7 +176,6 @@ def annotate_taxonomy(): def generate_taxonomy(app): - taxonomy = app.client.list_taxonomy() # Update the color list with existing colors (consistency) @@ -209,7 +203,6 @@ def generate_taxonomy(app): def update_criteria(): - updates = {} repo_uid = request.form.get("repo_uid") username = request.form.get("username") @@ -231,7 +224,6 @@ def update_criteria(): def update_taxonomy(): - uids = [] repo_uid = request.form.get("repo_uid") username = request.form.get("username") diff --git a/rse/client/__init__.py b/rse/client/__init__.py index e4af8bf..a222184 100644 --- a/rse/client/__init__.py +++ b/rse/client/__init__.py @@ -2,7 +2,7 @@ """ -Copyright (C) 2020 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/client/add.py b/rse/client/add.py index b5d9f5a..c4e6e4f 100644 --- a/rse/client/add.py +++ b/rse/client/add.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -14,7 +14,6 @@ def main(args, extra): - # Create a research software encyclopedia enc = Encyclopedia(config_file=args.config_file, database=args.database) diff --git a/rse/client/annotate.py b/rse/client/annotate.py index 0bbed4e..08be618 100644 --- a/rse/client/annotate.py +++ b/rse/client/annotate.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -13,7 +13,6 @@ def main(args, extra): - # Create a research software encyclopedia client = Encyclopedia(config_file=args.config_file, database=args.database) diff --git a/rse/client/clear.py b/rse/client/clear.py index 4af08c0..8b94115 100644 --- a/rse/client/clear.py +++ b/rse/client/clear.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -12,7 +12,6 @@ def main(args, extra): - # Clear a parser, uid, or target enc = Encyclopedia(config_file=args.config_file, database=args.database) enc.clear(args.target, noprompt=args.force) diff --git a/rse/client/config.py b/rse/client/config.py index ccc4a9a..d180d36 100644 --- a/rse/client/config.py +++ b/rse/client/config.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -17,7 +17,6 @@ def main(args, extra): - bot = logging.getLogger("rse.client") # The user wants to set the database diff --git a/rse/client/exists.py b/rse/client/exists.py index 1ebffce..b65a44f 100644 --- a/rse/client/exists.py +++ b/rse/client/exists.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -16,7 +16,6 @@ def main(args, extra): - enc = Encyclopedia(config_file=args.config_file) # Case 1: empty list indicates listing all diff --git a/rse/client/export.py b/rse/client/export.py index ad41786..daae0bd 100644 --- a/rse/client/export.py +++ b/rse/client/export.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -19,7 +19,6 @@ def main(args, extra): - client = Encyclopedia(config_file=args.config_file, database=args.database) # Case 1: empty list indicates listing all @@ -28,7 +27,6 @@ def main(args, extra): # Export a list of repos if args.export_type == "repos-txt": - # We just want the unique id, the first result repos = [x[0] for x in client.list()] write_file("\n".join(repos), args.path) diff --git a/rse/client/generate.py b/rse/client/generate.py index f759177..e367ed0 100644 --- a/rse/client/generate.py +++ b/rse/client/generate.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -13,7 +13,6 @@ def main(args, extra): - punctuation = "!#$%&()*+,-./:;<=>?@^_{|}~" choices = string.ascii_letters + string.digits + punctuation selected = [random.SystemRandom().choice(choices) for _ in range(50)] diff --git a/rse/client/get.py b/rse/client/get.py index 2b0d408..5477e7d 100644 --- a/rse/client/get.py +++ b/rse/client/get.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -18,7 +18,6 @@ def main(args, extra): - # Create a research software encyclopedia enc = Encyclopedia(config_file=args.config_file, database=args.database) diff --git a/rse/client/imp.py b/rse/client/imp.py index 097a266..9367aef 100644 --- a/rse/client/imp.py +++ b/rse/client/imp.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -20,7 +20,6 @@ def parse_extra(extra): args = [] kwargs = {} for arg in extra: - # This is a key value pair (extra) if arg.startswith("--") and "=" in arg: key, val = arg.strip().split("=") @@ -33,7 +32,6 @@ def parse_extra(extra): def main(args, extra): - try: importer = get_importer(args.import_type) except: diff --git a/rse/client/init.py b/rse/client/init.py index 40a1765..8b7b496 100644 --- a/rse/client/init.py +++ b/rse/client/init.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -15,7 +15,6 @@ def main(args, extra): - # present working directory path = args.path if args.path == ".": diff --git a/rse/client/label.py b/rse/client/label.py index 8fcbe92..5414abc 100644 --- a/rse/client/label.py +++ b/rse/client/label.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -12,7 +12,6 @@ def main(args, extra): - enc = Encyclopedia(config_file=args.config_file, database=args.database) # Grab the repository uid, label key and value diff --git a/rse/client/listing.py b/rse/client/listing.py index 95bce0c..6b9a80c 100644 --- a/rse/client/listing.py +++ b/rse/client/listing.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -13,7 +13,6 @@ def main(args, extra): - # Create a queue object, run the command to match to an executor enc = Encyclopedia(config_file=args.config_file, database=args.database) diff --git a/rse/client/metrics.py b/rse/client/metrics.py index 682d5bb..1f792a0 100644 --- a/rse/client/metrics.py +++ b/rse/client/metrics.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/client/scrape.py b/rse/client/scrape.py index a566162..bd55854 100644 --- a/rse/client/scrape.py +++ b/rse/client/scrape.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -14,7 +14,6 @@ def main(args, extra): - try: scraper = get_named_scraper(args.scraper_name[0]) except: diff --git a/rse/client/search.py b/rse/client/search.py index c054c4a..defff84 100644 --- a/rse/client/search.py +++ b/rse/client/search.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -15,7 +15,6 @@ def main(args, extra): - enc = Encyclopedia(config_file=args.config_file, database=args.database) query = " ".join(args.query).strip() diff --git a/rse/client/shell.py b/rse/client/shell.py index 69acd72..0a5bb6a 100644 --- a/rse/client/shell.py +++ b/rse/client/shell.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -13,7 +13,6 @@ def main(args, extra): - lookup = {"ipython": ipython, "python": python, "bpython": bpython} shells = ["ipython", "python", "bpython"] diff --git a/rse/client/start.py b/rse/client/start.py index 68c1d3c..9aee61e 100644 --- a/rse/client/start.py +++ b/rse/client/start.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -14,7 +14,6 @@ def main(args, extra): - # Clear a parser, uid, or target enc = Encyclopedia(config_file=args.config_file, database=args.database) diff --git a/rse/client/topics.py b/rse/client/topics.py index 32d6ffb..ae948c5 100644 --- a/rse/client/topics.py +++ b/rse/client/topics.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -12,7 +12,6 @@ def main(args, extra): - # Create an encyclopedia object enc = Encyclopedia(config_file=args.config_file, database=args.database) diff --git a/rse/client/update.py b/rse/client/update.py index f712f55..86f698c 100644 --- a/rse/client/update.py +++ b/rse/client/update.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -12,7 +12,6 @@ def main(args, extra): - # Create a queue object, run the command to match to an executor enc = Encyclopedia(config_file=args.config_file, database=args.database) diff --git a/rse/defaults.py b/rse/defaults.py index a7e7682..13a755b 100644 --- a/rse/defaults.py +++ b/rse/defaults.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/exceptions.py b/rse/exceptions.py index de70c6b..ff99668 100644 --- a/rse/exceptions.py +++ b/rse/exceptions.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/logger/message.py b/rse/logger/message.py index 0321367..aebf977 100644 --- a/rse/logger/message.py +++ b/rse/logger/message.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -205,7 +205,6 @@ def show_progress( length: character length of bar (Int) """ if not self.level == QUIET: - percent = 100 * (iteration / float(total)) progress = int(length * iteration // total) diff --git a/rse/main/__init__.py b/rse/main/__init__.py index 6ad692c..049a1ad 100644 --- a/rse/main/__init__.py +++ b/rse/main/__init__.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -621,10 +621,8 @@ def annotate_criteria(self, username, unseen_only=True, repo=None, save=False): for repo, criteria in self.yield_criteria_annotation_repos( username, unseen_only, repo ): - # Only print repository if not seen yet if not last or repo.uid != last.uid: - # If we have a last repo, we need to save progress if last is not None and save is True: self.save_criteria(last) @@ -670,7 +668,6 @@ def annotate_taxonomy(self, username, unseen_only=True, repo=None, save=False): prefix = "0:%s or s to skip" % (len(items) - 1) for repo in self.yield_taxonomy_annotation_repos(username, unseen_only, repo): - message.info(f"\n{repo.url} [{repo.description}]:") print("How would you categorize this software? [enter one or more numbers]") for i, t in enumerate(items): diff --git a/rse/main/config/__init__.py b/rse/main/config/__init__.py index 96d2a64..43740df 100644 --- a/rse/main/config/__init__.py +++ b/rse/main/config/__init__.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/main/criteria.py b/rse/main/criteria.py index 507124f..0c87718 100644 --- a/rse/main/criteria.py +++ b/rse/main/criteria.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/main/database/__init__.py b/rse/main/database/__init__.py index 3ee3a25..60ce84e 100644 --- a/rse/main/database/__init__.py +++ b/rse/main/database/__init__.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/main/database/base.py b/rse/main/database/base.py index 4dbe2a3..d2f02e4 100644 --- a/rse/main/database/base.py +++ b/rse/main/database/base.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/main/database/filesystem.py b/rse/main/database/filesystem.py index 6bcbedf..1622efd 100644 --- a/rse/main/database/filesystem.py +++ b/rse/main/database/filesystem.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -164,7 +164,6 @@ def search(self, query, taxonomy=None, criteria=None): taxonomy_regex = "(%s)" "|".join(taxonomy or []) criteria_regex = "(%s)" "|".join(criteria or []) for repo in self.list_repos(): - if query: if re.search(query, repo[0], re.IGNORECASE): if query not in results: @@ -336,7 +335,6 @@ def create(self, should_exist=False): """ if should_exist: if not os.path.exists(self.filename): - # Might be provided prefix contenders = glob("%s*" % os.path.join(self.data_base, self.parser.uid)) if len(contenders) == 1: diff --git a/rse/main/database/models.py b/rse/main/database/models.py index b5f5e1f..38338b0 100644 --- a/rse/main/database/models.py +++ b/rse/main/database/models.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/main/database/relational.py b/rse/main/database/relational.py index 97c51fc..4425972 100644 --- a/rse/main/database/relational.py +++ b/rse/main/database/relational.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -114,7 +114,6 @@ def add(self, uid, data=None): parser = get_parser(uid, config=self.config) if not self.exists(parser.uid): - if not data: data = parser.get_metadata() else: diff --git a/rse/main/database/sqlite.py b/rse/main/database/sqlite.py index cf1a112..c17ac45 100644 --- a/rse/main/database/sqlite.py +++ b/rse/main/database/sqlite.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/main/export/__init__.py b/rse/main/export/__init__.py index 1fc0415..174e927 100644 --- a/rse/main/export/__init__.py +++ b/rse/main/export/__init__.py @@ -1,5 +1,5 @@ __author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2022, Vanessa Sochat" +__copyright__ = "Copyright 2021-2023, Vanessa Sochat" __license__ = "MPL 2.0" import json @@ -51,7 +51,6 @@ def export(self, repos): # Iterate through repos to generate data print("Generating jekyll web export") for repo in repos: - # url will be over-written by jekyll templates, etc. if "url" in repo and repo["url"]: repo["repo_url"] = repo["url"] diff --git a/rse/main/parsers/__init__.py b/rse/main/parsers/__init__.py index 72f53af..86ced4c 100644 --- a/rse/main/parsers/__init__.py +++ b/rse/main/parsers/__init__.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/main/parsers/base.py b/rse/main/parsers/base.py index 1ea05b3..c1e8804 100644 --- a/rse/main/parsers/base.py +++ b/rse/main/parsers/base.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/main/parsers/custom.py b/rse/main/parsers/custom.py index 99e74a6..2c0a0ed 100644 --- a/rse/main/parsers/custom.py +++ b/rse/main/parsers/custom.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2022 Vanessa Sochat. +Copyright (C) 2022-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -18,7 +18,6 @@ class CustomParser(ParserBase): - name = rse.defaults.RSE_CUSTOM_DATABASE_DIR matchstring = "custom" diff --git a/rse/main/parsers/github.py b/rse/main/parsers/github.py index cad9088..3fdd488 100644 --- a/rse/main/parsers/github.py +++ b/rse/main/parsers/github.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -22,7 +22,6 @@ class GitHubParser(ParserBase): - name = "github" matchstring = "github" diff --git a/rse/main/parsers/gitlab.py b/rse/main/parsers/gitlab.py index 50a0f87..c5826d5 100644 --- a/rse/main/parsers/gitlab.py +++ b/rse/main/parsers/gitlab.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -21,7 +21,6 @@ class GitLabParser(ParserBase): - name = "gitlab" matchstring = "gitlab" diff --git a/rse/main/parsers/zenodo.py b/rse/main/parsers/zenodo.py index fc5cd63..987fbd1 100644 --- a/rse/main/parsers/zenodo.py +++ b/rse/main/parsers/zenodo.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -19,7 +19,6 @@ class ZenodoParser(ParserBase): - name = "zenodo" matchstring = "^10[.][0-9]{4}/zenodo[.][0-9]{7}$" diff --git a/rse/main/scrapers/__init__.py b/rse/main/scrapers/__init__.py index ac142ed..5c830f7 100644 --- a/rse/main/scrapers/__init__.py +++ b/rse/main/scrapers/__init__.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/main/scrapers/ascl.py b/rse/main/scrapers/ascl.py index 4f3c4b5..b0562c2 100644 --- a/rse/main/scrapers/ascl.py +++ b/rse/main/scrapers/ascl.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2022 Vanessa Sochat. +Copyright (C) 2022-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -21,7 +21,6 @@ class AsclScraper(ScraperBase): - name = "ascl" matchstring = "ascl" @@ -64,7 +63,6 @@ def _parse_page(self, url): contenders = soup.find_all("div", {"class": "item"}) for contender in contenders: - # Get metadata from child for child in contender.children: if not isinstance(child, bs4.element.Tag): @@ -88,7 +86,6 @@ def _parse_page(self, url): result["description"] = child.text elif "title" in child.attrs.get("class", []) and child.contents: - # This has a link to the detail page title = child.find_next("a") if "href" not in title.attrs or not title.contents: diff --git a/rse/main/scrapers/base.py b/rse/main/scrapers/base.py index 274d8f6..94a231b 100644 --- a/rse/main/scrapers/base.py +++ b/rse/main/scrapers/base.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/main/scrapers/biogrids.py b/rse/main/scrapers/biogrids.py index 8ee5ed1..56d0fe5 100644 --- a/rse/main/scrapers/biogrids.py +++ b/rse/main/scrapers/biogrids.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2022 Vanessa Sochat. +Copyright (C) 2022-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -19,7 +19,6 @@ class BioGridsScraper(ScraperBase): - name = "biogrids" matchstring = "biogrids" @@ -42,7 +41,6 @@ def scrape(self, paginate=False, delay=None, query=""): "tr", {"itemtype": "http://schema.org/SoftwareApplication"} ) for contender in contenders: - result = {} # Find the title diff --git a/rse/main/scrapers/biotools.py b/rse/main/scrapers/biotools.py index 1fac991..5836560 100644 --- a/rse/main/scrapers/biotools.py +++ b/rse/main/scrapers/biotools.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -24,7 +24,6 @@ class BioToolsScraper(ScraperBase): - name = "biotools" matchstring = "(biotool|bio[.]tool)" @@ -54,7 +53,6 @@ def scrape(self, url, paginate=False, delay=None): # Handle pagination original_url = url while url is not None: - response = requests.get(url, headers={"User-Agent": get_user_agent()}) data = check_response(response) @@ -66,7 +64,6 @@ def scrape(self, url, paginate=False, delay=None): ) for entry in data.get("list", []): - # Look for GitHub / GitLab URL repo = {} for link in entry.get("link", []): diff --git a/rse/main/scrapers/csv.py b/rse/main/scrapers/csv.py index ee74f54..bd5f336 100644 --- a/rse/main/scrapers/csv.py +++ b/rse/main/scrapers/csv.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -36,7 +36,6 @@ def read_rows(filepath, newline=None, delim=None): class CSVImporter(ScraperBase): - name = "csv" def scrape(self, args, **kwargs): diff --git a/rse/main/scrapers/debian.py b/rse/main/scrapers/debian.py index b0e6daa..6f44127 100644 --- a/rse/main/scrapers/debian.py +++ b/rse/main/scrapers/debian.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -20,7 +20,6 @@ class DebianMedScraper(ScraperBase): - name = "debian-med" matchstring = "(debian|debianmed|debian-med)" @@ -42,7 +41,6 @@ def scrape(self, paginate=False, delay=None, query=""): contenders = soup.find_all("a", href=True) for contender in contenders: - # Each link to a subpage has a name, id, and href that are the same name = contender.attrs.get("name") identifier = contender.attrs.get("id") diff --git a/rse/main/scrapers/googlesheet.py b/rse/main/scrapers/googlesheet.py index 883cfc8..cc21d7b 100644 --- a/rse/main/scrapers/googlesheet.py +++ b/rse/main/scrapers/googlesheet.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2022 Vanessa Sochat. +Copyright (C) 2022-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -23,7 +23,6 @@ class GoogleSheetImporter(CSVImporter): - name = "googlesheet" def scrape(self, args, **kwargs): diff --git a/rse/main/scrapers/hal.py b/rse/main/scrapers/hal.py index 47a3a09..a1d7eba 100644 --- a/rse/main/scrapers/hal.py +++ b/rse/main/scrapers/hal.py @@ -26,7 +26,6 @@ class HalScraper(ScraperBase): - name = "hal" matchstring = "hal" diff --git a/rse/main/scrapers/imperial.py b/rse/main/scrapers/imperial.py index c7bc453..9209f56 100644 --- a/rse/main/scrapers/imperial.py +++ b/rse/main/scrapers/imperial.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -29,7 +29,6 @@ class ImperialCollegeLondonScraper(ScraperBase): - name = "imperial" matchstring = "imperial" diff --git a/rse/main/scrapers/joss.py b/rse/main/scrapers/joss.py index f780d56..7ac98df 100644 --- a/rse/main/scrapers/joss.py +++ b/rse/main/scrapers/joss.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -21,7 +21,6 @@ class JossScraper(ScraperBase): - name = "joss" matchstring = "(joss|journal of open source software)" @@ -57,7 +56,6 @@ def scrape(self, url, paginate=False, delay=None): soup = self.soupify(url) url = None for link in soup.find_all("link", href=True): - # Sleep for a random amount of time to give a rest! sleep(delay or random.choice(range(1, 10)) * 0.1) paper_url = link.attrs.get("href", "") diff --git a/rse/main/scrapers/molssi.py b/rse/main/scrapers/molssi.py index 55287d8..4fd8448 100644 --- a/rse/main/scrapers/molssi.py +++ b/rse/main/scrapers/molssi.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2022 Vanessa Sochat. +Copyright (C) 2022-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -24,7 +24,6 @@ class MolssiScraper(ScraperBase): - name = "molssi" matchstring = "molssi" @@ -113,7 +112,6 @@ def scrape(self, paginate=False, delay=None, query=""): if "row" in bolded.parent.parent.attrs.get("class"): nexts = list(bolded.parent.parent.parent.next_elements) for next_element in nexts: - # The citation is the first link under this bolded section if isinstance( next_element, bs4.element.Tag diff --git a/rse/main/scrapers/ropensci.py b/rse/main/scrapers/ropensci.py index b96f735..b6389d1 100644 --- a/rse/main/scrapers/ropensci.py +++ b/rse/main/scrapers/ropensci.py @@ -20,7 +20,6 @@ class ROpenSciScraper(ScraperBase): - name = "ropensci" matchstring = "ropensci" @@ -80,7 +79,6 @@ def scrape(self, paginate=False, delay=None): repos = parser.get_org_repos("ropensci", paginate=paginate, delay=delay) for entry in repos: - # We determine belonging based on the github url if entry["html_url"] not in names: bot.info("Skipping repository: %s" % entry["html_url"]) @@ -106,7 +104,6 @@ def scrape(self, paginate=False, delay=None): # E.g., there are repos in other orgs that won't be found above if paginate and names: for name in names: - parser = GitHubParser(uid=name) # Topics will be added here! entry = parser.get_metadata() diff --git a/rse/main/scrapers/rsnl.py b/rse/main/scrapers/rsnl.py index 2d5e97a..0db83ed 100644 --- a/rse/main/scrapers/rsnl.py +++ b/rse/main/scrapers/rsnl.py @@ -20,9 +20,9 @@ bot = logging.getLogger("rse.main.scrapers.rsnl") + # Research Software Netherlands class RSNLScraper(ScraperBase): - name = "rsnl" matchstring = "(researchsoftwarenl|rsnl)" @@ -59,7 +59,6 @@ def scrape(self, url, paginate=False, delay=0.0): data = check_response(response) or [] for entry in data: - # I only see GitHub urls repo_url = entry.get("repositoryURLs", {}).get("github") repo_url = repo_url[0] if repo_url else None diff --git a/rse/main/taxonomy.py b/rse/main/taxonomy.py index a52e59f..0b07318 100644 --- a/rse/main/taxonomy.py +++ b/rse/main/taxonomy.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed diff --git a/rse/utils/command.py b/rse/utils/command.py index f7ebcb8..ebe0bae 100644 --- a/rse/utils/command.py +++ b/rse/utils/command.py @@ -94,7 +94,6 @@ class Command: """ def __init__(self, cmd=None): - cmd = cmd or [] self.returncode = None self.out = [] diff --git a/rse/version.py b/rse/version.py index cf87527..1747ab7 100644 --- a/rse/version.py +++ b/rse/version.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2020-2022 Vanessa Sochat. +Copyright (C) 2020-2023 Vanessa Sochat. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed @@ -9,7 +9,7 @@ """ -__version__ = "0.0.47" +__version__ = "0.0.48" AUTHOR = "Vanessa Sochat" AUTHOR_EMAIL = "vsoch@users.noreply.github.io" NAME = "rse"