adding search for taxonomy/criteria for filesystem database (#52)

* adding search for taxonomy/criteria for filesystem database
* dont pin black
* fixing black

Signed-off-by: vsoch <[email protected]>

.github/workflows/main.yml

@@ -21,7 +21,7 @@ jobs:
         run: |
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate black
-          pip install black==19.10b0
+          pip install black==20.8b1
           black --check rse
 
       - name: Check imports with pyflakes

docs/_docs/getting-started/commands.md

@@ -358,6 +358,64 @@ INFO:rse.main:Database: filesystem
 1  github/singularityhub/sregistry
 ```
 
+For a filesystem database, you can also search across taxonomy and/or criteria items:
+
+```bash
+$ rse search --taxonomy package
+RSE-taxonomy-package-management
+1  github/easybuilders/easybuild
+2  github/spack/spack
+
+$ rse search --criteria research
+RSE-research-intention
+1  github/AA-ALERT/AstroData
+2  github/fair-software/howfairis
+3  github/BrianAronson/birankr
+4  github/3D-e-Chem/knime-sstea
+5  github/davidebolo1993/TRiCoLOR
+6  github/AA-ALERT/AMBER
+7  gitlab/davidtourigny/dynamic-fba
+8  github/Sulstice/cocktail-shaker
+9  github/spack/spack
+10 github/snakemake/snakemake
+11 github/potree/PotreeConverter
+12 github/Effective-Quadratures/Effective-Quadratures
+13 github/3D-e-Chem/knime-pharmacophore
+14 github/sunpy/sunpy
+15 github/AA-ALERT/frbcatdb
+16 github/AA-ALERT/frbcat-web
+17 github/Parsl/parsl
+18 github/JuliaOpt/JuMP.jl
+19 github/AA-ALERT/Dedispersion
+20 github/scikit-image/scikit-image
+21 github/3D-e-Chem/sygma
+22 github/nextflow-io/nextflow
+23 gitlab/LouisLab/PiVR
+24 github/3D-e-Chem/knime-gpcrdb
+25 gitlab/cosmograil/PyCS3
+26 github/sjvrijn/mf2
+27 github/KVSlab/turtleFSI
+28 github/ropensci/chirps
+29 gitlab/ampere2/metalwalls
+```
+
+The searches are independent, meaning that you might see the same repository in two
+results listings if it has more than one match for a given taxonomy or criteria item.
+The same is true for adding a search term at the onset:
+
+```python
+$ rse search singularity --taxonomy package
+singularity
+1  github/hpcng/singularity
+2  github/singularityhub/singularity-compose
+3  github/singularityhub/sregistry
+4  github/eWaterCycle/setup-singularity
+
+RSE-taxonomy-package-management
+1  github/spack/spack
+2  github/easybuilders/easybuild
+```
+
 <a id="summary">
 ## Summary
 
@@ -695,7 +753,7 @@ web-scraping
 webscraping
 ```
 
-Finally, you can provide one or more topics, and find repositories that are labeled 
+Finally, you can search for one or more topics, and find repositories that are labeled 
 as such:
 
 ```bash

rse/app/views/repositories.py

@@ -78,7 +78,12 @@ def annotate_criteria():
         if last is not None and last.uid != repo.uid:
             break
         last = repo
-        annotation_sets.append((repo, criteria,))
+        annotation_sets.append(
+            (
+                repo,
+                criteria,
+            )
+        )
 
     return render_template(
         "annotate/criteria.html",

rse/client/__init__.py

@@ -16,6 +16,7 @@
 import argparse
 import sys
 import logging
+import os
 
 
 def get_parser():
@@ -49,7 +50,10 @@ def get_parser():
 
     description = "actions for rse"
     subparsers = parser.add_subparsers(
-        help="rse actions", title="actions", description=description, dest="command",
+        help="rse actions",
+        title="actions",
+        description=description,
+        dest="command",
     )
 
     # print version and exit
@@ -106,7 +110,10 @@ def get_parser():
         "init", help="Add an rse.ini to the present working directory."
     )
     init.add_argument(
-        "path", help="Path to generate rse.ini file", nargs="?", default=".",
+        "path",
+        help="Path to generate rse.ini file",
+        nargs="?",
+        default=".",
     )
 
     # Config
@@ -173,7 +180,9 @@ def get_parser():
         "analyze", help="View metrics for a specific repository."
     )
     analyze.add_argument(
-        "repo", help="Software repository to show", default=None,
+        "repo",
+        help="Software repository to show",
+        default=None,
     )
     analyze.add_argument(
         "--ct",
@@ -244,13 +253,17 @@ def get_parser():
 
     # Search for software
     search = subparsers.add_parser(
-        "search", help="Search for a piece of research software",
+        "search",
+        help="Search for a piece of research software",
     )
     search.add_argument("query", nargs="*")
+    search.add_argument("--taxonomy", nargs="*")
+    search.add_argument("--criteria", nargs="*")
 
     # Scrape for new repos
     scrape = subparsers.add_parser(
-        "scrape", help="Add new software repositories from a resource.",
+        "scrape",
+        help="Add new software repositories from a resource.",
     )
     scrape.add_argument("scraper_name", nargs=1)
     scrape.add_argument("query", nargs="?")
@@ -423,6 +436,8 @@ def help(return_code=0):
     args, extra = parser.parse_known_args()
 
     # Set the logging level
+    os.putenv("RSE_LOG_LEVEL", args.log_level)
+    RSE_LOG_LEVEL = args.log_level
     logging.basicConfig(level=getattr(logging, args.log_level))
     bot = logging.getLogger("rse.client")
     bot.setLevel(getattr(logging, args.log_level))

rse/client/search.py

@@ -17,8 +17,15 @@ def main(args, extra):
 
     enc = Encyclopedia(config_file=args.config_file, database=args.database)
     query = " ".join(args.query).strip()
-    if not query:
+
+    # We can search taxonomy, criteria, or both
+    taxonomy = args.taxonomy or []
+    criteria = args.criteria or []
+    if not query and not taxonomy and not criteria:
         sys.exit("Please provide a query to search for.")
-    results = enc.search(query)
+    results = enc.search(query, taxonomy=taxonomy, criteria=criteria)
     if results:
-        bot.table(results)
+        for key, listing in results.items():
+            bot.info(key)
+            bot.table(listing)
+            bot.newline()

rse/logger/message.py

@@ -295,13 +295,14 @@ def is_quiet(self):
 
     # Terminal ------------------------------------------
 
-    def table(self, rows, col_width=2):
+    def table(self, rows, col_width=2, labels=None):
         """table will print a table of entries. If the rows is
         a dictionary, the keys are interpreted as column names. if
         not, a numbered list is used.
         """
+        if not labels:
+            labels = [str(x) for x in range(1, len(rows) + 1)]
 
-        labels = [str(x) for x in range(1, len(rows) + 1)]
         if isinstance(rows, dict):
             labels = list(rows.keys())
             rows = list(rows.values())

rse/main/__init__.py

@@ -62,7 +62,7 @@ def initdb(self, database):
             or "filesystem"
         )
         database_string = self.config.get("DEFAULT", "databaseconnect")
-        bot.info("Database: %s" % self.database)
+        bot.debug("Database: %s" % self.database)
 
         # Supported database options
         valid = ("sqlite", "postgresql", "mysql+pymysql", "filesystem")
@@ -196,12 +196,12 @@ def label(self, uid, key, value, force=False):
         except RepoNotFoundError:
             bot.error(f"{uid} does not exist.")
 
-    def search(self, query):
+    def search(self, query, taxonomy=None, criteria=None):
         """Search across commands and general metadata for a string of interest.
         We use regular expressions (re.search) so they are supported.
         Search is only available for non-filesystem databases.
         """
-        results = self.db.search(query)
+        results = self.db.search(query, taxonomy=taxonomy, criteria=criteria)
         if results:
             return results
         bot.info(f"No results matching {query}")