Fix vocabulary term initialization (#412)

* Remove file from vocabulary file list that doesn't appear to exist

* Fix for #411

* Use the right method name (thanks mypy)

Author: dannymandel

isb_lib/core.py

@@ -11,6 +11,8 @@
 from signal import SIGINT
 
 import igsn_lib.time
+import term_store
+from sqlmodel import Session
 
 from isamples_metadata.metadata_constants import METADATA_SAMPLE_IDENTIFIER, METADATA_AT_ID, METADATA_LABEL, \
     METADATA_HAS_CONTEXT_CATEGORY, \
@@ -27,6 +29,7 @@
 from isamples_metadata.solr_field_constants import SOLR_PRODUCED_BY_SAMPLING_SITE_ELEVATION_IN_METERS, \
     SOLR_CURATION_LABEL, SOLR_CURATION_DESCRIPTION, SOLR_CURATION_ACCESS_CONSTRAINTS, SOLR_CURATION_LOCATION, \
     SOLR_CURATION_RESPONSIBILITY
+from isamples_metadata.vocabularies import vocabulary_mapper
 from isb_lib.models.thing import Thing
 from isamples_metadata.Transformer import Transformer, geo_to_h3
 import dateparser
@@ -36,10 +39,13 @@
 import shapely.wkt
 import shapely.geometry
 
+from isb_lib.vocabulary import vocab_adapter
 from isb_web import sqlmodel_database
 from isb_web.sqlmodel_database import SQLModelDAO
 from typing import Optional
 
+from isb_web.vocabulary import SAMPLEDFEATURE_URI, MATERIAL_URI, MATERIALSAMPLEOBJECTTYPE_URI
+
 RECOGNIZED_DATE_FORMATS = [
     "%Y",  # e.g. 1985
     "%Y-%m-%d",  # e.g. 1947-08-06
@@ -103,6 +109,16 @@ def things_main(ctx, db_url, solr_url, verbosity="INFO"):
     ctx.obj["solr_url"] = solr_url
 
 
+def initialize_vocabularies(session: Session):
+    repository = term_store.get_repository(session)
+    vocab_adapter.uijson_vocabulary_dict(SAMPLEDFEATURE_URI, repository)
+    vocab_adapter.uijson_vocabulary_dict(MATERIAL_URI, repository)
+    vocab_adapter.uijson_vocabulary_dict(MATERIALSAMPLEOBJECTTYPE_URI, repository)
+    vocabulary_mapper.sampled_feature_type()
+    vocabulary_mapper.material_type()
+    vocabulary_mapper.specimen_type()
+
+
 def datetimeToSolrStr(dt):
     if dt is None:
         return None

isb_web/main.py

@@ -21,13 +21,11 @@
 
 import isb_web
 import isamples_metadata.GEOMETransformer
-from isamples_metadata.vocabularies import vocabulary_mapper
-from isb_lib.core import MEDIA_GEO_JSON, MEDIA_JSON, MEDIA_NQUADS, SOLR_TIME_FORMAT
+from isb_lib.core import MEDIA_GEO_JSON, MEDIA_JSON, MEDIA_NQUADS, SOLR_TIME_FORMAT, initialize_vocabularies
 from isb_lib.localcontexts.localcontexts_client import local_contexts_info_for_resolved_content
 from isb_lib.models.thing import Thing
 from isb_lib.utilities import h3_utilities
 from isb_lib.utilities.url_utilities import full_url_from_suffix
-from isb_lib.vocabulary import vocab_adapter
 from isb_web import sqlmodel_database, analytics, manage, debug, metrics, vocabulary, export, auth
 from isb_web.analytics import AnalyticsEvent
 from isb_web import schemas
@@ -46,7 +44,6 @@
 from isb_web.schemas import ThingPage
 from isb_web.sqlmodel_database import SQLModelDAO, taxonomy_name_to_kingdom_map
 import isb_lib.stac
-from isb_web.vocabulary import SAMPLEDFEATURE_URI, MATERIAL_URI, MATERIALSAMPLEOBJECTTYPE_URI
 
 THIS_PATH = os.path.dirname(os.path.abspath(__file__))
 
@@ -148,14 +145,7 @@ def on_startup():
     dao.connect_sqlmodel(isb_web.config.Settings().database_url)
     session = dao.get_session()
     orcid_ids = sqlmodel_database.all_orcid_ids(session)
-    # preload each of these into memory to avoid performance issues on hyde
-    repository = term_store.get_repository(session)
-    vocab_adapter.uijson_vocabulary_dict(SAMPLEDFEATURE_URI, repository)
-    vocab_adapter.uijson_vocabulary_dict(MATERIAL_URI, repository)
-    vocab_adapter.uijson_vocabulary_dict(MATERIALSAMPLEOBJECTTYPE_URI, repository)
-    vocabulary_mapper.sampled_feature_type()
-    vocabulary_mapper.material_type()
-    vocabulary_mapper.specimen_type()
+    initialize_vocabularies(session)
     # Force this into memory so it's cached when we need it later
     global TAXONOMY_NAME_TO_KINGDOM_MAP
     if config.Settings().taxon_cache_enabled:

scripts/geome_things.py

@@ -350,6 +350,9 @@ def populateIsbCoreSolr(ctx, ignore_last_modified: bool):
         solr_url=solr_url,
         min_time_created=max_solr_updated_date
     )
+    dao = SQLModelDAO(db_url)
+    session = dao.get_session()
+    isb_lib.core.initialize_vocabularies(session)
     allkeys = solr_importer.run_solr_import(isb_lib.geome_adapter.reparseAsCoreRecord)
     logger.info(f"Total keys= {len(allkeys)}")
 

scripts/opencontext_things.py

@@ -11,7 +11,7 @@
 
 from isamples_metadata import OpenContextTransformer
 from isb_lib import opencontext_adapter
-from isb_lib.core import MEDIA_JSON
+from isb_lib.core import MEDIA_JSON, initialize_vocabularies
 from isb_lib.opencontext_adapter import OPENCONTEXT_PAGE_SIZE
 from isb_web import sqlmodel_database
 from isb_web.sqlmodel_database import SQLModelDAO, save_thing, DatabaseBulkUpdater
@@ -163,6 +163,8 @@ def populate_isb_core_solr(ctx, ignore_last_modified: bool):
             url=solr_url,
             authority_id=isb_lib.opencontext_adapter.OpenContextItem.AUTHORITY_ID,
         )
+    session = SQLModelDAO(db_url).get_session()
+    initialize_vocabularies(session)
     L.info(f"Going to index Things with tcreated > {max_solr_updated_date}")
     solr_importer = isb_lib.core.CoreSolrImporter(
         db_url=db_url,

scripts/smithsonian_things.py

@@ -128,6 +128,8 @@ def populate_isb_core_solr(ctx):
     logger = isb_lib.core.getLogger()
     db_url = ctx.obj["db_url"]
     solr_url = ctx.obj["solr_url"]
+    session = SQLModelDAO(db_url).get_session()
+    isb_lib.initialize_vocabularies(session)
     solr_importer = isb_lib.core.CoreSolrImporter(
         db_url=db_url,
         authority_id=isb_lib.smithsonian_adapter.SmithsonianItem.AUTHORITY_ID,