Fix stopwords error and fix upload to google storage (#1786)

* fix file path so google cloud storage works

* fix: hard code stop words in

* fix how google storage secrets are accessed

Author: EPortman

api/namex/analytics/solr.py

@@ -900,15 +900,11 @@ def remove_stopwords_designations(cls, name):
             'llp', 'ltd.', 'ltd', 'ltee', 'sencrl', 'societe a responsabilite limitee',
             'societe en nom collectif a responsabilite limitee', 'limited', 'srl', 'ulc', 'unlimited liability company']
 
-        stop_words = []
-        try:
-            with open('stopwords.txt') as stop_words_file:
-                stop_words = []
-                for line in stop_words_file.readlines():
-                    if line.find('#') == -1:
-                        stop_words.append(line.strip('\n').strip())
-        except Exception as err:
-            current_app.logger.error(err)
+        # TODO: these should be loaded from somewhere.
+        stop_words = [
+            'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'corp', 'if', 'in', 'incorporation', 'into', 'is', 'it',
+            'no', 'not', 'of', 'on', 'or', 'such', 'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to' 
+        ]
 
         # remove designations if they are at the end of the name
         for designation in designations:

jobs/nr-duplicates-report/nr_duplicates_report/config.py

@@ -1,5 +1,6 @@
 import os
 from dotenv import load_dotenv, find_dotenv
+import base64
 
 # this will load all the envars from a .env file located in the project root (api)
 load_dotenv(find_dotenv())
@@ -33,6 +34,23 @@ class Config(object):
     OCP_RELAY_URL = os.getenv('OCP_RELAY_URL', '')
 
     # Email
-    NOTIFY_API_URL = f"{os.getenv("NOTIFY_API_URL", "") + os.getenv("NOTIFY_API_VERSION", "")}/notify"
+    NOTIFY_API_URL = f"{os.getenv('NOTIFY_API_URL', '') + os.getenv('NOTIFY_API_VERSION', '')}/notify"
     REPORT_RECIPIENTS = os.getenv('REPORT_RECIPIENTS', '')
     ERROR_EMAIL_RECIPIENTS = os.getenv('ERROR_EMAIL_RECIPIENTS', '')
+
+    # Google Storage
+    TYPE = os.getenv('TYPE', '')
+    PROJECT_ID = os.getenv('PROJECT_ID', '')
+    CLIENT_ID = os.getenv('CLIENT_ID', '')
+    CLIENT_EMAIL = os.getenv('CLIENT_EMAIL', '')
+    AUTH_URI = os.getenv('AUTH_URI', '')
+    TOKEN_URI = os.getenv('TOKEN_URI', '')
+    AUTH_PROVIDER_X509_cert_URL = os.getenv('AUTH_PROVIDER_X509_cert_URL', '')
+    CLIENT_X509_CERT_URL = os.getenv('CLIENT_X509_CERT_URL', '')
+    PRIVATE_KEY_ID = os.getenv('PRIVATE_KEY_ID', '')
+    try:
+        pk_raw = os.getenv('PRIVATE_KEY', '').strip()
+        pk_padded = pk_raw + '=' * (-len(pk_raw) % 4)
+        PRIVATE_KEY = base64.b64decode(pk_padded).decode()
+    except Exception:
+        PRIVATE_KEY = os.getenv('PRIVATE_KEY', '')

jobs/nr-duplicates-report/nr_duplicates_report/daily/nr-duplicates.ipynb

@@ -273,16 +273,16 @@
     "    if not result_frame.empty:\n",
     "        # Config data dictionary\n",
     "        dictionary = {   \n",
-    "            \"type\": os.getenv('TYPE', ''),\n",
-    "            \"project_id\": os.getenv('PROJECT_ID', ''),\n",
-    "            \"private_key_id\": os.getenv('PRIVATE_KEY_ID', ''),\n",
-    "            \"private_key\": os.getenv('PRIVATE_KEY', ''),\n",
-    "            \"client_email\": os.getenv('CLIENT_EMAIL', ''),\n",
-    "            \"client_id\": os.getenv('CLIENT_ID', ''),\n",
-    "            \"auth_uri\": os.getenv('AUTH_URI', ''),\n",
-    "            \"token_uri\": os.getenv('TOKEN_URI', ''),\n",
-    "            \"auth_provider_x509_cert_url\": os.getenv('AUTH_PROVIDER_X509_cert_URL', ''),\n",
-    "            \"client_x509_cert_url\": os.getenv('CLIENT_X509_CERT_URL', '')\n",
+    "            \"type\": Config.TYPE,\n",
+    "            \"project_id\": Config.PROJECT_ID,\n",
+    "            \"private_key_id\": Config.PRIVATE_KEY_ID,\n",
+    "            \"private_key\": Config.PRIVATE_KEY,\n",
+    "            \"client_email\": Config.CLIENT_EMAIL,\n",
+    "            \"client_id\": Config.CLIENT_ID,\n",
+    "            \"auth_uri\": Config.AUTH_URI,\n",
+    "            \"token_uri\": Config.TOKEN_URI,\n",
+    "            \"auth_provider_x509_cert_url\": Config.AUTH_PROVIDER_X509_cert_URL,\n",
+    "            \"client_x509_cert_url\": Config.CLIENT_X509_CERT_URL\n",
     "        }\n",
     "\n",
     "        file_path = os.path.join(os.getcwd(), \"nr_duplicates_report/data/\") + \"service_key.json\"\n",