Merge pull request #195 from lanl/schema_update

updated user api schema() to print out a db's create table statements

Author: jpulidojr

dsi/backends/duckdb.py

@@ -366,6 +366,16 @@ def get_table_names(self, query):
         tables = [table for from_tbl, join_tbl in all_names if (table := from_tbl or join_tbl)]
         return tables
 
+    def get_schema(self):
+        """
+        Returns the structural schema of this database in the form of CREATE TABLE statements.
+
+        `return`: str
+            Each table's CREATE TABLE statement is concatenated into one large string.
+        """
+        schema_stmts = self.query_artifacts(query="SELECT sql FROM duckdb_tables where sql NOT NULL ")
+        return schema_stmts["sql"].str.cat(sep="\n")
+    
     # OLD NAME OF notebook(). TO BE DEPRECATED IN FUTURE DSI RELEASE
     def inspect_artifacts(self, interactive=False):
         return self.notebook(interactive)

dsi/backends/sqlite.py

@@ -357,10 +357,20 @@ def get_table_names(self, query):
         tables = [table for from_tbl, join_tbl in all_names if (table := from_tbl or join_tbl)]
         return tables
 
+    def get_schema(self):
+        """
+        Returns the structural schema of this database in the form of CREATE TABLE statements.
+
+       `return`: str
+            Each table's CREATE TABLE statement is concatenated into one large string.
+        """
+        schema_stmts = self.query_artifacts(query="SELECT sql FROM sqlite_master where sql NOT NULL ORDER BY type, name")
+        return schema_stmts["sql"].str.cat(sep="\n")
+
     # OLD NAME OF notebook(). TO BE DEPRECATED IN FUTURE DSI RELEASE
     def inspect_artifacts(self, interactive=False):
         return self.notebook(interactive)
-    
+
     def notebook(self, interactive=False):
         """
         Generates a Jupyter notebook displaying all the data in the SQLite database.

dsi/core.py

@@ -621,6 +621,36 @@ def get_table(self, table_name, dict_return = False):
         if output is not None and isinstance(output, (pd.DataFrame, OrderedDict)):
             return output
 
+    def get_schema(self):
+        """
+        Returns the first loaded database's structural schema as several CREATE TABLE statements.
+
+        `return`: str
+            Each table's CREATE TABLE statement is concatenated into one large string.
+        """
+        if self.debug_level != 0:
+            self.logger.info("-------------------------------------")
+            self.logger.error(f'Getting the structural schema of the first loaded backend')
+        if len(self.loaded_backends) == 0:
+            if self.debug_level != 0:
+                self.logger.error('Need to load a valid backend to be able to get its structural schema')
+            raise NotImplementedError('Need to load a valid backend to be able to get its structural schema')
+        backend = self.loaded_backends[0]
+        parent_backend = backend.__class__.__bases__[0].__name__
+        if not self.valid_backend(backend, parent_backend):
+            if self.debug_level != 0:
+                self.logger.error("First loaded backend needs to have data to get its structural schema")
+            raise RuntimeError("First loaded backend needs to have data to get its structural schema")
+        start = datetime.now()
+
+        output = backend.get_schema()
+
+        end = datetime.now()
+        if self.debug_level != 0:
+            self.logger.info(f"Runtime: {end-start}")
+        
+        return output
+
     def find(self, query_object):
         """
         Find all instances of `query_object` across all tables, columns, and cells in the first loaded backend.

dsi/dsi.py

@@ -88,27 +88,33 @@ def list_backends(self):
         print("DuckDB : In-process SQL backend optimized for fast analytics on large datasets.\n")
         print()
 
-    def schema(self, filename):
+    def schema(self, filename = None):
         """
-       Loads a relational database schema into DSI from a specified `filename`
+        Either loads a relational database schema into DSI with a specified `filename` OR returns this database's structural schema.
 
-        `filename` : str
-            Path to a JSON file describing the structure of a relational database.
+        `filename` : str, optional
+            Path to a JSON file describing the relationships of the tables in a database.
             The schema should follow the format described in :ref:`user_schema_example_label`
-
-        **Must be called before reading in any data files associated with the schema**
+        
+        `return` : If filename = None, returns the structural schema of this database - table/col names and their units.
+        **If loading a relational schema, this function must be called before reading in any associated data files**
         """
-        if not os.path.exists(filename):
-            sys.exit("schema() ERROR: Input schema file must have a valid filepath. Please check again.")
+        if filename:
+            if not os.path.exists(filename):
+                sys.exit("schema() ERROR: Input schema file must have a valid filepath. Please check again.")
 
-        fnull = open(os.devnull, 'w')
-        with redirect_stdout(fnull):
-            self.t.load_module('plugin', 'Schema', 'reader', filename=filename)
-        self.schema_read = True
-        pk_tables = set(t[0] for t in self.t.active_metadata["dsi_relations"]["primary_key"])
-        fk_tables = set(t[0] for t in self.t.active_metadata["dsi_relations"]["foreign_key"] if t[0] != None)
-        self.schema_tables = pk_tables.union(fk_tables)
-        print(f"Successfully loaded the schema file: {filename}")
+            fnull = open(os.devnull, 'w')
+            with redirect_stdout(fnull):
+                self.t.load_module('plugin', 'Schema', 'reader', filename=filename)
+            self.schema_read = True
+            pk_tables = set(t[0] for t in self.t.active_metadata["dsi_relations"]["primary_key"])
+            fk_tables = set(t[0] for t in self.t.active_metadata["dsi_relations"]["foreign_key"] if t[0] != None)
+            self.schema_tables = pk_tables.union(fk_tables)
+            print(f"Successfully loaded the schema file: {filename}")
+        else:
+            fnull = open(os.devnull, 'w')
+            with redirect_stdout(fnull):
+                return self.t.get_schema()
 
     def list_readers(self):
         """

dsi/plugins/file_reader.py

@@ -769,9 +769,6 @@ def add_rows(self) -> None:
                 return (ValueError, f"Error in reading {filename} data card. Please ensure all fields included match the template")
 
         self.datacard_data["oceans11_datacard"] = temp_data
-        
-        self.datacard_data["oceans11_datacard"]["remote"] = [""] * len(self.datacard_files)
-        self.datacard_data["oceans11_datacard"]["local"] = [""] * len(self.datacard_files)
         self.set_schema_2(self.datacard_data)
 
 class DublinCoreDatacard(FileReader):

dsi/plugins/file_writer.py

@@ -102,7 +102,7 @@ def get_rows(self, collection) -> None:
                 html_table = f"{tableName} [label="
             html_table += f"<<TABLE CELLSPACING=\"0\"><TR><TD COLSPAN=\"{num_tbl_cols}\"><B>{tableName}</B></TD></TR>"
 
-            col_list = tableData.keys()
+            col_list = list(tableData.keys())
             if tableName == "dsi_units":
                 col_list = ["table_name", "column_name", "unit"]
             if self.max_cols is not None: