Merge pull request #204 from lanl/guuid_bugfix

Guuid bugfix

Author: jpulidojr

dsi/core.py

@@ -1440,7 +1440,7 @@ def index(self, local_loc, remote_loc, isVerbose=False):
         st_dict['n_links'] = []
         st_dict['uid'] = []
         st_dict['gid'] = []
-        #st_dict['uuid'] = []
+        st_dict['uuid'] = []
         st_dict['file_remote'] = []
 
         for file in file_list:
@@ -1462,7 +1462,7 @@ def index(self, local_loc, remote_loc, isVerbose=False):
             st_dict['n_links'].append(st.st_nlink)
             st_dict['uid'].append(st.st_uid)
             st_dict['gid'].append(st.st_gid)
-            #st_dict['uuid'].append(self.gen_uuid(st))
+            st_dict['uuid'].append(self.gen_uuid(st))
             st_dict['file_remote'].append(rfilepath)
             st_list.append(st)
 
@@ -1733,7 +1733,7 @@ def gen_uuid(self, st):
 
         file_uuid = uuid.uuid5(uuid.NAMESPACE_URL, unique_str)
         #print(f"UUID:{file_uuid}")
-        return file_uuid
+        return str(file_uuid)
 
 
 

examples/clover3d/genesis_datacard.xlsx

@@ -0,0 +1,26 @@
+{
+   "genesis_datacard": {
+       "primary_key": "Title"
+   },
+    "simulation": {
+       "primary_key": "sim_id",
+       "foreign_key": {
+             "dataset": ["genesis_datacard", "Title"]
+       }
+    }, 
+    "input": {
+       "foreign_key": {
+             "sim_id": ["simulation", "sim_id"]
+       }
+    }, 
+    "output": {
+       "foreign_key": {
+             "sim_id": ["simulation", "sim_id"]
+       }
+    },
+    "viz_files": {
+       "foreign_key": {
+             "sim_id": ["simulation", "sim_id"]
+       }
+    }
+}

examples/clover3d/schema2.json

@@ -10,11 +10,11 @@
     "The goal of the Data Science Infrastructure Project ([DSI](https://github.com/lanl/dsi)) is to provide a flexible, AI-ready metadata query capability which returns data subject to strict, POSIX-enforced file security. In this tutorial, you will learn how to:\n",
     " - initialize a DSI instance\n",
     " - load Tier 1 metadata into DSI\n",
-    " - check the data loaded\n",
-    " - query the data\n",
+    " - check the metadata loaded\n",
+    " - query the metadata\n",
     " - load Tier 2 and Tier 3 metadata into DSI\n",
     " - apply a complex schema\n",
-    " - use DSI Sync to index and move data\n",
+    " - use DSI Sync to index and move data and metadata\n",
     "\n",
     "This tutorial uses data from the [Cloverleaf3D](https://github.com/UK-MAC/CloverLeaf3D) Lagrangian-Eulerian hydrodynamics solver. Data is provided in dsi/examples/clover3d/. Prior to running the tutorial, follow the instructions in the [Quick Start: Installation](https://lanl.github.io/dsi/installation.html) to set up DSI.\n",
     "\n"
@@ -113,7 +113,7 @@
    "id": "51584519-2e77-4afa-983a-f69945a084c3",
    "metadata": {},
    "source": [
-    "# Exploring the loaded data"
+    "# Exploring the loaded metadata"
    ]
   },
   {
@@ -165,7 +165,7 @@
    "id": "b59cddcd-5a83-4fa3-8f4e-fd5d98381aa6",
    "metadata": {},
    "source": [
-    "# DSI Find to search within the data\n",
+    "# DSI Find to search within the metadata\n",
     "\n",
     "DSI's find capability lets you explore your data by performing queries with the following modifiers, such as >, <, >=, <=, =, ==, ~ (contains), ~~ (contains), !=, and (X, Y) for a range between values X and Y. Additionally, by adding a \"True\" input will return you a collection."
    ]
@@ -250,7 +250,7 @@
    "source": [
     "# Query DSI\n",
     "\n",
-    "DSI Supports direct SQL queries to the data that you have ingested"
+    "DSI Supports direct SQL queries to the metadata that you have ingested"
    ]
   },
   {
@@ -303,8 +303,14 @@
    "metadata": {},
    "source": [
     "{\n",
+    "   \"genesis_datacard\": {\n",
+    "       \"primary_key\": \"Title\"\n",
+    "   },\n",
     "    \"simulation\": {\n",
-    "       \"primary_key\": \"sim_id\"\n",
+    "       \"primary_key\": \"sim_id\",\n",
+    "       \"foreign_key\": {\n",
+    "             \"dataset\": [\"genesis_datacard\", \"Title\"]\n",
+    "       }\n",
     "    }, \n",
     "    \"input\": {\n",
     "       \"foreign_key\": {\n",
@@ -365,9 +371,11 @@
    ]
   },
   {
-   "cell_type": "raw",
-   "id": "e145ba60-4f5b-4cb3-99e0-239f56794b56",
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18d62564-30f0-45af-b664-30f1862b32d9",
    "metadata": {},
+   "outputs": [],
    "source": [
     "store.write(\"clover_er_diagram_no_schema.png\", \"ER_Diagram\")\n",
     "\n",
@@ -386,15 +394,15 @@
     "schema_store = DSI(\"diana_schema_tutorial.db\")\n",
     "\n",
     "# dsi.schema(filename)\n",
-    "schema_store.schema(\"./clover3d/schema.json\") # Schema neeeds to be defined before reading Cloverleaf data\n",
-    "\n",
-    "# Read in Datacard (Tier 1)\n",
-    "store.read(\"clover3d/genesis_datacard.xlsx\", 'GenesisDatacard')\n",
+    "schema_store.schema(\"./clover3d/schema2.json\") # Schema neeeds to be defined before reading Cloverleaf data\n",
     "\n",
     "# Read in Tier 2 and Tier 3 metadata\n",
     "# dsi.read(path, reader)\n",
     "schema_store.read(\"./clover3d/\", 'Cloverleaf')\n",
     "\n",
+    "# Read in Datacard (Tier 1)\n",
+    "schema_store.read(\"clover3d/genesis_datacard.xlsx\", 'GenesisDatacard')\n",
+    "\n",
     "# dsi.write(filename, writer)\n",
     "schema_store.write(\"clover_er_diagram.png\", \"ER_Diagram\")"
    ]
@@ -425,7 +433,7 @@
    "source": [
     "# DSI Write - CSV\n",
     "\n",
-    "DSI Support the output (write) of data if you would like to export into another project. For example, here we want to export the table \"input\" into a csv file."
+    "DSI Support the output (write) of metadata if you would like to export into another project. For example, here we want to export the table \"input\" into a csv file."
    ]
   },
   {
@@ -490,7 +498,7 @@
    "id": "8140e5fb-8932-434b-852b-a9944259024f",
    "metadata": {},
    "source": [
-    "# Moving your data with DSI"
+    "# Moving your data and metadata with DSI"
    ]
   },
   {