Ts/dataset extract update (#1481)

Co-authored-by: Yohann Paris <[email protected]> Co-authored-by: mwdchang <[email protected]>
DARPA-ASKEM · Jul 24, 2023 · 2cee15a · 2cee15a
1 parent e422cfc
commit 2cee15a
Show file tree

Hide file tree

Showing 7 changed files with 82 additions and 84 deletions.
diff --git a/packages/client/hmi-client/src/components/dataset/tera-dataset.vue b/packages/client/hmi-client/src/components/dataset/tera-dataset.vue
@@ -85,6 +85,7 @@
  :dialog-flavour="'dataset'"
  :publications="props.project?.assets?.publications"
  :project="project"
+ :assetId="assetId"
  />
  <Accordion :multiple="true" :activeIndex="[0, 1, 2]">
  <AccordionTab>
@@ -93,25 +94,7 @@
  </template>
  <section v-if="enriched">
  <div class="dataset-detail">
- <div class="full-width">
- <div class="detail-list">
- <h2 class="title">Dataset Details</h2>
- <ul>
- <li><strong>Dataset name:</strong> {{ dataset.name }}</li>
- <li><strong>Dataset overview:</strong> {{ dataset.description }}</li>
- <li>
- <strong>Dataset URL:</strong>
- <a :href="dataset.source">{{ dataset.source }}</a>
- </li>
- <li>
- <strong>Data size:</strong> This dataset currently contains
- {{ csvContent?.length || '-' }} rows.
- </li>
- </ul>
- </div>
- </div>
  <div class="column">
- <h3 class="title">{{ headers.DESCRIPTION }}</h3>
  <p class="content">{{ enrichedData.DESCRIPTION }}</p>
 
  <h3 class="subtitle">{{ headers.AUTHOR_NAME }}</h3>
@@ -140,27 +123,6 @@
  <h3 class="subtitle">{{ headers.LICENSE }}</h3>
  <p class="content">{{ enrichedData.LICENSE }}</p>
  </div>
- <div class="full-width">
- <h3 class="subtitle">{{ headers.EXAMPLES }}</h3>
- <ul class="example-list" v-if="enrichedData.EXAMPLES">
- <li
- class="example-item"
- v-for="(value, key) in JSON.parse(enrichedData.EXAMPLES)"
- :key="key"
- >
- <strong>{{ key }}:</strong>{{ value }}
- </li>
- </ul>
- </div>
- <div class="full-width">
- <h3 class="subtitle">{{ `Extraction Table` }}</h3>
- <DataTable :value="pd">
- <Column field="col_name" header="Column Name"></Column>
- <Column field="concept" header="Concept"></Column>
- <Column field="unit" header="Unit"></Column>
- <Column field="description" header="Description"></Column>
- </DataTable>
- </div>
  </div>
  </section>
  <p v-else>
@@ -173,9 +135,9 @@
  <header id="Source">Source</header>
  </template>
  This data is sourced from
- {{ dataset.metadata.documents ? dataset.metadata.documents[0].title : 'unknown' }}:
- <a :href="dataset.metadata.documents ? dataset.metadata.documents[0].url : ''">{{
- dataset.metadata.documents ? dataset.metadata.documents[0].url : ''
+ {{ dataset.metadata?.documents ? dataset.metadata.documents[0].title : 'unknown' }}:
+ <a :href="dataset.metadata?.documents ? dataset.metadata?.documents[0].url : ''">{{
+ dataset.metadata?.documents ? dataset.metadata?.documents[0].url : ''
  }}</a>
  </AccordionTab>
  <AccordionTab>
@@ -241,20 +203,21 @@
  <InputText
  class="p-inputtext-sm"
  type="text"
- v-if="rowEditList[index]"
- @focus="setSuggestedValue(index, '')"
+ v-if="rowEditList[index] && column.metadata"
+ v-model="column.metadata.unit"
+ @focus="setSuggestedValue(index, dataset.columns?.[index].metadata?.unit)"
  />
- <div class="variables-value" v-else>-</div>
- <!-- grounding -->
+ <div class="variables-value" v-else>{{ column.metadata?.unit ?? '--' }}</div>
+ <!-- Concept -->
  <InputText
  class="p-inputtext-sm"
  type="text"
- v-if="rowEditList[index]"
- v-model="groundingValues[index][0]"
- @focus="setSuggestedValue(index, originalGroundingValues[0])"
+ v-if="rowEditList[index] && column.metadata"
+ v-model="column.metadata.concept"
+ @focus="setSuggestedValue(index, column.metadata?.concept)"
  />
  <div class="variables-value" v-else>
- {{ groundingValues[index][0] }}
+ {{ column.metadata?.concept ?? '--' }}
  </div>
  <!-- extractions - field does not exist in tds yet -->
  <InputText
@@ -263,7 +226,7 @@
  v-if="rowEditList[index]"
  @focus="setSuggestedValue(index, '')"
  />
- <div class="variables-value" v-else></div>
+ <div class="variables-value" v-else>{{ column.metadata?.extraction ?? '--' }}</div>
  <div v-if="rowEditList[index]" class="row-edit-buttons">
  <Button
  text
@@ -293,7 +256,7 @@
  <span>Suggested value</span>
  <div>
  <div class="suggested-value-source">
- <i class="pi pi-file" />{{ dataset.metadata.documents[0].title }}
+ <i class="pi pi-file" />{{ dataset.metadata?.documents[0].title }}
  </div>
  <div class="suggested-value">{{ suggestedValues[index] }}</div>
  </div>
@@ -314,6 +277,17 @@
  </div>
  </div>
  </AccordionTab>
+ <AccordionTab v-if="pd.length > 0">
+ <template #header>
+ <header id="ExtractionTable">Extraction Table</header>
+ </template>
+ <DataTable :value="pd">
+ <Column field="col_name" header="Column Name"></Column>
+ <Column field="concept" header="Concept"></Column>
+ <Column field="unit" header="Unit"></Column>
+ <Column field="description" header="Description"></Column>
+ </DataTable>
+ </AccordionTab>
  </Accordion>
  </template>
  <template v-else-if="datasetView === DatasetView.DATA">
@@ -345,6 +319,7 @@ import Accordion from 'primevue/accordion';
 import Button from 'primevue/button';
 import AccordionTab from 'primevue/accordiontab';
 import Message from 'primevue/message';
+import InputText from 'primevue/inputtext';
 import * as textUtil from '@/utils/text';
 import { isString } from 'lodash';
 import { downloadRawFile, getDataset } from '@/services/dataset';
@@ -382,14 +357,12 @@ const pd = computed(() =>
 );
 
 const headers = ref({
- DESCRIPTION: 'Project Description',
  AUTHOR_NAME: 'Author Name',
  AUTHOR_EMAIL: 'Author Email',
  DATE: 'Date of Data',
  SCHEMA: 'Data Schema',
  PROVENANCE: 'Data Provenance',
  SENSITIVITY: 'Data Sensitivity',
- EXAMPLES: 'Example Data',
  LICENSE: 'License Information'
 });
 

diff --git a/packages/client/hmi-client/src/components/models/tera-model.vue b/packages/client/hmi-client/src/components/models/tera-model.vue
@@ -86,6 +86,7 @@
  :publications="publications"
  :project="project"
  :dialog-flavour="'model'"
+ :assetId="assetId"
  />
  <Accordion multiple :active-index="[0, 1, 2, 3, 4, 5, 6]">
  <!-- Description -->

diff --git a/packages/client/hmi-client/src/components/widgets/tera-related-publications.vue b/packages/client/hmi-client/src/components/widgets/tera-related-publications.vue
@@ -36,7 +36,7 @@
  <Button
  label="Use these resources to enrich descriptions"
  @click="
- sendForEnrichments(selectedResources);
+ sendForEnrichments();
  visible = false;
  "
  />
@@ -59,6 +59,7 @@ import { AcceptedExtensions } from '@/types/common';
 import { WASTE_WATER_SURVEILLANCE } from '@/temp/datasets/wasteWaterSurveillance';
 
 import { Artifact, DocumentAsset } from '@/types/Types';
+import { profileDataset, fetchExtraction } from '@/services/models/extractions';
 
 const visible = ref(false);
 const selectedResources = ref();
@@ -96,6 +97,7 @@ const props = defineProps<{
  project?: IProject;
  publications?: Array<DocumentAsset>;
  dialogFlavour: string;
+ assetId: string;
 }>();
 const emit = defineEmits(['extracted-metadata']);
 
@@ -104,12 +106,17 @@ const addResources = () => {
  // do something
 };
 
-function sendForEnrichments(_selectedResources) {
- console.log('sending these resources for enrichment:', _selectedResources);
+const sendForEnrichments = async (/* _selectedResources */) => {
+ // 1. Send dataset profile request
+ /* TODO: send selected resources along with dataset to backend for enrichment */
+ const resp = await profileDataset(props.assetId);
+
+ // 2. Poll
+ const pollResult = await fetchExtraction(resp);
+ console.log('enrichment poll', pollResult);
 
  emit('extracted-metadata', WASTE_WATER_SURVEILLANCE);
- /* TODO: send selected resources to backend for enrichment */
-}
+};
 </script>
 
 <style scoped>

diff --git a/packages/client/hmi-client/src/services/models/extractions.ts b/packages/client/hmi-client/src/services/models/extractions.ts
@@ -8,26 +8,28 @@ import { logger } from '@/utils/logger';
  * @param id
  * @return {Promise<PollerResult>}
  */
-async function fetchExtraction(id: string) {
+export async function fetchExtraction(id: string) {
  const pollerResult: PollResponse<any> = { data: null, progress: null, error: null };
- const poller = new Poller<object>().setPollAction(async () => {
- const response = await API.get(`/extract/status/${id}`);
+ const poller = new Poller<object>()
+ .setPollAction(async () => {
+ const response = await API.get(`/extract/status/${id}`);
 
- // Finished
- if (response?.status === 200 && response?.data?.status === 'finished') {
- pollerResult.data = response.data.result;
- return pollerResult;
- }
+  // Finished
+  if (response?.status === 200 && response?.data?.status === 'finished') {
+  pollerResult.data = response.data.result;
+  return pollerResult;
+  }
 
- // Failed
- if (response?.status === 200 && response?.data?.status === 'failed') {
- pollerResult.error = true;
- return pollerResult;
- }
+  // Failed
+  if (response?.status === 200 && response?.data?.status === 'failed') {
+  pollerResult.error = true;
+  return pollerResult;
+  }
 
- // Queued
- return pollerResult;
- });
+ // Queued
+ return pollerResult;
+ })
+ .setThreshold(30);
  return poller.start();
 }
 
@@ -89,4 +91,19 @@ const mathmlToAMR = async (mathml: string[], framework = 'petrinet'): Promise<Mo
  return null;
 };
 
+/**
+ * Given a dataset, enrich its metadata
+ * Returns a runId used to poll for result
+ */
+export const profileDataset = async (datasetId: string, artifactId: string | null = null) => {
+ let response: any = null;
+ if (artifactId) {
+ response = await API.post(`/extract/profile-dataset/${datasetId}?artifact_id=${artifactId}`);
+ } else {
+ response = await API.post(`/extract/profile-dataset/${datasetId}`);
+ }
+ console.log('data profile response', response.data);
+ return response.data.id;
+};
+
 export { mathmlToAMR, latexToAMR };
diff --git a/packages/client/hmi-client/src/temp/datasets/wasteWaterSurveillance.ts b/packages/client/hmi-client/src/temp/datasets/wasteWaterSurveillance.ts
@@ -14,8 +14,8 @@ export const WASTE_WATER_SURVEILLANCE = {
  '{"dates": "2021-05-14", "VAX_count": 236, "day": 430, "sdm": 0.4, "events": 0.1, "I_1": 85, "I_2": 49, "I_3": 62, "Y_1": 3, "Y_2": 0, "Y_3": 2, "V_1": 113192.37284202829, "V_2": 81612.73344185714, "V_3": 78832.64723781461, "Infected": 196, "Y": 5, "V": 87568.41792704807, "logV": 11.38017568597244}',
  LICENSE: 'UNKNOWN',
  DATA_PROFILING_RESULT: {
- dates: {
- col_name: 'dates',
+ date: {
+ col_name: 'date',
  concept: 'Time',
  unit: 'YYYY-MM-DD',
  description: 'The date when the data was recorded',

diff --git a/...ftware/uncharted/terarium/hmiserver/proxies/extractionservice/ExtractionServiceProxy.java b/...ftware/uncharted/terarium/hmiserver/proxies/extractionservice/ExtractionServiceProxy.java
@@ -84,11 +84,11 @@ Response postPDFExtractions(
  * @return the profiled dataset
  */
  @POST
- @Path("/profile_dataset")
+ @Path("/profile_dataset/{dataset_id}")
  @Consumes(MediaType.APPLICATION_JSON)
  Response postProfileDataset(
- @QueryParam("dataset_id") String datasetId,
- @QueryParam("document_text") String documentText
+ @PathParam("dataset_id") String datasetId,
+ @QueryParam("artifact_id") String artifactId
  );
 
  /**

diff --git a/...software/uncharted/terarium/hmiserver/resources/extractionservice/ExtractionResource.java b/...software/uncharted/terarium/hmiserver/resources/extractionservice/ExtractionResource.java
@@ -154,12 +154,12 @@ public Response postPDFExtractions(
  * @return the profiled dataset
  */
  @POST
- @Path("/profile-dataset")
+ @Path("/profile-dataset/{dataset_id}")
  @Consumes(MediaType.APPLICATION_JSON)
  public Response postProfileDataset(
- @QueryParam("dataset_id") String datasetId,
- @QueryParam("document_text") String documentText
+ @PathParam("dataset_id") String datasetId,
+ @QueryParam("artifact_id") String artifactId
  ) {
- return extractionProxy.postProfileDataset(datasetId, documentText);
+ return extractionProxy.postProfileDataset(datasetId, artifactId);
  };
 }