From 2cee15af1c3a93d5f5a3749a570de13f0aba541f Mon Sep 17 00:00:00 2001
From: Tom Szendrey <T.Szendrey@hotmail.com>
Date: Mon, 24 Jul 2023 10:08:20 -0400
Subject: [PATCH] Ts/dataset extract update (#1481)

Co-authored-by: Yohann Paris <github@yohannparis.com>
Co-authored-by: mwdchang <mwdchang@gmail.com>
---
 .../src/components/dataset/tera-dataset.vue   | 81 +++++++------------
 .../src/components/models/tera-model.vue      |  1 +
 .../widgets/tera-related-publications.vue     | 17 ++--
 .../src/services/models/extractions.ts        | 49 +++++++----
 .../temp/datasets/wasteWaterSurveillance.ts   |  4 +-
 .../ExtractionServiceProxy.java               |  6 +-
 .../extractionservice/ExtractionResource.java |  8 +-
 7 files changed, 82 insertions(+), 84 deletions(-)

diff --git a/packages/client/hmi-client/src/components/dataset/tera-dataset.vue b/packages/client/hmi-client/src/components/dataset/tera-dataset.vue
index b78e333bd1..3d82d7899d 100644
--- a/packages/client/hmi-client/src/components/dataset/tera-dataset.vue
+++ b/packages/client/hmi-client/src/components/dataset/tera-dataset.vue
@@ -85,6 +85,7 @@
 				:dialog-flavour="'dataset'"
 				:publications="props.project?.assets?.publications"
 				:project="project"
+				:assetId="assetId"
 			/>
 			<Accordion :multiple="true" :activeIndex="[0, 1, 2]">
 				<AccordionTab>
@@ -93,25 +94,7 @@
 					</template>
 					<section v-if="enriched">
 						<div class="dataset-detail">
-							<div class="full-width">
-								<div class="detail-list">
-									<h2 class="title">Dataset Details</h2>
-									<ul>
-										<li><strong>Dataset name:</strong> {{ dataset.name }}</li>
-										<li><strong>Dataset overview:</strong> {{ dataset.description }}</li>
-										<li>
-											<strong>Dataset URL:</strong>
-											<a :href="dataset.source">{{ dataset.source }}</a>
-										</li>
-										<li>
-											<strong>Data size:</strong> This dataset currently contains
-											{{ csvContent?.length || '-' }} rows.
-										</li>
-									</ul>
-								</div>
-							</div>
 							<div class="column">
-								<h3 class="title">{{ headers.DESCRIPTION }}</h3>
 								<p class="content">{{ enrichedData.DESCRIPTION }}</p>
 
 								<h3 class="subtitle">{{ headers.AUTHOR_NAME }}</h3>
@@ -140,27 +123,6 @@
 								<h3 class="subtitle">{{ headers.LICENSE }}</h3>
 								<p class="content">{{ enrichedData.LICENSE }}</p>
 							</div>
-							<div class="full-width">
-								<h3 class="subtitle">{{ headers.EXAMPLES }}</h3>
-								<ul class="example-list" v-if="enrichedData.EXAMPLES">
-									<li
-										class="example-item"
-										v-for="(value, key) in JSON.parse(enrichedData.EXAMPLES)"
-										:key="key"
-									>
-										<strong>{{ key }}:</strong>{{ value }}
-									</li>
-								</ul>
-							</div>
-							<div class="full-width">
-								<h3 class="subtitle">{{ `Extraction Table` }}</h3>
-								<DataTable :value="pd">
-									<Column field="col_name" header="Column Name"></Column>
-									<Column field="concept" header="Concept"></Column>
-									<Column field="unit" header="Unit"></Column>
-									<Column field="description" header="Description"></Column>
-								</DataTable>
-							</div>
 						</div>
 					</section>
 					<p v-else>
@@ -173,9 +135,9 @@
 						<header id="Source">Source</header>
 					</template>
 					This data is sourced from
-					{{ dataset.metadata.documents ? dataset.metadata.documents[0].title : 'unknown' }}:
-					<a :href="dataset.metadata.documents ? dataset.metadata.documents[0].url : ''">{{
-						dataset.metadata.documents ? dataset.metadata.documents[0].url : ''
+					{{ dataset.metadata?.documents ? dataset.metadata.documents[0].title : 'unknown' }}:
+					<a :href="dataset.metadata?.documents ? dataset.metadata?.documents[0].url : ''">{{
+						dataset.metadata?.documents ? dataset.metadata?.documents[0].url : ''
 					}}</a>
 				</AccordionTab>
 				<AccordionTab>
@@ -241,20 +203,21 @@
 							<InputText
 								class="p-inputtext-sm"
 								type="text"
-								v-if="rowEditList[index]"
-								@focus="setSuggestedValue(index, '')"
+								v-if="rowEditList[index] && column.metadata"
+								v-model="column.metadata.unit"
+								@focus="setSuggestedValue(index, dataset.columns?.[index].metadata?.unit)"
 							/>
-							<div class="variables-value" v-else>-</div>
-							<!-- grounding -->
+							<div class="variables-value" v-else>{{ column.metadata?.unit ?? '--' }}</div>
+							<!-- Concept -->
 							<InputText
 								class="p-inputtext-sm"
 								type="text"
-								v-if="rowEditList[index]"
-								v-model="groundingValues[index][0]"
-								@focus="setSuggestedValue(index, originalGroundingValues[0])"
+								v-if="rowEditList[index] && column.metadata"
+								v-model="column.metadata.concept"
+								@focus="setSuggestedValue(index, column.metadata?.concept)"
 							/>
 							<div class="variables-value" v-else>
-								{{ groundingValues[index][0] }}
+								{{ column.metadata?.concept ?? '--' }}
 							</div>
 							<!-- extractions - field does not exist in tds yet -->
 							<InputText
@@ -263,7 +226,7 @@
 								v-if="rowEditList[index]"
 								@focus="setSuggestedValue(index, '')"
 							/>
-							<div class="variables-value" v-else></div>
+							<div class="variables-value" v-else>{{ column.metadata?.extraction ?? '--' }}</div>
 							<div v-if="rowEditList[index]" class="row-edit-buttons">
 								<Button
 									text
@@ -293,7 +256,7 @@
 								<span>Suggested value</span>
 								<div>
 									<div class="suggested-value-source">
-										<i class="pi pi-file" />{{ dataset.metadata.documents[0].title }}
+										<i class="pi pi-file" />{{ dataset.metadata?.documents[0].title }}
 									</div>
 									<div class="suggested-value">{{ suggestedValues[index] }}</div>
 								</div>
@@ -314,6 +277,17 @@
 						</div>
 					</div>
 				</AccordionTab>
+				<AccordionTab v-if="pd.length > 0">
+					<template #header>
+						<header id="ExtractionTable">Extraction Table</header>
+					</template>
+					<DataTable :value="pd">
+						<Column field="col_name" header="Column Name"></Column>
+						<Column field="concept" header="Concept"></Column>
+						<Column field="unit" header="Unit"></Column>
+						<Column field="description" header="Description"></Column>
+					</DataTable>
+				</AccordionTab>
 			</Accordion>
 		</template>
 		<template v-else-if="datasetView === DatasetView.DATA">
@@ -345,6 +319,7 @@ import Accordion from 'primevue/accordion';
 import Button from 'primevue/button';
 import AccordionTab from 'primevue/accordiontab';
 import Message from 'primevue/message';
+import InputText from 'primevue/inputtext';
 import * as textUtil from '@/utils/text';
 import { isString } from 'lodash';
 import { downloadRawFile, getDataset } from '@/services/dataset';
@@ -382,14 +357,12 @@ const pd = computed(() =>
 );
 
 const headers = ref({
-	DESCRIPTION: 'Project Description',
 	AUTHOR_NAME: 'Author Name',
 	AUTHOR_EMAIL: 'Author Email',
 	DATE: 'Date of Data',
 	SCHEMA: 'Data Schema',
 	PROVENANCE: 'Data Provenance',
 	SENSITIVITY: 'Data Sensitivity',
-	EXAMPLES: 'Example Data',
 	LICENSE: 'License Information'
 });
 
diff --git a/packages/client/hmi-client/src/components/models/tera-model.vue b/packages/client/hmi-client/src/components/models/tera-model.vue
index 2c6dd756e1..ae92a2e1df 100644
--- a/packages/client/hmi-client/src/components/models/tera-model.vue
+++ b/packages/client/hmi-client/src/components/models/tera-model.vue
@@ -86,6 +86,7 @@
 				:publications="publications"
 				:project="project"
 				:dialog-flavour="'model'"
+				:assetId="assetId"
 			/>
 			<Accordion multiple :active-index="[0, 1, 2, 3, 4, 5, 6]">
 				<!-- Description -->
diff --git a/packages/client/hmi-client/src/components/widgets/tera-related-publications.vue b/packages/client/hmi-client/src/components/widgets/tera-related-publications.vue
index 39bbb6b76e..5b2d6d80b4 100644
--- a/packages/client/hmi-client/src/components/widgets/tera-related-publications.vue
+++ b/packages/client/hmi-client/src/components/widgets/tera-related-publications.vue
@@ -36,7 +36,7 @@
 					<Button
 						label="Use these resources to enrich descriptions"
 						@click="
-							sendForEnrichments(selectedResources);
+							sendForEnrichments();
 							visible = false;
 						"
 					/>
@@ -59,6 +59,7 @@ import { AcceptedExtensions } from '@/types/common';
 import { WASTE_WATER_SURVEILLANCE } from '@/temp/datasets/wasteWaterSurveillance';
 
 import { Artifact, DocumentAsset } from '@/types/Types';
+import { profileDataset, fetchExtraction } from '@/services/models/extractions';
 
 const visible = ref(false);
 const selectedResources = ref();
@@ -96,6 +97,7 @@ const props = defineProps<{
 	project?: IProject;
 	publications?: Array<DocumentAsset>;
 	dialogFlavour: string;
+	assetId: string;
 }>();
 const emit = defineEmits(['extracted-metadata']);
 
@@ -104,12 +106,17 @@ const addResources = () => {
 	// do something
 };
 
-function sendForEnrichments(_selectedResources) {
-	console.log('sending these resources for enrichment:', _selectedResources);
+const sendForEnrichments = async (/* _selectedResources */) => {
+	// 1. Send dataset profile request
+	/* TODO: send selected resources along with dataset to backend for enrichment */
+	const resp = await profileDataset(props.assetId);
+
+	// 2. Poll
+	const pollResult = await fetchExtraction(resp);
+	console.log('enrichment poll', pollResult);
 
 	emit('extracted-metadata', WASTE_WATER_SURVEILLANCE);
-	/* TODO: send selected resources to backend for enrichment */
-}
+};
 </script>
 
 <style scoped>
diff --git a/packages/client/hmi-client/src/services/models/extractions.ts b/packages/client/hmi-client/src/services/models/extractions.ts
index f735a3b91d..01f2550321 100644
--- a/packages/client/hmi-client/src/services/models/extractions.ts
+++ b/packages/client/hmi-client/src/services/models/extractions.ts
@@ -8,26 +8,28 @@ import { logger } from '@/utils/logger';
  * @param id
  * @return {Promise<PollerResult>}
  */
-async function fetchExtraction(id: string) {
+export async function fetchExtraction(id: string) {
 	const pollerResult: PollResponse<any> = { data: null, progress: null, error: null };
-	const poller = new Poller<object>().setPollAction(async () => {
-		const response = await API.get(`/extract/status/${id}`);
+	const poller = new Poller<object>()
+		.setPollAction(async () => {
+			const response = await API.get(`/extract/status/${id}`);
 
-		// Finished
-		if (response?.status === 200 && response?.data?.status === 'finished') {
-			pollerResult.data = response.data.result;
-			return pollerResult;
-		}
+			// Finished
+			if (response?.status === 200 && response?.data?.status === 'finished') {
+				pollerResult.data = response.data.result;
+				return pollerResult;
+			}
 
-		// Failed
-		if (response?.status === 200 && response?.data?.status === 'failed') {
-			pollerResult.error = true;
-			return pollerResult;
-		}
+			// Failed
+			if (response?.status === 200 && response?.data?.status === 'failed') {
+				pollerResult.error = true;
+				return pollerResult;
+			}
 
-		// Queued
-		return pollerResult;
-	});
+			// Queued
+			return pollerResult;
+		})
+		.setThreshold(30);
 	return poller.start();
 }
 
@@ -89,4 +91,19 @@ const mathmlToAMR = async (mathml: string[], framework = 'petrinet'): Promise<Mo
 	return null;
 };
 
+/**
+ * Given a dataset, enrich its metadata
+ * Returns a runId used to poll for result
+ */
+export const profileDataset = async (datasetId: string, artifactId: string | null = null) => {
+	let response: any = null;
+	if (artifactId) {
+		response = await API.post(`/extract/profile-dataset/${datasetId}?artifact_id=${artifactId}`);
+	} else {
+		response = await API.post(`/extract/profile-dataset/${datasetId}`);
+	}
+	console.log('data profile response', response.data);
+	return response.data.id;
+};
+
 export { mathmlToAMR, latexToAMR };
diff --git a/packages/client/hmi-client/src/temp/datasets/wasteWaterSurveillance.ts b/packages/client/hmi-client/src/temp/datasets/wasteWaterSurveillance.ts
index be294f4f84..1fbe3efd56 100644
--- a/packages/client/hmi-client/src/temp/datasets/wasteWaterSurveillance.ts
+++ b/packages/client/hmi-client/src/temp/datasets/wasteWaterSurveillance.ts
@@ -14,8 +14,8 @@ export const WASTE_WATER_SURVEILLANCE = {
 		'{"dates": "2021-05-14", "VAX_count": 236, "day": 430, "sdm": 0.4, "events": 0.1, "I_1": 85, "I_2": 49, "I_3": 62, "Y_1": 3, "Y_2": 0, "Y_3": 2, "V_1": 113192.37284202829, "V_2": 81612.73344185714, "V_3": 78832.64723781461, "Infected": 196, "Y": 5, "V": 87568.41792704807, "logV": 11.38017568597244}',
 	LICENSE: 'UNKNOWN',
 	DATA_PROFILING_RESULT: {
-		dates: {
-			col_name: 'dates',
+		date: {
+			col_name: 'date',
 			concept: 'Time',
 			unit: 'YYYY-MM-DD',
 			description: 'The date when the data was recorded',
diff --git a/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/proxies/extractionservice/ExtractionServiceProxy.java b/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/proxies/extractionservice/ExtractionServiceProxy.java
index 124758d624..809cd77ed7 100644
--- a/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/proxies/extractionservice/ExtractionServiceProxy.java
+++ b/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/proxies/extractionservice/ExtractionServiceProxy.java
@@ -84,11 +84,11 @@ Response postPDFExtractions(
 	 * @return the profiled dataset
 	 */
 	@POST
-	@Path("/profile_dataset")
+	@Path("/profile_dataset/{dataset_id}")
 	@Consumes(MediaType.APPLICATION_JSON)
 	Response postProfileDataset(
-		@QueryParam("dataset_id") String datasetId,
-		@QueryParam("document_text") String documentText
+		@PathParam("dataset_id") String datasetId,
+		@QueryParam("artifact_id") String artifactId
 	);
 
 	/**
diff --git a/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/resources/extractionservice/ExtractionResource.java b/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/resources/extractionservice/ExtractionResource.java
index ca5e648821..4ebe2d8b68 100644
--- a/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/resources/extractionservice/ExtractionResource.java
+++ b/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/resources/extractionservice/ExtractionResource.java
@@ -154,12 +154,12 @@ public Response postPDFExtractions(
 	 * @return the profiled dataset
 	 */
 	@POST
-	@Path("/profile-dataset")
+	@Path("/profile-dataset/{dataset_id}")
 	@Consumes(MediaType.APPLICATION_JSON)
 	public Response postProfileDataset(
-		@QueryParam("dataset_id") String datasetId,
-		@QueryParam("document_text") String documentText
+		@PathParam("dataset_id") String datasetId,
+		@QueryParam("artifact_id") String artifactId
 	) {
-		return extractionProxy.postProfileDataset(datasetId, documentText);
+		return extractionProxy.postProfileDataset(datasetId, artifactId);
 	};
 }