From 7ac14123ff404fcea7fd5b0a90d2d7b810152c75 Mon Sep 17 00:00:00 2001
From: mloppie <mloppie@uncharted.software>
Date: Wed, 2 Oct 2024 16:53:00 -0400
Subject: [PATCH] paste screenshot as equasion and convert to base64 (#5009)

---
 .../tera-model-from-equations-drilldown.vue   | 48 +++++++++++++++++--
 .../client/hmi-client/src/services/goLLM.ts   | 14 ++++++
 2 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/packages/client/hmi-client/src/components/workflow/ops/model-from-equations/tera-model-from-equations-drilldown.vue b/packages/client/hmi-client/src/components/workflow/ops/model-from-equations/tera-model-from-equations-drilldown.vue
index 364c10c1fb..77b015fd07 100644
--- a/packages/client/hmi-client/src/components/workflow/ops/model-from-equations/tera-model-from-equations-drilldown.vue
+++ b/packages/client/hmi-client/src/components/workflow/ops/model-from-equations/tera-model-from-equations-drilldown.vue
@@ -29,10 +29,12 @@
 							<section class="header-group">
 								<Textarea
 									v-model="multipleEquations"
+									ref="equationTextarea"
 									autoResize
 									rows="1"
 									placeholder="Add one or more LaTex equations, or paste in a screenshot"
 									class="w-full"
+									:disabled="multipleEquationsDisabled"
 								/>
 								<Button label="Add" @click="getEquations" class="ml-2" :disabled="isEmpty(multipleEquations)" />
 							</section>
@@ -134,15 +136,15 @@ import { AssetBlock, WorkflowNode } from '@/types/workflow';
 import Button from 'primevue/button';
 import Checkbox from 'primevue/checkbox';
 import Textarea from 'primevue/textarea';
-import TeraAssetBlock from '@/components/widgets/tera-asset-block.vue';
 import TeraDrilldown from '@/components/drilldown/tera-drilldown.vue';
 import TeraDrilldownPreview from '@/components/drilldown/tera-drilldown-preview.vue';
-import { computed, onMounted, ref, watch } from 'vue';
+import TeraAssetBlock from '@/components/widgets/tera-asset-block.vue';
+import { computed, onMounted, onBeforeUnmount, ref, watch } from 'vue';
 import type { Card, DocumentAsset, Model } from '@/types/Types';
 import { cloneDeep, isEmpty } from 'lodash';
 import { equationsToAMR, type EquationsToAMRRequest } from '@/services/knowledge';
 import { downloadDocumentAsset, getDocumentAsset, getDocumentFileAsText } from '@/services/document-assets';
-import { enrichModelMetadata } from '@/services/goLLM';
+import { enrichModelMetadata, equationsFromImage } from '@/services/goLLM';
 import { getModel, updateModel } from '@/services/model';
 import { useProjects } from '@/composables/project';
 import TeraOperatorPlaceholder from '@/components/operator/tera-operator-placeholder.vue';
@@ -203,14 +205,17 @@ const goLLMCard = computed<any>(() => document.value?.metadata?.gollmCard);
 
 const isGeneratingCard = ref(false);
 const multipleEquations = ref<string>('');
+const multipleEquationsDisabled = ref(false);
 
 const isDocViewerOpen = ref(true);
 const isInputOpen = ref(true);
 const isOutputOpen = ref(true);
 
+const equationTextarea = ref();
 const documentEquations = ref<AssetBlock<EquationBlock>[]>();
 
 onMounted(async () => {
+	window.addEventListener('paste', handlePasteEvent);
 	clonedState.value = cloneDeep(props.node.state);
 	if (selectedOutputId.value) {
 		onSelection(selectedOutputId.value);
@@ -267,6 +272,42 @@ onMounted(async () => {
 	assetLoading.value = false;
 });
 
+function handlePasteEvent(e) {
+	// checks if the user pasted a file or collection of files
+	if (e.clipboardData?.files.length) {
+		multipleEquationsDisabled.value = true;
+		Array.from(e.clipboardData.files).forEach((item) => {
+			const reader = new FileReader();
+			reader.onload = function ({ target }) {
+				if (target && document.value?.id) {
+					const base64 = arrayBufferToBase64(target.result);
+					// send base64 to gollm
+					equationsFromImage(document.value.id, base64).then((response) => {
+						const responseJson = JSON.parse(window.atob(response.output)).response;
+						multipleEquations.value = responseJson.equations.join('\n');
+					});
+				}
+			};
+			if (item instanceof Blob) {
+				reader.readAsArrayBuffer(item);
+			}
+		});
+	}
+}
+
+function arrayBufferToBase64(buffer) {
+	let binary = '';
+	const bytes = new Uint8Array(buffer);
+	for (let i = 0; i < bytes.byteLength; i++) {
+		binary += String.fromCharCode(bytes[i]);
+	}
+	return window.btoa(binary);
+}
+
+onBeforeUnmount(async () => {
+	window.removeEventListener('paste', handlePasteEvent);
+});
+
 const onSelection = (id: string) => {
 	emit('select-output', id);
 };
@@ -347,6 +388,7 @@ function getEquations() {
 	});
 	emit('update-state', clonedState.value);
 	multipleEquations.value = '';
+	multipleEquationsDisabled.value = false;
 }
 
 function getEquationErrorLabel(equation) {
diff --git a/packages/client/hmi-client/src/services/goLLM.ts b/packages/client/hmi-client/src/services/goLLM.ts
index b4efed17fa..8c7e50dd47 100644
--- a/packages/client/hmi-client/src/services/goLLM.ts
+++ b/packages/client/hmi-client/src/services/goLLM.ts
@@ -95,6 +95,20 @@ export async function configureModelFromDocument(
 	return data;
 }
 
+export async function equationsFromImage(documentId: string, base64ImageStr: string): Promise<TaskResponse> {
+	const { data } = await API.post<TaskResponse>(
+		'/gollm/equations-from-image',
+		{ base64ImageStr },
+		{
+			params: {
+				'document-id': documentId,
+				mode: 'SYNC'
+			}
+		}
+	);
+	return data;
+}
+
 export async function configureModelFromDataset(
 	modelId: string,
 	datasetId: string,