No messages yet
diff --git a/src/services/ai-worker.ts b/src/services/ai-worker.ts
deleted file mode 100644
index bef156d..0000000
--- a/src/services/ai-worker.ts
+++ /dev/null
@@ -1,123 +0,0 @@
-// @ts-ignore
-import { AutomaticSpeechRecognitionPipeline, pipeline } from '@xenova/transformers';
-
-let transcriber: AutomaticSpeechRecognitionPipeline | undefined = undefined;
-
-self.onmessage = async (e) => {
- if (e.data.type === 'transcribe') {
- return new Promise((resolve) => {
- localTranscribe(e.data.blob, e.data.model).then((transcription) => {
- self.postMessage({
- type: 'transcribe',
- transcription: transcription.text,
- });
- resolve(transcription);
- })
- })
- }
- else if (e.data.type === "load") {
- await loadTranscriber(e.data.model || "tiny");
- return Promise.resolve();
- }
- else {
- return Promise.reject('Unknown message type');
- }
-}
-
-export async function loadTranscriber(model: "tiny" | "base"): Promise
{
- return new Promise(async (resolve) => {
- if (!transcriber) {
- try {
- transcriber = await pipeline('automatic-speech-recognition', `Xenova/whisper-${model}`);
- console.log("Transcriber loaded", transcriber)
- }
- catch (err) {
- console.error("err", err);
- }
-
- resolve();
- }
- else {
- resolve();
- }
- })
-}
-
-export async function localTranscribe(audio: Blob, model: "tiny" | "base"): Promise {
- return new Promise(async (resolve, reject) => {
- await loadTranscriber(model);
-
- if (transcriber) {
- // @ts-ignore
- const output = await transcriber(audio, {
- top_k: 0,
- do_sample: false,
- return_timestamps: true,
- force_full_sequences: false,
- chunk_length_s: 30,
- stride_length_s: 5,
- callback_function: callback_function, // after each generation step
- chunk_callback: chunk_callback, // after each chunk is processed
- });
-
- resolve(output);
- }
- else {
- reject("No transcriber loaded");
- }
- })
-}
-
-// Storage for chunks to be processed. Initialise with an empty chunk.
-const chunks_to_process = [
- {
- tokens: [],
- finalised: false,
- },
-];
-
-function chunk_callback(chunk: any) {
- let last = chunks_to_process[chunks_to_process.length - 1];
-
- // Overwrite last chunk with new info
- Object.assign(last, chunk);
- last.finalised = true;
-
- // Create an empty chunk after, if it not the last chunk
- if (!chunk.is_last) {
- chunks_to_process.push({
- tokens: [],
- finalised: false,
- });
- }
-}
-
-
-// Inject custom callback function to handle merging of chunks
-function callback_function(item: any) {
- const time_precision =
- transcriber?.processor.feature_extractor.config.chunk_length /
- transcriber?.model.config.max_source_positions;
-
- const last: any = chunks_to_process[chunks_to_process.length - 1];
-
- // Update tokens of last chunk
- last.tokens = [...item[0].output_token_ids];
-
- if (last.tokens.length > 1) {
- // Merge text chunks
- // TODO optimise so we don't have to decode all chunks every time
-
- // @ts-ignore
- const data = transcriber?.tokenizer._decode_asr(chunks_to_process, {
- time_precision: time_precision,
- return_timestamps: true,
- force_full_sequences: false,
- });
-
- self.postMessage({
- type: 'transcribe-interim',
- transcription: data[0],
- });
- }
-}
\ No newline at end of file
diff --git a/src/services/llm.ts b/src/services/llm.ts
index 98781d4..d5e33f7 100644
--- a/src/services/llm.ts
+++ b/src/services/llm.ts
@@ -1,10 +1,12 @@
+// @ts-ignore
import * as ort from 'onnxruntime-web/webgpu';
ort.env.wasm.numThreads = 1;
ort.env.wasm.simd = true;
-ort.env.wasm.wasmPaths = document.location.pathname.replace('index.html', '') + 'public/';
+// todo: set this with env variable for easier dev / prod builds
+ort.env.wasm.wasmPaths = "/";
//
// load file from server or cache
diff --git a/src/services/phi.ts b/src/services/phi.ts
index 28730bf..7a42aff 100644
--- a/src/services/phi.ts
+++ b/src/services/phi.ts
@@ -1,13 +1,29 @@
import { env, AutoTokenizer } from '@xenova/transformers';
import { LLM } from './llm.js';
+onmessage = async (e) => {
+ if (e.data.type === 'Init') {
+ Init(false).then(() => {
+ console.log("loaded from worker");
+ postMessage({ type: 'loaded' });
+ })
+ }
+ else if (e.data.type === 'Query') {
+ Query(e.data.continuation, e.data.prompt).then(() => {
+ postMessage({ type: 'done' });
+ });
+ }
+}
+
+
const MODELS: any = {
"phi3": { name: "phi3", path: "microsoft/Phi-3-mini-4k-instruct-onnx-web", externaldata: true },
"phi3dev": { name: "phi3dev", path: "schmuell/Phi-3-mini-4k-instruct-onnx-web", externaldata: true },
};
+
function getConfig() {
- const query = window.location.search.substring(1);
+ const query = '';
let config: any = {
model: "phi3",
provider: "webgpu",
@@ -20,7 +36,7 @@ function getConfig() {
local: 0,
}
let vars = query.split("&");
- for (var i = 0; i < vars.length; i++) {
+ for (let i = 0; i < vars.length; i++) {
let pair = vars[i].split("=");
if (pair[0] in config) {
const key = pair[0];
@@ -58,52 +74,63 @@ function token_to_text(tokenizer: any, tokens: any, startidx: any) {
return txt;
}
-export async function Query(continuation: any, query: any, cb: Function) {
- let prompt = (continuation) ? query : `<|system|>\nYou are a friendly assistant.<|end|>\n<|user|>\n${query}<|end|>\n<|assistant|>\n`;
+export async function Query(continuation: any, query: any/*, cb: Function*/): Promise {
+ return new Promise(async (resolve) => {
+ let prompt = (continuation) ? query : `<|system|>\nYou are a friendly assistant.<|end|>\n<|user|>\n${query}<|end|>\n<|assistant|>\n`;
- const { input_ids } = await tokenizer(prompt, { return_tensor: false, padding: true, truncation: true });
+ const { input_ids } = await tokenizer(prompt, { return_tensor: false, padding: true, truncation: true });
- // clear caches
- // TODO: use kv_cache for continuation
- llm.initilize_feed();
+ // clear caches
+ // TODO: use kv_cache for continuation
+ llm.initilize_feed();
- const start_timer = performance.now();
- const output_index = llm.output_tokens.length + input_ids.length;
- const output_tokens = await llm.generate(input_ids, (output_tokens: any) => {
- if (output_tokens.length == input_ids.length + 1) {
- // time to first token
- const took = (performance.now() - start_timer) / 1000;
- console.log(`time to first token in ${took.toFixed(1)}sec, ${input_ids.length} tokens`);
- }
- cb(token_to_text(tokenizer, output_tokens, output_index));
- }, { max_tokens: config.max_tokens });
-
- const took = (performance.now() - start_timer) / 1000;
- cb(token_to_text(tokenizer, output_tokens, output_index));
- const seqlen = output_tokens.length - output_index;
- console.log(`${seqlen} tokens in ${took.toFixed(1)}sec, ${(seqlen / took).toFixed(2)} tokens/sec`);
+ const start_timer = performance.now();
+ const output_index = llm.output_tokens.length + input_ids.length;
+ const output_tokens = await llm.generate(input_ids, (output_tokens: any) => {
+ if (output_tokens.length == input_ids.length + 1) {
+ // time to first token
+ const took = (performance.now() - start_timer) / 1000;
+ console.log(`time to first token in ${took.toFixed(1)}sec, ${input_ids.length} tokens`);
+ }
+ // cb(token_to_text(tokenizer, output_tokens, output_index));
+ postMessage({ type: 'response', response: token_to_text(tokenizer, output_tokens, output_index) });
+ }, { max_tokens: config.max_tokens });
+
+ const took = (performance.now() - start_timer) / 1000;
+ // cb(token_to_text(tokenizer, output_tokens, output_index));
+ postMessage({ type: 'response', response: token_to_text(tokenizer, output_tokens, output_index) });
+ const seqlen = output_tokens.length - output_index;
+ console.log(`${seqlen} tokens in ${took.toFixed(1)}sec, ${(seqlen / took).toFixed(2)} tokens/sec`);
+
+ resolve();
+ });
}
//
// Load the model and tokenizer
//
-export async function Init(hasFP16: boolean) {
- try {
- tokenizer = await AutoTokenizer.from_pretrained(config.model.path);
-
- console.log("Loading model...");
- await llm.load(config.model, {
- provider: config.provider,
- profiler: config.profiler,
- verbose: config.verbose,
- local: config.local,
- max_tokens: config.max_tokens,
- hasFP16: hasFP16,
- });
- console.log("Ready.");
- } catch (error) {
- console.log(error);
- }
+export async function Init(hasFP16: boolean): Promise {
+ return new Promise(async (resolve, reject) => {
+ try {
+ tokenizer = await AutoTokenizer.from_pretrained(config.model.path);
+
+ console.log("Loading model...");
+ await llm.load(config.model, {
+ provider: config.provider,
+ profiler: config.profiler,
+ verbose: config.verbose,
+ local: config.local,
+ max_tokens: config.max_tokens,
+ hasFP16: hasFP16,
+ });
+ console.log("Ready.");
+
+ resolve("Ready");
+ } catch (error) {
+ console.log(error);
+ reject(error);
+ }
+ })
}
// e
\ No newline at end of file
diff --git a/src/services/whisper.ts b/src/services/whisper.ts
deleted file mode 100644
index 3e48c34..0000000
--- a/src/services/whisper.ts
+++ /dev/null
@@ -1,48 +0,0 @@
-let whisperWorker: Worker;
-
-// @ts-ignore
-import WhisperWorker from './ai-worker?worker'
-
-export async function loadTranscriber(model: "tiny" | "base"): Promise {
- whisperWorker = new WhisperWorker();
- whisperWorker.postMessage({
- type: "load",
- model: model || "tiny",
- });
-}
-
-export function doLocalWhisper(audioFile: Blob, model: "tiny" | "base"): Promise {
- return new Promise((resolve) => {
- const fileReader = new FileReader();
- fileReader.onloadend = async () => {
- const audioCTX = new AudioContext({
- sampleRate: 16000,
- });
- const arrayBuffer = fileReader.result as ArrayBuffer;
- const audioData = await audioCTX.decodeAudioData(arrayBuffer);
-
- let audio = audioData.getChannelData(0);
-
- whisperWorker.onmessage = async (e) => {
- if (e.data.type === "transcribe") {
- resolve(e.data);
- }
- else if (e.data.type === "transcribe-interim") {
- window.dispatchEvent(new CustomEvent('interim-transcription', {
- detail: {
- message: e.data.transcription,
- }
- }));
- }
- }
-
- whisperWorker.postMessage({
- type: "transcribe",
- blob: audio,
- model: model || "tiny",
- })
-
- };
- fileReader.readAsArrayBuffer(audioFile);
- })
-}
\ No newline at end of file