diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0fd303e
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+.DS_Store
+node_modules
+dist
+dev-dist
+build
+types
+.idea
+.github
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
new file mode 100644
index 0000000..6de784d
--- /dev/null
+++ b/.vscode/extensions.json
@@ -0,0 +1,8 @@
+{
+ "recommendations": [
+ "eg2.vscode-npm-script",
+ "christian-kohler.npm-intellisense",
+ "ms-edgedevtools.vscode-edge-devtools",
+ "PWABuilder.pwa-studio"
+ ]
+}
\ No newline at end of file
diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 0000000..6dfde9c
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,29 @@
+{
+ // Use IntelliSense to learn about possible attributes.
+ // Hover to view descriptions of existing attributes.
+ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "type": "pwa-msedge",
+ "request": "launch",
+ "name": "Run PWA",
+ "webRoot": "${workspaceFolder}/",
+ "runtimeArgs": [
+ "--app=http://localhost:5173"
+ ],
+ "sourceMapPathOverrides": {
+ "../../src": "${workspaceFolder}/src",
+ "../../src/*": "${workspaceFolder}/src/*"
+ },
+ "preLaunchTask": "npm run dev-task",
+ "postDebugTask": "postdebugKill"
+ },
+ {
+ "name": "Launch Microsoft Edge and open the Edge DevTools",
+ "request": "launch",
+ "type": "vscode-edge-devtools.debug",
+ "url": "" // Provide your project's url to finish configuring
+ }
+ ]
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..7ea2066
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,9 @@
+{
+ "files.trimTrailingWhitespace": true,
+ "markdownlint.config": {
+ "MD028": false,
+ "MD025": {
+ "front_matter_title": ""
+ }
+ },
+}
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
new file mode 100644
index 0000000..33372fc
--- /dev/null
+++ b/.vscode/tasks.json
@@ -0,0 +1,36 @@
+{
+ "version": "2.0.0",
+ "tasks": [
+ {
+ "label": "npm run dev-task",
+ "type": "npm",
+ "script": "dev-task",
+ "isBackground": true,
+ "problemMatcher": {
+ "owner": "custom",
+ "pattern": {
+ "regexp": "^$"
+ },
+ "background": {
+ "activeOnStart": true,
+ "beginsPattern": ".*",
+ "endsPattern": "ready in .+"
+ }
+ },
+ },
+ {
+ "label": "postdebugKill",
+ "command": "echo ${input:terminate}",
+ "type": "shell",
+ "problemMatcher": []
+ },
+ ],
+ "inputs": [
+ {
+ "id": "terminate",
+ "type": "command",
+ "command": "workbench.action.tasks.terminate",
+ "args": "npm run dev-task"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000..c7f3f0d
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,13 @@
+ManifoldJS
+
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..b6465f9
--- /dev/null
+++ b/README.md
@@ -0,0 +1,36 @@
+# PWA Whisper Starter
+
+[**Straight to Full Documentation**](https://docs.pwabuilder.com/#/starter/quick-start)
+
+The PWABuilder Whisper pwa-starter is based on our opinionated, best practices, production tested starter that we use to build all of our PWAs, including [PWABuilder itself](https://blog.pwabuilder.com/posts/introducing-the-brand-new-pwa-builder/)! The pwa-starter is a starter codebase, just like create-react-app or the Angular CLI can generate, that uses the PWABuilder team's preferred front-end tech stack. We also have a CLI tool to allow you to create a PWA template from the command line.
+
+This "Whisper" version of the starter sets you up with all the code you need to use Whisper to transcribe audio and video, on your device using [transformers.js](https://huggingface.co/docs/transformers.js/index), which uses [Onnxruntime](https://onnxruntime.ai/) to run Whisper client-side on your device, no cloud needed.
+
+## Jump Right In
+
+After you have cloned this starter to your machine, run through the following steps.
+
+Install Dependencies:
+
+`npm install`
+
+And then start developing your app with:
+
+`npm run dev`
+
+With that command, the app will open in the browser and will update when you make changes to the code.
+And that's it! Good luck on your Progressive Web App adventure!
+
+## More Info
+
+[](https://www.youtube.com/watch?v=u3pWKpmic_k)
+
+With it you get an app that:
+- Has no build system to set up and no boilerplate code to add. Everything is included out of the box.
+- Has a Service Worker system using [Workbox](https://developers.google.com/web/tools/workbox/)
+- Scores close to 100 on Lighthouse out of the box
+- Has everything needed to be installable in the browser
+- Is ready to be package for the app stores using [PWABuilder](https://www.pwabuilder.com)
+- Uses the [Azure Static Web Apps CLI](https://azure.github.io/static-web-apps-cli) which enables emulating your production environment locally, and gets you ready for deploying to Azure Static Web Apps!
+
+and all with just a few button clicks 😊.
diff --git a/index.html b/index.html
new file mode 100644
index 0000000..00b636d
--- /dev/null
+++ b/index.html
@@ -0,0 +1,57 @@
+
+
+
+
+ PWA Whisper Starter
+
+
+ About Page
+
+
+ Did you know?
+
+ PWAs have access to many useful APIs in modern browsers! These
+ APIs have enabled many new types of apps that can be built as PWAs, such as advanced graphics editing apps, games,
+ apps that use machine learning and more!
+
+
+ Check out these
+ docs to learn more about the advanced features that you can use in your PWA
+
+ `;
+ }
+}
diff --git a/src/pages/app-home.ts b/src/pages/app-home.ts
new file mode 100644
index 0000000..01d62a1
--- /dev/null
+++ b/src/pages/app-home.ts
@@ -0,0 +1,160 @@
+import { LitElement, css, html } from 'lit';
+import { property, state, customElement } from 'lit/decorators.js';
+
+import '@fluentui/web-components/button.js';
+import '@fluentui/web-components/text-input.js';
+import '@fluentui/web-components/label.js';
+
+import { styles } from '../styles/shared-styles';
+
+@customElement('app-home')
+export class AppHome extends LitElement {
+
+ // For more information on using properties and state in lit
+ // check out this link https://lit.dev/docs/components/properties/
+ @property() message = 'Welcome!';
+
+ @state() previousMessages: any[] = [];
+ @state() loaded: boolean = false;
+
+ query: string = "";
+
+ static styles = [
+ styles,
+ css`
+ main {
+ display: flex;
+ flex-direction: column;
+ gap: 8px;
+ }
+
+ fluent-button svg {
+ width: 11px;
+ height: 11px;
+ }
+
+ fluent-text {
+ min-height: 50vh;
+ background: #ffffff12;
+ border-radius: 4px;
+ padding: 8px;
+ }
+
+ #actions-menu {
+ display: flex;
+ gap: 8px;
+ flex-direction: row;
+ justify-content: space-between;
+
+ margin-bottom: 10px;
+ }
+
+ #main-action-block {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ }
+
+ #file-data-block {
+ display: flex;
+ gap: 4px;
+ }
+
+ #file-size {
+ color: grey;
+ font-size: 10px;
+ }
+
+ #file-name {
+ color: grey;
+ font-size: 12px;
+ font-weight: bold;
+
+ max-width: 169px;
+ white-space: nowrap;
+ text-overflow: ellipsis;
+ overflow-x: hidden;
+ }
+
+ #file-data-block {
+ display: flex;
+ flex-direction: column;
+ }
+
+ @media(prefers-color-scheme: light) {
+ fluent-text {
+ background: #f0f0f0;
+ }
+ }
+
+ @media(max-width: 640px) {
+ main {
+ display: flex;
+ flex-direction: column-reverse;
+ gap: 10px;
+ }
+ }
+ `];
+
+ async firstUpdated() {
+ // this method is a lifecycle even in lit
+ // for more info check out the lit docs https://lit.dev/docs/components/lifecycle/
+ console.log('This is your home page');
+
+ window.onload = async () => {
+ const { Init } = await import('../services/phi');
+ await Init(false);
+
+ this.loaded = true;
+ }
+ }
+
+ async sendMessage() {
+ const { Query } = await import('../services/phi');
+ Query(false, this.query, (message: string) => {
+ console.log("Message received: ", message);
+ });
+ }
+
+ handleInputChange(query: string) {
+ this.query = query;
+ }
+
+ private async copyTranscript() {
+
+ }
+
+ private async shareTranscript() {
+
+ }
+
+ private async downloadTranscript() {
+
+ }
+
+ render() {
+ return html`
+
+
+
+ ${
+ this.previousMessages.length > 0 ?
+ this.previousMessages.map((message: string) => html``) : null
+ }
+
+
+
+
+ Why is the sky blue?
+
+
+
+ Send message
+
+
+
+ `;
+ }
+}
diff --git a/src/router.ts b/src/router.ts
new file mode 100644
index 0000000..8e54d22
--- /dev/null
+++ b/src/router.ts
@@ -0,0 +1,50 @@
+// docs for router https://github.com/thepassle/app-tools/blob/master/router/README.md
+
+import { html } from 'lit';
+
+if (!(globalThis as any).URLPattern) {
+ await import("urlpattern-polyfill");
+}
+
+import "./services/phi.js";
+
+import { Router } from '@thepassle/app-tools/router.js';
+import { lazy } from '@thepassle/app-tools/router/plugins/lazy.js';
+
+// @ts-ignore
+import { title } from '@thepassle/app-tools/router/plugins/title.js';
+
+import './pages/app-home.js';
+
+const baseURL: string = (import.meta as any).env.BASE_URL;
+
+export const router = new Router({
+ routes: [
+ {
+ path: resolveRouterPath(),
+ title: 'Home',
+ render: () => html` {
+ return new Promise(async (resolve) => {
+ if (!transcriber) {
+ try {
+ transcriber = await pipeline('automatic-speech-recognition', `Xenova/whisper-${model}`);
+ console.log("Transcriber loaded", transcriber)
+ }
+ catch (err) {
+ console.error("err", err);
+ }
+
+ resolve();
+ }
+ else {
+ resolve();
+ }
+ })
+}
+
+export async function localTranscribe(audio: Blob, model: "tiny" | "base"): Promise {
+ return new Promise(async (resolve, reject) => {
+ await loadTranscriber(model);
+
+ if (transcriber) {
+ // @ts-ignore
+ const output = await transcriber(audio, {
+ top_k: 0,
+ do_sample: false,
+ return_timestamps: true,
+ force_full_sequences: false,
+ chunk_length_s: 30,
+ stride_length_s: 5,
+ callback_function: callback_function, // after each generation step
+ chunk_callback: chunk_callback, // after each chunk is processed
+ });
+
+ resolve(output);
+ }
+ else {
+ reject("No transcriber loaded");
+ }
+ })
+}
+
+// Storage for chunks to be processed. Initialise with an empty chunk.
+const chunks_to_process = [
+ {
+ tokens: [],
+ finalised: false,
+ },
+];
+
+function chunk_callback(chunk: any) {
+ let last = chunks_to_process[chunks_to_process.length - 1];
+
+ // Overwrite last chunk with new info
+ Object.assign(last, chunk);
+ last.finalised = true;
+
+ // Create an empty chunk after, if it not the last chunk
+ if (!chunk.is_last) {
+ chunks_to_process.push({
+ tokens: [],
+ finalised: false,
+ });
+ }
+}
+
+
+// Inject custom callback function to handle merging of chunks
+function callback_function(item: any) {
+ const time_precision =
+ transcriber?.processor.feature_extractor.config.chunk_length /
+ transcriber?.model.config.max_source_positions;
+
+ const last: any = chunks_to_process[chunks_to_process.length - 1];
+
+ // Update tokens of last chunk
+ last.tokens = [...item[0].output_token_ids];
+
+ if (last.tokens.length > 1) {
+ // Merge text chunks
+ // TODO optimise so we don't have to decode all chunks every time
+
+ // @ts-ignore
+ const data = transcriber?.tokenizer._decode_asr(chunks_to_process, {
+ time_precision: time_precision,
+ return_timestamps: true,
+ force_full_sequences: false,
+ });
+
+ self.postMessage({
+ type: 'transcribe-interim',
+ transcription: data[0],
+ });
+ }
+}
\ No newline at end of file
diff --git a/src/services/llm.ts b/src/services/llm.ts
new file mode 100644
index 0000000..98781d4
--- /dev/null
+++ b/src/services/llm.ts
@@ -0,0 +1,244 @@
+import * as ort from 'onnxruntime-web/webgpu';
+
+
+ort.env.wasm.numThreads = 1;
+ort.env.wasm.simd = true;
+ort.env.wasm.wasmPaths = document.location.pathname.replace('index.html', '') + 'public/';
+
+
+//
+// load file from server or cache
+//
+async function fetchAndCache(url: string) {
+ try {
+ const cache = await caches.open("onnx");
+ let cachedResponse = await cache.match(url);
+ if (cachedResponse === undefined) {
+ console.log(`${url} (network)`);
+ const buffer = await fetch(url).then(response => response.arrayBuffer());
+ try {
+ await cache.put(url, new Response(buffer));
+ } catch (error) {
+ console.error(error);
+ }
+ return buffer;
+ }
+ console.log(`${url} (cached)`);
+ const data = await cachedResponse.arrayBuffer();
+ return data;
+ } catch (error) {
+ console.log(`can't fetch ${url}`);
+ throw error;
+ }
+}
+
+//
+// class to handle a large language model on top of onnxruntime-web
+//
+export class LLM {
+ sess = undefined;
+ profiler = false;
+ feed = {};
+ output_tokens = [];
+ eos = 2;
+ need_position_ids = true;
+ stop = false;
+ kv_dims = [];
+ dtype = "float16";
+ max_tokens = 9999;
+
+ constructor() {
+ }
+
+ async load(model: any, options: any) {
+ const provider = options.provider || "webgpu";
+ const verbose = options.verbose;
+ const local = options.local;
+ const hasFP16 = (provider === "wasm") ? false : options.hasFP16;
+ this.profiler = options.profiler;
+
+ const model_path = (local) ? "models/" + model.path : "https://huggingface.co/" + model.path + "/resolve/main";
+ let model_file = model.file || "model";
+ model_file = (hasFP16) ? model_file + "_q4f16.onnx" : model_file + "_q4.onnx";
+
+ console.log(`loading... ${model.name}, ${provider}`);
+ const json_bytes = await fetchAndCache(model_path + "/config.json");
+ let textDecoder = new TextDecoder();
+ const model_config = JSON.parse(textDecoder.decode(json_bytes));
+
+ const model_bytes = await fetchAndCache(model_path + "/onnx/" + model_file);
+ const externaldata = (model.externaldata) ? await fetchAndCache(model_path + "/onnx/" + model_file + '_data') : false;
+ let modelSize = model_bytes.byteLength;
+ if (externaldata) {
+ modelSize += externaldata.byteLength;
+ }
+ console.log(`model size ${Math.round(modelSize / 1024 / 1024)} MB`);
+
+ const opt: any = {
+ executionProviders: [provider],
+ preferredOutputLocation: {},
+ }
+
+ switch (provider) {
+ case "webgpu":
+ for (let i = 0; i < model_config.num_hidden_layers; ++i) {
+ // @ts-ignore
+ opt.preferredOutputLocation[`present.${i}.key`] = 'gpu-buffer';
+ // @ts-ignore
+ opt.preferredOutputLocation[`present.${i}.value`] = 'gpu-buffer';
+ }
+ break;
+ }
+
+ if (externaldata !== undefined) {
+ opt.externalData = [
+ {
+ data: externaldata,
+ path: model_file + "_data",
+ },
+ ]
+ }
+ if (verbose) {
+ opt.logSeverityLevel = 0;
+ opt.logVerbosityLevel = 0;
+ ort.env.logLevel = "verbose";
+ }
+
+ ort.env.webgpu.profiling = {}
+ if (this.profiler) {
+ opt.enableProfiling = true;
+ ort.env.webgpu.profilingMode = 'default';
+ ort.env.webgpu.profiling.mode = 'default';
+ }
+
+ // @ts-ignore
+ this.sess = await ort.InferenceSession.create(model_bytes, opt);
+ this.eos = model_config.eos_token_id;
+ // @ts-ignore
+ this.kv_dims = [1, model_config.num_key_value_heads, 0, model_config.hidden_size / model_config.num_attention_heads];
+ this.dtype = (hasFP16) ? "float16" : "float32";
+ // @ts-ignore
+ this.num_layers = model_config.num_hidden_layers;
+ this.initilize_feed();
+ }
+
+ initilize_feed() {
+ const feed = this.feed;
+
+ // dispose of previous gpu buffers
+ for (const name in feed) {
+ // @ts-ignore
+ const t = feed[name];
+ if (t.location === 'gpu-buffer') {
+ t.dispose();
+ }
+ }
+ this.feed = {};
+ // key value cache is zero copy, just pass gpu buffer as referece
+ const empty = (this.dtype === "float16") ? new Uint16Array() : [];
+ // @ts-ignore
+ for (let i = 0; i < this.num_layers; ++i) {
+ // @ts-ignore
+ this.feed[`past_key_values.${i}.key`] = new ort.Tensor(this.dtype, empty, this.kv_dims)
+ // @ts-ignore
+ this.feed[`past_key_values.${i}.value`] = new ort.Tensor(this.dtype, empty, this.kv_dims)
+ }
+ this.output_tokens = [];
+ }
+
+ //
+ // poor mens argmax
+ argmax(t: any) {
+ const arr = t.data;
+ const start = t.dims[2] * (t.dims[1] - 1);
+ let max = arr[start];
+ let maxidx = 0;
+
+ for (let i = 0; i < t.dims[2]; i++) {
+ const val = arr[i + start];
+ if (!isFinite(val)) {
+ throw new Error("found infinitive in logits");
+ }
+ if (val > max) {
+ max = arr[i + start];
+ maxidx = i;
+ }
+ }
+ return maxidx;
+ }
+
+ //
+ // update key value cache
+ //
+ update_kv_cache(feed: any, outputs: any) {
+ for (const name in outputs) {
+ if (name.startsWith('present')) {
+ let newName = name.replace('present', 'past_key_values');
+ // dispose previous gpu buffers
+ const t = feed[newName];
+ if (t.location === 'gpu-buffer') {
+ t.dispose();
+ }
+ feed[newName] = outputs[name];
+ }
+ }
+ }
+
+ //
+ // tell generate to stop()
+ //
+ abort() {
+ this.stop = true;
+ }
+
+ //
+ // prefill prompt and generate tokens, greedy search only
+ //
+ async generate(tokens: any, callback: Function, options: any) {
+ const max_tokens = options.max_tokens || 256;
+ const feed = this.feed;
+ const input_ids = new ort.Tensor('int64', BigInt64Array.from(tokens.map(BigInt)), [1, tokens.length]);
+ // @ts-ignore
+ feed['input_ids'] = input_ids;
+ this.stop = false;
+
+ // @ts-ignore
+ this.output_tokens.push(...input_ids.data);
+
+ let last_token = 0n;
+ let seqlen = this.output_tokens.length;
+ const input_len = input_ids.size;
+
+ if (this.need_position_ids) {
+ // @ts-ignore
+ feed['position_ids'] = new ort.Tensor('int64', BigInt64Array.from({ length: input_len }, (_, i) => BigInt(seqlen - input_len + i)), [1, input_len]);
+ }
+
+ // @ts-ignore
+ while (last_token != this.eos && last_token != 32007 && seqlen < max_tokens && !this.stop) {
+ seqlen = this.output_tokens.length;
+ // @ts-ignore
+ feed['attention_mask'] = new ort.Tensor('int64', BigInt64Array.from({ length: seqlen }, () => 1n), [1, seqlen]);
+ // @ts-ignore
+ const outputs = await this.sess.run(feed);
+ last_token = BigInt(this.argmax(outputs.logits));
+ // @ts-ignore
+ this.output_tokens.push(last_token);
+ if (callback && !this.profiler) {
+ callback(this.output_tokens);
+ }
+ this.update_kv_cache(feed, outputs);
+ // @ts-ignore
+ feed['input_ids'] = new ort.Tensor('int64', BigInt64Array.from([last_token]), [1, 1]);
+ if (this.need_position_ids) {
+ // @ts-ignore
+ feed['position_ids'] = new ort.Tensor('int64', BigInt64Array.from([BigInt(seqlen)]), [1, 1]);
+ }
+ }
+ if (this.profiler) {
+ // @ts-ignore
+ this.sess.endProfiling();
+ }
+ return this.output_tokens;
+ }
+}
\ No newline at end of file
diff --git a/src/services/phi.ts b/src/services/phi.ts
new file mode 100644
index 0000000..28730bf
--- /dev/null
+++ b/src/services/phi.ts
@@ -0,0 +1,109 @@
+import { env, AutoTokenizer } from '@xenova/transformers';
+import { LLM } from './llm.js';
+
+const MODELS: any = {
+ "phi3": { name: "phi3", path: "microsoft/Phi-3-mini-4k-instruct-onnx-web", externaldata: true },
+ "phi3dev": { name: "phi3dev", path: "schmuell/Phi-3-mini-4k-instruct-onnx-web", externaldata: true },
+};
+
+function getConfig() {
+ const query = window.location.search.substring(1);
+ let config: any = {
+ model: "phi3",
+ provider: "webgpu",
+ profiler: 0,
+ verbose: 0,
+ threads: 1,
+ show_special: 0,
+ csv: 0,
+ max_tokens: 9999,
+ local: 0,
+ }
+ let vars = query.split("&");
+ for (var i = 0; i < vars.length; i++) {
+ let pair = vars[i].split("=");
+ if (pair[0] in config) {
+ const key = pair[0];
+ const value = decodeURIComponent(pair[1]);
+ if (typeof config[key] == "number") {
+ config[key] = parseInt(value);
+ }
+ else {
+ config[key] = value;
+ }
+ } else if (pair[0].length > 0) {
+ throw new Error("unknown argument: " + pair[0]);
+ }
+ }
+ if (MODELS[config.model] !== undefined) {
+ config.model = MODELS[config.model];
+ }
+ return config;
+}
+
+
+const config = getConfig();
+
+// setup for transformers.js tokenizer
+env.localModelPath = 'models';
+env.allowRemoteModels = config.local == 0;
+env.allowLocalModels = config.local == 1;
+
+let tokenizer: any = undefined;
+
+const llm = new LLM();
+
+function token_to_text(tokenizer: any, tokens: any, startidx: any) {
+ const txt = tokenizer.decode(tokens.slice(startidx), { skip_special_tokens: config.show_special != 1, });
+ return txt;
+}
+
+export async function Query(continuation: any, query: any, cb: Function) {
+ let prompt = (continuation) ? query : `<|system|>\nYou are a friendly assistant.<|end|>\n<|user|>\n${query}<|end|>\n<|assistant|>\n`;
+
+ const { input_ids } = await tokenizer(prompt, { return_tensor: false, padding: true, truncation: true });
+
+ // clear caches
+ // TODO: use kv_cache for continuation
+ llm.initilize_feed();
+
+ const start_timer = performance.now();
+ const output_index = llm.output_tokens.length + input_ids.length;
+ const output_tokens = await llm.generate(input_ids, (output_tokens: any) => {
+ if (output_tokens.length == input_ids.length + 1) {
+ // time to first token
+ const took = (performance.now() - start_timer) / 1000;
+ console.log(`time to first token in ${took.toFixed(1)}sec, ${input_ids.length} tokens`);
+ }
+ cb(token_to_text(tokenizer, output_tokens, output_index));
+ }, { max_tokens: config.max_tokens });
+
+ const took = (performance.now() - start_timer) / 1000;
+ cb(token_to_text(tokenizer, output_tokens, output_index));
+ const seqlen = output_tokens.length - output_index;
+ console.log(`${seqlen} tokens in ${took.toFixed(1)}sec, ${(seqlen / took).toFixed(2)} tokens/sec`);
+}
+
+//
+// Load the model and tokenizer
+//
+export async function Init(hasFP16: boolean) {
+ try {
+ tokenizer = await AutoTokenizer.from_pretrained(config.model.path);
+
+ console.log("Loading model...");
+ await llm.load(config.model, {
+ provider: config.provider,
+ profiler: config.profiler,
+ verbose: config.verbose,
+ local: config.local,
+ max_tokens: config.max_tokens,
+ hasFP16: hasFP16,
+ });
+ console.log("Ready.");
+ } catch (error) {
+ console.log(error);
+ }
+}
+
+// e
\ No newline at end of file
diff --git a/src/services/utils.ts b/src/services/utils.ts
new file mode 100644
index 0000000..15d7714
--- /dev/null
+++ b/src/services/utils.ts
@@ -0,0 +1,6 @@
+export function bytesToSize(bytes: number) {
+ const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
+ if (bytes === 0) return '0 Byte';
+ const i = Math.floor(Math.log(bytes) / Math.log(1024));
+ return parseFloat((bytes / Math.pow(1024, i)).toFixed(2)) + ' ' + sizes[i];
+}
\ No newline at end of file
diff --git a/src/services/whisper.ts b/src/services/whisper.ts
new file mode 100644
index 0000000..3e48c34
--- /dev/null
+++ b/src/services/whisper.ts
@@ -0,0 +1,48 @@
+let whisperWorker: Worker;
+
+// @ts-ignore
+import WhisperWorker from './ai-worker?worker'
+
+export async function loadTranscriber(model: "tiny" | "base"): Promise {
+ whisperWorker = new WhisperWorker();
+ whisperWorker.postMessage({
+ type: "load",
+ model: model || "tiny",
+ });
+}
+
+export function doLocalWhisper(audioFile: Blob, model: "tiny" | "base"): Promise {
+ return new Promise((resolve) => {
+ const fileReader = new FileReader();
+ fileReader.onloadend = async () => {
+ const audioCTX = new AudioContext({
+ sampleRate: 16000,
+ });
+ const arrayBuffer = fileReader.result as ArrayBuffer;
+ const audioData = await audioCTX.decodeAudioData(arrayBuffer);
+
+ let audio = audioData.getChannelData(0);
+
+ whisperWorker.onmessage = async (e) => {
+ if (e.data.type === "transcribe") {
+ resolve(e.data);
+ }
+ else if (e.data.type === "transcribe-interim") {
+ window.dispatchEvent(new CustomEvent('interim-transcription', {
+ detail: {
+ message: e.data.transcription,
+ }
+ }));
+ }
+ }
+
+ whisperWorker.postMessage({
+ type: "transcribe",
+ blob: audio,
+ model: model || "tiny",
+ })
+
+ };
+ fileReader.readAsArrayBuffer(audioFile);
+ })
+}
\ No newline at end of file
diff --git a/src/styles/global.css b/src/styles/global.css
new file mode 100644
index 0000000..2a6d7ae
--- /dev/null
+++ b/src/styles/global.css
@@ -0,0 +1,18 @@
+
+/*
+ This file is used for all of your global styles and CSS variables.
+ Check here https://developer.mozilla.org/en-US/docs/Web/CSS/Using_CSS_custom_properties for more info on using CSS variables.
+*/
+:root {
+ --font-family: sans-serif;
+}
+
+html, body {
+ font-family: var(--font-family);
+ padding: 0;
+ margin: 0;
+
+ height: 100%;
+
+ overflow-y: hidden;
+}
\ No newline at end of file
diff --git a/src/styles/shared-styles.ts b/src/styles/shared-styles.ts
new file mode 100644
index 0000000..b6091b6
--- /dev/null
+++ b/src/styles/shared-styles.ts
@@ -0,0 +1,16 @@
+import { css } from 'lit';
+
+// these styles can be imported from any component
+// for an example of how to use this, check /pages/about-about.ts
+export const styles = css`
+ @media(min-width: 1000px) {
+ sl-card {
+ max-width: 70vw;
+ }
+ }
+
+ main {
+ margin-top: 34px;
+ padding: 12px;
+ }
+`;
\ No newline at end of file
diff --git a/swa-cli.config.json b/swa-cli.config.json
new file mode 100644
index 0000000..fbb577f
--- /dev/null
+++ b/swa-cli.config.json
@@ -0,0 +1,12 @@
+{
+ "$schema": "https://aka.ms/azure/static-web-apps-cli/schema",
+ "configurations": {
+ "pwa-starter": {
+ "appLocation": ".",
+ "outputLocation": "dist",
+ "appBuildCommand": "npm run build --if-present",
+ "run": "npm start",
+ "appDevserverUrl": "http://localhost:3000"
+ }
+ }
+}
\ No newline at end of file
diff --git a/tsconfig.json b/tsconfig.json
new file mode 100644
index 0000000..08abbc4
--- /dev/null
+++ b/tsconfig.json
@@ -0,0 +1,28 @@
+{
+ "compilerOptions": {
+ "module": "esnext",
+ "target": "esnext",
+ "lib": ["es2017", "esnext", "dom", "dom.iterable"],
+ "declaration": true,
+ "emitDeclarationOnly": true,
+ "skipLibCheck": true,
+ "outDir": "./types",
+ "strict": true,
+ "noUnusedLocals": true,
+ "noUnusedParameters": true,
+ "noImplicitReturns": true,
+ "noFallthroughCasesInSwitch": true,
+ "moduleResolution": "bundler",
+ "allowSyntheticDefaultImports": true,
+ "experimentalDecorators": true,
+ "forceConsistentCasingInFileNames": true,
+ "useDefineForClassFields": false,
+ "isolatedModules": true,
+ "types": [
+ "vite-plugin-pwa/client",
+ "vite/client"
+ ]
+ },
+ "include": ["src/**/*.ts", "router.d.ts"],
+ "exclude": ["node_modules"]
+}
diff --git a/vite.config.ts b/vite.config.ts
new file mode 100644
index 0000000..a92dbb0
--- /dev/null
+++ b/vite.config.ts
@@ -0,0 +1,32 @@
+import { defineConfig } from 'vite';
+import { VitePWA } from 'vite-plugin-pwa';
+
+// https://vitejs.dev/config/
+export default defineConfig({
+ base: "/",
+ build: {
+ sourcemap: true,
+ assetsDir: "code",
+ target: ["esnext"],
+ cssMinify: true,
+ lib: false
+ },
+ plugins: [
+ VitePWA({
+ strategies: "injectManifest",
+ injectManifest: {
+ swSrc: 'public/sw.js',
+ swDest: 'dist/sw.js',
+ globDirectory: 'dist',
+ globPatterns: [
+ '**/*.{html,js,css,json,png}',
+ ],
+ },
+ injectRegister: false,
+ manifest: false,
+ devOptions: {
+ enabled: true
+ }
+ })
+ ]
+})