Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add websearch controls for assistants (#812)
* remove query modifiers from generateQuery * Add backend for assistant RAG * Add front-end for updating RAG assistant * enable web parser to return plaintext directly for matching headers * Update websearch flow for handling assistant rag preferences * Add our old blocklist to .env.template * Enable websearch to run on messages depending on assistant requirements * reorganized imports * Rename vars * use projection * Add environment variable for assistant rag * fix assistant rag on runwebsearch * fix styling if rag is disabled * make sure we always omit credentials when fetching web pages * Add new checks for SSRF, with a new env var `ENABLE_LOCAL_FETCH` * Use DNS to check if the links are local or not * Add a websearch indicator * Add more tags to parser * Add indicators * Display RAG options in settings view * ui * fix rag detection * bit more spacing * fix button position in assistant form * wording (mainly) * reduce number of tags * Bump max URLs from 3 to 10 * add ul and ol to parseWeb * change splitting string * link style * wording * add feedback link * Update src/routes/settings/(nav)/assistants/[assistantId]/+page.svelte Co-authored-by: Mishig <[email protected]> * Update src/routes/settings/(nav)/assistants/[assistantId]/+page.svelte Co-authored-by: Mishig <[email protected]> * Update src/routes/assistants/+page.svelte Co-authored-by: Mishig <[email protected]> * Update src/routes/settings/(nav)/assistants/[assistantId]/+page.svelte Co-authored-by: Mishig <[email protected]> * Update src/lib/components/chat/ChatWindow.svelte Co-authored-by: Mishig <[email protected]> * Update src/routes/settings/(nav)/assistants/[assistantId]/+page.svelte Co-authored-by: Mishig <[email protected]> * Update src/lib/components/AssistantSettings.svelte Co-authored-by: Mishig <[email protected]> * lint * throw error if not a string * simplify rag check --------- Co-authored-by: Mishig <[email protected]> Co-authored-by: Victor Mustar <[email protected]>
- Loading branch information
3 people
committed
Mar 14, 2024
1 parent
eb071be
commit 3f5871c
Showing
22 changed files
with
469 additions
and
84 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| import { isURLLocal } from "./isURLLocal"; | ||
| import { describe, expect, it } from "vitest"; | ||
|
|
||
| describe("isURLLocal", async () => { | ||
| it("should return true for localhost", async () => { | ||
| expect(await isURLLocal(new URL("http://localhost"))).toBe(true); | ||
| }); | ||
| it("should return true for 127.0.0.1", async () => { | ||
| expect(await isURLLocal(new URL("http://127.0.0.1"))).toBe(true); | ||
| }); | ||
| it("should return true for 127.254.254.254", async () => { | ||
| expect(await isURLLocal(new URL("http://127.254.254.254"))).toBe(true); | ||
| }); | ||
| it("should return false for huggingface.co", async () => { | ||
| expect(await isURLLocal(new URL("https://huggingface.co/"))).toBe(false); | ||
| }); | ||
| it("should return true for 127.0.0.1.nip.io", async () => { | ||
| expect(await isURLLocal(new URL("http://127.0.0.1.nip.io"))).toBe(true); | ||
| }); | ||
| it("should fail on ipv6", async () => { | ||
| await expect(isURLLocal(new URL("http://[::1]"))).rejects.toThrow(); | ||
| }); | ||
| it("should fail on ipv6 --1.sslip.io", async () => { | ||
| await expect(isURLLocal(new URL("http://--1.sslip.io"))).rejects.toThrow(); | ||
| }); | ||
| it("should fail on invalid domain names", async () => { | ||
| await expect( | ||
| isURLLocal(new URL("http://34329487239847329874923948732984.com/")) | ||
| ).rejects.toThrow(); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| import { Address6, Address4 } from "ip-address"; | ||
|
|
||
| import dns from "node:dns"; | ||
|
|
||
| export async function isURLLocal(URL: URL): Promise<boolean> { | ||
| const isLocal = new Promise<boolean>((resolve, reject) => { | ||
| dns.lookup(URL.hostname, (err, address, family) => { | ||
| if (err) { | ||
| reject(err); | ||
| } | ||
| if (family === 4) { | ||
| const addr = new Address4(address); | ||
| resolve(addr.isInSubnet(new Address4("127.0.0.0/8"))); | ||
| } else if (family === 6) { | ||
| const addr = new Address6(address); | ||
| resolve( | ||
| addr.isLoopback() || addr.isInSubnet(new Address6("::1/128")) || addr.isLinkLocal() | ||
| ); | ||
| } else { | ||
| reject(new Error("Unknown IP family")); | ||
| } | ||
| }); | ||
| }); | ||
|
|
||
| return isLocal; | ||
| } |
Oops, something went wrong.