V2 and OSS prep for dexter (#6)

Co-authored-by: Riley Tomasek <[email protected]>

.env.example

@@ -0,0 +1,15 @@
+# ------------------------------------------------------------------------------
+# This is an example .env file.
+#
+# All of these environment vars must be defined either in your environment or in
+# a local .env file in order to run the examples for this project.
+# ------------------------------------------------------------------------------
+
+OPENAI_API_KEY=
+
+# Most examples will work with a free-tier index, but you'll need a paid hybrid-compatible index to run the chatbot example
+PINECONE_API_KEY=
+PINECONE_BASE_URL=
+
+# Optional; only needed if using a hybrid datastore
+#SPLADE_SERVICE_URL=

.eslintignore

@@ -0,0 +1,2 @@
+dist
+.next

packages/model/.eslintrc → .eslintrc

@@ -1,4 +1,5 @@
 {
+  "root": true,
   "extends": ["@dexaai/eslint-config", "@dexaai/eslint-config/node"],
   "rules": {
     "no-console": "off"

.github/workflows/main.yml

@@ -0,0 +1,48 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    name: Test Node.js ${{ matrix.node-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        node-version:
+          - 21
+          - 20
+          - 18
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2
+        id: pnpm-install
+        with:
+          version: 8
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run test
+        run: pnpm run test

.gitignore

@@ -11,6 +11,10 @@ node_modules
 temp
 todo.md
 .env*
+!.env.example
 .dev*
 .env.local
 .local-files
+.vscode
+.next
+.vercel

.npmrc

@@ -0,0 +1 @@
+enable-pre-post-scripts=true

.prettierignore

@@ -6,3 +6,4 @@
 **/build
 **/dbschema
 **/public
+.next

contributing.md

@@ -0,0 +1,66 @@
+# Contributing
+
+Suggestions and pull requests are very welcome. 😊
+
+## Development
+
+To develop the project locally, you'll need `node >= 18` and `pnpm >= 8`.
+
+```bash
+git clone https://github.com/dexaai/dexter
+cd dexter
+pnpm i
+```
+
+You can now run the `tsc` dev server to automatically recompile the project whenever you make changes:
+
+```bash
+pnpm dev
+```
+
+## Testing
+
+You can run the test suite via:
+
+```bash
+pnpm test
+```
+
+Or just the [Vitest](https://vitest.dev) unit tests via:
+
+```bash
+pnpm test:unit
+```
+
+## Examples
+
+To run the included examples, clone this repo, run `pnpm install`, set up your `.env` file, and then run an example file using `tsx`.
+
+For example:
+
+```bash
+npx tsx examples/basic.ts
+```
+
+## Docs
+
+The `./docs/pages/docs` directory is autogenerated via [typedoc](https://typedoc.org) and [typedoc-plugin-markdown](https://github.com/tgreyuk/typedoc-plugin-markdown/tree/next/packages/typedoc-plugin-markdown).
+
+To rebuild the docs, run:
+
+```bash
+pnpm run docs
+```
+
+Please don't run this command when submitting PRs to keep them uncluttered, unless you're making changes to the docs themselves.
+
+The docs folder is a normal [Next.js](https://nextjs.org) pages app built using [Nextra](https://nextra.site/).
+
+You can run the docs dev server to preview your changes via:
+
+```bash
+cd docs
+pnpm dev
+```
+
+The docs are automatically deployed to [Vercel](https://vercel.com).

docs/bin/transform-docs.ts

@@ -0,0 +1,113 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { globby } from 'globby';
+import pMap from 'p-map';
+
+/**
+ * Cleans up TypeDoc's a automated output to make it more suitable for Nextra.
+ *
+ * Nextra insists on title-casing identifiers, so we need to manually set their
+ * display names.
+ */
+async function main() {
+  const docsDir = path.join('docs', 'pages', 'docs');
+
+  const docs = (
+    await globby('**/*.md', {
+      cwd: docsDir,
+    })
+  )
+    .map((relativePath) => {
+      const absolutePath = path.join(docsDir, relativePath);
+
+      return {
+        relativePath,
+        absolutePath,
+      };
+    })
+    .filter(Boolean)
+    .sort((a, b) => a.relativePath.localeCompare(b.relativePath));
+
+  const metaMap: Record<string, string[]> = {};
+
+  await pMap(
+    docs,
+    async (doc) => {
+      console.log(`processing ${doc.relativePath}`);
+      const parts = doc.relativePath.split('/');
+      if (parts.length <= 1) {
+        return;
+      }
+
+      const bucket = parts.slice(0, -1).join('/');
+      if (!metaMap[bucket]) {
+        metaMap[bucket] = [];
+      }
+
+      metaMap[bucket].push(doc.relativePath);
+    },
+    {
+      concurrency: 32,
+    }
+  );
+
+  {
+    // top-level nextra _meta.json file
+    const docsMeta = {
+      exports: 'API Reference',
+      namespaces: 'Namespaces',
+      classes: 'Classes',
+      interfaces: 'Interfaces',
+      functions: 'Functions',
+      'type-aliases': {
+        title: 'Type Aliases',
+        display: 'hidden',
+      },
+      README: {
+        display: 'hidden',
+      },
+    };
+
+    await fs.writeFile(
+      path.join(docsDir, '_meta.json'),
+      JSON.stringify(docsMeta, null, 2),
+      'utf-8'
+    );
+  }
+
+  // sub nextra _meta.json files
+  await pMap(
+    Object.keys(metaMap),
+    async (key) => {
+      const values = metaMap[key]
+        .map((value) => value.split('/').slice(-1)[0].split('.')[0].trim())
+        .filter(Boolean)
+        .sort((a, b) => a.localeCompare(b));
+
+      if (values.length <= 1) {
+        return;
+      }
+
+      const meta = values.reduce(
+        (acc, value) => ({
+          ...acc,
+          [value]: value,
+        }),
+        {}
+      );
+
+      const destDir = path.join(docsDir, key);
+
+      await fs.writeFile(
+        path.join(path.join(destDir, '_meta.json')),
+        JSON.stringify(meta, null, 2),
+        'utf-8'
+      );
+    },
+    {
+      concurrency: 4,
+    }
+  );
+}
+
+main();

docs/next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/basic-features/typescript for more information.

docs/next.config.mjs

@@ -0,0 +1,29 @@
+import nextra from 'nextra';
+
+const withNextra = nextra({
+  theme: 'nextra-theme-docs',
+  themeConfig: './theme.config.tsx',
+  staticImage: true,
+  latex: false,
+  flexsearch: false,
+  defaultShowCopyCode: true,
+});
+
+export default withNextra({
+  reactStrictMode: false,
+  eslint: {
+    ignoreDuringBuilds: true,
+  },
+  rewrites() {
+    return [
+      {
+        source: '/guide',
+        destination: '/guide/install',
+      },
+      {
+        source: '/docs',
+        destination: '/docs/exports',
+      },
+    ];
+  },
+});

docs/package.json

@@ -0,0 +1,18 @@
+{
+  "private": true,
+  "name": "dexter-docs",
+  "scripts": {
+    "dev": "next",
+    "build": "next build",
+    "start": "next start"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "@vercel/analytics": "^1.1.1",
+    "next": "^14.0.1",
+    "nextra": "^2.13.2",
+    "nextra-theme-docs": "^2.13.2",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0"
+  }
+}

docs/pages/_app.mdx

@@ -0,0 +1,12 @@
+import { Analytics } from '@vercel/analytics/react'
+
+export default function App({ Component, pageProps }) {
+  return (
+    <>
+      <Component {...pageProps} />
+
+      <Analytics />
+    </>
+
+)
+}