fix(orama): remove INTERNAL packages from index, and chunk reindexing

api/src/orama.rs

@@ -46,12 +46,15 @@ impl OramaClient {
 
   #[instrument(name = "OramaClient::upsert_package", skip(self))]
   pub fn upsert_package(&self, package: &Package, meta: &PackageVersionMeta) {
-    if package.version_count == 0
-      || package.is_archived
-      || package.description.starts_with("INTERNAL")
-    {
+    if package.version_count == 0 || package.is_archived {
       return;
     }
+
+    if package.description.starts_with("INTERNAL") {
+      self.delete_package(&package.scope, &package.name);
+      return;
+    }
+
     let id = format!("@{}/{}", package.scope, package.name);
     let score = package
       .latest_version

tools/orama_package_reindex.ts

@@ -1,6 +1,7 @@
 // Copyright 2024 the JSR authors. All rights reserved. MIT license.
 import type { List, Package } from "../frontend/utils/api_types.ts";
 import type { OramaPackageHit } from "../frontend/util.ts";
+import { chunk } from "jsr:@std/collections";
 
 const index = Deno.env.get("ORAMA_PACKAGE_INDEX_ID");
 const auth = Deno.env.get("ORAMA_PACKAGE_PRIVATE_API_KEY");
@@ -54,17 +55,19 @@ const entries: OramaPackageHit[] = packages
     id: `@${entry.scope}/${entry.name}`,
   }));
 
-const res2 = await fetch(`${ORAMA_URL}/${index}/notify`, {
-  method: "POST",
-  headers: {
-    authorization: `Bearer ${auth}`,
-    "Content-Type": "application/json",
-  },
-  body: JSON.stringify({ "upsert": entries }),
-});
-if (res2.status !== 200) {
-  console.log(await res2.text());
-  throw res2;
+for (const entriesChunk of chunk(entries, 1000)) {
+  const res2 = await fetch(`${ORAMA_URL}/${index}/notify`, {
+    method: "POST",
+    headers: {
+      authorization: `Bearer ${auth}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({ "upsert": entriesChunk }),
+  });
+  if (res2.status !== 200) {
+    console.log(await res2.text());
+    throw res2;
+  }
 }
 
 // deploy the index