Fix and add cron

Author: jmduke

app/api/crons/refresh/route.ts

@@ -0,0 +1,46 @@
+import fetch from "@/lib/data";
+import { db } from "@/lib/db/connection";
+import { reify } from "@/lib/db/domains";
+import { sql } from "kysely";
+import pino from "pino";
+
+const logger = pino({
+    name: "cron-refresh",
+});
+
+const RAW_QUERY = sql<{
+    domain: string;
+}>`
+    select domain from tranco TABLESAMPLE system (0.01)
+`;
+
+const getRandomDomains = async () => {
+	const domains = await RAW_QUERY.execute(db);
+    return domains.rows.map((row) => row.domain);
+};
+
+const MAXIMUM_DOMAINS = 10;
+
+export async function GET(
+	request: Request,
+	context: {
+		params: {
+			domain: string;
+		};
+	},
+) {
+    const domains = await getRandomDomains();
+    const selectedDomains = domains.slice(0, MAXIMUM_DOMAINS);
+    await Promise.all(selectedDomains.map(async (domain) => {
+        logger.info({
+            message: "refresh.started",
+            domain: domain,
+        });
+        const rawResponse = await fetch(domain);
+		await reify(domain, rawResponse);
+	}));
+
+	return Response.json({
+        domains: selectedDomains,
+    });
+}

app/api/domains/[domain]/route.ts

@@ -20,7 +20,6 @@ export async function GET(
 ) {
 	const rawResponse = await fetch(context.params.domain);
 	await reify(context.params.domain, rawResponse);
-	console.log(rawResponse.detected_technologies);
 
 	return Response.json({
 		domain: context.params.domain,

app/technology/[identifier]/page.tsx

@@ -80,14 +80,15 @@ export default async function TechnologyPage({
   const trancoCount = process.env.DISABLE_DATABASE
     ? 0
     : await db
-        .selectFrom("tranco")
+        .selectFrom("affiliations")
         .innerJoin(
           "detected_technologies",
-          "tranco.domain",
+          "affiliations.domain",
           "detected_technologies.domain"
         )
         .where("detected_technologies.technology", "=", params.identifier)
-        .select(db.fn.count("tranco.domain").as("count"))
+        .where("affiliations.identifier", "=", "tranco")
+        .select(db.fn.count("affiliations.domain").as("count"))
         .executeTakeFirst()
         .then((result) => Number(result?.count || 0));
 

lib/parsers/html.ts

@@ -28,7 +28,7 @@ const GENERIC_SOCIAL_MEDIA_PROVIDER = (html: string) => {
 				const username = match[match.length - 1];
 				return [
 					{
-						identifier: potentialMatch?.identifier,
+						identifier: potentialMatch?.identifier as string,
 						metadata: {
 							username: username.split("?")[0],
 						},
@@ -119,7 +119,7 @@ const JSONLD_RULE = (html: string) => {
 							);
 							if (service) {
 								return {
-									identifier: service.identifier.split("?")[0],
+									identifier: service.identifier.split("?")[0] as string,
 									metadata: {
 										username: url.split("/").pop() || "",
 									},
@@ -199,7 +199,7 @@ const SUBDOMAIN_RULE = (html: string, domain: string) => {
 
 const RULES: ((html: string, domain: string) => DetectedTechnology[])[] = [
 	...Object.values(REGISTRY).map((service) => {
-		return (html: string, domain: string) => {
+		return (html: string) => {
 			const potentialMatches = service.substrings?.filter((substring) =>
 				html.includes(substring),
 			);

vercel.json

@@ -0,0 +1,8 @@
+{
+    "crons": [
+      {
+        "path": "/api/crons/refresh",
+        "schedule": "* * * * *"
+      }
+    ]
+}