From 2fa811792410a54ee8813a3adfeb03406037ca44 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 19 Apr 2023 23:25:39 +0200 Subject: [PATCH] wip: checkpoint --- docker-compose.yml | 2 +- indexer/fetch.ts | 50 ++++++------ indexer/fsm/machine.test.ts | 2 +- indexer/fsm/machine.ts | 5 +- indexer/fsm/services.ts | 70 +++++++---------- indexer/fsm/types.ts | 43 ----------- indexer/main.ts | 54 +++++-------- indexer/queries.ts | 8 +- indexer/queue/handlers.ts | 76 ------------------- .../queue/processes/fetching_commentables.ts | 56 ++++++++++++++ indexer/queue/processes/fetching_comments.ts | 54 +++++++++++++ .../queue/processes/storing_commentables.ts | 5 ++ indexer/queue/processes/storing_comments.ts | 12 +-- indexer/store.ts | 4 +- lib/types/comment.ts | 6 +- lib/types/index.ts | 6 +- lib/types/indexer.ts | 47 ++++++++++++ lib/types/job.ts | 8 ++ lib/types/query.ts | 23 +++--- yarn.lock | 5 ++ 20 files changed, 281 insertions(+), 255 deletions(-) delete mode 100644 indexer/queue/handlers.ts create mode 100644 lib/types/indexer.ts create mode 100644 lib/types/job.ts diff --git a/docker-compose.yml b/docker-compose.yml index 97d1aaa..e2fc879 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: - .env environment: <<: *psql-env - <<: *redis-env +# <<: *redis-env # NODE_ENV: # command: "yarn index && sleep 500" volumes: diff --git a/indexer/fetch.ts b/indexer/fetch.ts index 82f278e..f826732 100644 --- a/indexer/fetch.ts +++ b/indexer/fetch.ts @@ -1,5 +1,5 @@ import {GraphQLClient} from "graphql-request"; -import {issueCommentsQuery} from "./queries.js"; +import {issueCommentsQuery} from "./queries.ts"; import { Comment, Commentable, @@ -7,10 +7,10 @@ import { CommentableType, CommentsPage, IssueCommentsResponse, - nextQueryArgs, + QueryVars, ResponseComment, ResponseIssue -} from "../lib/types/index.js"; +} from "../lib/types/index.ts"; import {FetchResult} from "../lib/types/fetch"; // TODO: move @@ -63,21 +63,20 @@ function byCreatedAt(a: Timestamped, b: Timestamped) { return new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(); } -export async function fetchIssueComments(client: GraphQLClient, { - owner, - name, - issues -}: nextQueryArgs): Promise { +export async function fetchIssueComments(client: GraphQLClient, queryVars: QueryVars): Promise { console.log("fetchIssueComments") + + const {owner, name, pageSize} = queryVars; + const pageVars = queryVars.pageVars ?? {}; const issuesResponse: IssueCommentsResponse = await client.request(issueCommentsQuery, { owner, name, - maxIssues: issues?.max, - maxComments: issues?.comments.max, - afterIssue: issues?.comments.after, - beforeIssue: issues?.comments.before, - afterComment: issues?.comments.after, - }) + maxIssues: pageSize.commentables, + maxComments: pageSize.comments, + afterIssue: pageVars?.commentables?.after, + beforeIssue: pageVars?.commentables?.before, + afterComment: pageVars?.comments?.after, + }); console.log("got issue comments resopnse") const {nodes: responseIssues, pageInfo: {hasNextPage, endCursor}} = issuesResponse.repository.issues @@ -90,18 +89,24 @@ export async function fetchIssueComments(client: GraphQLClient, { }; } + // collect & normalize all issues const normalizedIssues: Commentable[] = responseIssues.map(issue => ({ id: issue.id, type: CommentableType.ISSUE, })); - const [issuesWithoutNext, issuesWithNext] = partition(responseIssues, hasMoreComments) - // TODO: refactor - separate normalized issues from normalized comments... - const normalizedIssueComments = issuesWithoutNext.flatMap((issue: ResponseIssue) => issue.comments.nodes) - .map(normalizeResponseComment) + // collect & normalize `Comment`s from issues w/o more comments + const normalizedIssueComments = responseIssues + .flatMap((issue: ResponseIssue) => issue.comments.nodes) + .map(normalizeResponseComment); + + // partition issues into w/ and w/o more comments + // TODO: refactor + const [_, issuesWithMoreComments] = partition(responseIssues, hasMoreComments); + // create `CommentsPage`s from issues w/ more comments // TODO: refactor (normalize) - const pendingCommentsPages = issuesWithNext.map(issue => { + const nextCommentsPages = issuesWithMoreComments.map(issue => { const { pageInfo: {endCursor} } = issue.comments; @@ -110,16 +115,15 @@ export async function fetchIssueComments(client: GraphQLClient, { commentableId: issue.id, afterComment: endCursor, } as CommentsPage; - }) + }); return { - // TODO: normalize issues (drop unused properties) commentables: normalizedIssues, // sort issueComments by createdAt in descending order (i.e. newest first) comments: normalizedIssueComments.sort(byCreatedAt), next: { commentables: nextIssuePage, - comments: pendingCommentsPages, + comments: nextCommentsPages, }, }; } @@ -129,7 +133,7 @@ export async function fetchCommentsPage(client: GraphQLClient, page: CommentsPag name, PRs, issues, -}: nextQueryArgs): Promise { +}: QueryVars): Promise { const {commentableId, afterComment} = page; // issues: { diff --git a/indexer/fsm/machine.test.ts b/indexer/fsm/machine.test.ts index a8c9fde..9fb8f84 100644 --- a/indexer/fsm/machine.test.ts +++ b/indexer/fsm/machine.test.ts @@ -1,8 +1,8 @@ import {interpret} from "xstate"; import newUpdateMachine from "./machine"; -import {Context} from "./types"; import {GraphQLClient} from "graphql-request"; import DoneCallback = jest.DoneCallback; +import {Context} from "../../lib/types/indexer"; jest.mock('graphql-request'); diff --git a/indexer/fsm/machine.ts b/indexer/fsm/machine.ts index 0b0595c..a387aef 100644 --- a/indexer/fsm/machine.ts +++ b/indexer/fsm/machine.ts @@ -5,8 +5,9 @@ import { upsertAll, logError, logUpdated -} from "./services.js"; -import {Context} from "./types.js"; +} from "./services.ts"; + +import {Context} from "../../lib/types/indexer"; // TODO: decompose into multiple machines export function newUpdateMachine(context: Context) { diff --git a/indexer/fsm/services.ts b/indexer/fsm/services.ts index a9c16c3..8216ee9 100644 --- a/indexer/fsm/services.ts +++ b/indexer/fsm/services.ts @@ -1,8 +1,6 @@ -import Queue from "bull"; - -import {Context} from "./types.js"; -import {Comment, CommentsPage, Commentable} from "../../lib/types/index.js"; -import {fetchIssueComments} from "../fetch.js"; +import {Comment, Commentable, CommentablesPage, CommentableType, CommentsPage} from "../../lib/types/index.ts"; +import {Context} from "../../lib/types/indexer"; +import {Job} from "../../lib/types/job"; export async function logUpdated(ctx: Context, event: any) { console.log("updated state!"); @@ -13,55 +11,41 @@ export async function logError(ctx: Context, event: any) { } export async function queueAll(ctx: Context, event: any) { - const {queryVars: {owner, name, pageSize}} = ctx; - const {commentables, comments, next} = await fetchIssueComments(ctx.clients.github, { - owner, name, - PRs: { - max: pageSize.commentables, - comments: {max: pageSize.comments} - }, - issues: { - max: pageSize.commentables, - comments: {max: pageSize.comments} - }, - }); + const {queues: {fetching, storing}} = ctx; + + // TODO: move? + const firstIssuePage = {commentableType: CommentableType.ISSUE,}; + const firstIssueJob: Job = {ctx, nextPage: firstIssuePage}; + fetching.commentables.add("enqueue", firstIssueJob); + + // TODO: PRs and reviews... + - const {queues} = ctx; - // Enqueue normalized commentables and comments - commentables.forEach((commentable: Commentable) => - queues.storing.commentables.add("enqueue", {commentable}) - ); - comments.forEach((comment: Comment) => - queues.storing.comments.add("enqueue", {comment}) - ); - // Enqueue CommentablePage and CommentPage objects for the next pages - if (next.commentables) { - queues.fetching.commentables.add("enqueue", {nextPage: next.commentables}); - } - if (next.comments) { - next.comments.forEach((commentPage: CommentsPage) => - queues.fetching.comments.add("enqueue", {commentPage}) - ); - } + const {commentables, comments} = ctx.processes.fetching; - // Process nextCommentables queue - const {storing, fetching} = ctx.processes; - queues.fetching.commentables.process( - fetching.commentables.concurrency, - fetching.commentables.scriptPath, + // Kick-off fetching commentables queue processing (dequeuing) + fetching.commentables.process( + commentables.concurrency, + commentables.scriptPath, ); - // Process nextComments queue - queues.fetching.comments.process( - fetching.comments.concurrency, - fetching.comments.scriptPath, + // Kick-off fetching comments queue processing (dequeuing) + fetching.comments.process( + comments.concurrency, + comments.scriptPath, ); } export async function upsertAll(ctx: Context, event: any) { + + // const {endpointURL} = ctx.gqlClientConfigs.postgraphile; const {commentables, comments} = ctx.processes.storing; + // TODO: add authentication + // postgraphile graphql client + // const client = new GraphQLClient(endpointURL); + // Process storing commentables queue ctx.queues.storing.commentables.process( commentables.concurrency, diff --git a/indexer/fsm/types.ts b/indexer/fsm/types.ts index 1610e9f..e69de29 100644 --- a/indexer/fsm/types.ts +++ b/indexer/fsm/types.ts @@ -1,43 +0,0 @@ -import {Queue} from "bull"; -import {GraphQLClient} from "graphql-request"; - -// TODO: consolidate w/ lib/types (?) -export interface Context { - clients: { - github: GraphQLClient, - postgraphile: GraphQLClient, - } - queryVars: { - owner: string; - name: string; - pageSize: { - commentables: number, - comments: number, - } - } - queues: { - fetching: { - commentables: Queue, - comments: Queue, - }, - storing: { - commentables: Queue, - comments: Queue, - } - } - processes: { - fetching: { - commentables: ProcessConfig, - comments: ProcessConfig, - }, - storing: { - commentables: ProcessConfig, - comments: ProcessConfig, - } - } -} - -export interface ProcessConfig { - concurrency: number; - scriptPath: string; -} \ No newline at end of file diff --git a/indexer/main.ts b/indexer/main.ts index 057b1ec..30c915f 100644 --- a/indexer/main.ts +++ b/indexer/main.ts @@ -1,9 +1,7 @@ -import Queue from "bull"; -import {GraphQLClient} from 'graphql-request'; import {interpret} from "xstate"; -import {newUpdateMachine} from "./fsm/machine.js"; -import {Context} from "./fsm/types.js"; -import {Comment, Commentable, CommentablesPage, CommentsPage} from "../lib/types/index.js"; +import {newUpdateMachine} from "./fsm/machine.ts"; + +import {Context} from "../lib/types/indexer"; // TODO: check `NODE_ENV` const defaults = { @@ -26,39 +24,27 @@ const commentablesPageSize = 3; const commentsPageSize = 100; // TODO: parameterize (?) -const storingCommentablesProcessPath = "./proceses/storing_commentables.js", - storingCommentsProcessPath = "./processes/storing_comments.js", - fetchCommentablesProcessPath = "./proceses/fetch_commentables.js", - fetchCommentsProcessPath = "./processes/fetch_comments.js", +// TODO: fix paths; weird relativeness; OS-agnostic +const storingCommentablesProcessPath = `${__dirname}/queue/processes/storing_commentables.ts`, + storingCommentsProcessPath = `${__dirname}/queue/processes/storing_comments.ts`, + fetchCommentablesProcessPath = `${__dirname}/queue/processes/fetching_commentables.ts`, + fetchCommentsProcessPath = `${__dirname}/queue/processes/fetching_comments.ts`, storingConcurrency = 2, fetchingConcurrency = 2 ; const [owner, name] = ghRepo.split("/") - -// github graphql client -const ghClient = new GraphQLClient(GITHUB_GRAPHQL_URL, { - headers: { - Authorization: `Bearer ${ghAccessToken}`, - }, -}); - -// TODO: add authentication -// postgraphile graphql client -const pgClient = new GraphQLClient(POSTGRAPHILE_URL); - -// TODO: replace with `Bull` queues -const storingCommentablesQueue = new Queue("pendingCommentables"); -const storingCommentsQueue = new Queue("pendingComments"); -const fetchingCommentablesQueue = new Queue("nextCommentables"); -const fetchingCommentsQueue = new Queue("nextComments"); - async function run() { const context: Context = { - clients: { - github: ghClient, - postgraphile: pgClient, + clientConfigs: { + github: { + endpointURL: GITHUB_GRAPHQL_URL, + classicAccessToken: ghAccessToken, + }, + postgraphile: { + endpointURL: POSTGRAPHILE_URL, + }, }, queryVars: { owner, name, @@ -69,12 +55,12 @@ async function run() { }, queues: { fetching: { - commentables: fetchingCommentablesQueue, - comments: fetchingCommentsQueue, + commentables: "nextCommentables", + comments: "nextComments", }, storing: { - commentables: storingCommentablesQueue, - comments: storingCommentsQueue, + commentables: "pendingCommentables", + comments: "pendingComments", }, }, processes: { diff --git a/indexer/queries.ts b/indexer/queries.ts index 69ffaec..cd67254 100644 --- a/indexer/queries.ts +++ b/indexer/queries.ts @@ -12,7 +12,7 @@ export const pullRequestCommentsQuery = gql` $beforeComment: String ) { repository(owner: $owner, name: $name) { - pullRequests(first: $maxPRs, after: $afterPR, before: $beforePR) { + pullRequests(first: $maxPRs, after: $afterPR, before: $beforePR, orderBy: DESC) { pageInfo { hasNextPage endCursor @@ -20,7 +20,7 @@ export const pullRequestCommentsQuery = gql` nodes { createdAt updatedAt - comments(first: $maxComments, after: $afterComment, before: $beforeComment) { + comments(first: $maxComments, after: $afterComment, before: $beforeComment, orderBy: DESC) { pageInfo { hasNextPage endCursor @@ -54,7 +54,7 @@ export const issueCommentsQuery = gql` $beforeComment: String ) { repository(owner: $owner, name: $name) { - issues(first: $maxIssues, after: $afterIssue, before: $beforeIssue) { + issues(first: $maxIssues, after: $afterIssue, before: $beforeIssue, orderBy: DESC) { pageInfo { hasNextPage endCursor @@ -62,7 +62,7 @@ export const issueCommentsQuery = gql` nodes { createdAt updatedAt - comments(first: $maxComments, after: $afterComment, before: $beforeComment) { + comments(first: $maxComments, after: $afterComment, before: $beforeComment, orderBy: DESC) { pageInfo { hasNextPage endCursor diff --git a/indexer/queue/handlers.ts b/indexer/queue/handlers.ts deleted file mode 100644 index 19370ec..0000000 --- a/indexer/queue/handlers.ts +++ /dev/null @@ -1,76 +0,0 @@ -import {GraphQLClient} from 'graphql-request'; -import {addCommentsToDatabase} from '../store'; -import {fetchIssueComments, fetchCommentsPage} from '../fetch'; -import {Commentable, Comment} from "../../lib/types"; -import {Context} from "../fsm/types"; - -export function processPendingCommentablesFactory(ctx: Context) { - return async function (job: any) { - const {commentable} = job.data; - console.log("Processing pending commentable:", commentable); - // TODO: Mutate commentables using Postgraphile - }; -} - -export function processPendingCommentsFactory(ctx: Context) { - return async function (job: any) { - const {comment} = job.data; - console.log("Processing pending comment:", comment); - await addCommentsToDatabase(ctx.pgClient, [comment]); - }; -} - -export function processNextCommentablesFactory(ctx: Context) { - return async function (job: any) { - const {nextPage} = job.data; - console.log("Processing next commentables page:", nextPage); - const {commentables: nextCommentables, comments: nextComments} = - await fetchIssueComments(ctx.ghClient, { - owner: ctx.owner, - name: ctx.name, - PRs: { - max: ctx.commentablesPageSize, - comments: {max: ctx.commentsPageSize}, - }, - issues: { - max: ctx.commentablesPageSize, - comments: {max: ctx.commentsPageSize}, - }, - ...nextPage, - }); - - nextCommentables.forEach((commentable: Commentable) => { - console.log("Enqueueing next commentable:", commentable); - ctx.pendingCommentablesQueue.add('enqueue', {commentable}); - }); - - nextComments.forEach((comment: Comment) => { - console.log("Enqueueing next comment:", comment); - ctx.pendingCommentsQueue.add('enqueue', {comment}); - }); - }; -} - -export function processNextCommentsFactory(ctx: Context) { - return async function (job: any) { - const {commentPage} = job.data; - console.log("Processing next comments page:", commentPage); - const fetchedComments = await fetchCommentsPage(ctx.ghClient, commentPage, { - owner: ctx.owner, - name: ctx.name, - PRs: { - max: ctx.commentablesPageSize, - comments: {max: ctx.commentsPageSize}, - }, - issues: { - max: ctx.commentablesPageSize, - comments: {max: ctx.commentsPageSize}, - }, - }); - - fetchedComments.forEach((comment: Comment) => { - console.log("Enqueueing fetched comment:", comment); - ctx.pendingCommentsQueue.add('enqueue', {comment}); - }); - }; -} diff --git a/indexer/queue/processes/fetching_commentables.ts b/indexer/queue/processes/fetching_commentables.ts index e69de29..969db52 100644 --- a/indexer/queue/processes/fetching_commentables.ts +++ b/indexer/queue/processes/fetching_commentables.ts @@ -0,0 +1,56 @@ +import Queue from "bull"; +import {Commentable, Comment, CommentablesPage, CommentsPage} from "../../../lib/types"; +import {Job} from "../../../lib/types/job"; +import {fetchIssueComments} from "../../fetch"; +import {GraphQLClient} from "graphql-request"; + +export default async function (job: Job) { + console.log("processing fetching commentables queue"); + + const {ctx, commentablesPage,} = job; + const { + queryVars, + queues: {fetching, storing}, + clientConfigs: {github}, + } = ctx; + + const fetchingCommentablesQueue = new Queue(storing.commentables), + fetchingCommentsQueue = new Queue(fetching.comments), + storingCommentablesQueue = new Queue(storing.commentables), + storingCommentsQueue = new Queue(storing.comments) + ; + + // GitHub GraphQL client + const client = new GraphQLClient(github.endpointURL, { + headers: {Authorization: `Bearer ${github.classicAccessToken}`}, + }); + + // TODO: enqueue a page for each... + // Fetch initial issues and issue comments + const {commentables, comments, next} = + await fetchIssueComments(client, queryVars); + + // Enqueue normalized commentables and comments + commentables.forEach((commentable: Commentable) => { + const job: Job = {ctx, commentable}; + storingCommentablesQueue.add("enqueue", job); + }); + + comments.forEach((comment: Comment) => { + const job: Job = {ctx, comment}; + storingCommentsQueue.add("enqueue", job); + }); + + // Enqueue CommentablePage and CommentPage objects to fetch next pages + if (next.commentables) { + const job: Job = {ctx, nextPage: next.commentables}; + fetchingCommentablesQueue.add("enqueue", job); + } + + if (next.comments) { + next.comments.forEach((commentsPage: CommentsPage) => { + const job: Job = {ctx, commentsPage}; + fetchingCommentsQueue.add("enqueue", job); + }); + } +} diff --git a/indexer/queue/processes/fetching_comments.ts b/indexer/queue/processes/fetching_comments.ts index e69de29..dff6b78 100644 --- a/indexer/queue/processes/fetching_comments.ts +++ b/indexer/queue/processes/fetching_comments.ts @@ -0,0 +1,54 @@ +import Queue from "bull"; +import {Commentable, Comment, CommentablesPage, CommentsPage, QueryVars} from "../../../lib/types"; +import {Job} from "../../../lib/types/job"; +import {fetchIssueComments} from "../../fetch"; +import {GraphQLClient} from "graphql-request"; + +export default async function (job: Job) { + console.log("processing fetching comments queue"); + + const {ctx, commentsPage,} = job; + const { + queryVars, + queues: {fetching, storing}, + clientConfigs: {github}, + } = ctx; + + const fetchingCommentsQueue = new Queue(fetching.comments), + storingCommentsQueue = new Queue(storing.comments) + ; + + // GitHub GraphQL client + const client = new GraphQLClient(github.endpointURL, { + headers: {Authorization: `Bearer ${github.classicAccessToken}`}, + }); + + // TODO: refactor + const {owner, name, pageSize} = queryVars; + const nextCommentPageVars: QueryVars = { + owner, name, + pageSize: { + commentables: 1, + comments: pageSize.comments, + }, + pageVars: queryVars.pageVars ?? {}, + }; + + // TODO: enqueue a page for each... + // Fetch initial issues and issue comments + const {comments, next} = + await fetchIssueComments(client, nextCommentPageVars); + + comments.forEach((comment: Comment) => { + const job: Job = {ctx, comment}; + storingCommentsQueue.add("enqueue", job); + }); + + // Enqueue `CommentPage` object to fetch next page + if (next.comments) { + next.comments.forEach((commentsPage: CommentsPage) => { + const job: Job = {ctx, commentsPage}; + fetchingCommentsQueue.add("enqueue", job); + }); + } +} diff --git a/indexer/queue/processes/storing_commentables.ts b/indexer/queue/processes/storing_commentables.ts index e69de29..872fe94 100644 --- a/indexer/queue/processes/storing_commentables.ts +++ b/indexer/queue/processes/storing_commentables.ts @@ -0,0 +1,5 @@ +export default function(job) { + console.log("processing pendingCommentables queue"); + const {commentable} = job.data; + // TODO: Mutate commentables using Postgraphile +} \ No newline at end of file diff --git a/indexer/queue/processes/storing_comments.ts b/indexer/queue/processes/storing_comments.ts index 794937d..fde9d03 100644 --- a/indexer/queue/processes/storing_comments.ts +++ b/indexer/queue/processes/storing_comments.ts @@ -1,9 +1,5 @@ -import Queue from "bull"; -import {processNextCommentablesFactory} from "../handlers"; -import {Context} from "../../fsm/types"; - -async function run(ctx: Context): Promise { - processNextCommentablesFactory(ctx) +export default function(job) { + console.log("processing pendingComments queue"); + const {comment} = job.data; + await addCommentsToDatabase(context.pgClient, [comment]); } - -export default run; diff --git a/indexer/store.ts b/indexer/store.ts index c0043af..7693c81 100644 --- a/indexer/store.ts +++ b/indexer/store.ts @@ -1,6 +1,6 @@ import {GraphQLClient} from 'graphql-request'; -import {Comment} from "../lib/types/index.js"; -import {createCommentsMutation} from "./mutations.js"; +import {Comment} from "../lib/types/index.ts"; +import {createCommentsMutation} from "./mutations.ts"; // TODO: move (?) interface CreateCommentPayload { diff --git a/lib/types/comment.ts b/lib/types/comment.ts index 2876468..6d1e542 100644 --- a/lib/types/comment.ts +++ b/lib/types/comment.ts @@ -24,11 +24,11 @@ export interface Commentable { export interface CommentsPage { commentableType: CommentableType; - commentableId: string; - afterComment: string; + commentableId?: string; + afterComment?: string; } export interface CommentablesPage { commentableType: CommentableType; - after: string; + after?: string; } \ No newline at end of file diff --git a/lib/types/index.ts b/lib/types/index.ts index 64602ef..c469438 100644 --- a/lib/types/index.ts +++ b/lib/types/index.ts @@ -1,3 +1,3 @@ -export * from "./response.js" -export * from "./query.js" -export * from "./comment.js" \ No newline at end of file +export * from "./response.ts" +export * from "./query.ts" +export * from "./comment.ts" \ No newline at end of file diff --git a/lib/types/indexer.ts b/lib/types/indexer.ts new file mode 100644 index 0000000..94ec4fc --- /dev/null +++ b/lib/types/indexer.ts @@ -0,0 +1,47 @@ +import Queue from "bull"; +import {QueryVars} from "./query"; + +export interface GitHubGQLConfig { + endpointURL: string; + classicAccessToken: string; +} + +export interface PostgraphileGQLConfig { + endpointURL: string; +} + +// TODO: consolidate w/ lib/types (?) +// Context MUST be serializable (included in Bull job objects) +export interface Context { + clientConfigs: { + github: GitHubGQLConfig; + postgraphile: PostgraphileGQLConfig; + } + queryVars: QueryVars + queues: { + fetching: { + // TODO: rename + commentables: string; + comments: string; + }, + storing: { + commentables: string; + comments: string; + } + } + processes: { + fetching: { + commentables: ProcessConfig; + comments: ProcessConfig; + }, + storing: { + commentables: ProcessConfig; + comments: ProcessConfig; + } + } +} + +export interface ProcessConfig { + concurrency: number; + scriptPath: string; +} \ No newline at end of file diff --git a/lib/types/job.ts b/lib/types/job.ts new file mode 100644 index 0000000..61c54d4 --- /dev/null +++ b/lib/types/job.ts @@ -0,0 +1,8 @@ +import {Context} from "./indexer"; + +// TODO: re-consider second `Record` type param +export interface Job { + ctx: Context; + // TODO: something better + [key: string]: T | Context +} \ No newline at end of file diff --git a/lib/types/query.ts b/lib/types/query.ts index 713d1d2..4021c7a 100644 --- a/lib/types/query.ts +++ b/lib/types/query.ts @@ -1,19 +1,18 @@ export interface paginatedQueryArgs { - max: number, after?: string, before?: string, } -export interface commentsQueryArgs extends paginatedQueryArgs { - comments: paginatedQueryArgs -} - -// TODO: refactor -export interface nextQueryArgs { - owner: string, - // TODO: rename to `repo` (?) - name: string, - PRs?: commentsQueryArgs, - issues?: commentsQueryArgs, +export interface QueryVars { + owner: string; + name: string; + pageSize: { + commentables: number; + comments: number; + } + pageVars?: { + commentables?: paginatedQueryArgs; + comments?: paginatedQueryArgs; + }; } diff --git a/yarn.lock b/yarn.lock index e2c0fe7..255c1f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -990,6 +990,11 @@ bull@^4.10.4: semver "^7.3.2" uuid "^8.3.0" +bun-types@^0.5.8: + version "0.5.8" + resolved "https://registry.yarnpkg.com/bun-types/-/bun-types-0.5.8.tgz#953722062116e0b87ce00e4e4cc807673926b834" + integrity sha512-VHwD0MAHo3wraYAeqTWH2NDmXOdGfC3wWWOnZvK93ytI6yq/LkgsCjDudWNmN7MlfPvJb2zoLMnkzhjxNwLsLw== + bytes@3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5"