From c1e36647c3e102883c30401129446b66a77e4a0b Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 6 May 2024 15:13:43 +0100 Subject: [PATCH 1/2] WIP env var management API --- apps/webapp/app/models/project.server.ts | 11 ++ .../route.tsx | 1 + ...rojects.$projectRef.envvars.$slug.$name.ts | 177 ++++++++++++++++++ ...i.v1.projects.$projectRef.envvars.$slug.ts | 127 +++++++++++++ .../environmentVariablesRepository.server.ts | 157 ++++++++++++++-- .../app/v3/environmentVariables/repository.ts | 22 ++- packages/core/src/v3/schemas/api.ts | 17 ++ 7 files changed, 498 insertions(+), 14 deletions(-) create mode 100644 apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.$name.ts create mode 100644 apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.ts diff --git a/apps/webapp/app/models/project.server.ts b/apps/webapp/app/models/project.server.ts index 47d9097ca5..aa0fd3f550 100644 --- a/apps/webapp/app/models/project.server.ts +++ b/apps/webapp/app/models/project.server.ts @@ -110,3 +110,14 @@ export async function findProjectBySlug(orgSlug: string, projectSlug: string, us }, }); } + +export async function findProjectByRef(externalRef: string, userId: string) { + return await prisma.project.findFirst({ + where: { + externalRef, + organization: { + members: { some: { userId } }, + }, + }, + }); +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables/route.tsx index e0d5c40a23..c83059a776 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables/route.tsx @@ -334,6 +334,7 @@ function EditEnvironmentVariablePanel({ name={`values[${index}].value`} placeholder="Not set" defaultValue={value} + type="password" /> ); diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.$name.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.$name.ts new file mode 100644 index 0000000000..e1048ae510 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.$name.ts @@ -0,0 +1,177 @@ +import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { UpdateEnvironmentVariableRequestBody } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { findProjectByRef } from "~/models/project.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; + +const ParamsSchema = z.object({ + projectRef: z.string(), + slug: z.string(), + name: z.string(), +}); + +export async function action({ params, request }: ActionFunctionArgs) { + const parsedParams = ParamsSchema.safeParse(params); + + if (!parsedParams.success) { + return json({ error: "Invalid params" }, { status: 400 }); + } + + const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request); + + if (!authenticationResult) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ + where: { + id: authenticationResult.userId, + }, + }); + + if (!user) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const project = await findProjectByRef(parsedParams.data.projectRef, user.id); + + if (!project) { + return json({ error: "Project not found" }, { status: 404 }); + } + + const environment = await prisma.runtimeEnvironment.findFirst({ + where: { + projectId: project.id, + slug: parsedParams.data.slug, + }, + }); + + if (!environment) { + return json({ error: "Environment not found" }, { status: 404 }); + } + + // Find the environment variable + const variable = await prisma.environmentVariable.findFirst({ + where: { + key: parsedParams.data.name, + projectId: project.id, + }, + }); + + if (!variable) { + return json({ error: "Environment variable not found" }, { status: 404 }); + } + + const repository = new EnvironmentVariablesRepository(); + + switch (request.method.toUpperCase()) { + case "DELETE": { + const result = await repository.deleteValue(project.id, user.id, { + id: variable.id, + environmentId: environment.id, + }); + + if (result.success) { + return json({ success: true }); + } else { + return json({ error: result.error }, { status: 400 }); + } + } + case "PUT": + case "POST": { + const jsonBody = await request.json(); + + const body = UpdateEnvironmentVariableRequestBody.safeParse(jsonBody); + + if (!body.success) { + return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 }); + } + + const result = await repository.edit(project.id, user.id, { + values: [ + { + value: body.data.value, + environmentId: environment.id, + }, + ], + id: variable.id, + keepEmptyValues: true, + }); + + if (result.success) { + return json({ success: true }); + } else { + return json({ error: result.error }, { status: 400 }); + } + } + } +} + +export async function loader({ params, request }: LoaderFunctionArgs) { + const parsedParams = ParamsSchema.safeParse(params); + + if (!parsedParams.success) { + return json({ error: "Invalid params" }, { status: 400 }); + } + + const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request); + + if (!authenticationResult) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ + where: { + id: authenticationResult.userId, + }, + }); + + if (!user) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const project = await findProjectByRef(parsedParams.data.projectRef, user.id); + + if (!project) { + return json({ error: "Project not found" }, { status: 404 }); + } + + const environment = await prisma.runtimeEnvironment.findFirst({ + where: { + projectId: project.id, + slug: parsedParams.data.slug, + }, + }); + + if (!environment) { + return json({ error: "Environment not found" }, { status: 404 }); + } + + // Find the environment variable + const variable = await prisma.environmentVariable.findFirst({ + where: { + key: parsedParams.data.name, + projectId: project.id, + }, + }); + + if (!variable) { + return json({ error: "Environment variable not found" }, { status: 404 }); + } + + const repository = new EnvironmentVariablesRepository(); + + const variables = await repository.getEnvironment(project.id, user.id, environment.id, true); + + const environmentVariable = variables.find((v) => v.key === parsedParams.data.name); + + if (!environmentVariable) { + return json({ error: "Environment variable not found" }, { status: 404 }); + } + + return json({ + value: environmentVariable.value, + }); +} diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.ts new file mode 100644 index 0000000000..e0c2dcd1eb --- /dev/null +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.ts @@ -0,0 +1,127 @@ +import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { CreateEnvironmentVariableRequestBody } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { findProjectByRef } from "~/models/project.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; + +const ParamsSchema = z.object({ + projectRef: z.string(), + slug: z.string(), +}); + +export async function action({ params, request }: ActionFunctionArgs) { + const parsedParams = ParamsSchema.safeParse(params); + + if (!parsedParams.success) { + return json({ error: "Invalid params" }, { status: 400 }); + } + + const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request); + + if (!authenticationResult) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const jsonBody = await request.json(); + + const body = CreateEnvironmentVariableRequestBody.safeParse(jsonBody); + + if (!body.success) { + return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 }); + } + + const user = await prisma.user.findUnique({ + where: { + id: authenticationResult.userId, + }, + }); + + if (!user) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const project = await findProjectByRef(parsedParams.data.projectRef, user.id); + + if (!project) { + return json({ error: "Project not found" }, { status: 404 }); + } + + const environment = await prisma.runtimeEnvironment.findFirst({ + where: { + projectId: project.id, + slug: parsedParams.data.slug, + }, + }); + + if (!environment) { + return json({ error: "Environment not found" }, { status: 404 }); + } + + const repository = new EnvironmentVariablesRepository(); + + const result = await repository.create(project.id, user.id, { + overwrite: true, + environmentIds: [environment.id], + variables: [ + { + key: body.data.name, + value: body.data.value, + }, + ], + }); + + if (result.success) { + return json({ success: true }); + } else { + return json({ error: result.error, variableErrors: result.variableErrors }, { status: 400 }); + } +} + +export async function loader({ params, request }: LoaderFunctionArgs) { + const parsedParams = ParamsSchema.safeParse(params); + + if (!parsedParams.success) { + return json({ error: "Invalid params" }, { status: 400 }); + } + + const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request); + + if (!authenticationResult) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ + where: { + id: authenticationResult.userId, + }, + }); + + if (!user) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const project = await findProjectByRef(parsedParams.data.projectRef, user.id); + + if (!project) { + return json({ error: "Project not found" }, { status: 404 }); + } + + const environment = await prisma.runtimeEnvironment.findFirst({ + where: { + projectId: project.id, + slug: parsedParams.data.slug, + }, + }); + + if (!environment) { + return json({ error: "Environment not found" }, { status: 404 }); + } + + const repository = new EnvironmentVariablesRepository(); + + const variables = await repository.getEnvironment(project.id, user.id, environment.id, true); + + return json(variables); +} diff --git a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts index 3a4e6ba6ce..79b62527ad 100644 --- a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts +++ b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts @@ -7,6 +7,8 @@ import { getSecretStore } from "~/services/secrets/secretStore.server"; import { generateFriendlyId } from "../friendlyIdentifiers"; import { CreateResult, + DeleteEnvironmentVariable, + DeleteEnvironmentVariableValue, EnvironmentVariable, ProjectEnvironmentVariable, Repository, @@ -218,7 +220,11 @@ export class EnvironmentVariablesRepository implements Repository { async edit( projectId: string, userId: string, - options: { values: { value: string; environmentId: string }[]; id: string } + options: { + values: { value: string; environmentId: string }[]; + id: string; + keepEmptyValues?: boolean; + } ): Promise { const project = await this.prismaClient.project.findUnique({ where: { @@ -266,12 +272,15 @@ export class EnvironmentVariablesRepository implements Repository { //add in empty values for environments that don't have a value const environmentIds = project.environments.map((e) => e.id); - for (const environmentId of environmentIds) { - if (!values.some((v) => v.environmentId === environmentId)) { - values.push({ - environmentId, - value: "", - }); + + if (!options.keepEmptyValues) { + for (const environmentId of environmentIds) { + if (!values.some((v) => v.environmentId === environmentId)) { + values.push({ + environmentId, + value: "", + }); + } } } @@ -447,7 +456,8 @@ export class EnvironmentVariablesRepository implements Repository { async getEnvironment( projectId: string, userId: string, - environmentId: string + environmentId: string, + excludeInternalVariables?: boolean ): Promise { const project = await this.prismaClient.project.findUnique({ where: { @@ -477,7 +487,7 @@ export class EnvironmentVariablesRepository implements Repository { return []; } - return this.getEnvironmentVariables(projectId, environmentId); + return this.getEnvironmentVariables(projectId, environmentId, excludeInternalVariables); } async #getTriggerEnvironmentVariables(environmentId: string): Promise { @@ -621,15 +631,25 @@ export class EnvironmentVariablesRepository implements Repository { async getEnvironmentVariables( projectId: string, - environmentId: string + environmentId: string, + excludeInternalVariables?: boolean ): Promise { const secretEnvVars = await this.#getSecretEnvironmentVariables(projectId, environmentId); + + if (excludeInternalVariables) { + return secretEnvVars; + } + const triggerEnvVars = await this.#getTriggerEnvironmentVariables(environmentId); return [...secretEnvVars, ...triggerEnvVars]; } - async delete(projectId: string, userId: string, options: { id: string }): Promise { + async delete( + projectId: string, + userId: string, + options: DeleteEnvironmentVariable + ): Promise { const project = await this.prismaClient.project.findUnique({ where: { id: projectId, @@ -703,7 +723,7 @@ export class EnvironmentVariablesRepository implements Repository { prismaClient: tx, }); - //create the secret values and references + //delete the secret values and references for (const value of environmentVariable.values) { const key = secretKey(projectId, value.environmentId, environmentVariable.key); await secretStore.deleteSecret(key); @@ -728,4 +748,117 @@ export class EnvironmentVariablesRepository implements Repository { }; } } + + async deleteValue( + projectId: string, + userId: string, + options: DeleteEnvironmentVariableValue + ): Promise { + const project = await this.prismaClient.project.findUnique({ + where: { + id: projectId, + organization: { + members: { + some: { + userId, + }, + }, + }, + deletedAt: null, + }, + select: { + environments: { + select: { + id: true, + }, + where: { + OR: [ + { + orgMember: null, + }, + { + orgMember: { + userId, + }, + }, + ], + }, + }, + }, + }); + + if (!project) { + return { success: false as const, error: "Project not found" }; + } + + const environmentVariable = await this.prismaClient.environmentVariable.findUnique({ + select: { + id: true, + key: true, + values: { + select: { + id: true, + environmentId: true, + valueReference: { + select: { + key: true, + }, + }, + }, + }, + }, + where: { + id: options.id, + }, + }); + + if (!environmentVariable) { + return { success: false as const, error: "Environment variable not found" }; + } + + const value = environmentVariable.values.find((v) => v.environmentId === options.environmentId); + + if (!value) { + return { success: false as const, error: "Environment variable value not found" }; + } + + // If this is the last value, delete the whole variable + if (environmentVariable.values.length === 1) { + return this.delete(projectId, userId, { id: options.id }); + } + + try { + await $transaction(this.prismaClient, async (tx) => { + const secretStore = getSecretStore("DATABASE", { + prismaClient: tx, + }); + + const key = secretKey(projectId, options.environmentId, environmentVariable.key); + await secretStore.deleteSecret(key); + + if (value.valueReference) { + await tx.secretReference.delete({ + where: { + key: value.valueReference.key, + }, + }); + } + + await tx.environmentVariableValue.delete({ + where: { + id: value.id, + }, + }); + }); + + return { + success: true as const, + }; + } catch (error) { + return { + success: false as const, + error: error instanceof Error ? error.message : "Something went wrong", + }; + } + } } diff --git a/apps/webapp/app/v3/environmentVariables/repository.ts b/apps/webapp/app/v3/environmentVariables/repository.ts index 920720290d..332e33d591 100644 --- a/apps/webapp/app/v3/environmentVariables/repository.ts +++ b/apps/webapp/app/v3/environmentVariables/repository.ts @@ -31,14 +31,22 @@ export const EditEnvironmentVariable = z.object({ value: z.string(), }) ), + keepEmptyValues: z.boolean().optional(), }); export type EditEnvironmentVariable = z.infer; export const DeleteEnvironmentVariable = z.object({ id: z.string(), + environmentId: z.string().optional(), }); export type DeleteEnvironmentVariable = z.infer; +export const DeleteEnvironmentVariableValue = z.object({ + id: z.string(), + environmentId: z.string(), +}); +export type DeleteEnvironmentVariableValue = z.infer; + export type Result = | { success: true; @@ -75,8 +83,18 @@ export interface Repository { getEnvironment( projectId: string, userId: string, - environmentId: string + environmentId: string, + excludeInternalVariables?: boolean + ): Promise; + getEnvironmentVariables( + projectId: string, + environmentId: string, + excludeInternalVariables?: boolean ): Promise; - getEnvironmentVariables(projectId: string, environmentId: string): Promise; delete(projectId: string, userId: string, options: DeleteEnvironmentVariable): Promise; + deleteValue( + projectId: string, + userId: string, + options: DeleteEnvironmentVariableValue + ): Promise; } diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index fc3a1e5040..97311adb87 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -362,3 +362,20 @@ export const RetrieveRunResponse = z.object({ }); export type RetrieveRunResponse = z.infer; + +export const CreateEnvironmentVariableRequestBody = z.object({ + name: z.string(), + value: z.string(), +}); + +export type CreateEnvironmentVariableRequestBody = z.infer< + typeof CreateEnvironmentVariableRequestBody +>; + +export const UpdateEnvironmentVariableRequestBody = z.object({ + value: z.string(), +}); + +export type UpdateEnvironmentVariableRequestBody = z.infer< + typeof UpdateEnvironmentVariableRequestBody +>; From a337bbb17066f4971482f6d20746bc20f74fc0ad Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 6 May 2024 22:03:19 +0100 Subject: [PATCH 2/2] Add import env var API endpoint --- ...ojects.$projectRef.envvars.$slug.import.ts | 98 +++++++++++++++++++ .../environmentVariablesRepository.server.ts | 9 ++ packages/core/src/v3/schemas/api.ts | 9 ++ 3 files changed, 116 insertions(+) create mode 100644 apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.import.ts diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.import.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.import.ts new file mode 100644 index 0000000000..f4e72d6cd2 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.import.ts @@ -0,0 +1,98 @@ +import { ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { ImportEnvironmentVariablesRequestBody } from "@trigger.dev/core/v3"; +import { parse } from "dotenv"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { findProjectByRef } from "~/models/project.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; + +const ParamsSchema = z.object({ + projectRef: z.string(), + slug: z.string(), +}); + +export async function action({ params, request }: ActionFunctionArgs) { + const parsedParams = ParamsSchema.safeParse(params); + + if (!parsedParams.success) { + return json({ error: "Invalid params" }, { status: 400 }); + } + + const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request); + + if (!authenticationResult) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ + where: { + id: authenticationResult.userId, + }, + }); + + if (!user) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const project = await findProjectByRef(parsedParams.data.projectRef, user.id); + + if (!project) { + return json({ error: "Project not found" }, { status: 404 }); + } + + const environment = await prisma.runtimeEnvironment.findFirst({ + where: { + projectId: project.id, + slug: parsedParams.data.slug, + }, + }); + + if (!environment) { + return json({ error: "Environment not found" }, { status: 404 }); + } + + const repository = new EnvironmentVariablesRepository(); + + const body = await parseImportBody(request); + + const result = await repository.create(project.id, user.id, { + overwrite: body.overwrite === true ? true : false, + environmentIds: [environment.id], + variables: Object.entries(body.variables).map(([key, value]) => ({ + key, + value, + })), + }); + + if (result.success) { + return json({ success: true }); + } else { + return json({ error: result.error, variableErrors: result.variableErrors }, { status: 400 }); + } +} + +async function parseImportBody(request: Request): Promise { + const contentType = request.headers.get("content-type") ?? "application/json"; + + if (contentType.includes("application/octet-stream")) { + // We have a "dotenv" formatted file uploaded + const buffer = await request.arrayBuffer(); + + const variables = parse(Buffer.from(buffer)); + + const overwrite = request.headers.get("x-overwrite") === "true"; + + return { variables, overwrite }; + } else { + const rawBody = await request.json(); + + const body = ImportEnvironmentVariablesRequestBody.safeParse(rawBody); + + if (!body.success) { + throw json({ error: "Invalid body" }, { status: 400 }); + } + + return body.data; + } +} diff --git a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts index 79b62527ad..f9e317739c 100644 --- a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts +++ b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts @@ -94,6 +94,15 @@ export class EnvironmentVariablesRepository implements Repository { return { success: false as const, error: `Environment not found` }; } + // Check to see if any of the variables are `TRIGGER_SECRET_KEY` or `TRIGGER_API_URL` + const triggerKeys = options.variables.map((v) => v.key); + if (triggerKeys.includes("TRIGGER_SECRET_KEY") || triggerKeys.includes("TRIGGER_API_URL")) { + return { + success: false as const, + error: `You cannot set the variables TRIGGER_SECRET_KEY or TRIGGER_API_URL as they will be set automatically`, + }; + } + //get rid of empty variables const values = options.variables.filter((v) => v.key.trim() !== "" && v.value.trim() !== ""); if (values.length === 0) { diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 97311adb87..9861d921e2 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -379,3 +379,12 @@ export const UpdateEnvironmentVariableRequestBody = z.object({ export type UpdateEnvironmentVariableRequestBody = z.infer< typeof UpdateEnvironmentVariableRequestBody >; + +export const ImportEnvironmentVariablesRequestBody = z.object({ + variables: z.record(z.string()), + overwrite: z.boolean().optional(), +}); + +export type ImportEnvironmentVariablesRequestBody = z.infer< + typeof ImportEnvironmentVariablesRequestBody +>;