Skip to content

Commit

Permalink
feat: custom test harnesses (#7249)
Browse files Browse the repository at this point in the history
  • Loading branch information
eladb authored Jan 27, 2025
1 parent 80b06d0 commit 94ab383
Show file tree
Hide file tree
Showing 20 changed files with 201 additions and 195 deletions.
14 changes: 9 additions & 5 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -214,31 +214,35 @@ jobs:

e2e-test:
name: "E2E / ${{ matrix.runner }} + Node${{ matrix.node }} [${{ matrix.shard }}]"
runs-on: "${{ matrix.runner }}-latest"
runs-on: "${{ matrix.runner }}"
needs:
- build
if: needs.build.outputs.e2e-changed == 'true' || contains(github.event.pull_request.labels.*.name, '🧪 pr/e2e-full')
strategy:
fail-fast: true
matrix:
runner: [windows, macos, ubuntu]
runner: [ubuntu-latest, macos-13]
node: ["20", "18"]
shard: ["1/2", "2/2"]
full_run:
# Do a full run on push or when the PR is labeled "pr/e2e-full"
- ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, '🧪 pr/e2e-full') }}
exclude:
- runner: macos
- runner: macos-13
full_run: false
- runner: windows
- runner: windows-latest
full_run: false
- runner: ubuntu
- runner: ubuntu-latest
node: "18"
full_run: false
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Setup Docker on macOS
if: runner.os == 'macOS'
uses: douglascamata/setup-docker-macos-action@v1-alpha

- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
Expand Down
26 changes: 3 additions & 23 deletions packages/@winglang/compiler/src/compile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { BuiltinPlatform } from "./constants";
import { PreflightError } from "./errors";
import { readFile } from "fs/promises";
import { fork } from "child_process";

import { PlatformManager } from "@winglang/sdk/lib/platform";
// increase the stack trace limit to 50, useful for debugging Rust panics
// (not setting the limit too high in case of infinite recursion)
Error.stackTraceLimit = 50;
Expand All @@ -19,14 +19,6 @@ const WINGC_COMPILE = "wingc_compile";
const WINGC_PREFLIGHT = "preflight.cjs";
const DOT_WING = ".wing";

const BUILTIN_PLATFORMS = [
BuiltinPlatform.SIM,
BuiltinPlatform.TF_AWS,
BuiltinPlatform.TF_AZURE,
BuiltinPlatform.TF_GCP,
BuiltinPlatform.AWSCDK, // TODO: remove this when awscdk platform is implemented external platform
];

const defaultSynthDir = (model: string): string => {
switch (model) {
case BuiltinPlatform.TF_AWS:
Expand Down Expand Up @@ -110,20 +102,8 @@ function resolveSynthDir(outDir: string, entrypoint: string, target: string, tes
* @returns the resolved model
*/
export function determineTargetFromPlatforms(platforms: string[]): string {
if (platforms.length === 0) {
return "";
}
// determine target based on first platform
const platform = platforms[0];

// If its a builtin platform just return
if (BUILTIN_PLATFORMS.includes(platform)) {
return platform;
}

// load custom platform to retrieve the target
const { _loadCustomPlatform } = require("@winglang/sdk/lib/platform");
return _loadCustomPlatform(platform).target;
const p = new PlatformManager({ platformPaths: platforms });
return p.primary.target;
}

export interface CompileOutput {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,48 +1,45 @@
import { readFile, rm } from "fs/promises";
import { ITestRunnerClient } from "@winglang/sdk/lib/std";
import { Util } from "@winglang/sdk/lib/util";
import { ITestHarness } from "./api";
import { withSpinner } from "../../../util";
import { execCapture } from "../util";
import { ITestHarness } from "@winglang/sdk/lib/platform";
import * as path from "path";

const ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK = "WingTestRunnerFunctionArns";
export const WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK = "WingTestRunnerFunctionArns";

const OUTPUT_FILE = "output.json";

export class AwsCdkTestHarness implements ITestHarness {
public async deploy(synthDir: string): Promise<ITestRunnerClient> {
const opts = {
cwd: synthDir,
inheritEnv: true,
};
try {
await execCapture("cdk version --ci true", { cwd: synthDir });
Util.exec("cdk", ["version", "--ci", "true"], opts);
} catch (err) {
throw new Error(
"AWS-CDK is not installed. Please install AWS-CDK to run tests in the cloud (npm i -g aws-cdk).",
);
}

await withSpinner("cdk deploy", () =>
execCapture("cdk deploy --require-approval never --ci true -O ./output.json --app . ", {
cwd: synthDir,
}),
);
Util.exec("cdk", ["deploy", "--require-approval", "never", "--ci", "true", "-O", OUTPUT_FILE, "--app", "."], opts);

const stackName = process.env.CDK_STACK_NAME! + Util.sha256(synthDir).slice(-8);
const testArns = await this.getFunctionArnsOutput(synthDir, stackName);

const { TestRunnerClient } = await import("@winglang/sdk/lib/shared-aws/test-runner.inflight");
const runner = new TestRunnerClient({ $tests: testArns });
return runner;
return new TestRunnerClient({ $tests: testArns });
}

public async cleanup(synthDir: string): Promise<void> {
await withSpinner("aws-cdk destroy", async () => {
await rm(synthDir.concat("/output.json"));
await execCapture("cdk destroy -f --ci true --app ./", { cwd: synthDir });
});

await rm(path.join(synthDir, OUTPUT_FILE));
Util.exec("cdk", ["destroy", "-f", "--ci", "true", "--app", "."], { cwd: synthDir, inheritEnv: true });
await rm(synthDir, { recursive: true, force: true });
}

private async getFunctionArnsOutput(synthDir: string, stackName: string) {
const file = await readFile(synthDir.concat("/output.json"));
const file = await readFile(path.join(synthDir, OUTPUT_FILE));
const parsed = JSON.parse(Buffer.from(file).toString());
return parsed[stackName][ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK];
return parsed[stackName][WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK];
}
}
6 changes: 6 additions & 0 deletions packages/@winglang/platform-awscdk/src/platform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import { Website } from "./website";
import { cloud } from "@winglang/sdk";
import { Construct } from "constructs";
import { Service } from "./service";
import { AwsCdkTestHarness } from "./harness";
import { ITestHarness } from "@winglang/sdk/lib/platform";

const {
API_FQN,
Expand Down Expand Up @@ -99,4 +101,8 @@ export class Platform implements platform.IPlatform {
}
return undefined;
}

public async createTestHarness(): Promise<ITestHarness> {
return new AwsCdkTestHarness();
}
}
5 changes: 2 additions & 3 deletions packages/@winglang/platform-awscdk/src/test-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@ import { CfnOutput, Lazy } from "aws-cdk-lib";
import { Construct } from "constructs";
import { core, std } from "@winglang/sdk";
import { isAwsCdkFunction } from "./function";

const OUTPUT_TEST_RUNNER_FUNCTION_ARNS = "WingTestRunnerFunctionArns";
import { WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK } from "./harness";

/**
* AWS implementation of `cloud.TestRunner`.
Expand Down Expand Up @@ -32,7 +31,7 @@ export class TestRunner extends std.TestRunner {
}),
});

output.overrideLogicalId(OUTPUT_TEST_RUNNER_FUNCTION_ARNS);
output.overrideLogicalId(WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK);
}

public onLift(host: std.IInflightHost, ops: string[]): void {
Expand Down
13 changes: 11 additions & 2 deletions packages/@winglang/sdk/src/platform/platform-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,13 @@ export class PlatformManager {
return new ClassFactory(newInstanceOverrides, resolveTypeOverrides);
}

/**
* Returns the target platform (the first platform in the list)
*/
public get primary(): IPlatform {
return this.platformInstances[0];
}

// This method is called from preflight.js in order to return an App instance
// that can be synthesized
public createApp(appProps: Omit<AppProps, "classFactory">): App {
Expand All @@ -133,7 +140,7 @@ export class PlatformManager {
}
(globalThis as any).$ClassFactory = this.createClassFactory();

let appCall = this.platformInstances[0].newApp;
let appCall = this.primary.newApp;

if (!appCall) {
throw new Error(
Expand Down Expand Up @@ -265,7 +272,9 @@ export function _loadCustomPlatform(customPlatformPath: string): any {
? "Ensure the path to the platform is correct"
: `Ensure you have installed the platform provider by running 'npm install ${customPlatformPath}'`;
console.error(
`An error occurred while loading the custom platform: ${customPlatformPath}\n\n(hint: ${hint})`,
`An error occurred while loading the custom platform: ${customPlatformPath}\n\n(hint: ${hint})\n${
(error as any).stack
}`,
);
}
}
Expand Down
24 changes: 24 additions & 0 deletions packages/@winglang/sdk/src/platform/platform.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { Construct } from "constructs";
import { App, AppProps } from "../core";
import { ITestRunnerClient } from "../std";

/**
* Platform interface
Expand Down Expand Up @@ -66,4 +67,27 @@ export interface IPlatform {
* Hook for creating and storing secrets
*/
storeSecrets?(secrets: { [name: string]: string }): Promise<void>;

/**
* Create a Wing test harness for this platform.
*/
createTestHarness?(): Promise<ITestHarness>;
}

/**
* API for running wing tests.
*/
export interface ITestHarness {
/**
* Deploys the test program synthesized in the given directory and return an `ITestRunnerClient`
* that can be used to run the tests.
* @param synthDir - The directory containing the synthesized test program.
*/
deploy(synthDir: string): Promise<ITestRunnerClient>;

/**
* Cleans up the test harness after the tests have been run.
* @param synthDir - The directory containing the synthesized test program.
*/
cleanup(synthDir: string): Promise<void>;
}
72 changes: 72 additions & 0 deletions packages/@winglang/sdk/src/shared-tf/harness.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import { rm } from "fs/promises";
import { ITestHarness } from "../platform";
import { ITestRunnerClient } from "../std";
import { Util } from "../util";

export const WING_TEST_RUNNER_FUNCTION_IDENTIFIERS =
"WING_TEST_RUNNER_FUNCTION_IDENTIFIERS";

export interface TerraformTestHarnessOptions {
readonly parallelism?: number;
readonly clientModule: string;
}

export class TerraformTestHarness implements ITestHarness {
private readonly options: TerraformTestHarnessOptions;
private readonly parallelism: string;

constructor(options: TerraformTestHarnessOptions) {
this.options = options;
this.parallelism = options.parallelism
? `-parallelism=${options.parallelism}`
: "";
}

public async deploy(synthDir: string): Promise<ITestRunnerClient> {
const opts = {
cwd: synthDir,
inheritEnv: true,
};

// Check if Terraform is installed
const tfVersion = Util.exec("terraform", ["version"], opts);
const installed = tfVersion.stdout.startsWith("Terraform v");
if (!installed) {
throw new Error(
"Terraform is not installed. Please install Terraform to run tests in the cloud.",
);
}

Util.exec("terraform", ["init"], opts);
Util.exec("terraform", ["apply", "-auto-approve", this.parallelism], opts);

// Get the test runner function ARNs
const output = Util.exec("terraform", ["output", "-json"], opts);

const parsed = JSON.parse(output.stdout);
const testArns = parsed[WING_TEST_RUNNER_FUNCTION_IDENTIFIERS]?.value;
if (!testArns) {
throw new Error(
`terraform output ${WING_TEST_RUNNER_FUNCTION_IDENTIFIERS} not found`,
);
}

// Create the test runner client
const { TestRunnerClient } = await import(this.options.clientModule);
const runner = new TestRunnerClient({ $tests: testArns });
return runner;
}

public async cleanup(synthDir: string): Promise<void> {
try {
Util.exec("terraform", ["destroy", "-auto-approve", this.parallelism], {
cwd: synthDir,
inheritEnv: true,
});

await rm(synthDir, { recursive: true, force: true });
} catch (e) {
console.error(e);
}
}
}
9 changes: 8 additions & 1 deletion packages/@winglang/sdk/src/target-tf-aws/platform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ import {
TOPIC_FQN,
WEBSITE_FQN,
} from "../cloud";
import { IPlatform } from "../platform";
import { IPlatform, ITestHarness } from "../platform";
import { Domain } from "../shared-aws/domain";
import { TerraformTestHarness } from "../shared-tf/harness";
import { TEST_RUNNER_FQN } from "../std";

/**
Expand Down Expand Up @@ -207,4 +208,10 @@ export class Platform implements IPlatform {
`${Object.keys(secrets).length} secret(s) stored AWS Secrets Manager`,
);
}

public async createTestHarness(): Promise<ITestHarness> {
return new TerraformTestHarness({
clientModule: require.resolve("../shared-aws/test-runner.inflight"),
});
}
}
6 changes: 2 additions & 4 deletions packages/@winglang/sdk/src/target-tf-aws/test-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,9 @@ import { Lazy } from "cdktf/lib/tokens";
import { Construct } from "constructs";
import { Function as AwsFunction } from "./function";
import * as core from "../core";
import { WING_TEST_RUNNER_FUNCTION_IDENTIFIERS } from "../shared-tf/harness";
import * as std from "../std";

const OUTPUT_TEST_RUNNER_FUNCTION_IDENTIFIERS =
"WING_TEST_RUNNER_FUNCTION_IDENTIFIERS";

/**
* AWS implementation of `cloud.TestRunner`.
*
Expand Down Expand Up @@ -37,7 +35,7 @@ export class TestRunner extends std.TestRunner {
}),
});

output.overrideLogicalId(OUTPUT_TEST_RUNNER_FUNCTION_IDENTIFIERS);
output.overrideLogicalId(WING_TEST_RUNNER_FUNCTION_IDENTIFIERS);
}

public onLift(host: std.IInflightHost, ops: string[]): void {
Expand Down
10 changes: 9 additions & 1 deletion packages/@winglang/sdk/src/target-tf-azure/platform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import { Counter } from "./counter";
import { Function } from "./function";
import { TestRunner } from "./test-runner";
import { BUCKET_FQN, COUNTER_FQN, FUNCTION_FQN } from "../cloud";
import { IPlatform } from "../platform";
import { IPlatform, ITestHarness } from "../platform";
import { TerraformTestHarness } from "../shared-tf/harness";
import { TEST_RUNNER_FQN } from "../std";

/**
Expand Down Expand Up @@ -50,4 +51,11 @@ export class Platform implements IPlatform {

return undefined;
}

public async createTestHarness(): Promise<ITestHarness> {
return new TerraformTestHarness({
parallelism: 5,
clientModule: require.resolve("../shared-azure/test-runner.inflight"),
});
}
}
Loading

0 comments on commit 94ab383

Please sign in to comment.