Loadtesting E2Es (#6956)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/loadtest.yml

@@ -0,0 +1,28 @@
+name: loadtest
+
+on:
+ pull_request:
+ branches:
+ - master
+ push:
+ branches:
+ - master
+
+jobs:
+ e2e:
+ name: e2e / node ${{matrix.node-version}}
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ node-version: [18, 20, 21]
+ steps:
+ - name: checkout
+ uses: actions/checkout@v4
+ - uses: the-guild-org/shared-config/setup@main
+ name: setup env
+ with:
+ nodeVersion: ${{matrix.node-version}}
+ packageManagerVersion: modern
+ - name: loadtest
+ run: yarn loadtest:e2e

e2e/auto-type-merging/auto-type-merging.loadtest.ts

@@ -0,0 +1,55 @@
+import os from 'os';
+import { createTbench, Tbench, TbenchResult } from '@e2e/tbench';
+import { Container, createTenv } from '@e2e/tenv';
+
+const { serve, compose, service, container } = createTenv(__dirname);
+
+let tbench: Tbench;
+let petstore!: Container;
+beforeAll(async () => {
+ tbench = await createTbench(
+ // to give space for jest and the serve process.
+ os.availableParallelism() - 2,
+ );
+ petstore = await container({
+ name: 'petstore',
+ image: 'swaggerapi/petstore3:1.0.7',
+ containerPort: 8080,
+ healthcheck: ['CMD-SHELL', 'wget --spider http://0.0.0.0:8080'],
+ });
+});
+
+const threshold: TbenchResult = {
+ maxCpu: Infinity, // we dont care
+ maxMem: 500, // MB
+ slowestRequest: 1, // second
+};
+
+it(`should perform within threshold ${JSON.stringify(threshold)}`, async () => {
+ const { output } = await compose({
+ services: [petstore, await service('vaccination')],
+ output: 'graphql',
+ });
+ const server = await serve({ fusiongraph: output });
+ const result = await tbench.sustain({
+ server,
+ params: {
+ query: /* GraphQL */ `
+ query GetPet {
+ getPetById(petId: 1) {
+ __typename
+ id
+ name
+ vaccinated
+ }
+ }
+ `,
+ },
+ });
+
+ console.debug(result);
+
+ expect(result.maxCpu).toBeLessThan(threshold.maxCpu);
+ expect(result.maxMem).toBeLessThan(threshold.maxMem);
+ expect(result.slowestRequest).toBeLessThan(threshold.slowestRequest);
+});

e2e/federation-example/federation-example.loadtest.ts

@@ -0,0 +1,129 @@
+import os from 'os';
+import path from 'path';
+import { createTbench, Tbench, TbenchResult } from '@e2e/tbench';
+import { createTenv, Service } from '@e2e/tenv';
+
+const { fs, serve, service, spawn } = createTenv(__dirname);
+
+let tbench: Tbench;
+beforeAll(async () => {
+ tbench = await createTbench(
+ // to give space for jest and the serve process.
+ os.availableParallelism() - 2,
+ );
+});
+
+let services!: Service[];
+let supergraph!: string;
+beforeAll(async () => {
+ services = [
+ await service('accounts'),
+ await service('inventory'),
+ await service('products'),
+ await service('reviews'),
+ ];
+
+ const supergraphConfig = { subgraphs: {} };
+ for (const service of services) {
+ supergraphConfig.subgraphs[service.name] = {
+ routing_url: `http://0.0.0.0:${service.port}/graphql`,
+ schema: {
+ file: path.join(__dirname, 'services', service.name, 'typeDefs.graphql'),
+ },
+ };
+ }
+
+ const supergraphConfigFile = await fs.tempfile('supergraph.json');
+ await fs.write(supergraphConfigFile, JSON.stringify(supergraphConfig));
+
+ const [proc, waitForExit] = await spawn(
+ `yarn rover supergraph compose --config ${supergraphConfigFile}`,
+ );
+ await waitForExit;
+
+ supergraph = proc.getStd('out');
+});
+
+const threshold: TbenchResult = {
+ maxCpu: Infinity, // we dont care
+ maxMem: 500, // MB
+ slowestRequest: 1, // second
+};
+
+it(`should perform within threshold ${JSON.stringify(threshold)}`, async () => {
+ const supergraphFile = await fs.tempfile('supergraph.graphql');
+ await fs.write(supergraphFile, supergraph);
+ const server = await serve({ supergraph: supergraphFile });
+
+ const result = await tbench.sustain({
+ server,
+ params: {
+ query: /* GraphQL */ `
+ fragment User on User {
+ id
+ username
+ name
+ }
+
+ fragment Review on Review {
+ id
+ body
+ }
+
+ fragment Product on Product {
+ inStock
+ name
+ price
+ shippingEstimate
+ upc
+ weight
+ }
+
+ query TestQuery {
+ users {
+ ...User
+ reviews {
+ ...Review
+ product {
+ ...Product
+ reviews {
+ ...Review
+ author {
+ ...User
+ reviews {
+ ...Review
+ product {
+ ...Product
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ topProducts {
+ ...Product
+ reviews {
+ ...Review
+ author {
+ ...User
+ reviews {
+ ...Review
+ product {
+ ...Product
+ }
+ }
+ }
+ }
+ }
+ }
+ `,
+ },
+ });
+
+ console.debug(result);
+
+ expect(result.maxCpu).toBeLessThan(threshold.maxCpu);
+ expect(result.maxMem).toBeLessThan(threshold.maxMem);
+ expect(result.slowestRequest).toBeLessThan(threshold.slowestRequest);
+});

e2e/mysql-employees/mysql-employees.loadtest.ts

@@ -0,0 +1,79 @@
+import os from 'os';
+import { createTbench, Tbench, TbenchResult } from '@e2e/tbench';
+import { Container, createTenv } from '@e2e/tenv';
+
+const { compose, serve, container } = createTenv(__dirname);
+
+let tbench: Tbench;
+beforeAll(async () => {
+ tbench = await createTbench(
+ // to give space for jest and the serve process.
+ os.availableParallelism() - 2,
+ );
+});
+
+let mysql!: Container;
+beforeAll(async () => {
+ mysql = await container({
+ name: 'employees',
+ image: 'genschsa/mysql-employees',
+ containerPort: 3306,
+ healthcheck: [
+ 'CMD',
+ 'mysqladmin',
+ 'ping',
+ '--host=127.0.0.1', // use the network connection (and not the socket file). making sure we dont check the temporary/setup database
+ ],
+ env: {
+ MYSQL_ROOT_PASSWORD: 'passwd', // used in mesh.config.ts
+ },
+ });
+});
+
+const threshold: TbenchResult = {
+ maxCpu: Infinity, // we dont care
+ maxMem: 500, // MB
+ slowestRequest: 1, // second
+};
+
+it(`should perform within threshold ${JSON.stringify(threshold)}`, async () => {
+ const { output } = await compose({ output: 'graphql', services: [mysql] });
+ const server = await serve({ fusiongraph: output });
+ const result = await tbench.sustain({
+ server,
+ params: {
+ query: /* GraphQL */ `
+ query GetSomeEmployees {
+ employees(limit: 5, orderBy: { emp_no: asc }) {
+ __typename
+ emp_no
+ # TODO: dates are different in GH actions
+ # birth_date
+ first_name
+ last_name
+ gender
+ # TODO: dates are different in GH actions
+ # hire_date
+ dept_emp {
+ emp_no
+ dept_no
+ # TODO: dates are different in GH actions
+ # from_date
+ # to_date
+ departments {
+ dept_no
+ dept_name
+ }
+ }
+ }
+ }
+ `,
+ },
+ });
+
+ console.debug(result);
+
+ expect(result.maxCpu).toBeLessThan(threshold.maxCpu);
+ expect(result.maxMem).toBeLessThan(threshold.maxMem);
+ expect(result.slowestRequest).toBeLessThan(threshold.slowestRequest);
+});

e2e/sqlite-chinook/sqlite-chinook.loadtest.ts

@@ -0,0 +1,46 @@
+import os from 'os';
+import { createTbench, Tbench, TbenchResult } from '@e2e/tbench';
+import { createTenv } from '@e2e/tenv';
+
+const { serve, compose } = createTenv(__dirname);
+
+let tbench: Tbench;
+beforeAll(async () => {
+ tbench = await createTbench(
+ // to give space for jest and the serve process.
+ os.availableParallelism() - 2,
+ );
+});
+
+const threshold: TbenchResult = {
+ maxCpu: Infinity, // we dont care
+ maxMem: 500, // MB
+ slowestRequest: 1, // second
+};
+
+it(`should perform within threshold ${JSON.stringify(threshold)}`, async () => {
+ const { output } = await compose({ output: 'graphql' });
+ const server = await serve({ fusiongraph: output });
+ const result = await tbench.sustain({
+ server,
+ params: {
+ query: /* GraphQL */ `
+ query Albums {
+ albums(limit: 2) {
+ albumId
+ title
+ artist {
+ name
+ }
+ }
+ }
+ `,
+ },
+ });
+
+ console.debug(result);
+
+ expect(result.maxCpu).toBeLessThan(threshold.maxCpu);
+ expect(result.maxMem).toBeLessThan(threshold.maxMem);
+ expect(result.slowestRequest).toBeLessThan(threshold.slowestRequest);
+});

e2e/thrift-calculator/thrift-calculator.loadtest.ts

@@ -0,0 +1,43 @@
+import os from 'os';
+import { createTbench, Tbench, TbenchResult } from '@e2e/tbench';
+import { createTenv } from '@e2e/tenv';
+
+const { serve, compose, service } = createTenv(__dirname);
+
+let tbench: Tbench;
+beforeAll(async () => {
+ tbench = await createTbench(
+ // to give space for jest and the serve process.
+ os.availableParallelism() - 2,
+ );
+});
+
+const threshold: TbenchResult = {
+ maxCpu: Infinity, // we dont care
+ maxMem: 500, // MB
+ slowestRequest: 1, // second
+};
+
+it(`should perform within threshold ${JSON.stringify(threshold)}`, async () => {
+ const { output } = await compose({
+ services: [await service('calculator')],
+ output: 'graphql',
+ });
+ const server = await serve({ fusiongraph: output });
+ const result = await tbench.sustain({
+ server,
+ params: {
+ query: /* GraphQL */ `
+ query Add {
+ add(request: { left: 2, right: 3 })
+ }
+ `,
+ },
+ });
+
+ console.debug(result);
+
+ expect(result.maxCpu).toBeLessThan(threshold.maxCpu);
+ expect(result.maxMem).toBeLessThan(threshold.maxMem);
+ expect(result.slowestRequest).toBeLessThan(threshold.slowestRequest);
+});