Merge pull request #40 from eresearchqut/qa

Prod Release - Transcription Refresh

.git-blame-ignore-revs

@@ -1,3 +1,6 @@
+# Apply prettier formatting and linting to frontend
+94c54775039489c0353e3cdba1d4e2bed990aaac
+
 # Apply formatting and linting to frontend
 6d17b0c103574aed8e53dc371f17ff96f1d5121e
 

.github/workflows/deploy.yaml

@@ -16,9 +16,11 @@ jobs:
       actions: write
       contents: read
       id-token: write
-    if: github.ref == 'refs/heads/dev'
+    if: github.event.workflow_run.head_branch == 'dev'
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.workflow_run.head_branch }}
       - name: Setup api
         uses: actions/setup-node@v4
         with:
@@ -33,13 +35,15 @@ jobs:
         run: npm run test
   deploy:
     runs-on: ubuntu-latest
-    environment: ${{ github.ref_name }}
+    environment: ${{ github.event.workflow_run.head_branch }}
     permissions:
       actions: write
       contents: read
       id-token: write
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.workflow_run.head_branch }}
       - name: Create frontend output dir
         run: mkdir -p frontend/out
       - name: Setup api

README.md

@@ -2,8 +2,9 @@
 
 [eResearch QUT](https://www.qut.edu.au/research/office-of-eresearch) Transcription Service
 
-![homepage](images/homepage.png)
-![transcription](images/transcription.png)
+![upload](images/upload.png)
+![transcriptions](images/transcriptions.png)
+![player](images/player.png)
 
 ## Linting and Formatting
 

api/src/event/fileUploadHandler.ts

@@ -1,6 +1,9 @@
 import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";
 import {
   LanguageCode,
+  PiiEntityType,
+  RedactionOutput,
+  RedactionType,
   StartTranscriptionJobCommand,
   TranscribeClient,
 } from "@aws-sdk/client-transcribe";
@@ -42,48 +45,81 @@ export const handler = async (event: S3Event) => {
           continue;
         }
 
-        const languageCode = headResponse.Metadata["languagecode"];
 
-        if (languageCode) {
-          const cognitoGuid = cognitoId.split("%3A")[1];
+        const languages: string[] =
+          headResponse.Metadata["languages"].split(/,\s?/);
+        const enablePiiRedaction: boolean = JSON.parse(
+          headResponse.Metadata["enablePiiRedaction".toLowerCase()],
+        );
 
-          // Member must satisfy regular expression pattern: [a-zA-Z0-9-_.!*'()/]{1,1024}$, i.e. no colons or escaped colons
-          const outputKey = `transcription/${cognitoGuid}/${identityId}/${jobId}.json`;
-          const params = {
-            TranscriptionJobName: `${identityId}_${jobId}`,
-            LanguageCode: languageCode as LanguageCode,
-            Media: {
-              MediaFileUri: `https://s3-${region}.amazonaws.com/${bucketName}/${key}`,
-            },
-            OutputBucketName: transcribeBucket,
-            OutputKey: outputKey,
-            Settings: {
-              ShowSpeakerLabels: true,
-              ShowAlternatives: true,
-              MaxAlternatives: 10,
-              MaxSpeakerLabels: 10,
-            },
-          };
-          const transcriptionResponse = await transcribeClient.send(
-            new StartTranscriptionJobCommand(params),
-          );
-          try {
-            await jobStarted(
-              identityId,
-              jobId,
-              outputKey,
-              record["s3"],
-              transcriptionResponse,
-              headResponse.Metadata,
-            );
-          } catch (error) {
-            console.error("Failed to save job details", error);
-          }
-        } else {
-          console.error(
-            "Missing language code in metadata.",
+        const languageParams = {
+          ...(enablePiiRedaction
+            ? {
+                LanguageCode: LanguageCode.EN_US,
+              }
+            : languages.length > 1
+            ? {
+                IdentifyMultipleLanguages: true,
+                LanguageOptions: languages.map(
+                  (language) => language as LanguageCode,
+                ),
+              }
+            : languages.length > 0
+            ? {
+                LanguageCode: languages.at(0) as LanguageCode,
+              }
+            : {
+                IdentifyLanguage: true,
+              }),
+        };
+
+        const piiParams = {
+          ...(enablePiiRedaction
+            ? {
+                ContentRedaction: {
+                  RedactionOutput: RedactionOutput.REDACTED,
+                  RedactionType: RedactionType.PII,
+                  PiiEntityTypes: [PiiEntityType.ALL],
+                },
+              }
+            : {}),
+        };
+
+        const cognitoGuid = cognitoId.split("%3A")[1];
+
+        // Member must satisfy regular expression pattern: [a-zA-Z0-9-_.!*'()/]{1,1024}$, i.e. no colons or escaped colons
+        const outputKey = `transcription/${cognitoGuid}/${identityId}/${jobId}.json`;
+        const params = {
+          TranscriptionJobName: `${identityId}_${jobId}`,
+          ...languageParams,
+          ...piiParams,
+          Media: {
+            MediaFileUri: `https://s3-${region}.amazonaws.com/${bucketName}/${key}`,
+          },
+          OutputBucketName: transcribeBucket,
+          OutputKey: outputKey,
+          Settings: {
+            ShowSpeakerLabels: true,
+            ShowAlternatives: true,
+            MaxAlternatives: 10,
+            MaxSpeakerLabels: 10,
+          },
+        };
+        console.log("transcription params", params);
+        const transcriptionResponse = await transcribeClient.send(
+          new StartTranscriptionJobCommand(params),
+        );
+        try {
+          await jobStarted(
+            identityId,
+            jobId,
+            outputKey,
+            record["s3"],
+            transcriptionResponse,
             headResponse.Metadata,
           );
+        } catch (error) {
+          console.error("Failed to save job details", error);
         }
       } else {
         console.error("Unexpected key: ", key);

api/src/routes/transcriptionRoutes.ts

@@ -1,6 +1,9 @@
 import express from "express";
 
-import { getTranscriptions } from "../service/transcriptionService";
+import {
+  getTranscription,
+  getTranscriptions,
+} from "../service/transcriptionService";
 import { getIdentityId } from "../util/requestUtils";
 
 const router = express.Router();
@@ -12,4 +15,12 @@ router.get("/", (request, response) => {
   );
 });
 
+router.get("/:jobId", (request, response) => {
+  const identityId = getIdentityId(request);
+  const jobId = request.params.jobId;
+  getTranscription(identityId, jobId).then((transcription) =>
+    response.json(transcription),
+  );
+});
+
 export default router;

api/src/service/transcriptionService.ts

@@ -1,11 +1,15 @@
 import { StartTranscriptionJobResponse } from "@aws-sdk/client-transcribe";
 
 import {
+  getResource,
   getResources,
   putResource,
   updateResource,
 } from "../repository/repository";
 
+const normaliseJobId = (jobId: string): string =>
+  jobId.split("redacted-").at(-1) ?? jobId;
+
 export const jobStarted = (
   identityId: string,
   jobId: string,
@@ -28,7 +32,7 @@ export const jobStatusUpdated = (
 ) =>
   updateResource(
     identityId,
-    jobId,
+    normaliseJobId(jobId),
     "jobStatusUpdated",
     JSON.parse(JSON.stringify(jobStatusUpdated)),
   );
@@ -37,7 +41,11 @@ export const downloadKey = (
   identityId: string,
   jobId: string,
   downloadKey: string,
-) => updateResource(identityId, jobId, "downloadKey", downloadKey);
+) =>
+  updateResource(identityId, normaliseJobId(jobId), "downloadKey", downloadKey);
 
 export const getTranscriptions = (identityId: string) =>
   getResources(identityId);
+
+export const getTranscription = (identityId: string, jobId: string) =>
+  getResource(identityId, jobId);

deployment/bin/deployment.ts

@@ -4,7 +4,9 @@ import { ApiStack } from "../lib/api-stack";
 import { GitHubStack } from "../lib/github-stack";
 import { FrontEndStack } from "../lib/front-end-stack";
 import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm";
+import { execSync } from "child_process";
 
+const GIT_CURRENT_BRANCH_COMMAND = 'git rev-parse --abbrev-ref HEAD';
 type Environment = "dev" | "qa" | "prod";
 
 interface EnvironmentConfig {
@@ -35,7 +37,8 @@ const owner = "eresearchqut";
 const repo = "transcription";
 
 const githubFilters = process.env.GITHUB_FILTERS ? process.env.GITHUB_FILTERS.split(",") : undefined;
-const envName = (process.env.GITHUB_REF_NAME ?? "dev") as Environment;
+const currentBranch = execSync(GIT_CURRENT_BRANCH_COMMAND).toString("utf8").trim();
+const envName = (currentBranch ?? "dev") as Environment;
 
 const apiStackName = `${envName}-${repo}`;
 const frontendStackName = `${envName}-${repo}-frontend`;

deployment/lib/api-stack.ts

@@ -15,6 +15,7 @@ import { HttpMethods } from "aws-cdk-lib/aws-s3";
 import * as s3n from "aws-cdk-lib/aws-s3-notifications";
 import * as wafv2 from "aws-cdk-lib/aws-wafv2";
 import { NodejsFunction } from "aws-cdk-lib/aws-lambda-nodejs";
+import { Duration } from "aws-cdk-lib";
 
 /**
  * Format environment variables for use in a shell script
@@ -93,6 +94,10 @@ export class ApiStack extends cdk.Stack {
       ]
     });
     (dataBucket.node.defaultChild! as s3.CfnBucket).overrideLogicalId("TranscriptionBucket");
+    dataBucket.addLifecycleRule({
+      enabled: true,
+      expiration: Duration.days(14)
+    })
 
     const vpc = ec2.Vpc.fromLookup(this, "Vpc", { vpcId: props.parameters.VpcId });
     const apiSecurityGroup = new ec2.SecurityGroup(this, "ApiSecurityGroup", {

frontend/.eslintrc.json

@@ -1,3 +1,74 @@
 {
-  "extends": "next/core-web-vitals"
+  "plugins": [
+    "prettier"
+  ],
+  "extends": [
+    "next",
+    "next/core-web-vitals",
+    "prettier"
+  ],
+  "rules": {
+    "prettier/prettier": [
+      "error",
+      {
+        "endOfLine": "auto"
+      }
+    ],
+    "camelcase": "off",
+    "import/prefer-default-export": "off",
+    "react/jsx-filename-extension": "off",
+    "react/jsx-props-no-spreading": "off",
+    "react/no-unused-prop-types": "off",
+    "react/require-default-props": "off",
+    "react/display-name": "off",
+    "import/extensions": [
+      "error",
+      "ignorePackages",
+      {
+        "ts": "never",
+        "tsx": "never",
+        "js": "never",
+        "jsx": "never"
+      }
+    ],
+    "quotes": "off",
+    "jsx-a11y/anchor-is-valid": [
+      "error",
+      {
+        "components": [
+          "Link"
+        ],
+        "specialLink": [
+          "hrefLeft",
+          "hrefRight"
+        ],
+        "aspects": [
+          "invalidHref",
+          "preferButton"
+        ]
+      }
+    ]
+  },
+  "overrides": [
+    {
+      "files": "**/*.+(ts|tsx)",
+      "parser": "@typescript-eslint/parser",
+      "plugins": [
+        "@typescript-eslint/eslint-plugin"
+      ],
+      "extends": [
+        "plugin:@typescript-eslint/recommended",
+        "prettier"
+      ],
+      "rules": {
+        "@typescript-eslint/no-use-before-define": "off",
+        "@typescript-eslint/explicit-function-return-type": "off",
+        "@typescript-eslint/explicit-module-boundary-types": "off",
+        "@typescript-eslint/no-unused-vars": ["error", { "ignoreRestSiblings": true }],
+        "@typescript-eslint/no-explicit-any": "off",
+        "@typescript-eslint/no-var-requires": "off",
+        "@typescript-eslint/prefer-as-const": "off"
+      }
+    }
+  ]
 }

frontend/.gitignore

@@ -37,3 +37,5 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+*storybook.log

frontend/.storybook/main.ts

@@ -0,0 +1,19 @@
+import type { StorybookConfig } from "@storybook/nextjs";
+
+const config: StorybookConfig = {
+  stories: [
+    "../**/*.mdx",
+    "../**/*.stories.@(js|jsx|mjs|ts|tsx)",
+  ],
+  addons: [
+    "@storybook/addon-links",
+    "@storybook/addon-essentials",
+    "@chromatic-com/storybook",
+    "@storybook/addon-interactions",
+  ],
+  framework: {
+    name: "@storybook/nextjs",
+    options: {},
+  },
+};
+export default config;