WSTEAM1-547: Update to Node 18 (#11015)

* Update Next.JS to 13.x

* Update next.config.js

* Update to NextJS 13.4.1 - Include new Undici agent resolver

* Add 'react' type pathing to tsconfig

* convert jest.config to .ts file - add necessary devDeps

* why do we need react type defs?

* Force resolution of `types/react` as the latest version doesn't work in latest Next.JS

* Update undici to memoize and add test

* Moving resolution to Express package-json

* Remove transformIgnorePatterns in favour of default behaviour

* Test removing resolution

* Update types/react-dom

* Yarn cache updates

* Add undici

* Update to Next 13.4.9

* Move undici to NextJS app

* NextJS update

* NextJS dep updates

* Update Next to `13.4.12`

* Use old getAgent until after Node 18 upgrade

* Maybe it does need to be used...

* Yarn cache

* Yarn checksum fix

* Pin `@types/react`

* WSTEAM1-547: Change node version

* WSTEAM1-547: Update ramda

* WSTEAM1-547: Change ramda order

* WSTEAM1-547: Revert related content section

* WSTEAM1-547: Update ramda types

* WSTEAM1-547: Update mostread snapshots

* Update to `13.4.18`

* WSTEAM1-547: Add shebang

* WSTEAM1-547: Merge undici

* Revert

* Set `--no-experimental-fetch` flag on app start

* Test Lighthouse v11

* Remove `expectCt` header and `prefetch-src` header

- Newer versions of Lighthouse drop the 'bestPractices' score significantly if these now deprecated APIs are set.

* Remove deprecated `prefetch-src` csp

* Increase Lighthouse score budget

* Test removing Puppeteer version pin

* Revert "Test removing Puppeteer version pin"

This reverts commit b451061.

* Disable experimental fetch on Lambda start

* Test logging preview to console for debugging

* Revert "Test logging preview to console for debugging"

This reverts commit 0940c88.

---------

Co-authored-by: Aaron Moore <[email protected]>

.github/workflows/simorgh-e2e-tests.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [18.x]
     env:
       CI: true
       LOG_LEVEL: 'error'

.github/workflows/simorgh-integration-tests.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [18.x]
     env:
       CI: true
       LOG_LEVEL: 'error'

.github/workflows/simorgh-local-server-tests.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [18.x]
     env:
       CI: true
       LOG_LEVEL: 'error'

.github/workflows/simorgh-misc-checks.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [18.x]
     env:
       CI: true
 

.github/workflows/simorgh-release-asset.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [18.x]
     env:
       CI: true
 

.github/workflows/simorgh-unit-tests.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [18.x]
     env:
       CI: true
       LOG_LEVEL: 'error'

.nvmrc

@@ -1 +1 @@
-v16.17.1
+v18.17.1

cypress/support/helpers/onDemandRadioTv.js

@@ -32,10 +32,12 @@ export const getEmbedUrl = ({ body, language, isAmp }) => {
   return isAmp ? `${embedUrl}/amp` : embedUrl;
 };
 
-export const isAvailable = pathEq(
-  ['content', 'blocks', '0', 'availability'],
-  'available',
-);
+export const isAvailable = pathEq('available', [
+  'content',
+  'blocks',
+  '0',
+  'availability',
+]);
 
 export const getEpisodeAvailability = path([
   'content',

package.json

@@ -40,12 +40,12 @@
     "cypress:adhoc": "yarn cypress:interactive -- --project ./AdHocCypress/. --config-file cypress-simorgh.json",
     "setupDevEnv": "cp envConfig/local.env .env && rm -rf envConfig/secret.env && ./scripts/checkSecretEnvVariables.sh",
     "dependencyFreshness": "node ./scripts/dependencyFreshness",
-    "dev": "yarn setupDevEnv && rm -rf build && run-p webpack:dev:client webpack:dev:server",
+    "dev": "yarn setupDevEnv && rm -rf build && NODE_OPTIONS=--no-experimental-fetch run-p webpack:dev:client webpack:dev:server",
     "lighthouse": "./scripts/lighthouseRun.sh",
     "postshrinkwrap": "test -z $CI && ./scripts/packagelockHttps.sh; git update-index --assume-unchanged .env",
     "prepare": "husky install",
     "preinstall": "node scripts/check-package-manager.js",
-    "start": "NODE_ENV=production node  --max-old-space-size=3500 build/server.js",
+    "start": "NODE_ENV=production node --no-experimental-fetch --max-old-space-size=3500 build/server.js",
     "stop": "./scripts/killApp.sh",
     "storybook": "node .storybook/buildFontPreloads && start-storybook -p 9001 -s .storybook/static -c .storybook",
     "test": "yarn build && yarn test:local",
@@ -111,7 +111,7 @@
     "morgan": "1.10.0",
     "path-to-regexp": "6.2.1",
     "polyfill-crypto.getrandomvalues": "1.0.0",
-    "ramda": "0.28.0",
+    "ramda": "0.29.0",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-helmet": "6.1.0",
@@ -163,7 +163,7 @@
     "@types/js-cookie": "^3.0.3",
     "@types/jsdom": "^20.0.0",
     "@types/loadable__component": "5.13.4",
-    "@types/ramda": "0.28.15",
+    "@types/ramda": "0.29.3",
     "@types/react": "18.2.20",
     "@types/react-dom": "18.2.7",
     "@types/react-helmet": "6.1.6",

scripts/checkSecretEnvVariables.sh

@@ -1,3 +1,4 @@
+#!/bin/sh
 if [ -z "$BFF_PATH" ]; then 
     echo No BFF_PATH set as environment variable, you will not have access to topics; 
 fi

scripts/lighthouseBudget.js

@@ -72,8 +72,8 @@ const exitResult = isPassing => {
 // START OF SCRIPT
 
 const budget = {
-  ally: 85,
-  bestPractises: 85,
+  ally: 90,
+  bestPractises: 90,
   seo: 100,
 };
 

scripts/lighthouseRun.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 set -e
 
-npx -p lighthouse@6 lighthouse http://localhost:7080/news/articles/cn7k01xp8kxo --chrome-flags="--no-sandbox --headless --disable-gpu" --output json --output html --output-path simorgh --config-path scripts/lighthouseConfig.js && node scripts/lighthouseBudget.js run
-npx -p lighthouse@6 lighthouse http://localhost:7080/kyrgyz --chrome-flags="--no-sandbox --headless --disable-gpu" --output json --output html --output-path simorgh --config-path scripts/lighthouseConfig.js && node scripts/lighthouseBudget.js run
+npx -p lighthouse@11 lighthouse http://localhost:7080/news/articles/cn7k01xp8kxo --chrome-flags="--no-sandbox --headless --disable-gpu" --output json --output html --output-path simorgh --config-path scripts/lighthouseConfig.js && node scripts/lighthouseBudget.js run
+npx -p lighthouse@11 lighthouse http://localhost:7080/kyrgyz --chrome-flags="--no-sandbox --headless --disable-gpu" --output json --output html --output-path simorgh --config-path scripts/lighthouseConfig.js && node scripts/lighthouseBudget.js run

src/app/components/RelatedContentSection/index.tsx

@@ -109,15 +109,15 @@ const RelatedContentSection = ({ content }: { content: OptimoBlock[] }) => {
   const eventTrackingDataSend = path<object>(['block'], eventTrackingData);
   const viewRef = useViewTracker(eventTrackingDataSend);
 
-  if (!pathEq(['type'], 'relatedContent', blocks)) return null;
+  if (!pathEq('relatedContent', ['type'], blocks)) return null;
 
   if (!blocks) return null;
 
   const items = pathOr([], ['model', 'blocks'], blocks);
   const LABEL_ID = 'related-content-heading';
 
   const customTitle =
-    pathEq([0, 'type'], 'title', items) &&
+    pathEq('title', [0, 'type'], items) &&
     pathOr(
       [],
       [0, 'model', 'blocks', 0, 'model', 'blocks', 0, 'model', 'text'],

src/app/pages/ArticlePage/ArticlePage.jsx

@@ -90,7 +90,7 @@ const ArticlePage = ({ pageData }) => {
   const aboutTags = getAboutTags(pageData);
   const topics = path(['metadata', 'topics'], pageData);
   const blocks = pathOr([], ['content', 'model', 'blocks'], pageData);
-  const startsWithHeading = propEq('type', 'headline')(blocks[0] || {});
+  const startsWithHeading = propEq('headline', 'type')(blocks[0] || {});
 
   const bylineBlock = blocks.find(block => block.type === 'byline');
   const bylineContribBlocks = pathOr([], ['model', 'blocks'], bylineBlock);

src/app/pages/ArticlePage/PagePromoSections/RelatedContentSection/index.jsx

@@ -83,15 +83,15 @@ const RelatedContentSection = ({ content }) => {
   const eventTrackingDataSend = path(['block'], eventTrackingData);
   const viewRef = useViewTracker(eventTrackingDataSend);
 
-  if (!pathEq(['type'], 'relatedContent', blocks)) return null;
+  if (!pathEq('relatedContent', ['type'], blocks)) return null;
 
   if (!blocks) return null;
 
   const items = pathOr([], ['model', 'blocks'], blocks);
   const LABEL_ID = 'related-content-heading';
 
   const customTitle =
-    pathEq([0, 'type'], 'title', items) &&
+    pathEq('title', [0, 'type'], items) &&
     pathOr(
       [],
       [0, 'model', 'blocks', 0, 'model', 'blocks', 0, 'model', 'text'],

src/app/routes/cpsAsset/getInitialData/augmentWithDisclaimer/index.js

@@ -5,7 +5,7 @@ import insert from 'ramda/src/insert';
 
 const getBlocks = pathOr([], ['content', 'model', 'blocks']);
 const setBlocks = assocPath(['content', 'model', 'blocks']);
-const isDisclaimerToggledOn = pathEq(['disclaimer', 'enabled'], true);
+const isDisclaimerToggledOn = pathEq(true, ['disclaimer', 'enabled']);
 const disclaimerBlock = {
   type: 'disclaimer',
   model: {},

src/app/routes/cpsAsset/getInitialData/insertPodcastPromo/index.js

@@ -53,7 +53,7 @@ const getPromoPosition = pageData => {
   const blocks = getBlocks(pageData);
   const targetIndex = findNthIndex(
     EARLIEST_PARAGRAPH_INSERTION_POINT,
-    pathEq(['type'], 'paragraph'),
+    pathEq('paragraph', ['type']),
     blocks,
   );
   if (targetIndex < 0) return targetIndex;

src/app/routes/cpsAsset/getInitialData/insertPodcastPromo/index.test.js

@@ -77,7 +77,7 @@ describe('insertPodcastPromo', () => {
     ({ inputBlocks, expectedIndex }) => {
       const findClearedIndex = transformedData =>
         transformedData.content.blocks.findIndex(
-          pathEq(['simorghMetadata', 'clear'], true),
+          pathEq(true, ['simorghMetadata', 'clear']),
         );
 
       pipe(

src/app/routes/utils/sharedDataTransformers/augmentWithTimestamp/index.js

@@ -43,7 +43,7 @@ const insertTimestampBlock = (originalJson, timestampBlock) => {
 
   const blocks = path(blockPath, json);
   const gistPosition = pathOr([], blockPath, json).findIndex(
-    pathEq(['type'], 'group'),
+    pathEq('group', ['type']),
   );
 
   if (gistPosition >= 0) {

src/integration/utils/fetchDom.js

@@ -1,5 +1,12 @@
 /* eslint-disable no-console */
+
 const fetch = require('isomorphic-fetch');
+
+// https://github.com/node-fetch/node-fetch/issues/1624#issuecomment-1407717012
+const dns = require('node:dns');
+
+dns.setDefaultResultOrder('ipv4first');
+
 const { JSDOM } = require('jsdom');
 const retry = require('retry');
 

src/server/index.jsx

@@ -84,6 +84,7 @@ server
   .use(compression())
   .use(
     helmet({
+      expectCt: false,
       frameguard: { action: 'deny' },
       contentSecurityPolicy: false,
     }),
@@ -181,7 +182,6 @@ server.get(
     injectResourceHintsHeader,
   ],
   async ({ url, query, headers, path: urlPath }, res) => {
-    res.removeHeader('Expect-CT');
     logger.info(SERVER_SIDE_RENDER_REQUEST_RECEIVED, {
       url,
       headers: removeSensitiveHeaders(headers),

src/server/utilities/cspHeader/index.js

@@ -61,7 +61,6 @@ const advertisingDirectives = {
     'https://*.webcontentassessor.com',
     ...advertisingServiceCountryDomains,
   ],
-  prefetchSrc: ['https://*.googlesyndication.com'],
   defaultSrc: [...bbcDomains, 'https://*.googlesyndication.com'],
   styleSrc: ['https://fonts.googleapis.com'],
   fontSrc: ['https://fonts.gstatic.com'],
@@ -334,12 +333,6 @@ const directives = {
     ampNonLive: [...bbcDomains],
     canonicalNonLive: [...bbcDomains],
   },
-  prefetchSrc: {
-    ampLive: [...advertisingDirectives.prefetchSrc],
-    canonicalLive: [...advertisingDirectives.prefetchSrc],
-    ampNonLive: [...advertisingDirectives.prefetchSrc],
-    canonicalNonLive: [...advertisingDirectives.prefetchSrc],
-  },
 };
 
 export const generateChildSrc = ({ isAmp }) => (isAmp ? ['blob:'] : ["'self'"]);
@@ -402,13 +395,6 @@ export const generateWorkerSrc = ({ isAmp }) =>
     ? ['blob:', '*.bbc.co.uk', '*.bbc.com']
     : ['blob:', "'self'", '*.bbc.co.uk', '*.bbc.com'];
 
-export const generatePrefetchSrc = ({ isAmp, isLive }) => {
-  if (!isLive && isAmp) return directives.prefetchSrc.ampNonLive.sort();
-  if (!isLive && !isAmp) return directives.prefetchSrc.canonicalNonLive.sort();
-  if (isLive && isAmp) return directives.prefetchSrc.ampLive.sort();
-  return directives.prefetchSrc.canonicalLive.sort();
-};
-
 const helmetCsp = ({ isAmp, isLive, reportOnlyOnLive }) => ({
   directives: {
     'default-src': generateDefaultSrc(),
@@ -421,7 +407,6 @@ const helmetCsp = ({ isAmp, isLive, reportOnlyOnLive }) => ({
     'style-src': generateStyleSrc({ isAmp, isLive }),
     'media-src': generateMediaSrc({ isAmp, isLive }),
     'worker-src': generateWorkerSrc({ isAmp }),
-    'prefetch-src': generatePrefetchSrc({ isAmp, isLive }),
     'report-to': 'worldsvc',
     'upgrade-insecure-requests': [],
   },

src/server/utilities/cspHeader/index.test.js

@@ -9,7 +9,6 @@ import injectCspHeader, {
   generateStyleSrc,
   generateMediaSrc,
   generateWorkerSrc,
-  generatePrefetchSrc,
 } from '.';
 
 import { bbcDomains, advertisingServiceCountryDomains } from './domainLists';
@@ -121,7 +120,6 @@ describe('cspHeader', () => {
       styleSrcExpectation: [...bbcDomains, "'unsafe-inline'"].sort(),
       mediaSrcExpectation: [...bbcDomains].sort(),
       workerSrcExpectation: ['blob:', '*.bbc.co.uk', '*.bbc.com'],
-      prefetchSrcExpectation: ['https://*.googlesyndication.com'],
     },
     {
       isAmp: false,
@@ -242,7 +240,6 @@ describe('cspHeader', () => {
       ].sort(),
       mediaSrcExpectation: [...bbcDomains].sort(),
       workerSrcExpectation: ['blob:', "'self'", '*.bbc.co.uk', '*.bbc.com'],
-      prefetchSrcExpectation: ['https://*.googlesyndication.com'],
     },
     {
       isAmp: true,
@@ -324,7 +321,6 @@ describe('cspHeader', () => {
       styleSrcExpectation: [...bbcDomains, "'unsafe-inline'"].sort(),
       mediaSrcExpectation: [...bbcDomains].sort(),
       workerSrcExpectation: ['blob:', '*.bbc.co.uk', '*.bbc.com'],
-      prefetchSrcExpectation: ['https://*.googlesyndication.com'],
     },
     {
       isAmp: false,
@@ -450,7 +446,6 @@ describe('cspHeader', () => {
       ].sort(),
       mediaSrcExpectation: [...bbcDomains].sort(),
       workerSrcExpectation: ['blob:', "'self'", '*.bbc.co.uk', '*.bbc.com'],
-      prefetchSrcExpectation: ['https://*.googlesyndication.com'].sort(),
     },
   ].forEach(
     ({
@@ -468,7 +463,6 @@ describe('cspHeader', () => {
       styleSrcExpectation,
       mediaSrcExpectation,
       workerSrcExpectation,
-      prefetchSrcExpectation,
     }) => {
       describe(`Given isAmp ${isAmp} & isLive ${isLive}`, () => {
         it(`Then it has this childSrc`, () => {
@@ -527,12 +521,6 @@ describe('cspHeader', () => {
           expect(generateWorkerSrc({ isAmp })).toEqual(workerSrcExpectation);
         });
 
-        it(`Then it has this prefetchSrc`, () => {
-          expect(generatePrefetchSrc({ isAmp, isLive })).toEqual(
-            prefetchSrcExpectation,
-          );
-        });
-
         it(`Then injectCspHeader middleware applies the correct Content-Security-Policy header`, () => {
           process.env.SIMORGH_APP_ENV = isLive ? 'live' : 'test';
 
@@ -551,7 +539,6 @@ describe('cspHeader', () => {
             `style-src ${styleSrcExpectation.join(' ')};` +
             `media-src ${mediaSrcExpectation.join(' ')};` +
             `worker-src ${workerSrcExpectation.join(' ')};` +
-            `prefetch-src ${prefetchSrcExpectation.join(' ')};` +
             `report-to worldsvc;` +
             `upgrade-insecure-requests`;
 

ws-nextjs-app/package.json

@@ -9,8 +9,8 @@
     "build:test": "cp ../envConfig/test.env .env && NODE_ENV=production && next build",
     "build:live": "cp ../envConfig/live.env .env && NODE_ENV=production && next build",
     "build": "yarn build:local",
-    "dev": "yarn setupDevEnv && next dev -p 7081",
-    "start": "next start -p 7081 && NODE_ENV=production",
+    "dev": "yarn setupDevEnv && NODE_OPTIONS=--no-experimental-fetch next dev -p 7081",
+    "start": "NODE_OPTIONS=--no-experimental-fetch next start -p 7081 && NODE_ENV=production",
     "stop": "lsof -t -i:7081 | xargs kill",
     "test": "jest --ci --colors --selectProjects='Unit Tests'",
     "test:integration": "jest --ci --colors --selectProjects='Integration Tests - Canonical'",

ws-nextjs-app/run.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 # Execute server.js, which is output using "next build"
-exec node ws-nextjs-app/server.js
+exec node --no-experimental-fetch ws-nextjs-app/server.js