feat: migration to es8 (#218)

* feat: migration to es8

* chore: update node version

* chore: update versions

* chore: update versions

* chore: update ci

* fix: remove logs

* fix: remove withSearch option

.github/workflows/generate-cache.yaml

@@ -11,7 +11,8 @@ jobs:
       - name: Set up Node
         uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 20.3.1
+          cache: "yarn"
       - name: Cache
         run: |
           yarn

.github/workflows/ingest_day.yml

@@ -15,7 +15,8 @@ jobs:
       - name: Set up Node
         uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 20.3.1
+          cache: "yarn"
       - name: Download
         uses: docker://mcr.microsoft.com/azure-cli
         with:

.github/workflows/node-js.yml

@@ -21,15 +21,21 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: 14
-          cache: yarn
+          node-version: 20.3.1
+          cache: "yarn"
+      - name: Runs Elasticsearch
+        uses: elastic/elastic-github-actions/elasticsearch@master
+        with:
+          stack-version: 8.13.4
+          security-enabled: false
+      - name: Verify Elasticsearch connection
+        run: |
+          curl -fsSL "http://localhost:9200/_cat/health?h=status"
       - run: yarn --frozen-lockfile
       - run: yarn lint
       - run: yarn test
         env:
           CDTN_API_URL: https://code-du-travail-numerique-dev.dev.fabrique.social.gouv.fr
-          ELASTICSEARCH_URL: ${{ secrets.ELASTIC_URL }}
-          API_KEY: ${{ secrets.ELASTIC_API_KEY }}
       - run: yarn build:bin
       - run: yarn build:lib
       - run: yarn start --version

.github/workflows/release.yml

@@ -15,8 +15,8 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: lts/*
-          cache: yarn
+          node-version: 20.3.1
+          cache: "yarn"
       - run: yarn --frozen-lockfile
       - run: yarn build:lib
       - run: yarn test --coverage

.github/workflows/schedule.yaml

@@ -13,7 +13,8 @@ jobs:
       - name: Set up Node
         uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 20.3.1
+          cache: "yarn"
       - name: Backup
         uses: docker://mcr.microsoft.com/azure-cli
         with:

.github/workflows/schedule_day.yml

@@ -15,7 +15,8 @@ jobs:
       - name: Set up Node
         uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 20.3.1
+          cache: "yarn"
       - name: Backup
         uses: docker://mcr.microsoft.com/azure-cli
         with:

.node-version

@@ -1 +1 @@
-14
+20.3.1

Dockerfile

@@ -1,4 +1,5 @@
-FROM node:14.18.3-alpine3.14 as builder
+ARG NODE_VERSION=20.3.1-alpine
+FROM node:$NODE_VERSION as builder
 
 WORKDIR /app
 
@@ -9,7 +10,7 @@ COPY ./src/ src/
 RUN yarn install
 RUN yarn build:bin
 
-FROM node:14.18.3-alpine3.14
+FROM node:$NODE_VERSION
 WORKDIR /app
 COPY --from=builder /app/bin .
 

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3"
 services:
   monolog-elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4
     environment:
       - node.name=monolog-elasticsearch
       - cluster.name=es-docker-cluster
@@ -10,7 +10,7 @@ services:
       - 9200:9200
   kibana:
     container_name: monolog-kibana
-    image: docker.elastic.co/kibana/kibana:7.15.2
+    image: docker.elastic.co/kibana/kibana:8.13.4
     environment:
       - ELASTICSEARCH_HOSTS=http://monolog-elasticsearch:9200
     ports:

package.json

@@ -19,7 +19,7 @@
   "scripts": {
     "dev": "jest --watch",
     "lint": "eslint src/ --ext .ts --fix",
-    "test": "CDTN_API_URL=https://code-du-travail-numerique-preprod.dev.fabrique.social.gouv.fr jest",
+    "test": "CDTN_API_URL=https://code-du-travail-numerique-preprod.ovh.fabrique.social.gouv.fr jest",
     "test-one": "jest analysis --watch",
     "monolog": "node -r ts-node/register --max-old-space-size=45000 src/bin/index.ts",
     "build": "yarn run build:bin && yarn run build:lib",
@@ -29,7 +29,7 @@
     "check:types": "tsc"
   },
   "dependencies": {
-    "@elastic/elasticsearch": "^7.15.0",
+    "@elastic/elasticsearch": "8.13.1",
     "@socialgouv/cdtn-logger": "^1.10.4",
     "@socialgouv/cdtn-sources": "^4.55.1",
     "data-forge": "^1.8.26",
@@ -49,10 +49,10 @@
     "@socialgouv/eslint-config-recommended": "^1.84.3",
     "@types/jest": "^27.0.2",
     "@types/murmurhash-js": "^1.0.3",
-    "@types/node-fetch": "^2.5.12",
+    "@types/node-fetch": "^2.6.11",
     "@typescript-eslint/eslint-plugin": "^4.28.5",
     "@typescript-eslint/parser": "^4.28.5",
-    "@vercel/ncc": "^0.29.2",
+    "@vercel/ncc": "0.38.1",
     "eslint": "^7.32.0",
     "esm": "^3.2.25",
     "husky": "^7.0.4",
@@ -83,6 +83,6 @@
     }
   },
   "volta": {
-    "node": "14.18.3"
+    "node": "20.3.1"
   }
 }

src/bin/index.ts

@@ -1,4 +1,4 @@
-import { ConnectionError } from "@elastic/elasticsearch/lib/errors";
+import { errors } from "@elastic/elasticsearch";
 import { logger } from "@socialgouv/cdtn-logger";
 import yargs from "yargs/yargs";
 
@@ -99,7 +99,7 @@ const main = async () => {
       // .strict()
       .help().argv;
   } catch (err: any) {
-    if (err.name != undefined && err.name == ConnectionError.name) {
+    if (err instanceof errors.ConnectionError) {
       logger.error("Cannot access Elastic : " + JSON.stringify(err, null, 2));
     } else {
       console.log(err);

src/es/elastic.ts

@@ -34,23 +34,17 @@ export const getDocumentsFromES = async (
   query: any,
   // TODO use fp-ts option here
   aggs: any = undefined,
-  withDocs = true,
-  withSearch = false
+  withDocs = true
 ): Promise<DocumentResponse> => {
-  const initResponse = !withSearch
-    ? await esClient.count({
-        body: { aggs, query },
-        index,
-      })
-    : await esClient.search({
-        body: { aggs, query },
-        index,
-        size: BATCH_SIZE,
-      });
+  const initResponse: any = await esClient.search<any>({
+    aggs,
+    index,
+    query,
+    size: BATCH_SIZE,
+  });
 
-  const total = !withSearch ? initResponse.body.count : initResponse.body.hits.total.value;
-
-  const { aggregations } = initResponse.body;
+  const total = initResponse.hits.total.value;
+  const aggregations = initResponse.aggregations;
 
   const docs: any[] = [];
 
@@ -66,7 +60,7 @@ export const getDocumentsFromES = async (
       return hits[hits.length - 1].sort[0];
     };
 
-    const pointInTimeId = (await esClient.openPointInTime({ index, keep_alive: "10m" })).body.id;
+    const pointInTimeId = (await esClient.openPointInTime({ index, keep_alive: "10m" })).id;
 
     let searchAfter;
 
@@ -87,8 +81,8 @@ export const getDocumentsFromES = async (
       if (docs.length % 50000 == 0) {
         logger.debug(docs.length);
       }
-      if (!response?.body?.hits?.hits || !response.body.hits.hits.length) break;
-      searchAfter = treatResponse({ hits: response.body.hits.hits });
+      if (!response?.hits?.hits || !response.hits.hits.length) break;
+      searchAfter = treatResponse({ hits: response.hits.hits });
     }
   }
   return { aggregations, docs };
@@ -99,15 +93,14 @@ const getDocuments = async (
   query: any,
   // TODO use fp-ts option here
   aggs: any = undefined,
-  withDocs = true,
-  withSearch = false
-): Promise<DocumentResponse> => getDocumentsFromES(esClient, index, query, aggs, withDocs, withSearch);
+  withDocs = true
+): Promise<DocumentResponse> => getDocumentsFromES(esClient, index, query, aggs, withDocs);
 
 // we ensure index exists otherwise we create it
 const testAndCreateIndex = async (index: string, mappings: any) => {
-  const { body } = await esClient.indices.exists({ index });
+  const exist = await esClient.indices.exists({ index });
 
-  if (!body) {
+  if (!exist) {
     try {
       await esClient.indices.create({
         body: {
@@ -135,26 +128,23 @@ const testAndCreateIndex = async (index: string, mappings: any) => {
 };
 
 const deleteIfExists = async (index: string) => {
-  const { body } = await esClient.indices.exists({ index });
-  if (body) {
+  const exist = await esClient.indices.exists({ index });
+  if (exist) {
     await esClient.indices.delete({ index });
     logger.info(`Index ${index} deleted.`);
   }
 };
 
 const insertDocuments = async (index: string, documents: any) => {
   try {
-    const header = { index: { _index: index, _type: "_doc" } };
+    const header = { create: { _index: index } };
     const body = documents.flatMap((doc: any) => {
       return [header, doc];
     });
-    const resp = await esClient.bulk({
-      body,
-      index,
-    });
-    console.log(resp);
-    if (resp.body.errors) {
-      resp.body.items.forEach((element: any) => {
+    const resp = await esClient.bulk({ body, refresh: true });
+    console.log(`Batch ${resp.items.length} (with errors : ${resp.errors})`);
+    if (resp.errors) {
+      resp.items.forEach((element: any) => {
         if (element.index.status == 400) {
           logger.error(`Error during insertion : ${element.index.error.reason}`);
         }

src/ingestion/__tests__/ingester.test.ts

@@ -24,7 +24,7 @@ test("ingest Matomo dump to ES", async () => {
   await ingest(dumpfile, index);
   await wait(2000);
   const resp = await elastic.esClient.count({ index });
-  expect(resp.body.count).toBe(299);
+  expect(resp.count).toBe(299);
 
   const docs = await elastic.getDocuments(index, { match_all: {} });
   expect(docs).toMatchSnapshot();

src/reader/logReader.ts

@@ -145,7 +145,7 @@ export const countVisits = async (index: string, days: string[]): Promise<DataFr
 
   const getCount = (day: string) => {
     const { query, aggs } = makeAgg(day);
-    return getDocuments(index, query, aggs, false, true).then(({ aggregations }) => ({
+    return getDocuments(index, query, aggs, false).then(({ aggregations }) => ({
       count: aggregations.visit_count.value,
       day,
     }));