Proposal to add Gitlab functionalities

config/default.json

@@ -56,9 +56,22 @@
  "updateTemplateId": 7
  }
  },
+ "reporter": {
+ "githubIssues": {
+ "repositories": {
+ "declarations": "OpenTermsArchive/contrib-declarations"
+ }
+ },
+ "gitlabIssues": {
+ "repositories": {
+ "declarations": "p2b/contrib-declarations"
+ }
+ }
+ },
  "dataset": {
  "title": "sandbox",
  "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox",
+ "versionsRepositoryURLGitLab": "https://gitlab.com/p2b/contrib-versions"
  "publishingSchedule": "30 8 * * MON"
  }
  }

env.example

@@ -0,0 +1,3 @@
+GITHUB_TOKEN=
+GITLAB_TOKEN=
+GITLAB_RELEASES_TOKEN=

scripts/dataset/assets/README.templateGitLab.js

@@ -0,0 +1,65 @@
+import config from 'config';
+
+const LOCALE = 'en-EN';
+const DATE_OPTIONS = { year: 'numeric', month: 'long', day: 'numeric' };
+
+export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate }) {
+ return `# Open Terms Archive — ${title({ releaseDate })}
+
+${body({ servicesCount, firstVersionDate, lastVersionDate })}`;
+}
+
+export function title({ releaseDate }) {
+ releaseDate = releaseDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
+
+ const title = config.get('dataset.title');
+
+ return `${title} — ${releaseDate} dataset`;
+}
+
+export function body({ servicesCount, firstVersionDate, lastVersionDate }) {
+ firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
+ lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
+
+ const versionsRepositoryURLGitLab = config.get('dataset.versionsRepositoryURLGitLab');
+
+ return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}.
+
+This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [${versionsRepositoryURLGitLab}](${versionsRepositoryURLGitLab}).
+
+It has been generated with [Open Terms Archive](https://opentermsarchive.org).
+
+### Dataset format
+
+This dataset represents each version of a document as a separate [Markdown](https://spec.commonmark.org/0.30/) file, nested in a directory with the name of the service provider and in a directory with the name of the terms type. The filesystem layout will look like below.
+
+\`\`\`
+├ README.md
+├┬ Service provider 1 (e.g. Facebook)
+│├┬ Terms type 1 (e.g. Terms of Service)
+││├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-08-01T01-03-12Z.md)
+┆┆┆
+││└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-10-03T08-12-25Z.md)
+┆┆
+│└┬ Terms type X (e.g. Privacy Policy)
+│ ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md)
+┆ ┆
+│ └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md)
+┆
+└┬ Service provider Y (e.g. Google)
+ ├┬ Terms type 1 (e.g. Developer Terms)
+ │├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2019-03-12T04-18-22Z.md)
+ ┆┆
+ │└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-12-04T22-47-05Z.md)
+ └┬ Terms type Z (e.g. Privacy Policy)
+ ┆
+ ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md)
+ ┆
+ └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md)
+\`\`\`
+
+### License
+
+This dataset is made available under an [Open Database (OdBL) License](https://opendatacommons.org/licenses/odbl/1.0/) by Open Terms Archive Contributors.
+`;
+}

scripts/dataset/index.js

@@ -6,10 +6,11 @@ import config from 'config';
 import generateRelease from './export/index.js';
 import logger from './logger/index.js';
 import publishRelease from './publish/index.js';
+import publishReleaseGitLab from './publishGitLab/index.js';
 
 export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }) {
  const releaseDate = new Date();
- const archiveName = fileName || `dataset-${config.get('@opentermsarchive/engine.dataset.title')}-${releaseDate.toISOString().replace(/T.*/, '')}`;
+ const archiveName = fileName || `dataset-${config.get('dataset.title')}-${releaseDate.toISOString().replace(/T.*/, '')}`;
  const archivePath = `${path.basename(archiveName, '.zip')}.zip`; // allow to pass filename or filename.zip as the archive name and have filename.zip as the result name
 
  logger.info('Start exporting dataset…');
@@ -24,13 +25,23 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }
 
  logger.info('Start publishing dataset…');
 
- const releaseUrl = await publishRelease({
- archivePath,
- releaseDate,
- stats,
- });
+ if (typeof process.env.GITHUB_TOKEN !== 'undefined') {
+ const releaseUrl = await publishRelease({
+ archivePath,
+ releaseDate,
+ stats,
+ });
+ logger.info(`Dataset published to ${releaseUrl}`);
+ }
 
- logger.info(`Dataset published to ${releaseUrl}`);
+ if (typeof process.env.GITLAB_RELEASES_TOKEN !== 'undefined') {
+ const releaseUrl = await publishReleaseGitLab({
+ archivePath,
+ releaseDate,
+ stats,
+ });
+ logger.info(`Dataset published to ${releaseUrl}`);
+ }
 
  if (!shouldRemoveLocalCopy) {
  return;

scripts/dataset/publishGitLab/index.js

@@ -0,0 +1,81 @@
+import fsApi from 'fs';
+import path from 'path';
+import url from 'url';
+
+import config from 'config';
+import dotenv from 'dotenv';
+import { Octokit } from 'octokit';
+
+import axios from 'axios';
+import FormData from 'form-data';
+
+import * as readme from '../assets/README.templateGitLab.js';
+
+dotenv.config();
+
+const gitlabAPIUrl = "https://gitlab.com/api/v4";
+const gitlabUrl = "https://gitlab.com";
+
+export default async function publishReleaseGitLab({ archivePath, releaseDate, stats }) {
+ let projectId = null; 
+
+ // const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
+
+ const [ owner, repo ] = url.parse(config.get('dataset.versionsRepositoryURLGitLab')).pathname.split('/').filter(component => component);
+ const commonParams = { owner, repo };
+
+ try {
+ const repositoryPath = `${commonParams.owner}/${commonParams.repo}`;
+ const response = await axios.get(`${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`, {
+ headers: {
+ 'Authorization': `Bearer ${process.env.GITLAB_RELEASES_TOKEN}`
+ }
+ });
+ projectId = response.data.id;
+ } catch (error) {
+ //logger.error(`🤖 Error while obtaining projectId: ${error}`);
+ projectId = null;
+ }
+
+ const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+
+ try {
+ // First, create the release
+ const releaseResponse = await axios.post(`${gitlabAPIUrl}/projects/${projectId}/releases`, {
+ ref: 'main',
+ tag_name: tagName,
+ name: readme.title({ releaseDate }),
+ description: readme.body(stats) 
+ }, {
+ headers: {
+ 'Authorization': `Bearer ${process.env.GITLAB_RELEASES_TOKEN}`,
+ 'Content-Type': 'application/json'
+ }
+ });
+
+ const releaseId = releaseResponse.data.commit.id;
+
+ // Then, upload the ZIP file as an asset to the release
+ const formData = new FormData();
+ formData.append('name', archivePath);
+ formData.append('url', `${gitlabUrl}/${commonParams.owner}/${commonParams.repo}/-/archive/${tagName}/${archivePath}`);
+ formData.append('file', fsApi.createReadStream(archivePath), {
+ filename: path.basename(archivePath)
+ });
+
+ const uploadResponse = await axios.post(`${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`, formData, {
+ headers: {
+ ...formData.getHeaders(),
+ 'Authorization': `Bearer ${process.env.GITLAB_RELEASES_TOKEN}`
+ }
+ });
+
+ const releaseUrl = uploadResponse.data.direct_asset_url;
+
+ return releaseUrl;
+
+ } catch (error) {
+ console.error('Failed to create release or upload ZIP file:', error);
+ throw error;
+ }
+}

src/index.js

@@ -1,29 +1,24 @@
-import { createRequire } from 'module';
-
 import config from 'config';
 import cron from 'croner';
-import cronstrue from 'cronstrue';
 
 import Archivist from './archivist/index.js';
 import logger from './logger/index.js';
 import Notifier from './notifier/index.js';
 import Reporter from './reporter/index.js';
-
-const require = createRequire(import.meta.url);
+import ReporterGitlab from './reporterGitlab/index.js';
 
 export default async function track({ services, types, extractOnly, schedule }) {
  const archivist = new Archivist({
- recorderConfig: config.get('@opentermsarchive/engine.recorder'),
- fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
+ recorderConfig: config.get('recorder'),
+ fetcherConfig: config.get('fetcher'),
  });
 
  archivist.attach(logger);
 
  await archivist.initialize();
 
- const { version } = require('../package.json');
-
- logger.info(`Start Open Terms Archive engine v${version}\n`);
+ console.log('Running from src');
+ logger.info('Start Open Terms Archive\n');
 
  if (services?.length) {
  services = services.filter(serviceId => {
@@ -45,48 +40,46 @@ export default async function track({ services, types, extractOnly, schedule })
  return;
  }
 
- if (process.env.OTA_ENGINE_SENDINBLUE_API_KEY) {
- try {
- archivist.attach(new Notifier(archivist.services));
- } catch (error) {
- logger.error('Cannot instantiate the Notifier module; it will be ignored:', error);
- }
+ if (process.env.SENDINBLUE_API_KEY) {
+ archivist.attach(new Notifier(archivist.services));
  } else {
- logger.warn('Environment variable "OTA_ENGINE_SENDINBLUE_API_KEY" was not found; the Notifier module will be ignored');
+ logger.warn('Environment variable "SENDINBLUE_API_KEY" was not found; the Notifier module will be ignored');
  }
 
- if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
- if (config.has('@opentermsarchive/engine.reporter.githubIssues.repositories.declarations')) {
- try {
- const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
+ if (process.env.GITHUB_TOKEN) {
+ if (config.has('reporter.githubIssues.repositories.declarations')) {
+ const reporter = new Reporter(config.get('reporter'));
 
- await reporter.initialize();
- archivist.attach(reporter);
- } catch (error) {
- logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
- }
+ await reporter.initialize();
+ archivist.attach(reporter);
  } else {
  logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
  }
  } else {
- logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored');
+ logger.warn('Environment variable "GITHUB_TOKEN" was not found; the Reporter module will be ignored');
  }
 
+ if (process.env.GITLAB_TOKEN) {
+ if (config.has('reporter.gitlabIssues.repositories.declarations')) {
+ const reporter = new ReporterGitlab(config.get('reporter'));
+
+ await reporter.initialize();
+ archivist.attach(reporter);
+ } else {
+ logger.warn('Configuration key "reporter.gitlabIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
+ }
+ } else {
+ logger.warn('Environment variable "GITLAB_TOKEN" was not found; the ReporterGitlab module will be ignored');
+ } 
+
  if (!schedule) {
  await archivist.track({ services, types });
 
  return;
  }
 
- const trackingSchedule = config.get('@opentermsarchive/engine.trackingSchedule');
- const humanReadableSchedule = cronstrue.toString(trackingSchedule);
-
  logger.info('The scheduler is running…');
- logger.info(`Terms will be tracked ${humanReadableSchedule.toLowerCase()} in the timezone of this machine`);
+ logger.info('Terms will be tracked every six hours starting at half past midnight');
 
- cron(
- trackingSchedule,
- { protect: job => logger.warn(`Tracking scheduled at ${new Date().toISOString()} were blocked by an unfinished tracking started at ${job.currentRun().toISOString()}`) },
- () => archivist.track({ services, types }),
- );
+ cron('30 */6 * * *', () => archivist.track({ services, types }));
 }