From 1aac6a594fbffba15f1be6af9bc329a04e2b25d8 Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Thu, 19 Dec 2024 10:22:18 +0100 Subject: [PATCH 1/7] dev --- packages/cubejs-cli/package.json | 1 + packages/cubejs-cli/src/command/auth.ts | 2 +- packages/cubejs-cli/src/command/deploy.ts | 112 +++---------- packages/cubejs-cli/src/config.ts | 183 +--------------------- packages/cubejs-cli/src/deploy.ts | 63 -------- 5 files changed, 29 insertions(+), 332 deletions(-) delete mode 100644 packages/cubejs-cli/src/deploy.ts diff --git a/packages/cubejs-cli/package.json b/packages/cubejs-cli/package.json index 3c4fde300142f..964521914fddf 100644 --- a/packages/cubejs-cli/package.json +++ b/packages/cubejs-cli/package.json @@ -33,6 +33,7 @@ "@cubejs-backend/dotenv": "^9.0.2", "@cubejs-backend/schema-compiler": "1.1.11", "@cubejs-backend/shared": "1.1.10", + "@cubejs-backend/cloud": "1.1.10", "chalk": "^2.4.2", "cli-progress": "^3.10", "commander": "^2.19.0", diff --git a/packages/cubejs-cli/src/command/auth.ts b/packages/cubejs-cli/src/command/auth.ts index fb771a0713da9..45721fc895316 100644 --- a/packages/cubejs-cli/src/command/auth.ts +++ b/packages/cubejs-cli/src/command/auth.ts @@ -1,7 +1,7 @@ import type { CommanderStatic } from 'commander'; +import { Config } from '@cubejs-backend/cloud'; import { displayError, event } from '../utils'; -import { Config } from '../config'; const authenticate = async (currentToken: string) => { const config = new Config(); diff --git a/packages/cubejs-cli/src/command/deploy.ts b/packages/cubejs-cli/src/command/deploy.ts index aadfb2e531ce7..21c85af9dc709 100644 --- a/packages/cubejs-cli/src/command/deploy.ts +++ b/packages/cubejs-cli/src/command/deploy.ts @@ -2,30 +2,25 @@ import fs from 'fs-extra'; import path from 'path'; import cliProgress from 'cli-progress'; import { CommanderStatic } from 'commander'; +import { CubeCloudClient, DeployController } from '@cubejs-backend/cloud'; +import { ConfigCli } from '../config'; -import { DeployDirectory } from '../deploy'; import { logStage, displayError, event } from '../utils'; -import { Config } from '../config'; -const deploy = async ({ directory, auth, uploadEnv, token }: any) => { +const deploy = async ({ directory, auth, token }: any) => { if (!(await fs.pathExists(path.join(process.cwd(), 'node_modules', '@cubejs-backend/server-core')))) { await displayError( '@cubejs-backend/server-core dependency not found. Please run deploy command from project root directory and ensure npm install has been run.' ); } + const config = new ConfigCli(); if (token) { - const config = new Config(); await config.addAuthToken(token); - - await event({ - event: 'Cube Cloud CLI Authenticate' - }); - + await event({ event: 'Cube Cloud CLI Authenticate' }); console.log('Token successfully added!'); } - const config = new Config(); const bar = new cliProgress.SingleBar({ format: '- Uploading files | {bar} | {percentage}% || {value} / {total} | {file}', barCompleteChar: '\u2588', @@ -33,83 +28,26 @@ const deploy = async ({ directory, auth, uploadEnv, token }: any) => { hideCursor: true }); - const deployDir = new DeployDirectory({ directory }); - const fileHashes: any = await deployDir.fileHashes(); - - const upstreamHashes = await config.cloudReq({ - url: (deploymentId: string) => `build/deploy/${deploymentId}/files`, - method: 'GET', - auth - }); - - const { transaction, deploymentName } = await config.cloudReq({ - url: (deploymentId: string) => `build/deploy/${deploymentId}/start-upload`, - method: 'POST', - auth - }); - - if (uploadEnv) { - const envVariables = await config.envFile(`${directory}/.env`); - await config.cloudReq({ - url: (deploymentId) => `build/deploy/${deploymentId}/set-env`, - method: 'POST', - body: { - envVariables: JSON.stringify(envVariables), - }, - auth - }); - } - - await logStage(`Deploying ${deploymentName}...`, 'Cube Cloud CLI Deploy'); - - const files = Object.keys(fileHashes); - const fileHashesPosix = {}; - - bar.start(files.length, 0, { - file: '' - }); - - try { - for (let i = 0; i < files.length; i++) { - const file = files[i]; + const cubeCloudClient = new CubeCloudClient(auth || (await config.deployAuthForCurrentDir())); + const deployController = new DeployController(cubeCloudClient, { + onStart: async (deploymentName, files) => { + await logStage(`Deploying ${deploymentName}...`, 'Cube Cloud CLI Deploy'); + bar.start(files.length, 0, { + file: '' + }); + }, + onUpdate: (i, { file }) => { bar.update(i, { file }); - - const filePosix = file.split(path.sep).join(path.posix.sep); - fileHashesPosix[filePosix] = fileHashes[file]; - - if (!upstreamHashes[filePosix] || upstreamHashes[filePosix].hash !== fileHashes[file].hash) { - await config.cloudReq({ - url: (deploymentId: string) => `build/deploy/${deploymentId}/upload-file`, - method: 'POST', - formData: { - transaction: JSON.stringify(transaction), - fileName: filePosix, - file: { - value: fs.createReadStream(path.join(directory, file)), - options: { - filename: path.basename(file), - contentType: 'application/octet-stream' - } - } - }, - auth - }); - } + }, + onUpload: (files) => { + bar.update(files.length, { file: 'Post processing...' }); + }, + onFinally: () => { + bar.stop(); } - bar.update(files.length, { file: 'Post processing...' }); - await config.cloudReq({ - url: (deploymentId: string) => `build/deploy/${deploymentId}/finish-upload`, - method: 'POST', - body: { - transaction, - files: fileHashesPosix - }, - auth - }); - } finally { - bar.stop(); - } + }); + await deployController.deploy(directory); await logStage('Done 🎉', 'Cube Cloud CLI Deploy Success'); }; @@ -119,12 +57,8 @@ export function configureDeployCommand(program: CommanderStatic) { .description('Deploy project to Cube Cloud') .option('--upload-env', 'Upload .env file to CubeCloud') .option('--token ', 'Add auth token to CubeCloud') - .option('--directory [path]', 'Specify path to conf directory', './') .action( - (options) => deploy({ - ...options, - directory: path.join(process.cwd(), options.directory) - }) + (options) => deploy({ directory: process.cwd(), ...options }) .catch(e => displayError(e.stack || e)) ) .on('--help', () => { diff --git a/packages/cubejs-cli/src/config.ts b/packages/cubejs-cli/src/config.ts index aa30251ed1cdf..7c23ec4d8ef32 100644 --- a/packages/cubejs-cli/src/config.ts +++ b/packages/cubejs-cli/src/config.ts @@ -1,89 +1,7 @@ import inquirer from 'inquirer'; -import fs from 'fs-extra'; -import rp, { RequestPromiseOptions } from 'request-promise'; -import jwt from 'jsonwebtoken'; -import path from 'path'; -import os from 'os'; -import dotenv from '@cubejs-backend/dotenv'; -import { isFilePath } from '@cubejs-backend/shared'; -import { displayWarning } from './utils'; - -type ConfigurationFull = { - auth: { - [organizationUrl: string]: { - auth: string, - } - } -}; - -type Configuration = Partial; - -export class Config { - protected async loadConfig(): Promise { - const { configFile } = this.configFile(); - - if (await fs.pathExists(configFile)) { - return fs.readJson(configFile); - } - - return {}; - } - - protected async writeConfig(config) { - const { cubeCloudConfigPath, configFile } = this.configFile(); - await fs.mkdirp(cubeCloudConfigPath); - await fs.writeJson(configFile, config); - } - - protected configFile() { - const cubeCloudConfigPath = this.cubeCloudConfigPath(); - const configFile = path.join(cubeCloudConfigPath, 'config.json'); - - return { cubeCloudConfigPath, configFile }; - } - - public async envFile(envFile: string) { - if (await fs.pathExists(envFile)) { - const env = dotenv.config({ path: envFile, multiline: 'line-breaks' }).parsed; - if (env) { - if ('CUBEJS_DEV_MODE' in env) { - delete env.CUBEJS_DEV_MODE; - } - - const resolvePossibleFiles = [ - 'CUBEJS_DB_SSL_CA', - 'CUBEJS_DB_SSL_CERT', - 'CUBEJS_DB_SSL_KEY', - ]; - - // eslint-disable-next-line no-restricted-syntax - for (const [key, value] of Object.entries(env)) { - if (resolvePossibleFiles.includes(key) && isFilePath(value)) { - if (fs.existsSync(value)) { - env[key] = fs.readFileSync(value, 'ascii'); - } else { - displayWarning(`Unable to resolve file "${value}" from ${key}`); - - env[key] = ''; - } - } - } - - return env; - } - } - - return {}; - } - - protected cubeEnvConfigPath() { - return path.join(os.homedir(), '.env'); - } - - protected cubeCloudConfigPath() { - return path.join(os.homedir(), '.cubecloud'); - } +import { Config } from '@cubejs-backend/cloud'; +export class ConfigCli extends Config { public async deployAuth(url?: string) { const config = await this.loadConfig(); @@ -103,39 +21,7 @@ export class Config { return (await this.addAuthToken(auth.auth, config)).auth; } - public async addAuthToken(authToken: string, config?: Configuration): Promise { - if (!config) { - config = await this.loadConfig(); - } - - const payload = jwt.decode(authToken); - if (payload && typeof payload === 'object' && payload.url) { - config.auth = config.auth || {}; - config.auth[payload.url] = { - auth: authToken - }; - - if (payload.deploymentId) { - const dotCubeCloud = await this.loadDotCubeCloud(); - dotCubeCloud.url = payload.url; - dotCubeCloud.deploymentId = payload.deploymentId; - await this.writeDotCubeCloud(dotCubeCloud); - } - - await this.writeConfig(config); - return config; - } - - const answer = await this.cloudTokenReq(authToken); - if (answer) { - return this.addAuthToken(answer, config); - } - - // eslint-disable-next-line no-throw-literal - throw 'Malformed Cube Cloud token'; - } - - protected async deployAuthForCurrentDir() { + public async deployAuthForCurrentDir() { const dotCubeCloud = await this.loadDotCubeCloud(); if (dotCubeCloud.url && dotCubeCloud.deploymentId) { const deployAuth = await this.deployAuth(dotCubeCloud.url); @@ -163,11 +49,7 @@ export class Config { } const authToken = auth[url]; - const deployments = await this.cloudReq({ - url: () => 'build/deploy/deployments', - method: 'GET', - auth: { ...authToken, url } - }); + const deployments = await this.cubeCloudClient.getDeploymentsList({ auth: { ...authToken, url } }); if (!Array.isArray(deployments)) { throw new Error(deployments.toString()); @@ -200,61 +82,4 @@ export class Config { deploymentId }; } - - protected dotCubeCloudFile() { - return '.cubecloud'; - } - - protected async loadDotCubeCloud() { - if (await fs.pathExists(this.dotCubeCloudFile())) { - return fs.readJson(this.dotCubeCloudFile()); - } - - return {}; - } - - protected async writeDotCubeCloud(config) { - await fs.writeJson(this.dotCubeCloudFile(), config); - } - - public async cloudReq(options: { - url: (deploymentId: string) => string, - auth: { auth: string, deploymentId?: string, url?: string }, - } & RequestPromiseOptions) { - const { url, auth, ...restOptions } = options; - - const authorization = auth || await this.deployAuthForCurrentDir(); - if (!authorization) { - throw new Error('Auth isn\'t set'); - } - - return rp({ - headers: { - authorization: authorization.auth - }, - ...restOptions, - url: `${authorization.url}/${url(authorization.deploymentId)}`, - json: true - }); - } - - protected async cloudTokenReq(authToken: string) { - const res = await rp({ - url: `${process.env.CUBE_CLOUD_HOST || 'https://cubecloud.dev'}/v1/token`, - method: 'POST', - headers: { - 'Content-type': 'application/json' - }, - json: true, - body: { - token: authToken - } - }); - - if (res && res.error) { - throw res.error; - } - - return res.jwt; - } } diff --git a/packages/cubejs-cli/src/deploy.ts b/packages/cubejs-cli/src/deploy.ts deleted file mode 100644 index 1b6158f83e12b..0000000000000 --- a/packages/cubejs-cli/src/deploy.ts +++ /dev/null @@ -1,63 +0,0 @@ -import crypto from 'crypto'; -import fs from 'fs-extra'; -import path from 'path'; - -type DeployDirectoryOptions = { - directory: string, -}; - -export class DeployDirectory { - public constructor( - protected readonly options: DeployDirectoryOptions - ) { } - - public async fileHashes(directory: string = this.options.directory) { - let result = {}; - - const files = await fs.readdir(directory); - // eslint-disable-next-line no-restricted-syntax - for (const file of files) { - const filePath = path.resolve(directory, file); - if (!this.filter(filePath)) { - // eslint-disable-next-line no-continue - continue; - } - const stat = await fs.stat(filePath); - if (stat.isDirectory()) { - result = { ...result, ...await this.fileHashes(filePath) }; - } else { - result[path.relative(this.options.directory, filePath)] = { - hash: await this.fileHash(filePath) - }; - } - } - return result; - } - - protected filter(file: string) { - const baseName = path.basename(file); - - // whitelist - if (['.gitignore'].includes(baseName)) { - return true; - } - - // blacklist - if (['dashboard-app', 'node_modules'].includes(baseName)) { - return false; - } - - return baseName.charAt(0) !== '.'; - } - - protected fileHash(file: string) { - return new Promise((resolve, reject) => { - const hash = crypto.createHash('sha1'); - const stream = fs.createReadStream(file); - - stream.on('error', err => reject(err)); - stream.on('data', chunk => hash.update(chunk)); - stream.on('end', () => resolve(hash.digest('hex'))); - }); - } -} From f007b4d27fc369b8473d75f48ed9c2e16d753738 Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Thu, 19 Dec 2024 12:12:08 +0100 Subject: [PATCH 2/7] dev --- packages/cubejs-backend-cloud/src/deploy.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cubejs-backend-cloud/src/deploy.ts b/packages/cubejs-backend-cloud/src/deploy.ts index 977319d93f099..920702730e395 100644 --- a/packages/cubejs-backend-cloud/src/deploy.ts +++ b/packages/cubejs-backend-cloud/src/deploy.ts @@ -64,10 +64,10 @@ export class DeployDirectory { } type DeployHooks = { - onStart?: Function, - onUpdate?: Function, - onUpload?: Function, - onFinally?: Function + onStart?: (deploymentName: string, files: string[]) => void, + onUpdate?: (i: number, { file }: { file: string}) => void, + onUpload?: (files: string[], file: string) => void, + onFinally?: () => void }; export class DeployController { From 1c3a73f3210d9e62eddc2a8774704e06319ad66c Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Thu, 19 Dec 2024 12:55:48 +0100 Subject: [PATCH 3/7] dev --- packages/cubejs-backend-cloud/src/deploy.ts | 6 ++++++ packages/cubejs-cli/src/command/deploy.ts | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-backend-cloud/src/deploy.ts b/packages/cubejs-backend-cloud/src/deploy.ts index 920702730e395..aa550aa61f07d 100644 --- a/packages/cubejs-backend-cloud/src/deploy.ts +++ b/packages/cubejs-backend-cloud/src/deploy.ts @@ -1,6 +1,7 @@ import crypto from 'crypto'; import path from 'path'; import fs from 'fs-extra'; +import { DotenvParseOutput } from '@cubejs-backend/dotenv'; import { CubeCloudClient } from './cloud'; type DeployDirectoryOptions = { @@ -73,6 +74,7 @@ type DeployHooks = { export class DeployController { public constructor( protected readonly cubeCloudClient: CubeCloudClient, + protected envVariables: DotenvParseOutput = {}, protected readonly hooks: DeployHooks = {} ) { } @@ -85,6 +87,10 @@ export class DeployController { const upstreamHashes = await this.cubeCloudClient.getUpstreamHashes(); const { transaction, deploymentName } = await this.cubeCloudClient.startUpload(); + if (Object.keys(this.envVariables).length) { + await this.cubeCloudClient.setEnvVars({ envVariables: this.envVariables }); + } + const files = Object.keys(fileHashes); const fileHashesPosix: Record = {}; if (this.hooks.onStart) this.hooks.onStart(deploymentName, files); diff --git a/packages/cubejs-cli/src/command/deploy.ts b/packages/cubejs-cli/src/command/deploy.ts index 21c85af9dc709..7d710a764cc44 100644 --- a/packages/cubejs-cli/src/command/deploy.ts +++ b/packages/cubejs-cli/src/command/deploy.ts @@ -7,7 +7,7 @@ import { ConfigCli } from '../config'; import { logStage, displayError, event } from '../utils'; -const deploy = async ({ directory, auth, token }: any) => { +const deploy = async ({ directory, auth, uploadEnv, token }: any) => { if (!(await fs.pathExists(path.join(process.cwd(), 'node_modules', '@cubejs-backend/server-core')))) { await displayError( '@cubejs-backend/server-core dependency not found. Please run deploy command from project root directory and ensure npm install has been run.' @@ -28,8 +28,10 @@ const deploy = async ({ directory, auth, token }: any) => { hideCursor: true }); + const envVariables = uploadEnv ? await config.envFile(`${directory}/.env`) : {}; + const cubeCloudClient = new CubeCloudClient(auth || (await config.deployAuthForCurrentDir())); - const deployController = new DeployController(cubeCloudClient, { + const deployController = new DeployController(cubeCloudClient, envVariables, { onStart: async (deploymentName, files) => { await logStage(`Deploying ${deploymentName}...`, 'Cube Cloud CLI Deploy'); bar.start(files.length, 0, { From 87bcc163807ddc29a7871b3c9aaec127b2295bae Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Thu, 19 Dec 2024 13:04:47 +0100 Subject: [PATCH 4/7] clear --- packages/cubejs-cli/src/command/deploy.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-cli/src/command/deploy.ts b/packages/cubejs-cli/src/command/deploy.ts index 7d710a764cc44..e391c8559a173 100644 --- a/packages/cubejs-cli/src/command/deploy.ts +++ b/packages/cubejs-cli/src/command/deploy.ts @@ -59,8 +59,12 @@ export function configureDeployCommand(program: CommanderStatic) { .description('Deploy project to Cube Cloud') .option('--upload-env', 'Upload .env file to CubeCloud') .option('--token ', 'Add auth token to CubeCloud') + .option('--directory [path]', 'Specify path to conf directory', './') .action( - (options) => deploy({ directory: process.cwd(), ...options }) + (options) => deploy({ + ...options, + directory: path.join(process.cwd(), options.directory) + }) .catch(e => displayError(e.stack || e)) ) .on('--help', () => { From caed6c559456bb459420df4aa1f373fabbc1ff00 Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Mon, 13 Jan 2025 12:34:32 +0100 Subject: [PATCH 5/7] clear --- packages/cubejs-cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-cli/package.json b/packages/cubejs-cli/package.json index 18a747b05edd8..c1d3fe25d2f8d 100644 --- a/packages/cubejs-cli/package.json +++ b/packages/cubejs-cli/package.json @@ -31,7 +31,7 @@ ], "dependencies": { "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/cloud": "1.1.15", + "@cubejs-backend/cloud": "1.1.13", "@cubejs-backend/schema-compiler": "1.1.15", "@cubejs-backend/shared": "1.1.12", "chalk": "^2.4.2", From 9578b4929aa345aad3fb9d6d8fdccaf25c582c09 Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Mon, 13 Jan 2025 12:54:14 +0100 Subject: [PATCH 6/7] fix --- packages/cubejs-backend-cloud/src/cloud.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/cubejs-backend-cloud/src/cloud.ts b/packages/cubejs-backend-cloud/src/cloud.ts index 93d2dfd31073b..2c14e3cff8d3a 100644 --- a/packages/cubejs-backend-cloud/src/cloud.ts +++ b/packages/cubejs-backend-cloud/src/cloud.ts @@ -113,22 +113,18 @@ export class CubeCloudClient { const formData = new FormData(); formData.append('transaction', JSON.stringify(transaction)); formData.append('fileName', fileName); - formData.append('file', { - value: data, - options: { - filename: path.basename(fileName), - contentType: 'application/octet-stream' - } + formData.append('file', data, { + filename: path.basename(fileName), + contentType: 'application/octet-stream' }); // Get the form data buffer and headers - const formDataBuffer = formData.getBuffer(); const formDataHeaders = formData.getHeaders(); return this.request({ url: (deploymentId: string) => `build/deploy/${deploymentId}/upload-file${this.extendRequestByLivePreview()}`, method: 'POST', - body: formDataBuffer, + body: formData, headers: { ...formDataHeaders, }, From 285339f2c62eb6e89cea20d1318be3f763cfbece Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Mon, 13 Jan 2025 13:02:57 +0100 Subject: [PATCH 7/7] clear --- packages/cubejs-backend-cloud/src/deploy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-backend-cloud/src/deploy.ts b/packages/cubejs-backend-cloud/src/deploy.ts index 136576fc8e98e..9abcfc0fa1049 100644 --- a/packages/cubejs-backend-cloud/src/deploy.ts +++ b/packages/cubejs-backend-cloud/src/deploy.ts @@ -79,7 +79,7 @@ export interface DeployResponse { export class DeployController { public constructor( protected readonly cubeCloudClient: CubeCloudClient, - protected envVariables: DotenvParseOutput = {}, + protected readonly envVariables: DotenvParseOutput = {}, protected readonly hooks: DeployHooks = {} ) { }