[DEMO ONLY] Speed up CI by using "threaded installs" for pnpm

common/config/rush/repo-state.json

@@ -1,5 +1,5 @@
 // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush.
 {
- "pnpmShrinkwrapHash": "f7e84216fc4d6afef8aa7da108503ae8a6b0e9c3",
+ "pnpmShrinkwrapHash": "76ccb36747864d99980450a9e92f581ba3decf75",
  "preferredVersionsHash": "1926a5b12ac8f4ab41e76503a0d1d0dccc9c0e06"
 }

rush-plugins/rush-phased-install-plugin/.eslintrc.js

@@ -0,0 +1,11 @@
+// This is a workaround for https://github.com/eslint/eslint/issues/3458
+require('@rushstack/eslint-config/patch/modern-module-resolution');
+
+module.exports = {
+ extends: [
+ '@rushstack/eslint-config/profile/node',
+ '@rushstack/eslint-config/mixins/friendly-locals',
+ '@rushstack/eslint-config/mixins/tsdoc'
+ ],
+ parserOptions: { tsconfigRootDir: __dirname }
+};

rush-plugins/rush-phased-install-plugin/LICENSE

@@ -0,0 +1,24 @@
+@rushstack/rush-phased-install-plugin
+
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

rush-plugins/rush-phased-install-plugin/README.md

@@ -0,0 +1,8 @@
+The @rushstack/rush-phased-install-plugin package is a demonstration of various optimizations to the package manager dependency installation process.
+Notably:
+1. Moving TAR integrity checking, decompression, parsing and unpacking off of the main thread
+1. Specifically handling the authentication redirect pattern encountered in Azure DevOps Artifacts feeds
+1. Tuning of network parameters for CI environments
+
+To use the plugin, define in command-line.json a phased command called "phased-install", containing a single phase "_phase:prepare".
+You will also need to install the plugin.

rush-plugins/rush-phased-install-plugin/config/rig.json

@@ -0,0 +1,7 @@
+{
+ // The "rig.json" file directs tools to look for their config files in an external package.
+ // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package
+ "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+
+ "rigPackageName": "@rushstack/heft-node-rig"
+}

rush-plugins/rush-phased-install-plugin/package.json

@@ -0,0 +1,25 @@
+{
+ "name": "@rushstack/rush-phased-install-plugin",
+ "version": "0.1.0",
+ "private": true,
+ "description": "Plugin to perform CI install as a phased command.",
+ "scripts": {
+ "build": "heft build --clean",
+ "clean": "heft clean",
+ "start": "heft build-watch --clean",
+ "_phase:build": "heft run --only build -- --clean"
+ },
+ "author": "dmichon-msft",
+ "devDependencies": {
+ "@rushstack/eslint-config": "workspace:*",
+ "@rushstack/heft": "workspace:*",
+ "@rushstack/heft-node-rig": "workspace:*",
+ "@rushstack/node-core-library": "workspace:*",
+ "@rushstack/rush-sdk": "workspace:*",
+ "@rushstack/ts-command-line": "workspace:*",
+ "@rushstack/worker-pool": "workspace:*",
+ "@types/node": "14.18.36",
+ "eslint": "~8.7.0"
+ },
+ "sideEffects": false
+}

rush-plugins/rush-phased-install-plugin/rush-plugin-manifest.json

@@ -0,0 +1,11 @@
+{
+ "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush-plugin-manifest.schema.json",
+ "plugins": [
+ {
+ "pluginName": "rush-phased-install-plugin",
+ "description": "Rush plugin that implements package manager installation in a phased command.",
+ "entryPoint": "dist/rush-phased-install-plugin.js",
+ "associatedCommands": ["phased-install"]
+ }
+ ]
+}

rush-plugins/rush-phased-install-plugin/src/externals.ts

@@ -0,0 +1,64 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+// This file is purely for optimizing the loading of this code when running inside of Rush.
+
+import type { ChildProcess } from 'node:child_process';
+import type Module from 'node:module';
+import type { Operation as OperationType, OperationStatus as OperationStatusType } from '@rushstack/rush-sdk';
+import type * as rushSdkType from '@rushstack/rush-sdk';
+import type { IPnpmLockYaml } from './types';
+
+// Ultra-cheap "I am a Rush plugin" import of rush-lib
+// eslint-disable-next-line @typescript-eslint/naming-convention
+declare const ___rush___rushLibModule: typeof rushSdkType;
+
+const { Operation, OperationStatus } = ___rush___rushLibModule;
+// eslint-disable-next-line @typescript-eslint/no-redeclare
+type Operation = OperationType;
+// eslint-disable-next-line @typescript-eslint/no-redeclare
+type OperationStatus = OperationStatusType;
+
+export { Operation, OperationStatus };
+
+// eslint-disable-next-line @typescript-eslint/naming-convention
+declare const __non_webpack_require__: typeof require;
+
+const entryModule: Module = __non_webpack_require__.main!;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function getExternal(name: string): any {
+ const externalPath: string = __non_webpack_require__.resolve(name, {
+ paths: entryModule.paths
+ });
+
+ return __non_webpack_require__(externalPath);
+}
+
+// Private Rush APIs
+export const PnpmShrinkwrapFile: {
+ loadFromString(data: string): IPnpmLockYaml;
+} = getExternal('@microsoft/rush-lib/lib/logic/pnpm/PnpmShrinkwrapFile').PnpmShrinkwrapFile;
+
+export const Utilities: {
+ executeLifecycleCommandAsync(
+ command: string,
+ options: {
+ rushConfiguration: undefined;
+ workingDirectory: string;
+ initCwd: string;
+ handleOutput: boolean;
+ environmentPathOptions?: {
+ additionalPathFolders: string[];
+ };
+ }
+ ): ChildProcess;
+} = getExternal('@microsoft/rush-lib/lib/utilities/Utilities').Utilities;
+
+// Avoid bundling expensive stuff that's already part of Rush.
+export const Async: typeof import('@rushstack/node-core-library/lib/Async').Async = getExternal(
+ `@rushstack/node-core-library/lib/Async`
+).Async;
+
+export const JsonFile: typeof import('@rushstack/node-core-library/lib/JsonFile').JsonFile = getExternal(
+ `@rushstack/node-core-library/lib/JsonFile`
+).JsonFile;

rush-plugins/rush-phased-install-plugin/src/index.ts

@@ -0,0 +1,43 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import { writeFileSync, mkdirSync } from 'fs';
+
+import type {
+ RushConfiguration,
+ RushSession,
+ IRushPlugin,
+ IPhasedCommand,
+ IRushCommand
+} from '@rushstack/rush-sdk';
+
+class RushPhasedInstallPlugin implements IRushPlugin {
+ public readonly pluginName: 'RushPhasedInstallPlugin' = 'RushPhasedInstallPlugin';
+ public apply(session: RushSession, configuration: RushConfiguration): void {
+ session.hooks.runAnyPhasedCommand.tapPromise(this.pluginName, async (action: IPhasedCommand) => {
+ if (action.actionName.includes('phased-install')) {
+ // Perform an asyn import so that booting Rush isn't slowed down by this plugin, even if it is registered
+ const handler: typeof import('./phasedInstallHandler') = await import(
+ /* webpackChunkName: 'handler' */
+ /* webpackMode: 'eager' */
+ /* webpackExports: ["apply"] */
+ './phasedInstallHandler.js'
+ );
+ await handler.apply(this, session, configuration, action);
+ }
+ });
+ // Exploit that the initialize hook runs before anything in the phased command
+ session.hooks.initialize.tap(this.pluginName, (action: IRushCommand) => {
+ if (action.actionName.includes('phased-install')) {
+ // Rush checks for the marker flag file before allowing a phased command to proceed, so create it preemptively.
+ const { commonTempFolder } = configuration;
+ const flagPath: string = `${commonTempFolder}/last-link.flag`;
+ session.terminalProvider.write(`Writing ${flagPath}\n`, 0);
+ mkdirSync(commonTempFolder, { recursive: true });
+ writeFileSync(flagPath, '{}', 'utf8');
+ }
+ });
+ }
+}
+
+export default RushPhasedInstallPlugin;

rush-plugins/rush-phased-install-plugin/src/operationRunners/AuthenticateOperationRunner.ts

@@ -0,0 +1,124 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import type { ClientRequest, IncomingMessage } from 'node:http';
+import https, { type Agent } from 'node:https';
+import type { IOperationRunner, IOperationRunnerContext } from '@rushstack/rush-sdk';
+import type { IDependencyMetadata, IRefCount } from '../types';
+import { OperationStatus } from '../externals';
+
+function noop(): void {
+ // Do nothing.
+}
+
+const AZURE_DEVOPS_ORGANIZATION: string = `INSERT ORGANIZATION NAME HERE`;
+const AUTH_HEADER: string = `AUTHORIZATION TODO`;
+
+/**
+ * Runner that queries the npm registry for a given package and version to get the authenticated Azure Blob Storage URL.
+ */
+export class AuthenticateOperationRunner implements IOperationRunner {
+ public readonly name: string;
+ // Reporting timing here would be very noisy
+ public readonly reportTiming: boolean = false;
+ public silent: boolean = true;
+ // Has side effects
+ public isSkipAllowed: boolean = false;
+ // Doesn't block cache writes
+ public isCacheWriteAllowed: boolean = true;
+ // Nothing will get logged, no point allowing warnings
+ public readonly warningsAreAllowed: boolean = false;
+
+ public readonly data: IDependencyMetadata;
+
+ private readonly _agent: IRefCount<Agent>;
+
+ private _promise: Promise<void> | undefined;
+
+ public constructor(name: string, data: IDependencyMetadata, agent: IRefCount<Agent>) {
+ this.name = name;
+ this.data = data;
+ this._agent = agent;
+ agent.count++;
+ }
+
+ public async executeAsync(context: IOperationRunnerContext): Promise<OperationStatus> {
+ try {
+ await this.fetchTarballURLAsync();
+
+ return OperationStatus.Success;
+ } catch (err) {
+ this.silent = false;
+ context.collatedWriter.terminal.writeStderrLine(`Authenticate: ${err.toString()}`);
+ return OperationStatus.Failure;
+ } finally {
+ if (--this._agent.count === 0) {
+ this._agent.ref.destroy();
+ }
+ }
+ }
+
+ public async fetchTarballURLAsync(): Promise<void> {
+ return this._promise ?? (this._promise = this._fetchTarballURLIntenalAsync());
+ }
+
+ private async _fetchTarballURLIntenalAsync(): Promise<void> {
+ const { packageName, version, tarball } = this.data;
+
+ if (!tarball) {
+ throw new Error(`No tarball for ${packageName}@${version}`);
+ }
+
+ // This prototype was optimized specifically for Azure DevOps Artifacts; replace with your configured registry. May need to revisit the API call flow
+ // if the authentication bounce doesn't return a 303 for your NPM registry.
+ const host: string = `${AZURE_DEVOPS_ORGANIZATION}.pkgs.visualstudio.com`;
+ const pathname: string = tarball.initialPath;
+
+ const options: https.RequestOptions = {
+ // Reuse of the connection is critical to performance
+ headers: { Authorization: AUTH_HEADER, Connection: 'keep-alive' },
+ protocol: 'https:',
+ // Set timeout at 1 minute, since that is longer than a normal install takes in total on CI.
+ timeout: 60000,
+ host,
+ agent: this._agent.ref,
+ path: pathname
+ };
+
+ // eslint-disable-next-line require-atomic-updates
+ tarball.storageUrl = await this._fetchInternal(options, 3);
+ }
+
+ private async _fetchInternal(options: https.RequestOptions, retryLeft: number): Promise<string> {
+ const { host, path: pathname } = options;
+ try {
+ const storageUrl: string = await new Promise((resolve, reject) => {
+ const request: ClientRequest = https.request(options, (res: IncomingMessage) => {
+ // Fully consume the stream without processing any data.
+ // If we do not consume the stream, the connection cannot be reused, which kills performance.
+ res.resume().on('end', noop);
+ if (res.statusCode === 303) {
+ // This logic is applicable to Azure DevOps Artifacts feeds; not sure if it also applies to the public npmjs registry.
+ resolve(res.headers.location!);
+ } else {
+ reject(
+ new Error(`request failed with status code ${res.statusCode}: https://${host}${pathname}`)
+ );
+ }
+ });
+ request.on('error', (e: Error) =>
+ reject(new Error(`request failed\nhttps://${host}${pathname}\n${e.message}`))
+ );
+ request.end();
+ });
+
+ return storageUrl;
+ } catch (e) {
+ if (retryLeft === 0) {
+ throw e;
+ } else {
+ return await this._fetchInternal(options, retryLeft - 1);
+ }
+ }
+ }
+}