From 0407e47dff2857e6bc618384efafd2f0499c7f8e Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Mon, 10 Jun 2024 14:27:30 -0700 Subject: [PATCH] feat(shim): invoke local turbo version via npx if not installed (#8385) ### Description I highly recommend reviewing the first 4 commits on their own as they are primarily breaking up the massive `shim` module. This PR adds fallback option when global turbo doesn't find a local turbo install. If a local install isn't found we will look at: - lockfiles - package.json - turbo.json to identify if turbo 1 is expected If one of those result in finding a version of `turbo`, then we'll invoke `turbo` via `npx` to download the correct version on the fly. This behavior can be disabled with by setting `TURBO_DOWNLOAD_LOCAL_DISABLED` to `1` or `true` ### Testing Instructions Added unit tests for detection behavior. Manual testing of repos with/without a local install of `turbo`. --- .../fixtures/local_config/turbo.v1.json | 6 + .../fixtures/local_config/turbo.v2.json | 6 + .../local_config/turbov2.package-lock.json | 116 +++ crates/turborepo-lib/src/engine/builder.rs | 4 + crates/turborepo-lib/src/shim.rs | 830 ------------------ .../src/shim/local_turbo_config.rs | 270 ++++++ .../src/shim/local_turbo_state.rs | 221 +++++ crates/turborepo-lib/src/shim/mod.rs | 330 +++++++ crates/turborepo-lib/src/shim/parser.rs | 278 ++++++ crates/turborepo-lib/src/shim/turbo_state.rs | 70 ++ .../fixtures/pnpm6turbo.yaml | 96 ++ .../fixtures/pnpm8turbo.yaml | 96 ++ crates/turborepo-lockfiles/src/berry/mod.rs | 19 +- crates/turborepo-lockfiles/src/bun/mod.rs | 4 + crates/turborepo-lockfiles/src/lib.rs | 3 + crates/turborepo-lockfiles/src/npm.rs | 12 + crates/turborepo-lockfiles/src/pnpm/data.rs | 26 + crates/turborepo-lockfiles/src/yarn1/mod.rs | 17 + crates/turborepo-repository/src/inference.rs | 42 +- .../src/package_graph/mod.rs | 4 + .../src/package_manager/mod.rs | 13 + turborepo-tests/helpers/setup.sh | 1 + turborepo-tests/helpers/setup_example_test.sh | 1 + .../integration/tests/find-turbo/setup.sh | 1 + .../tests/lockfile-aware-caching/setup.sh | 1 + 25 files changed, 1623 insertions(+), 844 deletions(-) create mode 100644 crates/turborepo-lib/fixtures/local_config/turbo.v1.json create mode 100644 crates/turborepo-lib/fixtures/local_config/turbo.v2.json create mode 100644 crates/turborepo-lib/fixtures/local_config/turbov2.package-lock.json delete mode 100644 crates/turborepo-lib/src/shim.rs create mode 100644 crates/turborepo-lib/src/shim/local_turbo_config.rs create mode 100644 crates/turborepo-lib/src/shim/local_turbo_state.rs create mode 100644 crates/turborepo-lib/src/shim/mod.rs create mode 100644 crates/turborepo-lib/src/shim/parser.rs create mode 100644 crates/turborepo-lib/src/shim/turbo_state.rs create mode 100644 crates/turborepo-lockfiles/fixtures/pnpm6turbo.yaml create mode 100644 crates/turborepo-lockfiles/fixtures/pnpm8turbo.yaml diff --git a/crates/turborepo-lib/fixtures/local_config/turbo.v1.json b/crates/turborepo-lib/fixtures/local_config/turbo.v1.json new file mode 100644 index 0000000000000..0baf1b96b32a4 --- /dev/null +++ b/crates/turborepo-lib/fixtures/local_config/turbo.v1.json @@ -0,0 +1,6 @@ +{ + // some comments + "pipeline": { + "build": {"dependsOn": ["^build"]} + } +} diff --git a/crates/turborepo-lib/fixtures/local_config/turbo.v2.json b/crates/turborepo-lib/fixtures/local_config/turbo.v2.json new file mode 100644 index 0000000000000..6e03bbedd06ca --- /dev/null +++ b/crates/turborepo-lib/fixtures/local_config/turbo.v2.json @@ -0,0 +1,6 @@ +{ + // some comments + "tasks": { + "build": {"dependsOn": ["^build"]} + } +} diff --git a/crates/turborepo-lib/fixtures/local_config/turbov2.package-lock.json b/crates/turborepo-lib/fixtures/local_config/turbov2.package-lock.json new file mode 100644 index 0000000000000..0616f589f2cda --- /dev/null +++ b/crates/turborepo-lib/fixtures/local_config/turbov2.package-lock.json @@ -0,0 +1,116 @@ +{ + "name": "pm-and-lockfile", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "pm-and-lockfile", + "workspaces": [ + "apps/*" + ], + "devDependencies": { + "turbo": "latest" + } + }, + "apps/web": {}, + "node_modules/turbo": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/turbo/-/turbo-2.0.3.tgz", + "integrity": "sha512-jF1K0tTUyryEWmgqk1V0ALbSz3VdeZ8FXUo6B64WsPksCMCE48N5jUezGOH2MN0+epdaRMH8/WcPU0QQaVfeLA==", + "dev": true, + "bin": { + "turbo": "bin/turbo" + }, + "optionalDependencies": { + "turbo-darwin-64": "2.0.3", + "turbo-darwin-arm64": "2.0.3", + "turbo-linux-64": "2.0.3", + "turbo-linux-arm64": "2.0.3", + "turbo-windows-64": "2.0.3", + "turbo-windows-arm64": "2.0.3" + } + }, + "node_modules/turbo-darwin-64": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/turbo-darwin-64/-/turbo-darwin-64-2.0.3.tgz", + "integrity": "sha512-v7ztJ8sxdHw3SLfO2MhGFeeU4LQhFii1hIGs9uBiXns/0YTGOvxLeifnfGqhfSrAIIhrCoByXO7nR9wlm10n3Q==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/turbo-darwin-arm64": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/turbo-darwin-arm64/-/turbo-darwin-arm64-2.0.3.tgz", + "integrity": "sha512-LUcqvkV9Bxtng6QHbevp8IK8zzwbIxM6HMjCE7FEW6yJBN1KwvTtRtsGBwwmTxaaLO0wD1Jgl3vgkXAmQ4fqUw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/turbo-linux-64": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/turbo-linux-64/-/turbo-linux-64-2.0.3.tgz", + "integrity": "sha512-xpdY1suXoEbsQsu0kPep2zrB8ijv/S5aKKrntGuQ62hCiwDFoDcA/Z7FZ8IHQ2u+dpJARa7yfiByHmizFE0r5Q==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/turbo-linux-arm64": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/turbo-linux-arm64/-/turbo-linux-arm64-2.0.3.tgz", + "integrity": "sha512-MBACTcSR874L1FtLL7gkgbI4yYJWBUCqeBN/iE29D+8EFe0d3fAyviFlbQP4K/HaDYet1i26xkkOiWr0z7/V9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/turbo-windows-64": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/turbo-windows-64/-/turbo-windows-64-2.0.3.tgz", + "integrity": "sha512-zi3YuKPkM9JxMTshZo3excPk37hUrj5WfnCqh4FjI26ux6j/LJK+Dh3SebMHd9mR7wP9CMam4GhmLCT+gDfM+w==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/turbo-windows-arm64": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/turbo-windows-arm64/-/turbo-windows-arm64-2.0.3.tgz", + "integrity": "sha512-wmed4kkenLvRbidi7gISB4PU77ujBuZfgVGDZ4DXTFslE/kYpINulwzkVwJIvNXsJtHqyOq0n6jL8Zwl3BrwDg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/web": { + "resolved": "apps/web", + "link": true + } + } +} diff --git a/crates/turborepo-lib/src/engine/builder.rs b/crates/turborepo-lib/src/engine/builder.rs index 4ceeeb99cac70..fe25eaa4ce69a 100644 --- a/crates/turborepo-lib/src/engine/builder.rs +++ b/crates/turborepo-lib/src/engine/builder.rs @@ -579,6 +579,10 @@ mod test { fn global_change(&self, _other: &dyn Lockfile) -> bool { unreachable!() } + + fn turbo_version(&self) -> Option { + None + } } struct MockDiscovery; diff --git a/crates/turborepo-lib/src/shim.rs b/crates/turborepo-lib/src/shim.rs deleted file mode 100644 index ee8421dfcc170..0000000000000 --- a/crates/turborepo-lib/src/shim.rs +++ /dev/null @@ -1,830 +0,0 @@ -use std::{ - backtrace::Backtrace, - env, - fs::{self}, - path::PathBuf, - process, - process::Stdio, - time::Duration, -}; - -use camino::Utf8PathBuf; -use const_format::formatcp; -use dunce::canonicalize as fs_canonicalize; -use itertools::Itertools; -use miette::{Diagnostic, SourceSpan}; -use semver::Version; -use serde::Deserialize; -use thiserror::Error; -use tiny_gradient::{GradientStr, RGB}; -use tracing::{debug, warn}; -use turbo_updater::display_update_check; -use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf}; -use turborepo_repository::{ - inference::{RepoMode, RepoState}, - package_json::PackageJson, -}; -use turborepo_ui::UI; - -use crate::{cli, get_version, spawn_child, tracing::TurboSubscriber}; - -const TURBO_GLOBAL_WARNING_DISABLED: &str = "TURBO_GLOBAL_WARNING_DISABLED"; - -#[derive(Debug, Error, Diagnostic)] -#[error("cannot have multiple `--cwd` flags in command")] -#[diagnostic(code(turbo::shim::multiple_cwd))] -pub struct MultipleCwd { - #[backtrace] - backtrace: Backtrace, - #[source_code] - args_string: String, - #[label("first flag declared here")] - flag1: Option, - #[label("but second flag declared here")] - flag2: Option, - #[label("and here")] - flag3: Option, - // The user should get the idea after the first 4 examples. - #[label("and here")] - flag4: Option, -} - -#[derive(Debug, Error, Diagnostic)] -pub enum Error { - #[error(transparent)] - #[diagnostic(transparent)] - MultipleCwd(Box), - #[error("No value assigned to `--cwd` flag")] - #[diagnostic(code(turbo::shim::empty_cwd))] - EmptyCwd { - #[backtrace] - backtrace: Backtrace, - #[source_code] - args_string: String, - #[label = "Requires a path to be passed after it"] - flag_range: SourceSpan, - }, - #[error(transparent)] - #[diagnostic(transparent)] - Cli(#[from] cli::Error), - #[error(transparent)] - Inference(#[from] turborepo_repository::inference::Error), - #[error("failed to execute local turbo process")] - LocalTurboProcess(#[source] std::io::Error), - #[error("failed to resolve local turbo path: {0}")] - LocalTurboPath(String), - #[error("failed to resolve repository root: {0}")] - RepoRootPath(AbsoluteSystemPathBuf), - #[error(transparent)] - Path(#[from] turbopath::PathError), -} - -// all arguments that result in a stdout that much be directly parsable and -// should not be paired with additional output (from the update notifier for -// example) -static TURBO_PURE_OUTPUT_ARGS: [&str; 6] = [ - "--json", - "--dry", - "--dry-run", - "--dry=json", - "--graph", - "--dry-run=json", -]; - -static TURBO_SKIP_NOTIFIER_ARGS: [&str; 5] = - ["--help", "--h", "--version", "--v", "--no-update-notifier"]; - -fn turbo_version_has_shim(version: &str) -> bool { - let version = Version::parse(version).unwrap(); - // only need to check major and minor (this will include canaries) - if version.major == 1 { - return version.minor >= 7; - } - - version.major > 1 -} - -#[derive(Debug)] -struct ShimArgs { - cwd: AbsoluteSystemPathBuf, - invocation_dir: AbsoluteSystemPathBuf, - skip_infer: bool, - verbosity: usize, - force_update_check: bool, - remaining_turbo_args: Vec, - forwarded_args: Vec, - color: bool, - no_color: bool, -} - -impl ShimArgs { - pub fn parse() -> Result { - let mut cwd_flag_idx = None; - let mut cwds = Vec::new(); - let mut skip_infer = false; - let mut found_verbosity_flag = false; - let mut verbosity = 0; - let mut force_update_check = false; - let mut remaining_turbo_args = Vec::new(); - let mut forwarded_args = Vec::new(); - let mut is_forwarded_args = false; - let mut color = false; - let mut no_color = false; - - let args = env::args().skip(1); - for (idx, arg) in args.enumerate() { - // We've seen a `--` and therefore we do no parsing - if is_forwarded_args { - forwarded_args.push(arg); - } else if arg == "--skip-infer" { - skip_infer = true; - } else if arg == "--check-for-update" { - force_update_check = true; - } else if arg == "--" { - // If we've hit `--` we've reached the args forwarded to tasks. - is_forwarded_args = true; - } else if arg == "--verbosity" { - // If we see `--verbosity` we expect the next arg to be a number. - remaining_turbo_args.push(arg); - found_verbosity_flag = true - } else if arg.starts_with("--verbosity=") || found_verbosity_flag { - let verbosity_count = if found_verbosity_flag { - found_verbosity_flag = false; - &arg - } else { - arg.strip_prefix("--verbosity=").unwrap_or("0") - }; - - verbosity = verbosity_count.parse::().unwrap_or(0); - remaining_turbo_args.push(arg); - } else if arg == "-v" || arg.starts_with("-vv") { - verbosity = arg[1..].len(); - remaining_turbo_args.push(arg); - } else if cwd_flag_idx.is_some() { - // We've seen a `--cwd` and therefore add this to the cwds list along with - // the index of the `--cwd` (*not* the value) - cwds.push((AbsoluteSystemPathBuf::from_cwd(arg)?, idx - 1)); - cwd_flag_idx = None; - } else if arg == "--cwd" { - // If we see a `--cwd` we expect the next arg to be a path. - cwd_flag_idx = Some(idx) - } else if let Some(cwd_arg) = arg.strip_prefix("--cwd=") { - // In the case where `--cwd` is passed as `--cwd=./path/to/foo`, that - // entire chunk is a single arg, so we need to split it up. - cwds.push((AbsoluteSystemPathBuf::from_cwd(cwd_arg)?, idx)); - } else if arg == "--color" { - color = true; - } else if arg == "--no-color" { - no_color = true; - } else { - remaining_turbo_args.push(arg); - } - } - - if let Some(idx) = cwd_flag_idx { - let (spans, args_string) = - Self::get_spans_in_args_string(vec![idx], env::args().skip(1)); - - return Err(Error::EmptyCwd { - backtrace: Backtrace::capture(), - args_string, - flag_range: spans[0], - }); - } - - if cwds.len() > 1 { - let (indices, args_string) = Self::get_spans_in_args_string( - cwds.iter().map(|(_, idx)| *idx).collect(), - env::args().skip(1), - ); - - let mut flags = indices.into_iter(); - return Err(Error::MultipleCwd(Box::new(MultipleCwd { - backtrace: Backtrace::capture(), - args_string, - flag1: flags.next(), - flag2: flags.next(), - flag3: flags.next(), - flag4: flags.next(), - }))); - } - - let invocation_dir = AbsoluteSystemPathBuf::cwd()?; - let cwd = cwds - .pop() - .map(|(cwd, _)| cwd) - .unwrap_or_else(|| invocation_dir.clone()); - - Ok(ShimArgs { - cwd, - invocation_dir, - skip_infer, - verbosity, - force_update_check, - remaining_turbo_args, - forwarded_args, - color, - no_color, - }) - } - - /// Takes a list of indices into a Vec of arguments, i.e. ["--graph", "foo", - /// "--cwd"] and converts them into `SourceSpan`'s into the string of those - /// arguments, i.e. "-- graph foo --cwd". Returns the spans and the args - /// string - fn get_spans_in_args_string( - mut args_indices: Vec, - args: impl Iterator>, - ) -> (Vec, String) { - // Sort the indices to keep the invariant - // that if i > j then output[i] > output[j] - args_indices.sort(); - let mut indices_in_args_string = Vec::new(); - let mut i = 0; - let mut current_args_string_idx = 0; - - for (idx, arg) in args.enumerate() { - let Some(arg_idx) = args_indices.get(i) else { - break; - }; - - let arg = arg.into(); - - if idx == *arg_idx { - indices_in_args_string.push((current_args_string_idx, arg.len()).into()); - i += 1; - } - current_args_string_idx += arg.len() + 1; - } - - let args_string = env::args().skip(1).join(" "); - - (indices_in_args_string, args_string) - } - - // returns true if any flags result in pure json output to stdout - fn has_json_flags(&self) -> bool { - self.remaining_turbo_args - .iter() - .any(|arg| TURBO_PURE_OUTPUT_ARGS.contains(&arg.as_str())) - } - - // returns true if any flags should bypass the update notifier - fn has_notifier_skip_flags(&self) -> bool { - self.remaining_turbo_args - .iter() - .any(|arg| TURBO_SKIP_NOTIFIER_ARGS.contains(&arg.as_str())) - } - - pub fn should_check_for_update(&self) -> bool { - if self.force_update_check { - return true; - } - - if self.has_notifier_skip_flags() || self.has_json_flags() { - return false; - } - - true - } - - pub fn ui(&self) -> UI { - if self.no_color { - UI::new(true) - } else if self.color { - // Do our best to enable ansi colors, but even if the terminal doesn't support - // still emit ansi escape sequences. - Self::supports_ansi(); - UI::new(false) - } else if Self::supports_ansi() { - // If the terminal supports ansi colors, then we can infer if we should emit - // colors - UI::infer() - } else { - UI::new(true) - } - } - - #[cfg(windows)] - fn supports_ansi() -> bool { - // This call has the side effect of setting ENABLE_VIRTUAL_TERMINAL_PROCESSING - // to true. https://learn.microsoft.com/en-us/windows/console/setconsolemode - crossterm::ansi_support::supports_ansi() - } - - #[cfg(not(windows))] - fn supports_ansi() -> bool { - true - } -} - -#[derive(Debug, Clone, Deserialize)] -#[serde(rename_all = "camelCase")] -struct YarnRc { - pnp_unplugged_folder: Utf8PathBuf, -} - -impl Default for YarnRc { - fn default() -> Self { - Self { - pnp_unplugged_folder: [".yarn", "unplugged"].iter().collect(), - } - } -} - -#[derive(Debug)] -pub struct TurboState { - bin_path: Option, - version: &'static str, - repo_state: Option, -} - -impl Default for TurboState { - fn default() -> Self { - Self { - bin_path: env::current_exe().ok(), - version: get_version(), - repo_state: None, - } - } -} - -impl TurboState { - pub const fn platform_name() -> &'static str { - const ARCH: &str = { - #[cfg(target_arch = "x86_64")] - { - "64" - } - #[cfg(target_arch = "aarch64")] - { - "arm64" - } - #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] - { - "unknown" - } - }; - - const OS: &str = { - #[cfg(target_os = "macos")] - { - "darwin" - } - #[cfg(target_os = "windows")] - { - "windows" - } - #[cfg(target_os = "linux")] - { - "linux" - } - #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] - { - "unknown" - } - }; - - formatcp!("{}-{}", OS, ARCH) - } - - pub const fn platform_package_name() -> &'static str { - formatcp!("turbo-{}", TurboState::platform_name()) - } - - pub const fn binary_name() -> &'static str { - { - #[cfg(windows)] - { - "turbo.exe" - } - #[cfg(not(windows))] - { - "turbo" - } - } - } - - #[allow(dead_code)] - pub fn version() -> &'static str { - include_str!("../../../version.txt") - .lines() - .next() - .expect("Failed to read version from version.txt") - } -} - -#[derive(Debug)] -pub struct LocalTurboState { - bin_path: PathBuf, - version: String, -} - -impl LocalTurboState { - // Hoisted strategy: - // - `bun install` - // - `npm install` - // - `yarn` - // - `yarn install --flat` - // - berry (nodeLinker: "node-modules") - // - // This also supports people directly depending upon the platform version. - fn generate_hoisted_path(root_path: &AbsoluteSystemPath) -> Option { - Some(root_path.join_component("node_modules")) - } - - // Nested strategy: - // - `npm install --install-strategy=shallow` (`npm install --global-style`) - // - `npm install --install-strategy=nested` (`npm install --legacy-bundling`) - // - berry (nodeLinker: "pnpm") - fn generate_nested_path(root_path: &AbsoluteSystemPath) -> Option { - Some(root_path.join_components(&["node_modules", "turbo", "node_modules"])) - } - - // Linked strategy: - // - `pnpm install` - // - `npm install --install-strategy=linked` - fn generate_linked_path(root_path: &AbsoluteSystemPath) -> Option { - // root_path/node_modules/turbo is a symlink. Canonicalize the symlink to what - // it points to. We do this _before_ traversing up to the parent, - // because on Windows, if you canonicalize a path that ends with `/..` - // it traverses to the parent directory before it follows the symlink, - // leading to the wrong place. We could separate the Windows - // implementation, but this workaround works for other platforms as - // well. - let canonical_path = - fs_canonicalize(root_path.as_path().join("node_modules").join("turbo")).ok()?; - - AbsoluteSystemPathBuf::try_from(canonical_path.parent()?).ok() - } - - // The unplugged directory doesn't have a fixed path. - fn get_unplugged_base_path(root_path: &AbsoluteSystemPath) -> Utf8PathBuf { - let yarn_rc_filename = - env::var("YARN_RC_FILENAME").unwrap_or_else(|_| String::from(".yarnrc.yml")); - let yarn_rc_filepath = root_path.as_path().join(yarn_rc_filename); - - let yarn_rc_yaml_string = fs::read_to_string(yarn_rc_filepath).unwrap_or_default(); - let yarn_rc: YarnRc = serde_yaml::from_str(&yarn_rc_yaml_string).unwrap_or_default(); - - root_path.as_path().join(yarn_rc.pnp_unplugged_folder) - } - - // Unplugged strategy: - // - berry 2.1+ - fn generate_unplugged_path(root_path: &AbsoluteSystemPath) -> Option { - let platform_package_name = TurboState::platform_package_name(); - let unplugged_base_path = Self::get_unplugged_base_path(root_path); - - unplugged_base_path - .read_dir_utf8() - .ok() - .and_then(|mut read_dir| { - // berry includes additional metadata in the filename. - // We actually have to find the platform package. - read_dir.find_map(|item| match item { - Ok(entry) => { - let file_name = entry.file_name(); - if file_name.starts_with(platform_package_name) { - AbsoluteSystemPathBuf::new( - unplugged_base_path.join(file_name).join("node_modules"), - ) - .ok() - } else { - None - } - } - Err(_) => None, - }) - }) - } - - // We support six per-platform packages and one `turbo` package which handles - // indirection. We identify the per-platform package and execute the appropriate - // binary directly. We can choose to operate this aggressively because the - // _worst_ outcome is that we run global `turbo`. - // - // In spite of that, the only known unsupported local invocation is Yarn/Berry < - // 2.1 PnP - pub fn infer(root_path: &AbsoluteSystemPath) -> Option { - let platform_package_name = TurboState::platform_package_name(); - let binary_name = TurboState::binary_name(); - - let platform_package_json_path_components = [platform_package_name, "package.json"]; - let platform_package_executable_path_components = - [platform_package_name, "bin", binary_name]; - - // These are lazy because the last two are more expensive. - let search_functions = [ - Self::generate_hoisted_path, - Self::generate_nested_path, - Self::generate_linked_path, - Self::generate_unplugged_path, - ]; - - // Detecting the package manager is more expensive than just doing an exhaustive - // search. - for root in search_functions - .iter() - .filter_map(|search_function| search_function(root_path)) - { - // Needs borrow because of the loop. - #[allow(clippy::needless_borrow)] - let bin_path = root.join_components(&platform_package_executable_path_components); - match fs_canonicalize(&bin_path) { - Ok(bin_path) => { - let resolved_package_json_path = - root.join_components(&platform_package_json_path_components); - let platform_package_json = - PackageJson::load(&resolved_package_json_path).ok()?; - let local_version = platform_package_json.version?; - - debug!("Local turbo path: {}", bin_path.display()); - debug!("Local turbo version: {}", &local_version); - return Some(Self { - bin_path, - version: local_version, - }); - } - Err(_) => debug!("No local turbo binary found at: {}", bin_path), - } - } - - None - } - - fn supports_skip_infer_and_single_package(&self) -> bool { - turbo_version_has_shim(&self.version) - } - - /// Check to see if the detected local executable is the one currently - /// running. - fn local_is_self(&self) -> bool { - std::env::current_exe().is_ok_and(|current_exe| { - fs_canonicalize(current_exe) - .is_ok_and(|canonical_current_exe| canonical_current_exe == self.bin_path) - }) - } -} - -/// Attempts to run correct turbo by finding nearest package.json, -/// then finding local turbo installation. If the current binary is the -/// local turbo installation, then we run current turbo. Otherwise we -/// kick over to the local turbo installation. -/// -/// # Arguments -/// -/// * `turbo_state`: state for current execution -/// -/// returns: Result -fn run_correct_turbo( - repo_state: RepoState, - shim_args: ShimArgs, - subscriber: &TurboSubscriber, - ui: UI, -) -> Result { - if let Some(turbo_state) = LocalTurboState::infer(&repo_state.root) { - try_check_for_updates(&shim_args, &turbo_state.version); - - if turbo_state.local_is_self() { - env::set_var( - cli::INVOCATION_DIR_ENV_VAR, - shim_args.invocation_dir.as_path(), - ); - debug!("Currently running turbo is local turbo."); - Ok(cli::run(Some(repo_state), subscriber, ui)?) - } else { - spawn_local_turbo(&repo_state, turbo_state, shim_args) - } - } else { - let version = get_version(); - try_check_for_updates(&shim_args, version); - // cli::run checks for this env var, rather than an arg, so that we can support - // calling old versions without passing unknown flags. - env::set_var( - cli::INVOCATION_DIR_ENV_VAR, - shim_args.invocation_dir.as_path(), - ); - debug!("Running command as global turbo"); - let should_warn_on_global = env::var(TURBO_GLOBAL_WARNING_DISABLED) - .map_or(true, |disable| !matches!(disable.as_str(), "1" | "true")); - if should_warn_on_global { - warn!("No locally installed `turbo` found. Using version: {version}."); - } - Ok(cli::run(Some(repo_state), subscriber, ui)?) - } -} - -fn spawn_local_turbo( - repo_state: &RepoState, - local_turbo_state: LocalTurboState, - mut shim_args: ShimArgs, -) -> Result { - let local_turbo_path = fs_canonicalize(&local_turbo_state.bin_path).map_err(|_| { - Error::LocalTurboPath(local_turbo_state.bin_path.to_string_lossy().to_string()) - })?; - debug!( - "Running local turbo binary in {}\n", - local_turbo_path.display() - ); - - let supports_skip_infer_and_single_package = - local_turbo_state.supports_skip_infer_and_single_package(); - let already_has_single_package_flag = shim_args - .remaining_turbo_args - .contains(&"--single-package".to_string()); - let should_add_single_package_flag = repo_state.mode == RepoMode::SinglePackage - && !already_has_single_package_flag - && supports_skip_infer_and_single_package; - - debug!( - "supports_skip_infer_and_single_package {:?}", - supports_skip_infer_and_single_package - ); - let cwd = fs_canonicalize(&repo_state.root) - .map_err(|_| Error::RepoRootPath(repo_state.root.clone()))?; - - let mut raw_args: Vec<_> = if supports_skip_infer_and_single_package { - vec!["--skip-infer".to_string()] - } else { - Vec::new() - }; - - raw_args.append(&mut shim_args.remaining_turbo_args); - - // We add this flag after the raw args to avoid accidentally passing it - // as a global flag instead of as a run flag. - if should_add_single_package_flag { - raw_args.push("--single-package".to_string()); - } - - raw_args.push("--".to_string()); - raw_args.append(&mut shim_args.forwarded_args); - - // We spawn a process that executes the local turbo - // that we've found in node_modules/.bin/turbo. - let mut command = process::Command::new(local_turbo_path); - command - .args(&raw_args) - // rather than passing an argument that local turbo might not understand, set - // an environment variable that can be optionally used - .env( - cli::INVOCATION_DIR_ENV_VAR, - shim_args.invocation_dir.as_path(), - ) - .current_dir(cwd) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()); - - let child = spawn_child(command).map_err(Error::LocalTurboProcess)?; - - let exit_status = child.wait().map_err(Error::LocalTurboProcess)?; - let exit_code = exit_status.code().unwrap_or_else(|| { - debug!("go-turbo failed to report exit code"); - #[cfg(unix)] - { - use std::os::unix::process::ExitStatusExt; - let signal = exit_status.signal(); - let core_dumped = exit_status.core_dumped(); - debug!( - "go-turbo caught signal {:?}. Core dumped? {}", - signal, core_dumped - ); - } - 2 - }); - - Ok(exit_code) -} - -/// Checks for `TURBO_BINARY_PATH` variable. If it is set, -/// we do not try to find local turbo, we simply run the command as -/// the current binary. This is due to legacy behavior of `TURBO_BINARY_PATH` -/// that lets users dynamically set the path of the turbo binary. Because -/// that conflicts with finding a local turbo installation and -/// executing that binary, these two features are fundamentally incompatible. -fn is_turbo_binary_path_set() -> bool { - env::var("TURBO_BINARY_PATH").is_ok() -} - -fn try_check_for_updates(args: &ShimArgs, current_version: &str) { - if args.should_check_for_update() { - // custom footer for update message - let footer = format!( - "Follow {username} for updates: {url}", - username = "@turborepo".gradient([RGB::new(0, 153, 247), RGB::new(241, 23, 18)]), - url = "https://x.com/turborepo" - ); - - let interval = if args.force_update_check { - // force update check - Some(Duration::ZERO) - } else { - // use default (24 hours) - None - }; - // check for updates - let _ = display_update_check( - "turbo", - "https://github.com/vercel/turbo", - Some(&footer), - current_version, - // use default for timeout (800ms) - None, - interval, - ); - } -} - -pub fn run() -> Result { - let args = ShimArgs::parse()?; - let ui = args.ui(); - if ui.should_strip_ansi { - // Let's not crash just because we failed to set up the hook - let _ = miette::set_hook(Box::new(|_| { - Box::new( - miette::MietteHandlerOpts::new() - .color(false) - .unicode(false) - .build(), - ) - })); - } - let subscriber = TurboSubscriber::new_with_verbosity(args.verbosity, &ui); - - debug!("Global turbo version: {}", get_version()); - - // If skip_infer is passed, we're probably running local turbo with - // global turbo having handled the inference. We can run without any - // concerns. - if args.skip_infer { - return Ok(cli::run(None, &subscriber, ui)?); - } - - // If the TURBO_BINARY_PATH is set, we do inference but we do not use - // it to execute local turbo. We simply use it to set the `--single-package` - // and `--cwd` flags. - if is_turbo_binary_path_set() { - let repo_state = RepoState::infer(&args.cwd)?; - debug!("Repository Root: {}", repo_state.root); - return Ok(cli::run(Some(repo_state), &subscriber, ui)?); - } - - match RepoState::infer(&args.cwd) { - Ok(repo_state) => { - debug!("Repository Root: {}", repo_state.root); - run_correct_turbo(repo_state, args, &subscriber, ui) - } - Err(err) => { - // If we cannot infer, we still run global turbo. This allows for global - // commands like login/logout/link/unlink to still work - debug!("Repository inference failed: {}", err); - debug!("Running command as global turbo"); - Ok(cli::run(None, &subscriber, ui)?) - } - } -} - -#[cfg(test)] -mod test { - use miette::SourceSpan; - use test_case::test_case; - - use super::turbo_version_has_shim; - use crate::shim::ShimArgs; - - #[test] - fn test_skip_infer_version_constraint() { - let canary = "1.7.0-canary.0"; - let newer_canary = "1.7.0-canary.1"; - let newer_minor_canary = "1.7.1-canary.6"; - let release = "1.7.0"; - let old = "1.6.3"; - let old_canary = "1.6.2-canary.1"; - let new = "1.8.0"; - let new_major = "2.1.0"; - - assert!(turbo_version_has_shim(release)); - assert!(turbo_version_has_shim(canary)); - assert!(turbo_version_has_shim(newer_canary)); - assert!(turbo_version_has_shim(newer_minor_canary)); - assert!(turbo_version_has_shim(new)); - assert!(turbo_version_has_shim(new_major)); - assert!(!turbo_version_has_shim(old)); - assert!(!turbo_version_has_shim(old_canary)); - } - - #[test_case(vec![3], vec!["--graph", "foo", "--cwd", "apple"], vec![(18, 5).into()])] - #[test_case(vec![0], vec!["--graph", "foo", "--cwd"], vec![(0, 7).into()])] - #[test_case(vec![0, 2], vec!["--graph", "foo", "--cwd"], vec![(0, 7).into(), (12, 5).into()])] - #[test_case(vec![], vec!["--cwd"], vec![])] - fn test_get_indices_in_arg_string( - arg_indices: Vec, - args: Vec<&'static str>, - expected_indices_in_arg_string: Vec, - ) { - let (indices_in_args_string, _) = - ShimArgs::get_spans_in_args_string(arg_indices, args.into_iter()); - assert_eq!(indices_in_args_string, expected_indices_in_arg_string); - } -} diff --git a/crates/turborepo-lib/src/shim/local_turbo_config.rs b/crates/turborepo-lib/src/shim/local_turbo_config.rs new file mode 100644 index 0000000000000..b0d716eabe18d --- /dev/null +++ b/crates/turborepo-lib/src/shim/local_turbo_config.rs @@ -0,0 +1,270 @@ +use std::env; + +use jsonc_parser::{ + ast::{ObjectPropName, Value}, + parse_to_ast, +}; +use tracing::debug; +use turborepo_repository::{inference::RepoState, package_manager::PackageManager}; + +const TURBO_DOWNLOAD_LOCAL_DISABLED: &str = "TURBO_DOWNLOAD_LOCAL_DISABLED"; + +/// Struct containing information about the desired local turbo version +/// according to lockfiles, package.jsons, and if all else fails turbo.json +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LocalTurboConfig { + turbo_version: String, +} + +impl LocalTurboConfig { + pub fn infer(repo_state: &RepoState) -> Option { + // Don't attempt a download if user has opted out + if env::var(TURBO_DOWNLOAD_LOCAL_DISABLED) + .map_or(false, |disable| matches!(disable.as_str(), "1" | "true")) + { + debug!("downloading correct local version disabled"); + return None; + } + let turbo_version = Self::turbo_version_from_lockfile(repo_state) + .or_else(|| { + debug!( + "No turbo version found in a lockfile. Attempting to read version from root \ + package.json" + ); + Self::turbo_version_from_package_json(repo_state) + }) + .or_else(|| { + debug!("No turbo version found in package.json. Checking if turbo.json is for v1"); + Self::turbo_version_from_turbo_json_schema(repo_state) + })?; + Some(Self { turbo_version }) + } + + pub fn turbo_version(&self) -> &str { + &self.turbo_version + } + + fn turbo_version_from_lockfile(repo_state: &RepoState) -> Option { + if let Ok(package_manager) = &repo_state.package_manager { + let lockfile = package_manager + .read_lockfile(&repo_state.root, &repo_state.root_package_json) + .ok()?; + return lockfile.turbo_version(); + } + + // If there isn't a package manager, just try to parse all known lockfiles + // This isn't the most effecient, but since we'll be hitting network to download + // the correct binary the unnecessary file reads aren't costly relative to the + // download. + PackageManager::supported_managers().iter().find_map(|pm| { + let lockfile = pm + .read_lockfile(&repo_state.root, &repo_state.root_package_json) + .ok()?; + lockfile.turbo_version() + }) + } + + fn turbo_version_from_package_json(repo_state: &RepoState) -> Option { + let package_json = &repo_state.root_package_json; + // Look for turbo as a root dependency + package_json + .all_dependencies() + .find_map(|(name, version)| (name == "turbo").then(|| version.clone())) + } + + fn turbo_version_from_turbo_json_schema(repo_state: &RepoState) -> Option { + let turbo_json_path = repo_state.root.join_component("turbo.json"); + let turbo_json_contents = turbo_json_path.read_existing_to_string().ok().flatten()?; + // We explicitly do not use regular path for parsing turbo.json as that will + // fail if it sees unexpected keys. Future versions of turbo might add + // keys and we don't want to crash in that situation. + let turbo_json = parse_to_ast( + &turbo_json_contents, + &Default::default(), + &Default::default(), + ) + .ok()?; + + if let Value::Object(turbo_json) = turbo_json.value? { + let has_pipeline = turbo_json.properties.iter().any(|property| { + let ObjectPropName::String(name) = &property.name else { + return false; + }; + name.value == "pipeline" + }); + if has_pipeline { + // All we can determine is that the turbo.json is meant for a turbo v1 + return Some("^1".to_owned()); + } + } + // We do not check for the existence of `tasks` as it provides us no beneficial + // information. We're already a turbo 2 binary so we'll continue + // execution. + None + } +} + +#[cfg(test)] +mod test { + use tempfile::TempDir; + use turbopath::AbsoluteSystemPath; + use turborepo_repository::{ + inference::RepoMode, package_json::PackageJson, package_manager::Error, + }; + + use super::*; + + #[test] + fn test_package_manager_and_lockfile() { + let tmpdir = TempDir::with_prefix("local_config").unwrap(); + let root = AbsoluteSystemPath::from_std_path(tmpdir.path()).unwrap(); + let repo = RepoState { + root: root.to_owned(), + mode: RepoMode::MultiPackage, + root_package_json: PackageJson::default(), + package_manager: Ok(PackageManager::Npm), + }; + let lockfile = root.join_component("package-lock.json"); + lockfile + .create_with_contents(include_bytes!( + "../../fixtures/local_config/turbov2.package-lock.json" + )) + .unwrap(); + + assert_eq!( + LocalTurboConfig::infer(&repo), + Some(LocalTurboConfig { + turbo_version: "2.0.3".into() + }) + ); + } + + #[test] + fn test_just_lockfile() { + let tmpdir = TempDir::with_prefix("local_config").unwrap(); + let root = AbsoluteSystemPath::from_std_path(tmpdir.path()).unwrap(); + let repo = RepoState { + root: root.to_owned(), + mode: RepoMode::MultiPackage, + root_package_json: PackageJson::default(), + package_manager: Err(Error::MissingPackageManager), + }; + let lockfile = root.join_component("package-lock.json"); + lockfile + .create_with_contents(include_bytes!( + "../../fixtures/local_config/turbov2.package-lock.json" + )) + .unwrap(); + + assert_eq!( + LocalTurboConfig::infer(&repo), + Some(LocalTurboConfig { + turbo_version: "2.0.3".into() + }) + ); + } + + #[test] + fn test_package_json_dep() { + let tmpdir = TempDir::with_prefix("local_config").unwrap(); + let root = AbsoluteSystemPath::from_std_path(tmpdir.path()).unwrap(); + let repo = RepoState { + root: root.to_owned(), + mode: RepoMode::MultiPackage, + root_package_json: PackageJson { + dependencies: Some( + vec![("turbo".into(), "^2.0.0".into())] + .into_iter() + .collect(), + ), + ..Default::default() + }, + package_manager: Err(Error::MissingPackageManager), + }; + + assert_eq!( + LocalTurboConfig::infer(&repo), + Some(LocalTurboConfig { + turbo_version: "^2.0.0".into() + }) + ); + } + + #[test] + fn test_package_json_dev_dep() { + let tmpdir = TempDir::with_prefix("local_config").unwrap(); + let root = AbsoluteSystemPath::from_std_path(tmpdir.path()).unwrap(); + let repo = RepoState { + root: root.to_owned(), + mode: RepoMode::MultiPackage, + root_package_json: PackageJson { + dev_dependencies: Some( + vec![("turbo".into(), "^2.0.0".into())] + .into_iter() + .collect(), + ), + ..Default::default() + }, + package_manager: Err(Error::MissingPackageManager), + }; + + assert_eq!( + LocalTurboConfig::infer(&repo), + Some(LocalTurboConfig { + turbo_version: "^2.0.0".into() + }) + ); + } + + #[test] + fn test_v1_schema() { + let tmpdir = TempDir::with_prefix("local_config").unwrap(); + let root = AbsoluteSystemPath::from_std_path(tmpdir.path()).unwrap(); + let repo = RepoState { + root: root.to_owned(), + mode: RepoMode::MultiPackage, + root_package_json: PackageJson::default(), + package_manager: Err(Error::MissingPackageManager), + }; + let turbo_json = root.join_component("turbo.json"); + turbo_json + .create_with_contents(include_bytes!("../../fixtures/local_config/turbo.v1.json")) + .unwrap(); + assert_eq!( + LocalTurboConfig::infer(&repo), + Some(LocalTurboConfig { + turbo_version: "^1".into() + }) + ); + } + + #[test] + fn test_v2_schema() { + let tmpdir = TempDir::with_prefix("local_config").unwrap(); + let root = AbsoluteSystemPath::from_std_path(tmpdir.path()).unwrap(); + let repo = RepoState { + root: root.to_owned(), + mode: RepoMode::MultiPackage, + root_package_json: PackageJson::default(), + package_manager: Err(Error::MissingPackageManager), + }; + let turbo_json = root.join_component("turbo.json"); + turbo_json + .create_with_contents(include_bytes!("../../fixtures/local_config/turbo.v2.json")) + .unwrap(); + assert_eq!(LocalTurboConfig::infer(&repo), None,); + } + + #[test] + fn nothing() { + let tmpdir = TempDir::with_prefix("local_config").unwrap(); + let root = AbsoluteSystemPath::from_std_path(tmpdir.path()).unwrap(); + let repo = RepoState { + root: root.to_owned(), + mode: RepoMode::MultiPackage, + root_package_json: PackageJson::default(), + package_manager: Err(Error::MissingPackageManager), + }; + assert_eq!(LocalTurboConfig::infer(&repo), None,); + } +} diff --git a/crates/turborepo-lib/src/shim/local_turbo_state.rs b/crates/turborepo-lib/src/shim/local_turbo_state.rs new file mode 100644 index 0000000000000..9f972c51a4975 --- /dev/null +++ b/crates/turborepo-lib/src/shim/local_turbo_state.rs @@ -0,0 +1,221 @@ +use std::{ + env, fs, + path::{Path, PathBuf}, +}; + +use camino::Utf8PathBuf; +use dunce::canonicalize as fs_canonicalize; +use semver::Version; +use serde::Deserialize; +use tracing::debug; +use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf}; +use turborepo_repository::package_json::PackageJson; + +use super::TurboState; + +/// Structure that holds information on an existing local turbo install +#[derive(Debug)] +pub struct LocalTurboState { + bin_path: PathBuf, + version: String, +} + +impl LocalTurboState { + pub fn version(&self) -> &str { + &self.version + } + + pub fn binary(&self) -> &Path { + &self.bin_path + } + // Hoisted strategy: + // - `bun install` + // - `npm install` + // - `yarn` + // - `yarn install --flat` + // - berry (nodeLinker: "node-modules") + // + // This also supports people directly depending upon the platform version. + fn generate_hoisted_path(root_path: &AbsoluteSystemPath) -> Option { + Some(root_path.join_component("node_modules")) + } + + // Nested strategy: + // - `npm install --install-strategy=shallow` (`npm install --global-style`) + // - `npm install --install-strategy=nested` (`npm install --legacy-bundling`) + // - berry (nodeLinker: "pnpm") + fn generate_nested_path(root_path: &AbsoluteSystemPath) -> Option { + Some(root_path.join_components(&["node_modules", "turbo", "node_modules"])) + } + + // Linked strategy: + // - `pnpm install` + // - `npm install --install-strategy=linked` + fn generate_linked_path(root_path: &AbsoluteSystemPath) -> Option { + // root_path/node_modules/turbo is a symlink. Canonicalize the symlink to what + // it points to. We do this _before_ traversing up to the parent, + // because on Windows, if you canonicalize a path that ends with `/..` + // it traverses to the parent directory before it follows the symlink, + // leading to the wrong place. We could separate the Windows + // implementation, but this workaround works for other platforms as + // well. + let canonical_path = + fs_canonicalize(root_path.as_path().join("node_modules").join("turbo")).ok()?; + + AbsoluteSystemPathBuf::try_from(canonical_path.parent()?).ok() + } + + // The unplugged directory doesn't have a fixed path. + fn get_unplugged_base_path(root_path: &AbsoluteSystemPath) -> Utf8PathBuf { + let yarn_rc_filename = + env::var("YARN_RC_FILENAME").unwrap_or_else(|_| String::from(".yarnrc.yml")); + let yarn_rc_filepath = root_path.as_path().join(yarn_rc_filename); + + let yarn_rc_yaml_string = fs::read_to_string(yarn_rc_filepath).unwrap_or_default(); + let yarn_rc: YarnRc = serde_yaml::from_str(&yarn_rc_yaml_string).unwrap_or_default(); + + root_path.as_path().join(yarn_rc.pnp_unplugged_folder) + } + + // Unplugged strategy: + // - berry 2.1+ + fn generate_unplugged_path(root_path: &AbsoluteSystemPath) -> Option { + let platform_package_name = TurboState::platform_package_name(); + let unplugged_base_path = Self::get_unplugged_base_path(root_path); + + unplugged_base_path + .read_dir_utf8() + .ok() + .and_then(|mut read_dir| { + // berry includes additional metadata in the filename. + // We actually have to find the platform package. + read_dir.find_map(|item| match item { + Ok(entry) => { + let file_name = entry.file_name(); + if file_name.starts_with(platform_package_name) { + AbsoluteSystemPathBuf::new( + unplugged_base_path.join(file_name).join("node_modules"), + ) + .ok() + } else { + None + } + } + Err(_) => None, + }) + }) + } + + // We support six per-platform packages and one `turbo` package which handles + // indirection. We identify the per-platform package and execute the appropriate + // binary directly. We can choose to operate this aggressively because the + // _worst_ outcome is that we run global `turbo`. + // + // In spite of that, the only known unsupported local invocation is Yarn/Berry < + // 2.1 PnP + pub fn infer(root_path: &AbsoluteSystemPath) -> Option { + let platform_package_name = TurboState::platform_package_name(); + let binary_name = TurboState::binary_name(); + + let platform_package_json_path_components = [platform_package_name, "package.json"]; + let platform_package_executable_path_components = + [platform_package_name, "bin", binary_name]; + + // These are lazy because the last two are more expensive. + let search_functions = [ + Self::generate_hoisted_path, + Self::generate_nested_path, + Self::generate_linked_path, + Self::generate_unplugged_path, + ]; + + // Detecting the package manager is more expensive than just doing an exhaustive + // search. + for root in search_functions + .iter() + .filter_map(|search_function| search_function(root_path)) + { + // Needs borrow because of the loop. + #[allow(clippy::needless_borrow)] + let bin_path = root.join_components(&platform_package_executable_path_components); + match fs_canonicalize(&bin_path) { + Ok(bin_path) => { + let resolved_package_json_path = + root.join_components(&platform_package_json_path_components); + let platform_package_json = + PackageJson::load(&resolved_package_json_path).ok()?; + let local_version = platform_package_json.version?; + + debug!("Local turbo path: {}", bin_path.display()); + debug!("Local turbo version: {}", &local_version); + return Some(Self { + bin_path, + version: local_version, + }); + } + Err(_) => debug!("No local turbo binary found at: {}", bin_path), + } + } + + None + } + + /// Check to see if the detected local executable is the one currently + /// running. + pub fn local_is_self(&self) -> bool { + std::env::current_exe().is_ok_and(|current_exe| { + fs_canonicalize(current_exe) + .is_ok_and(|canonical_current_exe| canonical_current_exe == self.bin_path) + }) + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct YarnRc { + pnp_unplugged_folder: Utf8PathBuf, +} + +impl Default for YarnRc { + fn default() -> Self { + Self { + pnp_unplugged_folder: [".yarn", "unplugged"].iter().collect(), + } + } +} + +pub fn turbo_version_has_shim(version: &str) -> bool { + let version = Version::parse(version).unwrap(); + // only need to check major and minor (this will include canaries) + if version.major == 1 { + return version.minor >= 7; + } + + version.major > 1 +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_skip_infer_version_constraint() { + let canary = "1.7.0-canary.0"; + let newer_canary = "1.7.0-canary.1"; + let newer_minor_canary = "1.7.1-canary.6"; + let release = "1.7.0"; + let old = "1.6.3"; + let old_canary = "1.6.2-canary.1"; + let new = "1.8.0"; + let new_major = "2.1.0"; + + assert!(turbo_version_has_shim(release)); + assert!(turbo_version_has_shim(canary)); + assert!(turbo_version_has_shim(newer_canary)); + assert!(turbo_version_has_shim(newer_minor_canary)); + assert!(turbo_version_has_shim(new)); + assert!(turbo_version_has_shim(new_major)); + assert!(!turbo_version_has_shim(old)); + assert!(!turbo_version_has_shim(old_canary)); + } +} diff --git a/crates/turborepo-lib/src/shim/mod.rs b/crates/turborepo-lib/src/shim/mod.rs new file mode 100644 index 0000000000000..5d933c7f49984 --- /dev/null +++ b/crates/turborepo-lib/src/shim/mod.rs @@ -0,0 +1,330 @@ +mod local_turbo_config; +mod local_turbo_state; +mod parser; +mod turbo_state; + +use std::{backtrace::Backtrace, env, process, process::Stdio, time::Duration}; + +use dunce::canonicalize as fs_canonicalize; +use local_turbo_config::LocalTurboConfig; +use local_turbo_state::{turbo_version_has_shim, LocalTurboState}; +use miette::{Diagnostic, SourceSpan}; +use parser::{MultipleCwd, ShimArgs}; +use thiserror::Error; +use tiny_gradient::{GradientStr, RGB}; +use tracing::{debug, warn}; +pub use turbo_state::TurboState; +use turbo_updater::display_update_check; +use turbopath::AbsoluteSystemPathBuf; +use turborepo_repository::inference::{RepoMode, RepoState}; +use turborepo_ui::UI; +use which::which; + +use crate::{cli, get_version, spawn_child, tracing::TurboSubscriber}; + +const TURBO_GLOBAL_WARNING_DISABLED: &str = "TURBO_GLOBAL_WARNING_DISABLED"; + +#[derive(Debug, Error, Diagnostic)] +pub enum Error { + #[error(transparent)] + #[diagnostic(transparent)] + MultipleCwd(Box), + #[error("No value assigned to `--cwd` flag")] + #[diagnostic(code(turbo::shim::empty_cwd))] + EmptyCwd { + #[backtrace] + backtrace: Backtrace, + #[source_code] + args_string: String, + #[label = "Requires a path to be passed after it"] + flag_range: SourceSpan, + }, + #[error(transparent)] + #[diagnostic(transparent)] + Cli(#[from] cli::Error), + #[error(transparent)] + Inference(#[from] turborepo_repository::inference::Error), + #[error("failed to execute local turbo process")] + LocalTurboProcess(#[source] std::io::Error), + #[error("failed to resolve local turbo path: {0}")] + LocalTurboPath(String), + #[error("failed to find npx: {0}")] + Which(#[from] which::Error), + #[error("failed to execute turbo via npx")] + NpxTurboProcess(#[source] std::io::Error), + #[error("failed to resolve repository root: {0}")] + RepoRootPath(AbsoluteSystemPathBuf), + #[error(transparent)] + Path(#[from] turbopath::PathError), +} + +/// Attempts to run correct turbo by finding nearest package.json, +/// then finding local turbo installation. If the current binary is the +/// local turbo installation, then we run current turbo. Otherwise we +/// kick over to the local turbo installation. +/// +/// # Arguments +/// +/// * `turbo_state`: state for current execution +/// +/// returns: Result +fn run_correct_turbo( + repo_state: RepoState, + shim_args: ShimArgs, + subscriber: &TurboSubscriber, + ui: UI, +) -> Result { + if let Some(turbo_state) = LocalTurboState::infer(&repo_state.root) { + try_check_for_updates(&shim_args, turbo_state.version()); + + if turbo_state.local_is_self() { + env::set_var( + cli::INVOCATION_DIR_ENV_VAR, + shim_args.invocation_dir.as_path(), + ); + debug!("Currently running turbo is local turbo."); + Ok(cli::run(Some(repo_state), subscriber, ui)?) + } else { + spawn_local_turbo(&repo_state, turbo_state, shim_args) + } + } else if let Some(local_config) = LocalTurboConfig::infer(&repo_state) { + debug!( + "Found configuration for turbo version {}", + local_config.turbo_version() + ); + spawn_npx_turbo(&repo_state, local_config.turbo_version(), shim_args) + } else { + let version = get_version(); + try_check_for_updates(&shim_args, version); + // cli::run checks for this env var, rather than an arg, so that we can support + // calling old versions without passing unknown flags. + env::set_var( + cli::INVOCATION_DIR_ENV_VAR, + shim_args.invocation_dir.as_path(), + ); + debug!("Running command as global turbo"); + let should_warn_on_global = env::var(TURBO_GLOBAL_WARNING_DISABLED) + .map_or(true, |disable| !matches!(disable.as_str(), "1" | "true")); + if should_warn_on_global { + warn!("No locally installed `turbo` found. Using version: {version}."); + } + Ok(cli::run(Some(repo_state), subscriber, ui)?) + } +} + +fn spawn_local_turbo( + repo_state: &RepoState, + local_turbo_state: LocalTurboState, + mut shim_args: ShimArgs, +) -> Result { + let local_turbo_path = fs_canonicalize(local_turbo_state.binary()).map_err(|_| { + Error::LocalTurboPath(local_turbo_state.binary().to_string_lossy().to_string()) + })?; + debug!( + "Running local turbo binary in {}\n", + local_turbo_path.display() + ); + let cwd = fs_canonicalize(&repo_state.root) + .map_err(|_| Error::RepoRootPath(repo_state.root.clone()))?; + + let raw_args = modify_args_for_local(&mut shim_args, repo_state, local_turbo_state.version()); + + // We spawn a process that executes the local turbo + // that we've found in node_modules/.bin/turbo. + let mut command = process::Command::new(local_turbo_path); + command + .args(&raw_args) + // rather than passing an argument that local turbo might not understand, set + // an environment variable that can be optionally used + .env( + cli::INVOCATION_DIR_ENV_VAR, + shim_args.invocation_dir.as_path(), + ) + .current_dir(cwd) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + spawn_child_turbo(command, Error::LocalTurboProcess) +} + +fn spawn_npx_turbo( + repo_state: &RepoState, + turbo_version: &str, + mut shim_args: ShimArgs, +) -> Result { + debug!("Running turbo@{turbo_version} via npx"); + let npx_path = which("npx")?; + let cwd = fs_canonicalize(&repo_state.root) + .map_err(|_| Error::RepoRootPath(repo_state.root.clone()))?; + + let raw_args = modify_args_for_local(&mut shim_args, repo_state, turbo_version); + + let mut command = process::Command::new(npx_path); + command.arg("-y"); + command.arg(&format!("turbo@{turbo_version}")); + + // rather than passing an argument that local turbo might not understand, set + // an environment variable that can be optionally used + command + .args(&raw_args) + .env( + cli::INVOCATION_DIR_ENV_VAR, + shim_args.invocation_dir.as_path(), + ) + .current_dir(cwd) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + spawn_child_turbo(command, Error::NpxTurboProcess) +} + +fn modify_args_for_local( + shim_args: &mut ShimArgs, + repo_state: &RepoState, + local_version: &str, +) -> Vec { + let supports_skip_infer_and_single_package = turbo_version_has_shim(local_version); + let already_has_single_package_flag = shim_args + .remaining_turbo_args + .contains(&"--single-package".to_string()); + let should_add_single_package_flag = repo_state.mode == RepoMode::SinglePackage + && !already_has_single_package_flag + && supports_skip_infer_and_single_package; + + debug!( + "supports_skip_infer_and_single_package {:?}", + supports_skip_infer_and_single_package + ); + + let mut raw_args: Vec<_> = if supports_skip_infer_and_single_package { + vec!["--skip-infer".to_string()] + } else { + Vec::new() + }; + + raw_args.append(&mut shim_args.remaining_turbo_args); + + // We add this flag after the raw args to avoid accidentally passing it + // as a global flag instead of as a run flag. + if should_add_single_package_flag { + raw_args.push("--single-package".to_string()); + } + + raw_args.push("--".to_string()); + raw_args.append(&mut shim_args.forwarded_args); + + raw_args +} + +fn spawn_child_turbo( + command: process::Command, + err: fn(std::io::Error) -> Error, +) -> Result { + let child = spawn_child(command).map_err(err)?; + + let exit_status = child.wait().map_err(err)?; + let exit_code = exit_status.code().unwrap_or_else(|| { + debug!("child turbo failed to report exit code"); + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + let signal = exit_status.signal(); + let core_dumped = exit_status.core_dumped(); + debug!( + "child turbo caught signal {:?}. Core dumped? {}", + signal, core_dumped + ); + } + 2 + }); + + Ok(exit_code) +} + +/// Checks for `TURBO_BINARY_PATH` variable. If it is set, +/// we do not try to find local turbo, we simply run the command as +/// the current binary. This is due to legacy behavior of `TURBO_BINARY_PATH` +/// that lets users dynamically set the path of the turbo binary. Because +/// that conflicts with finding a local turbo installation and +/// executing that binary, these two features are fundamentally incompatible. +fn is_turbo_binary_path_set() -> bool { + env::var("TURBO_BINARY_PATH").is_ok() +} + +fn try_check_for_updates(args: &ShimArgs, current_version: &str) { + if args.should_check_for_update() { + // custom footer for update message + let footer = format!( + "Follow {username} for updates: {url}", + username = "@turborepo".gradient([RGB::new(0, 153, 247), RGB::new(241, 23, 18)]), + url = "https://x.com/turborepo" + ); + + let interval = if args.force_update_check { + // force update check + Some(Duration::ZERO) + } else { + // use default (24 hours) + None + }; + // check for updates + let _ = display_update_check( + "turbo", + "https://github.com/vercel/turbo", + Some(&footer), + current_version, + // use default for timeout (800ms) + None, + interval, + ); + } +} + +pub fn run() -> Result { + let args = ShimArgs::parse()?; + let ui = args.ui(); + if ui.should_strip_ansi { + // Let's not crash just because we failed to set up the hook + let _ = miette::set_hook(Box::new(|_| { + Box::new( + miette::MietteHandlerOpts::new() + .color(false) + .unicode(false) + .build(), + ) + })); + } + let subscriber = TurboSubscriber::new_with_verbosity(args.verbosity, &ui); + + debug!("Global turbo version: {}", get_version()); + + // If skip_infer is passed, we're probably running local turbo with + // global turbo having handled the inference. We can run without any + // concerns. + if args.skip_infer { + return Ok(cli::run(None, &subscriber, ui)?); + } + + // If the TURBO_BINARY_PATH is set, we do inference but we do not use + // it to execute local turbo. We simply use it to set the `--single-package` + // and `--cwd` flags. + if is_turbo_binary_path_set() { + let repo_state = RepoState::infer(&args.cwd)?; + debug!("Repository Root: {}", repo_state.root); + return Ok(cli::run(Some(repo_state), &subscriber, ui)?); + } + + match RepoState::infer(&args.cwd) { + Ok(repo_state) => { + debug!("Repository Root: {}", repo_state.root); + run_correct_turbo(repo_state, args, &subscriber, ui) + } + Err(err) => { + // If we cannot infer, we still run global turbo. This allows for global + // commands like login/logout/link/unlink to still work + debug!("Repository inference failed: {}", err); + debug!("Running command as global turbo"); + Ok(cli::run(None, &subscriber, ui)?) + } + } +} diff --git a/crates/turborepo-lib/src/shim/parser.rs b/crates/turborepo-lib/src/shim/parser.rs new file mode 100644 index 0000000000000..70a38f9f1ed25 --- /dev/null +++ b/crates/turborepo-lib/src/shim/parser.rs @@ -0,0 +1,278 @@ +use std::{backtrace::Backtrace, env}; + +use itertools::Itertools; +use miette::{Diagnostic, SourceSpan}; +use turbopath::AbsoluteSystemPathBuf; +use turborepo_ui::UI; + +use super::Error; + +// all arguments that result in a stdout that much be directly parsable and +// should not be paired with additional output (from the update notifier for +// example) +static TURBO_PURE_OUTPUT_ARGS: [&str; 6] = [ + "--json", + "--dry", + "--dry-run", + "--dry=json", + "--graph", + "--dry-run=json", +]; + +static TURBO_SKIP_NOTIFIER_ARGS: [&str; 5] = + ["--help", "--h", "--version", "--v", "--no-update-notifier"]; + +#[derive(Debug, thiserror::Error, Diagnostic)] +#[error("cannot have multiple `--cwd` flags in command")] +#[diagnostic(code(turbo::shim::multiple_cwd))] +pub struct MultipleCwd { + #[backtrace] + backtrace: Backtrace, + #[source_code] + args_string: String, + #[label("first flag declared here")] + flag1: Option, + #[label("but second flag declared here")] + flag2: Option, + #[label("and here")] + flag3: Option, + // The user should get the idea after the first 4 examples. + #[label("and here")] + flag4: Option, +} + +#[derive(Debug)] +pub struct ShimArgs { + pub cwd: AbsoluteSystemPathBuf, + pub invocation_dir: AbsoluteSystemPathBuf, + pub skip_infer: bool, + pub verbosity: usize, + pub force_update_check: bool, + pub remaining_turbo_args: Vec, + pub forwarded_args: Vec, + pub color: bool, + pub no_color: bool, +} + +impl ShimArgs { + pub fn parse() -> Result { + let mut cwd_flag_idx = None; + let mut cwds = Vec::new(); + let mut skip_infer = false; + let mut found_verbosity_flag = false; + let mut verbosity = 0; + let mut force_update_check = false; + let mut remaining_turbo_args = Vec::new(); + let mut forwarded_args = Vec::new(); + let mut is_forwarded_args = false; + let mut color = false; + let mut no_color = false; + + let args = env::args().skip(1); + for (idx, arg) in args.enumerate() { + // We've seen a `--` and therefore we do no parsing + if is_forwarded_args { + forwarded_args.push(arg); + } else if arg == "--skip-infer" { + skip_infer = true; + } else if arg == "--check-for-update" { + force_update_check = true; + } else if arg == "--" { + // If we've hit `--` we've reached the args forwarded to tasks. + is_forwarded_args = true; + } else if arg == "--verbosity" { + // If we see `--verbosity` we expect the next arg to be a number. + remaining_turbo_args.push(arg); + found_verbosity_flag = true + } else if arg.starts_with("--verbosity=") || found_verbosity_flag { + let verbosity_count = if found_verbosity_flag { + found_verbosity_flag = false; + &arg + } else { + arg.strip_prefix("--verbosity=").unwrap_or("0") + }; + + verbosity = verbosity_count.parse::().unwrap_or(0); + remaining_turbo_args.push(arg); + } else if arg == "-v" || arg.starts_with("-vv") { + verbosity = arg[1..].len(); + remaining_turbo_args.push(arg); + } else if cwd_flag_idx.is_some() { + // We've seen a `--cwd` and therefore add this to the cwds list along with + // the index of the `--cwd` (*not* the value) + cwds.push((AbsoluteSystemPathBuf::from_cwd(arg)?, idx - 1)); + cwd_flag_idx = None; + } else if arg == "--cwd" { + // If we see a `--cwd` we expect the next arg to be a path. + cwd_flag_idx = Some(idx) + } else if let Some(cwd_arg) = arg.strip_prefix("--cwd=") { + // In the case where `--cwd` is passed as `--cwd=./path/to/foo`, that + // entire chunk is a single arg, so we need to split it up. + cwds.push((AbsoluteSystemPathBuf::from_cwd(cwd_arg)?, idx)); + } else if arg == "--color" { + color = true; + } else if arg == "--no-color" { + no_color = true; + } else { + remaining_turbo_args.push(arg); + } + } + + if let Some(idx) = cwd_flag_idx { + let (spans, args_string) = + Self::get_spans_in_args_string(vec![idx], env::args().skip(1)); + + return Err(Error::EmptyCwd { + backtrace: Backtrace::capture(), + args_string, + flag_range: spans[0], + }); + } + + if cwds.len() > 1 { + let (indices, args_string) = Self::get_spans_in_args_string( + cwds.iter().map(|(_, idx)| *idx).collect(), + env::args().skip(1), + ); + + let mut flags = indices.into_iter(); + return Err(Error::MultipleCwd(Box::new(MultipleCwd { + backtrace: Backtrace::capture(), + args_string, + flag1: flags.next(), + flag2: flags.next(), + flag3: flags.next(), + flag4: flags.next(), + }))); + } + + let invocation_dir = AbsoluteSystemPathBuf::cwd()?; + let cwd = cwds + .pop() + .map(|(cwd, _)| cwd) + .unwrap_or_else(|| invocation_dir.clone()); + + Ok(ShimArgs { + cwd, + invocation_dir, + skip_infer, + verbosity, + force_update_check, + remaining_turbo_args, + forwarded_args, + color, + no_color, + }) + } + + /// Takes a list of indices into a Vec of arguments, i.e. ["--graph", "foo", + /// "--cwd"] and converts them into `SourceSpan`'s into the string of those + /// arguments, i.e. "-- graph foo --cwd". Returns the spans and the args + /// string + fn get_spans_in_args_string( + mut args_indices: Vec, + args: impl Iterator>, + ) -> (Vec, String) { + // Sort the indices to keep the invariant + // that if i > j then output[i] > output[j] + args_indices.sort(); + let mut indices_in_args_string = Vec::new(); + let mut i = 0; + let mut current_args_string_idx = 0; + + for (idx, arg) in args.enumerate() { + let Some(arg_idx) = args_indices.get(i) else { + break; + }; + + let arg = arg.into(); + + if idx == *arg_idx { + indices_in_args_string.push((current_args_string_idx, arg.len()).into()); + i += 1; + } + current_args_string_idx += arg.len() + 1; + } + + let args_string = env::args().skip(1).join(" "); + + (indices_in_args_string, args_string) + } + + // returns true if any flags result in pure json output to stdout + fn has_json_flags(&self) -> bool { + self.remaining_turbo_args + .iter() + .any(|arg| TURBO_PURE_OUTPUT_ARGS.contains(&arg.as_str())) + } + + // returns true if any flags should bypass the update notifier + fn has_notifier_skip_flags(&self) -> bool { + self.remaining_turbo_args + .iter() + .any(|arg| TURBO_SKIP_NOTIFIER_ARGS.contains(&arg.as_str())) + } + + pub fn should_check_for_update(&self) -> bool { + if self.force_update_check { + return true; + } + + if self.has_notifier_skip_flags() || self.has_json_flags() { + return false; + } + + true + } + + pub fn ui(&self) -> UI { + if self.no_color { + UI::new(true) + } else if self.color { + // Do our best to enable ansi colors, but even if the terminal doesn't support + // still emit ansi escape sequences. + Self::supports_ansi(); + UI::new(false) + } else if Self::supports_ansi() { + // If the terminal supports ansi colors, then we can infer if we should emit + // colors + UI::infer() + } else { + UI::new(true) + } + } + + #[cfg(windows)] + fn supports_ansi() -> bool { + // This call has the side effect of setting ENABLE_VIRTUAL_TERMINAL_PROCESSING + // to true. https://learn.microsoft.com/en-us/windows/console/setconsolemode + crossterm::ansi_support::supports_ansi() + } + + #[cfg(not(windows))] + fn supports_ansi() -> bool { + true + } +} + +#[cfg(test)] +mod test { + use miette::SourceSpan; + use test_case::test_case; + + use super::ShimArgs; + + #[test_case(vec![3], vec!["--graph", "foo", "--cwd", "apple"], vec![(18, 5).into()])] + #[test_case(vec![0], vec!["--graph", "foo", "--cwd"], vec![(0, 7).into()])] + #[test_case(vec![0, 2], vec!["--graph", "foo", "--cwd"], vec![(0, 7).into(), (12, 5).into()])] + #[test_case(vec![], vec!["--cwd"], vec![])] + fn test_get_indices_in_arg_string( + arg_indices: Vec, + args: Vec<&'static str>, + expected_indices_in_arg_string: Vec, + ) { + let (indices_in_args_string, _) = + ShimArgs::get_spans_in_args_string(arg_indices, args.into_iter()); + assert_eq!(indices_in_args_string, expected_indices_in_arg_string); + } +} diff --git a/crates/turborepo-lib/src/shim/turbo_state.rs b/crates/turborepo-lib/src/shim/turbo_state.rs new file mode 100644 index 0000000000000..5ce14fec42e89 --- /dev/null +++ b/crates/turborepo-lib/src/shim/turbo_state.rs @@ -0,0 +1,70 @@ +use const_format::formatcp; + +/// Struct containing helper methods for querying information about the +/// currently running turbo binary. +#[derive(Debug)] +pub struct TurboState; + +impl TurboState { + pub const fn platform_name() -> &'static str { + const ARCH: &str = { + #[cfg(target_arch = "x86_64")] + { + "64" + } + #[cfg(target_arch = "aarch64")] + { + "arm64" + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + { + "unknown" + } + }; + + const OS: &str = { + #[cfg(target_os = "macos")] + { + "darwin" + } + #[cfg(target_os = "windows")] + { + "windows" + } + #[cfg(target_os = "linux")] + { + "linux" + } + #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] + { + "unknown" + } + }; + + formatcp!("{}-{}", OS, ARCH) + } + + pub const fn platform_package_name() -> &'static str { + formatcp!("turbo-{}", TurboState::platform_name()) + } + + pub const fn binary_name() -> &'static str { + { + #[cfg(windows)] + { + "turbo.exe" + } + #[cfg(not(windows))] + { + "turbo" + } + } + } + + pub fn version() -> &'static str { + include_str!("../../../../version.txt") + .lines() + .next() + .expect("Failed to read version from version.txt") + } +} diff --git a/crates/turborepo-lockfiles/fixtures/pnpm6turbo.yaml b/crates/turborepo-lockfiles/fixtures/pnpm6turbo.yaml new file mode 100644 index 0000000000000..ca55831ab31c1 --- /dev/null +++ b/crates/turborepo-lockfiles/fixtures/pnpm6turbo.yaml @@ -0,0 +1,96 @@ +lockfileVersion: "6.0" + +settings: + autoInstallPeers: false + excludeLinksFromLockfile: false + +importers: + .: + devDependencies: + turbo: + specifier: ^2.0.3 + version: 2.0.3 + + apps/web: {} + +packages: + /turbo-darwin-64@2.0.3: + resolution: + { + integrity: sha512-v7ztJ8sxdHw3SLfO2MhGFeeU4LQhFii1hIGs9uBiXns/0YTGOvxLeifnfGqhfSrAIIhrCoByXO7nR9wlm10n3Q==, + } + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /turbo-darwin-arm64@2.0.3: + resolution: + { + integrity: sha512-LUcqvkV9Bxtng6QHbevp8IK8zzwbIxM6HMjCE7FEW6yJBN1KwvTtRtsGBwwmTxaaLO0wD1Jgl3vgkXAmQ4fqUw==, + } + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /turbo-linux-64@2.0.3: + resolution: + { + integrity: sha512-xpdY1suXoEbsQsu0kPep2zrB8ijv/S5aKKrntGuQ62hCiwDFoDcA/Z7FZ8IHQ2u+dpJARa7yfiByHmizFE0r5Q==, + } + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /turbo-linux-arm64@2.0.3: + resolution: + { + integrity: sha512-MBACTcSR874L1FtLL7gkgbI4yYJWBUCqeBN/iE29D+8EFe0d3fAyviFlbQP4K/HaDYet1i26xkkOiWr0z7/V9A==, + } + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /turbo-windows-64@2.0.3: + resolution: + { + integrity: sha512-zi3YuKPkM9JxMTshZo3excPk37hUrj5WfnCqh4FjI26ux6j/LJK+Dh3SebMHd9mR7wP9CMam4GhmLCT+gDfM+w==, + } + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /turbo-windows-arm64@2.0.3: + resolution: + { + integrity: sha512-wmed4kkenLvRbidi7gISB4PU77ujBuZfgVGDZ4DXTFslE/kYpINulwzkVwJIvNXsJtHqyOq0n6jL8Zwl3BrwDg==, + } + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /turbo@2.0.3: + resolution: + { + integrity: sha512-jF1K0tTUyryEWmgqk1V0ALbSz3VdeZ8FXUo6B64WsPksCMCE48N5jUezGOH2MN0+epdaRMH8/WcPU0QQaVfeLA==, + } + hasBin: true + optionalDependencies: + turbo-darwin-64: 2.0.3 + turbo-darwin-arm64: 2.0.3 + turbo-linux-64: 2.0.3 + turbo-linux-arm64: 2.0.3 + turbo-windows-64: 2.0.3 + turbo-windows-arm64: 2.0.3 + dev: true diff --git a/crates/turborepo-lockfiles/fixtures/pnpm8turbo.yaml b/crates/turborepo-lockfiles/fixtures/pnpm8turbo.yaml new file mode 100644 index 0000000000000..bbc4f50fd09b2 --- /dev/null +++ b/crates/turborepo-lockfiles/fixtures/pnpm8turbo.yaml @@ -0,0 +1,96 @@ +lockfileVersion: "6.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + .: + devDependencies: + turbo: + specifier: ^2.0.3 + version: 2.0.3 + + apps/web: {} + +packages: + /turbo-darwin-64@2.0.3: + resolution: + { + integrity: sha512-v7ztJ8sxdHw3SLfO2MhGFeeU4LQhFii1hIGs9uBiXns/0YTGOvxLeifnfGqhfSrAIIhrCoByXO7nR9wlm10n3Q==, + } + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /turbo-darwin-arm64@2.0.3: + resolution: + { + integrity: sha512-LUcqvkV9Bxtng6QHbevp8IK8zzwbIxM6HMjCE7FEW6yJBN1KwvTtRtsGBwwmTxaaLO0wD1Jgl3vgkXAmQ4fqUw==, + } + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /turbo-linux-64@2.0.3: + resolution: + { + integrity: sha512-xpdY1suXoEbsQsu0kPep2zrB8ijv/S5aKKrntGuQ62hCiwDFoDcA/Z7FZ8IHQ2u+dpJARa7yfiByHmizFE0r5Q==, + } + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /turbo-linux-arm64@2.0.3: + resolution: + { + integrity: sha512-MBACTcSR874L1FtLL7gkgbI4yYJWBUCqeBN/iE29D+8EFe0d3fAyviFlbQP4K/HaDYet1i26xkkOiWr0z7/V9A==, + } + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /turbo-windows-64@2.0.3: + resolution: + { + integrity: sha512-zi3YuKPkM9JxMTshZo3excPk37hUrj5WfnCqh4FjI26ux6j/LJK+Dh3SebMHd9mR7wP9CMam4GhmLCT+gDfM+w==, + } + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /turbo-windows-arm64@2.0.3: + resolution: + { + integrity: sha512-wmed4kkenLvRbidi7gISB4PU77ujBuZfgVGDZ4DXTFslE/kYpINulwzkVwJIvNXsJtHqyOq0n6jL8Zwl3BrwDg==, + } + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /turbo@2.0.3: + resolution: + { + integrity: sha512-jF1K0tTUyryEWmgqk1V0ALbSz3VdeZ8FXUo6B64WsPksCMCE48N5jUezGOH2MN0+epdaRMH8/WcPU0QQaVfeLA==, + } + hasBin: true + optionalDependencies: + turbo-darwin-64: 2.0.3 + turbo-darwin-arm64: 2.0.3 + turbo-linux-64: 2.0.3 + turbo-linux-arm64: 2.0.3 + turbo-windows-64: 2.0.3 + turbo-windows-arm64: 2.0.3 + dev: true diff --git a/crates/turborepo-lockfiles/src/berry/mod.rs b/crates/turborepo-lockfiles/src/berry/mod.rs index c6417a22a3157..ae6a886159cfa 100644 --- a/crates/turborepo-lockfiles/src/berry/mod.rs +++ b/crates/turborepo-lockfiles/src/berry/mod.rs @@ -11,7 +11,7 @@ use std::{ }; use de::Entry; -use identifiers::{Descriptor, Locator}; +use identifiers::{Descriptor, Ident, Locator}; use protocol_resolver::DescriptorResolver; use serde::Deserialize; use thiserror::Error; @@ -520,6 +520,16 @@ impl Lockfile for BerryLockfile { true } } + + fn turbo_version(&self) -> Option { + let turbo_ident = Ident::try_from("turbo").expect("'turbo' is valid identifier"); + let key = self + .locator_package + .keys() + .find(|key| turbo_ident == key.ident)?; + let entry = self.locator_package.get(key)?; + Some(entry.version.clone()) + } } impl LockfileData { @@ -1148,4 +1158,11 @@ mod test { ] ); } + + #[test] + fn test_turbo_version() { + let data = LockfileData::from_bytes(include_bytes!("../../fixtures/berry.lock")).unwrap(); + let lockfile = BerryLockfile::new(data, None).unwrap(); + assert_eq!(lockfile.turbo_version().as_deref(), Some("1.4.6")); + } } diff --git a/crates/turborepo-lockfiles/src/bun/mod.rs b/crates/turborepo-lockfiles/src/bun/mod.rs index bc607352c7ee9..363d443fe3705 100644 --- a/crates/turborepo-lockfiles/src/bun/mod.rs +++ b/crates/turborepo-lockfiles/src/bun/mod.rs @@ -118,6 +118,10 @@ impl Lockfile for BunLockfile { // if the types don't match then we changed package managers any_other.downcast_ref::().is_none() } + + fn turbo_version(&self) -> Option { + None + } } impl Entry { diff --git a/crates/turborepo-lockfiles/src/lib.rs b/crates/turborepo-lockfiles/src/lib.rs index 9434bc1b1a93c..c8fbb9d3d2f45 100644 --- a/crates/turborepo-lockfiles/src/lib.rs +++ b/crates/turborepo-lockfiles/src/lib.rs @@ -62,6 +62,9 @@ pub trait Lockfile: Send + Sync + Any + std::fmt::Debug { /// Determine if there's a global change between two lockfiles fn global_change(&self, other: &dyn Lockfile) -> bool; + + /// Return any turbo version found in the lockfile + fn turbo_version(&self) -> Option; } /// Takes a lockfile, and a map of workspace directory paths -> (package name, diff --git a/crates/turborepo-lockfiles/src/npm.rs b/crates/turborepo-lockfiles/src/npm.rs index 191550f608135..2422a190e8ff8 100644 --- a/crates/turborepo-lockfiles/src/npm.rs +++ b/crates/turborepo-lockfiles/src/npm.rs @@ -147,6 +147,11 @@ impl Lockfile for NpmLockfile { true } } + + fn turbo_version(&self) -> Option { + let turbo_entry = self.packages.get("node_modules/turbo")?; + turbo_entry.version.clone() + } } impl NpmLockfile { @@ -450,4 +455,11 @@ mod test { assert!(closures.get("packages/c").unwrap().is_empty()); Ok(()) } + + #[test] + fn test_turbo_version() -> Result<(), Error> { + let lockfile = NpmLockfile::load(include_bytes!("../fixtures/npm-lock.json"))?; + assert_eq!(lockfile.turbo_version().as_deref(), Some("1.5.5")); + Ok(()) + } } diff --git a/crates/turborepo-lockfiles/src/pnpm/data.rs b/crates/turborepo-lockfiles/src/pnpm/data.rs index ea028cf0eabe0..dd83afb632711 100644 --- a/crates/turborepo-lockfiles/src/pnpm/data.rs +++ b/crates/turborepo-lockfiles/src/pnpm/data.rs @@ -514,6 +514,16 @@ impl crate::Lockfile for PnpmLockfile { true } } + + fn turbo_version(&self) -> Option { + let turbo_version = self + .importers + .values() + // Look through all of the workspace packages for a turbo dependency + // grab the first one we find. + .find_map(|project| project.dependencies.turbo_version())?; + Some(turbo_version.to_owned()) + } } impl DependencyInfo { @@ -547,6 +557,11 @@ impl DependencyInfo { fn get_resolution<'a, V>(maybe_map: &'a Option>, key: &str) -> Option<&'a V> { maybe_map.as_ref().and_then(|maybe_map| maybe_map.get(key)) } + + fn turbo_version(&self) -> Option<&str> { + let (_specifier, version) = self.find_resolution("turbo")?; + Some(version) + } } impl Dependency { @@ -605,6 +620,8 @@ mod tests { const PNPM_V7_PEER: &[u8] = include_bytes!("../../fixtures/pnpm-v7-peer.yaml").as_slice(); const PNPM_V7_PATCH: &[u8] = include_bytes!("../../fixtures/pnpm-v7-patch.yaml").as_slice(); const PNPM_V9: &[u8] = include_bytes!("../../fixtures/pnpm-v9.yaml").as_slice(); + const PNPM6_TURBO: &[u8] = include_bytes!("../../fixtures/pnpm6turbo.yaml").as_slice(); + const PNPM8_TURBO: &[u8] = include_bytes!("../../fixtures/pnpm8turbo.yaml").as_slice(); use super::*; use crate::{Lockfile, Package}; @@ -1242,4 +1259,13 @@ c: "contains patched dependency" ); } + + #[test_case(PNPM6, None ; "v6 missing")] + #[test_case(PNPM6_TURBO, Some("2.0.3") ; "v6")] + #[test_case(PNPM8_TURBO, Some("2.0.3") ; "v8")] + #[test_case(PNPM_V9, Some("1.13.3-canary.1") ; "v9")] + fn test_turbo_version(lockfile: &[u8], expected: Option<&str>) { + let lockfile = PnpmLockfile::from_bytes(lockfile).unwrap(); + assert_eq!(lockfile.turbo_version().as_deref(), expected); + } } diff --git a/crates/turborepo-lockfiles/src/yarn1/mod.rs b/crates/turborepo-lockfiles/src/yarn1/mod.rs index 0d5918912c060..f93f1c6d837c4 100644 --- a/crates/turborepo-lockfiles/src/yarn1/mod.rs +++ b/crates/turborepo-lockfiles/src/yarn1/mod.rs @@ -117,6 +117,16 @@ impl Lockfile for Yarn1Lockfile { // if the types don't match then we changed package managers any_other.downcast_ref::().is_none() } + + fn turbo_version(&self) -> Option { + // Yarn lockfiles can have multiple descriptors as a key e.g. turbo@latest, + // turbo@1.2.3 We just check if the first descriptor is for turbo and + // return that. Using multiple versions of turbo in a single project is + // not supported. + let key = self.inner.keys().find(|key| key.starts_with("turbo@"))?; + let entry = self.inner.get(key)?; + Some(entry.version.clone()) + } } pub fn yarn_subgraph(contents: &[u8], packages: &[String]) -> Result, crate::Error> { @@ -176,4 +186,11 @@ mod test { ); } } + + #[test_case(MINIMAL, "1.9.3" ; "minimal lockfile")] + #[test_case(FULL, "1.4.6" ; "full lockfile")] + fn test_turbo_version(lockfile: &str, expected: &str) { + let lockfile = Yarn1Lockfile::from_str(lockfile).unwrap(); + assert_eq!(lockfile.turbo_version().as_deref(), Some(expected)); + } } diff --git a/crates/turborepo-repository/src/inference.rs b/crates/turborepo-repository/src/inference.rs index b704d595de463..cd2fbc9cb8187 100644 --- a/crates/turborepo-repository/src/inference.rs +++ b/crates/turborepo-repository/src/inference.rs @@ -16,6 +16,7 @@ pub enum RepoMode { pub struct RepoState { pub root: AbsoluteSystemPathBuf, pub mode: RepoMode, + pub root_package_json: PackageJson, pub package_manager: Result, } @@ -30,6 +31,7 @@ struct InferInfo { path: AbsoluteSystemPathBuf, workspace_globs: Option, package_manager: Result, + package_json: PackageJson, } impl InferInfo { @@ -57,6 +59,7 @@ impl From for RepoState { mode: root.repo_mode(), package_manager: root.package_manager, root: root.path, + root_package_json: root.package_json, } } } @@ -87,6 +90,7 @@ impl RepoState { path: path.to_owned(), workspace_globs, package_manager, + package_json, } }) }) @@ -113,7 +117,7 @@ mod test { use turbopath::AbsoluteSystemPathBuf; use super::{RepoMode, RepoState}; - use crate::package_manager::PackageManager; + use crate::{package_json::PackageJson, package_manager::PackageManager}; fn tmp_dir() -> (tempfile::TempDir, AbsoluteSystemPathBuf) { let tmp_dir = tempfile::tempdir().unwrap(); @@ -149,11 +153,11 @@ mod test { irrelevant.create_dir_all().unwrap(); let monorepo_root = tmp_dir.join_component("monorepo_root"); let monorepo_pkg_json = monorepo_root.join_component("package.json"); + let monorepo_contents = + "{\"workspaces\": [\"packages/*\"], \"packageManager\": \"npm@7.0.0\"}"; monorepo_pkg_json.ensure_dir().unwrap(); monorepo_pkg_json - .create_with_contents( - "{\"workspaces\": [\"packages/*\"], \"packageManager\": \"npm@7.0.0\"}", - ) + .create_with_contents(monorepo_contents) .unwrap(); monorepo_root .join_component("package-lock.json") @@ -171,9 +175,10 @@ mod test { let standalone = monorepo_root.join_component("standalone"); let standalone_pkg_json = standalone.join_component("package.json"); + let standalone_contents = "{\"name\":\"standalone\"}"; standalone_pkg_json.ensure_dir().unwrap(); standalone_pkg_json - .create_with_contents("{\"name\":\"standalone\"}") + .create_with_contents(standalone_contents) .unwrap(); standalone .join_component("package-lock.json") @@ -181,17 +186,17 @@ mod test { .unwrap(); let standalone_monorepo = monorepo_root.join_component("standalone_monorepo"); + let standalone_monorepo_package_json = standalone_monorepo.join_component("package.json"); + let standalone_monorepo_contents = + "{\"workspaces\": [\"packages/*\"], \"packageManager\": \"npm@7.0.0\"}"; let app_2 = standalone_monorepo.join_components(&["packages", "app-2"]); app_2.create_dir_all().unwrap(); app_2 .join_component("package.json") .create_with_contents("{\"name\":\"app-2\"}") .unwrap(); - standalone_monorepo - .join_component("package.json") - .create_with_contents( - "{\"workspaces\": [\"packages/*\"], \"packageManager\": \"npm@7.0.0\"}", - ) + standalone_monorepo_package_json + .create_with_contents(standalone_monorepo_contents) .unwrap(); standalone_monorepo .join_component("package-lock.json") @@ -200,10 +205,11 @@ mod test { let single_root = tmp_dir.join_component("single_root"); let single_root_src = single_root.join_component("src"); + let single_root_contents = "{\"name\": \"single-root\"}"; + let single_root_package_json = single_root.join_component("package.json"); single_root_src.create_dir_all().unwrap(); - single_root - .join_component("package.json") - .create_with_contents("{\"name\": \"single-root\"}") + single_root_package_json + .create_with_contents(single_root_contents) .unwrap(); single_root .join_component("package-lock.json") @@ -219,6 +225,7 @@ mod test { root: monorepo_root.clone(), mode: RepoMode::MultiPackage, package_manager: Ok(pnpm), + root_package_json: PackageJson::load(&monorepo_pkg_json).unwrap(), }), ), ( @@ -227,6 +234,7 @@ mod test { root: monorepo_root.clone(), mode: RepoMode::MultiPackage, package_manager: Ok(pnpm), + root_package_json: PackageJson::load(&monorepo_pkg_json).unwrap(), }), ), ( @@ -235,6 +243,7 @@ mod test { root: monorepo_root.clone(), mode: RepoMode::MultiPackage, package_manager: Ok(pnpm), + root_package_json: PackageJson::load(&monorepo_pkg_json).unwrap(), }), ), ( @@ -243,6 +252,7 @@ mod test { root: single_root.clone(), mode: RepoMode::SinglePackage, package_manager: Ok(pnpm), + root_package_json: PackageJson::load(&single_root_package_json).unwrap(), }), ), ( @@ -251,6 +261,7 @@ mod test { root: single_root.clone(), mode: RepoMode::SinglePackage, package_manager: Ok(pnpm), + root_package_json: PackageJson::load(&single_root_package_json).unwrap(), }), ), // Nested, technically not supported @@ -260,6 +271,7 @@ mod test { root: standalone.clone(), mode: RepoMode::SinglePackage, package_manager: Ok(pnpm), + root_package_json: PackageJson::load(&standalone_pkg_json).unwrap(), }), ), ( @@ -268,6 +280,8 @@ mod test { root: standalone_monorepo.clone(), mode: RepoMode::MultiPackage, package_manager: Ok(pnpm), + root_package_json: PackageJson::load(&standalone_monorepo_package_json) + .unwrap(), }), ), ( @@ -276,6 +290,8 @@ mod test { root: standalone_monorepo.clone(), mode: RepoMode::MultiPackage, package_manager: Ok(pnpm), + root_package_json: PackageJson::load(&standalone_monorepo_package_json) + .unwrap(), }), ), ]; diff --git a/crates/turborepo-repository/src/package_graph/mod.rs b/crates/turborepo-repository/src/package_graph/mod.rs index aafb9682013f5..0f7a77dfbf0a4 100644 --- a/crates/turborepo-repository/src/package_graph/mod.rs +++ b/crates/turborepo-repository/src/package_graph/mod.rs @@ -680,6 +680,10 @@ mod test { fn global_change(&self, _other: &dyn Lockfile) -> bool { unreachable!("global change detection not necessary for package graph construction") } + + fn turbo_version(&self) -> Option { + None + } } #[tokio::test] diff --git a/crates/turborepo-repository/src/package_manager/mod.rs b/crates/turborepo-repository/src/package_manager/mod.rs index 7530682f96ec7..b6b127bf81c7b 100644 --- a/crates/turborepo-repository/src/package_manager/mod.rs +++ b/crates/turborepo-repository/src/package_manager/mod.rs @@ -321,6 +321,19 @@ static PACKAGE_MANAGER_PATTERN: Lazy = lazy_regex!(r"(?Pbun|npm|pnpm|yarn)@(?P\d+\.\d+\.\d+(-.+)?)"); impl PackageManager { + pub fn supported_managers() -> &'static [Self] { + [ + Self::Npm, + Self::Pnpm9, + Self::Pnpm, + Self::Pnpm6, + Self::Yarn, + Self::Berry, + Self::Bun, + ] + .as_slice() + } + pub fn command(&self) -> &'static str { match self { PackageManager::Npm => "npm", diff --git a/turborepo-tests/helpers/setup.sh b/turborepo-tests/helpers/setup.sh index 884c15ec6609a..9e15edfa48032 100755 --- a/turborepo-tests/helpers/setup.sh +++ b/turborepo-tests/helpers/setup.sh @@ -12,4 +12,5 @@ fi # disable the first-run telemetry message export TURBO_TELEMETRY_MESSAGE_DISABLED=1 export TURBO_GLOBAL_WARNING_DISABLED=1 +export TURBO_DOWNLOAD_LOCAL_DISABLED=1 TURBO=${MONOREPO_ROOT_DIR}/target/debug/turbo${EXT} diff --git a/turborepo-tests/helpers/setup_example_test.sh b/turborepo-tests/helpers/setup_example_test.sh index 40c1f1eeb88e7..f4fd970947d9b 100644 --- a/turborepo-tests/helpers/setup_example_test.sh +++ b/turborepo-tests/helpers/setup_example_test.sh @@ -3,6 +3,7 @@ set -eo pipefail export TURBO_TELEMETRY_MESSAGE_DISABLED=1 +export TURBO_DOWNLOAD_LOCAL_DISABLED=1 # Start by figuring out which example we're testing and its package manager example_path=$1 diff --git a/turborepo-tests/integration/tests/find-turbo/setup.sh b/turborepo-tests/integration/tests/find-turbo/setup.sh index 6ebd0e962131e..bafa637cf0d21 100644 --- a/turborepo-tests/integration/tests/find-turbo/setup.sh +++ b/turborepo-tests/integration/tests/find-turbo/setup.sh @@ -1,5 +1,6 @@ #!/bin/bash +export TURBO_DOWNLOAD_LOCAL_DISABLED=1 SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) TARGET_DIR=$1 FIXTURE_DIR=$2 diff --git a/turborepo-tests/integration/tests/lockfile-aware-caching/setup.sh b/turborepo-tests/integration/tests/lockfile-aware-caching/setup.sh index 4ad5b1ff42638..3280fe5e03e09 100755 --- a/turborepo-tests/integration/tests/lockfile-aware-caching/setup.sh +++ b/turborepo-tests/integration/tests/lockfile-aware-caching/setup.sh @@ -1,6 +1,7 @@ #!/bin/bash export TURBO_GLOBAL_WARNING_DISABLED=1 +export TURBO_DOWNLOAD_LOCAL_DISABLED=1 SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) TARGET_DIR=$1 FIXTURE_DIR=$2