chore: remove unused legacy infra (#1339)

lets wait at least to end of week to land this.

The buckets will be orphaned, so we will keep them around a while longer
just as a precaution.

4h downloads is removed in favour of a broader analytics engine query
that runs less frequently. keeps similar behaviour than before while
removing one the largest tables we have.

Author: crowlKats

README.md

@@ -24,13 +24,13 @@ This is the source code for https://jsr.io, the new JavaScript registry.
 
 **Implementation details**
 
-- Modules and package metadata are stored on Google Cloud Storage (GCS)
-- npm compatibility tarballs are stored on Google Cloud Storage (GCS)
+- Modules and package metadata are stored on Cloudflare R2
+- npm compatibility tarballs are stored on Cloudflare R2
 - Management API is implemented in Rust and runs on Google Cloud Run
 - Frontend uses Fresh and is running on Google Cloud Run in 6 regions
 - https://jsr.io, https://api.jsr.io, and https://npm.jsr.io are served by a
   Cloudflare Workers worker
-  - Module, package metadata, and npm tarballs is served directly from GCS
+  - Module, package metadata, and npm tarballs is served directly from R2
   - /api requests are proxied to the management API
   - All other requests are proxied to the frontend
 - Data is stored in PostgreSQL (using Google Cloud SQL)

api/.env.example

@@ -3,7 +3,6 @@ GITHUB_CLIENT_ID=xxx
 GITHUB_CLIENT_SECRET=xxx
 GITLAB_CLIENT_ID=xxx
 GITLAB_CLIENT_SECRET=xxx
-GCS_ENDPOINT=http://localhost:4080
 OTLP_ENDPOINT=http://localhost:4317
 MODULES_BUCKET=modules
 PUBLISHING_BUCKET=publishing

api/.sqlx/query-68d3663094b9cf79de357c436b8fa561176160d508e54868515c0f55bf4bd832.json

@@ -46,7 +46,6 @@ use url::Url;
 use crate::db::DependencyKind;
 use crate::db::ExportsMap;
 use crate::db::PackageVersionMeta;
-use crate::gcs_paths;
 use crate::ids::PackageName;
 use crate::ids::PackagePath;
 use crate::ids::ScopeName;
@@ -56,6 +55,7 @@ use crate::npm::NpmTarballFiles;
 use crate::npm::NpmTarballOptions;
 use crate::npm::create_npm_tarball;
 use crate::s3::BucketWithQueue;
+use crate::s3_paths;
 use crate::tarball::PublishError;
 
 pub struct PackageAnalysisData {
@@ -574,7 +574,7 @@ async fn rebuild_npm_tarball_inner(
     .build(
       roots.clone(),
       vec![],
-      &GcsLoader {
+      &S3Loader {
         files: &files,
         bucket: &modules_bucket,
         scope: &scope,
@@ -633,15 +633,15 @@ async fn rebuild_npm_tarball_inner(
   Ok(npm_tarball)
 }
 
-struct GcsLoader<'a> {
+struct S3Loader<'a> {
   files: &'a HashSet<PackagePath>,
   bucket: &'a BucketWithQueue,
   scope: &'a ScopeName,
   name: &'a PackageName,
   version: &'a Version,
 }
 
-impl GcsLoader<'_> {
+impl S3Loader<'_> {
   fn load_inner(
     &self,
     specifier: &ModuleSpecifier,
@@ -655,12 +655,12 @@ impl GcsLoader<'_> {
         if !self.files.contains(&path) {
           return async move { Ok(None) }.boxed();
         };
-        let gcs_path =
-          gcs_paths::file_path(self.scope, self.name, self.version, &path);
+        let s3_path =
+          s3_paths::file_path(self.scope, self.name, self.version, &path);
         let bucket = self.bucket.clone();
         async move {
           let Some(bytes) = bucket
-            .download(gcs_path.into())
+            .download(s3_path.into())
             .await
             .map_err(|e| LoadError::Other(Arc::new(JsErrorBox::from_err(e))))?
           else {
@@ -691,7 +691,7 @@ impl GcsLoader<'_> {
   }
 }
 
-impl deno_graph::source::Loader for GcsLoader<'_> {
+impl deno_graph::source::Loader for S3Loader<'_> {
   fn load(
     &self,
     specifier: &ModuleSpecifier,

api/src/analysis.rs

@@ -1,8 +1,8 @@
 // Copyright 2024 the JSR authors. All rights reserved. MIT license.
 use crate::NpmUrl;
 use crate::RegistryUrl;
-use crate::buckets::Buckets;
 use crate::external::orama::OramaClient;
+use crate::s3::Buckets;
 use hyper::Body;
 use hyper::Request;
 use routerify::Router;

api/src/api/admin.rs

@@ -5,7 +5,6 @@ use std::borrow::Cow;
 
 use super::ApiPublishingTask;
 use crate::errors;
-use crate::gcp::GcsError;
 use crate::s3::S3Error;
 
 errors!(
@@ -400,12 +399,6 @@ impl
   }
 }
 
-impl From<GcsError> for ApiError {
-  fn from(error: GcsError) -> ApiError {
-    anyhow::Error::from(error).into()
-  }
-}
-
 impl From<crate::docs::DocNodeCacheError> for ApiError {
   fn from(error: crate::docs::DocNodeCacheError) -> ApiError {
     anyhow::Error::from(error).into()

api/src/api/errors.rs

@@ -53,7 +53,6 @@ use crate::RegistryUrl;
 use crate::analysis::JsrResolver;
 use crate::analysis::ModuleParser;
 use crate::auth;
-use crate::buckets::Buckets;
 use crate::db::CreatePackageResult;
 use crate::db::CreatePublishingTaskResult;
 use crate::db::Database;
@@ -66,8 +65,6 @@ use crate::docs::DocsRequest;
 use crate::docs::GeneratedDocsOutput;
 use crate::external::orama::OramaClient;
 use crate::gcp;
-use crate::gcp::CACHE_CONTROL_DO_NOT_CACHE;
-use crate::gcp::GcsUploadOptions;
 use crate::iam::ReqIamExt;
 use crate::ids::PackageName;
 use crate::ids::PackagePath;
@@ -78,6 +75,9 @@ use crate::metadata::VersionMetadata;
 use crate::npm::generate_npm_version_manifest;
 use crate::provenance;
 use crate::publish::publish_task;
+use crate::s3::Buckets;
+use crate::s3::CACHE_CONTROL_DO_NOT_CACHE;
+use crate::s3::S3UploadOptions;
 use crate::s3::UploadTaskBody;
 use crate::tarball::bucket_tarball_path;
 use crate::util;
@@ -615,7 +615,7 @@ async fn update_description(
   }
 
   let npm_version_manifest_path =
-    crate::gcs_paths::npm_version_manifest_path(scope, &package.name);
+    crate::s3_paths::npm_version_manifest_path(scope, &package.name);
   let npm_version_manifest =
     generate_npm_version_manifest(db, npm_url, scope, &package.name).await?;
   let content = serde_json::to_vec_pretty(&npm_version_manifest)?;
@@ -624,7 +624,7 @@ async fn update_description(
     .upload(
       npm_version_manifest_path.into(),
       crate::s3::UploadTaskBody::Bytes(content.into()),
-      GcsUploadOptions {
+      S3UploadOptions {
         content_type: Some("application/json".into()),
         cache_control: Some(CACHE_CONTROL_DO_NOT_CACHE.into()),
         gzip_encoded: false,
@@ -898,7 +898,7 @@ pub async fn version_publish_handler(
     }
   };
 
-  let gcs_path = bucket_tarball_path(publishing_task.id);
+  let s3_path = bucket_tarball_path(publishing_task.id);
 
   let body = req.into_body();
   let total_size = Arc::new(AtomicU64::new(0));
@@ -923,9 +923,9 @@ pub async fn version_publish_handler(
   let upload_result = buckets
     .publishing_bucket
     .upload(
-      gcs_path.into(),
+      s3_path.into(),
       crate::s3::UploadTaskBody::Stream(Box::new(stream)),
-      GcsUploadOptions {
+      S3UploadOptions {
         content_type: Some("application/x-tar".into()),
         cache_control: None,
         gzip_encoded: true,
@@ -1063,7 +1063,7 @@ pub async fn version_update_handler(
   .await?;
 
   let package_metadata_path =
-    crate::gcs_paths::package_metadata(&scope, &package);
+    crate::s3_paths::package_metadata(&scope, &package);
   let package_metadata = PackageMetadata::create(db, &scope, &package).await?;
 
   let content = serde_json::to_vec(&package_metadata)?;
@@ -1072,7 +1072,7 @@ pub async fn version_update_handler(
     .upload(
       package_metadata_path.into(),
       UploadTaskBody::Bytes(content.into()),
-      GcsUploadOptions {
+      S3UploadOptions {
         content_type: Some("application/json".into()),
         cache_control: Some(CACHE_CONTROL_DO_NOT_CACHE.into()),
         gzip_encoded: false,
@@ -1081,7 +1081,7 @@ pub async fn version_update_handler(
     .await?;
 
   let npm_version_manifest_path =
-    crate::gcs_paths::npm_version_manifest_path(&scope, &package);
+    crate::s3_paths::npm_version_manifest_path(&scope, &package);
   let npm_version_manifest =
     generate_npm_version_manifest(db, npm_url, &scope, &package).await?;
   let content = serde_json::to_vec_pretty(&npm_version_manifest)?;
@@ -1090,7 +1090,7 @@ pub async fn version_update_handler(
     .upload(
       npm_version_manifest_path.into(),
       crate::s3::UploadTaskBody::Bytes(content.into()),
-      GcsUploadOptions {
+      S3UploadOptions {
         content_type: Some("application/json".into()),
         cache_control: Some(CACHE_CONTROL_DO_NOT_CACHE.into()),
         gzip_encoded: false,
@@ -1143,20 +1143,20 @@ pub async fn version_delete_handler(
   db.delete_package_version(&staff.id, &scope, &package, &version)
     .await?;
 
-  let v1_path = crate::gcs_paths::docs_v1_path(&scope, &package, &version);
-  let v2_path = crate::gcs_paths::docs_v2_path(&scope, &package, &version);
+  let v1_path = crate::s3_paths::docs_v1_path(&scope, &package, &version);
+  let v2_path = crate::s3_paths::docs_v2_path(&scope, &package, &version);
   buckets.docs_bucket.delete_file(v1_path.into()).await?;
   buckets.docs_bucket.delete_file(v2_path.into()).await?;
 
-  let path = crate::gcs_paths::version_metadata(&scope, &package, &version);
+  let path = crate::s3_paths::version_metadata(&scope, &package, &version);
   buckets.modules_bucket.delete_file(path.into()).await?;
 
   let path =
-    crate::gcs_paths::file_path_root_directory(&scope, &package, &version);
+    crate::s3_paths::file_path_root_directory(&scope, &package, &version);
   buckets.modules_bucket.delete_directory(path.into()).await?;
 
   let package_metadata_path =
-    crate::gcs_paths::package_metadata(&scope, &package);
+    crate::s3_paths::package_metadata(&scope, &package);
   let package_metadata = PackageMetadata::create(db, &scope, &package).await?;
 
   let content = serde_json::to_vec(&package_metadata)?;
@@ -1165,7 +1165,7 @@ pub async fn version_delete_handler(
     .upload(
       package_metadata_path.into(),
       UploadTaskBody::Bytes(content.into()),
-      GcsUploadOptions {
+      S3UploadOptions {
         content_type: Some("application/json".into()),
         cache_control: Some(CACHE_CONTROL_DO_NOT_CACHE.into()),
         gzip_encoded: false,
@@ -1174,7 +1174,7 @@ pub async fn version_delete_handler(
     .await?;
 
   let npm_version_manifest_path =
-    crate::gcs_paths::npm_version_manifest_path(&scope, &package);
+    crate::s3_paths::npm_version_manifest_path(&scope, &package);
   let npm_version_manifest =
     generate_npm_version_manifest(db, npm_url, &scope, &package).await?;
   let content = serde_json::to_vec_pretty(&npm_version_manifest)?;
@@ -1183,7 +1183,7 @@ pub async fn version_delete_handler(
     .upload(
       npm_version_manifest_path.into(),
       crate::s3::UploadTaskBody::Bytes(content.into()),
-      GcsUploadOptions {
+      S3UploadOptions {
         content_type: Some("application/json".into()),
         cache_control: Some(CACHE_CONTROL_DO_NOT_CACHE.into()),
         gzip_encoded: false,
@@ -1296,14 +1296,14 @@ pub async fn get_docs_handler(
     && version.readme_path.is_some();
 
   let readme_fut = if has_readme {
-    let gcs_path = crate::gcs_paths::file_path(
+    let s3_path = crate::s3_paths::file_path(
       &scope,
       &package_name,
       &version.version,
       version.readme_path.as_ref().unwrap(),
     )
     .into();
-    Either::Left(buckets.modules_bucket.download(gcs_path))
+    Either::Left(buckets.modules_bucket.download(s3_path))
   } else {
     Either::Right(futures::future::ready(Ok(None)))
   };
@@ -1578,14 +1578,14 @@ pub async fn get_source_handler(
   let version = maybe_version.ok_or(ApiError::PackageVersionNotFound)?;
 
   let file = if path == "meta.json" {
-    let source_file_path = crate::gcs_paths::package_metadata(&scope, &package);
+    let source_file_path = crate::s3_paths::package_metadata(&scope, &package);
     buckets
       .modules_bucket
       .download(source_file_path.into())
       .await?
   } else if path == format!("{}_meta.json", version.version) {
     let source_file_path =
-      crate::gcs_paths::version_metadata(&scope, &package, &version.version);
+      crate::s3_paths::version_metadata(&scope, &package, &version.version);
     buckets
       .modules_bucket
       .download(source_file_path.into())
@@ -1596,7 +1596,7 @@ pub async fn get_source_handler(
       ApiError::MalformedRequest { msg }
     })?;
 
-    let source_file_path = crate::gcs_paths::file_path(
+    let source_file_path = crate::s3_paths::file_path(
       &scope,
       &package,
       &version.version,
@@ -2047,7 +2047,7 @@ impl DepTreeLoader {
         async move {
           let Some(bytes) = bucket
             .download(
-              crate::gcs_paths::file_path(&scope, &package, &version, &path)
+              crate::s3_paths::file_path(&scope, &package, &version, &path)
                 .into(),
             )
             .await
@@ -2556,11 +2556,11 @@ pub async fn get_dependencies_graph_handler(
   Span::current().record("version", field::display(&version));
 
   let buckets = req.data::<Buckets>().unwrap().clone();
-  let gcs_path =
-    crate::gcs_paths::version_metadata(&scope, &package, &version).into();
+  let s3_path =
+    crate::s3_paths::version_metadata(&scope, &package, &version).into();
   let version_meta = buckets
     .modules_bucket
-    .download(gcs_path)
+    .download(s3_path)
     .await?
     .ok_or(ApiError::PackageVersionNotFound)?;
   let version_meta = serde_json::from_slice::<VersionMetadata>(&version_meta)?;