feat: clickhouse schemas v2

.github/workflows/job_test_api_local.yaml

@@ -30,8 +30,6 @@ jobs:
         uses: ./.github/actions/setup-go
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Install Goose
-        run: go install github.com/pressly/goose/v3/cmd/goose@latest
       - name: Build
         run: pnpm turbo run build --filter=./apps/api...
         env:
@@ -42,12 +40,6 @@ jobs:
         env:
           DRIZZLE_DATABASE_URL: "mysql://unkey:password@localhost:3306/unkey"
           CI: 1
-      - name: Migrate ClickHouse
-        run: goose up
-        env:
-          GOOSE_DRIVER: clickhouse
-          GOOSE_DBSTRING: "tcp://default:[email protected]:9000"
-          GOOSE_MIGRATION_DIR: ./internal/clickhouse/schema
       - name: Test
         run: pnpm vitest run -c vitest.integration.ts --bail=1
         working-directory: apps/api

.github/workflows/job_test_unit.yaml

@@ -15,12 +15,6 @@ jobs:
         run: |
           sudo chmod 666 /var/run/docker.sock
           docker version
-      - name: Install goose
-        run: |
-          wget -qO- https://github.com/pressly/goose/releases/download/v3.20.0/goose_linux_x86_64 > /tmp/goose
-          chmod +x /tmp/goose
-          sudo mv /tmp/goose /usr/local/bin/goose
-          goose --version
       - name: Setup Node
         uses: ./.github/actions/setup-node
         with:

Makefile

@@ -12,18 +12,6 @@ down:
 up: down build
 	docker compose -f ./deployment/docker-compose.yaml up -d
 
-migrate-clickhouse:
-	@export GOOSE_DRIVER=clickhouse && \
-	export GOOSE_DBSTRING="tcp://default:[email protected]:9000" && \
-	export GOOSE_MIGRATION_DIR=./internal/clickhouse/schema && \
-	goose up
-
-migrate-clickhouse-reset:
-	@export GOOSE_DRIVER=clickhouse && \
-	export GOOSE_DBSTRING="tcp://default:[email protected]:9000" && \
-	export GOOSE_MIGRATION_DIR=./internal/clickhouse/schema && \
-	goose down-to 0
-
 integration: up
 	@cd apps/api && \
 	$(MAKE) seed && \

deployment/Dockerfile.clickhouse

@@ -0,0 +1,8 @@
+FROM bitnami/clickhouse:25.6.4
+
+# Copy ClickHouse schemas
+COPY go/pkg/clickhouse/migrations /opt/clickhouse-schemas/
+
+# Create initialization script that will execute our SQL files on first run
+# (script is already made executable on host)
+COPY deployment/init-clickhouse.sh /docker-entrypoint-initdb.d/init-clickhouse.sh

deployment/docker-compose.yaml

@@ -110,7 +110,9 @@ services:
       VAULT_MASTER_KEYS: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U="
       CLICKHOUSE_URL: "clickhouse://default:password@clickhouse:9000"
   clickhouse:
-    image: bitnami/clickhouse:25.6.4
+    build:
+      context: ..
+      dockerfile: deployment/Dockerfile.clickhouse
     container_name: clickhouse
     environment:
       CLICKHOUSE_ADMIN_USER: default
@@ -138,13 +140,7 @@ services:
       retries: 10
       start_period: 30s
       interval: 5s
-  clickhouse_migrator:
-    container_name: clickhouse_migrator
-    build:
-      context: ../internal/clickhouse
-      dockerfile: ./Dockerfile
-    depends_on:
-      - clickhouse
+
   s3:
     container_name: s3
     image: bitnami/minio:2025.4.3

deployment/init-clickhouse.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+echo "Initializing ClickHouse schemas..."
+
+# Execute SQL files in order from our schemas directory
+
+# Execute SQL files in numerical order
+for sql_file in /opt/clickhouse-schemas/*.sql; do
+      echo "Executing: $sql_file"
+
+        if ! clickhouse-client --host localhost --user "$CLICKHOUSE_ADMIN_USER" --password "$CLICKHOUSE_ADMIN_PASSWORD" --queries-file "$sql_file"; then
+            echo "Error executing $sql_file - stopping initialization"
+            exit 1
+        fi
+done
+
+echo "ClickHouse schema initialization complete!"

go/GO_DOCUMENTATION_GUIDELINES.md

@@ -46,7 +46,7 @@ Create a `doc.go` file in each package with this structure:
 //
 // # Key Types
 //
-// The main entry point is [RateLimiter], which provides the [RateLimiter.Allow] 
+// The main entry point is [RateLimiter], which provides the [RateLimiter.Allow]
 // method for checking rate limits. Configuration is handled through [Config].
 //
 // # Usage
@@ -99,7 +99,7 @@ Every exported function and method must be documented. Focus on:
 Every public function must be thoroughly documented. Here's what comprehensive documentation looks like:
 
 ```go
-// Allow determines whether the specified identifier can perform the requested number 
+// Allow determines whether the specified identifier can perform the requested number
 // of operations within the configured rate limit window.
 //
 // This method implements distributed rate limiting with strong consistency guarantees
@@ -121,11 +121,6 @@ Every public function must be thoroughly documented. Here's what comprehensive d
 //   - Updates the rate limit counters atomically if the request is allowed
 //   - Implements fair queuing to prevent starvation under high load
 //
-// Performance:
-//   - Typical latency: <1ms for cached identifiers, <10ms for distributed coordination
-//   - Scales linearly with cluster size up to ~100 nodes
-//   - Memory usage: ~100 bytes per active identifier
-//
 // Returns:
 //   - (true, nil): Request is allowed and counters have been updated
 //   - (false, nil): Request is rate limited, no error condition
@@ -200,10 +195,10 @@ Document all exported types, focusing on:
 type Config struct {
     // Window is the time period over which operations are counted
     Window time.Duration
-    
+
     // Limit is the maximum number of operations allowed within Window
     Limit int64
-    
+
     // ClusterNodes lists all nodes participating in distributed rate limiting.
     // Must include at least the local node.
     ClusterNodes []string
@@ -225,7 +220,7 @@ type Cache[T any] interface {
     // Get retrieves a value by key. Returns the value and whether it was found.
     // A cache miss (found=false) is not an error.
     Get(ctx context.Context, key string) (value T, found bool, err error)
-    
+
     // Set stores a value. The value will be replicated to other cache nodes
     // asynchronously. Use SetSync if you need immediate consistency.
     Set(ctx context.Context, key string, value T) error
@@ -241,7 +236,7 @@ var (
     // ErrRateLimited is returned when an operation exceeds the configured rate limit.
     // This is expected behavior, not a system error.
     ErrRateLimited = errors.New("rate limit exceeded")
-    
+
     // ErrClusterUnavailable indicates that the required number of cluster nodes
     // are not reachable. Operations may still succeed if configured to fail-open.
     ErrClusterUnavailable = errors.New("insufficient cluster nodes available")
@@ -264,7 +259,7 @@ const (
     // DefaultWindow is the standard rate limiting window for new limiters.
     // Chosen as a balance between memory usage and granularity for most use cases.
     DefaultWindow = time.Minute
-    
+
     // MaxBurstRatio determines how much bursting is allowed above the base rate.
     // Set to 1.5 based on analysis of traffic patterns in production.
     MaxBurstRatio = 1.5
@@ -301,7 +296,7 @@ func (r *RateLimiter) distributeTokens(ctx context.Context, required int64) (gra
     if r.localTokens.Load() >= required {
         // ... implementation
     }
-    
+
     // Cluster coordination required
     // We use Raft consensus here instead of eventual consistency because
     // rate limiting must be strictly enforced for security and billing
@@ -321,25 +316,25 @@ func Example_basicUsage() {
         Limit:  1000,
         ClusterNodes: []string{"localhost:8080"},
     }
-    
+
     limiter, err := New(cfg)
     if err != nil {
         log.Fatal(err)
     }
     defer limiter.Close()
-    
+
     // Check if user can make 5 API calls
     allowed, err := limiter.Allow(context.Background(), "user:alice", 5)
     if err != nil {
         log.Printf("System error: %v", err)
         return
     }
-    
+
     if !allowed {
         log.Println("Rate limit exceeded")
         return
     }
-    
+
     log.Println("Request allowed")
     // Output: Request allowed
 }
@@ -395,7 +390,7 @@ Follow these formatting and style conventions:
 9. **Self-contained documentation** - provide all necessary information
 10. **Cross-references** using Go's `[Reference]` format:
     - `[OtherFunc]` for functions
-    - `[TypeName]` for structs/interfaces  
+    - `[TypeName]` for structs/interfaces
     - `[ConstantName]` for constants
 
 ## Best Practices and Anti-Patterns
@@ -409,11 +404,11 @@ Follow these formatting and style conventions:
 //
 // IMPORTANT: Do not call Allow() in a loop without backoff - this can
 // overwhelm the system. Instead use:
-//   
+//
 //   // Bad:
 //   for !limiter.Allow(ctx, id, 1) { /* busy wait */ }
-//   
-//   // Good:  
+//
+//   // Good:
 //   if allowed, err := limiter.Allow(ctx, id, 1); !allowed {
 //       return ErrRateLimited
 //   }
@@ -427,7 +422,7 @@ Before submitting code, verify:
 
 - [ ] **Every package has a dedicated `doc.go` file** with comprehensive package documentation
 - [ ] Every exported function, method, type, constant, and variable is documented
-- [ ] Package documentation in `doc.go` explains purpose, key concepts, and includes examples
+- [ ] Package documentation in `doc.go` explains purpose, key concepts but not details
 - [ ] Internal code explains "why" decisions were made, not just "what" it does
 - [ ] Error conditions and return values are clearly explained
 - [ ] Complex algorithms include reasoning for the chosen approach
@@ -436,19 +431,19 @@ Before submitting code, verify:
 - [ ] All documentation follows Go formatting conventions (proper line breaks, etc.)
 - [ ] Cross-references use proper `[Reference]` format
 - [ ] Edge cases and non-obvious behaviors are documented
-- [ ] Anti-patterns are prevented with clear guidance
+- [ ] Concurrency guarantees are documented if and only if the code is designed to be concurrently safe
 
 ## Deprecation and Breaking Changes
 
 When deprecating APIs, provide clear migration paths:
 
 ```go
 // Deprecated: Use NewRateLimiterV2 instead. This function will be removed in v2.0.
-// 
+//
 // Migration example:
 //   // Old:
 //   limiter := NewRateLimiter(100, time.Minute)
-//   
+//
 //   // New:
 //   limiter := NewRateLimiterV2(Config{Limit: 100, Window: time.Minute})
 func NewRateLimiter(limit int, window time.Duration) *RateLimiter

go/Makefile

@@ -30,17 +30,7 @@ pull:
 
 up: pull
 	@docker compose -f ../deployment/docker-compose.yaml up -d planetscale mysql redis clickhouse s3 otel
-	@echo "Starting ClickHouse migrations (will retry if ClickHouse isn't ready)..."
-	@for i in {1..10}; do \
-		echo "Migration attempt $$i..."; \
-		if docker compose -f ../deployment/docker-compose.yaml run --rm clickhouse_migrator; then \
-			echo "Migrations completed successfully!"; \
-			break; \
-		else \
-			echo "Migration failed, retrying in 5 seconds..."; \
-			sleep 5; \
-		fi; \
-	done
+
 
 clean:
 	@docker compose -f ../deployment/docker-compose.yaml down --volumes

go/go.mod

@@ -41,6 +41,7 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v1.37.0
 	go.opentelemetry.io/otel/trace v1.37.0
 	golang.org/x/net v0.43.0
+	golang.org/x/perf v0.0.0-20250813145418-2f7363a06fe1
 	golang.org/x/text v0.28.0
 	google.golang.org/protobuf v1.36.8
 )

go/go.sum

@@ -432,6 +432,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
 golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
+golang.org/x/perf v0.0.0-20250813145418-2f7363a06fe1 h1:stGRioFgvBd3x8HoGVg9bb41lLTWLjBMFT/dMB7f4mQ=
+golang.org/x/perf v0.0.0-20250813145418-2f7363a06fe1/go.mod h1:rjfRjhHXb3XNVh/9i5Jr2tXoTd0vOlZN5rzsM8cQE6k=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

go/pkg/array/doc.go

@@ -0,0 +1,41 @@
+// Package array provides generic utility functions for slice generation and manipulation.
+//
+// The package implements core operations for test data generation: creating slices with
+// generated values ([Fill]), selecting random elements ([Random]), transforming slices ([Map]),
+// and aggregating slice data ([Reduce]).
+//
+// All functions use Go generics for type safety and are optimized for high-frequency use
+// in test data generation scenarios.
+//
+// # Usage
+//
+// Generate slices with [Fill]:
+//
+//	userIDs := array.Fill(1000, func() string {
+//	    return fmt.Sprintf("user_%d", rand.Intn(100000))
+//	})
+//
+// Select random elements with [Random]:
+//
+//	outcomes := []string{"VALID", "INVALID", "EXPIRED"}
+//	outcome := array.Random(outcomes)
+//
+// Transform data with [Map]:
+//
+//	numbers := []int{1, 2, 3}
+//	strings := array.Map(numbers, strconv.Itoa)
+//
+// Aggregate data with [Reduce]:
+//
+//	sum := array.Reduce(numbers, func(acc, val int) int { return acc + val }, 0)
+//
+// # Performance
+//
+// [Fill] uses single allocation with O(n) time complexity. [Random] operates in O(1) constant time.
+// [Map] and [Reduce] both use O(n) time with single allocation where applicable.
+//
+// # Concurrency
+//
+// All functions are safe for concurrent use when operating on different slices. [Random] uses
+// Go's global RNG which may become a bottleneck under extreme concurrency.
+package array

go/pkg/array/fill.go

@@ -0,0 +1,33 @@
+package array
+
+// Fill creates a new slice with the specified length where each element is generated by calling the provided function.
+//
+// The generator function is called exactly length times in sequential order. Returns an empty slice
+// for zero or negative length.
+//
+// Parameters:
+//   - length: Number of elements to create
+//   - generator: Function called once per element to generate the value
+//
+// The returned slice has capacity equal to length for optimal memory allocation.
+//
+//	// Generate test data
+//	userIDs := array.Fill(1000, func() string {
+//	    return fmt.Sprintf("user_%d", rand.Intn(10000))
+//	})
+//
+//	// Create structs with varying properties
+//	testCases := array.Fill(100, func() TestCase {
+//	    return TestCase{ID: rand.Int(), Created: time.Now()}
+//	})
+func Fill[T any](length int, generator func() T) []T {
+	if length <= 0 {
+		return []T{}
+	}
+
+	slice := make([]T, length)
+	for i := range length {
+		slice[i] = generator()
+	}
+	return slice
+}