quix.yaml

# Quix Project Descriptor
# This file describes the data pipeline and configuration of resources of a Quix Project.

metadata:
  version: 1.0

# This section describes the Deployments of the data pipeline
deployments:
  - name: Quix TS Datalake API
    image: quixcontainerregistry.azurecr.io/quix-datalake-ts-api:20260420.2-master
    deploymentType: Service
    blobStorage:
      bind: true
    resources:
      cpu: 200
      memory: 1000
      replicas: 1
    publicAccess:
      enabled: true
      urlPrefix: quixlake
    state:
      enabled: true
      size: 1
    network:
      serviceName: quixlake
      ports:
        - port: 80
          targetPort: 80
    variables:
      - name: CATALOG_URL
        inputType: FreeText
        description: REST Catalog service URL
        value: http://iceberg-catalog:80
  - name: Quix TS Query UI
    image: quixcontainerregistry.azurecr.io/quix-datalake-ts-ui:20260420.2-master
    deploymentType: Service
    resources:
      cpu: 200
      memory: 500
      replicas: 1
    publicAccess:
      enabled: true
      urlPrefix: query-ui
    variables:
      - name: QUERY_API_URL
        inputType: FreeText
        description: URL of the DuckDB Query API service to connect to
        required: true
        value: http://quixlake
      - name: DEBUG_LOG_LEVEL
        inputType: FreeText
        description: debug logging level
        value: DEBUG
    plugin:
      embeddedView:
        enabled: true
      sidebarItem:
        show: true
        label: Data Explorer
        icon: search
  - name: Quix TS Datalake Catalog
    image: quixcontainerregistry.azurecr.io/quix-datalake-ts-catalog:20260420.2-master
    deploymentType: Service
    blobStorage:
      bind: true
    resources:
      cpu: 200
      memory: 500
      replicas: 1
    publicAccess:
      enabled: true
      urlPrefix: iceberg-catalog
    network:
      serviceName: iceberg-catalog
      ports:
        - port: 80
          targetPort: 5001
    variables:
      - name: POSTGRES_HOST
        inputType: FreeText
        description: PostgreSQL host address
        value: postgresql
      - name: POSTGRES_PORT
        inputType: FreeText
        description: PostgreSQL port
        value: 80
      - name: POSTGRES_DB
        inputType: FreeText
        description: PostgreSQL database name
        value: iceberg_catalog
      - name: POSTGRES_USER
        inputType: FreeText
        description: PostgreSQL username
        value: admin
      - name: POSTGRES_PASSWORD
        inputType: Secret
        description: PostgreSQL password
        secretKey: postgres_password
      - name: CATALOG_BACKEND
        inputType: FreeText
        required: true
        value: postgres
  - name: postgreSQL
    application: postgresql
    version: latest
    deploymentType: Service
    resources:
      cpu: 200
      memory: 2000
      replicas: 1
    network:
      serviceName: postgresql
      ports:
        - port: 80
          targetPort: 5432
    variables:
      - name: POSTGRES_USER
        inputType: FreeText
        description: The root username to initialize PostgreSQL with
        required: true
        value: admin
      - name: POSTGRES_PASSWORD
        inputType: Secret
        description: The root password to initialize MongoDB with
        required: true
        secretKey: postgres_password
      - name: POSTGRES_DB
        inputType: FreeText
        description: The default database name to initialize PostgreSQL with
        required: true
        value: quix
  - name: TSBS Quix Data Generator
    group: Example pipeline
    application: tsbs-quix-data-generator
    version: latest
    deploymentType: Job
    resources:
      cpu: 200
      memory: 1000
      replicas: 1
    variables:
      - name: output
        inputType: OutputTopic
        description: This is the Quix Topic that will receive the generated data
        required: true
        value: tsbs_data
      - name: TSBS_USE_CASE
        inputType: FreeText
        description: 'The type of data; options: cpu-only, devops, iot'
        required: true
        value: cpu-only
      - name: TSBS_SEED
        inputType: FreeText
        description: 'an integer PRNG seed for deterministic generation. '
        required: true
        value: 123
      - name: TSBS_SCALE
        inputType: FreeText
        description: The number of devices / trucks to generate data for
        required: true
        value: 5
      - name: TSBS_TIMESTAMP_START
        inputType: FreeText
        description: A start time for the data's timestamps, formatted as Y-M-DTH:M:SZ
        required: true
        value: 2025-01-08T00:00:00Z
      - name: TSBS_TIMESTAMP_END
        inputType: FreeText
        description: The end time for the data's timestamps, formatted as Y-M-DTH:M:SZ
        required: true
        value: 2025-01-09T00:00:00Z
      - name: TSBS_LOG_INTERVAL
        inputType: FreeText
        description: how much time should be between each reading per device, formatted as Nt
        required: true
        value: 10s
  - name: TSBS Transformer
    group: Example pipeline
    application: tsbs-transformer
    version: latest
    deploymentType: Service
    resources:
      cpu: 200
      memory: 500
      replicas: 1
    variables:
      - name: input
        inputType: InputTopic
        description: Name of the input topic to listen to.
        value: tsbs_data
      - name: output
        inputType: OutputTopic
        description: Name of the output topic to write to.
        value: tsbs_data_transformed
  - name: quix-ts-datalake-sink
    group: Example pipeline
    application: quix-ts-datalake-sink
    version: latest
    deploymentType: Service
    blobStorage:
      bind: true
    resources:
      cpu: 200
      memory: 500
      replicas: 1
    variables:
      - name: input
        inputType: InputTopic
        description: Name of the Kafka input topic to consume from
        required: true
        value: tsbs_data_transformed
      - name: TABLE_NAME
        inputType: FreeText
        description: Table name for data organization and registration, else defaults to topic name
        value: sensordata
      - name: HIVE_COLUMNS
        inputType: FreeText
        description: Comma-separated list of columns for Hive partitioning. Include year/month/day/hour to extract from TIMESTAMP_COLUMN (e.g., location,year,month,day,sensor_type)
        value: region,datacenter,hostname
      - name: TIMESTAMP_COLUMN
        inputType: FreeText
        description: Column containing timestamp values to extract year/month/day/hour from
        value: ts_ms
      - name: CATALOG_URL
        inputType: FreeText
        description: REST Catalog URL for optional table registration (leave empty to skip)
        value: http://iceberg-catalog
      - name: CATALOG_AUTH_TOKEN
        inputType: Secret
        description: If using a catalog, the respective auth token to access it
        required: true
        secretKey: config_ui_auth_token
      - name: AUTO_DISCOVER
        inputType: FreeText
        description: Automatically register table in REST Catalog on first write
        value: true
      - name: CATALOG_NAMESPACE
        inputType: FreeText
        description: Catalog namespace for table registration
        value: default
      - name: BATCH_SIZE
        inputType: FreeText
        description: Number of messages to batch before writing to S3
        value: 1000
      - name: COMMIT_INTERVAL
        inputType: FreeText
        description: Kafka commit interval in seconds
        value: 30
      - name: KAFKA_KEY_DESERIALIZER
        inputType: FreeText
        description: The key deserializer to use
        value: str
      - name: KAFKA_VALUE_DESERIALIZER
        inputType: FreeText
        description: the value deserializer to use
        value: json
      - name: CONSUMER_GROUP
        inputType: FreeText
        description: Kafka consumer group name
        value: s3_direct_sink_v1.1
      - name: AUTO_OFFSET_RESET
        inputType: FreeText
        description: Where to start consuming if no offset exists
        value: earliest
      - name: LOGLEVEL
        inputType: FreeText
        description: set application logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
        value: INFO
      - name: MAX_WRITE_WORKERS
        inputType: FreeText
        description: How many files can be written in parallel to S3 at once (based on partitioning + batch size)
        value: 10

# This section describes the Topics of the data pipeline
topics:
  - name: tsbs_data
  - name: tsbs_data_transformed
  - name: tsbs_data_transformed_000