hdfs-example.yaml

---
apiVersion: stable.opencore.com/v1
kind: HDFS

metadata:
  name: hdfs-prod
  labels:
    environment: production
    paidForBy: HR
spec:
  config:
    # Global configuration for all HDFS things go here
  configMapRef:
    # Global ConfigMaps go here

  # TODO: Further Ranger properties. Are there any properties that we need to "inject" into Ranger as well? How'd that work?
  rangerServiceRef: ...
  rangerPolicySetRef: prod-policies # Potentially allow more than one

  # Only needed when HA is enabled
  zooKeeperRef: ...

  roleConfig:
    - selector:
        diskType: ssd
      config:
      configMapRef:
      priority:
    - selector:
        diskCount: 8
      config:
      configMapRef:


  nameNode: # This also needs a Failover controller when HA is enabled
    minNodeCount: 1
    maxNodeCount: 3
    highAvailable: true
    runAsLinuxUser: hdfs
    nodeSelector:
      env: production
      type: master
      diskType: ssd
      paidForBy: Marketing
      hostName: server-0[1-3].opencore.io
    jvmOptions: ...
    config:
      dfs.namenode.disks: /foo/01, /foo/02 # TODO: Can commaseparated values be provided in a more structured way?
      dfs.namenode.http.principal: HTTP
    configMapRef: # TODO: Do we need more than one? How do we deal with priorities/conflicts?
      name: special-config
      priority: 1
    roleConfig:
      - selector:
          diskType: ssd
        config:
          ...
        configMapRef:
      - selector:
          hostName: server-03.opencore.io
        config:
          dfs.datanode.disks:/foo/01,/foo/03 # /foo/02 is broken
          ...
        configMapRef:


  secondaryNameNode: # Optional if HA is not needed
    ...

  journalNode: # TODO: Optional if HA is disabled
    minNodeCount: 2
    maxNodeCount: 3
    nodeSelector:
      env: production
      type: master
    config:
      dfs.namenode.disks: /foo/01, /foo/02
    configMapRef:
      name: special-config
      priority: 1

  dataNode:
    minNodeCount: 3
    maxNodeCount: 10
    nodeSelector:
      env: production
      type: master
    config:
      dfs.namenode.disks: /foo/01, /foo/02
    configMapRef:
      name: special-config
      priority: 1

  kerberos:
    enabled: true
    customKeytabScript: /opt/distro/keytabs.sh
    nodeSelector: # Selects the machine where we can find the Keytab Script
      ...

    # TODO: Move to Secrets or separate Object?
    adRef: centalAd
    adHost:
    adPassword:
    adUsername:
    ipa: ...

  tls:
    enabled: true
    certOrg: ...
    certCN: ...
    certExtendedKeyUsage: clientAuth, serverAuth
    certProviderRef: foobar

  logging:
    retention: ...
    rollingInterval: ...
    defaultExternalLogStorageRef: prod-elk
    # TODO: Potentially introduce a new object for this as well to make it reusable
    nameNodeAuditLog:
      collect: true
      externalLogStorageRef: prod-audit-elk
    nameNodeStdout:
      collect: true

  metrics:
    metricsStorageRef: ...
    blacklist: ...
    whitelist: ...


# TODO: Should HttpFS, NFS Gateway be their own services or part of this one?