diff --git a/charts/data-prepper/CHANGELOG.md b/charts/data-prepper/CHANGELOG.md index dd5feb5f..99817c03 100644 --- a/charts/data-prepper/CHANGELOG.md +++ b/charts/data-prepper/CHANGELOG.md @@ -4,10 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.3.0] +## [0.3.1] ### Added - Added configurable `initContainers` +## [0.3.0] +### Added +- Demo pipeline not part of values.yaml. New value `pipelineConfig.demoPipeline` controls creation of demo pipeline. + ## [0.2.0] ### Added - Added configurable `global.dockerRegistry` diff --git a/charts/data-prepper/README.md b/charts/data-prepper/README.md index e72c2ef9..6f7cee13 100644 --- a/charts/data-prepper/README.md +++ b/charts/data-prepper/README.md @@ -33,12 +33,12 @@ helm repo update helm install my-data-prepper-release opensearch/data-prepper ``` -Replace my-data-prepper-release with your desired release name. +Replace my-data-prepper-release with your desired release name. When no explicit pipeline is defined, this will configure a demo pipeline using a random source and stdout sink. ## Configuration The Data Prepper Helm chart comes with a variety of configuration options to tailor the deployment to your needs. -The default values are specified in the [values.yaml](values.yaml) file. You can override these values by providing your own values.yaml file during installation or by specifying configuration options with --set flags. +The default values are specified in the [values.yaml](values.yaml) file. You can override these values by providing your own `values.yaml` file during installation or by specifying configuration options with --set flags. For a detailed list of configuration options, refer to the values.yaml file or the [Data Prepper documentation](https://opensearch.org/docs/latest/data-prepper/managing-data-prepper/configuring-data-prepper/). @@ -99,8 +99,11 @@ We welcome contributions! Please read our [CONTRIBUTING.md](../../CONTRIBUTING.m | ingress.tls | list | `[]` | | | nameOverride | string | `""` | Override the default name for the deployment | | nodeSelector | object | `{}` | | -| pipelineConfig | object | `{"config":{"simple-sample-pipeline":{"buffer":{"bounded_blocking":{"batch_size":256,"buffer_size":1024}},"delay":5000,"processor":[{"string_converter":{"upper_case":true}}],"sink":[{"stdout":null}],"source":{"random":null},"workers":2}},"enabled":true,"existingSecret":""}` | Pipeline configuration | -| pipelineConfig.existingSecret | string | `""` | The name of the existing secret containing the pipeline configuration. If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`. | +| pipelineConfig | object | (See below) | Pipeline configuration | +| pipelineConfig.enabled | boolean | `false` | Enable inline configuration in `config` sub key. | +| pipelineConfig.config | object | `{}` | Pipeline configuration file inline if `enabled` is set to true | +| pipelineConfig.demoPipeline | boolean | "" | If set, a demo pipeline will be provisioned with source `random` and sink `stdout`. | +| pipelineConfig.existingSecret | string | `""` | The name of an existing secret containing the pipeline configuration. If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`. | | podAnnotations | object | `{}` | | | podLabels | object | `{}` | | | podSecurityContext | object | `{}` | | diff --git a/charts/data-prepper/templates/NOTES.txt b/charts/data-prepper/templates/NOTES.txt index cae9ebaf..6814d818 100644 --- a/charts/data-prepper/templates/NOTES.txt +++ b/charts/data-prepper/templates/NOTES.txt @@ -20,3 +20,11 @@ echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT {{- end }} + +{{- if eq "true" (include "data-prepper.demoPipeline" .) }} +The demo pipeline configuration is enabled, using the `random` source and `stdout` sink. +{{- else if .Values.pipelineConfig.enabled }} +Inline pipeline configuration is enabled. Please refer to the values.yaml file for the configuration. +{{- else }} +Pipeline configuration from secret {{ .Values.pipelineConfig.existingSecret }} is enabled. +{{- end }} \ No newline at end of file diff --git a/charts/data-prepper/templates/_helpers.tpl b/charts/data-prepper/templates/_helpers.tpl index 4f808c76..e10f00a2 100644 --- a/charts/data-prepper/templates/_helpers.tpl +++ b/charts/data-prepper/templates/_helpers.tpl @@ -71,3 +71,25 @@ Create the dockerRegistry prefix if defined {{- .Values.global.dockerRegistry | trimSuffix "/" | printf "%s/" -}} {{- end -}} {{- end -}} + +{{/* +Utility to decide whether demoPipeline should be enabled. +If pipelineConfig.demoPipeline is true/false, return that value. +If pipelineConfig.demoPipeline is undefined, return true if no other pipeline is configured. +*/}} +{{- define "data-prepper.demoPipeline" -}} +{{- $demoPipeline := .Values.pipelineConfig.demoPipeline | toString -}} +{{- if eq $demoPipeline "true" -}} +{{ true }} +{{- else if eq $demoPipeline "false" -}} +{{ false }} +{{- else if or (not $demoPipeline) (eq $demoPipeline "") -}} +{{- if and (not .Values.pipelineConfig.enabled) (not .Values.pipelineConfig.existingSecret) -}} +{{ true }} +{{- else -}} +{{ false }} +{{- end -}} +{{- else -}} +{{ false }} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/data-prepper/templates/demoPipeline.yaml b/charts/data-prepper/templates/demoPipeline.yaml new file mode 100644 index 00000000..5c22c985 --- /dev/null +++ b/charts/data-prepper/templates/demoPipeline.yaml @@ -0,0 +1,25 @@ +{{- if eq "true" (include "data-prepper.demoPipeline" .) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "data-prepper.fullname" . }}-demo-pipeline + labels: + {{- include "data-prepper.labels" . | nindent 4 }} +type: Opaque +stringData: + pipelines.yaml: | + simple-sample-pipeline: + workers: 2 # the number of workers + delay: 5000 # in milliseconds, how long workers wait between read attempts + source: + random: {} + buffer: + bounded_blocking: + buffer_size: 1024 # max number of records the buffer accepts + batch_size: 256 # max number of records the buffer drains after each read + processor: + - string_converter: + upper_case: true + sink: + - stdout: {} +{{- end }} diff --git a/charts/data-prepper/templates/deployment.yaml b/charts/data-prepper/templates/deployment.yaml index 4546e04c..d03efa27 100644 --- a/charts/data-prepper/templates/deployment.yaml +++ b/charts/data-prepper/templates/deployment.yaml @@ -90,10 +90,12 @@ spec: name: {{ include "data-prepper.fullname" . }}-config - name: data-prepper-pipelines secret: - {{- if .Values.pipelineConfig.enabled }} + {{- if eq "true" (include "data-prepper.demoPipeline" .) }} + secretName: {{ include "data-prepper.fullname" . }}-demo-pipeline + {{- else if .Values.pipelineConfig.enabled }} secretName: {{ include "data-prepper.fullname" . }}-pipeline {{- else }} - secretName: {{ required "A valid .Values.pipelineConfig.existingSecret entry required!" .Values.pipelineConfig.existingSecret }} + secretName: {{ required "A valid pipeline configuration is required! Please set one of:\npipelineConfig.existingSecret - for your custom pipeline secret\npipelineConfig.enabled - for inline configuration in values.yaml or\npipelineConfig.demoPipeline=true - for a sample demo pipeline using random source and stdout sink." .Values.pipelineConfig.existingSecret }} {{- end }} {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} diff --git a/charts/data-prepper/values.yaml b/charts/data-prepper/values.yaml index 23a57e9e..39e2b8b0 100644 --- a/charts/data-prepper/values.yaml +++ b/charts/data-prepper/values.yaml @@ -101,160 +101,31 @@ config: # -- Pipeline configuration pipelineConfig: - # If enabled, a secret containing the pipeline configuration will be created based on the 'config' section below. - enabled: true + # If 'true', a secret containing a demo pipeline configuration with random source and stdout sink will be created. + # If left undefined, the demo pipeline will be used only when no other pipeline is configured below + demoPipeline: "" # -- The name of the existing secret containing the pipeline configuration. # If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`. existingSecret: "" + # If enabled, a secret containing the pipeline configuration will be created based on the 'config' section below. + enabled: false # The configuration of the pipeline see https://opensearch.org/docs/2.4/data-prepper/pipelines/pipelines/ config: - ## Simple Example - simple-sample-pipeline: - workers: 2 # the number of workers - delay: 5000 # in milliseconds, how long workers wait between read attempts - source: - random: - buffer: - bounded_blocking: - buffer_size: 1024 # max number of records the buffer accepts - batch_size: 256 # max number of records the buffer drains after each read - processor: - - string_converter: - upper_case: true - sink: - - stdout: - - ## More Complex example - # otel-logs-pipeline: - # workers: 5 - # delay: 10 - # source: - # otel_logs_source: - # ssl: false - # buffer: - # bounded_blocking: - # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: custom - # index: events-%{yyyy.MM.dd} - # #max_retries: 20 - # bulk_size: 4 - # otel-trace-pipeline: - # # workers is the number of threads processing data in each pipeline. - # # We recommend same value for all pipelines. - # # default value is 1, set a value based on the machine you are running Data Prepper - # workers: 8 - # # delay in milliseconds is how often the worker threads should process data. - # # Recommend not to change this config as we want the otel-trace-pipeline to process as quick as possible - # # default value is 3_000 ms - # delay: "100" - # source: - # otel_trace_source: - # ssl: false # Change this to enable encryption in transit - # buffer: - # bounded_blocking: - # # buffer_size is the number of ExportTraceRequest from otel-collector the data prepper should hold in memeory. - # # We recommend to keep the same buffer_size for all pipelines. - # # Make sure you configure sufficient heap - # # default value is 12800 - # buffer_size: 25600 - # # This is the maximum number of request each worker thread will process within the delay. - # # Default is 200. - # # Make sure buffer_size >= workers * batch_size - # batch_size: 400 - # sink: - # - pipeline: - # name: "raw-traces-pipeline" - # - pipeline: - # name: "otel-service-map-pipeline" - # raw-traces-pipeline: - # workers: 5 - # delay: 3000 - # source: - # pipeline: - # name: "otel-trace-pipeline" - # buffer: - # bounded_blocking: - # buffer_size: 25600 # max number of records the buffer accepts - # batch_size: 400 # max number of records the buffer drains after each read - # processor: - # - otel_traces: - # - otel_trace_group: - # hosts: [ "https://opensearch-cluster-master:9200" ] - # insecure: true - # username: "admin" - # password: "admin" - # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: trace-analytics-raw - # otel-service-map-pipeline: - # workers: 5 - # delay: 3000 - # source: - # pipeline: - # name: "otel-trace-pipeline" - # processor: - # - service_map: - # # The window duration is the maximum length of time the data prepper stores the most recent trace data to evaluvate service-map relationships. - # # The default is 3 minutes, this means we can detect relationships between services from spans reported in last 3 minutes. - # # Set higher value if your applications have higher latency. - # window_duration: 180 - # buffer: - # bounded_blocking: - # # buffer_size is the number of ExportTraceRequest from otel-collector the data prepper should hold in memeory. - # # We recommend to keep the same buffer_size for all pipelines. - # # Make sure you configure sufficient heap - # # default value is 12800 - # buffer_size: 25600 - # # This is the maximum number of request each worker thread will process within the delay. - # # Default is 200. - # # Make sure buffer_size >= workers * batch_size - # batch_size: 400 - # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: trace-analytics-service-map - # #index: otel-v1-apm-span-%{yyyy.MM.dd} - # #max_retries: 20 - # bulk_size: 4 - # otel-metrics-pipeline: - # workers: 8 - # delay: 3000 + ## Provide your pipeline configuration here if 'enabled' is set to true. See documentation for more advanced pipelines + # simple-sample-pipeline: + # workers: 2 # the number of workers + # delay: 5000 # in milliseconds, how long workers wait between read attempts # source: - # otel_metrics_source: - # health_check_service: true - # ssl: false + # random: {} # buffer: # bounded_blocking: - # buffer_size: 1024 # max number of records the buffer accepts - # batch_size: 1024 # max number of records the buffer drains after each read + # buffer_size: 1024 # max number of records the buffer accepts + # batch_size: 256 # max number of records the buffer drains after each read # processor: - # - otel_metrics: - # calculate_histogram_buckets: true - # calculate_exponential_histogram_buckets: true - # exponential_histogram_max_allowed_scale: 10 - # flatten_attributes: false + # - string_converter: + # upper_case: true # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: custom - # index: metrics-%{yyyy.MM.dd} - # #max_retries: 20 - # bulk_size: 4 + # - stdout: {} # -- Data Prepper ports ports: diff --git a/charts/opensearch/CHANGELOG.md b/charts/opensearch/CHANGELOG.md index 89c63825..79d42e2f 100644 --- a/charts/opensearch/CHANGELOG.md +++ b/charts/opensearch/CHANGELOG.md @@ -14,6 +14,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed ### Security --- +## [2.30.1] +### Added +- Added scheme for serviceMonitor and optional tlsConfig +### Changed +### Deprecated +### Removed +### Fixed +### Security +--- ## [2.30.0] ### Added - Added plugins.removeList to allow remove plugins @@ -557,8 +566,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed ### Security -[Unreleased]: https://github.com/opensearch-project/helm-charts/compare/opensearch-2.30.0...HEAD -[2.29.0]: https://github.com/opensearch-project/helm-charts/compare/opensearch-2.29.0...opensearch-2.30.0 +[Unreleased]: https://github.com/opensearch-project/helm-charts/compare/opensearch-2.30.1...HEAD +[2.30.1]: https://github.com/opensearch-project/helm-charts/compare/opensearch-2.30.0...opensearch-2.30.1 +[2.30.0]: https://github.com/opensearch-project/helm-charts/compare/opensearch-2.29.0...opensearch-2.30.0 [2.29.0]: https://github.com/opensearch-project/helm-charts/compare/opensearch-2.28.0...opensearch-2.29.0 [2.28.0]: https://github.com/opensearch-project/helm-charts/compare/opensearch-2.27.1...opensearch-2.28.0 [2.27.1]: https://github.com/opensearch-project/helm-charts/compare/opensearch-2.27.0...opensearch-2.27.1 diff --git a/charts/opensearch/Chart.yaml b/charts/opensearch/Chart.yaml index 809a7cff..e9591d3d 100644 --- a/charts/opensearch/Chart.yaml +++ b/charts/opensearch/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 2.30.0 +version: 2.30.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/opensearch/README.md b/charts/opensearch/README.md index cb3561ed..d32292c8 100644 --- a/charts/opensearch/README.md +++ b/charts/opensearch/README.md @@ -132,6 +132,8 @@ helm uninstall my-release | `serviceMonitor.basicAuth.existingSecret` | When using basicAuth for the serviceMonitor, use an existing secret | `""` | | `serviceMonitor.basicAuth.username` | Username to be used for basic auth | `""` | | `serviceMonitor.basicAuth.password` | Password to be used for basic auth | `""` | +| `serviceMonitor.scheme` | scheme to be used for scraping the metrics | `"http"` | +| `serviceMonitor.tlsConfig` | optional tlsConfig to be used for scraping | `{}` | [anti-affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity [environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables diff --git a/charts/opensearch/templates/serviceMonitor.yaml b/charts/opensearch/templates/serviceMonitor.yaml index 85bdd011..0abda693 100644 --- a/charts/opensearch/templates/serviceMonitor.yaml +++ b/charts/opensearch/templates/serviceMonitor.yaml @@ -17,9 +17,14 @@ spec: - port: {{ .Values.service.httpPortName | default "http" }} interval: {{ .Values.serviceMonitor.interval }} path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} {{- if .Values.serviceMonitor.basicAuth.enabled }} basicAuth: - username: + username: {{- if .Values.serviceMonitor.basicAuth.existingSecret }} name: {{ .Values.serviceMonitor.basicAuth.existingSecret }} {{- else }} diff --git a/charts/opensearch/values.yaml b/charts/opensearch/values.yaml index 76373736..9a52b920 100644 --- a/charts/opensearch/values.yaml +++ b/charts/opensearch/values.yaml @@ -550,6 +550,9 @@ serviceMonitor: # Ensure this matches your OpenSearch service configuration. path: /_prometheus/metrics + # Scheme to use for scraping. + scheme: http + # Frequency at which Prometheus will scrape metrics. # Adjust based on your needs. interval: 10s @@ -559,6 +562,9 @@ serviceMonitor: # k8s.example.com/prometheus: kube-prometheus labels: {} + # additional tlsConfig to be added to the ServiceMonitor + tlsConfig: {} + # Basic Auth configuration for the service monitor # You can either use existingSecret, which expects a secret to be already present with data.username and data.password # or set the credentials over the helm values, making helm create a secret for you