✨ feat: otel integration for backend services (#584)

* ✨ Add Opentelemetry Auto-Instrumentation to Backend Services (#553)

* Ignore .logs used for development

* Add basic otel collector config

* add docker compose file for otel-collector

* Add packages needed for serilog to include span ids with logs and update System.Text.Encoding.Web to mitigate CVE-2021-26701

* Add code to get otel endpoint from an environment variable because config from appsettings is not available at this stage. Add code for Serilog to use otel

* Add Packages serilog to attach spans to log messages for otel, and  to mitigate CVE-2021-24112 in System.Drawing.Common

* Add code to get otel endpoint from an environment variable because config from appsettings is not available at this stage. Add code for Serilog to use otel

* Add code for featbit-das to support otel

* A utility powershell script to set otel environment variables on windows

* Add otel components to dockerfiles

* WIP: add prometheus, seq, and jaeger to otel compose

* WIP: Add env config and addtional collector config

* WIP: add prometheus, jaeger and seq

* configure for local development

* Update OTEL_VERION To 1.1.0 in backend api and evaluation api Dockerfiles

* Set SSOEnabled to true in docker dev compose

* Remove unnecessary reference to Serilog.Sinks.Console

* Fix Duplicate Traces and cleanup

* Remove MessageTemplateMD5HashAttribute

* clean

* chore

* fix

* remove unused import

* code clean

* add ENABLE_OPENTELEMETRY environment variable

* add docker-compose-otel.yml

---------

Co-authored-by: rbrennan <[email protected]>

.gitignore

@@ -12,3 +12,6 @@ benchmark/k6-scripts/test-results/
 .vs/
 /.vs/VSWorkspaceState.json
 /.vs/featbit/v17/.wsuo
+
+#Ignore dev logs
+.logs/

docker/composes/docker-compose-otel-collector-contrib.yml

@@ -0,0 +1,65 @@
+name: featbit-otel-collector
+version: "3"
+services:
+ otel-collector:
+ image: otel/opentelemetry-collector-contrib
+ container_name: otel-collector
+ volumes:
+ - ./infra/otel/otel-collector-config.yml:/etc/otelcol-contrib/config.yaml
+ - ./.logs/otel:/otel:rw
+ ports:
+ - 1888:1888 # pprof extension
+ - 8888:8888 # Prometheus metrics exposed by the Collector
+ - 8889:8889 # Prometheus exporter metrics
+ - 13133:13133 # health_check extension
+ - 4317:4317 # OTLP gRPC receiver
+ - 4318:4318 # OTLP http receiver
+ - 55679:55679 # zpages extension
+ networks:
+ - featbit-network
+ depends_on:
+ jaeger:
+ condition: service_started
+ prometheus:
+ condition: service_started
+ seq:
+ condition: service_started
+ jaeger:
+ image: jaegertracing/all-in-one:1.46.0
+ container_name: jaeger
+ environment:
+ - COLLECTOR_OTLP_ENABLED=true
+ networks:
+ - featbit-network
+ ports:
+ - "16686:16686" # Jaeger Web UI
+ # - "14317" # trace port
+ # - "14268"
+ - "14250"
+
+ prometheus:
+ image: prom/prometheus:v2.42.0
+ container_name: prometheus
+ volumes:
+ - ./infra/otel/prometheus.yml:/etc/prometheus/prometheus.yml
+ networks:
+ - featbit-network
+ ports:
+ - "9090:9090"
+ seq:
+ image: datalust/seq:latest
+ container_name: seq
+ networks:
+ - featbit-network
+ ports:
+ - 8082:80
+ - 5341:5341
+ environment:
+ - ACCEPT_EULA=Y 
+networks:
+ featbit-network:
+ name: featbit-network
+ driver: bridge
+ ipam:
+ config:
+ - subnet: 172.10.0.0/16

docker/composes/docker-compose-otel.yml

@@ -0,0 +1,126 @@
+name: featbit-otel
+version: "3"
+services:
+ ui:
+ image: featbit/ui:dev
+ container_name: ui
+ build:
+ context: ./modules/front-end
+ dockerfile: ./Dockerfile
+ environment:
+ - API_URL=http://localhost:5000
+ - DEMO_URL=http://localhost:5173
+ - EVALUATION_URL=http://localhost:5100
+ depends_on:
+ - api-server
+ - demo-dino-game-vue
+ ports:
+ - "8081:80"
+ networks:
+ - featbit-network
+
+ api-server:
+ image: featbit/api:dev
+ container_name: api
+ build:
+ context: ./modules/back-end
+ dockerfile: ./deploy/Dockerfile
+ environment:
+ - SSOEnabled=true
+ env_file:
+ - ./infra/otel/env/otel-api.env
+ depends_on:
+ - mongodb
+ - redis
+ - da-server
+ ports:
+ - "5000:5000"
+ networks:
+ - featbit-network
+
+ evaluation-server:
+ image: featbit/evaluation-server:dev
+ container_name: evaluation-server
+ build:
+ context: ./modules/evaluation-server
+ dockerfile: ./deploy/Dockerfile
+ depends_on:
+ - mongodb
+ - redis
+ env_file:
+ - ./infra/otel/env/otel-els.env
+ ports:
+ - "5100:5100"
+ networks:
+ - featbit-network
+
+ da-server:
+ image: featbit/data-analytics-server:dev
+ container_name: data-analytics-server
+ build:
+ context: ./modules/data-analytics
+ dockerfile: ./Dockerfile
+ depends_on:
+ - mongodb
+ - redis
+ ports:
+ - "8200:80"
+ networks:
+ - featbit-network
+ environment:
+ REDIS_HOST: redis
+ REDIS_PORT: 6379
+ MONGO_URI: mongodb://admin:password@mongodb:27017
+ MONGO_INITDB_DATABASE: featbit
+ MONGO_HOST: mongodb
+ env_file:
+ - ./infra/otel/env/otel-das.env
+
+ mongodb:
+ image: mongo:5.0.14
+ container_name: mongodb
+ restart: on-failure
+ ports:
+ - "27017:27017"
+ environment:
+ - MONGO_INITDB_ROOT_USERNAME=admin
+ - MONGO_INITDB_ROOT_PASSWORD=password
+ - MONGO_INITDB_DATABASE=featbit
+ volumes:
+ - mongodb:/data/db
+ - ./infra/mongodb/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/
+ networks:
+ - featbit-network
+
+ redis:
+ image: bitnami/redis:6.2.10
+ container_name: redis
+ restart: on-failure
+ environment:
+ - ALLOW_EMPTY_PASSWORD=yes
+ networks:
+ - featbit-network
+ ports:
+ - "6379:6379"
+ volumes:
+ - redis:/bitnami/redis/data
+
+ demo-dino-game-vue:
+ image: featbit/demo-dino-game-vue:latest
+ container_name: demo-dino-game-vue
+ networks:
+ - featbit-network
+ ports:
+ - "5173:80"
+
+networks:
+ featbit-network:
+ name: featbit-network
+ driver: bridge
+ ipam:
+ config:
+ - subnet: 172.1.0.0/16
+
+volumes:
+ mongodb:
+ redis:

infra/otel/env/otel-api.env

@@ -0,0 +1,24 @@
+ENABLE_OPENTELEMETRY=true
+
+# OpenTelemetry .NET Automatic Instrumentation
+CORECLR_ENABLE_PROFILING="1"
+CORECLR_PROFILER='{918728DD-259F-4A6A-AC2B-B85E1B658318}'
+CORECLR_PROFILER_PATH="/app/linux-x64/OpenTelemetry.AutoInstrumentation.Native.so"
+DOTNET_ADDITIONAL_DEPS="/app/AdditionalDeps"
+DOTNET_SHARED_STORE="/app/store"
+DOTNET_STARTUP_HOOKS="/app/net/OpenTelemetry.AutoInstrumentation.StartupHook.dll"
+OTEL_DOTNET_AUTO_HOME="/app"
+# Uncomment the following lines to enable verbose logging of OpenTelemetry itself
+# OTEL_LOG_LEVEL="debug"
+# COREHOST_TRACE=1
+# COREHOST_TRACEFILE=corehost_verbose_tracing.log
+
+OTEL_SERVICE_NAME=featbit-api
+OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+OTEL_EXPORTER_OTLP_ENDPOINT_GRPC=http://otel-collector:4317
+OTEL_EXPORTER_OTLP_ENDPOINT_HTTP=http://otel-collector:4318
+OTEL_DOTNET_AUTO_LOGS_CONSOLE_EXPORTER_ENABLED="true"
+OTEL_DOTNET_AUTO_METRICS_CONSOLE_EXPORTER_ENABLED="true"
+OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED="true"
+OTEL_DOTNET_AUTO_TRACES_ADDITIONAL_SOURCES="Api"
+OTEL_DOTNET_AUTO_METRICS_ADDITIONAL_SOURCES="Api"

infra/otel/env/otel-das.env

@@ -0,0 +1,15 @@
+ENABLE_OPENTELEMETRY=true
+
+OTEL_SERVICE_NAME=featbit-das
+OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+
+OTEL_EXPORTER_OTLP_INSECURE=true
+
+OTEL_TRACES_EXPORTER=otlp
+OTEL_METRICS_EXPORTER=otlp
+OTEL_LOGS_EXPORTER=otlp
+
+OTEL_PYTHON_LOG_CORRELATION=true
+OTEL_PYTHON_LOG_FORMAT="%(msg)s [span_id=%(span_id)s]"
+OTEL_PYTHON_LOG_LEVEL=debug
+OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true

infra/otel/env/otel-els.env

@@ -0,0 +1,24 @@
+ENABLE_OPENTELEMETRY=true
+
+# OpenTelemetry .NET Automatic Instrumentation
+CORECLR_ENABLE_PROFILING="1"
+CORECLR_PROFILER='{918728DD-259F-4A6A-AC2B-B85E1B658318}'
+CORECLR_PROFILER_PATH="/app/linux-x64/OpenTelemetry.AutoInstrumentation.Native.so"
+DOTNET_ADDITIONAL_DEPS="/app/AdditionalDeps"
+DOTNET_SHARED_STORE="/app/store"
+DOTNET_STARTUP_HOOKS="/app/net/OpenTelemetry.AutoInstrumentation.StartupHook.dll"
+OTEL_DOTNET_AUTO_HOME="/app"
+# Uncomment the following lines to enable verbose logging of OpenTelemetry itself
+# OTEL_LOG_LEVEL="debug"
+# COREHOST_TRACE=1
+# COREHOST_TRACEFILE=corehost_verbose_tracing.log
+
+OTEL_SERVICE_NAME=featbit-els
+OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+OTEL_EXPORTER_OTLP_ENDPOINT_GRPC=http://otel-collector:4317
+OTEL_EXPORTER_OTLP_ENDPOINT_HTTP=http://otel-collector:4318
+OTEL_DOTNET_AUTO_LOGS_CONSOLE_EXPORTER_ENABLED="true"
+OTEL_DOTNET_AUTO_METRICS_CONSOLE_EXPORTER_ENABLED="true"
+OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED="true"
+OTEL_DOTNET_AUTO_TRACES_ADDITIONAL_SOURCES="Api"
+OTEL_DOTNET_AUTO_METRICS_ADDITIONAL_SOURCES="Api"