From d104a78f9b0a40956a40e99676f509c2900c2c3e Mon Sep 17 00:00:00 2001 From: Steve Flanders Date: Fri, 8 May 2020 09:46:47 -0400 Subject: [PATCH 1/5] Add OpenTelemetry distributed tracing * Add Otel Java auto-instrumentation with Jaeger export * Add Otel Collector * Configure data to be sent to Zipkin --- collector.yaml | 68 ++++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 26 ++++++++++++++---- docker/Dockerfile | 16 ++++++++++- 3 files changed, 103 insertions(+), 7 deletions(-) create mode 100644 collector.yaml diff --git a/collector.yaml b/collector.yaml new file mode 100644 index 0000000000..efd7a1de96 --- /dev/null +++ b/collector.yaml @@ -0,0 +1,68 @@ +extensions: + health_check: + zpages: + endpoint: 0.0.0.0:55679 + +receivers: + + jaeger: + protocols: + grpc: + endpoint: "0.0.0.0:14250" + + otlp: + endpoint: "0.0.0.0:55680" + + prometheus: + config: + scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 10s + static_configs: + - targets: ['localhost:8888'] + # Used to query metrics + labels: + environment: test + metric_relabel_configs: + - source_labels: [ __name__ ] + regex: '.*grpc_io.*' + action: drop + +processors: + + batch: + + attributes/copyfromexistingkey: + actions: + - key: environment + from_attribute: env + action: upsert + attributes/newenvironment: + actions: + - key: environment + value: "test" + action: insert + + queued_retry: + +exporters: + + logging: + + zipkin: + url: "http://tracing-server:9411/api/v2/spans" + +service: + + pipelines: + + traces: + receivers: [jaeger, otlp] + processors: [batch, attributes/copyfromexistingkey, attributes/newenvironment, queued_retry] + exporters: [logging, zipkin] + + #metrics: + # receivers: [prometheus] + # exporters: [logging] + + extensions: [health_check, zpages] diff --git a/docker-compose.yml b/docker-compose.yml index 93c0a42d34..c609bcb719 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: mem_limit: 512M depends_on: - config-server - entrypoint: ["./dockerize","-wait=tcp://config-server:8888","-timeout=60s","--","java", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://config-server:8888","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=discovery", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] ports: - 8761:8761 @@ -25,7 +25,7 @@ services: depends_on: - config-server - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=customers", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] ports: - 8081:8081 @@ -36,7 +36,7 @@ services: depends_on: - config-server - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=visits", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] ports: - 8082:8082 @@ -47,7 +47,7 @@ services: depends_on: - config-server - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=vets", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] ports: - 8083:8083 @@ -58,7 +58,7 @@ services: depends_on: - config-server - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=api-gateway", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] ports: - 8080:8080 @@ -78,7 +78,7 @@ services: depends_on: - config-server - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=admin", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] ports: - 9090:9090 @@ -97,3 +97,17 @@ services: mem_limit: 256M ports: - 9091:9090 + + # Collector + otel-collector: + image: omnition/opentelemetry-collector-contrib:latest + command: ["--config=/etc/otelcol-config.yaml", "--new-metrics", "--legacy-metrics=false"] + depends_on: + - tracing-server + volumes: + - ./collector.yaml:/etc/otelcol-config.yaml + ports: + - "13133:13133" # health_check extension + - "14250:14250" # Jaeger gRPC + - "55679:55679" # zpages extension + diff --git a/docker/Dockerfile b/docker/Dockerfile index 52aef406eb..a1bb2bc8ca 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,14 +6,28 @@ ARG DOCKERIZE_VERSION RUN wget -O dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz RUN tar xzf dockerize.tar.gz RUN chmod +x dockerize +RUN wget -O opentelemetry-auto-0.2.2.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.2.2/opentelemetry-auto-0.2.2.jar +RUN wget -O opentelemetry-auto-exporters-jaeger-0.2.2.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.2.2/opentelemetry-auto-exporters-jaeger-0.2.2.jar # This is the first layer that won't be cached ARG ARTIFACT_NAME ADD ${ARTIFACT_NAME}.jar /app.jar +RUN wget -O /opt/opentelemetry-auto-0.2.2.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.2.2/opentelemetry-auto-0.2.2.jar +RUN wget -O /opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.2.2/opentelemetry-auto-exporters-jaeger-0.2.2.jar ARG EXPOSED_PORT EXPOSE ${EXPOSED_PORT} ENV SPRING_PROFILES_ACTIVE docker -ENTRYPOINT ["java", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] +ENTRYPOINT ["java", \ + "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", \ + "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", \ + "-Dota.exporter.jaeger.endpoint=otel-collector:14250", \ + "-Dota.exporter.jaeger.service.name=config", \ + "-XX:+UnlockExperimentalVMOptions", \ + "-XX:+UseCGroupMemoryLimitForHeap", \ + "-Djava.security.egd=file:/dev/./urandom", \ + "-jar", \ + "/app.jar" \ +] From da1a9230ac7b23e50a86dcfdf0109ac5af45a79f Mon Sep 17 00:00:00 2001 From: Steve Flanders Date: Mon, 18 May 2020 12:41:27 -0400 Subject: [PATCH 2/5] Update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 03cbf9ee96..651d2d4c42 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ If everything goes well, you can access the following services at given location * Config Server - http://localhost:8888 * AngularJS frontend (API Gateway) - http://localhost:8080 * Customers, Vets and Visits Services - random port, check Eureka Dashboard +* Otel Collector - http://localhost:55680 * Tracing Server (Zipkin) - http://localhost:9411/zipkin/ (we use [openzipkin](https://github.com/openzipkin/zipkin/tree/master/zipkin-server)) * Admin Server (Spring Boot Admin) - http://localhost:9090 * Grafana Dashboards - http://localhost:3000 From 0ddd19f7db179bdf8f41e64a0abfb9735bbcee1c Mon Sep 17 00:00:00 2001 From: Steve Flanders Date: Thu, 23 Jul 2020 15:28:13 -0400 Subject: [PATCH 3/5] Upgrade to v0.6.0 --- collector.yaml | 40 ++++++---------------------------------- docker-compose.yml | 30 +++++++++++++++++++++--------- docker/Dockerfile | 12 ++++-------- 3 files changed, 31 insertions(+), 51 deletions(-) diff --git a/collector.yaml b/collector.yaml index efd7a1de96..1a73f0c83b 100644 --- a/collector.yaml +++ b/collector.yaml @@ -5,42 +5,18 @@ extensions: receivers: - jaeger: - protocols: - grpc: - endpoint: "0.0.0.0:14250" - otlp: - endpoint: "0.0.0.0:55680" - - prometheus: - config: - scrape_configs: - - job_name: 'otel-collector' - scrape_interval: 10s - static_configs: - - targets: ['localhost:8888'] - # Used to query metrics - labels: - environment: test - metric_relabel_configs: - - source_labels: [ __name__ ] - regex: '.*grpc_io.*' - action: drop + protocols: + grpc: processors: batch: - attributes/copyfromexistingkey: - actions: - - key: environment - from_attribute: env - action: upsert attributes/newenvironment: actions: - key: environment - value: "test" + value: "petclinic" action: insert queued_retry: @@ -50,19 +26,15 @@ exporters: logging: zipkin: - url: "http://tracing-server:9411/api/v2/spans" + endpoint: "http://tracing-server:9411/api/v2/spans" service: pipelines: traces: - receivers: [jaeger, otlp] - processors: [batch, attributes/copyfromexistingkey, attributes/newenvironment, queued_retry] + receivers: [otlp] + processors: [batch, attributes/newenvironment, queued_retry] exporters: [logging, zipkin] - #metrics: - # receivers: [prometheus] - # exporters: [logging] - extensions: [health_check, zpages] diff --git a/docker-compose.yml b/docker-compose.yml index c609bcb719..3a0c881d88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,9 @@ services: mem_limit: 512M depends_on: - config-server - entrypoint: ["./dockerize","-wait=tcp://config-server:8888","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=discovery", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://config-server:8888","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + environment: + OTEL_RESOURCE_ATTRIBUTES: "service.name=discovery" ports: - 8761:8761 @@ -25,7 +27,9 @@ services: depends_on: - config-server - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=customers", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + environment: + OTEL_RESOURCE_ATTRIBUTES: "service.name=customers" ports: - 8081:8081 @@ -36,7 +40,9 @@ services: depends_on: - config-server - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=visits", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + environment: + OTEL_RESOURCE_ATTRIBUTES: "service.name=visits" ports: - 8082:8082 @@ -47,7 +53,9 @@ services: depends_on: - config-server - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=vets", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + environment: + OTEL_RESOURCE_ATTRIBUTES: "service.name=vets" ports: - 8083:8083 @@ -58,7 +66,9 @@ services: depends_on: - config-server - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=api-gateway", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + environment: + OTEL_RESOURCE_ATTRIBUTES: "service.name=shopping" ports: - 8080:8080 @@ -78,7 +88,9 @@ services: depends_on: - config-server - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=60s","--","java", "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", "-Dota.exporter.jaeger.endpoint=otel-collector:14250", "-Dota.exporter.jaeger.service.name=admin", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + environment: + OTEL_RESOURCE_ATTRIBUTES: "service.name=admin" ports: - 9090:9090 @@ -100,14 +112,14 @@ services: # Collector otel-collector: - image: omnition/opentelemetry-collector-contrib:latest - command: ["--config=/etc/otelcol-config.yaml", "--new-metrics", "--legacy-metrics=false"] + image: otel/opentelemetry-collector-contrib:latest + command: ["--config=/etc/otelcol-config.yaml"] depends_on: - tracing-server volumes: - ./collector.yaml:/etc/otelcol-config.yaml ports: - "13133:13133" # health_check extension - - "14250:14250" # Jaeger gRPC + - "55680:55680" # Jaeger gRPC - "55679:55679" # zpages extension diff --git a/docker/Dockerfile b/docker/Dockerfile index a1bb2bc8ca..80941ebf9a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,14 +6,12 @@ ARG DOCKERIZE_VERSION RUN wget -O dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz RUN tar xzf dockerize.tar.gz RUN chmod +x dockerize -RUN wget -O opentelemetry-auto-0.2.2.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.2.2/opentelemetry-auto-0.2.2.jar -RUN wget -O opentelemetry-auto-exporters-jaeger-0.2.2.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.2.2/opentelemetry-auto-exporters-jaeger-0.2.2.jar +RUN wget -O opentelemetry-javaagent-all.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.6.0/opentelemetry-javaagent-all.jar # This is the first layer that won't be cached ARG ARTIFACT_NAME ADD ${ARTIFACT_NAME}.jar /app.jar -RUN wget -O /opt/opentelemetry-auto-0.2.2.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.2.2/opentelemetry-auto-0.2.2.jar -RUN wget -O /opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.2.2/opentelemetry-auto-exporters-jaeger-0.2.2.jar +RUN wget -O /opt/opentelemetry-javaagent-all.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.6.0/opentelemetry-javaagent-all.jar ARG EXPOSED_PORT EXPOSE ${EXPOSED_PORT} @@ -21,10 +19,8 @@ EXPOSE ${EXPOSED_PORT} ENV SPRING_PROFILES_ACTIVE docker ENTRYPOINT ["java", \ - "-javaagent:/opt/opentelemetry-auto-0.2.2.jar", \ - "-Dota.exporter.jar=/opt/opentelemetry-auto-exporters-jaeger-0.2.2.jar", \ - "-Dota.exporter.jaeger.endpoint=otel-collector:14250", \ - "-Dota.exporter.jaeger.service.name=config", \ + "-javaagent:/opt/opentelemetry-javaagent-all.jar", \ + "-Dotel.otlp.endpoint=otel-collector:55680", \ "-XX:+UnlockExperimentalVMOptions", \ "-XX:+UseCGroupMemoryLimitForHeap", \ "-Djava.security.egd=file:/dev/./urandom", \ From e8f196dab6f17249040dfca9bb56eb589b39b0d0 Mon Sep 17 00:00:00 2001 From: Jakub Wach Date: Mon, 21 Sep 2020 21:52:23 +0200 Subject: [PATCH 4/5] set newest otel collector dependency --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 80941ebf9a..6a472f5524 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,12 +6,12 @@ ARG DOCKERIZE_VERSION RUN wget -O dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz RUN tar xzf dockerize.tar.gz RUN chmod +x dockerize -RUN wget -O opentelemetry-javaagent-all.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.6.0/opentelemetry-javaagent-all.jar +RUN wget -O opentelemetry-javaagent-all.jar https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent-all.jar # This is the first layer that won't be cached ARG ARTIFACT_NAME ADD ${ARTIFACT_NAME}.jar /app.jar -RUN wget -O /opt/opentelemetry-javaagent-all.jar https://github.com/open-telemetry/opentelemetry-auto-instr-java/releases/download/v0.6.0/opentelemetry-javaagent-all.jar +RUN wget -O /opt/opentelemetry-javaagent-all.jar https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent-all.jar ARG EXPOSED_PORT EXPOSE ${EXPOSED_PORT} From 54a32e6691cfe9e741c0cf3e4cf10400f3068802 Mon Sep 17 00:00:00 2001 From: Jakub Wach Date: Wed, 7 Apr 2021 22:54:46 +0200 Subject: [PATCH 5/5] clean-up, configured with latest splunk otel distro --- .gitignore | 1 + README.md | 52 +++--- docker-compose.yml | 151 +++++++++++------- docker/Dockerfile | 4 - collector.yaml => otel/collector.yaml | 13 +- .../src/main/resources/application.properties | 4 + .../src/main/resources/application.properties | 4 + .../src/main/resources/application.properties | 4 + start-instrumented.sh | 5 + 9 files changed, 136 insertions(+), 102 deletions(-) rename collector.yaml => otel/collector.yaml (86%) create mode 100755 start-instrumented.sh diff --git a/.gitignore b/.gitignore index 3b29741b01..3245a3e8e7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ target/ # Branch switching generated/ +/otel/splunk-otel-javaagent-all.jar diff --git a/README.md b/README.md index 651d2d4c42..9b520c0d45 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,21 @@ -# Distributed version of the Spring PetClinic Sample Application built with Spring Cloud +# Spring PetClinic [![Build Status](https://travis-ci.org/spring-petclinic/spring-petclinic-microservices.svg?branch=master)](https://travis-ci.org/spring-petclinic/spring-petclinic-microservices/) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +Distributed version of the Sample Application built with Spring Cloud. Instrumented with the Splunk distribution of the OpenTelemetry Java Instrumentation and accompanied by OpenTelemetry Collector. + + This microservices branch was initially derived from [AngularJS version](https://github.com/spring-petclinic/spring-petclinic-angular1) to demonstrate how to split sample Spring application into [microservices](http://www.martinfowler.com/articles/microservices.html). To achieve that goal we use Spring Cloud Gateway, Spring Cloud Circuit Breaker, Spring Cloud Config, Spring Cloud Sleuth, Resilience4j, Micrometer and the Eureka Service Discovery from the [Spring Cloud Netflix](https://github.com/spring-cloud/spring-cloud-netflix) technology stack. -## Starting services locally without Docker +## Starting the application with instrumentation + +In order to start the application as configured for Splunk OpenTelemetry distribution, simply run `start.sh`. The script will: +- download the agent +- build Docker images +- run all microservices using Docker compose -Every microservice is a Spring Boot application and can be started locally using IDE or `../mvnw spring-boot:run` command. Please note that supporting services (Config and Discovery Server) must be started before any other application (Customers, Vets, Visits and API). -Startup of Tracing server, Admin server, Grafana and Prometheus is optional. If everything goes well, you can access the following services at given location: * Discovery Server - http://localhost:8761 * Config Server - http://localhost:8888 @@ -21,9 +27,14 @@ If everything goes well, you can access the following services at given location * Grafana Dashboards - http://localhost:3000 * Prometheus - http://localhost:9091 -You can tell Config Server to use your local Git repository by using `native` Spring profile and setting -`GIT_REPO` environment variable, for example: -`-Dspring.profiles.active=native -DGIT_REPO=/projects/spring-petclinic-microservices-config` +## Downloading OpenTelemetry Java Agent + +Download the [latest release](https://github.com/signalfx/splunk-otel-java/releases/latest/download/splunk-otel-javaagent-all.jar) + of the Splunk Distribution of OpenTelemetry Java Instrumentation. For example use: + ```bash + $ # download the newest version of the agent + $ sudo curl -vsSL -o /opt/splunk-otel-javaagent-all.jar 'https://github.com/signalfx/splunk-otel-java/releases/latest/download/splunk-otel-javaagent-all.jar' + ``` ## Starting services locally with docker-compose In order to start entire infrastructure using Docker, you have to build images by executing `./mvnw clean install -P buildDocker` @@ -61,33 +72,6 @@ Our issue tracker is available here: https://github.com/spring-petclinic/spring- ## Database configuration In its default configuration, Petclinic uses an in-memory database (HSQLDB) which gets populated at startup with data. -A similar setup is provided for MySql in case a persistent database configuration is needed. -Dependency for Connector/J, the MySQL JDBC driver is already included in the `pom.xml` files. - -### Start a MySql database - -You may start a MySql database with docker: - -``` -docker run -e MYSQL_ROOT_PASSWORD=petclinic -e MYSQL_DATABASE=petclinic -p 3306:3306 mysql:5.7.8 -``` -or download and install the MySQL database (e.g., MySQL Community Server 5.7 GA), which can be found here: https://dev.mysql.com/downloads/ - -### Use the Spring 'mysql' profile - -To use a MySQL database, you have to start 3 microservices (`visits-service`, `customers-service` and `vets-services`) -with the `mysql` Spring profile. Add the `--spring.profiles.active=mysql` as programm argument. - -By default, at startup, database schema will be created and data will be populated. -You may also manually create the PetClinic database and data by executing the `"db/mysql/{schema,data}.sql"` scripts of each 3 microservices. -In the `application.yml` of the [Configuration repository], set the `initialization-mode` to `never`. - -If you are running the microservices with Docker, you have to add the `mysql` profile into the (Dockerfile)[docker/Dockerfile]: -``` -ENV SPRING_PROFILES_ACTIVE docker,mysql -``` -In the `mysql section` of the `application.yml` from the [Configuration repository], you have to change -the host and port of your MySQL JDBC connection string. ## Custom metrics monitoring diff --git a/docker-compose.yml b/docker-compose.yml index 3a0c881d88..0bad4d2128 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,94 +5,151 @@ services: image: springcommunity/spring-petclinic-config-server container_name: config-server mem_limit: 512M + volumes: + - ./otel:/otel + environment: + JAVA_TOOL_OPTIONS: -javaagent:/otel/splunk-otel-javaagent-all.jar + OTEL_TRACES_EXPORTER: otlp + OTEL_METRICS_EXPORTER: none + OTEL_EXPORTER_OTLP_ENDPOINT: http://tracing-server:55680 + OTEL_RESOURCE_ATTRIBUTES: service.name=config-server ports: - - 8888:8888 + - 8888:8888 discovery-server: image: springcommunity/spring-petclinic-discovery-server container_name: discovery-server mem_limit: 512M + volumes: + - ./otel:/otel + environment: + JAVA_TOOL_OPTIONS: -javaagent:/otel/splunk-otel-javaagent-all.jar + OTEL_TRACES_EXPORTER: otlp + OTEL_METRICS_EXPORTER: none + OTEL_EXPORTER_OTLP_ENDPOINT: http://tracing-server:55680 + OTEL_RESOURCE_ATTRIBUTES: service.name=discovery-server depends_on: - config-server - entrypoint: ["./dockerize","-wait=tcp://config-server:8888","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] - environment: - OTEL_RESOURCE_ATTRIBUTES: "service.name=discovery" + entrypoint: ["./dockerize","-wait=tcp://config-server:8888","-timeout=300s","--","java", "-jar","/app.jar"] ports: - - 8761:8761 + - 8761:8761 customers-service: image: springcommunity/spring-petclinic-customers-service container_name: customers-service mem_limit: 512M - depends_on: - - config-server - - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + volumes: + - ./otel:/otel environment: - OTEL_RESOURCE_ATTRIBUTES: "service.name=customers" + JAVA_TOOL_OPTIONS: -javaagent:/otel/splunk-otel-javaagent-all.jar + OTEL_TRACES_EXPORTER: otlp + OTEL_METRICS_EXPORTER: none + OTEL_EXPORTER_OTLP_ENDPOINT: http://tracing-server:55680 + OTEL_RESOURCE_ATTRIBUTES: service.name=customers-service + depends_on: + - config-server + - discovery-server + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=300s","--","java", "-jar","/app.jar"] ports: - - 8081:8081 + - 8081:8081 visits-service: image: springcommunity/spring-petclinic-visits-service container_name: visits-service mem_limit: 512M - depends_on: - - config-server - - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + volumes: + - ./otel:/otel environment: - OTEL_RESOURCE_ATTRIBUTES: "service.name=visits" + JAVA_TOOL_OPTIONS: -javaagent:/otel/splunk-otel-javaagent-all.jar + OTEL_TRACES_EXPORTER: otlp + OTEL_METRICS_EXPORTER: none + OTEL_EXPORTER_OTLP_ENDPOINT: http://tracing-server:55680 + OTEL_RESOURCE_ATTRIBUTES: service.name=visits-service + depends_on: + - config-server + - discovery-server + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=300s","--","java", "-jar","/app.jar"] ports: - - 8082:8082 + - 8082:8082 vets-service: image: springcommunity/spring-petclinic-vets-service container_name: vets-service mem_limit: 512M - depends_on: - - config-server - - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + volumes: + - ./otel:/otel environment: - OTEL_RESOURCE_ATTRIBUTES: "service.name=vets" + JAVA_TOOL_OPTIONS: -javaagent:/otel/splunk-otel-javaagent-all.jar + OTEL_TRACES_EXPORTER: otlp + OTEL_METRICS_EXPORTER: none + OTEL_EXPORTER_OTLP_ENDPOINT: http://tracing-server:55680 + OTEL_RESOURCE_ATTRIBUTES: service.name=vets-service + depends_on: + - config-server + - discovery-server + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=300s","--","java", "-jar","/app.jar"] ports: - - 8083:8083 + - 8083:8083 api-gateway: image: springcommunity/spring-petclinic-api-gateway container_name: api-gateway mem_limit: 512M - depends_on: - - config-server - - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + volumes: + - ./otel:/otel environment: - OTEL_RESOURCE_ATTRIBUTES: "service.name=shopping" + JAVA_TOOL_OPTIONS: -javaagent:/otel/splunk-otel-javaagent-all.jar + OTEL_TRACES_EXPORTER: otlp + OTEL_METRICS_EXPORTER: none + OTEL_EXPORTER_OTLP_ENDPOINT: http://tracing-server:55680 + OTEL_RESOURCE_ATTRIBUTES: service.name=api-gateway + depends_on: + - config-server + - discovery-server + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=300s","--","java", "-jar","/app.jar"] ports: - - 8080:8080 + - 8080:8080 tracing-server: - image: openzipkin/zipkin + image: otel/opentelemetry-collector container_name: tracing-server mem_limit: 512M + volumes: + - ./otel:/otel + depends_on: + - zipkin-server + command: --config /otel/collector.yaml + ports: + - 55679:55679 + - 55680:55680 + + zipkin-server: + image: openzipkin/zipkin + container_name: zipkin-server + mem_limit: 512M environment: - - JAVA_OPTS=-XX:+UnlockExperimentalVMOptions -Djava.security.egd=file:/dev/./urandom + - JAVA_OPTS=-XX:+UnlockExperimentalVMOptions -Djava.security.egd=file:/dev/./urandom ports: - - 9411:9411 + - 9411:9411 admin-server: image: springcommunity/spring-petclinic-admin-server container_name: admin-server mem_limit: 512M - depends_on: - - config-server - - discovery-server - entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=240s","--","java", "-javaagent:/opt/opentelemetry-javaagent-all.jar", "-Dotel.otlp.endpoint=otel-collector:55680", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] + volumes: + - ./otel:/otel environment: - OTEL_RESOURCE_ATTRIBUTES: "service.name=admin" + JAVA_TOOL_OPTIONS: -javaagent:/otel/splunk-otel-javaagent-all.jar + OTEL_TRACES_EXPORTER: otlp + OTEL_METRICS_EXPORTER: none + OTEL_EXPORTER_OTLP_ENDPOINT: http://tracing-server:55680 + OTEL_RESOURCE_ATTRIBUTES: service.name=admin-server + depends_on: + - config-server + - discovery-server + entrypoint: ["./dockerize","-wait=tcp://discovery-server:8761","-timeout=300s","--","java", "-jar","/app.jar"] ports: - - 9090:9090 + - 9090:9090 ## Grafana / Prometheus @@ -101,25 +158,11 @@ services: container_name: grafana-server mem_limit: 256M ports: - - 3000:3000 + - 3000:3000 prometheus-server: build: ./docker/prometheus container_name: prometheus-server mem_limit: 256M ports: - - 9091:9090 - - # Collector - otel-collector: - image: otel/opentelemetry-collector-contrib:latest - command: ["--config=/etc/otelcol-config.yaml"] - depends_on: - - tracing-server - volumes: - - ./collector.yaml:/etc/otelcol-config.yaml - ports: - - "13133:13133" # health_check extension - - "55680:55680" # Jaeger gRPC - - "55679:55679" # zpages extension - + - 9091:9090 diff --git a/docker/Dockerfile b/docker/Dockerfile index 6a472f5524..7745c637a9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,12 +6,10 @@ ARG DOCKERIZE_VERSION RUN wget -O dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz RUN tar xzf dockerize.tar.gz RUN chmod +x dockerize -RUN wget -O opentelemetry-javaagent-all.jar https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent-all.jar # This is the first layer that won't be cached ARG ARTIFACT_NAME ADD ${ARTIFACT_NAME}.jar /app.jar -RUN wget -O /opt/opentelemetry-javaagent-all.jar https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent-all.jar ARG EXPOSED_PORT EXPOSE ${EXPOSED_PORT} @@ -19,8 +17,6 @@ EXPOSE ${EXPOSED_PORT} ENV SPRING_PROFILES_ACTIVE docker ENTRYPOINT ["java", \ - "-javaagent:/opt/opentelemetry-javaagent-all.jar", \ - "-Dotel.otlp.endpoint=otel-collector:55680", \ "-XX:+UnlockExperimentalVMOptions", \ "-XX:+UseCGroupMemoryLimitForHeap", \ "-Djava.security.egd=file:/dev/./urandom", \ diff --git a/collector.yaml b/otel/collector.yaml similarity index 86% rename from collector.yaml rename to otel/collector.yaml index 1a73f0c83b..dc8dd80bef 100644 --- a/collector.yaml +++ b/otel/collector.yaml @@ -4,34 +4,27 @@ extensions: endpoint: 0.0.0.0:55679 receivers: - otlp: protocols: grpc: processors: - batch: - + queued_retry: attributes/newenvironment: actions: - key: environment value: "petclinic" action: insert - queued_retry: - exporters: - logging: - + logLevel: debug zipkin: - endpoint: "http://tracing-server:9411/api/v2/spans" + endpoint: "http://zipkin-server:9411/api/v2/spans" service: - pipelines: - traces: receivers: [otlp] processors: [batch, attributes/newenvironment, queued_retry] diff --git a/spring-petclinic-customers-service/src/main/resources/application.properties b/spring-petclinic-customers-service/src/main/resources/application.properties index e69de29bb2..ba3e5428a6 100644 --- a/spring-petclinic-customers-service/src/main/resources/application.properties +++ b/spring-petclinic-customers-service/src/main/resources/application.properties @@ -0,0 +1,4 @@ +spring.datasource.schema=classpath*:db/hsqldb/schema.sql +spring.datasource.data=classpath*:db/hsqldb/data.sql +spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +spring.datasource.url=jdbc:hsqldb:mem:testdb;DB_CLOSE_DELAY=-1 diff --git a/spring-petclinic-vets-service/src/main/resources/application.properties b/spring-petclinic-vets-service/src/main/resources/application.properties index 68f17e806f..2a08689997 100644 --- a/spring-petclinic-vets-service/src/main/resources/application.properties +++ b/spring-petclinic-vets-service/src/main/resources/application.properties @@ -1,2 +1,6 @@ spring.profiles.active=production spring.cache.cache-names=vets +spring.datasource.schema=classpath*:db/hsqldb/schema.sql +spring.datasource.data=classpath*:db/hsqldb/data.sql +spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +spring.datasource.url=jdbc:hsqldb:mem:testdb;DB_CLOSE_DELAY=-1 diff --git a/spring-petclinic-visits-service/src/main/resources/application.properties b/spring-petclinic-visits-service/src/main/resources/application.properties index e69de29bb2..ba3e5428a6 100644 --- a/spring-petclinic-visits-service/src/main/resources/application.properties +++ b/spring-petclinic-visits-service/src/main/resources/application.properties @@ -0,0 +1,4 @@ +spring.datasource.schema=classpath*:db/hsqldb/schema.sql +spring.datasource.data=classpath*:db/hsqldb/data.sql +spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver +spring.datasource.url=jdbc:hsqldb:mem:testdb;DB_CLOSE_DELAY=-1 diff --git a/start-instrumented.sh b/start-instrumented.sh new file mode 100755 index 0000000000..601d026eae --- /dev/null +++ b/start-instrumented.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +curl -vsSL -o ./otel/splunk-otel-javaagent-all.jar 'https://github.com/signalfx/splunk-otel-java/releases/latest/download/splunk-otel-javaagent-all.jar' +./mvnw clean install -P buildDocker +docker-compose up