add opentelemery helper scripts (#119)

tests/test_rapids-get-telemetry-trace-id.py

@@ -1,3 +1,4 @@
+import sys
 import os.path
 import subprocess
 
@@ -9,7 +10,7 @@ def test_rapids_compute_trace_id():
         env={
             "GITHUB_REPOSITORY": "rapidsai/gha-tools",
             "GITHUB_RUN_ID": "1123123",
-            "RUN_ATTEMPT": "1"
+            "GITHUB_RUN_ATTEMPT": "1"
         },
         text=True,
         capture_output=True,
@@ -19,31 +20,80 @@ def test_rapids_compute_trace_id():
     assert result.returncode == 0
 
 def test_rapids_get_traceparent():
+        # this should raise, because OTEL_SERVICE_NAME isn't set
+    try:
+        result = subprocess.run(
+            [os.path.join(TOOLS_DIR, "rapids-get-telemetry-traceparent"), "my_job"],
+            env={
+                "GITHUB_REPOSITORY": "rapidsai/gha-tools",
+                "GITHUB_RUN_ID": "1123123",
+                "GITHUB_RUN_ATTEMPT": "1"
+            },
+            text=True,
+            capture_output=True,
+        )
+    except subprocess.CalledProcessError:
+        pass
     result = subprocess.run(
-        [os.path.join(TOOLS_DIR, "rapids-get-telemetry-traceparent"), "my_job"],
+        os.path.join(TOOLS_DIR, "rapids-get-telemetry-traceparent"),
         env={
             "GITHUB_REPOSITORY": "rapidsai/gha-tools",
             "GITHUB_RUN_ID": "1123123",
-            "RUN_ATTEMPT": "1"
+            "GITHUB_RUN_ATTEMPT": "1",
+            "OTEL_SERVICE_NAME": "my job"
         },
         text=True,
         capture_output=True,
     )
-    assert result.stdout.strip() == "00-22ab4ec60f37f446b4a95917e86660df-5f57388b5b07a3e8-01"
+    assert result.stdout.strip() == "00-22ab4ec60f37f446b4a95917e86660df-737c2f0e9d9bd9b0-01"
     assert result.stderr == ""
     assert result.returncode == 0
 
 def test_rapids_get_traceparent_with_step():
     result = subprocess.run(
-        [os.path.join(TOOLS_DIR, "rapids-get-telemetry-traceparent"), "my_job", "my step"],
+        [os.path.join(TOOLS_DIR, "rapids-get-telemetry-traceparent"), "my step"],
         env={
             "GITHUB_REPOSITORY": "rapidsai/gha-tools",
             "GITHUB_RUN_ID": "1123123",
-            "RUN_ATTEMPT": "1"
+            "GITHUB_RUN_ATTEMPT": "1",
+            "OTEL_SERVICE_NAME": "my_job",
         },
         text=True,
         capture_output=True,
     )
-    assert result.stdout.strip() == "00-22ab4ec60f37f446b4a95917e86660df-a6e5bc57fad91889-01"
+    assert result.stdout.strip() == "00-22ab4ec60f37f446b4a95917e86660df-5f57388b5b07a3e8-01"
     assert result.stderr == ""
+    assert result.returncode == 0
+
+
+def test_wrap_otel():
+    result = subprocess.run(
+        [os.path.join(TOOLS_DIR, "rapids-otel-wrap"), "echo", "bob"],
+        text=True,
+        capture_output=True,
+    )
+    assert result.stdout == "bob\n"
+    assert result.stderr == "Skipping instrumentation, running \"echo bob\"\n"
+    assert result.returncode == 0
+
+def test_wrap_otel_with_spaces():
+    result = subprocess.run(
+        [os.path.join(TOOLS_DIR, "rapids-otel-wrap"), "echo", "-n", "bob is here"],
+        text=True,
+        capture_output=True,
+    )
+    # Note: no newline here, because echo -n shouldn't end with a newline
+    assert result.stdout == "bob is here"
+    assert result.stderr == "Skipping instrumentation, running \"echo -n bob is here\"\n"
+    assert result.returncode == 0
+
+def test_wrap_otel_with_spaces_and_parens():
+    result = subprocess.run(
+        [os.path.join(TOOLS_DIR, "rapids-otel-wrap"), "python", "-c", "import sys; print(sys.version)"],
+        text=True,
+        capture_output=True,
+    )
+    # Note: no newline here, because echo -n shouldn't end with a newline
+    assert result.stderr == "Skipping instrumentation, running \"python -c import sys; print(sys.version)\"\n"
+    assert result.stdout == "{}\n".format(sys.version)
     assert result.returncode == 0

tools/rapids-get-telemetry-trace-id

@@ -2,5 +2,10 @@
 # This is a global, per-run identifier. It is the same across all jobs and all steps within all jobs.
 # It is constant from the source repo, to shared-workflows, to shared-actions.
 
-sha="$(echo "${GITHUB_REPOSITORY}+${GITHUB_RUN_ID}+${RUN_ATTEMPT}" | sha256sum | cut -f1 -d' ')"
+if [ "$GITHUB_REPOSITORY" = "" ] || [ "${GITHUB_RUN_ID}" = "" ] || [ "${GITHUB_RUN_ATTEMPT}" = "" ]; then
+    echo "Error: one or more inputs to trace id is empty. The variables that must be set are:"
+    echo "    GITHUB_REPOSITORY, GITHUB_RUN_ID, and GITHUB_RUN_ATTEMPT"
+    exit 1
+fi
+sha="$(echo "${GITHUB_REPOSITORY}+${GITHUB_RUN_ID}+${GITHUB_RUN_ATTEMPT}" | sha256sum | cut -f1 -d' ')"
 echo "${sha:0:32}"

tools/rapids-get-telemetry-traceparent

@@ -11,15 +11,22 @@
 # A step name MAY be provided as the second argument. If it is specified, the output corresponds to
 #     the step within the context of its job.
 
-JOB_NAME=$1
 STEP_NAME=${2:-}
 
+if [ "$OTEL_SERVICE_NAME" = "" ]; then
+    echo "ERROR: OTEL_SERVICE_NAME env var is empty. This means your trace doesn't identify anything."
+    exit 1
+fi
+
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 TRACE_ID="$("${SCRIPT_DIR}"/rapids-get-telemetry-trace-id)"
-JOB_SPAN_ID="${TRACE_ID}-${JOB_NAME}"
+JOB_SPAN_ID="${TRACE_ID}-${OTEL_SERVICE_NAME}"
 STEP_SPAN_ID="${JOB_SPAN_ID}-${STEP_NAME}"
 
+# echo "JOB_SPAN_ID pre-hash: "$JOB_SPAN_ID"" 1>&2
+# echo "STEP_SPAN_ID pre-hash: "$STEP_SPAN_ID"" 1>&2
+
 JOB_TRACEPARENT=$(echo -n "${JOB_SPAN_ID}" | sha256sum | cut -f1 -d' ')
 STEP_TRACEPARENT=$(echo -n "${STEP_SPAN_ID}" | sha256sum | cut -f1 -d' ')
 

tools/rapids-otel-wrap

@@ -10,13 +10,14 @@ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-${OTEL
 OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/metrics}"
 OTEL_EXPORTER_OTLP_LOGS_ENDPOINT="${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT:-${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/logs}"
 export TRACEPARENT="${TRACEPARENT}"
+export OTEL_SERVICE_NAME="${OTEL_SERVICE_NAME}"
 
 if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 ]] && [ "$TRACEPARENT" != "" ]; then
-    echo "Running command with OpenTelemetry instrumentation";
+    echo "Running command with OpenTelemetry instrumentation" >&2;
 
     set -x
     if [ "$OTEL_SERVICE_NAME" = "" ]; then
-        echo "WARNING: OTEL_SERVICE_NAME variable not provided. Traces from different steps may not be associated correctly."
+        echo "WARNING: OTEL_SERVICE_NAME variable not provided. Traces from different steps may not be associated correctly. >&2"
     fi
 
     # Some commands have instrumentation. For example, conda-build has monkey-patched instrumentation
@@ -25,28 +26,39 @@ if [[ $(type otel-cli >/dev/null 2>&1) -eq 0 ]] && [ "$TRACEPARENT" != "" ]; the
     # of otel-cli exec, so that flags don't get confused.
     case "$1" in
         conda* )
-        echo "using opentelemetry-instrument for command";
-        command="opentelemetry-instrument $*"
+        echo "using opentelemetry-instrument for command" >&2;
+        echo "TRACEPARENT prior to otel-cli exec is: \"${TRACEPARENT}\"" >&2;
+        STEP_TRACEPARENT=$("${SCRIPT_DIR}/rapids-get-telemetry-traceparent" "${STEP_NAME}")
+
+        # otel-cli creates a span for us that bridges the traceparent from the parent process
+        # into the command we're wrapping
+        # shellcheck disable=SC2086,SC2048
+        otel-cli exec \
+            --service "${OTEL_SERVICE_NAME}" \
+            --name "Run instrumented \"$*\"" \
+            --force-parent-span-id "$(cut -d'-' -f3 <<<"$STEP_TRACEPARENT")" \
+            --verbose \
+            -- \
+                opentelemetry-instrument \
+                --service_name "${OTEL_SERVICE_NAME}" \
+                $*
         ;;
         * )
-        command="$*"
+        # shellcheck disable=SC2086,SC2048
+        otel-cli exec \
+            --service "${OTEL_SERVICE_NAME}" \
+            --name "Run instrumented \"$*\"" \
+            --force-parent-span-id "$(cut -d'-' -f3 <<<"$STEP_TRACEPARENT")" \
+            --verbose \
+            -- $*
         ;;
     esac
 
-    echo "TRACEPARENT prior to otel-cli exec is: \"${TRACEPARENT}\""
-    STEP_TRACEPARENT=$("${SCRIPT_DIR}/rapids-get-telemetry-traceparent" "${JOB_NAME}" "${OTEL_SERVICE_NAME}")
 
-    # otel-cli creates a span for us that bridges the traceparent from the parent process
-    # into the command we're wrapping
-    otel-cli exec \
-        --name "Run instrumented $*" \
-        --force-parent-span-id "$(cut -d'-' -f3 <<<"$STEP_TRACEPARENT")" \
-        --verbose \
-        -- "${command}"
         RETURN_STATUS=$?
 else
-    echo "Skipping instrumentation, running \"${*}\"";
-    eval "$*"
+    echo "Skipping instrumentation, running \"$*\"" >&2;
+    "$@"
     RETURN_STATUS=$?
 fi