From 694cccead8be8c195b5fe6b39bf07b5ca4e73bf0 Mon Sep 17 00:00:00 2001 From: jdahmenlemur <142763346+jdahmen-splunk@users.noreply.github.com> Date: Mon, 27 Jan 2025 12:15:43 -0800 Subject: [PATCH] Add slow span scenario Adds a slow span scenario to the recommendation service - add new feature flag - add delay code to service - add scenario description markdown page for cds --- README.md | 4 ++ course_scenarios.md | 47 +++++++++++++++++++++ src/flagd/demo.flagd.json | 11 ++++- src/recommendation/recommendation_server.py | 12 +++++- 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 course_scenarios.md diff --git a/README.md b/README.md index 67e6ac183c..a81c0950c2 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,10 @@ For detailed documentation, see [Demo Documentation][docs]. If you're curious about a specific feature, the [docs landing page][docs] can point you in the right direction. +## Splunk Course Dev Scenarios + +For more information about the scenarios present in the feature flags please refer to the [Scenario Documentation]. + ## Demos featuring the Astronomy Shop We welcome any vendor to fork the project to demonstrate their services and diff --git a/course_scenarios.md b/course_scenarios.md new file mode 100644 index 0000000000..121cb549a0 --- /dev/null +++ b/course_scenarios.md @@ -0,0 +1,47 @@ +## Course Scenarios + +You can find the original scenario documentation provided by the opentelemetry demo project [here](https://opentelemetry.io/docs/demo/feature-flags/) + +The demo provides several feature flags that you can use to simulate different +scenarios. These flags are managed by [`flagd`](https://flagd.dev), a simple +feature flag service that supports [OpenFeature](https://openfeature.dev). + +Flag values can be changed through the user interface provided at + when running the demo. Changing the values +through this user interface will be reflected in the flagd service. + +There are two options when it comes to changing the feature flags through the +user interface: + +- **Basic View**: A user friendly view in which default variants (the same + options that need to be changed when configuring through the raw file) can be + selected and saved for each feature flag. Currently, the basic view does not + support fractional targeting. + +- **Advanced View**: A view in which the raw configuration JSON file is loaded + and can be edited within the browser. The view provides the flexibility that + comes with editing a raw JSON file, however it also provides schema checking + to ensure that the JSON is valid and that the provided configuration values + are correct. + +## OpenTelemetry Project Scenarios + +| Scenario | Service(s) | Description | +| ----------------------------------- | ---------------- | --------------------------------------------------------------------------------------------------------- | +| `adServiceFailure` | Ad Service | Generate an error for `GetAds` 1/10th of the time | +| `adServiceManualGc` | Ad Service | Trigger full manual garbage collections in the ad service | +| `adServiceHighCpu` | Ad Service | Trigger high cpu load in the ad service. If you want to demo cpu throttling, set cpu resource limits | +| `cartServiceFailure` | Cart Service | Generate an error whenever `EmptyCart` is called | +| `productCatalogFailure` | Product Catalog | Generate an error for `GetProduct` requests with product ID: `OLJCESPC7Z` | +| `recommendationServiceCacheFailure` | Recommendation | Create a memory leak due to an exponentially growing cache. 1.4x growth, 50% of requests trigger growth. | +| `paymentServiceFailure` | Payment Service | Generate an error when calling the `charge` method. | +| `paymentServiceUnreachable` | Checkout Service | Use a bad address when calling the PaymentService to make it seem like the PaymentService is unavailable. | +| `loadgeneratorFloodHomepage` | Loadgenerator | Start flooding the homepage with a huge amount of requests, configurable by changing flagd JSON on state. | +| `kafkaQueueProblems` | Kafka | Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike. | +| `imageSlowLoad` | Frontend | Utilizes envoy fault injection, produces a delay in loading of product images in the frontend. | + +## Splunk O11y CD Scenarios + +| Scenario | Service(s) | Description | +| ----------------------------------- | ---------------- | --------------------------------------------------------------------------------------------------------- | +| `slowRecommendationSpan` | Recommendation | Create a slow 5sec span in the recommendation service (get_product_list) | \ No newline at end of file diff --git a/src/flagd/demo.flagd.json b/src/flagd/demo.flagd.json index b4febb1570..6c3a04e36f 100644 --- a/src/flagd/demo.flagd.json +++ b/src/flagd/demo.flagd.json @@ -19,6 +19,15 @@ }, "defaultVariant": "off" }, + "slowRecommendationSpan": { + "description": "Create a slow 5sec span in the recommendation service (get_product_list)", + "state": "ENABLED", + "variants": { + "on": true, + "off": false + }, + "defaultVariant": "off" + }, "adManualGc": { "description": "Triggers full manual garbage collections in the ad service", "state": "ENABLED", @@ -107,4 +116,4 @@ "defaultVariant": "off" } } -} +} \ No newline at end of file diff --git a/src/recommendation/recommendation_server.py b/src/recommendation/recommendation_server.py index df681bfccc..d5864b7a76 100644 --- a/src/recommendation/recommendation_server.py +++ b/src/recommendation/recommendation_server.py @@ -9,6 +9,8 @@ import random from concurrent import futures +import time + # Pip import grpc from opentelemetry import trace, metrics @@ -74,6 +76,14 @@ def get_product_list(request_product_ids): request_product_ids_str = ''.join(request_product_ids) request_product_ids = request_product_ids_str.split(',') + #Feature flag Scenario - Slow Span + if check_feature_flag("slowRecommendationSpan"): + span.set_attribute("app.recommendation.slow_span_enabled", True) + logger.info("get_product_list: slow span enabled") + time.sleep(5) + else: + span.set_attribute("app.recommendation.slow_span_enabled", False) + # Feature flag scenario - Cache Leak if check_feature_flag("recommendationCacheFailure"): span.set_attribute("app.recommendation.cache_enabled", True) @@ -123,7 +133,7 @@ def must_map_env(key: str): def check_feature_flag(flag_name: str): # Initialize OpenFeature client = api.get_client() - return client.get_boolean_value("recommendationCacheFailure", False) + return client.get_boolean_value(flag_name, False) if __name__ == "__main__":