sample-mlops-bestpractices/sagemaker-automated-drift-and-trend-monitoring/.env.example at main · aws-samples/sample-mlops-bestpractices · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
# MLflow Configuration
# Copy this file to .env and update with your actual values

# Required: MLflow Tracking Server URI
# Get this from: AWS SageMaker Partner AI Apps -> MLflow -> View Details
MLFLOW_TRACKING_URI=arn:aws:sagemaker:us-east-1:YOUR_ACCOUNT_ID:mlflow-app/app-YOUR_APP_ID

# Optional: Custom experiment name (defaults to credit-card-fraud-detection)
MLFLOW_EXPERIMENT_NAME=credit-card-fraud-detection

# Optional: Custom model name (defaults to xgboost-fraud-detector)
MLFLOW_MODEL_NAME=xgboost-fraud-detector

# Optional: AWS region (if different from tracking URI)
# AWS_DEFAULT_REGION=us-east-1

# Required for deployment: SageMaker Execution Role ARN
# Get this from: IAM Console -> Roles -> Search for SageMaker execution role
# Example: arn:aws:iam::YOUR_ACCOUNT_ID:role/service-role/AmazonSageMaker-ExecutionRole-YYYYMMDDTHHMMSS
SAGEMAKER_EXEC_ROLE=arn:aws:iam::YOUR_ACCOUNT_ID:role/YOUR_SAGEMAKER_ROLE

# Athena Configuration (for SageMaker pipeline with Athena integration)
ATHENA_DATABASE=fraud_detection
ATHENA_WORKGROUP=primary
ATHENA_OUTPUT_S3=s3://fraud-detection-data-lake/athena-query-results/

# S3 Bucket for Data Lake
DATA_S3_BUCKET=fraud-detection-data-lake
DATA_S3_PREFIX=fraud-detection/

# SageMaker Training Configuration
SAGEMAKER_TRAINING_INSTANCE=ml.m5.xlarge

# Inference Logging Configuration
ENABLE_ATHENA_LOGGING=true
INFERENCE_LOG_BATCH_SIZE=100
INFERENCE_LOG_FLUSH_INTERVAL=300

# SQS Queue URL
SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/<YOUR_ACCOUNT_ID>/<NAME>

# SQS Queue URL for monitoring results (drift metrics → Athena)
MONITORING_SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/<YOUR_ACCOUNT_ID>/fraud-monitoring-results

# Lambda logger function name
LAMBDA_LOGGER_NAME=fraud-inference-log-consumer

# Lambda monitoring writer function name
LAMBDA_MONITORING_WRITER_NAME=fraud-monitoring-results-writer

# ============================================================================
# Evidently Monitoring Configuration
# ============================================================================

# Enable Evidently for drift detection (true/false)
USE_EVIDENTLY=true

# Fallback to legacy PSI/KS detection if Evidently fails (true/false)
EVIDENTLY_FALLBACK_TO_LEGACY=true

# Drift detection thresholds
EVIDENTLY_DRIFT_THRESHOLD=0.2
EVIDENTLY_STAT_TEST_THRESHOLD=0.05

# Statistical test configuration
# Categorical tests: chi_square, psi, jensenshannon, kl_div
EVIDENTLY_CAT_STAT_TEST=chi_square

# Numerical tests: wasserstein, ks, kl_div, psi, jensenshannon
EVIDENTLY_NUM_STAT_TEST=ks

# Baseline data configuration
EVIDENTLY_BASELINE_SAMPLES=10000
EVIDENTLY_BASELINE_S3_PREFIX=baselines

# KS Test Configuration
KS_DRIFT_THRESHOLD=0.1
KS_PVALUE_THRESHOLD=0.05
KS_ENABLE_CDF_PLOTS=true
KS_MAX_FEATURES_TO_PLOT=10

# ============================================================================
# Monitoring Pipeline Configuration
# ============================================================================

# SNS Topic for Monitoring Alerts
MONITORING_SNS_TOPIC_ARN=arn:aws:sns:us-east-1:YOUR_ACCOUNT_ID:fraud-monitoring-alerts

# CloudWatch Alarm Configuration
MONITORING_ALARM_PREFIX=FraudDetection-Monitoring

# Drift Detection Thresholds
MONITORING_KS_THRESHOLD=0.1
MONITORING_F1_THRESHOLD=0.70
MONITORING_DRIFTED_FEATURES_THRESHOLD=5
MONITORING_PERFORMANCE_DROP_PCT=5.0

# Ground Truth Simulation Configuration
GT_HIGH_CONFIDENCE_THRESHOLD=0.9
GT_HIGH_CONFIDENCE_ACCURACY=0.95
GT_MEDIUM_CONFIDENCE_ACCURACY=0.85
GT_LOW_CONFIDENCE_ACCURACY=0.70

# Lambda Execution Role (for monitoring pipeline Lambdas)
# Must have trust policy allowing lambda.amazonaws.com to assume it
LAMBDA_EXEC_ROLE=arn:aws:iam::YOUR_ACCOUNT_ID:role/YOUR_LAMBDA_EXEC_ROLE

# Optional: Email address for monitoring alert notifications
MONITORING_EMAIL=your-email@example.com

# ============================================================================
# Auto-Generated Resources (created by setup scripts)
# ============================================================================
# These values are automatically written by setup scripts. Do not edit manually.
# They will be populated when you run:
# - python src/scripts/setup_monitoring_pipeline.py --create-sns-topic
# - python src/scripts/setup_monitoring_pipeline.py --create-pipeline

# MONITORING_SNS_TOPIC_ARN=arn:aws:sns:us-east-1:123456789012:fraud-monitoring-alerts
# MONITORING_PIPELINE_ARN=arn:aws:sagemaker:us-east-1:123456789012:pipeline/fraud-detection-monitoring-pipeline