Demo for Kyverno

.gitignore

@@ -3,3 +3,6 @@
 webapp/node_modules/
 webapp/package-lock.json
 iac/aws/terraform/creating-custom-vpc/.terraform/
+iac/demo/textract/.terraform.lock.hcl
+iac/demo/textract/.terraform/*
+

iac/demo/kyverno/demo/0-development-namespace.yml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: development
+spec: {}
+status: {}

iac/demo/kyverno/demo/1.1-kyverno-policy-image-source.yml

@@ -0,0 +1,21 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: enforce-image-source
+  namespace: development
+spec:
+  validationFailureAction: Enforce
+  rules:
+    - name: validate-image-source
+      match:
+        resources:
+          kinds:
+            - Deployment
+      validate:
+        message: "Only images from the Docker Hub account vinod827/ are allowed."
+        pattern:
+          spec:
+            template:
+              spec:
+                containers:
+                  - image: "vinod827/*"

iac/demo/kyverno/demo/1.2-sample-app-invalid-image-source.yml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: myapp    
+  name: myapp
+  namespace: development
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: myapp
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        app: myapp
+    spec:
+      containers:
+      - image: nginx
+        name: nginx
+status: {}

iac/demo/kyverno/demo/1.3-sample-app-valid-image-source.yml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: myapp    
+  name: myapp
+  namespace: development
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: myapp
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        app: myapp
+    spec:
+      containers:
+      - image: vinod827/myapp:1.0.1
+        name: myapp
+status: {}

iac/demo/kyverno/demo/2.1-kyverno-clusterpolicy-mutate-label.yml

@@ -0,0 +1,19 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: add-app-label
+spec:
+  rules:
+    - name: add-app-label-to-pods-deployment
+      match:
+        resources:
+          kinds:
+            - Pod
+            - Deployment
+      mutate:
+        patchStrategicMerge:
+          metadata:
+            labels: # Adds these labels if not provided
+              company: "mycompany"  
+              app: "Observability"
+              cost: "shared-infra"

iac/demo/kyverno/demo/3.1-kyverno-policy-generate-configmap.yml

@@ -0,0 +1,32 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: generate-postgres-configmap
+spec:
+  rules:
+    - name: generate-postgres-configmap
+      match:
+        resources:
+          kinds:
+            - Namespace
+      preconditions:
+        - key: "{{ request.object.metadata.labels.team }}"
+          operator: Equals
+          value: "infra"
+      generate:
+        apiVersion: v1
+        kind: ConfigMap
+        name: postgres-config
+        namespace: "{{ request.object.metadata.name }}"
+        synchronize: true
+        data:
+          postgres.conf: |
+            # Sample PostgreSQL Configuration
+            listen_addresses = '*'
+            port = 5432
+            max_connections = 100
+            shared_buffers = 128MB
+            work_mem = 4MB
+            maintenance_work_mem = 64MB
+            timezone = 'UTC'
+            log_statement = 'ddl'

iac/demo/kyverno/demo/3.2-sample-namespace.yml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test-name-2
+  labels:
+    team: infra

iac/demo/kyverno/demo/4.1-kyverno-policy-cleanup.yml

@@ -0,0 +1,16 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: cleanup-completed-jobs
+  namespace: development
+spec:
+  rules:
+    - name: cleanup-jobs
+      match:
+        resources:
+          kinds:
+            - Job
+      mutate:
+        patchStrategicMerge:
+          spec:
+            ttlSecondsAfterFinished: 60

iac/demo/kyverno/demo/4.2-sample-job.yml

@@ -0,0 +1,13 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: test-job
+  namespace: development
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: busybox
+          image: busybox
+          command: ["echo", "Hello, Welcome to the DevOps Malayalam community! Let's deep dive into Kyverno today."]

iac/demo/textract/Makefile

@@ -0,0 +1,27 @@
+# Python Lambda files
+LAMBDA1_FILE = lambda_function.py
+LAMBDA2_FILE = sqs_to_csv_lambda.py
+
+# Output zip files
+LAMBDA1_ZIP = lambda_function.zip
+LAMBDA2_ZIP = sqs_to_csv_lambda.zip
+
+# Default target
+all: zip-lambdas
+
+# Zip the Lambda functions
+zip-lambdas: $(LAMBDA1_ZIP) $(LAMBDA2_ZIP)
+
+$(LAMBDA1_ZIP): $(LAMBDA1_FILE)
+	@echo "Zipping $(LAMBDA1_FILE) into $(LAMBDA1_ZIP)..."
+	zip $(LAMBDA1_ZIP) $(LAMBDA1_FILE)
+
+$(LAMBDA2_ZIP): $(LAMBDA2_FILE)
+	@echo "Zipping $(LAMBDA2_FILE) into $(LAMBDA2_ZIP)..."
+	zip $(LAMBDA2_ZIP) $(LAMBDA2_FILE)
+
+# Clean the zip files
+clean:
+	rm -f $(LAMBDA1_ZIP) $(LAMBDA2_ZIP)
+	@echo "Cleaned up old zip files!"
+

iac/demo/textract/lambda_function.py

@@ -0,0 +1,136 @@
+import boto3
+import json
+import os
+
+s3_client = boto3.client('s3')
+textract_client = boto3.client('textract')
+sns_client = boto3.client('sns')
+
+SNS_TOPIC_ARN = os.environ['SNS_TOPIC_ARN']  # Environment variable for SNS topic ARN
+
+SUPPORTED_IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg']
+SUPPORTED_PDF_EXTENSION = '.pdf'
+
+def lambda_handler(event, context):
+    try:
+        print(f"Event: {json.dumps(event)}")
+
+        # Get S3 bucket and object key from the S3 event
+        bucket_name = event['Records'][0]['s3']['bucket']['name']
+        object_key = event['Records'][0]['s3']['object']['key']
+
+        print('bucket_name::', bucket_name, ' - object_key::', object_key)
+
+        # Validate file extension
+        if any(object_key.lower().endswith(ext) for ext in SUPPORTED_IMAGE_EXTENSIONS):
+            # Process image files
+            response = textract_client.detect_document_text(
+                Document={'S3Object': {'Bucket': bucket_name, 'Name': object_key}}
+            )
+
+            print('Textract detect_document_text response::', response)
+
+            # Extract text blocks
+            # text_blocks = [block['Text'] for block in response['Blocks'] if block['BlockType'] == 'LINE']
+            # extracted_text = '\n'.join(text_blocks)
+            # print('extracted_text::', extracted_text)
+
+
+            # Extract text blocks with confidence scores
+            text_blocks_with_confidence = []
+            for block in response['Blocks']:
+                if block['BlockType'] == 'LINE':
+                    text_blocks_with_confidence.append((block['Text'], block['Confidence']))
+
+
+            # Format the extracted text with confidence scores
+            extracted_text_with_confidence = '\n'.join([f"{text} (Confidence: {confidence:.2f})" for text, confidence in text_blocks_with_confidence])
+            print('extracted_text_with_confidence::', extracted_text_with_confidence)
+
+
+            # Send extracted text to SNS
+            sns_client.publish(
+                TopicArn=SNS_TOPIC_ARN,
+                Message=json.dumps({
+                    'bucket': bucket_name,
+                    'key': object_key,
+                    'text': extracted_text_with_confidence
+                }),
+                Subject='Textract Extracted Text from Image'
+            )
+
+        elif object_key.lower().endswith(SUPPORTED_PDF_EXTENSION):
+            # Process PDF files
+            try:
+                response = textract_client.analyze_document(
+                    Document={'S3Object': {'Bucket': bucket_name, 'Name': object_key}},
+                    FeatureTypes=['QUERIES'],
+                    QueriesConfig={
+                        'Queries': [
+                            {'Text': 'What is the event name?', 'Alias': 'EventName'},
+                            {'Text': 'What is the Location?', 'Alias': 'Location'}
+                        ]
+                    }
+                )
+
+                print('Textract analyze_document response::', response)
+
+                # # Extract key-value pairs from QUERY_RESULT blocks
+                # query_results = {
+                #     block['QueryResult']['Alias']: block['Text']
+                #     for block in response['Blocks']
+                #     if block['BlockType'] == 'QUERY_RESULT'
+                # }
+
+
+                query_results = {}
+                for block in response['Blocks']:
+                    if block['BlockType'] == 'QUERY_RESULT':
+                        for relationship in block['Relationships']:
+                            if relationship['Type'] == 'ANSWER':
+                                query_id = relationship['Ids'][0]
+                                for query_block in response['Blocks']:
+                                    if query_block['Id'] == query_id:
+                                        query_alias = query_block['Query']['Alias']
+                                        query_results[query_alias] = {
+                                            'Text': block['Text'],
+                                            'Confidence': block['Confidence']
+                                        }
+
+                print('query_results::', query_results)
+
+
+                # Send key-value pairs to SNS
+                sns_client.publish(
+                    TopicArn=SNS_TOPIC_ARN,
+                    Message=json.dumps({
+                        'bucket': bucket_name,
+                        'key': object_key,
+                        'key_value_pairs': query_results
+                    }),
+                    Subject='Textract Extracted Key-Value Pairs from PDF'
+                )
+
+            except Exception as e:
+                print(f"Error processing file: {e}")
+
+        else:
+            raise ValueError(f"Unsupported file extension for file: {object_key}")
+
+        return {
+            'statusCode': 200,
+            'body': json.dumps('File processed successfully and data sent to SNS')
+        }
+
+    except textract_client.exceptions.UnsupportedDocumentException as e:
+        print(f"Unsupported document format: {e}")
+        return {
+            'statusCode': 400,
+            'body': json.dumps('Unsupported document format')
+        }
+    except Exception as e:
+        print(f"Error processing file: {e}")
+        return {
+            'statusCode': 500,
+            'body': json.dumps(f"Error processing file: {str(e)}")
+        }