Merge pull request #1082 from solarwinds/generateKubernetesServiceRoutesToRelations

Generate `KubernetesServiceRoutesTo` relations

Author: gantrior

deploy/helm/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: swo-k8s-collector
-version: 4.8.0-alpha.4
-appVersion: 0.127.5
+version: 4.8.0-alpha.5
+appVersion: 0.127.7
 description: SolarWinds Kubernetes Integration
 keywords:
   - monitoring

deploy/helm/events-collector-config.yaml

@@ -18,6 +18,8 @@
     "clusterroles" "ClusterRole"
     "clusterrolebindings" "ClusterRoleBinding"
     "serviceaccounts" "ServiceAccount"
+    "endpoints" "Endpoints"
+    "endpointslices" "EndpointSlice"
 
 }}
 {{- $arrayOfWatchedResources := keys $resourceMap | sortAlpha }}
@@ -41,6 +43,7 @@ exporters:
       storage: file_storage/sending_queue
 {{- end }}
     timeout: {{ .Values.otel.events.timeout }}
+
 extensions:
 {{- if .Values.otel.events.sending_queue.offload_to_disk }}
   file_storage/sending_queue:
@@ -128,12 +131,83 @@ connectors:
             - sw.k8s.cluster.uid
             - k8s.namespace.name
             - k8s.service.name
-        - entity: KubernetesStatefulSet 
+        - entity: KubernetesStatefulSet
           id:
             - sw.k8s.cluster.uid
             - k8s.namespace.name
             - k8s.statefulset.name
 {{- end }}
+  solarwindsentity/serviceendpointsmapping:
+    source_prefix: "source."
+    destination_prefix: "dest."
+    schema:
+      entities:
+        - entity: KubernetesService
+          id:
+            - sw.k8s.cluster.uid
+            - k8s.namespace.name
+            - k8s.service.name
+        - entity: KubernetesDaemonSet
+          id:
+            - sw.k8s.cluster.uid
+            - k8s.namespace.name
+            - k8s.daemonset.name
+        - entity: KubernetesDeployment
+          id:
+            - sw.k8s.cluster.uid
+            - k8s.namespace.name
+            - k8s.deployment.name
+        - entity: KubernetesStatefulSet
+          id:
+            - sw.k8s.cluster.uid
+            - k8s.namespace.name
+            - k8s.statefulset.name
+        - entity: KubernetesReplicaSet
+          id:
+            - sw.k8s.cluster.uid
+            - k8s.namespace.name
+            - k8s.replicaset.name
+      events:
+        relationships:
+          - type: KubernetesServiceRoutesTo
+            source_entity: KubernetesService
+            destination_entity: KubernetesDaemonSet
+            conditions: []
+            context: "log"
+            attributes: []
+            action: "update"
+          - type: KubernetesServiceRoutesTo
+            source_entity: KubernetesService
+            destination_entity: KubernetesDeployment
+            conditions: []
+            context: "log"
+            attributes: []
+            action: "update"
+          - type: KubernetesServiceRoutesTo
+            source_entity: KubernetesService
+            destination_entity: KubernetesStatefulSet
+            conditions: []
+            context: "log"
+            attributes: []
+            action: "update"
+          - type: KubernetesServiceRoutesTo
+            source_entity: KubernetesService
+            destination_entity: KubernetesReplicaSet
+            conditions: []
+            context: "log"
+            attributes: []
+            action: "update"
+
+  routing/manifests:
+    default_pipelines: [logs/manifests-export]
+    table:
+      - context: resource
+        pipelines:
+          - logs/serviceendpointsmapping
+        condition: |
+          attributes["sw.k8s.log.type"] == "serviceendpointsmapping"
+
+  forward/exportserviceendpointsmapping: {}
 
 processors:
   k8seventgeneration:
@@ -277,8 +351,12 @@ processors:
 {{- range $index, $resourceName := $arrayOfWatchedResources }}
       {{- $resourceKind := get $resourceMap $resourceName }}
       {{- $prefix := lower $resourceKind }}
-            - set(log.attributes["k8s.{{$prefix}}.name"], log.body["metadata"]["name"]) where log.body["kind"] == "{{$resourceKind}}"
+            - set(log.attributes["k8s.{{$prefix}}.name"], log.body["metadata"]["name"]) where log.body["metadata"]["name"] != nil and log.body["kind"] == "{{$resourceKind}}"
 {{- end }}
+            # rename k8s.endpoints.name to k8s.endpoint.name if exists
+            - set(log.attributes["k8s.endpoint.name"], log.attributes["k8s.endpoints.name"]) where log.attributes["k8s.endpoints.name"] != nil
+            - delete_key(log.attributes, "k8s.endpoints.name") where log.attributes["k8s.endpoints.name"] != nil
+
             - set(log.attributes["k8s.namespace.name"], log.body["metadata"]["namespace"])
             - set(log.attributes["k8s.ingress.name"], log.body["metadata"]["name"]) where log.body["kind"] == "Ingress" or log.body["kind"] == "VirtualService"
             - set(log.attributes["sw.k8s.ingress.type"], log.body["kind"]) where log.body["kind"] == "Ingress" or log.body["kind"] == "VirtualService"
@@ -289,6 +367,33 @@ processors:
             - delete_key(log.attributes, "k8s.resource.name")
             - delete_key(log.attributes, "event.name")
             - delete_key(log.attributes, "event.domain")
+
+  groupbyattrs/serviceendpointsmapping:
+    keys:
+      - sw.k8s.cluster.uid
+      - sw.k8s.workload.ip
+      - k8s.service.name
+      - k8s.namespace.name
+
+  transform/serviceendpointsmapping-renamepodip:
+    error_mode: ignore
+    log_statements:
+      - statements:
+          - set(resource.attributes["k8s.pod.ip"], resource.attributes["sw.k8s.workload.ip"]) where resource.attributes["sw.k8s.log.type"] == "serviceendpointsmapping" and resource.attributes["sw.k8s.workload.ip"] != nil
+
+  transform/serviceendpointsmapping:
+    error_mode: ignore
+    log_statements:
+      - statements:
+          - set(resource.attributes["source.k8s.service.name"], resource.attributes["k8s.service.name"]) where resource.attributes["k8s.service.name"] != nil
+          - set(resource.attributes["source.k8s.namespace.name"], resource.attributes["k8s.namespace.name"]) where resource.attributes["k8s.namespace.name"] != nil
+          
+          - set(resource.attributes["dest.k8s.namespace.name"], resource.attributes["k8s.namespace.name"]) where resource.attributes["k8s.namespace.name"] != nil
+          - set(resource.attributes["dest.k8s.deployment.name"], resource.attributes["k8s.deployment.name"]) where resource.attributes["k8s.deployment.name"] != nil
+          - set(resource.attributes["dest.k8s.statefulset.name"], resource.attributes["k8s.statefulset.name"]) where resource.attributes["k8s.statefulset.name"] != nil
+          - set(resource.attributes["dest.k8s.replicaset.name"], resource.attributes["k8s.replicaset.name"]) where resource.attributes["k8s.replicaset.name"] != nil
+          - set(resource.attributes["dest.k8s.daemonset.name"], resource.attributes["k8s.daemonset.name"]) where resource.attributes["k8s.daemonset.name"] != nil
+
   transform/scope:
     log_statements:
       - statements:
@@ -316,6 +421,8 @@ processors:
       - k8s.clusterrole.name
       - k8s.clusterrolebinding.name
       - k8s.serviceaccount.name
+      - k8s.endpoint.name
+      - k8s.endpointslice.name
 
   resource/events:
     attributes:
@@ -431,6 +538,18 @@ processors:
   k8sattributes:
 {{ include "common.k8s-instrumentation" . | indent 4 }}
 
+  logdedup/entitystateevents: 
+    interval: {{ quote .Values.otel.manifests.pull_every }}
+    conditions: 
+      - instrumentation_scope.attributes["otel.entity.event_as_log"] == true
+    include_fields:
+      - attributes.otel.entity.event.type
+      - attributes.otel.entity_relationship.source_entity.id
+      - attributes.otel.entity_relationship.destination_entity.id
+      - attributes.otel.entity_relationship.type
+      - attributes.otel.entity_relationship.source_entity.type
+      - attributes.otel.entity_relationship.destination_entity.type
+
 receivers:
   k8s_events:
 {{- $root := . }}
@@ -517,6 +636,7 @@ service:
       exporters:
         - otlp
 {{- end }}
+
     logs/manifests:
       receivers:
         - swok8sobjects
@@ -525,26 +645,47 @@ service:
         - transform/manifest
         - groupbyattrs/manifest
         - transform/set_labels_and_annotations_for_entities
+{{- if not (and .Values.otel.events.enabled .Values.otel.manifests.enabled) }}
+        - filter/k8s_collector_config_include
+{{- end }}
         - transform/extract_resource_attrs_from_manifests
         - transform/stringify_body
         - resource/manifest
         - resourcedetection/providers
-        - k8sattributes
         - k8seventgeneration
-{{- if not (and .Values.otel.events.enabled .Values.otel.manifests.enabled) }}
-        - filter/k8s_collector_config_include
-{{- end }}
+        - groupbyattrs/serviceendpointsmapping
+        - transform/serviceendpointsmapping-renamepodip
+        - k8sattributes
 {{- if eq (include "isNamespacesFilterEnabled" .) "true" }}
         - filter/namespaces
-{{- end }}  
+{{- end }}
 {{- if .Values.otel.manifests.filter }}
         - filter/manifests
 {{- end }}
+      exporters:
+        - routing/manifests
+
+    logs/serviceendpointsmapping:
+      receivers:
+        - routing/manifests
+      processors:
+        - memory_limiter
+        - transform/serviceendpointsmapping
+      exporters:
+        - solarwindsentity/serviceendpointsmapping
+
+    logs/manifests-export:
+      receivers:
+        - routing/manifests
+        - solarwindsentity/serviceendpointsmapping
+      processors:
+        - memory_limiter
+        - logdedup/entitystateevents
         - transform/scope
         - batch
       exporters:
         - otlp
-    
+
 {{- if and .Values.otel.manifests.enabled .Values.otel.manifests.keepalive_events.enabled }}
     logs/manifests-keepalive:
       receivers:
@@ -553,15 +694,15 @@ service:
         - memory_limiter
         - transform/manifest
         - groupbyattrs/manifest
-        - resource/manifest
-        - k8sattributes
-        - k8seventgeneration
 {{- if not (and .Values.otel.events.enabled .Values.otel.manifests.enabled) }}
         - filter/k8s_collector_config_include
 {{- end }}
+        - resource/manifest
+        - k8sattributes
+        - k8seventgeneration
 {{- if eq (include "isNamespacesFilterEnabled" .) "true" }}
         - filter/namespaces
-{{- end }}  
+{{- end }}
 {{- if .Values.otel.manifests.filter }}
         - filter/manifests
 {{- end }}
@@ -578,7 +719,7 @@ service:
       exporters:
         - otlp
 {{- end }}
-        
+
   telemetry:
 {{- if .Values.otel.events.telemetry.logs.enabled }}
     logs:

deploy/helm/gateway-collector-config.yaml

@@ -539,7 +539,7 @@ connectors:
             - sw.server.address.fqdn
       events:
         entities:
-          - type: "PublicNetworkLocation"
+          - entity: "PublicNetworkLocation"
             context: "metric"
             action: "update"
 

deploy/helm/templates/_common-discovery-config.tpl

@@ -460,7 +460,7 @@ solarwindsentity/istio-workload-service:
           - sw.server.address.fqdn
     events:
       entities:
-        - type: "PublicNetworkLocation"
+        - entity: "PublicNetworkLocation"
           context: "metric"
           action: "update"
       relationships:

deploy/helm/templates/_helpers.tpl

@@ -137,6 +137,7 @@ auth_type: "serviceAccount"
 passthrough: false
 extract:
   metadata:
+    - k8s.pod.name
     - k8s.deployment.name
     - k8s.replicaset.name
     - k8s.daemonset.name
@@ -150,6 +151,11 @@ pod_association:
         name: k8s.pod.name
       - from: resource_attribute
         name: k8s.namespace.name
+  - sources:
+      - from: resource_attribute
+        name: k8s.pod.ip
+      - from: resource_attribute
+        name: k8s.namespace.name
 {{- end -}}
 
 {{/*

deploy/helm/templates/cluster-role.yaml

@@ -97,6 +97,14 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - "discovery.k8s.io"
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
   - nonResourceURLs:
       - /metrics
     verbs:

deploy/helm/tests/__snapshot__/discovery-collector_test.yaml.snap

@@ -82,7 +82,7 @@ Discovery collector spec should match snapshot when using default values:
               entities:
                 - action: update
                   context: metric
-                  type: PublicNetworkLocation
+                  entity: PublicNetworkLocation
               relationships:
                 - action: update
                   attributes: