Nested App Stats Collection and Publications

- Docker-Compose 'app' operation is through Patch-Envelope mechanism and
  is not known to the pillar side
- in zedrouter/appcontainer.go to extend the deployment type of 'Docker'
  for query the list of 'Docker-Compose' apps and their stats
- periodically querying the runtime agent of Docker-Compose
- publish the NestedAppDomainStatus, at least for 'newlogd' to comsume
- publish the AppContainerStats to be included in App stats upload

Signed-off-by: Naiming Shen <[email protected]>

Author: naiming-zededa

pkg/pillar/cmd/domainmgr/domainmgr.go

@@ -1372,6 +1372,7 @@ func handleCreate(ctx *domainContext, key string, config *types.DomainConfig) {
 		VmConfig:       config.VmConfig,
 		Service:        config.Service,
 		NodeName:       ctx.nodeName,
+		DeploymentType: config.DeploymentType,
 	}
 
 	status.VmConfig.CPUs = make([]int, 0)

pkg/pillar/cmd/zedagent/parseconfig.go

@@ -716,7 +716,14 @@ func parseAppInstanceConfig(getconfigCtx *getconfigContext,
 		parseVolumeRefList(appInstance.VolumeRefConfigList, cfgApp.GetVolumeRefList(), appInstance.UUIDandVersion.UUID)
 
 		// fill in the collect stats IP address of the App
+		// Get the runtime deployment type of the App
 		appInstance.CollectStatsIPAddr = net.ParseIP(cfgApp.GetCollectStatsIPAddr())
+		switch cfgApp.GetRuntimeType() {
+		case zconfig.AppRuntimeType_APP_RUNTIME_TYPE_DOCKER:
+			appInstance.DeploymentType = types.AppRuntimeTypeDocker
+		default:
+			appInstance.DeploymentType = types.AppRuntimeTypeUnSpecified
+		}
 
 		// fill the app adapter config
 		parseAppNetworkConfig(&appInstance, cfgApp, config.Networks,

pkg/pillar/cmd/zedmanager/handledomainmgr.go

@@ -64,6 +64,8 @@ func MaybeAddDomainConfig(ctx *zedmanagerContext,
 		// This isDNiDnode will be set to true even if the App is not in cluster mode,
 		// This will be set in zedagent parseConfig for the case of single node/device App case.
 		IsDNidNode: aiConfig.IsDesignatedNodeID,
+		// DeploymentType is set to the value of the DeploymentType of the AppInstanceConfig
+		DeploymentType: aiConfig.DeploymentType,
 	}
 
 	dc.DiskConfigList = make([]types.DiskConfig, 0, len(aiStatus.VolumeRefStatusList))

pkg/pillar/cmd/zedmanager/handlezedrouter.go

@@ -75,6 +75,7 @@ func MaybeAddAppNetworkConfig(ctx *zedmanagerContext,
 			CloudInitUserData: aiConfig.CloudInitUserData,
 			CipherBlockStatus: aiConfig.CipherBlockStatus,
 			MetaDataType:      aiConfig.MetaDataType,
+			DeploymentType:    aiConfig.DeploymentType, // can not be dynamically changed
 		}
 		nc.AppNetAdapterList = make([]types.AppNetAdapterConfig,
 			len(aiConfig.AppNetAdapterList))

pkg/pillar/cmd/zedrouter/appcontainer.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2020 Zededa, Inc.
+// Copyright (c) 2020-2025 Zededa, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
 package zedrouter
@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"strconv"
@@ -16,10 +17,14 @@ import (
 	apitypes "github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/google/go-cmp/cmp"
+	nestedapp "github.com/lf-edge/eve-tools/runtimemetrics/go/nestedappinstancemetrics"
 	"github.com/lf-edge/eve/pkg/pillar/agentlog"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	"github.com/lf-edge/eve/pkg/pillar/utils"
+	uuid "github.com/satori/go.uuid"
 	"github.com/sirupsen/logrus"
+	"google.golang.org/protobuf/encoding/protojson"
 )
 
 // dockerAPIPort - unencrypted docker socket for remote password-less access
@@ -28,6 +33,19 @@ const dockerAPIPort int = 2375
 // dockerAPIVersion - docker API version used
 const dockerAPIVersion string = "1.40"
 
+const (
+	// See detail in https://github.com/lf-edge/eve-tools/blob/master/runtimemetrics/README.md
+
+	// nestedAppDomainAppPort - TCP port for nested domain app stats for runtime to provide app list and metrics
+	// this is used in the http://<runtime-ip>:57475, where the <runtime-ip> is the 'GetStatsIPAddr' of the AppNetworkStatus
+	// and the deployment type is 'Docker'
+	nestedAppDomainAppPort int = 57475
+	// nestedAppDomainAppListURL - URL to get nested domain app list
+	nestedAppDomainAppListURL = "/api/v1/inventory/nested-app-id"
+	// nestedAppDomainAppMetricsURL - URL to get nested domain app metrics with nested-app uuid
+	nestedAppDomainAppMetricsURL = "/api/v1/metrics/nested-app-id/"
+)
+
 // check if we need to launch the goroutine to collect App container stats
 func (z *zedrouter) checkAppContainerStatsCollecting(config *types.AppNetworkConfig,
 	status *types.AppNetworkStatus) {
@@ -36,11 +54,13 @@ func (z *zedrouter) checkAppContainerStatsCollecting(config *types.AppNetworkCon
 	if config != nil {
 		if !status.GetStatsIPAddr.Equal(config.GetStatsIPAddr) {
 			status.GetStatsIPAddr = config.GetStatsIPAddr
+			status.DeploymentType = config.DeploymentType
 			changed = true
 		}
 	} else {
 		if status.GetStatsIPAddr != nil {
 			status.GetStatsIPAddr = nil
+			status.DeploymentType = types.AppRuntimeTypeUnSpecified
 			changed = true
 		}
 	}
@@ -72,28 +92,16 @@ func (z *zedrouter) collectAppContainerStats() {
 			collectTime := time.Now() // all apps collection assign the same timestamp
 			for _, st := range items {
 				status := st.(types.AppNetworkStatus)
+				// When the GetStatsIPAddr is configured, we need to handle collecting stats
+				// for various deployment types, defined by the DeploymentType. At this moment,
+				// we have two types of deployment for stats collection: Docker-Compose and IoT-Edge.
 				if status.GetStatsIPAddr != nil {
-					// get a list of containers and client handle
-					cli, containers, err := z.getAppContainers(status)
-					if err != nil {
-						z.log.Errorf(
-							"collectAppContainerStats: can't get App Containers %s on %s, %v",
-							status.UUIDandVersion.UUID.String(), status.GetStatsIPAddr.String(),
-							err)
-						continue
-					}
-					acNum += len(containers)
-
-					// collect container stats, and publish to zedclient
-					acMetrics := z.getAppContainerStats(cli, containers)
-					if len(acMetrics.StatsList) > 0 {
-						acMetrics.UUIDandVersion = status.UUIDandVersion
-						acMetrics.CollectTime = collectTime
-						z.pubAppContainerStats.Publish(acMetrics.Key(), acMetrics)
+					switch status.DeploymentType {
+					case types.AppRuntimeTypeDocker:
+						z.getNestedDomainAppMetrics(status, &acNum)
+					default:
+						z.getIotEdgeMetricsAndLogs(status, collectTime, lastLogTime, &acNum, &numlogs)
 					}
-
-					// collect container logs and send through the logging system
-					numlogs += z.getAppContainerLogs(status, lastLogTime, cli, containers)
 				}
 			}
 			// log output every 5 min, see this goroutine running status and number
@@ -328,3 +336,185 @@ func (z *zedrouter) getAppContainers(status types.AppNetworkStatus) (
 
 	return cli, containers, nil
 }
+
+// getIotEdgeMetricsAndLogs collects the metrics and logs for IoT-Edge
+func (z *zedrouter) getIotEdgeMetricsAndLogs(status types.AppNetworkStatus,
+	collectTime time.Time, lastLogTime map[string]time.Time, acNum, numlogs *int) {
+	// get a list of containers and client handle
+	cli, containers, err := z.getAppContainers(status)
+	if err != nil {
+		z.log.Errorf(
+			"getIotEdgeMetricsAndLogs: can't get App Containers %s on %s, %v",
+			status.UUIDandVersion.UUID.String(), status.GetStatsIPAddr.String(),
+			err)
+		return
+	}
+	*acNum += len(containers)
+
+	// collect container stats, and publish to zedclient
+	acMetrics := z.getAppContainerStats(cli, containers)
+	if len(acMetrics.StatsList) > 0 {
+		acMetrics.UUIDandVersion = status.UUIDandVersion
+		acMetrics.CollectTime = collectTime
+		z.pubAppContainerStats.Publish(acMetrics.Key(), acMetrics)
+	}
+
+	// collect container logs and send through the logging system
+	*numlogs += z.getAppContainerLogs(status, lastLogTime, cli, containers)
+}
+
+// Helper function to construct the URL for nested app operations
+func buildNestedAppURL(status types.AppNetworkStatus, endpoint string, appID string) string {
+	baseURL := fmt.Sprintf("http://%s:%d%s", status.GetStatsIPAddr.String(), nestedAppDomainAppPort, endpoint)
+	if appID != "" {
+		return baseURL + appID
+	}
+	return baseURL
+}
+
+// getNestedDomainAppMetrics collects the metrics for nested domain apps
+// this does several tasks:
+// - http request to runtime agent to get the list of nested domain apps
+// - publish the nested domain apps, currently it can be used by 'newlogd'
+// - http request to runtime agent to get the metrics for each nested domain app
+// - publish the metrics to zedagent w/ types.AppContainerStats
+func (z *zedrouter) getNestedDomainAppMetrics(status types.AppNetworkStatus, acNum *int) {
+	// first get the list of nested domain apps
+	nestedApps, err := z.getNestedDomainAppList(status)
+	if err != nil {
+		z.log.Errorf("getNestedDomainAppMetrics: failed to get nested app list, error: %v", err)
+		return
+	}
+
+	*acNum += len(nestedApps)
+	var acMetrics types.AppContainerMetrics
+	acMetrics.UUIDandVersion = status.UUIDandVersion
+	acMetrics.CollectTime = time.Now()
+
+	// for each nested domain app, get the metrics
+	// this list of nested app metrics is published to zedclient
+	// and to be uploaded to the controller along with the runtime or parent app metrics
+	for _, nestedApp := range nestedApps {
+		url := buildNestedAppURL(status, nestedAppDomainAppMetricsURL, nestedApp.UUIDandVersion.UUID.String())
+
+		data, err := fetchHTTPData(url)
+		if err != nil {
+			z.log.Errorf("getNestedDomainAppMetrics: %v", err)
+			continue
+		}
+
+		var nastat nestedapp.NestedAppMetrics
+		if err := protojson.Unmarshal(data, &nastat); err != nil {
+			z.log.Errorf("getNestedDomainAppMetrics: failed to decode JSON data, error: %v", err)
+			continue
+		}
+
+		acStats := types.AppContainerStats{
+			ContainerName:  nastat.Id,
+			Status:         nastat.Status,
+			Pids:           nastat.Pids,
+			Uptime:         nastat.Uptime,
+			CPUTotal:       nastat.CPUTotal,
+			SystemCPUTotal: nastat.SystemCPUTotal,
+			UsedMem:        nastat.UsedMem,
+			AllocatedMem:   nastat.AllocatedMem,
+			TxBytes:        nastat.TxBytes,
+			RxBytes:        nastat.RxBytes,
+			ReadBytes:      nastat.ReadBytes,
+			WriteBytes:     nastat.WriteBytes,
+		}
+		acMetrics.StatsList = append(acMetrics.StatsList, acStats)
+	}
+	// send for zedagent to pack w/ parent app metrics
+	z.pubAppContainerStats.Publish(acMetrics.Key(), acMetrics)
+
+	z.log.Functionf("getNestedDomainAppMetrics: collected metrics %+v", acMetrics)
+}
+
+// getNestedDomainAppList gets the list of nested domain apps
+func (z *zedrouter) getNestedDomainAppList(status types.AppNetworkStatus) ([]types.NestedAppDomainStatus, error) {
+	pub := z.pubNestedAppDomainStatus
+	existingItems := pub.GetAll()
+	existingNestedApps := make(map[string]types.NestedAppDomainStatus)
+
+	// Save existing items for later comparison
+	for _, st := range existingItems {
+		nestedApp := st.(types.NestedAppDomainStatus)
+		existingNestedApps[nestedApp.UUIDandVersion.UUID.String()] = nestedApp
+	}
+
+	// Get the JSON data from the Runtime endpoint
+	url := buildNestedAppURL(status, nestedAppDomainAppListURL, "")
+
+	data, err := fetchHTTPData(url)
+	if err != nil {
+		z.log.Errorf("getNestedDomainAppMetrics: %v", err)
+		return nil, err
+	}
+
+	var nestedAppInventory nestedapp.NestedAppInventory
+	// Decode the JSON data into the protobuf struct
+	if err := protojson.Unmarshal(data, &nestedAppInventory); err != nil {
+		z.log.Errorf("getNestedAppListAndMetrics: failed to decode JSON data using protojson, error: %v", err)
+		return nil, err
+	}
+
+	// Process the nested app IDs
+	var nestedapps []types.NestedAppDomainStatus
+	newNestedApps := make(map[string]types.NestedAppDomainStatus)
+	for _, nestedAppID := range nestedAppInventory.Apps {
+		nestedAppUUID, err := uuid.FromString(nestedAppID.AppId)
+		if err != nil {
+			z.log.Errorf("getNestedAppListAndMetrics: invalid UUID %s, error: %v", nestedAppID.AppId, err)
+			continue
+		}
+
+		nestedApp := types.NestedAppDomainStatus{
+			UUIDandVersion: types.UUIDandVersion{UUID: nestedAppUUID},
+			DisplayName:    nestedAppID.AppName,
+			DisableLogs:    nestedAppID.DisableLogs,
+			ParentAppUUID:  status.UUIDandVersion.UUID,
+		}
+
+		newNestedApps[nestedAppID.AppId] = nestedApp
+		nestedapps = append(nestedapps, nestedApp)
+	}
+
+	// Compare old and new sets of nested apps and publish if different
+	for uuidStr, newNestedApp := range newNestedApps {
+		if existingNestedApp, exists := existingNestedApps[uuidStr]; !exists || !cmp.Equal(existingNestedApp, newNestedApp) {
+			z.log.Functionf("getNestedAppListAndMetrics: publish nestedApp %+v", newNestedApp)
+			z.pubNestedAppDomainStatus.Publish(newNestedApp.Key(), newNestedApp)
+		}
+	}
+
+	// handle removed nested apps
+	for uuidStr := range existingNestedApps {
+		if _, exists := newNestedApps[uuidStr]; !exists {
+			z.log.Functionf("getNestedAppListAndMetrics: remove nestedApp with UUID %s", uuidStr)
+			z.pubNestedAppDomainStatus.Unpublish(uuidStr)
+		}
+	}
+
+	return nestedapps, nil
+}
+
+// fetchHTTPData fetches data from the given URL
+func fetchHTTPData(url string) ([]byte, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch data from %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, url)
+	}
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body from %s: %w", url, err)
+	}
+
+	return data, nil
+}

pkg/pillar/cmd/zedrouter/pubsubhandlers.go

@@ -660,6 +660,11 @@ func (z *zedrouter) handleAppNetworkDelete(ctxArg interface{}, key string,
 	// Write out what we modified to AppNetworkStatus aka delete
 	z.unpublishAppNetworkStatus(status)
 
+	// Unpublish AppContainerStats
+	if config.GetStatsIPAddr != nil {
+		z.pubAppContainerStats.Unpublish(status.Key())
+	}
+
 	// Free all numbers allocated for this app network.
 	appNumKey := types.UuidToNumKey{UUID: status.UUIDandVersion.UUID}
 	err := z.appNumAllocator.Free(appNumKey, false)

pkg/pillar/cmd/zedrouter/zedrouter.go

@@ -154,6 +154,9 @@ type zedrouter struct {
 	// Kubernetes networking
 	withKubeNetworking bool
 	cniRequests        chan *rpcRequest
+
+	// publist nested App Status
+	pubNestedAppDomainStatus pubsub.Publication
 }
 
 // AddAgentSpecificCLIFlags adds CLI options
@@ -578,6 +581,15 @@ func (z *zedrouter) initPublications() (err error) {
 		return err
 	}
 
+	z.pubNestedAppDomainStatus, err = z.pubSub.NewPublication(
+		pubsub.PublicationOptions{
+			AgentName: agentName,
+			TopicType: types.NestedAppDomainStatus{},
+		})
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 

pkg/pillar/go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/lf-edge/edge-containers v0.0.0-20240207093504-5dfda0619b80
 	github.com/lf-edge/eve-api/go v0.0.0-20250310225738-c77ab6f8c73a
 	github.com/lf-edge/eve-libs v0.0.0-20250313200311-28f858e8e99b
+	github.com/lf-edge/eve-tools/runtimemetrics/go v0.0.0-20250320220227-713ea9d6c6d2
 	github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d
 	github.com/lf-edge/go-qemu v0.0.0-20231121152149-4c467eda0c56
 	github.com/linuxkit/linuxkit/src/cmd/linuxkit v0.0.0-20240507172735-6d37353ca1ee

pkg/pillar/go.sum

@@ -1461,6 +1461,8 @@ github.com/lf-edge/eve-api/go v0.0.0-20250310225738-c77ab6f8c73a h1:AI1EbTLgXv+v
 github.com/lf-edge/eve-api/go v0.0.0-20250310225738-c77ab6f8c73a/go.mod h1:ot6MhAhBXapUDl/hXklaX4kY88T3uC4PTg0D2wD8DzA=
 github.com/lf-edge/eve-libs v0.0.0-20250313200311-28f858e8e99b h1:sprMvkZ9Ogls4WDo2rgtRwRFPpdykjYtUqntzwy3NKI=
 github.com/lf-edge/eve-libs v0.0.0-20250313200311-28f858e8e99b/go.mod h1:We4FeQWOAxUiB8ZH+eIfRdn2QG9Hfr1W5/IoufU1t64=
+github.com/lf-edge/eve-tools/runtimemetrics/go v0.0.0-20250320220227-713ea9d6c6d2 h1:rCZhnv3q+g8bcNUHp9wf0NlIB7k4ORckSnoZdHGcMtU=
+github.com/lf-edge/eve-tools/runtimemetrics/go v0.0.0-20250320220227-713ea9d6c6d2/go.mod h1:l5jh2deQzJnZ7gW4F7dvzJcLRlkYBgFjcHs+eBnBUxo=
 github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d h1:tUBb9M6u42LXwHAYHyh22wJeUUQlTpDkXwRXalpRqbo=
 github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d/go.mod h1:Nn3juMJJ1G8dyHOebdZyS4jOB/fuxAd5fIajBaWjHr8=
 github.com/lf-edge/go-qemu v0.0.0-20231121152149-4c467eda0c56 h1:LmFp0jbNSwPLuxJA+nQ+mMQrQ53ESkvHP4CVMqR0zrY=

pkg/pillar/types/domainmgrtypes.go

@@ -66,6 +66,8 @@ type DomainConfig struct {
 	// OemWindowsLicenseKeyInfo provides the information required to propagate the OEM license key
 	// to the VM.
 	OemWindowsLicenseKeyInfo OemWindowsLicenseKeyInfo
+
+	DeploymentType AppRuntimeType
 }
 
 // MetaDataType of metadata service for app
@@ -340,6 +342,8 @@ type DomainStatus struct {
 	// PassthroughWindowsLicenseKey is true if eveything it available to propagate
 	// the OEM license key to the VM.
 	PassthroughWindowsLicenseKey bool
+	// DeploymentType is the type of deployment for the app
+	DeploymentType AppRuntimeType
 }
 
 func (status DomainStatus) Key() string {