Implement agent introspection server in ecs-agent (#4482)

implement introspection server in shared ecs-agent

To make the Agent Introspection Server available to the Fargate agent,
the introspection server is now implemented in the shared ecs-agent
library, where it can be consumed by both agents.

ecs-agent/introspection/handlers.go

@@ -0,0 +1,69 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package introspection
+
+import (
+	"errors"
+	"net/http"
+
+	v1 "github.com/aws/amazon-ecs-agent/ecs-agent/introspection/v1"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/logger"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/logger/field"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/metrics"
+	tmdsutils "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/utils"
+)
+
+const (
+	requestTypeLicense = "introspection/license"
+	licensePath        = "/license"
+)
+
+// licenseHandler creates response for '/license' API.
+func licenseHandler(agentState v1.AgentState, metricsFactory metrics.EntryFactory) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		text, err := agentState.GetLicenseText()
+		if err != nil {
+			logger.Error("Failed to get v1 license.", logger.Fields{
+				field.Error: err,
+			})
+			metricsFactory.New(metrics.IntrospectionInternalServerError).Done(err)
+			tmdsutils.WriteStringToResponse(w, http.StatusInternalServerError, "", requestTypeLicense)
+		} else {
+			tmdsutils.WriteStringToResponse(w, http.StatusOK, text, requestTypeLicense)
+		}
+	}
+}
+
+// panicHandler handler will gracefully close the connection if a panic occurs and emit a metric.
+func panicHandler(next http.Handler, metricsFactory metrics.EntryFactory) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if r := recover(); r != nil {
+				var err error
+				switch x := r.(type) {
+				case string:
+					err = errors.New(x)
+				case error:
+					err = x
+				default:
+					err = errors.New("unknown panic")
+				}
+				w.Header().Set("Connection", "close")
+				w.WriteHeader(http.StatusInternalServerError)
+				metricsFactory.New(metrics.IntrospectionCrash).Done(err)
+			}
+		}()
+		next.ServeHTTP(w, r)
+	})
+}

ecs-agent/introspection/handlers_test.go

@@ -0,0 +1,103 @@
+//go:build unit
+// +build unit
+
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+package introspection
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	v1 "github.com/aws/amazon-ecs-agent/ecs-agent/introspection/v1"
+	mock_v1 "github.com/aws/amazon-ecs-agent/ecs-agent/introspection/v1/mocks"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/metrics"
+	mock_metrics "github.com/aws/amazon-ecs-agent/ecs-agent/metrics/mocks"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type IntrospectionResponse interface {
+	string |
+		*v1.AgentMetadataResponse |
+		*v1.TaskResponse |
+		*v1.TasksResponse
+}
+
+type IntrospectionTestCase[R IntrospectionResponse] struct {
+	Path          string
+	AgentResponse R
+	Err           error
+	MetricName    string
+}
+
+const (
+	licenseText       = "Licensed under the Apache License ..."
+	internalErrorText = "some internal error"
+)
+
+func testHandlerSetup[R IntrospectionResponse](t *testing.T, testCase IntrospectionTestCase[R]) (
+	*gomock.Controller, *mock_v1.MockAgentState, *mock_metrics.MockEntryFactory, *http.Request, *httptest.ResponseRecorder) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	agentState := mock_v1.NewMockAgentState(ctrl)
+	metricsFactory := mock_metrics.NewMockEntryFactory(ctrl)
+
+	req, err := http.NewRequest("GET", testCase.Path, nil)
+	require.NoError(t, err)
+	recorder := httptest.NewRecorder()
+
+	return ctrl, agentState, metricsFactory, req, recorder
+}
+
+func TestLicenseHandler(t *testing.T) {
+	var performMockRequest = func(t *testing.T, testCase IntrospectionTestCase[string]) *httptest.ResponseRecorder {
+		mockCtrl, mockAgentState, mockMetricsFactory, req, recorder := testHandlerSetup(t, testCase)
+		mockAgentState.EXPECT().
+			GetLicenseText().
+			Return(testCase.AgentResponse, testCase.Err)
+		if testCase.Err != nil {
+			mockEntry := mock_metrics.NewMockEntry(mockCtrl)
+			mockEntry.EXPECT().Done(testCase.Err)
+			mockMetricsFactory.EXPECT().
+				New(testCase.MetricName).Return(mockEntry)
+		}
+		licenseHandler(mockAgentState, mockMetricsFactory)(recorder, req)
+		return recorder
+	}
+
+	t.Run("happy case", func(t *testing.T) {
+		recorder := performMockRequest(t, IntrospectionTestCase[string]{
+			Path:          licensePath,
+			AgentResponse: licenseText,
+			Err:           nil,
+		})
+		assert.Equal(t, http.StatusOK, recorder.Code)
+		assert.Equal(t, licenseText, recorder.Body.String())
+	})
+
+	t.Run("internal error", func(t *testing.T) {
+		recorder := performMockRequest(t, IntrospectionTestCase[string]{
+			Path:          licensePath,
+			AgentResponse: "",
+			Err:           errors.New(internalErrorText),
+			MetricName:    metrics.IntrospectionInternalServerError,
+		})
+		assert.Equal(t, http.StatusInternalServerError, recorder.Code)
+		assert.Equal(t, "", recorder.Body.String())
+	})
+}

ecs-agent/introspection/server.go

@@ -0,0 +1,161 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package introspection
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/pprof"
+	"strconv"
+	"time"
+
+	v1 "github.com/aws/amazon-ecs-agent/ecs-agent/introspection/v1"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/introspection/v1/handlers"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/metrics"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/tmds/logging"
+)
+
+const (
+	Port = 51678
+	// With pprof we need to increase the timeout so that there is enough time to do the profiling. Since the profiling
+	// time window for CPU is configurable in the request, this timeout effectively means the CPU profiling will be
+	// capped to 5 min.
+	writeTimeoutForPprof = time.Minute * 5
+	pprofBasePath        = "/debug/pprof/"
+	pprofCMDLinePath     = pprofBasePath + "cmdline"
+	pprofProfilePath     = pprofBasePath + "profile"
+	pprofSymbolPath      = pprofBasePath + "symbol"
+	pprofTracePath       = pprofBasePath + "trace"
+)
+
+var (
+	// Injection points for testing
+	pprofIndexHandler   = pprof.Index
+	pprofCmdlineHandler = pprof.Cmdline
+	pprofProfileHandler = pprof.Profile
+	pprofSymbolHandler  = pprof.Symbol
+	pprofTraceHandler   = pprof.Trace
+)
+
+type rootResponse struct {
+	AvailableCommands []string
+}
+
+// Configuration for Introspection Server
+type Config struct {
+	readTimeout        time.Duration // http server read timeout
+	writeTimeout       time.Duration // http server write timeout
+	enableRuntimeStats bool          // enable profiling handlers
+}
+
+// Function type for updating Introspection Server config
+type ConfigOpt func(*Config)
+
+// Set if Introspection Server should accept profiling requests
+func WithRuntimeStats(enableRuntimeStats bool) ConfigOpt {
+	return func(c *Config) {
+		c.enableRuntimeStats = enableRuntimeStats
+	}
+}
+
+// Set Introspection Server read timeout
+func WithReadTimeout(readTimeout time.Duration) ConfigOpt {
+	return func(c *Config) {
+		c.readTimeout = readTimeout
+	}
+}
+
+// Set Introspection Server write timeout
+func WithWriteTimeout(writeTimeout time.Duration) ConfigOpt {
+	return func(c *Config) {
+		c.writeTimeout = writeTimeout
+	}
+}
+
+// Create a new HTTP Introspection Server
+func NewServer(agentState v1.AgentState, metricsFactory metrics.EntryFactory, options ...ConfigOpt) (*http.Server, error) {
+	config := new(Config)
+	for _, opt := range options {
+		opt(config)
+	}
+	return setup(agentState, metricsFactory, config)
+}
+
+func v1HandlersSetup(serverMux *http.ServeMux,
+	agentState v1.AgentState,
+	metricsFactory metrics.EntryFactory) {
+	serverMux.HandleFunc(handlers.V1AgentMetadataPath, handlers.AgentMetadataHandler(agentState, metricsFactory))
+	serverMux.HandleFunc(handlers.V1TasksMetadataPath, handlers.TasksMetadataHandler(agentState, metricsFactory))
+	serverMux.HandleFunc(licensePath, licenseHandler(agentState, metricsFactory))
+}
+
+func pprofHandlerSetup(serverMux *http.ServeMux) {
+	serverMux.HandleFunc(pprofBasePath, pprofIndexHandler)
+	serverMux.HandleFunc(pprofCMDLinePath, pprofCmdlineHandler)
+	serverMux.HandleFunc(pprofProfilePath, pprofProfileHandler)
+	serverMux.HandleFunc(pprofSymbolPath, pprofSymbolHandler)
+	serverMux.HandleFunc(pprofTracePath, pprofTraceHandler)
+}
+
+func setup(
+	agentState v1.AgentState,
+	metricsFactory metrics.EntryFactory,
+	config *Config,
+) (*http.Server, error) {
+	if agentState == nil {
+		return nil, errors.New("state cannot be nil")
+	}
+	if metricsFactory == nil {
+		return nil, errors.New("metrics factory cannot be nil")
+	}
+
+	paths := []string{handlers.V1AgentMetadataPath, handlers.V1TasksMetadataPath, licensePath}
+
+	if config.enableRuntimeStats {
+		paths = append(paths, pprofBasePath, pprofCMDLinePath, pprofProfilePath, pprofSymbolPath, pprofTracePath)
+	}
+
+	availableCommands := &rootResponse{paths}
+	// Autogenerated list of the above serverFunctions paths
+	availableCommandResponse, err := json.Marshal(&availableCommands)
+	if err != nil {
+		return nil, fmt.Errorf("error marshaling JSON in introspection server setup: %s", err)
+	}
+
+	defaultHandler := func(w http.ResponseWriter, r *http.Request) {
+		w.Write(availableCommandResponse)
+	}
+
+	serveMux := http.NewServeMux()
+	serveMux.HandleFunc("/", defaultHandler)
+
+	v1HandlersSetup(serveMux, agentState, metricsFactory)
+	wTimeout := config.writeTimeout
+	if config.enableRuntimeStats {
+		pprofHandlerSetup(serveMux)
+		wTimeout = writeTimeoutForPprof
+	}
+
+	loggingServeMux := http.NewServeMux()
+	loggingServeMux.Handle("/", logging.NewLoggingHandler(serveMux))
+
+	return &http.Server{
+		Addr:         ":" + strconv.Itoa(Port),
+		Handler:      panicHandler(loggingServeMux, metricsFactory),
+		ReadTimeout:  config.readTimeout,
+		WriteTimeout: wTimeout,
+	}, nil
+}