Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

clean meta about deleted k8s pods #778

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

clean meta about deleted k8s pods #778

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
976c6f8
try to upgrade to 20241010143419
DmitryRomanov Feb 20, 2025
cfd565d
Revert "try to upgrade to 20241010143419"
DmitryRomanov Feb 20, 2025
58865ed
k8s.io/client-go v0.29.14
DmitryRomanov Feb 20, 2025
df13d20
meta about deleted pods expires too
DmitryRomanov Feb 20, 2025
2cd40c2
metric for metric on len k8s meta
DmitryRomanov Feb 24, 2025
7e623bf
Revert "metric for metric on len k8s meta"
DmitryRomanov Feb 25, 2025
984f5f3
deletedPodsCounter
DmitryRomanov Feb 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ require (
google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.29.0
k8s.io/apimachinery v0.29.0
k8s.io/client-go v0.29.0
k8s.io/api v0.29.14
k8s.io/apimachinery v0.29.14
k8s.io/client-go v0.29.14
sigs.k8s.io/yaml v1.3.0
)

Expand Down Expand Up @@ -81,7 +81,7 @@ require (
github.com/go-openapi/swag v0.22.3 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
Expand Down
20 changes: 8 additions & 12 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,15 +117,13 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
Expand Down Expand Up @@ -565,8 +563,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002 h1:V7Da7qt0MkY3noVANIMVBk28nOnijADeOR3i5Hcvpj4=
google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand All @@ -592,12 +588,12 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A=
k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA=
k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o=
k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis=
k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8=
k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38=
k8s.io/api v0.29.14 h1:JWFh5ufowH3Y6tCgEzY3URVJHb27f0tEDEej0nCjWDw=
k8s.io/api v0.29.14/go.mod h1:IV8YqKxMm8JGLBLlHM13Npn5lCITH10XYipWEW+YEOQ=
k8s.io/apimachinery v0.29.14 h1:IDhwnGNCp836SLOwW1SoEfFNV77wxIklhxeAHX9vmSo=
k8s.io/apimachinery v0.29.14/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y=
k8s.io/client-go v0.29.14 h1:OSnzZ9DClaFRgl3zMAY2kGZhNjdGJkEb+RDz+MW2h6k=
k8s.io/client-go v0.29.14/go.mod h1:XtZt5n5UxKfPJ+sCoTPcEavWgZbLFFxMnAFFRQGK1RY=
k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
Expand Down
17 changes: 11 additions & 6 deletions plugin/input/k8s/meta/gatherer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ var (
DisableMetaUpdates = false
metaAddedCounter atomic.Int64
expiredItemsCounter atomic.Int64
deletedPodsCounter atomic.Int64

CriType = "docker"
NodeLabels = make(map[string]string)
Expand Down Expand Up @@ -163,6 +164,7 @@ func initInformer() {
pod := obj.(*corev1.Pod)
PutMeta(pod)
deletedPodsCache.Add(PodName(pod.Name), true)
deletedPodsCounter.Inc()
},
}, cache.Indexers{})
controller = c
Expand All @@ -189,11 +191,19 @@ func removeExpired() {
cleanUpItems(expiredItems)

if MaintenanceInterval > time.Second {
localLogger.Infof("k8s meta stat for last %d seconds: total=%d, updated=%d, expired=%d", MaintenanceInterval/time.Second, getTotalItems(), metaAddedCounter.Load(), expiredItemsCounter.Load())
localLogger.Infof(
"k8s meta stat for last %d seconds: total=%d, updated=%d, expired=%d, deleted=%d",
MaintenanceInterval/time.Second,
getTotalItems(),
metaAddedCounter.Load(),
expiredItemsCounter.Load(),
deletedPodsCounter.Load(),
)
}

metaAddedCounter.Swap(0)
expiredItemsCounter.Swap(0)
deletedPodsCounter.Swap(0)
}

func maintenance() {
Expand Down Expand Up @@ -234,11 +244,6 @@ func getExpiredItems(out []*MetaItem) []*MetaItem {
// find pods which aren't in k8s pod list for some time and add them to the expiration list
for ns, podNames := range MetaData {
for pod, containerIDs := range podNames {
isDeleted := deletedPodsCache.Contains(pod)
if isDeleted {
// information about deleted pods will never change again
continue
}
for cid, podData := range containerIDs {
if now.Sub(podData.updateTime) > MetaExpireDuration {
out = append(out, &MetaItem{
Expand Down
2 changes: 1 addition & 1 deletion plugin/input/k8s/pipeline_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func TestPipeline(t *testing.T) {
k8sContainerID = strings.Clone(e.Root.Dig("k8s_container_id").AsString())
wg.Done()
})
p.Start()

item := &meta.MetaItem{
Namespace: "sre",
Expand All @@ -51,7 +52,6 @@ func TestPipeline(t *testing.T) {
meta.PutMeta(getPodInfo(item, true))
filename := getLogFilename(dir, item)

p.Start()
file, err := os.Create(filename)
if err != nil {
logger.Fatalf("Error creating file: %s", err.Error())
Expand Down