From f65b244de81942ce127f021c5996d04f511ed929 Mon Sep 17 00:00:00 2001 From: wwatson Date: Wed, 17 Apr 2024 13:42:57 -0500 Subject: [PATCH 1/8] Update pod_memory_hog_spec.cr --- spec/workload/resilience/pod_memory_hog_spec.cr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spec/workload/resilience/pod_memory_hog_spec.cr b/spec/workload/resilience/pod_memory_hog_spec.cr index 5c2622c3d..ee66e1305 100644 --- a/spec/workload/resilience/pod_memory_hog_spec.cr +++ b/spec/workload/resilience/pod_memory_hog_spec.cr @@ -14,7 +14,8 @@ describe "Resilience pod memory hog Chaos" do it "'pod_memory_hog' A 'Good' CNF should not crash when pod memory hog occurs", tags: ["pod_memory_hog"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` + install_log = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` + LOGGING.info = install_log $?.success?.should be_true response_s = `./cnf-testsuite pod_memory_hog verbose` LOGGING.info response_s @@ -27,4 +28,4 @@ describe "Resilience pod memory hog Chaos" do $?.success?.should be_true end end -end \ No newline at end of file +end From cf82ac1fc5ace1689da34fed841408685ec4a5e6 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 17 Apr 2024 19:46:53 +0000 Subject: [PATCH 2/8] Add debug --- .github/workflows/actions.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 3d3c5c35d..9ff499f1e 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -296,7 +296,10 @@ jobs: echo "RUNNER RATE LIMIT: $ANONYMOUS_RUNNER_RATE_LIMIT" echo "CLUSTER RATE LIMIT: $CLUSTER_RATE_LIMIT" echo "DOCKER USER RATE LIMIT: $AUTH_RATE_LIMIT" - LOG_LEVEL=info crystal spec --warnings none --tag ${{ matrix.tag }} -v + #LOG_LEVEL=info crystal spec --warnings none --tag ${{ matrix.tag }} -v + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + build: name: Build Release From 760210da0b5689cba36c16cfa6baba84371d99f0 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Thu, 18 Apr 2024 19:31:42 +0000 Subject: [PATCH 3/8] Upgrade Litmus & hardcode/patch bug with pod memory test cnti-testcatalog/testsuite#1973 --- src/tasks/litmus_setup.cr | 3 +- src/tasks/workload/reliability.cr | 50 +++++++------------ .../chaos_templates/pod_memory_hog.yml.ecr | 6 +-- 3 files changed, 23 insertions(+), 36 deletions(-) diff --git a/src/tasks/litmus_setup.cr b/src/tasks/litmus_setup.cr index 750a284bc..2a0412695 100644 --- a/src/tasks/litmus_setup.cr +++ b/src/tasks/litmus_setup.cr @@ -45,7 +45,8 @@ end module LitmusManager - Version = "2.14.0" + Version = "3.6.0" + RBAC_VERSION = "2.6.0" # Version = "1.13.8" # Version = "3.0.0-beta12" NODE_LABEL = "kubernetes.io/hostname" diff --git a/src/tasks/workload/reliability.cr b/src/tasks/workload/reliability.cr index 0e5a71152..df2fd0ca8 100644 --- a/src/tasks/workload/reliability.cr +++ b/src/tasks/workload/reliability.cr @@ -132,11 +132,10 @@ task "pod_network_latency", ["install_litmus"] do |t, args| KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/lat-experiment.yaml") KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/lat-rbac.yaml") else - # experiment_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-network-latency/experiment.yaml" - experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-network-latency/experiment.yaml" - # rbac_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-network-latency/rbac.yaml" - # https://raw.githubusercontent.com/litmuschaos/chaos-charts/v2.14.x/charts/generic/pod-network-latency/rbac.yaml - rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-network-latency/rbac.yaml" + experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/faults/kubernetes/pod-network-latency/fault.yaml" + rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::RBAC_VERSION}/charts/generic/pod-network-latency/rbac.yaml" + + experiment_path = LitmusManager.download_template(experiment_url, "#{t.name}_experiment.yaml") KubectlClient::Apply.file(experiment_path, namespace: app_namespace) @@ -216,10 +215,8 @@ task "pod_network_corruption", ["install_litmus"] do |t, args| KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/corr-experiment.yaml") KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/corr-rbac.yaml") else - experiment_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-network-corruption/experiment.yaml" - experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-network-corruption/experiment.yaml" - # rbac_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-network-corruption/rbac.yaml" - rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-network-corruption/rbac.yaml" + experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/faults/kubernetes/pod-network-corruption/fault.yaml" + rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::RBAC_VERSION}/charts/generic/pod-network-corruption/rbac.yaml" experiment_path = LitmusManager.download_template(experiment_url, "#{t.name}_experiment.yaml") KubectlClient::Apply.file(experiment_path, namespace: app_namespace) @@ -282,10 +279,8 @@ task "pod_network_duplication", ["install_litmus"] do |t, args| KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/dup-experiment.yaml") KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/dup-rbac.yaml") else - # experiment_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-network-duplication/experiment.yaml" - experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-network-duplication/experiment.yaml" - # rbac_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-network-duplication/rbac.yaml" - rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-network-duplication/rbac.yaml" + experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/faults/kubernetes/pod-network-duplication/fault.yaml" + rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::RBAC_VERSION}/charts/generic/pod-network-duplication/rbac.yaml" experiment_path = LitmusManager.download_template(experiment_url, "#{t.name}_experiment.yaml") KubectlClient::Apply.file(experiment_path, namespace: app_namespace) @@ -346,10 +341,8 @@ task "disk_fill", ["install_litmus"] do |t, args| KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/disk-fill-experiment.yaml") KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/disk-fill-rbac.yaml") else - # experiment_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/disk-fill/experiment.yaml" - experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/disk-fill/experiment.yaml" - # rbac_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/disk-fill/rbac.yaml" - rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/disk-fill/rbac.yaml" + experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/faults/kubernetes/disk-fill/fault.yaml" + rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::RBAC_VERSION}/charts/generic/disk-fill/rbac.yaml" experiment_path = LitmusManager.download_template(experiment_url, "#{t.name}_experiment.yaml") KubectlClient::Apply.file(experiment_path, namespace: app_namespace) @@ -439,9 +432,8 @@ task "pod_delete", ["install_litmus"] do |t, args| KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/pod-delete-experiment.yaml") KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/pod-delete-rbac.yaml") else - # experiment_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-delete/experiment.yaml" - experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-delete/experiment.yaml" - experiment_path = LitmusManager.download_template(experiment_url, "#{t.name}_experiment.yaml") + experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/faults/kubernetes/pod-delete/fault.yaml" + rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::RBAC_VERSION}/charts/generic/pod-delete/rbac.yaml" # rbac_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-delete/rbac.yaml" rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-delete/rbac.yaml" @@ -523,10 +515,8 @@ task "pod_memory_hog", ["install_litmus"] do |t, args| KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/pod-memory-hog-experiment.yaml") KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/pod-memory-hog-rbac.yaml") else - # experiment_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-memory-hog/experiment.yaml" - experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-memory-hog/experiment.yaml" - # rbac_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-memory-hog/rbac.yaml" - rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-memory-hog/rbac.yaml" + experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/faults/kubernetes/pod-memory-hog/fault.yaml" + rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::RBAC_VERSION}/charts/generic/pod-memory-hog/rbac.yaml" experiment_path = LitmusManager.download_template(experiment_url, "#{t.name}_experiment.yaml") KubectlClient::Apply.file(experiment_path, namespace: app_namespace) @@ -591,10 +581,8 @@ task "pod_io_stress", ["install_litmus"] do |t, args| KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/pod-io-stress-experiment.yaml") KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/pod-io-stress-rbac.yaml") else - # experiment_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-io-stress/experiment.yaml" - experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-io-stress/experiment.yaml" - # rbac_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-io-stress/rbac.yaml" - rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-io-stress/rbac.yaml" + experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/faults/kubernetes/pod-io-stress/fault.yaml" + rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::RBAC_VERSION}/charts/generic/pod-io-stress/rbac.yaml" experiment_path = LitmusManager.download_template(experiment_url, "#{t.name}_experiment.yaml") KubectlClient::Apply.file(experiment_path, namespace: app_namespace) @@ -666,10 +654,8 @@ task "pod_dns_error", ["install_litmus"] do |t, args| KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/pod-dns-error-experiment.yaml") KubectlClient::Apply.file("#{OFFLINE_MANIFESTS_PATH}/pod-dns-error-rbac.yaml") else - # experiment_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-dns-error/experiment.yaml" - experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-dns-error/experiment.yaml" - # rbac_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-dns-error/rbac.yaml" - rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-dns-error/rbac.yaml" + experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/faults/kubernetes/pod-dns-error/fault.yaml" + rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::RBAC_VERSION}/charts/generic/pod-dns-error/rbac.yaml" experiment_path = LitmusManager.download_template(experiment_url, "#{t.name}_experiment.yaml") KubectlClient::Apply.file(experiment_path, namespace: app_namespace) diff --git a/src/templates/chaos_templates/pod_memory_hog.yml.ecr b/src/templates/chaos_templates/pod_memory_hog.yml.ecr index 6a27a8ebc..0871900ec 100644 --- a/src/templates/chaos_templates/pod_memory_hog.yml.ecr +++ b/src/templates/chaos_templates/pod_memory_hog.yml.ecr @@ -19,9 +19,9 @@ spec: components: env: # Enter the amount of memory in megabytes to be consumed by the application pod + #TODO: Temp Fix, mem hardcode to made dynamic. - name: MEMORY_CONSUMPTION - value: '500' - + value: '100' - name: TOTAL_CHAOS_DURATION value: '<%= @total_chaos_duration %>' - name: TARGET_PODS @@ -34,4 +34,4 @@ spec: # provide the socket file path # applicable only for containerd and crio runtime - name: SOCKET_PATH - value: '/run/containerd/containerd.sock' \ No newline at end of file + value: '/run/containerd/containerd.sock' From 6b8cdc3f345fb19331578aa4bd19dc0670e93114 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Thu, 18 Apr 2024 19:33:32 +0000 Subject: [PATCH 4/8] Remove debug --- .github/workflows/actions.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 9ff499f1e..e1fe375e9 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -196,9 +196,6 @@ jobs: LOG_LEVEL=info crystal spec --warnings none --tag ${{ matrix.spec }} -v - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v2 - - name: Delete Cluster if: ${{ always() }} run: | From 872443e87532e548c8a5b8fc06aaa5315d0c7d18 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Thu, 18 Apr 2024 19:39:04 +0000 Subject: [PATCH 5/8] Add missing path for pod_delete cnti-testcatalog/testsuite#1973 --- src/tasks/workload/reliability.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tasks/workload/reliability.cr b/src/tasks/workload/reliability.cr index df2fd0ca8..9dac043a7 100644 --- a/src/tasks/workload/reliability.cr +++ b/src/tasks/workload/reliability.cr @@ -435,13 +435,15 @@ task "pod_delete", ["install_litmus"] do |t, args| experiment_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/faults/kubernetes/pod-delete/fault.yaml" rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::RBAC_VERSION}/charts/generic/pod-delete/rbac.yaml" - # rbac_url = "https://hub.litmuschaos.io/api/chaos/#{LitmusManager::Version}?file=charts/generic/pod-delete/rbac.yaml" + experiment_path = LitmusManager.download_template(experiment_url, "#{t.name}_experiment.yaml") + rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-delete/rbac.yaml" rbac_path = LitmusManager.download_template(rbac_url, "#{t.name}_rbac.yaml") rbac_yaml = File.read(rbac_path) rbac_yaml = rbac_yaml.gsub("namespace: default", "namespace: #{app_namespace}") File.write(rbac_path, rbac_yaml) + KubectlClient::Apply.file(experiment_path, namespace: app_namespace) KubectlClient::Apply.file(rbac_path) end From bee501f969a8fc3365c2e567647eb2d3c411aa4c Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Thu, 18 Apr 2024 19:43:16 +0000 Subject: [PATCH 6/8] Remove extra debug cnti-testcatalog/testsuite#1973 --- .github/workflows/actions.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index e1fe375e9..b2d079bad 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -293,9 +293,7 @@ jobs: echo "RUNNER RATE LIMIT: $ANONYMOUS_RUNNER_RATE_LIMIT" echo "CLUSTER RATE LIMIT: $CLUSTER_RATE_LIMIT" echo "DOCKER USER RATE LIMIT: $AUTH_RATE_LIMIT" - #LOG_LEVEL=info crystal spec --warnings none --tag ${{ matrix.tag }} -v - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 + LOG_LEVEL=info crystal spec --warnings none --tag ${{ matrix.tag }} -v build: From 7a7ce1ccc3b26758351fc772064ac65acbef3c3a Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Thu, 18 Apr 2024 23:53:00 +0000 Subject: [PATCH 7/8] Fix pod delete spec --- src/tasks/workload/reliability.cr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tasks/workload/reliability.cr b/src/tasks/workload/reliability.cr index 9dac043a7..7df39e326 100644 --- a/src/tasks/workload/reliability.cr +++ b/src/tasks/workload/reliability.cr @@ -437,7 +437,6 @@ task "pod_delete", ["install_litmus"] do |t, args| experiment_path = LitmusManager.download_template(experiment_url, "#{t.name}_experiment.yaml") - rbac_url = "https://raw.githubusercontent.com/litmuschaos/chaos-charts/#{LitmusManager::Version}/charts/generic/pod-delete/rbac.yaml" rbac_path = LitmusManager.download_template(rbac_url, "#{t.name}_rbac.yaml") rbac_yaml = File.read(rbac_path) rbac_yaml = rbac_yaml.gsub("namespace: default", "namespace: #{app_namespace}") From 0998ac3ce8e646ddf62f058ed735c7c3de8c948d Mon Sep 17 00:00:00 2001 From: "W. Watson" Date: Fri, 19 Apr 2024 20:03:22 +0000 Subject: [PATCH 8/8] #1973 pod disk fill now uses new configuration --- src/templates/chaos_templates/disk_fill.yml.ecr | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/templates/chaos_templates/disk_fill.yml.ecr b/src/templates/chaos_templates/disk_fill.yml.ecr index 931b8f814..116dd0035 100644 --- a/src/templates/chaos_templates/disk_fill.yml.ecr +++ b/src/templates/chaos_templates/disk_fill.yml.ecr @@ -19,12 +19,17 @@ spec: components: env: # specify the fill percentage according to the disk pressure required - - name: EPHEMERAL_STORAGE_MEBIBYTES - value: '500' - + #- name: EPHEMERAL_STORAGE_MEBIBYTES + #value: '500' + - name: TOTAL_CHAOS_DURATION + value: '60' + - name: CHAOS_INTERVAL + value: '10' + - name: DISK_FILL_PERCENTAGE + value: '80' - name: TARGET_CONTAINER value: '' - - name: FILL_PERCENTAGE - value: '' + #- name: FILL_PERCENTAGE + # value: '' - name: CONTAINER_PATH value: '/var/lib/containerd/io.containerd.grpc.v1.cri/containers/'