From 1f8191d5f6eb07d73f207173fa2a4eb19012edcb Mon Sep 17 00:00:00 2001 From: Marcin Grzejszczak Date: Tue, 20 Apr 2021 08:28:01 +0000 Subject: [PATCH] Added chaos monkey setup (#177) * Added chaos monkey setup * Update scripts/chaos/call_chaos.sh Co-authored-by: Jonatan Ivanov * Added docs Co-authored-by: Jonatan Ivanov --- .gitignore | 3 + README.md | 6 + pom.xml | 6 + scripts/chaos/README.md | 24 +++ scripts/chaos/attacks_disable.json | 6 + scripts/chaos/attacks_enable_exception.json | 4 + .../chaos/attacks_enable_killapplication.json | 3 + scripts/chaos/attacks_enable_latency.json | 5 + scripts/chaos/attacks_enable_memory.json | 4 + scripts/chaos/call_chaos.sh | 49 ++++++ scripts/chaos/watcher_disable.json | 7 + scripts/chaos/watcher_enable_component.json | 3 + scripts/chaos/watcher_enable_controller.json | 3 + scripts/chaos/watcher_enable_repository.json | 3 + .../chaos/watcher_enable_restcontroller.json | 3 + scripts/chaos/watcher_enable_service.json | 3 + scripts/run_all.sh | 29 +++ spring-petclinic-api-gateway/.factorypath | 159 ----------------- spring-petclinic-customers-service/pom.xml | 4 + spring-petclinic-vets-service/.factorypath | 166 ------------------ spring-petclinic-vets-service/pom.xml | 4 + spring-petclinic-visits-service/pom.xml | 4 + 22 files changed, 173 insertions(+), 325 deletions(-) create mode 100644 scripts/chaos/README.md create mode 100644 scripts/chaos/attacks_disable.json create mode 100644 scripts/chaos/attacks_enable_exception.json create mode 100644 scripts/chaos/attacks_enable_killapplication.json create mode 100644 scripts/chaos/attacks_enable_latency.json create mode 100644 scripts/chaos/attacks_enable_memory.json create mode 100755 scripts/chaos/call_chaos.sh create mode 100644 scripts/chaos/watcher_disable.json create mode 100644 scripts/chaos/watcher_enable_component.json create mode 100644 scripts/chaos/watcher_enable_controller.json create mode 100644 scripts/chaos/watcher_enable_repository.json create mode 100644 scripts/chaos/watcher_enable_restcontroller.json create mode 100644 scripts/chaos/watcher_enable_service.json create mode 100755 scripts/run_all.sh delete mode 100644 spring-petclinic-api-gateway/.factorypath delete mode 100644 spring-petclinic-vets-service/.factorypath diff --git a/.gitignore b/.gitignore index 3b29741b0..e282e3b23 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,8 @@ target/ .idea *.iml +# Visual Studio Code +.factorypath + # Branch switching generated/ diff --git a/README.md b/README.md index 2812b8ea0..6e1a26724 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,12 @@ The `master` branch uses an Alpine linux with JRE 8 as Docker base. You will f *NOTE: Under MacOSX or Windows, make sure that the Docker VM has enough memory to run the microservices. The default settings are usually not enough and make the `docker-compose up` painfully slow.* + +## Starting services locally with docker-compose and Java +If you experience issues with running the system via docker-compose you can try running the `./scripts/run_all.sh` script that will start the infrastructure services via docker-compose and all the Java based applications via standard `nohup java -jar ...` command. The logs will be available under `${ROOT}/target/nameoftheapp.log`. + +Each of the java based applications is started with the `chaos-monkey` profile in order to interact with Spring Boot Chaos Monkey. You can check out the (README)[scripts/chaos/README.md] for more information about how to use the `./scripts/chaos/call_chaos.sh` helper script to enable assaults. + ## Understanding the Spring Petclinic application [See the presentation of the Spring Petclinic Framework version](http://fr.slideshare.net/AntoineRey/spring-framework-petclinic-sample-application) diff --git a/pom.xml b/pom.xml index 88598252d..8c5f791a3 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ 2.4.2 2020.0.1 + 2.3.2 2.22.0 @@ -62,6 +63,11 @@ + + de.codecentric + chaos-monkey-spring-boot + ${chaos-monkey-spring-boot.version} + org.assertj assertj-core diff --git a/scripts/chaos/README.md b/scripts/chaos/README.md new file mode 100644 index 000000000..e7bbc9ed3 --- /dev/null +++ b/scripts/chaos/README.md @@ -0,0 +1,24 @@ +# Spring Boot Chaos Monkey Scripts + +You can read more about the possible configuration options [here](https://codecentric.github.io/chaos-monkey-spring-boot/latest/#_properties). + +## Scripts + +In order to active Spring Boot Chaos Monkey's assault options and component instrumentation, you need to call the project's API. For your convenience we're providing a script that turns on various watchers and attacks. To print out the usage description just call the script without any parameters. + +```bash +$ ./scripts/chaos/call_chaos.sh +usage: ./scripts/chaos/call_chaos.sh: +First pick either customers, visits or vets +Then pick what to enable. Order matters! +Example +./scripts/chaos/call_chaos.sh visits attacks_enable_exception watcher_enable_restcontroller +``` + +The script takes in at minimum 2 parameters. First provides the name of the application for which you want to turn on Chaos Monkey features. The subsequent ones will enable attacks and watchers. The name of the desired feature maps to a json file that gets updated to `http://localhost:${PORT}/actuator/chaosmonkey/assaults` and `http://localhost:${PORT}/actuator/chaosmonkey/watchers` respectively. Example of enabling exception assault via rest controllers for the visits microservice: + +```bash +$ ./scripts/chaos/call_chaos.sh visits attacks_enable_exception watcher_enable_restcontroller +``` + +The default assault configuration is set to fail every 5th request. That means that the first four will work as if Chaos Monkey was be disabled. diff --git a/scripts/chaos/attacks_disable.json b/scripts/chaos/attacks_disable.json new file mode 100644 index 000000000..cc60dbf86 --- /dev/null +++ b/scripts/chaos/attacks_disable.json @@ -0,0 +1,6 @@ +{ + "latencyActive": false, + "exceptionsActive": false, + "killApplicationActive": false, + "memoryActive": false +} diff --git a/scripts/chaos/attacks_enable_exception.json b/scripts/chaos/attacks_enable_exception.json new file mode 100644 index 000000000..4e24f5a14 --- /dev/null +++ b/scripts/chaos/attacks_enable_exception.json @@ -0,0 +1,4 @@ +{ + "exceptionsActive": true, + "level" : 5 +} diff --git a/scripts/chaos/attacks_enable_killapplication.json b/scripts/chaos/attacks_enable_killapplication.json new file mode 100644 index 000000000..ab6720853 --- /dev/null +++ b/scripts/chaos/attacks_enable_killapplication.json @@ -0,0 +1,3 @@ +{ + "killApplicationActive": true +} diff --git a/scripts/chaos/attacks_enable_latency.json b/scripts/chaos/attacks_enable_latency.json new file mode 100644 index 000000000..1ef154608 --- /dev/null +++ b/scripts/chaos/attacks_enable_latency.json @@ -0,0 +1,5 @@ +{ + "latencyActive": true, + "latencyRangeStart" : 1000, + "latencyRangeEnd" : 10000 +} diff --git a/scripts/chaos/attacks_enable_memory.json b/scripts/chaos/attacks_enable_memory.json new file mode 100644 index 000000000..b92066393 --- /dev/null +++ b/scripts/chaos/attacks_enable_memory.json @@ -0,0 +1,4 @@ +{ + "memoryActive": true, + "level" : 5 +} diff --git a/scripts/chaos/call_chaos.sh b/scripts/chaos/call_chaos.sh new file mode 100755 index 000000000..b412427e2 --- /dev/null +++ b/scripts/chaos/call_chaos.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail + +ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function usage { + echo "usage: $0: " + echo "First pick either customers, visits or vets" + echo "Then pick what to enable. Order matters!" + echo "Example" + echo "./scripts/chaos/call_chaos.sh visits attacks_enable_exception watcher_enable_restcontroller" + exit 1 +} + +if [[ $# -lt 2 ]]; then + usage +fi + +export PORT="${PORT:-}" + +while [[ $# > 0 ]] +do +key="$1" +case $1 in + customers) + PORT=8081 + ;; + visits) + PORT=8082 + ;; + vets) + PORT=8083 + ;; + attacks*) + ( cd "${ROOT_DIR}" && curl "http://localhost:${PORT}/actuator/chaosmonkey/assaults" -H "Content-Type: application/json" --data @"${1}".json --fail ) + ;; + watcher*) + ( cd "${ROOT_DIR}" && curl "http://localhost:${PORT}/actuator/chaosmonkey/watchers" -H "Content-Type: application/json" --data @"${1}".json --fail ) + ;; + *) + usage + ;; +esac +shift +done diff --git a/scripts/chaos/watcher_disable.json b/scripts/chaos/watcher_disable.json new file mode 100644 index 000000000..78779dc6b --- /dev/null +++ b/scripts/chaos/watcher_disable.json @@ -0,0 +1,7 @@ +{ + "controller": false, + "restController": false, + "service": false, + "repository": false, + "component": false +} diff --git a/scripts/chaos/watcher_enable_component.json b/scripts/chaos/watcher_enable_component.json new file mode 100644 index 000000000..10e7f9d58 --- /dev/null +++ b/scripts/chaos/watcher_enable_component.json @@ -0,0 +1,3 @@ +{ + "component": true +} diff --git a/scripts/chaos/watcher_enable_controller.json b/scripts/chaos/watcher_enable_controller.json new file mode 100644 index 000000000..b45401627 --- /dev/null +++ b/scripts/chaos/watcher_enable_controller.json @@ -0,0 +1,3 @@ +{ + "controller": true +} diff --git a/scripts/chaos/watcher_enable_repository.json b/scripts/chaos/watcher_enable_repository.json new file mode 100644 index 000000000..5e16469c3 --- /dev/null +++ b/scripts/chaos/watcher_enable_repository.json @@ -0,0 +1,3 @@ +{ + "repository": true +} diff --git a/scripts/chaos/watcher_enable_restcontroller.json b/scripts/chaos/watcher_enable_restcontroller.json new file mode 100644 index 000000000..e846f6d6d --- /dev/null +++ b/scripts/chaos/watcher_enable_restcontroller.json @@ -0,0 +1,3 @@ +{ + "restController": true +} diff --git a/scripts/chaos/watcher_enable_service.json b/scripts/chaos/watcher_enable_service.json new file mode 100644 index 000000000..f4dd8e7de --- /dev/null +++ b/scripts/chaos/watcher_enable_service.json @@ -0,0 +1,3 @@ +{ + "service": true +} diff --git a/scripts/run_all.sh b/scripts/run_all.sh new file mode 100755 index 000000000..8560c917e --- /dev/null +++ b/scripts/run_all.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail + +pkill -9 -f spring-petclinic || echo "Failed to kill any apps" + +docker-compose kill || echo "No docker containers are running" + +echo "Running infra" +docker-compose up -d grafana-server prometheus-server tracing-server + +echo "Running apps" +mkdir -p target +nohup java -jar spring-petclinic-config-server/target/*.jar --server.port=8888 --spring.profiles.active=chaos-monkey > target/config-server.log 2>&1 & +echo "Waiting for config server to start" +sleep 20 +nohup java -jar spring-petclinic-discovery-server/target/*.jar --server.port=8761 --spring.profiles.active=chaos-monkey > target/discovery-server.log 2>&1 & +echo "Waiting for discovery server to start" +sleep 20 +nohup java -jar spring-petclinic-customers-service/target/*.jar --server.port=8081 --spring.profiles.active=chaos-monkey > target/customers-service.log 2>&1 & +nohup java -jar spring-petclinic-visits-service/target/*.jar --server.port=8082 --spring.profiles.active=chaos-monkey > target/visits-service.log 2>&1 & +nohup java -jar spring-petclinic-vets-service/target/*.jar --server.port=8083 --spring.profiles.active=chaos-monkey > target/vets-service.log 2>&1 & +nohup java -jar spring-petclinic-api-gateway/target/*.jar --server.port=8080 --spring.profiles.active=chaos-monkey > target/gateway-service.log 2>&1 & +nohup java -jar spring-petclinic-admin-server/target/*.jar --server.port=9090 --spring.profiles.active=chaos-monkey > target/admin-server.log 2>&1 & +echo "Waiting for apps to start" +sleep 60 diff --git a/spring-petclinic-api-gateway/.factorypath b/spring-petclinic-api-gateway/.factorypath deleted file mode 100644 index 29bf92844..000000000 --- a/spring-petclinic-api-gateway/.factorypath +++ /dev/null @@ -1,159 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spring-petclinic-customers-service/pom.xml b/spring-petclinic-customers-service/pom.xml index de578ac6d..758c17202 100644 --- a/spring-petclinic-customers-service/pom.xml +++ b/spring-petclinic-customers-service/pom.xml @@ -85,6 +85,10 @@ io.micrometer micrometer-registry-prometheus + + de.codecentric + chaos-monkey-spring-boot + diff --git a/spring-petclinic-vets-service/.factorypath b/spring-petclinic-vets-service/.factorypath deleted file mode 100644 index ccb3d4aae..000000000 --- a/spring-petclinic-vets-service/.factorypath +++ /dev/null @@ -1,166 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spring-petclinic-vets-service/pom.xml b/spring-petclinic-vets-service/pom.xml index 570452cb3..4f6baa129 100644 --- a/spring-petclinic-vets-service/pom.xml +++ b/spring-petclinic-vets-service/pom.xml @@ -101,6 +101,10 @@ io.micrometer micrometer-registry-prometheus + + de.codecentric + chaos-monkey-spring-boot + diff --git a/spring-petclinic-visits-service/pom.xml b/spring-petclinic-visits-service/pom.xml index 670acbe68..275bbbb31 100644 --- a/spring-petclinic-visits-service/pom.xml +++ b/spring-petclinic-visits-service/pom.xml @@ -84,6 +84,10 @@ io.micrometer micrometer-registry-prometheus + + de.codecentric + chaos-monkey-spring-boot +