From 89471d624060882f580e244639f7b8915085f18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Garc=C3=ADa=20Montoro?= Date: Thu, 22 Feb 2024 15:53:45 +0100 Subject: [PATCH] Add note about --prometheus-url in local deployments (#698) * Add note about --prometheus-url in local deployments * Add note when retrieving Prom URL from deployment * Add prerequisites explaining Prometheus setup --- cmd/ltctl/report.go | 1 + docs/compare.md | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/cmd/ltctl/report.go b/cmd/ltctl/report.go index 4037204cd..0f7290990 100644 --- a/cmd/ltctl/report.go +++ b/cmd/ltctl/report.go @@ -76,6 +76,7 @@ func RunGenerateReportCmdF(cmd *cobra.Command, args []string) error { } if promURL == "" { + fmt.Printf("Flag --prometheus-url is not set. Defaulting to the deployment's Prometheus server...") config, err := getConfig(cmd) if err != nil { return err diff --git a/docs/compare.md b/docs/compare.md index aa473743a..53f0a2576 100644 --- a/docs/compare.md +++ b/docs/compare.md @@ -1,5 +1,15 @@ # Comparing load test results +## Prerequisites + +Test reports contain two types of data: +- API and store times averaged over the duration of the whole test. +- Granular data of user-specified queries, configured through [the Report subsection in `deployer.json`](config/deployer.md#report). + +All those data points are retrieved from Prometheus at the moment of generating the report, but the Prometheus service should have been scraping the metrics *during the test*. Terraform deployments automatically provide a metrics instance with a Prometheus service running, but locally, things may be different: +- If you're running a dev environment locally through the docker-compose file in the `mattermost` repository (i.e., using `make run` or `make run-server`), a Prometheus server should be already up and running at `localhost:9090`. +- If you're running Mattermost locally using any other method, you may need to [manually install and run a Prometheus server](https://prometheus.io/docs/introduction/first_steps/). Make sure to configure it to scrape both your local Mattermost metrics (usually at `localhost:8067`) and [the node exporter](https://prometheus.io/docs/guides/node-exporter/#monitoring-linux-host-metrics-with-the-node-exporter) (usually at `localhost:9100`). + ## Generating reports After a load test is complete, a report can be generated and saved to a file which can be used to compare different load test runs. This can be done using @@ -8,6 +18,8 @@ After a load test is complete, a report can be generated and saved to a file whi go run ./cmd/ltctl report generate --output=base.out --label=base "2020-06-23 07:23:35" "2020-06-23 07:33:35" ``` +Note that in the case of a local deployment, you'll need to set the `--prometheus-url ` flag to the URL of the Prometheus server containing your metrics: usually `http://localhost:9090`. If the flag is not set, the tool will assume a Terraform deployment is up and will try to connect to its Prometheus. + The timestamps _must_ be in UTC, and they indicate the range within which the data is to be collected for the test. It is recommended to keep the timestamps within a range during which the loadtest is running a stable number of users, and not in a ramp-up phase or an unstable state of adding/removing users. This is to get consistent results between different runs. The timestamp ranges for different load tests can be different. They will be compared with the base report. If a report has more data points than the base report, the extra ones will be ignored.