Merge remote-tracking branch 'origin/master' into feat/openid-users

.gitignore

@@ -88,3 +88,5 @@ config/metricswatcher.json
 mattermost-load-test.iml
 
 .terraform.tfstate.lock.info
+
+*.tar

cmd/ltctl/main.go

@@ -12,6 +12,7 @@ import (
 	"github.com/mattermost/mattermost-load-test-ng/deployment"
 	"github.com/mattermost/mattermost-load-test-ng/deployment/terraform"
 	"github.com/mattermost/mattermost-load-test-ng/logger"
+	"github.com/mattermost/mattermost/server/public/model"
 
 	"github.com/spf13/cobra"
 )
@@ -171,18 +172,28 @@ func getConfig(cmd *cobra.Command) (deployment.Config, error) {
 	return *cfg, nil
 }
 
+func setServiceEnv(cmd *cobra.Command) {
+	serviceEnv, _ := cmd.Flags().GetString("service_environment")
+	// Set it to test if it's neither prod nor test.
+	if serviceEnv != model.ServiceEnvironmentProduction && serviceEnv != model.ServiceEnvironmentTest {
+		serviceEnv = model.ServiceEnvironmentTest
+	}
+	os.Setenv("MM_SERVICEENVIRONMENT", serviceEnv)
+}
+
 func main() {
 	rootCmd := &cobra.Command{
 		Use:          "ltctl",
 		SilenceUsage: true,
 		Short:        "Manage and control load-test deployments",
 	}
 	rootCmd.PersistentFlags().StringP("config", "c", "", "path to the deployer configuration file to use")
+	rootCmd.PersistentFlags().StringP("service_environment", "s", model.ServiceEnvironmentTest, "value of the MM_SERVICEENVIRONMENT variable. Valid values are production, test")
 
 	deploymentCmd := &cobra.Command{
 		Use:               "deployment",
 		Short:             "Manage a load-test deployment",
-		PersistentPreRun:  func(_ *cobra.Command, _ []string) { os.Setenv("MM_SERVICEENVIRONMENT", "production") },
+		PersistentPreRun:  func(cmd *cobra.Command, _ []string) { setServiceEnv(cmd) },
 		PersistentPostRun: func(_ *cobra.Command, _ []string) { os.Unsetenv("MM_SERVICEENVIRONMENT") },
 	}
 
@@ -230,7 +241,7 @@ func main() {
 	loadtestCmd := &cobra.Command{
 		Use:               "loadtest",
 		Short:             "Manage the load-test",
-		PersistentPreRun:  func(_ *cobra.Command, _ []string) { os.Setenv("MM_SERVICEENVIRONMENT", "production") },
+		PersistentPreRun:  func(cmd *cobra.Command, _ []string) { setServiceEnv(cmd) },
 		PersistentPostRun: func(_ *cobra.Command, _ []string) { os.Unsetenv("MM_SERVICEENVIRONMENT") },
 	}
 
@@ -381,7 +392,7 @@ func main() {
 	comparisonCmd := &cobra.Command{
 		Use:               "comparison",
 		Short:             "Manage fully automated load-test comparisons environments",
-		PersistentPreRun:  func(_ *cobra.Command, _ []string) { os.Setenv("MM_SERVICEENVIRONMENT", "production") },
+		PersistentPreRun:  func(cmd *cobra.Command, _ []string) { setServiceEnv(cmd) },
 		PersistentPostRun: func(_ *cobra.Command, _ []string) { os.Unsetenv("MM_SERVICEENVIRONMENT") },
 	}
 	comparisonCmd.Flags().StringP("comparison-config", "", "", "path to the comparison config file to use")

config/deployer.sample.json

@@ -6,9 +6,9 @@
   "ClusterVpcID": "",
   "ClusterSubnetID": "",
   "AppInstanceCount": 1,
-  "AppInstanceType": "c5.xlarge",
+  "AppInstanceType": "c7i.xlarge",
   "AgentInstanceCount": 2,
-  "AgentInstanceType": "c5.xlarge",
+  "AgentInstanceType": "c7i.xlarge",
   "ElasticSearchSettings": {
     "InstanceCount": 0,
     "InstanceType": "r6g.large.search",
@@ -18,10 +18,10 @@
   },
   "JobServerSettings":{
     "InstanceCount": 0,
-    "InstanceType": "c5.xlarge"
+    "InstanceType": "c7i.xlarge"
   },
   "EnableAgentFullLogs": true,
-  "ProxyInstanceType": "c5.xlarge",
+  "ProxyInstanceType": "c7i.xlarge",
   "SSHPublicKey": "~/.ssh/id_rsa.pub",
   "TerraformStateDir" : "/var/lib/mattermost-load-test-ng",
   "S3BucketDumpURI" : "",
@@ -30,7 +30,7 @@
   "TerraformDBSettings": {
     "InstanceCount": 1,
     "InstanceEngine": "aurora-postgresql",
-    "InstanceType": "db.r6g.large",
+    "InstanceType": "db.r7g.large",
     "UserName": "mmuser",
     "Password": "mostest80098bigpass_",
     "EnablePerformanceInsights": true,
@@ -60,7 +60,7 @@
   "AdminEmail": "[email protected]",
   "AdminUsername": "sysadmin",
   "AdminPassword": "Sys@dmin-sample1",
-  "LoadTestDownloadURL": "https://github.com/mattermost/mattermost-load-test-ng/releases/download/v1.15.0-rc2/mattermost-load-test-ng-v1.15.0-rc2-linux-amd64.tar.gz",
+  "LoadTestDownloadURL": "https://github.com/mattermost/mattermost-load-test-ng/releases/download/v1.15.0/mattermost-load-test-ng-v1.15.0-linux-amd64.tar.gz",
   "LogSettings": {
     "EnableConsole": true,
     "ConsoleLevel": "INFO",

deployment/config.go

@@ -34,11 +34,11 @@ type Config struct {
 	// Number of application instances.
 	AppInstanceCount int `default:"1" validate:"range:[0,)"`
 	// Type of the EC2 instance for app.
-	AppInstanceType string `default:"c5.xlarge" validate:"notempty"`
+	AppInstanceType string `default:"c7i.xlarge" validate:"notempty"`
 	// Number of agents, first agent and coordinator will share the same instance.
 	AgentInstanceCount int `default:"2" validate:"range:[0,)"`
 	// Type of the EC2 instance for agent.
-	AgentInstanceType string `default:"c5.xlarge" validate:"notempty"`
+	AgentInstanceType string `default:"c7i.xlarge" validate:"notempty"`
 	// Logs the command output (stdout & stderr) to home directory.
 	EnableAgentFullLogs bool `default:"true"`
 	// Type of the EC2 instance for proxy.
@@ -73,7 +73,7 @@ type Config struct {
 	// URL from where to download load-test-ng binaries and configuration files.
 	// The configuration files provided in the package will be overridden in
 	// the deployment process.
-	LoadTestDownloadURL   string `default:"https://github.com/mattermost/mattermost-load-test-ng/releases/download/v1.15.0-rc2/mattermost-load-test-ng-v1.15.0-rc2-linux-amd64.tar.gz" validate:"url"`
+	LoadTestDownloadURL   string `default:"https://github.com/mattermost/mattermost-load-test-ng/releases/download/v1.15.0/mattermost-load-test-ng-v1.15.0-linux-amd64.tar.gz" validate:"url"`
 	ElasticSearchSettings ElasticSearchSettings
 	JobServerSettings     JobServerSettings
 	LogSettings           logger.Settings
@@ -136,7 +136,7 @@ type TerraformDBSettings struct {
 	// Number of DB instances.
 	InstanceCount int `default:"1" validate:"range:[0,)"`
 	// Type of the DB instance.
-	InstanceType string `default:"db.r6g.large" validate:"notempty"`
+	InstanceType string `default:"db.r7g.large" validate:"notempty"`
 	// Type of the DB instance - postgres or mysql.
 	InstanceEngine string `default:"aurora-postgresql" validate:"oneof:{aurora-mysql, aurora-postgresql}"`
 	// Username to connect to the DB.
@@ -243,7 +243,7 @@ type JobServerSettings struct {
 	// Job server instances count.
 	InstanceCount int `default:"0" validate:"range:[0,1]"`
 	// Job server instance type to be created.
-	InstanceType string `default:"c5.xlarge"`
+	InstanceType string `default:"c7i.xlarge"`
 }
 
 // DBParameter contains info regarding a single RDS DB specific parameter.

deployment/config_test.go

@@ -10,7 +10,7 @@ func TestConfigIsValid(t *testing.T) {
 	baseConfig := func() Config {
 		return Config{
 			MattermostDownloadURL: "https://latest.mattermost.com/mattermost-enterprise-linux",
-			LoadTestDownloadURL:   "https://github.com/mattermost/mattermost-load-test-ng/releases/download/v1.15.0-rc2/mattermost-load-test-ng-v1.15.0-rc2-linux-amd64.tar.gz",
+			LoadTestDownloadURL:   "https://github.com/mattermost/mattermost-load-test-ng/releases/download/v1.15.0/mattermost-load-test-ng-v1.15.0-linux-amd64.tar.gz",
 		}
 	}
 

deployment/terraform/assets/cluster.tf

@@ -174,12 +174,18 @@ resource "aws_instance" "proxy_server" {
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
       "echo 'tcp_bbr' | sudo tee -a /etc/modules",
       "sudo modprobe tcp_bbr",
+      "wget -qO - https://nginx.org/keys/nginx_signing.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/nginx.gpg",
+      "sudo sh -c 'echo \"deb [arch=amd64] http://nginx.org/packages/mainline/ubuntu/ $(lsb_release -cs) nginx\" > /etc/apt/sources.list.d/nginx.list'",
+      "sudo sh -c 'echo \"deb-src http://nginx.org/packages/mainline/ubuntu/ $(lsb_release -cs) nginx\" >> /etc/apt/sources.list.d/nginx.list'",
       "sudo apt-get -y update",
-      "sudo apt-get install -y prometheus-node-exporter",
       "sudo apt-get install -y nginx",
+      "sudo apt-get install -y prometheus-node-exporter",
       "sudo apt-get install -y numactl linux-tools-aws linux-tools-aws-lts-22.04",
       "sudo systemctl daemon-reload",
       "sudo systemctl enable nginx",
+      "sudo mkdir -p /etc/nginx/snippets",
+      "sudo mkdir -p /etc/nginx/sites-available",
+      "sudo mkdir -p /etc/nginx/sites-enabled",
       "sudo rm -f /etc/nginx/sites-enabled/default",
       "sudo ln -fs /etc/nginx/sites-available/mattermost /etc/nginx/sites-enabled/mattermost"
     ]

deployment/terraform/create.go

@@ -311,7 +311,7 @@ func (t *Terraform) setupAppServer(extAgent *ssh.ExtAgent, ip, siteURL, serviceF
 	// Upload files
 	batch := []uploadInfo{
 		{srcData: strings.TrimPrefix(serverSysctlConfig, "\n"), dstPath: "/etc/sysctl.conf"},
-		{srcData: strings.TrimSpace(serviceFile), dstPath: "/lib/systemd/system/mattermost.service"},
+		{srcData: strings.TrimSpace(fmt.Sprintf(serviceFile, os.Getenv("MM_SERVICEENVIRONMENT"))), dstPath: "/lib/systemd/system/mattermost.service"},
 		{srcData: strings.TrimPrefix(limitsConfig, "\n"), dstPath: "/etc/security/limits.conf"},
 	}
 
@@ -417,6 +417,7 @@ func (t *Terraform) setupProxyServer(extAgent *ssh.ExtAgent) {
 		mlog.Error("error in getting ssh connection", mlog.String("ip", ip), mlog.Err(err))
 		return
 	}
+
 	func() {
 		defer func() {
 			err := sshc.Close()
@@ -430,7 +431,22 @@ func (t *Terraform) setupProxyServer(extAgent *ssh.ExtAgent) {
 
 		backends := ""
 		for _, addr := range t.output.Instances {
-			backends += "server " + addr.PrivateIP + ":8065 max_fails=3;\n"
+			backends += "server " + addr.PrivateIP + ":8065 max_fails=0;\n"
+		}
+
+		cacheObjects := "10m"
+		cacheSize := "3g"
+		// Extracting the instance class from the type.
+		// Usually they are of the form (m7/c7).(large/2xlarge/4xlarge/..)
+		parts := strings.Split(t.config.ProxyInstanceType, ".")
+		if len(parts) > 1 {
+			// Hack alert. There can be other higher variants like 12xlarge or even metal. But we are keeping it simple for now.
+			switch parts[1] {
+			case "4", "8":
+				cacheObjects = "50m"
+				cacheSize = "16g" // Ideally we'd like half of the total server mem. But the mem consumption rarely exceeds 10G
+				// from my tests. So there's no point stretching it further.
+			}
 		}
 
 		nginxConfig, err := genNginxConfig()
@@ -439,10 +455,20 @@ func (t *Terraform) setupProxyServer(extAgent *ssh.ExtAgent) {
 			return
 		}
 
+		nginxSiteConfig, err := fillConfigTemplate(nginxSiteConfigTmpl, map[string]string{
+			"backends":     backends,
+			"cacheObjects": cacheObjects,
+			"cacheSize":    cacheSize,
+		})
+		if err != nil {
+			mlog.Error("Failed to generate nginx site config", mlog.Err(err))
+			return
+		}
+
 		batch := []uploadInfo{
 			{srcData: strings.TrimLeft(nginxProxyCommonConfig, "\n"), dstPath: "/etc/nginx/snippets/proxy.conf"},
 			{srcData: strings.TrimLeft(nginxCacheCommonConfig, "\n"), dstPath: "/etc/nginx/snippets/cache.conf"},
-			{srcData: strings.TrimLeft(fmt.Sprintf(nginxSiteConfig, backends), "\n"), dstPath: "/etc/nginx/sites-available/mattermost"},
+			{srcData: strings.TrimLeft(nginxSiteConfig, "\n"), dstPath: "/etc/nginx/sites-available/mattermost"},
 			{srcData: strings.TrimLeft(serverSysctlConfig, "\n"), dstPath: "/etc/sysctl.conf"},
 			{srcData: strings.TrimLeft(nginxConfig, "\n"), dstPath: "/etc/nginx/nginx.conf"},
 			{srcData: strings.TrimLeft(limitsConfig, "\n"), dstPath: "/etc/security/limits.conf"},
@@ -452,7 +478,7 @@ func (t *Terraform) setupProxyServer(extAgent *ssh.ExtAgent) {
 			return
 		}
 
-		cmd := "sudo sysctl -p && sudo service nginx reload"
+		cmd := "sudo sysctl -p && sudo service nginx restart"
 		if out, err := sshc.RunCommand(cmd); err != nil {
 			mlog.Error("error running ssh command", mlog.String("output", string(out)), mlog.String("cmd", cmd), mlog.Err(err))
 			return

deployment/terraform/strings.go

@@ -19,6 +19,7 @@ Group=ubuntu
 LimitNOFILE=49152
 Environment=MM_FEATUREFLAGS_POSTPRIORITY=true
 Environment=MM_FEATUREFLAGS_WEBSOCKETEVENTSCOPE=true
+Environment=MM_SERVICEENVIRONMENT=%s
 
 [Install]
 WantedBy=multi-user.target
@@ -165,7 +166,6 @@ http {
   access_log /var/log/nginx/access.log combined if=$loggable;
   error_log /var/log/nginx/error.log;
   gzip on;
-  include /etc/nginx/conf.d/*.conf;
   include /etc/nginx/sites-enabled/*;
 }
 `
@@ -197,16 +197,16 @@ proxy_cache_use_stale timeout;
 proxy_cache_lock on;
 `
 
-const nginxSiteConfig = `
+const nginxSiteConfigTmpl = `
 upstream backend {
-%s
+{{.backends}}
   keepalive 256;
 }
 
-proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=mattermost_cache:10m max_size=3g inactive=120m use_temp_path=off;
+proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=mattermost_cache:{{.cacheObjects}} max_size={{.cacheSize}} inactive=60m use_temp_path=off;
 
 server {
-  listen 80;
+  listen 80 reuseport;
   server_name _;
 
   location ~ /api/v[0-9]+/(users/)?websocket$ {
@@ -347,6 +347,7 @@ WorkingDirectory=/opt/mattermost
 User=ubuntu
 Group=ubuntu
 LimitNOFILE=49152
+Environment=MM_SERVICEENVIRONMENT=%s
 
 [Install]
 WantedBy=multi-user.target

docs/terraform_loadtest.md

@@ -57,6 +57,8 @@ Once done, it will output information about the entire cluster. Everything will
 
 If you see an error when running `deployment create` mentioning a "resource already exists", it is most likely because your `ClusterName` is not a unique value within your AWS account. Run `go run ./cmd/ltctl deployment destroy` to clean up the half created deployment. Then change the `ClusterName` to something more unique for your loadtest and try again.
 
+NOTE: If you are not a Mattermost staff, then most likely you would want to set the `--service_environment` flag to `production` so that the right license type is picked up. Internally, we use test licenses.
+
 ### Get information on the current deployment
 
 ```sh
@@ -108,6 +110,8 @@ go run ./cmd/ltctl loadtest start
 
 This will begin to run the load-test across the whole cluster of load-test agents.
 
+NOTE: If you are not a Mattermost staff, then most likely you would want to set the `--service_environment` flag to `production` so that the right license type is picked up. Internally, we use test licenses.
+
 ### Show the load-test status
 
 ```sh

loadtest/control/simulcontroller/actions.go

@@ -75,6 +75,9 @@ func (c *SimulController) reload(full bool) control.UserActionResponse {
 		if err := c.connect(); err != nil {
 			return control.UserActionResponse{Err: control.NewUserError(err)}
 		}
+		if resp := control.FetchStaticAssets(c.user); resp.Err != nil {
+			return resp
+		}
 	}
 
 	// A full reload always calls GET /api/v4/users?page=0&per_page=100,
@@ -132,7 +135,7 @@ func (c *SimulController) loginOrSignUp(u user.User) control.UserActionResponse
 		c.status <- c.newInfoStatus(resp.Info)
 		return c.login(u)
 	}
-	return resp
+	return control.FetchStaticAssets(u)
 }
 
 func (c *SimulController) login(u user.User) control.UserActionResponse {

loadtest/control/simulcontroller/websocket.go

@@ -36,6 +36,13 @@ func (c *SimulController) wsEventHandler(wg *sync.WaitGroup) {
 				break
 			}
 
+			if ack, ok := ev.GetData()["should_ack"]; ok && ack.(bool) {
+				if err := c.user.PostedAck(post.Id, "success", "", ""); err != nil {
+					c.status <- c.newErrorStatus(err)
+					break
+				}
+			}
+
 			cm, _ := c.user.Store().ChannelMember(post.ChannelId, c.user.Store().Id())
 			if cm.UserId != "" {
 				break

loadtest/user/user.go

@@ -46,6 +46,7 @@ type User interface {
 	UpdateActiveChannel(channelId string) error
 	UpdateActiveThread(channelId string) error
 	UpdateActiveTeam(teamId string) error
+	PostedAck(postId string, status string, reason string, postedData string) error
 
 	//server
 	// GetConfig fetches and stores the server's configuration.

loadtest/user/userentity/user.go

@@ -84,6 +84,13 @@ type teamPresenceMsg struct {
 	teamId string
 }
 
+type postedAckMsg struct {
+	postId     string
+	status     string
+	reason     string
+	postedData string
+}
+
 type ueTransport struct {
 	transport http.RoundTripper
 	ue        *UserEntity
@@ -160,7 +167,7 @@ func (ue *UserEntity) Connect() (<-chan error, error) {
 	}
 
 	ue.wsEventChan = make(chan *model.WebSocketEvent)
-	ue.dataChan = make(chan any)
+	ue.dataChan = make(chan any, 10)
 	go ue.listen(ue.wsErrorChan)
 	ue.connected = true
 	return ue.wsErrorChan, nil