Merge branch 'release/v1.23.0'

Author: axllent

CHANGELOG.md

@@ -2,6 +2,28 @@
 
 Notable changes to Mailpit will be documented in this file.
 
+## [v1.23.0]
+
+### Feature
+- Add configuration to disable SQLite WAL mode for NFS compatibility
+- Add configuration to explicitly disable HTTP compression in web UI/API ([#448](https://github.com/axllent/mailpit/issues/448))
+- Add configuration to set message compression level in db (0-3) ([#447](https://github.com/axllent/mailpit/issues/447) & [#448](https://github.com/axllent/mailpit/issues/448))
+
+### Chore
+- Update node dependencies
+- Update Go dependencies
+- Minor speed & memory improvements when storing messages
+- Optimize ZSTD encoder for fastest compression of messages ([#447](https://github.com/axllent/mailpit/issues/447))
+- Handle BLOB storage for default database differently to rqlite to reduce memory overhead ([#447](https://github.com/axllent/mailpit/issues/447))
+- Avoid shell in Docker health check ([#444](https://github.com/axllent/mailpit/issues/444))
+
+### Fix
+- Display the correct STARTTLS or TLS runtime option on startup ([#446](https://github.com/axllent/mailpit/issues/446))
+
+### Testing
+- Add tests for message compression levels
+
+
 ## [v1.22.3]
 
 ### Feature

Dockerfile

@@ -25,6 +25,6 @@ RUN apk upgrade --no-cache && apk add --no-cache tzdata
 
 EXPOSE 1025/tcp 1110/tcp 8025/tcp
 
-HEALTHCHECK --interval=15s --start-period=10s --start-interval=1s CMD /mailpit readyz
+HEALTHCHECK --interval=15s --start-period=10s --start-interval=1s CMD ["/mailpit", "readyz"]
 
 ENTRYPOINT ["/mailpit"]

cmd/root.go

@@ -83,6 +83,8 @@ func init() {
 	initConfigFromEnv()
 
 	rootCmd.Flags().StringVarP(&config.Database, "database", "d", config.Database, "Database to store persistent data")
+	rootCmd.Flags().BoolVar(&config.DisableWAL, "disable-wal", config.DisableWAL, "Disable WAL for local database (allows NFS mounted DBs)")
+	rootCmd.Flags().IntVar(&config.Compression, "compression", config.Compression, "Compression level to store raw messages (0-3)")
 	rootCmd.Flags().StringVar(&config.Label, "label", config.Label, "Optional label identify this Mailpit instance")
 	rootCmd.Flags().StringVar(&config.TenantID, "tenant-id", config.TenantID, "Database tenant ID to isolate data")
 	rootCmd.Flags().IntVarP(&config.MaxMessages, "max", "m", config.MaxMessages, "Max number of messages to store")
@@ -103,6 +105,7 @@ func init() {
 	rootCmd.Flags().BoolVar(&config.BlockRemoteCSSAndFonts, "block-remote-css-and-fonts", config.BlockRemoteCSSAndFonts, "Block access to remote CSS & fonts")
 	rootCmd.Flags().StringVar(&config.EnableSpamAssassin, "enable-spamassassin", config.EnableSpamAssassin, "Enable integration with SpamAssassin")
 	rootCmd.Flags().BoolVar(&config.AllowUntrustedTLS, "allow-untrusted-tls", config.AllowUntrustedTLS, "Do not verify HTTPS certificates (link checker & screenshots)")
+	rootCmd.Flags().BoolVar(&config.DisableHTTPCompression, "disable-http-compression", config.DisableHTTPCompression, "Disable HTTP compression support (web UI & API)")
 
 	// SMTP server
 	rootCmd.Flags().StringVarP(&config.SMTPListen, "smtp", "s", config.SMTPListen, "SMTP bind interface and port")
@@ -181,6 +184,12 @@ func initConfigFromEnv() {
 		config.Database = os.Getenv("MP_DATABASE")
 	}
 
+	config.DisableWAL = getEnabledFromEnv("MP_DISABLE_WAL")
+
+	if len(os.Getenv("MP_COMPRESSION")) > 0 {
+		config.Compression, _ = strconv.Atoi(os.Getenv("MP_COMPRESSION"))
+	}
+
 	config.TenantID = os.Getenv("MP_TENANT_ID")
 
 	config.Label = os.Getenv("MP_LABEL")
@@ -232,6 +241,9 @@ func initConfigFromEnv() {
 	if getEnabledFromEnv("MP_ALLOW_UNTRUSTED_TLS") {
 		config.AllowUntrustedTLS = true
 	}
+	if getEnabledFromEnv("MP_DISABLE_HTTP_COMPRESSION") {
+		config.DisableHTTPCompression = true
+	}
 
 	// SMTP server
 	if len(os.Getenv("MP_SMTP_BIND_ADDR")) > 0 {

config/config.go

@@ -28,6 +28,14 @@ var (
 	// Database for mail (optional)
 	Database string
 
+	// DisableWAL will disable Write-Ahead Logging in SQLite
+	// @see https://sqlite.org/wal.html
+	DisableWAL bool
+
+	// Compression is the compression level used to store raw messages in the database:
+	// 0 = off, 1 = fastest (default), 2 = standard, 3 = best compression
+	Compression = 1
+
 	// TenantID is an optional prefix to be applied to all database tables,
 	// allowing multiple isolated instances of Mailpit to share a database.
 	TenantID string
@@ -61,6 +69,9 @@ var (
 	// Webroot to define the base path for the UI and API
 	Webroot = "/"
 
+	// DisableHTTPCompression will explicitly disable HTTP compression in the web UI and API
+	DisableHTTPCompression bool
+
 	// SMTPTLSCert file
 	SMTPTLSCert string
 
@@ -250,6 +261,10 @@ func VerifyConfig() error {
 		Database = filepath.Join(Database, "mailpit.db")
 	}
 
+	if Compression < 0 || Compression > 3 {
+		return errors.New("[db] compression level must be between 0 and 3")
+	}
+
 	Label = tools.Normalize(Label)
 
 	if err := parseMaxAge(); err != nil {

go.mod

@@ -12,22 +12,22 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/gorilla/websocket v1.5.3
 	github.com/jhillyerd/enmime v1.3.0
-	github.com/klauspost/compress v1.17.11
+	github.com/klauspost/compress v1.18.0
 	github.com/kovidgoyal/imaging v1.6.4
 	github.com/leporo/sqlf v1.4.0
 	github.com/lithammer/shortuuid/v4 v4.2.0
 	github.com/mneis/go-telnet v0.0.0-20221017141824-6f643e477c62
 	github.com/rqlite/gorqlite v0.0.0-20250128004930-114c7828b55a
 	github.com/sirupsen/logrus v1.9.3
-	github.com/spf13/cobra v1.9.0
+	github.com/spf13/cobra v1.9.1
 	github.com/spf13/pflag v1.0.6
 	github.com/tg123/go-htpasswd v1.2.3
 	github.com/vanng822/go-premailer v1.23.0
 	golang.org/x/net v0.35.0
 	golang.org/x/text v0.22.0
 	golang.org/x/time v0.10.0
 	gopkg.in/yaml.v3 v3.0.1
-	modernc.org/sqlite v1.35.0
+	modernc.org/sqlite v1.36.0
 )
 
 require (
@@ -52,8 +52,8 @@ require (
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/vanng822/css v1.0.1 // indirect
-	golang.org/x/crypto v0.33.0 // indirect
-	golang.org/x/exp v0.0.0-20250215185904-eff6e970281f // indirect
+	golang.org/x/crypto v0.35.0 // indirect
+	golang.org/x/exp v0.0.0-20250228200357-dead58393ab7 // indirect
 	golang.org/x/image v0.24.0 // indirect
 	golang.org/x/sys v0.30.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect

go.sum

@@ -43,8 +43,8 @@ github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQykt
 github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
 github.com/jhillyerd/enmime v1.3.0 h1:LV5kzfLidiOr8qRGIpYYmUZCnhrPbcFAnAFUnWn99rw=
 github.com/jhillyerd/enmime v1.3.0/go.mod h1:6c6jg5HdRRV2FtvVL69LjiX1M8oE0xDX9VEhV3oy4gs=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kovidgoyal/imaging v1.6.4 h1:K0idhRPXnRrJBKnBYcTfI1HTWSNDeAn7hYDvf9I0dCk=
 github.com/kovidgoyal/imaging v1.6.4/go.mod h1:bEIgsaZmXlvFfkv/CUxr9rJook6AQkJnpB5EPosRfRY=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -92,8 +92,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/spf13/cobra v1.9.0 h1:Py5fIuq/lJsRYxcxfOtsJqpmwJWCMOUy2tMJYV8TNHE=
-github.com/spf13/cobra v1.9.0/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
+github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
+github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
 github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
 github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
@@ -128,10 +128,10 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
-golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
-golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
-golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9BdpmEkvPyi5YrBGXbamg=
-golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
+golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
+golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
+golang.org/x/exp v0.0.0-20250228200357-dead58393ab7 h1:aWwlzYV971S4BXRS9AmqwDLAD85ouC6X+pocatKY58c=
+golang.org/x/exp v0.0.0-20250228200357-dead58393ab7/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
 golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ=
 golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -244,8 +244,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
 modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
 modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
 modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
-modernc.org/sqlite v1.35.0 h1:yQps4fegMnZFdphtzlfQTCNBWtS0CZv48pRpW3RFHRw=
-modernc.org/sqlite v1.35.0/go.mod h1:9cr2sicr7jIaWTBKQmAxQLfBv9LL0su4ZTEV+utt3ic=
+modernc.org/sqlite v1.36.0 h1:EQXNRn4nIS+gfsKeUTymHIz1waxuv5BzU7558dHSfH8=
+modernc.org/sqlite v1.36.0/go.mod h1:7MPwH7Z6bREicF9ZVUR78P1IKuxfZ8mRIDHD0iD+8TU=
 modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
 modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

internal/dump/dump.go

@@ -50,7 +50,7 @@ func Sync(d string) error {
 	}
 
 	if !tools.IsDir(outDir) {
-		if err := os.MkdirAll(outDir, 0755); err != nil {
+		if err := os.MkdirAll(outDir, 0755); /* #nosec */ err != nil {
 			return err
 		}
 	}
@@ -149,7 +149,7 @@ func saveMessages() error {
 			}
 		}
 
-		if err := os.WriteFile(out, b, 0644); err != nil {
+		if err := os.WriteFile(out, b, 0644); /* #nosec */ err != nil {
 			logger.Log().Errorf("Error writing message %s: %s", m.ID, err.Error())
 			continue
 		}

internal/smtpd/main.go

@@ -34,7 +34,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) (string
 
 // SaveToDatabase will attempt to save a message to the database
 func SaveToDatabase(origin net.Addr, from string, to []string, data []byte) (string, error) {
-	if !config.SMTPStrictRFCHeaders {
+	if !config.SMTPStrictRFCHeaders && bytes.Contains(data, []byte("\r\r\n")) {
 		// replace all <CR><CR><LF> (\r\r\n) with <CR><LF> (\r\n)
 		// @see https://github.com/axllent/mailpit/issues/87 & https://github.com/axllent/mailpit/issues/153
 		data = bytes.ReplaceAll(data, []byte("\r\r\n"), []byte("\r\n"))
@@ -50,21 +50,9 @@ func SaveToDatabase(origin net.Addr, from string, to []string, data []byte) (str
 	// check / set the Return-Path based on SMTP from
 	returnPath := strings.Trim(msg.Header.Get("Return-Path"), "<>")
 	if returnPath != from {
-		if returnPath != "" {
-			// replace Return-Path
-			re := regexp.MustCompile(`(?i)(^|\n)(Return\-Path: .*\n)`)
-			replaced := false
-			data = re.ReplaceAllFunc(data, func(r []byte) []byte {
-				if replaced {
-					return r
-				}
-				replaced = true // only replace first occurrence
-
-				return re.ReplaceAll(r, []byte("${1}Return-Path: <"+from+">\r\n"))
-			})
-		} else {
-			// add Return-Path
-			data = append([]byte("Return-Path: <"+from+">\r\n"), data...)
+		data, err = tools.SetMessageHeader(data, "Return-Path", "<"+from+">")
+		if err != nil {
+			return "", err
 		}
 	}
 
@@ -108,23 +96,15 @@ func SaveToDatabase(origin net.Addr, from string, to []string, data []byte) (str
 
 	// add missing email addresses to Bcc (eg: Laravel doesn't include these in the headers)
 	if len(missingAddresses) > 0 {
+		bccVal := strings.Join(missingAddresses, ", ")
 		if hasBccHeader {
-			// email already has Bcc header, add to existing addresses
-			re := regexp.MustCompile(`(?i)(^|\n)(Bcc: )`)
-			replaced := false
-			data = re.ReplaceAllFunc(data, func(r []byte) []byte {
-				if replaced {
-					return r
-				}
-				replaced = true // only replace first occurrence
-
-				return re.ReplaceAll(r, []byte("${1}Bcc: "+strings.Join(missingAddresses, ", ")+", "))
-			})
+			b := msg.Header.Get("Bcc")
+			bccVal = ", " + b
+		}
 
-		} else {
-			// prepend new Bcc header
-			bcc := []byte(fmt.Sprintf("Bcc: %s\r\n", strings.Join(missingAddresses, ", ")))
-			data = append(bcc, data...)
+		data, err = tools.SetMessageHeader(data, "Bcc", bccVal)
+		if err != nil {
+			return "", err
 		}
 
 		logger.Log().Debugf("[smtpd] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", "))
@@ -282,10 +262,10 @@ func listenAndServe(addr string, handler MsgIDHandler, authHandler AuthHandler)
 		smtpType := "no encryption"
 
 		if config.SMTPTLSCert != "" {
-			if config.SMTPRequireSTARTTLS {
-				smtpType = "STARTTLS required"
-			} else if config.SMTPRequireTLS {
+			if config.SMTPRequireTLS {
 				smtpType = "SSL/TLS required"
+			} else if config.SMTPRequireSTARTTLS {
+				smtpType = "STARTTLS required"
 			} else {
 				smtpType = "STARTTLS optional"
 				if !config.SMTPAuthAllowInsecure && auth.SMTPCredentials != nil {

internal/storage/database.go

@@ -32,19 +32,39 @@ var (
 	dbLastAction time.Time
 
 	// zstd compression encoder & decoder
-	dbEncoder, _ = zstd.NewWriter(nil)
+	dbEncoder    *zstd.Encoder
 	dbDecoder, _ = zstd.NewReader(nil)
 
 	temporaryFiles = []string{}
 )
 
 // InitDB will initialise the database
 func InitDB() error {
+	// dbEncoder
 	var (
 		dsn string
 		err error
 	)
 
+	if config.Compression > 0 {
+		var compression zstd.EncoderLevel
+		switch config.Compression {
+		case 1:
+			compression = zstd.SpeedFastest
+		case 2:
+			compression = zstd.SpeedDefault
+		case 3:
+			compression = zstd.SpeedBestCompression
+		}
+		dbEncoder, err = zstd.NewWriter(nil, zstd.WithEncoderLevel(compression))
+		if err != nil {
+			return err
+		}
+		logger.Log().Debugf("[db] storing messages with compression: %s", compression.String())
+	} else {
+		logger.Log().Debug("[db] storing messages with no compression")
+	}
+
 	p := config.Database
 
 	if p == "" {
@@ -100,8 +120,13 @@ func InitDB() error {
 	db.SetMaxOpenConns(1)
 
 	if sqlDriver == "sqlite" {
-		// SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
-		_, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;")
+		if config.DisableWAL {
+			// disable WAL mode for SQLite, allows NFS mounted DBs
+			_, err = db.Exec("PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL;")
+		} else {
+			// SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
+			_, err = db.Exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;")
+		}
 		if err != nil {
 			return err
 		}