initial commit

Author: ganigeorgiev

.dockerignore

@@ -0,0 +1,19 @@
+/.vscode/
+.idea
+.DS_Store
+
+# plaintask todo files
+*.todo
+
+# generated markdown previews
+*.html
+
+# ignore generated pb_data
+pb_data/
+
+# binaries, deployment tools and configs
+app
+hop
+hop.yml
+fly
+fly.toml

.gitignore

@@ -0,0 +1,19 @@
+/.vscode/
+.idea
+.DS_Store
+
+# plaintask todo files
+*.todo
+
+# generated markdown previews
+*.html
+
+# ignore generated pb_data
+pb_data/
+
+# binaries, deployment tools and configs
+app
+hop
+hop.yml
+fly
+fly.toml

Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.20-alpine
+
+WORKDIR /pb
+
+COPY . .
+
+RUN go mod tidy
+
+RUN CGO_ENABLED=0 go build -o /pb/pocketbase
+
+EXPOSE 8090
+
+ADD docker_cmd.sh /docker_cmd.sh
+
+CMD /docker_cmd.sh

LICENSE.md

@@ -0,0 +1,17 @@
+The MIT License (MIT)
+Copyright (c) 2023 - present, Gani Georgiev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+and associated documentation files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or
+substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,99 @@
+PocketBase benchmarks
+======================================================================
+
+This is a test PocketBase application with various benchmarks to serve
+as a general idea of what you could expect from a basic PocketBase instance
+without extra performance tuning or deployment on expensive server.
+
+- [Results](#results)
+- [Takeaways and things we'll have to improve](#takeaways-and-things-well-have-to-improve)
+- [About the benchmarks](#about-the-benchmarks)
+
+
+## Results
+
+- **Hetzner CAX11 (2vCPU ARM64, 4GB RAM, €3.79)**
+    - [Pure Go results (default)](results/hetzner_cax11.md)
+    - [CGO results](results/hetzner_cax11_cgo.md)
+
+- **Hetzner CAX41 (16vCPU ARM64, 32GB RAM, €24.49)**
+    - [Pure Go results (default)](results/hetzner_cax41.md)
+    - [CGO results](results/hetzner_cax41_cgo.md)
+
+- **Hop.io (1vCPU, 512MB RAM, _free tier_)**
+    - [Pure Go results (default)](results/hop_free_tier.md)
+    _(fails the mixed `posts50k` tests most likely a result of some resources abuse protection)_
+
+- **Fly.io (1vCPU, 256MB RAM, _free tier_)**
+    - [Pure Go results (default)](results/hop_free_tier.md)
+    _(it requires swap to be configured to prevent OOM errors and restarts; fails the heavier `posts100k` tests most likely a result of some resources abuse protection)_
+
+_Keep in mind that the tests are performed on the same machine where the PocketBase instance is deployed so the app performance can be slightly affected by the benchmarks execution itself (most hosts providers have several protections in place and at the moment I don't have the time to create proper setup to run the tests from more than one IP._
+
+_There are several optimizations that can be done and the benchmarks will change in the future so that the tests can run as part of the development workflow to track regressions, but for now improving the overall PocketBase dev experience remains with higher priority._
+
+
+#### Takeaways and things we'll have to improve
+
+- The Go and JS (_will be available with v0.17.0+_) custom routes and app hooks perform almost the same for low and medium concurrency (this is because by default we have 100 prewarmed `goja.Runtime`).
+
+- At the moment there is no much difference in terms of query execution between the lower and higher spec Hetzner VMs (_probably because most of the operations are I/O bound_).
+
+- We could experiment with different SQLite `PRAGMA` options to see if there will be any significant difference since we currently use mostly the defaults (_eg. increasing `page_size`, `cache_size`, heap limits, etc._).
+
+- The default data DB connections limit (max:120, idle:20) could be changed to be dynamic based on the running hardware to improve the overall performance and reduce the memory usage.
+
+- With higher concurrency individual query performance degrades (_probably because the runtime has to do more work, there is context switching involved, locks, etc._) but still the overall requests completion is better for most of the times.
+
+- Basic API rules don't have significant impact on the performance. The performance starts to degrade when the rule condition requires a join with a large table (especially in case of a nested relation fields lookup). Joins over large datasets are slow even for simple queries like:
+
+    ```sql
+    SELECT COUNT(DISTINCT posts10k.rowid)
+    FROM posts10k
+    LEFT JOIN posts100k ON posts100k.author = posts10k.author
+    ```
+    Creating indexes for the relation fields can help speeding up the queries but further optimization techniques will have to be researched.
+
+- To prevent unnecessary `JOIN` statements we can implement internally a special condition that will treat single `relation` field statements like `rel.id = @request.auth.id` the same as `rel = @request.auth.id`.
+
+- The existing json column normalizations (eg. `CASE WHEN json_valid ...`) have some impact on the performance (~10%) and can be further optimized by removing them entirely and ensuring that the field values are always stored in the correct format before create/update persistence (either on app level or triggers).
+
+- The CGO driver for some queries in _large datasets_ can be ~1.5x-4x times faster.
+  Building with `CGO_ENABLED=1` is the easiest way to boost the query performance without changing your db structure or hardware but keep in mind that it complicates the cross-compilation.
+
+
+## About the benchmarks
+
+The application uses the `develop` branch of PocketBase.
+
+The test database is ~180MB and has the following collections structure:
+
+![db_erd](https://i.imgur.com/gYC8Qci.png)
+
+In order to emulate real usage scenarios as close as possible, the tests are grouped in several categories:
+
+- **create** - Tests the write (_insert_) HTTP API performance. It is also responsible for populating the test database.
+
+- **search** - Tests the list/search (aka. `getList()`) HTTP API performance.
+    It also contains scenarios with mixed list and update tests to observe how mixed read/write operations affects each other.
+
+- **custom** - Tests for custom Go and JS code (routes, middlewares, hooks, etc.).
+
+- **delete** - Test the delete HTTP API performance (including cascade delete).
+
+
+## Run the benchmarks
+
+To run the benchmarks locally or on your server, you can:
+
+0. _[Install Go 1.18+](https://go.dev/doc/install) (if you haven't already)_
+1. Clone/download the repo
+2. Run `GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build` (_https://go.dev/doc/install/source#environment_)
+3. Start the created executable by running `./app serve`.
+4. Navigate to `http:localhost:8090/benchmarks`
+
+_By default all tests are executed in the previously mentioned test categories order, but you can also
+specify which exact tests to run using the `/benchmarks?run=custom,delete` query parameter._
+
+The above will start the benchmarks in a new goroutine and once completed it will print its result to the terminal and in the `benchmarks` collection
+(_note that some of the tests are slow and it may take some time to complete; we write the test result as a collection record to workaround various host providers DDoS protections and restrictions like persistent connections limit, read/write timeouts, etc._).

benchmarks/bench.go

@@ -0,0 +1,81 @@
+package benchmarks
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+)
+
+type BenchResult struct {
+	Best      time.Duration
+	Worst     time.Duration
+	Completed time.Duration
+	Errors    []error
+}
+
+func (r BenchResult) String() string {
+	return fmt.Sprintf(
+		"```\n┌─ Best:      %s\n├─ Worst:     %s\n├─ Completed: %s\n└─ Errors:    %d\n```",
+		r.Best,
+		r.Worst,
+		r.Completed,
+		len(r.Errors),
+	)
+}
+
+// A negative concurrency indicates no limit
+// (aka. a go routine will be fired for each iteration).
+func bench(action func(i int) error, iterations int, concurrency int) (*BenchResult, error) {
+	if iterations < 1 {
+		return nil, errors.New("iterations must be >= 1")
+	}
+
+	totalStart := time.Now()
+
+	var errors []error
+
+	times := make([]time.Duration, iterations)
+
+	g := errgroup.Group{}
+	g.SetLimit(concurrency)
+
+	for i := 0; i < iterations; i++ {
+		i := i
+		g.Go(func() error {
+			start := time.Now()
+
+			if err := action(i); err != nil {
+				errors = append(errors, err)
+			}
+
+			times[i] = time.Since(start)
+
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+
+	result := &BenchResult{
+		Best:      times[0],
+		Worst:     times[0],
+		Completed: time.Since(totalStart),
+		Errors:    errors,
+	}
+
+	// find the best and worst time
+	for _, t := range times {
+		if t < result.Best {
+			result.Best = t
+		}
+		if t > result.Worst {
+			result.Worst = t
+		}
+	}
+
+	return result, nil
+}

benchmarks/request.go

@@ -0,0 +1,80 @@
+package benchmarks
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"time"
+)
+
+type Request struct {
+	Method  string
+	Url     string
+	Headers map[string]string
+	Body    io.Reader
+	Context context.Context // default to context.Background()
+}
+
+// If destBody is non-nil, it will read and unmarshal the request
+// response body into the specified variable.
+func (c *Request) Send(destBody any) error {
+	if c.Context == nil {
+		c.Context = context.Background()
+	}
+
+	req, err := http.NewRequestWithContext(c.Context, c.Method, c.Url, c.Body)
+	if err != nil {
+		return err
+	}
+
+	for k, v := range c.Headers {
+		req.Header.Add(k, v)
+	}
+
+	// set default content-type header (if missing)
+	if req.Header.Get("content-type") == "" {
+		req.Header.Set("content-type", "application/json")
+	}
+
+	dialer := &net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+	}
+
+	client := http.Client{
+		Transport: &http.Transport{
+			Proxy:                 http.ProxyFromEnvironment,
+			DialContext:           dialer.DialContext,
+			MaxIdleConns:          0,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode >= 400 {
+		return fmt.Errorf("request failed with status %d", res.StatusCode)
+	}
+
+	if destBody != nil {
+		bodyRaw, err := io.ReadAll(res.Body)
+		if err != nil {
+			return err
+		}
+
+		if err := json.Unmarshal(bodyRaw, destBody); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}