Skip to content

Commit

Permalink
Added a graceful shutdown mechanism with a configurable timeout (#8)
Browse files Browse the repository at this point in the history
  • Loading branch information
weisdd authored Mar 2, 2022
1 parent ace4750 commit ba75d2f
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 54 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# CHANGELOG

## 0.6.0

- Key changes:
- Added a graceful shutdown mechanism with a configurable timeout.

## 0.5.0

- Key changes:
Expand Down
49 changes: 25 additions & 24 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,30 +31,31 @@ OIDC roles are expected to be present in `roles` within a jwt token.

### Environment variables

| Module | Variable | Default Value | Description |
| -------------------- | ---------------------- | ------------- | ------------------------------------------------------------ |
| **General settings** | | | |
| | `OPTIMIZE_EXPRESSIONS` | `true` | Whether to automatically optimize expressions for non-full access requests. [More details](https://pkg.go.dev/github.com/VictoriaMetrics/metricsql#Optimize) |
| | | | |
| **Logging** | | | |
| | `DEBUG` | `false` | Whether to print out debug log messages. |
| | | | |
| **HTTP Server** | | | |
| | `PORT` | `8080` | Port the web server will listen on. |
| | `READ_TIMEOUT` | `10s` | `ReadTimeout` covers the time from when the connection is accepted to when the request body is fully read (if you do read the body, otherwise to the end of the headers). [More details](https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/) |
| | `WRITE_TIMEOUT` | `10s` | `WriteTimeout` normally covers the time from the end of the request header read to the end of the response write (a.k.a. the lifetime of the ServeHTTP). [More details](https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/) |
| | | | |
| **Proxy** | | | |
| | `UPSTREAM_URL` | | Prometheus URL, e.g. `http://prometheus.microk8s.localhost`. |
| | `SAFE_MODE` | `true` | Whether to block requests to sensitive endpoints like `/api/v1/admin/tsdb`, `/api/v1/insert`. |
| | `SET_PROXY_HEADERS` | `false` | Whether to set proxy headers (`X-Forwarded-For`, `X-Forwarded-Proto`, `X-Forwarded-Host`). |
| | | | |
| **OIDC** | | | |
| | `ACL_PATH` | `./acl.yaml` | Path to a file with ACL definitions (OIDC role to namespace bindings). |
| | `OIDC_REALM_URL` | | OIDC Realm URL, e.g. `https://auth.microk8s.localhost/auth/realms/cicd` |
| | `OIDC_CLIENT_ID` | | OIDC Client ID (1*) |

(1*): since it's grafana who obtains jwt-tokens in the first place, the specified client id must also be present in the forwarded token (the `audience` field). To put it simply, better to use the same client id for both Grafana and LFGW.
| Module | Variable | Default Value | Description |
| -------------------- | --------------------------- | ------------- | ------------------------------------------------------------ |
| **General settings** | | | |
| | `OPTIMIZE_EXPRESSIONS` | `true` | Whether to automatically optimize expressions for non-full access requests. [More details](https://pkg.go.dev/github.com/VictoriaMetrics/metricsql#Optimize) |
| | | | |
| **Logging** | | | |
| | `DEBUG` | `false` | Whether to print out debug log messages. |
| | | | |
| **HTTP Server** | | | |
| | `PORT` | `8080` | Port the web server will listen on. |
| | `READ_TIMEOUT` | `10s` | `ReadTimeout` covers the time from when the connection is accepted to when the request body is fully read (if you do read the body, otherwise to the end of the headers). [More details](https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/) |
| | `WRITE_TIMEOUT` | `10s` | `WriteTimeout` normally covers the time from the end of the request header read to the end of the response write (a.k.a. the lifetime of the ServeHTTP). [More details](https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/) |
| | `GRACEFUL_SHUTDOWN_TIMEOUT` | `20s` | Maximum amount of time to wait for all connections to be closed. [More details](https://pkg.go.dev/net/http#Server.Shutdown) |
| | | | |
| **Proxy** | | | |
| | `UPSTREAM_URL` | | Prometheus URL, e.g. `http://prometheus.microk8s.localhost`. |
| | `SAFE_MODE` | `true` | Whether to block requests to sensitive endpoints like `/api/v1/admin/tsdb`, `/api/v1/insert`. |
| | `SET_PROXY_HEADERS` | `false` | Whether to set proxy headers (`X-Forwarded-For`, `X-Forwarded-Proto`, `X-Forwarded-Host`). |
| | | | |
| **OIDC** | | | |
| | `ACL_PATH` | `./acl.yaml` | Path to a file with ACL definitions (OIDC role to namespace bindings). |
| | `OIDC_REALM_URL` | | OIDC Realm URL, e.g. `https://auth.microk8s.localhost/auth/realms/cicd` |
| | `OIDC_CLIENT_ID` | | OIDC Client ID (1*) |

(1*): since it's grafana who obtains jwt-tokens in the first place, the specified client id must also be present in the forwarded token (the `aud` claim). To put it simply, better to use the same client id for both Grafana and LFGW.

### acl.yaml syntax

Expand Down
51 changes: 21 additions & 30 deletions cmd/lfgw/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,8 @@ package main

import (
"context"
"fmt"
"io"
"log"
"net/http"
"net/http/httputil"
"net/url"
"os"
Expand All @@ -18,23 +16,24 @@ import (
// Define an application struct to hold the application-wide dependencies for the
// web application.
type application struct {
errorLog *log.Logger
infoLog *log.Logger
debugLog *log.Logger
ACLMap ACLMap
proxy *httputil.ReverseProxy
verifier *oidc.IDTokenVerifier
Debug bool `env:"DEBUG" envDefault:"false"`
UpstreamURL *url.URL `env:"UPSTREAM_URL,required"`
OptimizeExpressions bool `env:"OPTIMIZE_EXPRESSIONS" envDefault:"true"`
SafeMode bool `env:"SAFE_MODE" envDefault:"true"`
SetProxyHeaders bool `env:"SET_PROXY_HEADERS" envDefefault:"false"`
ACLPath string `env:"ACL_PATH" envDefault:"./acl.yaml"`
OIDCRealmURL string `env:"OIDC_REALM_URL,required"`
OIDCClientID string `env:"OIDC_CLIENT_ID,required"`
Port int `env:"PORT" envDefault:"8080"`
ReadTimeout time.Duration `env:"READ_TIMEOUT" envDefault:"10s"`
WriteTimeout time.Duration `env:"WRITE_TIMEOUT" envDefault:"10s"`
errorLog *log.Logger
infoLog *log.Logger
debugLog *log.Logger
ACLMap ACLMap
proxy *httputil.ReverseProxy
verifier *oidc.IDTokenVerifier
Debug bool `env:"DEBUG" envDefault:"false"`
UpstreamURL *url.URL `env:"UPSTREAM_URL,required"`
OptimizeExpressions bool `env:"OPTIMIZE_EXPRESSIONS" envDefault:"true"`
SafeMode bool `env:"SAFE_MODE" envDefault:"true"`
SetProxyHeaders bool `env:"SET_PROXY_HEADERS" envDefefault:"false"`
ACLPath string `env:"ACL_PATH" envDefault:"./acl.yaml"`
OIDCRealmURL string `env:"OIDC_REALM_URL,required"`
OIDCClientID string `env:"OIDC_CLIENT_ID,required"`
Port int `env:"PORT" envDefault:"8080"`
ReadTimeout time.Duration `env:"READ_TIMEOUT" envDefault:"10s"`
WriteTimeout time.Duration `env:"WRITE_TIMEOUT" envDefault:"10s"`
GracefulShutdownTimeout time.Duration `env:"GRACEFUL_SHUTDOWN_TIMEOUT" envDefault:"20s"`
}

type contextKey string
Expand Down Expand Up @@ -85,16 +84,8 @@ func main() {
app.proxy.ErrorLog = app.errorLog
app.proxy.FlushInterval = time.Millisecond * 200

srv := &http.Server{
Addr: fmt.Sprintf(":%d", app.Port),
ErrorLog: app.errorLog,
Handler: app.routes(),
IdleTimeout: time.Minute,
ReadTimeout: app.ReadTimeout,
WriteTimeout: app.WriteTimeout,
err = app.serve()
if err != nil {
app.errorLog.Fatal(err)
}

app.infoLog.Printf("Starting server on %d", app.Port)
err = srv.ListenAndServe()
app.errorLog.Fatal(err)
}
60 changes: 60 additions & 0 deletions cmd/lfgw/server.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package main

import (
"context"
"errors"
"fmt"
"net/http"
"os"
"os/signal"
"syscall"
"time"
)

// serve starts a web server and ensures graceful shutdown
func (app *application) serve() error {
srv := &http.Server{
Addr: fmt.Sprintf(":%d", app.Port),
ErrorLog: app.errorLog,
Handler: app.routes(),
IdleTimeout: time.Minute,
ReadTimeout: app.ReadTimeout,
WriteTimeout: app.WriteTimeout,
}

shutdownError := make(chan error)

go func() {
quit := make(chan os.Signal, 1)
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
s := <-quit

app.infoLog.Printf("Caught %s signal, waiting for all connections to be closed within %s", s, app.GracefulShutdownTimeout)

ctx, cancel := context.WithTimeout(context.Background(), app.GracefulShutdownTimeout)
defer cancel()

err := srv.Shutdown(ctx)
if err != nil {
shutdownError <- err
}

shutdownError <- nil
}()

app.infoLog.Printf("Starting server on %d", app.Port)

err := srv.ListenAndServe()
if !errors.Is(err, http.ErrServerClosed) {
return err
}

err = <-shutdownError
if err != nil {
return err
}

app.infoLog.Print("Successfully stopped server")

return nil
}

0 comments on commit ba75d2f

Please sign in to comment.