Merge pull request #58 from bobh66/add_reconcile_rate

Add max-reconcile-rate and use latest tools

Makefile

@@ -30,14 +30,6 @@ fallthrough: submodules
  @echo Initial setup complete. Running make again . . .
  @make
 
-crds.clean:
- @$(INFO) cleaning generated CRDs
- @find package/crds -name *.yaml -exec sed -i.sed -e '1,2d' {} \; || $(FAIL)
- @find package/crds -name *.yaml.sed -delete || $(FAIL)
- @$(OK) cleaned generated CRDs
-
-generate.done: crds.clean
-
 # integration tests
 e2e.run: test-integration
 
@@ -75,7 +67,7 @@ dev-clean: $(KIND) $(KUBECTL)
  @$(INFO) Deleting kind cluster
  @$(KIND) delete cluster --name=$(PROJECT_NAME)-dev
 
-.PHONY: reviewable submodules fallthrough test-integration run crds.clean dev dev-clean
+.PHONY: reviewable submodules fallthrough test-integration run dev dev-clean
 
 # ====================================================================================
 # Special Targets

README.md

@@ -223,11 +223,14 @@ applyArgs, destroyArgs and planArgs will be added to these default arguments.
 * You must either use remote state or ensure the provider container's `/tf`
  directory is not lost. `provider-terraform` __does not persist state__;
  consider using the [Kubernetes] remote state backend.
-* If the module takes longer than the supplied `--timeout` to apply the
+* If the module takes longer than the value of `--timeout` (default is 20m) to apply the
  underlying `terraform` process will be killed. You will potentially lose state
- and leak resources.
+ and leak resources. The workspace lock will also likely be left in place and need to be manually removed
+ before the Workspace can be reconciled again.
 * The provider won't emit an event until _after_ it has successfully applied the
  Terraform module, which can take a long time.
-
+* Setting --max-reconcile-rate to a value greater than 1 will potentially cause the provider
+ to use up to the same number of CPUs. Add a resources section to the ControllerConfig to restrict
+ CPU usage as needed.
 [Kubernetes]: https://www.terraform.io/docs/language/settings/backends/kubernetes.html
 [git credentials store]: https://git-scm.com/docs/git-credential-store

apis/generate.go

@@ -23,7 +23,7 @@ limitations under the License.
 //go:generate rm -rf ../package/crds
 
 // Generate deepcopy methodsets and CRD manifests
-//go:generate go run -tags generate sigs.k8s.io/controller-tools/cmd/controller-gen object:headerFile=../hack/boilerplate.go.txt paths=./... crd:trivialVersions=true,crdVersions=v1 output:artifacts:config=../package/crds
+//go:generate go run -tags generate sigs.k8s.io/controller-tools/cmd/controller-gen object:headerFile=../hack/boilerplate.go.txt paths=./... crd:crdVersions=v1 output:artifacts:config=../package/crds
 
 // Generate crossplane-runtime methodsets (resource.Claim, etc)
 //go:generate go run -tags generate github.com/crossplane/crossplane-tools/cmd/angryjet generate-methodsets --header-file=../hack/boilerplate.go.txt ./...

cmd/provider/main.go

@@ -19,6 +19,13 @@ package main
 import (
  "os"
  "path/filepath"
+ "time"
+
+ "github.com/crossplane/crossplane-runtime/pkg/controller"
+ "github.com/crossplane/crossplane-runtime/pkg/feature"
+ "go.uber.org/zap/zapcore"
+
+ "k8s.io/client-go/tools/leaderelection/resourcelock"
 
  "gopkg.in/alecthomas/kingpin.v2"
  ctrl "sigs.k8s.io/controller-runtime"
@@ -28,21 +35,22 @@ import (
  "github.com/crossplane/crossplane-runtime/pkg/ratelimiter"
 
  "github.com/crossplane-contrib/provider-terraform/apis"
- "github.com/crossplane-contrib/provider-terraform/internal/controller"
+ workspace "github.com/crossplane-contrib/provider-terraform/internal/controller"
 )
 
 func main() {
  var (
- app = kingpin.New(filepath.Base(os.Args[0]), "Terraform support for Crossplane.").DefaultEnvars()
- debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool()
- syncInterval = app.Flag("sync", "Sync interval controls how often all resources will be double checked for drift.").Short('s').Default("1h").Duration()
- pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("1m").Duration()
- timeout = app.Flag("timeout", "Controls how long Terraform processes may run before they are killed.").Default("20m").Duration()
- leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool()
+ app = kingpin.New(filepath.Base(os.Args[0]), "Terraform support for Crossplane.").DefaultEnvars()
+ debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool()
+ syncInterval = app.Flag("sync", "Sync interval controls how often all resources will be double checked for drift.").Short('s').Default("1h").Duration()
+ pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("1m").Duration()
+ timeout = app.Flag("timeout", "Controls how long Terraform processes may run before they are killed.").Default("20m").Duration()
+ leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").Envar("LEADER_ELECTION").Bool()
+ maxReconcileRate = app.Flag("max-reconcile-rate", "The maximum number of concurrent reconciliation operations.").Default("1").Int()
  )
  kingpin.MustParse(app.Parse(os.Args[1:]))
 
- zl := zap.New(zap.UseDevMode(*debug))
+ zl := zap.New(zap.UseDevMode(*debug), UseISO8601())
  log := logging.NewLogrLogger(zl.WithName("provider-terraform"))
  if *debug {
  // The controller-runtime runs with a no-op logger by default. It is
@@ -56,15 +64,41 @@ func main() {
  cfg, err := ctrl.GetConfig()
  kingpin.FatalIfError(err, "Cannot get API server rest config")
 
- mgr, err := ctrl.NewManager(cfg, ctrl.Options{
- LeaderElection: *leaderElection,
- LeaderElectionID: "crossplane-leader-election-provider-terraform",
- SyncPeriod: syncInterval,
+ mgr, err := ctrl.NewManager(ratelimiter.LimitRESTConfig(cfg, *maxReconcileRate), ctrl.Options{
+ SyncPeriod: syncInterval,
+
+ // controller-runtime uses both ConfigMaps and Leases for leader
+ // election by default. Leases expire after 15 seconds, with a
+ // 10 second renewal deadline. We've observed leader loss due to
+ // renewal deadlines being exceeded when under high load - i.e.
+ // hundreds of reconciles per second and ~200rps to the API
+ // server. Switching to Leases only and longer leases appears to
+ // alleviate this.
+ LeaderElection: *leaderElection,
+ LeaderElectionID: "crossplane-leader-election-provider-template",
+ LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
+ LeaseDuration: func() *time.Duration { d := 60 * time.Second; return &d }(),
+ RenewDeadline: func() *time.Duration { d := 50 * time.Second; return &d }(),
  })
  kingpin.FatalIfError(err, "Cannot create controller manager")
 
- rl := ratelimiter.NewDefaultProviderRateLimiter(ratelimiter.DefaultProviderRPS)
  kingpin.FatalIfError(apis.AddToScheme(mgr.GetScheme()), "Cannot add terraform APIs to scheme")
- kingpin.FatalIfError(controller.Setup(mgr, log, rl, *pollInterval, *timeout), "Cannot setup terraform controllers")
+
+ o := controller.Options{
+ Logger: log,
+ MaxConcurrentReconciles: *maxReconcileRate,
+ PollInterval: *pollInterval,
+ GlobalRateLimiter: ratelimiter.NewGlobal(*maxReconcileRate),
+ Features: &feature.Flags{},
+ }
+
+ kingpin.FatalIfError(workspace.Setup(mgr, o, *timeout), "Cannot setup Workspace controllers")
  kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")
 }
+
+// UseISO8601 sets the logger to use ISO8601 timestamp format
+func UseISO8601() zap.Opts {
+ return func(o *zap.Options) {
+ o.TimeEncoder = zapcore.ISO8601TimeEncoder
+ }
+}

go.mod

@@ -3,97 +3,96 @@ module github.com/crossplane-contrib/provider-terraform
 go 1.17
 
 require (
- github.com/crossplane/crossplane-runtime v0.14.0
- github.com/crossplane/crossplane-tools v0.0.0-20201201125637-9ddc70edfd0d
- github.com/google/go-cmp v0.5.2
+ github.com/crossplane/crossplane-runtime v0.17.0
+ github.com/crossplane/crossplane-tools v0.0.0-20220310165030-1f43fc12793e
+ github.com/google/go-cmp v0.5.6
  github.com/google/uuid v1.1.2
  github.com/hashicorp/go-getter v1.4.0
  github.com/pkg/errors v0.9.1
- github.com/spf13/afero v1.5.1
+ github.com/spf13/afero v1.8.0
+ go.uber.org/zap v1.19.1
+ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
  gopkg.in/alecthomas/kingpin.v2 v2.2.6
- k8s.io/api v0.20.1
- k8s.io/apimachinery v0.20.1
- k8s.io/client-go v0.20.1
- sigs.k8s.io/controller-runtime v0.8.0
- sigs.k8s.io/controller-tools v0.3.0
+ k8s.io/api v0.23.0
+ k8s.io/apimachinery v0.23.0
+ k8s.io/client-go v0.23.0
+ sigs.k8s.io/controller-runtime v0.11.0
+ sigs.k8s.io/controller-tools v0.8.0
 )
 
 require (
- cloud.google.com/go v0.54.0 // indirect
- cloud.google.com/go/storage v1.6.0 // indirect
- github.com/BurntSushi/toml v0.3.1 // indirect
+ cloud.google.com/go v0.81.0 // indirect
+ cloud.google.com/go/storage v1.14.0 // indirect
  github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
- github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
+ github.com/alecthomas/units v0.0.0-20210912230133-d1bdfacee922 // indirect
  github.com/aws/aws-sdk-go v1.15.78 // indirect
  github.com/beorn7/perks v1.0.1 // indirect
  github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
  github.com/cespare/xxhash/v2 v2.1.1 // indirect
- github.com/dave/jennifer v1.3.0 // indirect
+ github.com/dave/jennifer v1.4.1 // indirect
  github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/evanphx/json-patch v4.9.0+incompatible // indirect
- github.com/fatih/color v1.7.0 // indirect
- github.com/fsnotify/fsnotify v1.4.9 // indirect
- github.com/go-logr/logr v0.3.0 // indirect
- github.com/go-logr/zapr v0.2.0 // indirect
- github.com/gobuffalo/flect v0.2.0 // indirect
- github.com/gogo/protobuf v1.3.1 // indirect
- github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
- github.com/golang/protobuf v1.4.3 // indirect
+ github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+ github.com/fatih/color v1.12.0 // indirect
+ github.com/fsnotify/fsnotify v1.5.1 // indirect
+ github.com/go-logr/logr v1.2.0 // indirect
+ github.com/go-logr/zapr v1.2.0 // indirect
+ github.com/gobuffalo/flect v0.2.3 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+ github.com/golang/protobuf v1.5.2 // indirect
  github.com/google/gofuzz v1.1.0 // indirect
  github.com/googleapis/gax-go/v2 v2.0.5 // indirect
- github.com/googleapis/gnostic v0.5.1 // indirect
- github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+ github.com/googleapis/gnostic v0.5.5 // indirect
+ github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
  github.com/hashicorp/go-safetemp v1.0.0 // indirect
- github.com/hashicorp/go-version v1.1.0 // indirect
- github.com/hashicorp/golang-lru v0.5.4 // indirect
- github.com/imdario/mergo v0.3.10 // indirect
+ github.com/hashicorp/go-version v1.2.0 // indirect
+ github.com/imdario/mergo v0.3.12 // indirect
  github.com/inconshreveable/mousetrap v1.0.0 // indirect
  github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect
- github.com/json-iterator/go v1.1.10 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
  github.com/jstemmer/go-junit-report v0.9.1 // indirect
- github.com/mattn/go-colorable v0.1.2 // indirect
- github.com/mattn/go-isatty v0.0.8 // indirect
+ github.com/mattn/go-colorable v0.1.8 // indirect
+ github.com/mattn/go-isatty v0.0.12 // indirect
  github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
  github.com/mitchellh/go-homedir v1.1.0 // indirect
  github.com/mitchellh/go-testing-interface v1.0.0 // indirect
  github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
- github.com/modern-go/reflect2 v1.0.1 // indirect
- github.com/prometheus/client_golang v1.7.1 // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/prometheus/client_golang v1.11.0 // indirect
  github.com/prometheus/client_model v0.2.0 // indirect
- github.com/prometheus/common v0.10.0 // indirect
- github.com/prometheus/procfs v0.2.0 // indirect
- github.com/spf13/cobra v1.1.1 // indirect
+ github.com/prometheus/common v0.28.0 // indirect
+ github.com/prometheus/procfs v0.6.0 // indirect
+ github.com/spf13/cobra v1.2.1 // indirect
  github.com/spf13/pflag v1.0.5 // indirect
  github.com/ulikunitz/xz v0.5.5 // indirect
- go.opencensus.io v0.22.3 // indirect
- go.uber.org/atomic v1.6.0 // indirect
- go.uber.org/multierr v1.5.0 // indirect
- go.uber.org/zap v1.15.0 // indirect
- golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
- golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
- golang.org/x/mod v0.3.0 // indirect
- golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect
- golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
- golang.org/x/sys v0.0.0-20201112073958-5cba982894dd // indirect
- golang.org/x/text v0.3.4 // indirect
- golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
- golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 // indirect
+ go.opencensus.io v0.23.0 // indirect
+ go.uber.org/atomic v1.9.0 // indirect
+ go.uber.org/multierr v1.6.0 // indirect
+ golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
+ golang.org/x/mod v0.4.2 // indirect
+ golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect
+ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+ golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect
+ golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
+ golang.org/x/text v0.3.7 // indirect
+ golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+ golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff // indirect
  golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
- gomodules.xyz/jsonpatch/v2 v2.1.0 // indirect
- google.golang.org/api v0.20.0 // indirect
- google.golang.org/appengine v1.6.6 // indirect
- google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
- google.golang.org/grpc v1.27.1 // indirect
- google.golang.org/protobuf v1.25.0 // indirect
+ gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
+ google.golang.org/api v0.44.0 // indirect
+ google.golang.org/appengine v1.6.7 // indirect
+ google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect
+ google.golang.org/grpc v1.41.0 // indirect
+ google.golang.org/protobuf v1.27.1 // indirect
  gopkg.in/inf.v0 v0.9.1 // indirect
- gopkg.in/yaml.v2 v2.3.0 // indirect
- gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
- honnef.co/go/tools v0.0.1-2020.1.3 // indirect
- k8s.io/apiextensions-apiserver v0.20.1 // indirect
- k8s.io/component-base v0.20.1 // indirect
- k8s.io/klog/v2 v2.4.0 // indirect
- k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd // indirect
- k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
- sigs.k8s.io/structured-merge-diff/v4 v4.0.2 // indirect
- sigs.k8s.io/yaml v1.2.0 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
+ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+ k8s.io/apiextensions-apiserver v0.23.0 // indirect
+ k8s.io/component-base v0.23.0 // indirect
+ k8s.io/klog/v2 v2.30.0 // indirect
+ k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+ k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
+ sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+ sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect
+ sigs.k8s.io/yaml v1.3.0 // indirect
 )