From a49d1429d4c895d7914b8f7cdfb247ca13115d42 Mon Sep 17 00:00:00 2001 From: Luke Addison Date: Tue, 3 Jul 2018 10:48:28 +0100 Subject: [PATCH] Check for existance of bastion instance and don't write init token during read operation --- .../tarmak/resource_vault_cluster.go | 1 - .../tarmak/resource_vault_instance_role.go | 6 +- .../tarmak/rpc/bastion_instance_status.go | 46 +++++++++--- pkg/terraform/providers/tarmak/rpc/rpc.go | 4 +- .../tarmak/rpc/vault_cluster_status.go | 20 ++--- .../tarmak/rpc/vault_instance_role.go | 75 ++++++++++++++++--- .../vault-helper/pkg/kubernetes/kubernetes.go | 71 +++++++++--------- 7 files changed, 150 insertions(+), 73 deletions(-) diff --git a/pkg/terraform/providers/tarmak/resource_vault_cluster.go b/pkg/terraform/providers/tarmak/resource_vault_cluster.go index ece053078e..ee7403e9b5 100644 --- a/pkg/terraform/providers/tarmak/resource_vault_cluster.go +++ b/pkg/terraform/providers/tarmak/resource_vault_cluster.go @@ -109,7 +109,6 @@ func resourceTarmakVaultClusterRead(d *schema.ResourceData, meta interface{}) (e log.Print("[DEBUG] calling rpc vault cluster init status") var reply tarmakRPC.VaultClusterStatusReply - // TODO: verify that all Ensure operations have succeeded, not just initialisation err = client.Call(tarmakRPC.VaultClusterInitStatusCall, args, &reply) if err != nil { d.SetId("") diff --git a/pkg/terraform/providers/tarmak/resource_vault_instance_role.go b/pkg/terraform/providers/tarmak/resource_vault_instance_role.go index 1e87efde90..9bdf190a21 100644 --- a/pkg/terraform/providers/tarmak/resource_vault_instance_role.go +++ b/pkg/terraform/providers/tarmak/resource_vault_instance_role.go @@ -70,10 +70,10 @@ func resourceTarmakVaultInstanceRoleCreate(d *schema.ResourceData, meta interfac log.Printf("[DEBUG] calling rpc vault instance role for role %s", roleName) var reply tarmakRPC.VaultInstanceRoleReply - err = client.Call(tarmakRPC.VaultInstanceRole, args, &reply) + err = client.Call(tarmakRPC.VaultInstanceRoleCreate, args, &reply) if err != nil { d.SetId("") - return fmt.Errorf("call to %s failed: %s", tarmakRPC.VaultInstanceRole, err) + return fmt.Errorf("call to %s failed: %s", tarmakRPC.VaultInstanceRoleCreate, err) } if err = d.Set("init_token", reply.InitToken); err != nil { @@ -106,7 +106,7 @@ func resourceTarmakVaultInstanceRoleRead(d *schema.ResourceData, meta interface{ log.Printf("[DEBUG] calling rpc vault instance role for role %s", roleName) var reply tarmakRPC.VaultInstanceRoleReply - err = client.Call(tarmakRPC.VaultInstanceRole, args, &reply) + err = client.Call(tarmakRPC.VaultInstanceRoleRead, args, &reply) if err != nil { d.SetId("") return nil diff --git a/pkg/terraform/providers/tarmak/rpc/bastion_instance_status.go b/pkg/terraform/providers/tarmak/rpc/bastion_instance_status.go index 359c741845..90de70253a 100644 --- a/pkg/terraform/providers/tarmak/rpc/bastion_instance_status.go +++ b/pkg/terraform/providers/tarmak/rpc/bastion_instance_status.go @@ -8,6 +8,10 @@ import ( cluster "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" ) +const ( + bastionVerifyTimeoutSeconds = 120 +) + var ( BastionInstanceStatusCall = fmt.Sprintf("%s.BastionInstanceStatus", RPCName) ) @@ -29,17 +33,41 @@ func (r *tarmakRPC) BastionInstanceStatus(args *BastionInstanceStatusArgs, resul return nil } - var err error - for i := 1; i <= Retries; i++ { - if err = r.cluster.Environment().VerifyBastionAvailable(); err != nil { - r.tarmak.Log().Error(err) - time.Sleep(time.Second) - } else { - break + // check if instance exists + instances, err := r.cluster.Environment().Provider().ListHosts(r.cluster.Environment().Hub()) + if err != nil { + return fmt.Errorf("failed to list instances in hub: %s", err) + } + bastionExists := false + for _, instance := range instances { + for _, role := range instance.Roles() { + if role == cluster.InstancePoolTypeBastion { + bastionExists = true + } } } - if err != nil { - return fmt.Errorf("bastion instance is not ready: %s", err) + if !bastionExists { + return fmt.Errorf("bastion instance does not exist") + } + + // verify bastion responsiveness + verifyChannel := make(chan bool) + go func() { + for { + if err := r.cluster.Environment().VerifyBastionAvailable(); err != nil { + r.tarmak.Log().Error(err) + time.Sleep(time.Second) + continue + } + verifyChannel <- true + return + } + }() + + select { + case <-verifyChannel: + case <-time.After(bastionVerifyTimeoutSeconds * time.Second): + return fmt.Errorf("failed to verify bastion instance") } result.Status = "ready" diff --git a/pkg/terraform/providers/tarmak/rpc/rpc.go b/pkg/terraform/providers/tarmak/rpc/rpc.go index 2f0d7f691d..b74aefff0c 100644 --- a/pkg/terraform/providers/tarmak/rpc/rpc.go +++ b/pkg/terraform/providers/tarmak/rpc/rpc.go @@ -15,7 +15,6 @@ import ( const ( ConnectorSocket = "/tmp/tarmak-connector.sock" RPCName = "Tarmak" - Retries = 60 ) type tarmakRPC struct { @@ -38,7 +37,8 @@ type Tarmak interface { BastionInstanceStatus(*BastionInstanceStatusArgs, *BastionInstanceStatusReply) error VaultClusterStatus(*VaultClusterStatusArgs, *VaultClusterStatusReply) error VaultClusterInitStatus(*VaultClusterStatusArgs, *VaultClusterStatusReply) error - VaultInstanceRole(*VaultInstanceRoleArgs, *VaultInstanceRoleReply) error + VaultInstanceRoleCreate(*VaultInstanceRoleArgs, *VaultInstanceRoleReply) error + VaultInstanceRoleRead(*VaultInstanceRoleArgs, *VaultInstanceRoleReply) error Ping(*PingArgs, *PingReply) error log() *logrus.Entry } diff --git a/pkg/terraform/providers/tarmak/rpc/vault_cluster_status.go b/pkg/terraform/providers/tarmak/rpc/vault_cluster_status.go index 4e1a0c1f86..d7f5255be4 100644 --- a/pkg/terraform/providers/tarmak/rpc/vault_cluster_status.go +++ b/pkg/terraform/providers/tarmak/rpc/vault_cluster_status.go @@ -3,7 +3,6 @@ package rpc import ( "fmt" - "time" "github.com/jetstack/vault-helper/pkg/kubernetes" @@ -36,6 +35,7 @@ func (r *tarmakRPC) VaultClusterStatus(args *VaultClusterStatusArgs, result *Vau vault := r.cluster.Environment().Vault() + // initialise and unseal vault err := vault.VerifyInitFromFQDNs(args.VaultInternalFQDNs, args.VaultCA, args.VaultKMSKeyID, args.VaultUnsealKeyName) if err != nil { err = fmt.Errorf("failed to initialise vault cluster: %s", err) @@ -78,6 +78,7 @@ func (r *tarmakRPC) VaultClusterStatus(args *VaultClusterStatusArgs, result *Vau func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result *VaultClusterStatusReply) error { r.tarmak.Log().Debug("received rpc vault cluster status") + // if destroying, return with unknown state if r.tarmak.Cluster().GetState() == cluster.StateDestroy { result.Status = "unknown" return nil @@ -93,6 +94,7 @@ func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result } defer vaultTunnel.Stop() + // init vault client vaultClient := vaultTunnel.VaultClient() vaultRootToken, err := vault.RootToken() @@ -104,27 +106,21 @@ func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result vaultClient.SetToken(vaultRootToken) - up := false - err = nil - for i := 1; i <= Retries; i++ { - up, err = vaultClient.Sys().InitStatus() - if err != nil { - time.Sleep(time.Second) - continue - } - break - } + // retrieve vault init status + up, err := vaultClient.Sys().InitStatus() if err != nil { err = fmt.Errorf("failed to retrieve init status: %s", err) r.tarmak.Log().Error(err) return err } if !up { - err = fmt.Errorf("failed to initialised vault cluster") + err = fmt.Errorf("vault cluster is not initialised") r.tarmak.Log().Error(err) return err } + // TODO: verify that all Ensure operations have succeeded, not just initialisation + result.Status = "ready" return nil } diff --git a/pkg/terraform/providers/tarmak/rpc/vault_instance_role.go b/pkg/terraform/providers/tarmak/rpc/vault_instance_role.go index cf91d1a811..419b7277d4 100644 --- a/pkg/terraform/providers/tarmak/rpc/vault_instance_role.go +++ b/pkg/terraform/providers/tarmak/rpc/vault_instance_role.go @@ -10,7 +10,8 @@ import ( ) var ( - VaultInstanceRole = fmt.Sprintf("%s.VaultInstanceRole", RPCName) + VaultInstanceRoleCreate = fmt.Sprintf("%s.VaultInstanceRoleCreate", RPCName) + VaultInstanceRoleRead = fmt.Sprintf("%s.VaultInstanceRoleRead", RPCName) ) type VaultInstanceRoleArgs struct { @@ -25,8 +26,8 @@ type VaultInstanceRoleReply struct { InitToken string } -func (r *tarmakRPC) VaultInstanceRole(args *VaultInstanceRoleArgs, result *VaultInstanceRoleReply) error { - r.tarmak.Log().Debug("received rpc vault instance role") +func (r *tarmakRPC) VaultInstanceRoleCreate(args *VaultInstanceRoleArgs, result *VaultInstanceRoleReply) error { + r.tarmak.Log().Debug("received rpc vault instance role create") if r.tarmak.Cluster().GetState() == cluster.StateDestroy { result.InitToken = "" @@ -58,18 +59,74 @@ func (r *tarmakRPC) VaultInstanceRole(args *VaultInstanceRoleArgs, result *Vault k := kubernetes.New(vaultClient, r.tarmak.Log()) k.SetClusterID(r.tarmak.Cluster().ClusterName()) - if err := k.Ensure(); err != nil { - err = fmt.Errorf("vault cluster is not ready: %s", err) + initToken, err := k.NewInitToken(roleName) + if err != nil { + return fmt.Errorf("could not get init token for role %s: %s", roleName, err) + } + + err = initToken.Ensure() + if err != nil { + return fmt.Errorf("could not ensure init token for role %s: %s", roleName, err) + } + + initTokenString, err := initToken.InitToken() + if err != nil { + return fmt.Errorf("could not retrieve init token for role %s: %s", roleName, err) + } + + result.InitToken = initTokenString + + r.tarmak.Log().Debug(roleName, " init token ", initTokenString) + + return nil +} + +func (r *tarmakRPC) VaultInstanceRoleRead(args *VaultInstanceRoleArgs, result *VaultInstanceRoleReply) error { + r.tarmak.Log().Debug("received rpc vault instance role read") + + if r.tarmak.Cluster().GetState() == cluster.StateDestroy { + result.InitToken = "" + return nil + } + + roleName := args.RoleName + + vault := r.cluster.Environment().Vault() + vaultTunnel, err := vault.TunnelFromFQDNs(args.VaultInternalFQDNs, args.VaultCA) + if err != nil { + err := fmt.Errorf("failed to create vault tunnel: %s", err) + r.tarmak.Log().Error(err) + return err + } + defer vaultTunnel.Stop() + + vaultClient := vaultTunnel.VaultClient() + + vaultRootToken, err := vault.RootToken() + if err != nil { + err := fmt.Errorf("failed to retrieve root token: %s", err) r.tarmak.Log().Error(err) return err } - initTokens := k.InitTokens() - initToken, ok := initTokens[roleName] - if !ok { + vaultClient.SetToken(vaultRootToken) + + k := kubernetes.New(vaultClient, r.tarmak.Log()) + k.SetClusterID(r.tarmak.Cluster().ClusterName()) + + initToken, err := k.NewInitToken(roleName) + if err != nil { return fmt.Errorf("could not get init token for role %s: %s", roleName, err) } - result.InitToken = initToken + initTokenString, err := initToken.InitToken() + if err != nil { + return fmt.Errorf("could not retrieve init token for role %s: %s", roleName, err) + } + + result.InitToken = initTokenString + + r.tarmak.Log().Debug(roleName, " init token ", initTokenString) + return nil } diff --git a/vendor/github.com/jetstack/vault-helper/pkg/kubernetes/kubernetes.go b/vendor/github.com/jetstack/vault-helper/pkg/kubernetes/kubernetes.go index 8acf823560..71a4373acb 100644 --- a/vendor/github.com/jetstack/vault-helper/pkg/kubernetes/kubernetes.go +++ b/vendor/github.com/jetstack/vault-helper/pkg/kubernetes/kubernetes.go @@ -250,11 +250,6 @@ func (k *Kubernetes) Ensure() error { result = multierror.Append(result, err) } - // setup init tokens - if err := k.ensureInitTokens(); err != nil { - result = multierror.Append(result, err) - } - return result.ErrorOrNil() } @@ -277,7 +272,11 @@ func (k *Kubernetes) EnsureDryRun() (bool, error) { } if len(k.initTokens) == 0 { - k.initTokens = k.NewInitTokens() + initTokens, err := k.NewInitTokens() + if err != nil { + return true, err + } + k.initTokens = initTokens } for _, b := range k.backends() { @@ -377,48 +376,46 @@ func GetMountByPath(vaultClient Vault, mountPath string) (*vault.MountOutput, er return nil, nil } -func (k *Kubernetes) NewInitToken(role, expected string, policies []string) *InitToken { +func (k *Kubernetes) NewInitToken(role string) (*InitToken, error) { + + var policies []string + var expected string + switch { + case role == "etcd": + expected = k.FlagInitTokens.Etcd + policies = []string{k.etcdPolicy().Name} + case role == "master": + expected = k.FlagInitTokens.Master + policies = []string{k.masterPolicy().Name, k.workerPolicy().Name} + case role == "worker": + expected = k.FlagInitTokens.Worker + policies = []string{k.workerPolicy().Name} + case role == "all": + expected = k.FlagInitTokens.All + policies = []string{k.etcdPolicy().Name, k.masterPolicy().Name, k.workerPolicy().Name} + default: + return nil, fmt.Errorf("role %s is not supported", role) + } + return &InitToken{ Role: role, Policies: policies, kubernetes: k, ExpectedToken: expected, - } + }, nil } -func (k *Kubernetes) NewInitTokens() []*InitToken { +func (k *Kubernetes) NewInitTokens() ([]*InitToken, error) { var initTokens []*InitToken - initTokens = append(initTokens, k.NewInitToken("etcd", k.FlagInitTokens.Etcd, []string{ - k.etcdPolicy().Name, - })) - initTokens = append(initTokens, k.NewInitToken("master", k.FlagInitTokens.Master, []string{ - k.masterPolicy().Name, - k.workerPolicy().Name, - })) - initTokens = append(initTokens, k.NewInitToken("worker", k.FlagInitTokens.Worker, []string{ - k.workerPolicy().Name, - })) - initTokens = append(initTokens, k.NewInitToken("all", k.FlagInitTokens.All, []string{ - k.etcdPolicy().Name, - k.masterPolicy().Name, - k.workerPolicy().Name, - })) - - return initTokens -} - -func (k *Kubernetes) ensureInitTokens() error { - var result *multierror.Error - - k.initTokens = append(k.initTokens, k.NewInitTokens()...) - - for _, initToken := range k.initTokens { - if err := initToken.Ensure(); err != nil { - result = multierror.Append(result, err) + for _, role := range []string{"etcd", "master", "worker", "all"} { + initToken, err := k.NewInitToken(role) + if err != nil { + return nil, fmt.Errorf("failed to create init tokens: %s", err) } + initTokens = append(initTokens, initToken) } - return result + return initTokens, nil } func (k *Kubernetes) ensureDryRunInitTokens() (bool, error) {