[WIP] Improve Tarmak Terraform provider

pkg/terraform/providers/tarmak/resource_vault_cluster.go

@@ -109,7 +109,6 @@ func resourceTarmakVaultClusterRead(d *schema.ResourceData, meta interface{}) (e
 
  log.Print("[DEBUG] calling rpc vault cluster init status")
  var reply tarmakRPC.VaultClusterStatusReply
- // TODO: verify that all Ensure operations have succeeded, not just initialisation
  err = client.Call(tarmakRPC.VaultClusterInitStatusCall, args, &reply)
  if err != nil {
  d.SetId("")

pkg/terraform/providers/tarmak/resource_vault_instance_role.go

@@ -70,10 +70,10 @@ func resourceTarmakVaultInstanceRoleCreate(d *schema.ResourceData, meta interfac
 
  log.Printf("[DEBUG] calling rpc vault instance role for role %s", roleName)
  var reply tarmakRPC.VaultInstanceRoleReply
- err = client.Call(tarmakRPC.VaultInstanceRole, args, &reply)
+ err = client.Call(tarmakRPC.VaultInstanceRoleCreate, args, &reply)
  if err != nil {
  d.SetId("")
- return fmt.Errorf("call to %s failed: %s", tarmakRPC.VaultInstanceRole, err)
+ return fmt.Errorf("call to %s failed: %s", tarmakRPC.VaultInstanceRoleCreate, err)
  }
 
  if err = d.Set("init_token", reply.InitToken); err != nil {
@@ -106,7 +106,7 @@ func resourceTarmakVaultInstanceRoleRead(d *schema.ResourceData, meta interface{
 
  log.Printf("[DEBUG] calling rpc vault instance role for role %s", roleName)
  var reply tarmakRPC.VaultInstanceRoleReply
- err = client.Call(tarmakRPC.VaultInstanceRole, args, &reply)
+ err = client.Call(tarmakRPC.VaultInstanceRoleRead, args, &reply)
  if err != nil {
  d.SetId("")
  return nil

pkg/terraform/providers/tarmak/rpc/bastion_instance_status.go

@@ -8,6 +8,10 @@ import (
  cluster "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1"
 )
 
+const (
+ bastionVerifyTimeoutSeconds = 120
+)
+
 var (
  BastionInstanceStatusCall = fmt.Sprintf("%s.BastionInstanceStatus", RPCName)
 )
@@ -29,17 +33,41 @@ func (r *tarmakRPC) BastionInstanceStatus(args *BastionInstanceStatusArgs, resul
  return nil
  }
 
- var err error
- for i := 1; i <= Retries; i++ {
- if err = r.cluster.Environment().VerifyBastionAvailable(); err != nil {
- r.tarmak.Log().Error(err)
- time.Sleep(time.Second)
- } else {
- break
+ // check if instance exists
+ instances, err := r.cluster.Environment().Provider().ListHosts(r.cluster.Environment().Hub())
+ if err != nil {
+ return fmt.Errorf("failed to list instances in hub: %s", err)
+ }
+ bastionExists := false
+ for _, instance := range instances {
+ for _, role := range instance.Roles() {
+ if role == cluster.InstancePoolTypeBastion {
+ bastionExists = true
+ }
  }
  }
- if err != nil {
- return fmt.Errorf("bastion instance is not ready: %s", err)
+ if !bastionExists {
+ return fmt.Errorf("bastion instance does not exist")
+ }
+
+ // verify bastion responsiveness
+ verifyChannel := make(chan bool)
+ go func() {
+ for {
+ if err := r.cluster.Environment().VerifyBastionAvailable(); err != nil {
+ r.tarmak.Log().Error(err)
+ time.Sleep(time.Second)
+ continue
+ }
+ verifyChannel <- true
+ return
+ }
+ }()
+
+ select {
+ case <-verifyChannel:
+ case <-time.After(bastionVerifyTimeoutSeconds * time.Second):
+ return fmt.Errorf("failed to verify bastion instance")
  }
 
  result.Status = "ready"

pkg/terraform/providers/tarmak/rpc/rpc.go

@@ -15,7 +15,6 @@ import (
 const (
  ConnectorSocket = "/tmp/tarmak-connector.sock"
  RPCName = "Tarmak"
- Retries = 60
 )
 
 type tarmakRPC struct {
@@ -38,7 +37,8 @@ type Tarmak interface {
  BastionInstanceStatus(*BastionInstanceStatusArgs, *BastionInstanceStatusReply) error
  VaultClusterStatus(*VaultClusterStatusArgs, *VaultClusterStatusReply) error
  VaultClusterInitStatus(*VaultClusterStatusArgs, *VaultClusterStatusReply) error
- VaultInstanceRole(*VaultInstanceRoleArgs, *VaultInstanceRoleReply) error
+ VaultInstanceRoleCreate(*VaultInstanceRoleArgs, *VaultInstanceRoleReply) error
+ VaultInstanceRoleRead(*VaultInstanceRoleArgs, *VaultInstanceRoleReply) error
  Ping(*PingArgs, *PingReply) error
  log() *logrus.Entry
 }

pkg/terraform/providers/tarmak/rpc/vault_cluster_status.go

@@ -3,7 +3,6 @@ package rpc
 
 import (
  "fmt"
- "time"
 
  "github.com/jetstack/vault-helper/pkg/kubernetes"
 
@@ -36,6 +35,7 @@ func (r *tarmakRPC) VaultClusterStatus(args *VaultClusterStatusArgs, result *Vau
 
  vault := r.cluster.Environment().Vault()
 
+ // initialise and unseal vault
  err := vault.VerifyInitFromFQDNs(args.VaultInternalFQDNs, args.VaultCA, args.VaultKMSKeyID, args.VaultUnsealKeyName)
  if err != nil {
  err = fmt.Errorf("failed to initialise vault cluster: %s", err)
@@ -78,6 +78,7 @@ func (r *tarmakRPC) VaultClusterStatus(args *VaultClusterStatusArgs, result *Vau
 func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result *VaultClusterStatusReply) error {
  r.tarmak.Log().Debug("received rpc vault cluster status")
 
+ // if destroying, return with unknown state
  if r.tarmak.Cluster().GetState() == cluster.StateDestroy {
  result.Status = "unknown"
  return nil
@@ -93,6 +94,7 @@ func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result
  }
  defer vaultTunnel.Stop()
 
+ // init vault client
  vaultClient := vaultTunnel.VaultClient()
 
  vaultRootToken, err := vault.RootToken()
@@ -104,27 +106,21 @@ func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result
 
  vaultClient.SetToken(vaultRootToken)
 
- up := false
- err = nil
- for i := 1; i <= Retries; i++ {
- up, err = vaultClient.Sys().InitStatus()
- if err != nil {
- time.Sleep(time.Second)
- continue
- }
- break
- }
+ // retrieve vault init status
+ up, err := vaultClient.Sys().InitStatus()
  if err != nil {
  err = fmt.Errorf("failed to retrieve init status: %s", err)
  r.tarmak.Log().Error(err)
  return err
  }
  if !up {
- err = fmt.Errorf("failed to initialised vault cluster")
+ err = fmt.Errorf("vault cluster is not initialised")
  r.tarmak.Log().Error(err)
  return err
  }
 
+ // TODO: verify that all Ensure operations have succeeded, not just initialisation
+
  result.Status = "ready"
  return nil
 }

pkg/terraform/providers/tarmak/rpc/vault_instance_role.go

@@ -10,7 +10,8 @@ import (
 )
 
 var (
- VaultInstanceRole = fmt.Sprintf("%s.VaultInstanceRole", RPCName)
+ VaultInstanceRoleCreate = fmt.Sprintf("%s.VaultInstanceRoleCreate", RPCName)
+ VaultInstanceRoleRead = fmt.Sprintf("%s.VaultInstanceRoleRead", RPCName)
 )
 
 type VaultInstanceRoleArgs struct {
@@ -25,8 +26,8 @@ type VaultInstanceRoleReply struct {
  InitToken string
 }
 
-func (r *tarmakRPC) VaultInstanceRole(args *VaultInstanceRoleArgs, result *VaultInstanceRoleReply) error {
- r.tarmak.Log().Debug("received rpc vault instance role")
+func (r *tarmakRPC) VaultInstanceRoleCreate(args *VaultInstanceRoleArgs, result *VaultInstanceRoleReply) error {
+ r.tarmak.Log().Debug("received rpc vault instance role create")
 
  if r.tarmak.Cluster().GetState() == cluster.StateDestroy {
  result.InitToken = ""
@@ -58,18 +59,74 @@ func (r *tarmakRPC) VaultInstanceRole(args *VaultInstanceRoleArgs, result *Vault
  k := kubernetes.New(vaultClient, r.tarmak.Log())
  k.SetClusterID(r.tarmak.Cluster().ClusterName())
 
- if err := k.Ensure(); err != nil {
- err = fmt.Errorf("vault cluster is not ready: %s", err)
+ initToken, err := k.NewInitToken(roleName)
+ if err != nil {
+ return fmt.Errorf("could not get init token for role %s: %s", roleName, err)
+ }
+
+ err = initToken.Ensure()
+ if err != nil {
+ return fmt.Errorf("could not ensure init token for role %s: %s", roleName, err)
+ }
+
+ initTokenString, err := initToken.InitToken()
+ if err != nil {
+ return fmt.Errorf("could not retrieve init token for role %s: %s", roleName, err)
+ }
+
+ result.InitToken = initTokenString
+
+ r.tarmak.Log().Debug(roleName, " init token ", initTokenString)
+
+ return nil
+}
+
+func (r *tarmakRPC) VaultInstanceRoleRead(args *VaultInstanceRoleArgs, result *VaultInstanceRoleReply) error {
+ r.tarmak.Log().Debug("received rpc vault instance role read")
+
+ if r.tarmak.Cluster().GetState() == cluster.StateDestroy {
+ result.InitToken = ""
+ return nil
+ }
+
+ roleName := args.RoleName
+
+ vault := r.cluster.Environment().Vault()
+ vaultTunnel, err := vault.TunnelFromFQDNs(args.VaultInternalFQDNs, args.VaultCA)
+ if err != nil {
+ err := fmt.Errorf("failed to create vault tunnel: %s", err)
+ r.tarmak.Log().Error(err)
+ return err
+ }
+ defer vaultTunnel.Stop()
+
+ vaultClient := vaultTunnel.VaultClient()
+
+ vaultRootToken, err := vault.RootToken()
+ if err != nil {
+ err := fmt.Errorf("failed to retrieve root token: %s", err)
  r.tarmak.Log().Error(err)
  return err
  }
 
- initTokens := k.InitTokens()
- initToken, ok := initTokens[roleName]
- if !ok {
+ vaultClient.SetToken(vaultRootToken)
+
+ k := kubernetes.New(vaultClient, r.tarmak.Log())
+ k.SetClusterID(r.tarmak.Cluster().ClusterName())
+
+ initToken, err := k.NewInitToken(roleName)
+ if err != nil {
  return fmt.Errorf("could not get init token for role %s: %s", roleName, err)
  }
 
- result.InitToken = initToken
+ initTokenString, err := initToken.InitToken()
+ if err != nil {
+ return fmt.Errorf("could not retrieve init token for role %s: %s", roleName, err)
+ }
+
+ result.InitToken = initTokenString
+
+ r.tarmak.Log().Debug(roleName, " init token ", initTokenString)
+
  return nil
 }