feat(cli): Initial secrets implementation

cli/app.go

@@ -19,6 +19,7 @@ import (
  "github.com/kopia/kopia/internal/gather"
  "github.com/kopia/kopia/internal/passwordpersist"
  "github.com/kopia/kopia/internal/releasable"
+ "github.com/kopia/kopia/internal/secrets"
  "github.com/kopia/kopia/repo"
  "github.com/kopia/kopia/repo/blob"
  "github.com/kopia/kopia/repo/logging"
@@ -120,7 +121,7 @@ type App struct {
  initialUpdateCheckDelay time.Duration
  updateCheckInterval time.Duration
  updateAvailableNotifyInterval time.Duration
- password string
+ password *secrets.Secret
  configPath string
  traceStorage bool
  keyRingEnabled bool
@@ -252,7 +253,7 @@ func (c *App) setup(app *kingpin.Application) {
  app.Flag("config-file", "Specify the config file to use").Default("repository.config").Envar(c.EnvName("KOPIA_CONFIG_PATH")).StringVar(&c.configPath)
  app.Flag("trace-storage", "Enables tracing of storage operations.").Default("true").Hidden().BoolVar(&c.traceStorage)
  app.Flag("timezone", "Format time according to specified time zone (local, utc, original or time zone name)").Hidden().StringVar(&timeZone)
- app.Flag("password", "Repository password.").Envar(c.EnvName("KOPIA_PASSWORD")).Short('p').StringVar(&c.password)
+ secretVarWithEnv(app.Flag("password", "Repository password.").Short('p'), c.EnvName("KOPIA_PASSWORD"), &c.password)
  app.Flag("persist-credentials", "Persist credentials").Default("true").Envar(c.EnvName("KOPIA_PERSIST_CREDENTIALS_ON_CONNECT")).BoolVar(&c.persistCredentials)
  app.Flag("disable-internal-log", "Disable internal log").Hidden().Envar(c.EnvName("KOPIA_DISABLE_INTERNAL_LOG")).BoolVar(&c.disableInternalLog)
  app.Flag("advanced-commands", "Enable advanced (and potentially dangerous) commands.").Hidden().Envar(c.EnvName("KOPIA_ADVANCED_COMMANDS")).StringVar(&c.AdvancedCommands)

cli/command_repository_change_password.go

@@ -5,18 +5,19 @@ import (
 
  "github.com/pkg/errors"
 
+ "github.com/kopia/kopia/internal/secrets"
  "github.com/kopia/kopia/repo"
 )
 
 type commandRepositoryChangePassword struct {
- newPassword string
+ newPassword *secrets.Secret
 
  svc advancedAppServices
 }
 
 func (c *commandRepositoryChangePassword) setup(svc advancedAppServices, parent commandParent) {
  cmd := parent.Command("change-password", "Change repository password")
- cmd.Flag("new-password", "New password").Envar(svc.EnvName("KOPIA_NEW_PASSWORD")).StringVar(&c.newPassword)
+ secretVarWithEnv(cmd.Flag("new-password", "New password"), svc.EnvName("KOPIA_NEW_PASSWORD"), &c.newPassword)
 
  c.svc = svc
  cmd.Action(svc.directRepositoryWriteAction(c.run))
@@ -25,18 +26,19 @@ func (c *commandRepositoryChangePassword) setup(svc advancedAppServices, parent
 func (c *commandRepositoryChangePassword) run(ctx context.Context, rep repo.DirectRepositoryWriter) error {
  var newPass string
 
- if c.newPassword == "" {
+ if !c.newPassword.IsSet() {
  n, err := askForChangedRepositoryPassword(c.svc.stdout())
  if err != nil {
  return err
  }
 
  newPass = n
  } else {
- newPass = c.newPassword
+ _ = c.newPassword.Evaluate(nil, "")
+ newPass = c.newPassword.String()
  }
 
- if err := rep.FormatManager().ChangePassword(ctx, newPass); err != nil {
+ if err := rep.FormatManager().ChangePassword(ctx, newPass, rep); err != nil {
  return errors.Wrap(err, "unable to change password")
  }
 

cli/command_repository_change_password_test.go

@@ -3,6 +3,9 @@ package cli_test
 import (
  "testing"
 
+ "github.com/stretchr/testify/require"
+
+ "github.com/kopia/kopia/repo"
  "github.com/kopia/kopia/repo/format"
  "github.com/kopia/kopia/tests/testenv"
 )
@@ -28,8 +31,26 @@ func (s *formatSpecificTestSuite) TestRepositoryChangePassword(t *testing.T) {
  env2.RunAndExpectSuccess(t, "repo", "connect", "filesystem", "--path", env1.RepoDir, "--disable-repository-format-cache")
  env2.RunAndExpectSuccess(t, "snapshot", "ls")
 
+ // to test secret password-changes, artificially introduce a Secret token (which is not used for filesystem repos)
+ require.NoError(t, repo.AddSecretTokenForTest(env1.GetConfigFile(), testenv.TestRepoPassword))
+
+ lc, err := repo.LoadConfigFromFile(env1.GetConfigFile())
+ require.NoError(t, err)
+
+ encrypted, err := lc.SecretToken.Encrypt([]byte("secret"), testenv.TestRepoPassword)
+ require.NoError(t, err)
+
  env1.RunAndExpectSuccess(t, "repo", "change-password", "--new-password", "newPass")
 
+ // test that secrets use new password
+
+ lc, err = repo.LoadConfigFromFile(env1.GetConfigFile())
+ require.NoError(t, err)
+
+ decrypted, err := lc.SecretToken.Decrypt(encrypted, "newPass")
+ require.NoError(t, err)
+ require.Equal(t, string(decrypted), "secret")
+
  // at this point env2 stops working
  env2.RunAndExpectFailure(t, "snapshot", "ls")
 

cli/command_repository_connect.go

@@ -8,6 +8,7 @@ import (
  "github.com/pkg/errors"
 
  "github.com/kopia/kopia/internal/passwordpersist"
+ "github.com/kopia/kopia/internal/secrets"
  "github.com/kopia/kopia/repo"
  "github.com/kopia/kopia/repo/blob"
  "github.com/kopia/kopia/repo/content"
@@ -57,6 +58,8 @@ type connectOptions struct {
  connectPermissiveCacheLoading bool
  connectDescription string
  connectEnableActions bool
+ connectSecretAlgorithm string
+ connectSecretKeyDerivation string
 
  formatBlobCacheDuration time.Duration
  disableFormatBlobCache bool
@@ -81,6 +84,8 @@ func (c *connectOptions) setup(svc appServices, cmd *kingpin.CmdClause) {
  cmd.Flag("enable-actions", "Allow snapshot actions").BoolVar(&c.connectEnableActions)
  cmd.Flag("repository-format-cache-duration", "Duration of kopia.repository format blob cache").Hidden().DurationVar(&c.formatBlobCacheDuration)
  cmd.Flag("disable-repository-format-cache", "Disable caching of kopia.repository format blob").Hidden().BoolVar(&c.disableFormatBlobCache)
+ cmd.Flag("secret-algorithm", "Secret encryption algorithm").PlaceHolder("ALGO").Default(secrets.DefaultAlgorithm).EnumVar(&c.connectSecretAlgorithm, secrets.SupportedAlgorithms()...)
+ cmd.Flag("secret-key-derivation", "Secret key-derivation algorithm").PlaceHolder("ALGO").Default(secrets.DefaultKeyDerivation).EnumVar(&c.connectSecretKeyDerivation, secrets.SupportedKeyDerivations()...)
 }
 
 func (c *connectOptions) getFormatBlobCacheDuration() time.Duration {
@@ -112,6 +117,7 @@ func (c *connectOptions) toRepoConnectOptions() *repo.ConnectOptions {
  Description: c.connectDescription,
  EnableActions: c.connectEnableActions,
  FormatBlobCacheDuration: c.getFormatBlobCacheDuration(),
+ SecretToken: secrets.NewSigningKey(c.connectSecretAlgorithm, c.connectSecretKeyDerivation),
  },
  }
 }

cli/command_repository_status.go

@@ -183,6 +183,11 @@ func (c *commandRepositoryStatus) run(ctx context.Context, rep repo.Repository)
  c.out.printStdout("Unique ID: %x\n", dr.UniqueID())
  c.out.printStdout("Hash: %v\n", contentFormat.GetHashFunction())
  c.out.printStdout("Encryption: %v\n", contentFormat.GetEncryptionAlgorithm())
+
+ if rep.ClientOptions().SecretToken != nil {
+ c.out.printStdout("SecretAlgorithm: %v\n", rep.ClientOptions().SecretToken.Algorithm)
+ }
+
  c.out.printStdout("Splitter: %v\n", dr.ObjectFormat().Splitter)
  c.out.printStdout("Format version: %v\n", mp.Version)
  c.out.printStdout("Content compression: %v\n", mp.IndexVersion >= index.Version2)

cli/command_server.go

@@ -5,6 +5,7 @@ import (
  "github.com/pkg/errors"
 
  "github.com/kopia/kopia/internal/apiclient"
+ "github.com/kopia/kopia/internal/secrets"
 )
 
 type commandServer struct {
@@ -25,19 +26,19 @@ type commandServer struct {
 type serverFlags struct {
  serverAddress string
  serverUsername string
- serverPassword string
+ serverPassword *secrets.Secret
 }
 
 func (c *serverFlags) setup(svc appServices, cmd *kingpin.CmdClause) {
  cmd.Flag("address", "Server address").Default("http://127.0.0.1:51515").StringVar(&c.serverAddress)
  cmd.Flag("server-username", "HTTP server username (basic auth)").Envar(svc.EnvName("KOPIA_SERVER_USERNAME")).Default("kopia").StringVar(&c.serverUsername)
- cmd.Flag("server-password", "HTTP server password (basic auth)").Envar(svc.EnvName("KOPIA_SERVER_PASSWORD")).StringVar(&c.serverPassword)
+ secretVarWithEnv(cmd.Flag("server-password", "HTTP server password (basic auth)"), svc.EnvName("KOPIA_SERVER_PASSWORD"), &c.serverPassword)
 }
 
 type serverClientFlags struct {
  serverAddress string
  serverUsername string
- serverPassword string
+ serverPassword *secrets.Secret
  serverCertFingerprint string
 }
 
@@ -46,11 +47,11 @@ func (c *serverClientFlags) setup(svc appServices, cmd *kingpin.CmdClause) {
 
  cmd.Flag("address", "Address of the server to connect to").Envar(svc.EnvName("KOPIA_SERVER_ADDRESS")).Default("http://127.0.0.1:51515").StringVar(&c.serverAddress)
  cmd.Flag("server-control-username", "Server control username").Envar(svc.EnvName("KOPIA_SERVER_USERNAME")).StringVar(&c.serverUsername)
- cmd.Flag("server-control-password", "Server control password").PlaceHolder("PASSWORD").Envar(svc.EnvName("KOPIA_SERVER_PASSWORD")).StringVar(&c.serverPassword)
+ secretVarWithEnv(cmd.Flag("server-control-password", "Server control password").PlaceHolder("PASSWORD"), svc.EnvName("KOPIA_SERVER_PASSWORD"), &c.serverPassword)
 
  // aliases for backwards compat
  cmd.Flag("server-username", "Server control username").Hidden().StringVar(&c.serverUsername)
- cmd.Flag("server-password", "Server control password").Hidden().StringVar(&c.serverPassword)
+ secretVar(cmd.Flag("server-password", "Server control password").Hidden(), &c.serverPassword)
 
  cmd.Flag("server-cert-fingerprint", "Server certificate fingerprint").PlaceHolder("SHA256-FINGERPRINT").Envar(svc.EnvName("KOPIA_SERVER_CERT_FINGERPRINT")).StringVar(&c.serverCertFingerprint)
 }
@@ -79,10 +80,12 @@ func (c *serverClientFlags) serverAPIClientOptions() (apiclient.Options, error)
  return apiclient.Options{}, errors.Errorf("missing server address")
  }
 
+ _ = c.serverPassword.Evaluate(nil, "")
+
  return apiclient.Options{
  BaseURL: c.serverAddress,
  Username: c.serverUsername,
- Password: c.serverPassword,
+ Password: c.serverPassword.String(),
  TrustedServerCertificateFingerprint: c.serverCertFingerprint,
  }, nil
 }

cli/command_server_start.go

@@ -323,8 +323,10 @@ func (c *commandServerStart) getAuthenticator(ctx context.Context) (auth.Authent
 
  return nil, nil
 
- case c.sf.serverPassword != "":
- authenticators = append(authenticators, auth.AuthenticateSingleUser(c.sf.serverUsername, c.sf.serverPassword))
+ case c.sf.serverPassword.IsSet():
+ _ = c.sf.serverPassword.Evaluate(nil, "")
+
+ authenticators = append(authenticators, auth.AuthenticateSingleUser(c.sf.serverUsername, c.sf.serverPassword.String()))
 
  case c.serverStartRandomPassword:
  // generate very long random one-time password

cli/custom_types.go

@@ -0,0 +1,31 @@
+package cli
+
+import (
+ "github.com/alecthomas/kingpin/v2"
+
+ "github.com/kopia/kopia/internal/secrets"
+)
+
+// secretVar is called by kingpin to handle Secret arguments.
+func secretVar(s kingpin.Settings, target **secrets.Secret) {
+ if *target == nil {
+ secret := secrets.Secret{}
+ *target = &secret
+ }
+
+ s.SetValue(*target)
+}
+
+// secretVarWithEnv is called by kingpin to handle Secret arguments with a default environment variable.
+// Use this instead of kingpin's EnvName because it provides no limitations on the password value.
+func secretVarWithEnv(s kingpin.Settings, envvar string, target **secrets.Secret) {
+ if *target == nil {
+ secret := secrets.Secret{}
+ *target = &secret
+ }
+
+ (*target).Type = secrets.EnvVar
+ (*target).Input = envvar
+
+ s.SetValue(*target)
+}

cli/password.go

@@ -5,7 +5,6 @@ import (
  "fmt"
  "io"
  "os"
- "strings"
 
  "github.com/pkg/errors"
  "golang.org/x/term"
@@ -65,14 +64,16 @@ func askForExistingRepositoryPassword(out io.Writer) (string, error) {
 }
 
 func (c *App) setPasswordFromToken(pwd string) {
- c.password = pwd
+ _ = c.password.Set("plaintext:" + pwd)
 }
 
 func (c *App) getPasswordFromFlags(ctx context.Context, isCreate, allowPersistent bool) (string, error) {
  switch {
- case c.password != "":
+ case c.password.IsSet():
  // password provided via --password flag or KOPIA_PASSWORD environment variable
- return strings.TrimSpace(c.password), nil
+ _ = c.password.Evaluate(nil, "")
+
+ return c.password.String(), nil
  case isCreate:
  // this is a new repository, ask for password
  return askForNewRepositoryPassword(c.stdoutWriter)

cli/storage_azure.go

@@ -16,9 +16,9 @@ type storageAzureFlags struct {
 func (c *storageAzureFlags) Setup(svc StorageProviderServices, cmd *kingpin.CmdClause) {
  cmd.Flag("container", "Name of the Azure blob container").Required().StringVar(&c.azOptions.Container)
  cmd.Flag("storage-account", "Azure storage account name (overrides AZURE_STORAGE_ACCOUNT environment variable)").Required().Envar(svc.EnvName("AZURE_STORAGE_ACCOUNT")).StringVar(&c.azOptions.StorageAccount)
- cmd.Flag("storage-key", "Azure storage account key (overrides AZURE_STORAGE_KEY environment variable)").Envar(svc.EnvName("AZURE_STORAGE_KEY")).StringVar(&c.azOptions.StorageKey)
+ secretVarWithEnv(cmd.Flag("storage-key", "Azure storage account key (overrides AZURE_STORAGE_KEY environment variable)"), svc.EnvName("AZURE_STORAGE_KEY"), &c.azOptions.StorageKey)
  cmd.Flag("storage-domain", "Azure storage domain").Envar(svc.EnvName("AZURE_STORAGE_DOMAIN")).StringVar(&c.azOptions.StorageDomain)
- cmd.Flag("sas-token", "Azure SAS Token").Envar(svc.EnvName("AZURE_STORAGE_SAS_TOKEN")).StringVar(&c.azOptions.SASToken)
+ secretVarWithEnv(cmd.Flag("sas-token", "Azure SAS Token"), svc.EnvName("AZURE_STORAGE_SAS_TOKEN"), &c.azOptions.SASToken)
  cmd.Flag("prefix", "Prefix to use for objects in the bucket").StringVar(&c.azOptions.Prefix)
  cmd.Flag("tenant-id", "Azure service principle tenant ID (overrides AZURE_TENANT_ID environment variable)").Envar(svc.EnvName("AZURE_TENANT_ID")).StringVar(&c.azOptions.TenantID)
  cmd.Flag("client-id", "Azure service principle client ID (overrides AZURE_CLIENT_ID environment variable)").Envar(svc.EnvName("AZURE_CLIENT_ID")).StringVar(&c.azOptions.ClientID)

cli/storage_b2.go

@@ -4,6 +4,7 @@
  "context"
 
  "github.com/alecthomas/kingpin/v2"
+ "github.com/pkg/errors"
 
  "github.com/kopia/kopia/repo/blob"
  "github.com/kopia/kopia/repo/blob/b2"
@@ -16,14 +17,18 @@
 func (c *storageB2Flags) Setup(svc StorageProviderServices, cmd *kingpin.CmdClause) {
  cmd.Flag("bucket", "Name of the B2 bucket").Required().StringVar(&c.b2options.BucketName)
  cmd.Flag("key-id", "Key ID (overrides B2_KEY_ID environment variable)").Required().Envar(svc.EnvName("B2_KEY_ID")).StringVar(&c.b2options.KeyID)
- cmd.Flag("key", "Secret key (overrides B2_KEY environment variable)").Required().Envar(svc.EnvName("B2_KEY")).StringVar(&c.b2options.Key)
+ secretVarWithEnv(cmd.Flag("key", "Secret key (overrides B2_KEY environment variable)"), svc.EnvName("B2_KEY"), &c.b2options.Key)
  cmd.Flag("prefix", "Prefix to use for objects in the bucket").StringVar(&c.b2options.Prefix)
  commonThrottlingFlags(cmd, &c.b2options.Limits)
 }
 
 func (c *storageB2Flags) Connect(ctx context.Context, isCreate bool, formatVersion int) (blob.Storage, error) {
  _ = formatVersion
 
+ if !c.b2options.Key.IsSet() {
+ return nil, errors.New("Must specify secret key")
+ }
+
  //nolint:wrapcheck
  return b2.New(ctx, &c.b2options, isCreate)
 }

cli/storage_gcs.go

@@ -2,12 +2,12 @@
 
 import (
  "context"
- "encoding/json"
  "os"
 
  "github.com/alecthomas/kingpin/v2"
  "github.com/pkg/errors"
 
+ "github.com/kopia/kopia/internal/secrets"
  "github.com/kopia/kopia/repo/blob"
  "github.com/kopia/kopia/repo/blob/gcs"
 )
@@ -37,7 +37,7 @@
  return nil, errors.Wrap(err, "unable to open service account credentials file")
  }
 
- c.options.ServiceAccountCredentialJSON = json.RawMessage(data)
+ c.options.ServiceAccountCredentialJSON = secrets.NewSecret(string(data))
  c.options.ServiceAccountCredentialsFile = ""
  }
 

cli/storage_gdrive.go

@@ -2,12 +2,12 @@
 
 import (
  "context"
- "encoding/json"
  "os"
 
  "github.com/alecthomas/kingpin/v2"
  "github.com/pkg/errors"
 
+ "github.com/kopia/kopia/internal/secrets"
  "github.com/kopia/kopia/repo/blob"
  "github.com/kopia/kopia/repo/blob/gdrive"
 )
@@ -36,7 +36,7 @@
  return nil, errors.Wrap(err, "unable to open service account credentials file")
  }
 
- c.options.ServiceAccountCredentialJSON = json.RawMessage(data)
+ c.options.ServiceAccountCredentialJSON = secrets.NewSecret(string(data))
  c.options.ServiceAccountCredentialsFile = ""
  }
 

cli/storage_s3.go

@@ -24,8 +24,8 @@
  cmd.Flag("endpoint", "Endpoint to use").Default("s3.amazonaws.com").StringVar(&c.s3options.Endpoint)
  cmd.Flag("region", "S3 Region").Default("").StringVar(&c.s3options.Region)
  cmd.Flag("access-key", "Access key ID (overrides AWS_ACCESS_KEY_ID environment variable)").Required().Envar(svc.EnvName("AWS_ACCESS_KEY_ID")).StringVar(&c.s3options.AccessKeyID)
- cmd.Flag("secret-access-key", "Secret access key (overrides AWS_SECRET_ACCESS_KEY environment variable)").Required().Envar(svc.EnvName("AWS_SECRET_ACCESS_KEY")).StringVar(&c.s3options.SecretAccessKey)
- cmd.Flag("session-token", "Session token (overrides AWS_SESSION_TOKEN environment variable)").Envar(svc.EnvName("AWS_SESSION_TOKEN")).StringVar(&c.s3options.SessionToken)
+ secretVarWithEnv(cmd.Flag("secret-access-key", "Secret access key (overrides AWS_SECRET_ACCESS_KEY environment variable)"), svc.EnvName("AWS_SECRET_ACCESS_KEY"), &c.s3options.SecretAccessKey)
+ secretVarWithEnv(cmd.Flag("session-token", "Session token (overrides AWS_SESSION_TOKEN environment variable)"), svc.EnvName("AWS_SESSION_TOKEN"), &c.s3options.SessionToken)
  cmd.Flag("prefix", "Prefix to use for objects in the bucket. Put trailing slash (/) if you want to use prefix as directory. e.g my-backup-dir/ would put repository contents inside my-backup-dir directory").StringVar(&c.s3options.Prefix)
  cmd.Flag("disable-tls", "Disable TLS security (HTTPS)").BoolVar(&c.s3options.DoNotUseTLS)
  cmd.Flag("disable-tls-verification", "Disable TLS (HTTPS) certificate verification").BoolVar(&c.s3options.DoNotVerifyTLS)
@@ -82,6 +82,10 @@
 func (c *storageS3Flags) Connect(ctx context.Context, isCreate bool, formatVersion int) (blob.Storage, error) {
  _ = formatVersion
 
+ if !c.s3options.SecretAccessKey.IsSet() {
+ return nil, errors.New("secret-access-key must be set")
+ }
+
  if isCreate && c.s3options.PointInTime != nil && !c.s3options.PointInTime.IsZero() {
  return nil, errors.New("Cannot specify a 'point-in-time' option when creating a repository")
  }