Skip to content

Commit

Permalink
merge main
Browse files Browse the repository at this point in the history
  • Loading branch information
max-hoffman committed Nov 22, 2024
2 parents 323870d + 4ad1957 commit 3428267
Show file tree
Hide file tree
Showing 26 changed files with 320 additions and 202 deletions.
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -713,7 +713,7 @@ func printStagedDiffs(wr io.Writer, stagedRows []sql.Row, printHelp bool) int {

lines := make([]string, 0, len(stagedRows))
for _, row := range stagedRows {
if !doltdb.IsReadOnlySystemTable(row.GetValue(0).(string)) {
if !doltdb.IsReadOnlySystemTable(doltdb.TableName{Name: row.GetValue(0).(string)}) {
switch row.GetValue(1).(string) {
case "new table":
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[diff.AddedTable], row.GetValue(0).(string)))
Expand Down
4 changes: 2 additions & 2 deletions go/cmd/dolt/commands/stashcmds/pop.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,15 +169,15 @@ func applyStashAtIdx(ctx *sql.Context, dEnv *env.DoltEnv, curWorkingRoot doltdb.
return false, err
}

var tablesWithConflict []string
var tablesWithConflict []doltdb.TableName
for tbl, stats := range result.Stats {
if stats.HasConflicts() {
tablesWithConflict = append(tablesWithConflict, tbl)
}
}

if len(tablesWithConflict) > 0 {
tblNames := strings.Join(tablesWithConflict, "', '")
tblNames := strings.Join(doltdb.FlattenTableNames(tablesWithConflict), "', '")
cli.Printf("error: Your local changes to the following tables would be overwritten by applying stash %d:\n"+
"\t{'%s'}\n"+
"Please commit your changes or stash them before you merge.\nAborting\n", idx, tblNames)
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ and have %v and %v different commits each, respectively.
cli.Println(stagedHeader)
cli.Println(stagedHeaderHelp)
for tableName, status := range data.stagedTables {
if !doltdb.IsReadOnlySystemTable(tableName) {
if !doltdb.IsReadOnlySystemTable(doltdb.TableName{Name: tableName}) {
text := fmt.Sprintf(statusFmt, status+":", tableName)
greenText := color.GreenString(text)
cli.Println(greenText)
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/tblcmds/rm.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func (cmd RmCmd) Exec(ctx context.Context, commandStr string, args []string, dEn
}

for _, tableName := range apr.Args {
if doltdb.IsReadOnlySystemTable(tableName) {
if doltdb.IsReadOnlySystemTable(doltdb.TableName{Name: tableName}) {
return commands.HandleVErrAndExitCode(
errhand.BuildDError("error removing table %s", tableName).AddCause(doltdb.ErrSystemTableCannotBeModified).Build(), usage)
}
Expand Down
5 changes: 3 additions & 2 deletions go/cmd/dolt/commands/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/go-mysql-server/sql/mysql_db"
"github.com/dolthub/vitess/go/mysql"
"github.com/fatih/color"
"github.com/gocraft/dbr/v2"
"github.com/gocraft/dbr/v2/dialect"
Expand Down Expand Up @@ -265,7 +266,7 @@ func newLateBindingEngine(
sqlCtx.SetCurrentDatabase(database)

rawDb := se.GetUnderlyingEngine().Analyzer.Catalog.MySQLDb
salt, err := rawDb.Salt()
salt, err := mysql.NewSalt()
if err != nil {
return nil, nil, nil, err
}
Expand Down Expand Up @@ -500,7 +501,7 @@ func buildAuthResponse(salt []byte, password string) []byte {
}

func ValidatePasswordWithAuthResponse(rawDb *mysql_db.MySQLDb, user, password string) error {
salt, err := rawDb.Salt()
salt, err := mysql.NewSalt()
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/doltversion/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@
package doltversion

const (
Version = "1.43.15"
Version = "1.43.16"
)
2 changes: 2 additions & 0 deletions go/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662 h1:aC17hZD6iwzBwwfO5M+3oBT5E5gGRiQPdn+vzpDXqIA=
github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
github.com/dolthub/go-mysql-server v0.18.2-0.20241122190136-dd8defd838e3 h1:bvSE64pO6euDX8j5hpml5qVVz9OXG3hVV9532bn+eZ0=
github.com/dolthub/go-mysql-server v0.18.2-0.20241122190136-dd8defd838e3/go.mod h1:2mA/v84EOCe8TQIKR8TN8ZRIQSbOqThGQHyevGRmawU=
github.com/dolthub/go-mysql-server v0.18.2-0.20241122200945-194ff9004d54 h1:mBN1ol/WEzmq0TsGjLKiE2DIRQMA6JAf8pq6bVBzExE=
github.com/dolthub/go-mysql-server v0.18.2-0.20241122200945-194ff9004d54/go.mod h1:2mA/v84EOCe8TQIKR8TN8ZRIQSbOqThGQHyevGRmawU=
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
Expand Down
9 changes: 4 additions & 5 deletions go/libraries/doltcore/cherry_pick/cherry_pick.go
Original file line number Diff line number Diff line change
Expand Up @@ -391,16 +391,16 @@ func rootsEqual(root1, root2 doltdb.RootValue) (bool, error) {

// stageCherryPickedTables stages the tables from |mergeStats| that don't have any merge artifacts – i.e.
// tables that don't have any data or schema conflicts and don't have any constraint violations.
func stageCherryPickedTables(ctx *sql.Context, mergeStats map[string]*merge.MergeStats) (err error) {
tablesToAdd := make([]string, 0, len(mergeStats))
func stageCherryPickedTables(ctx *sql.Context, mergeStats map[doltdb.TableName]*merge.MergeStats) (err error) {
tablesToAdd := make([]doltdb.TableName, 0, len(mergeStats))
for tableName, mergeStats := range mergeStats {
if mergeStats.HasArtifacts() {
continue
}

// Find any tables being deleted and make sure we stage those tables first
if mergeStats.Operation == merge.TableRemoved {
tablesToAdd = append([]string{tableName}, tablesToAdd...)
tablesToAdd = append([]doltdb.TableName{tableName}, tablesToAdd...)
} else {
tablesToAdd = append(tablesToAdd, tableName)
}
Expand All @@ -413,8 +413,7 @@ func stageCherryPickedTables(ctx *sql.Context, mergeStats map[string]*merge.Merg
return fmt.Errorf("unable to get roots for database '%s' from session", dbName)
}

// TODO: schema name
roots, err = actions.StageTables(ctx, roots, doltdb.ToTableNames(tablesToAdd, doltdb.DefaultSchemaName), true)
roots, err = actions.StageTables(ctx, roots, tablesToAdd, true)
if err != nil {
return err
}
Expand Down
20 changes: 19 additions & 1 deletion go/libraries/doltcore/diff/table_deltas.go
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,21 @@ func (td TableDelta) GetRowData(ctx context.Context) (from, to durable.Index, er
return from, to, nil
}

// GetUniqueSchemaNamesFromTableNames returns a list of unique schema names from a list of table names
func GetUniqueSchemaNamesFromTableDeltas(deltas []TableDelta) []string {
schemaMap := make(map[string]struct{})
var schemas []string

for _, td := range deltas {
if _, exists := schemaMap[td.ToName.Schema]; !exists {
schemaMap[td.ToName.Schema] = struct{}{}
schemas = append(schemas, td.ToName.Schema)
}
}

return schemas
}

// WorkingSetContainsOnlyIgnoredTables returns true if all changes in working set are ignored tables.
// Otherwise, if there are any non-ignored changes, returns false.
// Note that only unstaged tables are subject to dolt_ignore (this is consistent with what git does.)
Expand All @@ -638,7 +653,8 @@ func WorkingSetContainsOnlyIgnoredTables(ctx context.Context, roots doltdb.Roots
return false, nil
}

ignorePatterns, err := doltdb.GetIgnoredTablePatterns(ctx, roots)
schemas := GetUniqueSchemaNamesFromTableDeltas(unstaged)
ignorePatternMap, err := doltdb.GetIgnoredTablePatterns(ctx, roots, schemas)
if err != nil {
return false, err
}
Expand All @@ -647,6 +663,8 @@ func WorkingSetContainsOnlyIgnoredTables(ctx context.Context, roots doltdb.Roots
if !(tableDelta.IsAdd()) {
return false, nil
}

ignorePatterns := ignorePatternMap[tableDelta.ToName.Schema]
isIgnored, err := ignorePatterns.IsTableNameIgnored(tableDelta.ToName)
if err != nil {
return false, err
Expand Down
122 changes: 76 additions & 46 deletions go/libraries/doltcore/doltdb/ignore.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,79 +54,102 @@ const (

type IgnorePatterns []IgnorePattern

func GetIgnoredTablePatterns(ctx context.Context, roots Roots) (IgnorePatterns, error) {
var ignorePatterns []IgnorePattern
workingSet := roots.Working
table, found, err := workingSet.GetTable(ctx, TableName{Name: IgnoreTableName})
if err != nil {
return nil, err
}
if !found {
// dolt_ignore doesn't exist, so don't filter any tables.
return ignorePatterns, nil
}
index, err := table.GetRowData(ctx)
if table.Format() == types.Format_LD_1 {
// dolt_ignore is not supported for the legacy storage format.
return ignorePatterns, nil
}
if err != nil {
return nil, err
}
ignoreTableSchema, err := table.GetSchema(ctx)
if err != nil {
return nil, err
}
keyDesc, valueDesc := ignoreTableSchema.GetMapDescriptors()
// ConvertTupleToIgnoreBoolean is a function that converts a Tuple to a boolean for the ignore field. This is used to handle the Doltgres extended boolean type.
var ConvertTupleToIgnoreBoolean = convertTupleToIgnoreBoolean

if !keyDesc.Equals(val.NewTupleDescriptor(val.Type{Enc: val.StringEnc})) {
return nil, fmt.Errorf("dolt_ignore had unexpected key type, this should never happen")
func convertTupleToIgnoreBoolean(valueDesc val.TupleDesc, valueTuple val.Tuple) (bool, error) {
if !valueDesc.Equals(val.NewTupleDescriptor(val.Type{Enc: val.Int8Enc, Nullable: false})) {
return false, fmt.Errorf("dolt_ignore had unexpected value type, this should never happen")
}
if !valueDesc.Equals(val.NewTupleDescriptor(val.Type{Enc: val.Int8Enc, Nullable: true})) {
return nil, fmt.Errorf("dolt_ignore had unexpected value type, this should never happen")
ignore, ok := valueDesc.GetBool(0, valueTuple)
if !ok {
return false, fmt.Errorf("could not read boolean")
}
return ignore, nil
}

ignoreTableMap, err := durable.ProllyMapFromIndex(index).IterAll(ctx)
if err != nil {
return nil, err
}
for {
keyTuple, valueTuple, err := ignoreTableMap.Next(ctx)
if err == io.EOF {
break
func GetIgnoredTablePatterns(ctx context.Context, roots Roots, schemas []string) (map[string]IgnorePatterns, error) {
ignorePatternsForSchemas := make(map[string]IgnorePatterns)
workingSet := roots.Working

for _, schemaName := range schemas {
var ignorePatterns []IgnorePattern

tname := TableName{Name: IgnoreTableName, Schema: schemaName}
table, found, err := workingSet.GetTable(ctx, tname)
if err != nil {
return nil, err
}
if !found {
// dolt_ignore doesn't exist, so don't filter any tables.
continue
}
index, err := table.GetRowData(ctx)
if table.Format() == types.Format_LD_1 {
// dolt_ignore is not supported for the legacy storage format.
continue
}
if err != nil {
return nil, err
}
ignoreTableSchema, err := table.GetSchema(ctx)
if err != nil {
return nil, err
}
keyDesc, valueDesc := ignoreTableSchema.GetMapDescriptors()

if !keyDesc.Equals(val.NewTupleDescriptor(val.Type{Enc: val.StringEnc})) {
return nil, fmt.Errorf("dolt_ignore had unexpected key type, this should never happen")
}

ignoreTableMap, err := durable.ProllyMapFromIndex(index).IterAll(ctx)
if err != nil {
return nil, err
}
for {
keyTuple, valueTuple, err := ignoreTableMap.Next(ctx)
if err == io.EOF {
break
}
if err != nil {
return nil, err
}

pattern, ok := keyDesc.GetString(0, keyTuple)
if !ok {
return nil, fmt.Errorf("could not read pattern")
pattern, ok := keyDesc.GetString(0, keyTuple)
if !ok {
return nil, fmt.Errorf("could not read pattern")
}
ignore, err := ConvertTupleToIgnoreBoolean(valueDesc, valueTuple)
if err != nil {
return nil, err
}
ignorePatterns = append(ignorePatterns, NewIgnorePattern(pattern, ignore))
}
ignore, ok := valueDesc.GetBool(0, valueTuple)
ignorePatterns = append(ignorePatterns, NewIgnorePattern(pattern, ignore))

ignorePatternsForSchemas[schemaName] = ignorePatterns
}
return ignorePatterns, nil

return ignorePatternsForSchemas, nil
}

// ExcludeIgnoredTables takes a list of table names and removes any tables that should be ignored,
// as determined by the patterns in the dolt_ignore table.
// The ignore patterns are read from the dolt_ignore table in the working set.
func ExcludeIgnoredTables(ctx context.Context, roots Roots, tables []TableName) ([]TableName, error) {
ignorePatterns, err := GetIgnoredTablePatterns(ctx, roots)
schemas := GetUniqueSchemaNamesFromTableNames(tables)
ignorePatternMap, err := GetIgnoredTablePatterns(ctx, roots, schemas)
if err != nil {
return nil, err
}
filteredTables := []TableName{}
for _, tbl := range tables {
ignorePatterns := ignorePatternMap[tbl.Schema]
ignored, err := ignorePatterns.IsTableNameIgnored(tbl)
if err != nil {
return nil, err
}
if conflict := AsDoltIgnoreInConflict(err); conflict != nil {
// no-op
} else if err != nil {
return nil, err
} else if ignored == DontIgnore {
// no-op
} else if ignored == Ignore {
Expand Down Expand Up @@ -231,10 +254,17 @@ func resolveConflictingPatterns(trueMatches, falseMatches []string, tableName Ta
return IgnorePatternConflict, DoltIgnoreConflictError{Table: tableName, TruePatterns: conflictingTrueMatches, FalsePatterns: conflictingFalseMatches}
}

func isDoltRebaseTable(tableName TableName) bool {
if strings.EqualFold(tableName.Name, RebaseTableName) {
return true
}
return tableName.Schema == DoltNamespace && tableName.Name == GetRebaseTableName()
}

func (ip *IgnorePatterns) IsTableNameIgnored(tableName TableName) (IgnoreResult, error) {
// The dolt_rebase table is automatically ignored by Dolt – it shouldn't ever
// be checked in to a Dolt database.
if strings.EqualFold(tableName.Name, RebaseTableName) {
if isDoltRebaseTable(tableName) {
return Ignore, nil
}

Expand Down
19 changes: 18 additions & 1 deletion go/libraries/doltcore/doltdb/root_val.go
Original file line number Diff line number Diff line change
Expand Up @@ -835,6 +835,21 @@ func UnqualifiedTableNamesAsString(names []TableName) string {
return sb.String()
}

// GetUniqueSchemaNamesFromTableNames returns a list of unique schema names from a list of table names
func GetUniqueSchemaNamesFromTableNames(names []TableName) []string {
schemaMap := make(map[string]struct{})
var schemas []string

for _, t := range names {
if _, exists := schemaMap[t.Schema]; !exists {
schemaMap[t.Schema] = struct{}{}
schemas = append(schemas, t.Schema)
}
}

return schemas
}

// DefaultSchemaName is the name of the default schema. Tables with this schema name will be stored in the
// primary (unnamed) table store in a root.
var DefaultSchemaName = ""
Expand Down Expand Up @@ -1205,11 +1220,13 @@ func schemaNames(ctx context.Context, root RootValue) ([]string, error) {

// FilterIgnoredTables takes a slice of table names and divides it into new slices based on whether the table is ignored, not ignored, or matches conflicting ignore patterns.
func FilterIgnoredTables(ctx context.Context, tables []TableName, roots Roots) (ignoredTables IgnoredTables, err error) {
ignorePatterns, err := GetIgnoredTablePatterns(ctx, roots)
schemas := GetUniqueSchemaNamesFromTableNames(tables)
ignorePatternMap, err := GetIgnoredTablePatterns(ctx, roots, schemas)
if err != nil {
return ignoredTables, err
}
for _, tableName := range tables {
ignorePatterns := ignorePatternMap[tableName.Schema]
ignored, err := ignorePatterns.IsTableNameIgnored(tableName)
if conflict := AsDoltIgnoreInConflict(err); conflict != nil {
ignoredTables.Conflicts = append(ignoredTables.Conflicts, *conflict)
Expand Down
Loading

0 comments on commit 3428267

Please sign in to comment.