GTID:

- Added "reset-master-gtid-remove-own-uuid" command - ResetMasterGTIDOperation supports removal of self-UUID entries - setGTIDPurged respects --noop - Repoint() makes up a non-existent binlog file name when log file is empty (can be cause when switching to/from GTID with error)
outbrain-inc · Nov 24, 2015 · 3703cf6 · 3703cf6
1 parent 546ae8c
commit 3703cf6
Show file tree

Hide file tree

Showing 8 changed files with 96 additions and 38 deletions.
diff --git a/build.sh b/build.sh
@@ -6,7 +6,7 @@
 #
 set -e
 
-RELEASE_VERSION="1.4.542"
+RELEASE_VERSION="1.4.543"
 TOPDIR=/tmp/orchestrator-release
 export RELEASE_VERSION TOPDIR
 

diff --git a/go/app/cli.go b/go/app/cli.go
@@ -520,6 +520,15 @@ func Cli(command string, strict bool, instance string, destination string, owner
 			}
 			fmt.Println(instanceKey.DisplayString())
 		}
+	case registerCliCommand("reset-master-gtid-remove-own-uuid", "Replication, general", `Reset master on instance, remove GTID entries generated by instance`):
+		{
+			instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
+			_, err := inst.ResetMasterGTIDOperation(instanceKey, true, "")
+			if err != nil {
+				log.Fatale(err)
+			}
+			fmt.Println(instanceKey.DisplayString())
+		}
 	case registerCliCommand("skip-query", "Replication, general", `Skip a single statement on a slave; either when running with GTID or without`):
 		{
 			instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)

diff --git a/go/cmd/orchestrator/main.go b/go/cmd/orchestrator/main.go
@@ -339,7 +339,15 @@ Cheatsheet:
             Assuming slave replicates via GTID, disable GTID replication and resume standard file:pos replication. Example:
 
             orchestrator -c disable-gtid -i slave.replicating.via.gtid.com
-        
+
+        reset-master-gtid-remove-own-uuid
+            Assuming GTID is enabled, Reset master on instance, remove GTID entries generated by the instance.
+            This operation is only allowed on Oracle-GTID enabled servers that have no slaves.
+            Is is used for cleaning up the GTID mess incurred by mistakenly issuing queries on the slave (even such
+            queries as "FLUSH ENGINE LOGS" that happen to write to binary logs). Example:
+            
+            orchestrator -c reset-master-gtid-remove-own-uuid -i slave.running.with.gtid.com
+                    
         stop-slave
             Issues a STOP SLAVE; command. Example:
 

diff --git a/go/inst/binlog.go b/go/inst/binlog.go
@@ -76,6 +76,11 @@ func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool {
 	return this.LogFile == other.LogFile && this.LogPos == other.LogPos && this.Type == other.Type
 }
 
+// IsEmpty returns true if the log file is empty, unnamed
+func (this *BinlogCoordinates) IsEmpty() bool {
+	return this.LogFile == ""
+}
+
 // SmallerThan returns true if this coordinate is strictly smaller than the other.
 func (this *BinlogCoordinates) SmallerThan(other *BinlogCoordinates) bool {
 	if this.LogFile < other.LogFile {

diff --git a/go/inst/instance_test.go b/go/inst/instance_test.go
@@ -305,49 +305,58 @@ func (s *TestSuite) TestNextGTID(c *C) {
 }
 
 func (s *TestSuite) TestOracleGTIDSet(c *C) {
-	gtidSetString := `230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,
+	{
+		gtidSetString := `230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,
 316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-8935:8984-6124596,
 321f5c0d-70e5-11e5-adb2-ecf4bb2262ff:1-56`
-	gtidSet, err := inst.ParseGtidSet(gtidSetString)
-	c.Assert(err, IsNil)
-	c.Assert(len(gtidSet.GtidEntries), Equals, 3)
-	c.Assert(gtidSet.String(), Equals, `230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,
+		gtidSet, err := inst.ParseGtidSet(gtidSetString)
+		c.Assert(err, IsNil)
+		c.Assert(len(gtidSet.GtidEntries), Equals, 3)
+		c.Assert(gtidSet.String(), Equals, `230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,
 316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-8935:8984-6124596,
 321f5c0d-70e5-11e5-adb2-ecf4bb2262ff:1-56`)
 
-	c.Assert(gtidSet.GtidEntries[0].UUID, Equals, `230ea8ea-81e3-11e4-972a-e25ec4bd140a`)
-	c.Assert(gtidSet.GtidEntries[1].UUID, Equals, `316d193c-70e5-11e5-adb2-ecf4bb2262ff`)
-	c.Assert(gtidSet.GtidEntries[2].UUID, Equals, `321f5c0d-70e5-11e5-adb2-ecf4bb2262ff`)
-	c.Assert(gtidSet.GtidEntries[1].Ranges, Equals, `1-8935:8984-6124596`)
+		c.Assert(gtidSet.GtidEntries[0].UUID, Equals, `230ea8ea-81e3-11e4-972a-e25ec4bd140a`)
+		c.Assert(gtidSet.GtidEntries[1].UUID, Equals, `316d193c-70e5-11e5-adb2-ecf4bb2262ff`)
+		c.Assert(gtidSet.GtidEntries[2].UUID, Equals, `321f5c0d-70e5-11e5-adb2-ecf4bb2262ff`)
+		c.Assert(gtidSet.GtidEntries[1].Ranges, Equals, `1-8935:8984-6124596`)
 
-	removed := gtidSet.RemoveUUID(`ffffffff-70e5-11e5-adb2-ecf4bb2262ff`)
-	c.Assert(removed, Equals, false)
-	c.Assert(len(gtidSet.GtidEntries), Equals, 3)
+		removed := gtidSet.RemoveUUID(`ffffffff-70e5-11e5-adb2-ecf4bb2262ff`)
+		c.Assert(removed, Equals, false)
+		c.Assert(len(gtidSet.GtidEntries), Equals, 3)
 
-	removed = gtidSet.RemoveUUID(`316d193c-70e5-11e5-adb2-ecf4bb2262ff`)
-	c.Assert(removed, Equals, true)
-	c.Assert(len(gtidSet.GtidEntries), Equals, 2)
+		removed = gtidSet.RemoveUUID(`316d193c-70e5-11e5-adb2-ecf4bb2262ff`)
+		c.Assert(removed, Equals, true)
+		c.Assert(len(gtidSet.GtidEntries), Equals, 2)
 
-	c.Assert(gtidSet.String(), Equals, `230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,
+		c.Assert(gtidSet.String(), Equals, `230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,
 321f5c0d-70e5-11e5-adb2-ecf4bb2262ff:1-56`)
 
-	removed = gtidSet.RemoveUUID(`316d193c-70e5-11e5-adb2-ecf4bb2262ff`)
-	c.Assert(removed, Equals, false)
-	c.Assert(len(gtidSet.GtidEntries), Equals, 2)
+		removed = gtidSet.RemoveUUID(`316d193c-70e5-11e5-adb2-ecf4bb2262ff`)
+		c.Assert(removed, Equals, false)
+		c.Assert(len(gtidSet.GtidEntries), Equals, 2)
 
-	removed = gtidSet.RemoveUUID(`321f5c0d-70e5-11e5-adb2-ecf4bb2262ff`)
-	c.Assert(removed, Equals, true)
-	c.Assert(len(gtidSet.GtidEntries), Equals, 1)
-	c.Assert(gtidSet.String(), Equals, `230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539`)
+		removed = gtidSet.RemoveUUID(`321f5c0d-70e5-11e5-adb2-ecf4bb2262ff`)
+		c.Assert(removed, Equals, true)
+		c.Assert(len(gtidSet.GtidEntries), Equals, 1)
+		c.Assert(gtidSet.String(), Equals, `230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539`)
 
-	removed = gtidSet.RemoveUUID(`230ea8ea-81e3-11e4-972a-e25ec4bd140a`)
-	c.Assert(removed, Equals, true)
-	c.Assert(len(gtidSet.GtidEntries), Equals, 0)
+		removed = gtidSet.RemoveUUID(`230ea8ea-81e3-11e4-972a-e25ec4bd140a`)
+		c.Assert(removed, Equals, true)
+		c.Assert(len(gtidSet.GtidEntries), Equals, 0)
 
-	removed = gtidSet.RemoveUUID(`230ea8ea-81e3-11e4-972a-e25ec4bd140a`)
-	c.Assert(removed, Equals, false)
-	c.Assert(len(gtidSet.GtidEntries), Equals, 0)
-	c.Assert(gtidSet.String(), Equals, ``)
+		removed = gtidSet.RemoveUUID(`230ea8ea-81e3-11e4-972a-e25ec4bd140a`)
+		c.Assert(removed, Equals, false)
+		c.Assert(len(gtidSet.GtidEntries), Equals, 0)
+		c.Assert(gtidSet.String(), Equals, ``)
+	}
+	{
+		gtidSetString := `230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,
+:1-8935:8984-6124596,
+321f5c0d-70e5-11e5-adb2-ecf4bb2262ff:1-56`
+		_, err := inst.ParseGtidSet(gtidSetString)
+		c.Assert(err, Not(IsNil))
+	}
 }
 
 func (s *TestSuite) TestRemoveInstance(c *C) {

diff --git a/go/inst/instance_topology.go b/go/inst/instance_topology.go
@@ -719,6 +719,9 @@ func Repoint(instanceKey *InstanceKey, masterKey *InstanceKey, gtidHint Operatio
 	// See above, we are relaxed about the master being accessible/inaccessible.
 	// If accessible, we wish to do hostname-unresolve. If inaccessible, we can skip the test and not fail the
 	// ChangeMasterTo operation. This is why we pass "!masterIsAccessible" below.
+	if instance.ExecBinlogCoordinates.IsEmpty() {
+		instance.ExecBinlogCoordinates.LogFile = "orchestrator-unknown-log-file"
+	}
 	instance, err = ChangeMasterTo(instanceKey, masterKey, &instance.ExecBinlogCoordinates, !masterIsAccessible, gtidHint)
 	if err != nil {
 		goto Cleanup
@@ -1167,7 +1170,7 @@ func DisableGTID(instanceKey *InstanceKey) (*Instance, error) {
 // It will make sure the gtid_purged set matches the executed set value as read just before the RESET.
 // this will enable new slaves to be attached to given instance without complaints about missing/purged entries.
 // This function requires that the instance does not have slaves.
-func ResetMasterGTIDOperation(instanceKey *InstanceKey) (*Instance, error) {
+func ResetMasterGTIDOperation(instanceKey *InstanceKey, removeSelfUUID bool, uuidToRemove string) (*Instance, error) {
 	instance, err := ReadTopologyInstance(instanceKey)
 	if err != nil {
 		return instance, err
@@ -1181,7 +1184,7 @@ func ResetMasterGTIDOperation(instanceKey *InstanceKey) (*Instance, error) {
 
 	log.Infof("Will reset master on %+v", instanceKey)
 
-	var executedGtidSet string
+	var oracleGtidSet *OracleGtidSet
 	if maintenanceToken, merr := BeginMaintenance(instanceKey, GetMaintenanceOwner(), "reset-master-gtid"); merr != nil {
 		err = fmt.Errorf("Cannot begin maintenance on %+v", *instanceKey)
 		goto Cleanup
@@ -1195,13 +1198,28 @@ func ResetMasterGTIDOperation(instanceKey *InstanceKey) (*Instance, error) {
 			goto Cleanup
 		}
 	}
-	executedGtidSet = instance.ExecutedGtidSet
+
+	oracleGtidSet, err = ParseGtidSet(instance.ExecutedGtidSet)
+	if err != nil {
+		goto Cleanup
+	}
+	if removeSelfUUID {
+		uuidToRemove = instance.ServerUUID
+	}
+	if uuidToRemove != "" {
+		removed := oracleGtidSet.RemoveUUID(uuidToRemove)
+		if removed {
+			log.Debugf("Will remove UUID %s", uuidToRemove)
+		} else {
+			log.Debugf("UUID %s not found", uuidToRemove)
+		}
+	}
 
 	instance, err = ResetMaster(instanceKey)
 	if err != nil {
 		goto Cleanup
 	}
-	err = setGTIDPurged(instance, executedGtidSet)
+	err = setGTIDPurged(instance, oracleGtidSet.String())
 	if err != nil {
 		goto Cleanup
 	}

diff --git a/go/inst/instance_topology_dao.go b/go/inst/instance_topology_dao.go
@@ -577,6 +577,10 @@ func ResetMaster(instanceKey *InstanceKey) (*Instance, error) {
 
 // skipQueryClassic skips a query in normal binlog file:pos replication
 func setGTIDPurged(instance *Instance, gtidPurged string) error {
+	if *config.RuntimeCLIFlags.Noop {
+		return fmt.Errorf("noop: aborting set-gtid-purged operation on %+v; signalling error but nothing went wrong.", instance.Key)
+	}
+
 	_, err := ExecInstance(&instance.Key, fmt.Sprintf(`set global gtid_purged := '%s'`, gtidPurged))
 	return err
 }

diff --git a/go/inst/oracle_gtid_set_entry.go b/go/inst/oracle_gtid_set_entry.go
@@ -21,21 +21,26 @@ import (
 	"strings"
 )
 
-// OracleGtidSetEntry represents an entry in a set of GTID ranges, 
+// OracleGtidSetEntry represents an entry in a set of GTID ranges,
 // for example, the entry: "316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-8935:8984-6124596" (may include gaps)
 type OracleGtidSetEntry struct {
 	UUID   string
 	Ranges string
 }
 
-
 // NewOracleGtidSetEntry parses a single entry text
 func NewOracleGtidSetEntry(gtidRangeString string) (*OracleGtidSetEntry, error) {
 	gtidRangeString = strings.TrimSpace(gtidRangeString)
 	tokens := strings.SplitN(gtidRangeString, ":", 2)
 	if len(tokens) != 2 {
 		return nil, fmt.Errorf("Cannot parse OracleGtidSetEntry from %s", gtidRangeString)
 	}
+	if tokens[0] == "" {
+		return nil, fmt.Errorf("Unexpected UUID: %s", tokens[0])
+	}
+	if tokens[1] == "" {
+		return nil, fmt.Errorf("Unexpected GTID range: %s", tokens[1])
+	}
 	gtidRange := &OracleGtidSetEntry{UUID: tokens[0], Ranges: tokens[1]}
 	return gtidRange, nil
 }