Initial commit

vendor/github.com/siddontang/go-mysql/canal/BUILD.bazel

@@ -0,0 +1,42 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "canal.go",
+        "config.go",
+        "dump.go",
+        "handler.go",
+        "master.go",
+        "rows.go",
+        "sync.go",
+    ],
+    importmap = "go-common/vendor/github.com/siddontang/go-mysql/canal",
+    importpath = "github.com/siddontang/go-mysql/canal",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "//vendor/github.com/juju/errors:go_default_library",
+        "//vendor/github.com/satori/go.uuid:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/client:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/dump:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/mysql:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/replication:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/schema:go_default_library",
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/siddontang/go-mysql/canal/canal.go

@@ -0,0 +1,443 @@
+package canal
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/juju/errors"
+	"github.com/siddontang/go-mysql/client"
+	"github.com/siddontang/go-mysql/dump"
+	"github.com/siddontang/go-mysql/mysql"
+	"github.com/siddontang/go-mysql/replication"
+	"github.com/siddontang/go-mysql/schema"
+	log "github.com/sirupsen/logrus"
+)
+
+// Canal can sync your MySQL data into everywhere, like Elasticsearch, Redis, etc...
+// MySQL must open row format for binlog
+type Canal struct {
+	m sync.Mutex
+
+	cfg *Config
+
+	useGTID bool
+
+	master     *masterInfo
+	dumper     *dump.Dumper
+	dumpDoneCh chan struct{}
+	syncer     *replication.BinlogSyncer
+
+	eventHandler EventHandler
+
+	connLock sync.Mutex
+	conn     *client.Conn
+
+	tableLock          sync.RWMutex
+	tables             map[string]*schema.Table
+	errorTablesGetTime map[string]time.Time
+
+	tableMatchCache   map[string]bool
+	includeTableRegex []*regexp.Regexp
+	excludeTableRegex []*regexp.Regexp
+
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
+// canal will retry fetching unknown table's meta after UnknownTableRetryPeriod
+var UnknownTableRetryPeriod = time.Second * time.Duration(10)
+var ErrExcludedTable = errors.New("excluded table meta")
+
+func NewCanal(cfg *Config) (*Canal, error) {
+	c := new(Canal)
+	c.cfg = cfg
+
+	c.ctx, c.cancel = context.WithCancel(context.Background())
+
+	c.dumpDoneCh = make(chan struct{})
+	c.eventHandler = &DummyEventHandler{}
+
+	c.tables = make(map[string]*schema.Table)
+	if c.cfg.DiscardNoMetaRowEvent {
+		c.errorTablesGetTime = make(map[string]time.Time)
+	}
+	c.master = &masterInfo{}
+
+	var err error
+
+	if err = c.prepareDumper(); err != nil {
+		return nil, errors.Trace(err)
+	}
+
+	if err = c.prepareSyncer(); err != nil {
+		return nil, errors.Trace(err)
+	}
+
+	if err := c.checkBinlogRowFormat(); err != nil {
+		return nil, errors.Trace(err)
+	}
+
+	// init table filter
+	if n := len(c.cfg.IncludeTableRegex); n > 0 {
+		c.includeTableRegex = make([]*regexp.Regexp, n)
+		for i, val := range c.cfg.IncludeTableRegex {
+			reg, err := regexp.Compile(val)
+			if err != nil {
+				return nil, errors.Trace(err)
+			}
+			c.includeTableRegex[i] = reg
+		}
+	}
+
+	if n := len(c.cfg.ExcludeTableRegex); n > 0 {
+		c.excludeTableRegex = make([]*regexp.Regexp, n)
+		for i, val := range c.cfg.ExcludeTableRegex {
+			reg, err := regexp.Compile(val)
+			if err != nil {
+				return nil, errors.Trace(err)
+			}
+			c.excludeTableRegex[i] = reg
+		}
+	}
+
+	if c.includeTableRegex != nil || c.excludeTableRegex != nil {
+		c.tableMatchCache = make(map[string]bool)
+	}
+
+	return c, nil
+}
+
+func (c *Canal) prepareDumper() error {
+	var err error
+	dumpPath := c.cfg.Dump.ExecutionPath
+	if len(dumpPath) == 0 {
+		// ignore mysqldump, use binlog only
+		return nil
+	}
+
+	if c.dumper, err = dump.NewDumper(dumpPath,
+		c.cfg.Addr, c.cfg.User, c.cfg.Password); err != nil {
+		return errors.Trace(err)
+	}
+
+	if c.dumper == nil {
+		//no mysqldump, use binlog only
+		return nil
+	}
+
+	dbs := c.cfg.Dump.Databases
+	tables := c.cfg.Dump.Tables
+	tableDB := c.cfg.Dump.TableDB
+
+	if len(tables) == 0 {
+		c.dumper.AddDatabases(dbs...)
+	} else {
+		c.dumper.AddTables(tableDB, tables...)
+	}
+
+	charset := c.cfg.Charset
+	c.dumper.SetCharset(charset)
+
+	c.dumper.SkipMasterData(c.cfg.Dump.SkipMasterData)
+	c.dumper.SetMaxAllowedPacket(c.cfg.Dump.MaxAllowedPacketMB)
+
+	for _, ignoreTable := range c.cfg.Dump.IgnoreTables {
+		if seps := strings.Split(ignoreTable, ","); len(seps) == 2 {
+			c.dumper.AddIgnoreTables(seps[0], seps[1])
+		}
+	}
+
+	if c.cfg.Dump.DiscardErr {
+		c.dumper.SetErrOut(ioutil.Discard)
+	} else {
+		c.dumper.SetErrOut(os.Stderr)
+	}
+
+	return nil
+}
+
+// Run will first try to dump all data from MySQL master `mysqldump`,
+// then sync from the binlog position in the dump data.
+// It will run forever until meeting an error or Canal closed.
+func (c *Canal) Run() error {
+	return c.run()
+}
+
+// RunFrom will sync from the binlog position directly, ignore mysqldump.
+func (c *Canal) RunFrom(pos mysql.Position) error {
+	c.useGTID = false
+	c.master.Update(pos)
+
+	return c.Run()
+}
+
+func (c *Canal) StartFromGTID(set mysql.GTIDSet) error {
+	c.useGTID = true
+	c.master.UpdateGTID(set)
+
+	return c.Run()
+}
+
+func (c *Canal) run() error {
+	defer func() {
+		c.cancel()
+	}()
+
+	err := c.tryDump()
+	close(c.dumpDoneCh)
+
+	if err != nil {
+		log.Errorf("canal dump mysql err: %v", err)
+		return errors.Trace(err)
+	}
+
+	if err = c.runSyncBinlog(); err != nil {
+		log.Errorf("canal start sync binlog err: %v", err)
+		return errors.Trace(err)
+	}
+
+	return nil
+}
+
+func (c *Canal) Close() {
+	log.Infof("closing canal")
+
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	c.cancel()
+	c.connLock.Lock()
+	c.conn.Close()
+	c.conn = nil
+	c.connLock.Unlock()
+	c.syncer.Close()
+
+	c.eventHandler.OnPosSynced(c.master.Position(), true)
+}
+
+func (c *Canal) WaitDumpDone() <-chan struct{} {
+	return c.dumpDoneCh
+}
+
+func (c *Canal) Ctx() context.Context {
+	return c.ctx
+}
+
+func (c *Canal) checkTableMatch(key string) bool {
+	// no filter, return true
+	if c.tableMatchCache == nil {
+		return true
+	}
+
+	c.tableLock.RLock()
+	rst, ok := c.tableMatchCache[key]
+	c.tableLock.RUnlock()
+	if ok {
+		// cache hit
+		return rst
+	}
+	matchFlag := false
+	// check include
+	if c.includeTableRegex != nil {
+		for _, reg := range c.includeTableRegex {
+			if reg.MatchString(key) {
+				matchFlag = true
+				break
+			}
+		}
+	}
+	// check exclude
+	if matchFlag && c.excludeTableRegex != nil {
+		for _, reg := range c.excludeTableRegex {
+			if reg.MatchString(key) {
+				matchFlag = false
+				break
+			}
+		}
+	}
+	c.tableLock.Lock()
+	c.tableMatchCache[key] = matchFlag
+	c.tableLock.Unlock()
+	return matchFlag
+}
+
+func (c *Canal) GetTable(db string, table string) (*schema.Table, error) {
+	key := fmt.Sprintf("%s.%s", db, table)
+	// if table is excluded, return error and skip parsing event or dump
+	if !c.checkTableMatch(key) {
+		return nil, ErrExcludedTable
+	}
+	c.tableLock.RLock()
+	t, ok := c.tables[key]
+	c.tableLock.RUnlock()
+
+	if ok {
+		return t, nil
+	}
+
+	if c.cfg.DiscardNoMetaRowEvent {
+		c.tableLock.RLock()
+		lastTime, ok := c.errorTablesGetTime[key]
+		c.tableLock.RUnlock()
+		if ok && time.Now().Sub(lastTime) < UnknownTableRetryPeriod {
+			return nil, schema.ErrMissingTableMeta
+		}
+	}
+
+	t, err := schema.NewTable(c, db, table)
+	if err != nil {
+		// check table not exists
+		if ok, err1 := schema.IsTableExist(c, db, table); err1 == nil && !ok {
+			return nil, schema.ErrTableNotExist
+		}
+		// work around : RDS HAHeartBeat
+		// ref : https://github.com/alibaba/canal/blob/master/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java#L385
+		// issue : https://github.com/alibaba/canal/issues/222
+		// This is a common error in RDS that canal can't get HAHealthCheckSchema's meta, so we mock a table meta.
+		// If canal just skip and log error, as RDS HA heartbeat interval is very short, so too many HAHeartBeat errors will be logged.
+		if key == schema.HAHealthCheckSchema {
+			// mock ha_health_check meta
+			ta := &schema.Table{
+				Schema:  db,
+				Name:    table,
+				Columns: make([]schema.TableColumn, 0, 2),
+				Indexes: make([]*schema.Index, 0),
+			}
+			ta.AddColumn("id", "bigint(20)", "", "")
+			ta.AddColumn("type", "char(1)", "", "")
+			c.tableLock.Lock()
+			c.tables[key] = ta
+			c.tableLock.Unlock()
+			return ta, nil
+		}
+		// if DiscardNoMetaRowEvent is true, we just log this error
+		if c.cfg.DiscardNoMetaRowEvent {
+			c.tableLock.Lock()
+			c.errorTablesGetTime[key] = time.Now()
+			c.tableLock.Unlock()
+			// log error and return ErrMissingTableMeta
+			log.Errorf("canal get table meta err: %v", errors.Trace(err))
+			return nil, schema.ErrMissingTableMeta
+		}
+		return nil, err
+	}
+
+	c.tableLock.Lock()
+	c.tables[key] = t
+	if c.cfg.DiscardNoMetaRowEvent {
+		// if get table info success, delete this key from errorTablesGetTime
+		delete(c.errorTablesGetTime, key)
+	}
+	c.tableLock.Unlock()
+
+	return t, nil
+}
+
+// ClearTableCache clear table cache
+func (c *Canal) ClearTableCache(db []byte, table []byte) {
+	key := fmt.Sprintf("%s.%s", db, table)
+	c.tableLock.Lock()
+	delete(c.tables, key)
+	if c.cfg.DiscardNoMetaRowEvent {
+		delete(c.errorTablesGetTime, key)
+	}
+	c.tableLock.Unlock()
+}
+
+// Check MySQL binlog row image, must be in FULL, MINIMAL, NOBLOB
+func (c *Canal) CheckBinlogRowImage(image string) error {
+	// need to check MySQL binlog row image? full, minimal or noblob?
+	// now only log
+	if c.cfg.Flavor == mysql.MySQLFlavor {
+		if res, err := c.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_row_image"`); err != nil {
+			return errors.Trace(err)
+		} else {
+			// MySQL has binlog row image from 5.6, so older will return empty
+			rowImage, _ := res.GetString(0, 1)
+			if rowImage != "" && !strings.EqualFold(rowImage, image) {
+				return errors.Errorf("MySQL uses %s binlog row image, but we want %s", rowImage, image)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *Canal) checkBinlogRowFormat() error {
+	res, err := c.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_format";`)
+	if err != nil {
+		return errors.Trace(err)
+	} else if f, _ := res.GetString(0, 1); f != "ROW" {
+		return errors.Errorf("binlog must ROW format, but %s now", f)
+	}
+
+	return nil
+}
+
+func (c *Canal) prepareSyncer() error {
+	seps := strings.Split(c.cfg.Addr, ":")
+	if len(seps) != 2 {
+		return errors.Errorf("invalid mysql addr format %s, must host:port", c.cfg.Addr)
+	}
+
+	port, err := strconv.ParseUint(seps[1], 10, 16)
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	cfg := replication.BinlogSyncerConfig{
+		ServerID:        c.cfg.ServerID,
+		Flavor:          c.cfg.Flavor,
+		Host:            seps[0],
+		Port:            uint16(port),
+		User:            c.cfg.User,
+		Password:        c.cfg.Password,
+		Charset:         c.cfg.Charset,
+		HeartbeatPeriod: c.cfg.HeartbeatPeriod,
+		ReadTimeout:     c.cfg.ReadTimeout,
+		UseDecimal:      c.cfg.UseDecimal,
+	}
+
+	c.syncer = replication.NewBinlogSyncer(cfg)
+
+	return nil
+}
+
+// Execute a SQL
+func (c *Canal) Execute(cmd string, args ...interface{}) (rr *mysql.Result, err error) {
+	c.connLock.Lock()
+	defer c.connLock.Unlock()
+
+	retryNum := 3
+	for i := 0; i < retryNum; i++ {
+		if c.conn == nil {
+			c.conn, err = client.Connect(c.cfg.Addr, c.cfg.User, c.cfg.Password, "")
+			if err != nil {
+				return nil, errors.Trace(err)
+			}
+		}
+
+		rr, err = c.conn.Execute(cmd, args...)
+		if err != nil && !mysql.ErrorEqual(err, mysql.ErrBadConn) {
+			return
+		} else if mysql.ErrorEqual(err, mysql.ErrBadConn) {
+			c.conn.Close()
+			c.conn = nil
+			continue
+		} else {
+			return
+		}
+	}
+	return
+}
+
+func (c *Canal) SyncedPosition() mysql.Position {
+	return c.master.Position()
+}

vendor/github.com/siddontang/go-mysql/canal/config.go

@@ -0,0 +1,103 @@
+package canal
+
+import (
+	"io/ioutil"
+	"math/rand"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/juju/errors"
+	"github.com/siddontang/go-mysql/mysql"
+)
+
+type DumpConfig struct {
+	// mysqldump execution path, like mysqldump or /usr/bin/mysqldump, etc...
+	// If not set, ignore using mysqldump.
+	ExecutionPath string `toml:"mysqldump"`
+
+	// Will override Databases, tables is in database table_db
+	Tables  []string `toml:"tables"`
+	TableDB string   `toml:"table_db"`
+
+	Databases []string `toml:"dbs"`
+
+	// Ignore table format is db.table
+	IgnoreTables []string `toml:"ignore_tables"`
+
+	// If true, discard error msg, else, output to stderr
+	DiscardErr bool `toml:"discard_err"`
+
+	// Set true to skip --master-data if we have no privilege to do
+	// 'FLUSH TABLES WITH READ LOCK'
+	SkipMasterData bool `toml:"skip_master_data"`
+
+	// Set to change the default max_allowed_packet size
+	MaxAllowedPacketMB int `toml:"max_allowed_packet_mb"`
+}
+
+type Config struct {
+	Addr     string `toml:"addr"`
+	User     string `toml:"user"`
+	Password string `toml:"password"`
+
+	Charset         string        `toml:"charset"`
+	ServerID        uint32        `toml:"server_id"`
+	Flavor          string        `toml:"flavor"`
+	HeartbeatPeriod time.Duration `toml:"heartbeat_period"`
+	ReadTimeout     time.Duration `toml:"read_timeout"`
+
+	// IncludeTableRegex or ExcludeTableRegex should contain database name
+	// Only a table which matches IncludeTableRegex and dismatches ExcludeTableRegex will be processed
+	// eg, IncludeTableRegex : [".*\\.canal"], ExcludeTableRegex : ["mysql\\..*"]
+	//     this will include all database's 'canal' table, except database 'mysql'
+	// Default IncludeTableRegex and ExcludeTableRegex are empty, this will include all tables
+	IncludeTableRegex []string `toml:include_table_regex`
+	ExcludeTableRegex []string `toml:exclude_table_regex`
+
+	// discard row event without table meta
+	DiscardNoMetaRowEvent bool `toml:"discard_no_meta_row_event"`
+
+	Dump DumpConfig `toml:"dump"`
+
+	UseDecimal bool `toml:"use_decimal"`
+}
+
+func NewConfigWithFile(name string) (*Config, error) {
+	data, err := ioutil.ReadFile(name)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+
+	return NewConfig(string(data))
+}
+
+func NewConfig(data string) (*Config, error) {
+	var c Config
+
+	_, err := toml.Decode(data, &c)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+
+	return &c, nil
+}
+
+func NewDefaultConfig() *Config {
+	c := new(Config)
+
+	c.Addr = "127.0.0.1:3306"
+	c.User = "root"
+	c.Password = ""
+
+	c.Charset = mysql.DEFAULT_CHARSET
+	rand.Seed(time.Now().Unix())
+	c.ServerID = uint32(rand.Intn(1000)) + 1001
+
+	c.Flavor = "mysql"
+
+	c.Dump.ExecutionPath = "mysqldump"
+	c.Dump.DiscardErr = true
+	c.Dump.SkipMasterData = false
+
+	return c
+}

vendor/github.com/siddontang/go-mysql/canal/dump.go

@@ -0,0 +1,139 @@
+package canal
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/juju/errors"
+	"github.com/siddontang/go-mysql/dump"
+	"github.com/siddontang/go-mysql/mysql"
+	"github.com/siddontang/go-mysql/schema"
+	log "github.com/sirupsen/logrus"
+)
+
+type dumpParseHandler struct {
+	c    *Canal
+	name string
+	pos  uint64
+}
+
+func (h *dumpParseHandler) BinLog(name string, pos uint64) error {
+	h.name = name
+	h.pos = pos
+	return nil
+}
+
+func (h *dumpParseHandler) Data(db string, table string, values []string) error {
+	if err := h.c.ctx.Err(); err != nil {
+		return err
+	}
+
+	tableInfo, err := h.c.GetTable(db, table)
+	if err != nil {
+		e := errors.Cause(err)
+		if e == ErrExcludedTable ||
+			e == schema.ErrTableNotExist ||
+			e == schema.ErrMissingTableMeta {
+			return nil
+		}
+		log.Errorf("get %s.%s information err: %v", db, table, err)
+		return errors.Trace(err)
+	}
+
+	vs := make([]interface{}, len(values))
+
+	for i, v := range values {
+		if v == "NULL" {
+			vs[i] = nil
+		} else if v[0] != '\'' {
+			if tableInfo.Columns[i].Type == schema.TYPE_NUMBER {
+				n, err := strconv.ParseInt(v, 10, 64)
+				if err != nil {
+					log.Errorf("parse row %v at %d error %v, skip", values, i, err)
+					return dump.ErrSkip
+				}
+				vs[i] = n
+			} else if tableInfo.Columns[i].Type == schema.TYPE_FLOAT {
+				f, err := strconv.ParseFloat(v, 64)
+				if err != nil {
+					log.Errorf("parse row %v at %d error %v, skip", values, i, err)
+					return dump.ErrSkip
+				}
+				vs[i] = f
+			} else {
+				log.Errorf("parse row %v error, invalid type at %d, skip", values, i)
+				return dump.ErrSkip
+			}
+		} else {
+			vs[i] = v[1 : len(v)-1]
+		}
+	}
+
+	events := newRowsEvent(tableInfo, InsertAction, [][]interface{}{vs})
+	return h.c.eventHandler.OnRow(events)
+}
+
+func (c *Canal) AddDumpDatabases(dbs ...string) {
+	if c.dumper == nil {
+		return
+	}
+
+	c.dumper.AddDatabases(dbs...)
+}
+
+func (c *Canal) AddDumpTables(db string, tables ...string) {
+	if c.dumper == nil {
+		return
+	}
+
+	c.dumper.AddTables(db, tables...)
+}
+
+func (c *Canal) AddDumpIgnoreTables(db string, tables ...string) {
+	if c.dumper == nil {
+		return
+	}
+
+	c.dumper.AddIgnoreTables(db, tables...)
+}
+
+func (c *Canal) tryDump() error {
+	pos := c.master.Position()
+	gtid := c.master.GTID()
+	if (len(pos.Name) > 0 && pos.Pos > 0) || gtid != nil {
+		// we will sync with binlog name and position
+		log.Infof("skip dump, use last binlog replication pos %s or GTID %s", pos, gtid)
+		return nil
+	}
+
+	if c.dumper == nil {
+		log.Info("skip dump, no mysqldump")
+		return nil
+	}
+
+	h := &dumpParseHandler{c: c}
+
+	if c.cfg.Dump.SkipMasterData {
+		pos, err := c.GetMasterPos()
+		if err != nil {
+			return errors.Trace(err)
+		}
+		log.Infof("skip master data, get current binlog position %v", pos)
+		h.name = pos.Name
+		h.pos = uint64(pos.Pos)
+	}
+
+	start := time.Now()
+	log.Info("try dump MySQL and parse")
+	if err := c.dumper.DumpAndParse(h); err != nil {
+		return errors.Trace(err)
+	}
+
+	log.Infof("dump MySQL and parse OK, use %0.2f seconds, start binlog replication at (%s, %d)",
+		time.Now().Sub(start).Seconds(), h.name, h.pos)
+
+	pos = mysql.Position{h.name, uint32(h.pos)}
+	c.master.Update(pos)
+	c.eventHandler.OnPosSynced(pos, true)
+	return nil
+}

vendor/github.com/siddontang/go-mysql/canal/handler.go

@@ -0,0 +1,36 @@
+package canal
+
+import (
+	"github.com/siddontang/go-mysql/mysql"
+	"github.com/siddontang/go-mysql/replication"
+)
+
+type EventHandler interface {
+	OnRotate(roateEvent *replication.RotateEvent) error
+	OnDDL(nextPos mysql.Position, queryEvent *replication.QueryEvent) error
+	OnRow(e *RowsEvent) error
+	OnXID(nextPos mysql.Position) error
+	OnGTID(gtid mysql.GTIDSet) error
+	// OnPosSynced Use your own way to sync position. When force is true, sync position immediately.
+	OnPosSynced(pos mysql.Position, force bool) error
+	String() string
+}
+
+type DummyEventHandler struct {
+}
+
+func (h *DummyEventHandler) OnRotate(*replication.RotateEvent) error { return nil }
+func (h *DummyEventHandler) OnDDL(mysql.Position, *replication.QueryEvent) error {
+	return nil
+}
+func (h *DummyEventHandler) OnRow(*RowsEvent) error                 { return nil }
+func (h *DummyEventHandler) OnXID(mysql.Position) error             { return nil }
+func (h *DummyEventHandler) OnGTID(mysql.GTIDSet) error             { return nil }
+func (h *DummyEventHandler) OnPosSynced(mysql.Position, bool) error { return nil }
+func (h *DummyEventHandler) String() string                         { return "DummyEventHandler" }
+
+// `SetEventHandler` registers the sync handler, you must register your
+// own handler before starting Canal.
+func (c *Canal) SetEventHandler(h EventHandler) {
+	c.eventHandler = h
+}

vendor/github.com/siddontang/go-mysql/canal/master.go

@@ -0,0 +1,46 @@
+package canal
+
+import (
+	"sync"
+
+	"github.com/siddontang/go-mysql/mysql"
+	log "github.com/sirupsen/logrus"
+)
+
+type masterInfo struct {
+	sync.RWMutex
+
+	pos mysql.Position
+
+	gtid mysql.GTIDSet
+}
+
+func (m *masterInfo) Update(pos mysql.Position) {
+	log.Debugf("update master position %s", pos)
+
+	m.Lock()
+	m.pos = pos
+	m.Unlock()
+}
+
+func (m *masterInfo) UpdateGTID(gtid mysql.GTIDSet) {
+	log.Debugf("update master gtid %s", gtid.String())
+
+	m.Lock()
+	m.gtid = gtid
+	m.Unlock()
+}
+
+func (m *masterInfo) Position() mysql.Position {
+	m.RLock()
+	defer m.RUnlock()
+
+	return m.pos
+}
+
+func (m *masterInfo) GTID() mysql.GTIDSet {
+	m.RLock()
+	defer m.RUnlock()
+
+	return m.gtid
+}

vendor/github.com/siddontang/go-mysql/canal/rows.go

@@ -0,0 +1,69 @@
+package canal
+
+import (
+	"fmt"
+
+	"github.com/juju/errors"
+	"github.com/siddontang/go-mysql/schema"
+)
+
+const (
+	UpdateAction = "update"
+	InsertAction = "insert"
+	DeleteAction = "delete"
+)
+
+type RowsEvent struct {
+	Table  *schema.Table
+	Action string
+	// changed row list
+	// binlog has three update event version, v0, v1 and v2.
+	// for v1 and v2, the rows number must be even.
+	// Two rows for one event, format is [before update row, after update row]
+	// for update v0, only one row for a event, and we don't support this version.
+	Rows [][]interface{}
+}
+
+func newRowsEvent(table *schema.Table, action string, rows [][]interface{}) *RowsEvent {
+	e := new(RowsEvent)
+
+	e.Table = table
+	e.Action = action
+	e.Rows = rows
+
+	return e
+}
+
+// Get primary keys in one row for a table, a table may use multi fields as the PK
+func GetPKValues(table *schema.Table, row []interface{}) ([]interface{}, error) {
+	indexes := table.PKColumns
+	if len(indexes) == 0 {
+		return nil, errors.Errorf("table %s has no PK", table)
+	} else if len(table.Columns) != len(row) {
+		return nil, errors.Errorf("table %s has %d columns, but row data %v len is %d", table,
+			len(table.Columns), row, len(row))
+	}
+
+	values := make([]interface{}, 0, len(indexes))
+
+	for _, index := range indexes {
+		values = append(values, row[index])
+	}
+
+	return values, nil
+}
+
+// Get term column's value
+func GetColumnValue(table *schema.Table, column string, row []interface{}) (interface{}, error) {
+	index := table.FindColumn(column)
+	if index == -1 {
+		return nil, errors.Errorf("table %s has no column name %s", table, column)
+	}
+
+	return row[index], nil
+}
+
+// String implements fmt.Stringer interface.
+func (r *RowsEvent) String() string {
+	return fmt.Sprintf("%s %s %v", r.Action, r.Table, r.Rows)
+}

vendor/github.com/siddontang/go-mysql/canal/sync.go

@@ -0,0 +1,228 @@
+package canal
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/juju/errors"
+	"github.com/satori/go.uuid"
+	"github.com/siddontang/go-mysql/mysql"
+	"github.com/siddontang/go-mysql/replication"
+	"github.com/siddontang/go-mysql/schema"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	expCreateTable = regexp.MustCompile("(?i)^CREATE\\sTABLE(\\sIF\\sNOT\\sEXISTS)?\\s`{0,1}(.*?)`{0,1}\\.{0,1}`{0,1}([^`\\.]+?)`{0,1}\\s.*")
+	expAlterTable  = regexp.MustCompile("(?i)^ALTER\\sTABLE\\s.*?`{0,1}(.*?)`{0,1}\\.{0,1}`{0,1}([^`\\.]+?)`{0,1}\\s.*")
+	expRenameTable = regexp.MustCompile("(?i)^RENAME\\sTABLE\\s.*?`{0,1}(.*?)`{0,1}\\.{0,1}`{0,1}([^`\\.]+?)`{0,1}\\s{1,}TO\\s.*?")
+	expDropTable   = regexp.MustCompile("(?i)^DROP\\sTABLE(\\sIF\\sEXISTS){0,1}\\s`{0,1}(.*?)`{0,1}\\.{0,1}`{0,1}([^`\\.]+?)`{0,1}($|\\s)")
+)
+
+func (c *Canal) startSyncer() (*replication.BinlogStreamer, error) {
+	if !c.useGTID {
+		pos := c.master.Position()
+		s, err := c.syncer.StartSync(pos)
+		if err != nil {
+			return nil, errors.Errorf("start sync replication at binlog %v error %v", pos, err)
+		}
+		log.Infof("start sync binlog at binlog file %v", pos)
+		return s, nil
+	} else {
+		gset := c.master.GTID()
+		s, err := c.syncer.StartSyncGTID(gset)
+		if err != nil {
+			return nil, errors.Errorf("start sync replication at GTID %v error %v", gset, err)
+		}
+		log.Infof("start sync binlog at GTID %v", gset)
+		return s, nil
+	}
+}
+
+func (c *Canal) runSyncBinlog() error {
+	s, err := c.startSyncer()
+	if err != nil {
+		return err
+	}
+
+	savePos := false
+	force := false
+	for {
+		ev, err := s.GetEvent(c.ctx)
+
+		if err != nil {
+			return errors.Trace(err)
+		}
+		savePos = false
+		force = false
+		pos := c.master.Position()
+
+		curPos := pos.Pos
+		//next binlog pos
+		pos.Pos = ev.Header.LogPos
+
+		// We only save position with RotateEvent and XIDEvent.
+		// For RowsEvent, we can't save the position until meeting XIDEvent
+		// which tells the whole transaction is over.
+		// TODO: If we meet any DDL query, we must save too.
+		switch e := ev.Event.(type) {
+		case *replication.RotateEvent:
+			pos.Name = string(e.NextLogName)
+			pos.Pos = uint32(e.Position)
+			log.Infof("rotate binlog to %s", pos)
+			savePos = true
+			force = true
+			if err = c.eventHandler.OnRotate(e); err != nil {
+				return errors.Trace(err)
+			}
+		case *replication.RowsEvent:
+			// we only focus row based event
+			err = c.handleRowsEvent(ev)
+			if err != nil {
+				e := errors.Cause(err)
+				// if error is not ErrExcludedTable or ErrTableNotExist or ErrMissingTableMeta, stop canal
+				if e != ErrExcludedTable &&
+					e != schema.ErrTableNotExist &&
+					e != schema.ErrMissingTableMeta {
+					log.Errorf("handle rows event at (%s, %d) error %v", pos.Name, curPos, err)
+					return errors.Trace(err)
+				}
+			}
+			continue
+		case *replication.XIDEvent:
+			savePos = true
+			// try to save the position later
+			if err := c.eventHandler.OnXID(pos); err != nil {
+				return errors.Trace(err)
+			}
+		case *replication.MariadbGTIDEvent:
+			// try to save the GTID later
+			gtid := &e.GTID
+			c.master.UpdateGTID(gtid)
+			if err := c.eventHandler.OnGTID(gtid); err != nil {
+				return errors.Trace(err)
+			}
+		case *replication.GTIDEvent:
+			u, _ := uuid.FromBytes(e.SID)
+			gset, err := mysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d", u.String(), e.GNO))
+			if err != nil {
+				return errors.Trace(err)
+			}
+			c.master.UpdateGTID(gset)
+			if err := c.eventHandler.OnGTID(gset); err != nil {
+				return errors.Trace(err)
+			}
+		case *replication.QueryEvent:
+			var (
+				mb     [][]byte
+				schema []byte
+				table  []byte
+			)
+			regexps := []regexp.Regexp{*expCreateTable, *expAlterTable, *expRenameTable, *expDropTable}
+			for _, reg := range regexps {
+				mb = reg.FindSubmatch(e.Query)
+				if len(mb) != 0 {
+					break
+				}
+			}
+			mbLen := len(mb)
+			if mbLen == 0 {
+				continue
+			}
+
+			// the first last is table name, the second last is database name(if exists)
+			if len(mb[mbLen-2]) == 0 {
+				schema = e.Schema
+			} else {
+				schema = mb[mbLen-2]
+			}
+			table = mb[mbLen-1]
+
+			savePos = true
+			force = true
+			c.ClearTableCache(schema, table)
+			log.Infof("table structure changed, clear table cache: %s.%s\n", schema, table)
+			if err = c.eventHandler.OnDDL(pos, e); err != nil {
+				return errors.Trace(err)
+			}
+
+		default:
+			continue
+		}
+
+		if savePos {
+			c.master.Update(pos)
+			c.eventHandler.OnPosSynced(pos, force)
+		}
+	}
+
+	return nil
+}
+
+func (c *Canal) handleRowsEvent(e *replication.BinlogEvent) error {
+	ev := e.Event.(*replication.RowsEvent)
+
+	// Caveat: table may be altered at runtime.
+	schema := string(ev.Table.Schema)
+	table := string(ev.Table.Table)
+
+	t, err := c.GetTable(schema, table)
+	if err != nil {
+		return err
+	}
+	var action string
+	switch e.Header.EventType {
+	case replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2:
+		action = InsertAction
+	case replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2:
+		action = DeleteAction
+	case replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2:
+		action = UpdateAction
+	default:
+		return errors.Errorf("%s not supported now", e.Header.EventType)
+	}
+	events := newRowsEvent(t, action, ev.Rows)
+	return c.eventHandler.OnRow(events)
+}
+
+func (c *Canal) WaitUntilPos(pos mysql.Position, timeout time.Duration) error {
+	timer := time.NewTimer(timeout)
+	for {
+		select {
+		case <-timer.C:
+			return errors.Errorf("wait position %v too long > %s", pos, timeout)
+		default:
+			curPos := c.master.Position()
+			if curPos.Compare(pos) >= 0 {
+				return nil
+			} else {
+				log.Debugf("master pos is %v, wait catching %v", curPos, pos)
+				time.Sleep(100 * time.Millisecond)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *Canal) GetMasterPos() (mysql.Position, error) {
+	rr, err := c.Execute("SHOW MASTER STATUS")
+	if err != nil {
+		return mysql.Position{"", 0}, errors.Trace(err)
+	}
+
+	name, _ := rr.GetString(0, 0)
+	pos, _ := rr.GetInt(0, 1)
+
+	return mysql.Position{name, uint32(pos)}, nil
+}
+
+func (c *Canal) CatchMasterPos(timeout time.Duration) error {
+	pos, err := c.GetMasterPos()
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	return c.WaitUntilPos(pos, timeout)
+}