Initial commit

vendor/github.com/siddontang/go-mysql/dump/BUILD.bazel

@@ -0,0 +1,30 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dump.go",
+        "parser.go",
+    ],
+    importmap = "go-common/vendor/github.com/siddontang/go-mysql/dump",
+    importpath = "github.com/siddontang/go-mysql/dump",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/juju/errors:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/mysql:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/siddontang/go-mysql/dump/dump.go

@@ -0,0 +1,194 @@
+package dump
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/juju/errors"
+	. "github.com/siddontang/go-mysql/mysql"
+)
+
+// Unlick mysqldump, Dumper is designed for parsing and syning data easily.
+type Dumper struct {
+	// mysqldump execution path, like mysqldump or /usr/bin/mysqldump, etc...
+	ExecutionPath string
+
+	Addr     string
+	User     string
+	Password string
+
+	// Will override Databases
+	Tables  []string
+	TableDB string
+
+	Databases []string
+
+	Charset string
+
+	IgnoreTables map[string][]string
+
+	ErrOut io.Writer
+
+	masterDataSkipped bool
+	maxAllowedPacket  int
+}
+
+func NewDumper(executionPath string, addr string, user string, password string) (*Dumper, error) {
+	if len(executionPath) == 0 {
+		return nil, nil
+	}
+
+	path, err := exec.LookPath(executionPath)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+
+	d := new(Dumper)
+	d.ExecutionPath = path
+	d.Addr = addr
+	d.User = user
+	d.Password = password
+	d.Tables = make([]string, 0, 16)
+	d.Databases = make([]string, 0, 16)
+	d.Charset = DEFAULT_CHARSET
+	d.IgnoreTables = make(map[string][]string)
+	d.masterDataSkipped = false
+
+	d.ErrOut = os.Stderr
+
+	return d, nil
+}
+
+func (d *Dumper) SetCharset(charset string) {
+	d.Charset = charset
+}
+
+func (d *Dumper) SetErrOut(o io.Writer) {
+	d.ErrOut = o
+}
+
+// In some cloud MySQL, we have no privilege to use `--master-data`.
+func (d *Dumper) SkipMasterData(v bool) {
+	d.masterDataSkipped = v
+}
+
+func (d *Dumper) SetMaxAllowedPacket(i int) {
+	d.maxAllowedPacket = i
+}
+
+func (d *Dumper) AddDatabases(dbs ...string) {
+	d.Databases = append(d.Databases, dbs...)
+}
+
+func (d *Dumper) AddTables(db string, tables ...string) {
+	if d.TableDB != db {
+		d.TableDB = db
+		d.Tables = d.Tables[0:0]
+	}
+
+	d.Tables = append(d.Tables, tables...)
+}
+
+func (d *Dumper) AddIgnoreTables(db string, tables ...string) {
+	t, _ := d.IgnoreTables[db]
+	t = append(t, tables...)
+	d.IgnoreTables[db] = t
+}
+
+func (d *Dumper) Reset() {
+	d.Tables = d.Tables[0:0]
+	d.TableDB = ""
+	d.IgnoreTables = make(map[string][]string)
+	d.Databases = d.Databases[0:0]
+}
+
+func (d *Dumper) Dump(w io.Writer) error {
+	args := make([]string, 0, 16)
+
+	// Common args
+	seps := strings.Split(d.Addr, ":")
+	args = append(args, fmt.Sprintf("--host=%s", seps[0]))
+	if len(seps) > 1 {
+		args = append(args, fmt.Sprintf("--port=%s", seps[1]))
+	}
+
+	args = append(args, fmt.Sprintf("--user=%s", d.User))
+	args = append(args, fmt.Sprintf("--password=%s", d.Password))
+
+	if !d.masterDataSkipped {
+		args = append(args, "--master-data")
+	}
+
+	if d.maxAllowedPacket > 0 {
+		// mysqldump param should be --max-allowed-packet=%dM not be --max_allowed_packet=%dM
+		args = append(args, fmt.Sprintf("--max-allowed-packet=%dM", d.maxAllowedPacket))
+	}
+
+	args = append(args, "--single-transaction")
+	args = append(args, "--skip-lock-tables")
+
+	// Disable uncessary data
+	args = append(args, "--compact")
+	args = append(args, "--skip-opt")
+	args = append(args, "--quick")
+
+	// We only care about data
+	args = append(args, "--no-create-info")
+
+	// Multi row is easy for us to parse the data
+	args = append(args, "--skip-extended-insert")
+
+	for db, tables := range d.IgnoreTables {
+		for _, table := range tables {
+			args = append(args, fmt.Sprintf("--ignore-table=%s.%s", db, table))
+		}
+	}
+
+	if len(d.Tables) == 0 && len(d.Databases) == 0 {
+		args = append(args, "--all-databases")
+	} else if len(d.Tables) == 0 {
+		args = append(args, "--databases")
+		args = append(args, d.Databases...)
+	} else {
+		args = append(args, d.TableDB)
+		args = append(args, d.Tables...)
+
+		// If we only dump some tables, the dump data will not have database name
+		// which makes us hard to parse, so here we add it manually.
+
+		w.Write([]byte(fmt.Sprintf("USE `%s`;\n", d.TableDB)))
+	}
+
+	if len(d.Charset) != 0 {
+		args = append(args, fmt.Sprintf("--default-character-set=%s", d.Charset))
+	}
+
+	cmd := exec.Command(d.ExecutionPath, args...)
+
+	cmd.Stderr = d.ErrOut
+	cmd.Stdout = w
+
+	return cmd.Run()
+}
+
+// Dump MySQL and parse immediately
+func (d *Dumper) DumpAndParse(h ParseHandler) error {
+	r, w := io.Pipe()
+
+	done := make(chan error, 1)
+	go func() {
+		err := Parse(r, h, !d.masterDataSkipped)
+		r.CloseWithError(err)
+		done <- err
+	}()
+
+	err := d.Dump(w)
+	w.CloseWithError(err)
+
+	err = <-done
+
+	return errors.Trace(err)
+}

vendor/github.com/siddontang/go-mysql/dump/parser.go

@@ -0,0 +1,190 @@
+package dump
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/juju/errors"
+	"github.com/siddontang/go-mysql/mysql"
+)
+
+var (
+	ErrSkip = errors.New("Handler error, but skipped")
+)
+
+type ParseHandler interface {
+	// Parse CHANGE MASTER TO MASTER_LOG_FILE=name, MASTER_LOG_POS=pos;
+	BinLog(name string, pos uint64) error
+
+	Data(schema string, table string, values []string) error
+}
+
+var binlogExp *regexp.Regexp
+var useExp *regexp.Regexp
+var valuesExp *regexp.Regexp
+
+func init() {
+	binlogExp = regexp.MustCompile("^CHANGE MASTER TO MASTER_LOG_FILE='(.+)', MASTER_LOG_POS=(\\d+);")
+	useExp = regexp.MustCompile("^USE `(.+)`;")
+	valuesExp = regexp.MustCompile("^INSERT INTO `(.+?)` VALUES \\((.+)\\);$")
+}
+
+// Parse the dump data with Dumper generate.
+// It can not parse all the data formats with mysqldump outputs
+func Parse(r io.Reader, h ParseHandler, parseBinlogPos bool) error {
+	rb := bufio.NewReaderSize(r, 1024*16)
+
+	var db string
+	var binlogParsed bool
+
+	for {
+		line, err := rb.ReadString('\n')
+		if err != nil && err != io.EOF {
+			return errors.Trace(err)
+		} else if mysql.ErrorEqual(err, io.EOF) {
+			break
+		}
+
+		// Ignore '\n' on Linux or '\r\n' on Windows
+		line = strings.SplitAfter(line, ";")[0]
+
+		if parseBinlogPos && !binlogParsed {
+			if m := binlogExp.FindAllStringSubmatch(line, -1); len(m) == 1 {
+				name := m[0][1]
+				pos, err := strconv.ParseUint(m[0][2], 10, 64)
+				if err != nil {
+					return errors.Errorf("parse binlog %v err, invalid number", line)
+				}
+
+				if err = h.BinLog(name, pos); err != nil && err != ErrSkip {
+					return errors.Trace(err)
+				}
+
+				binlogParsed = true
+			}
+		}
+
+		if m := useExp.FindAllStringSubmatch(line, -1); len(m) == 1 {
+			db = m[0][1]
+		}
+
+		if m := valuesExp.FindAllStringSubmatch(line, -1); len(m) == 1 {
+			table := m[0][1]
+
+			values, err := parseValues(m[0][2])
+			if err != nil {
+				return errors.Errorf("parse values %v err", line)
+			}
+
+			if err = h.Data(db, table, values); err != nil && err != ErrSkip {
+				return errors.Trace(err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func parseValues(str string) ([]string, error) {
+	// values are seperated by comma, but we can not split using comma directly
+	// string is enclosed by single quote
+
+	// a simple implementation, may be more robust later.
+
+	values := make([]string, 0, 8)
+
+	i := 0
+	for i < len(str) {
+		if str[i] != '\'' {
+			// no string, read until comma
+			j := i + 1
+			for ; j < len(str) && str[j] != ','; j++ {
+			}
+			values = append(values, str[i:j])
+			// skip ,
+			i = j + 1
+		} else {
+			// read string until another single quote
+			j := i + 1
+
+			escaped := false
+			for j < len(str) {
+				if str[j] == '\\' {
+					// skip escaped character
+					j += 2
+					escaped = true
+					continue
+				} else if str[j] == '\'' {
+					break
+				} else {
+					j++
+				}
+			}
+
+			if j >= len(str) {
+				return nil, fmt.Errorf("parse quote values error")
+			}
+
+			value := str[i : j+1]
+			if escaped {
+				value = unescapeString(value)
+			}
+			values = append(values, value)
+			// skip ' and ,
+			i = j + 2
+		}
+
+		// need skip blank???
+	}
+
+	return values, nil
+}
+
+// unescapeString un-escapes the string.
+// mysqldump will escape the string when dumps,
+// Refer http://dev.mysql.com/doc/refman/5.7/en/string-literals.html
+func unescapeString(s string) string {
+	i := 0
+
+	value := make([]byte, 0, len(s))
+	for i < len(s) {
+		if s[i] == '\\' {
+			j := i + 1
+			if j == len(s) {
+				// The last char is \, remove
+				break
+			}
+
+			value = append(value, unescapeChar(s[j]))
+			i += 2
+		} else {
+			value = append(value, s[i])
+			i++
+		}
+	}
+
+	return string(value)
+}
+
+func unescapeChar(ch byte) byte {
+	// \" \' \\ \n \0 \b \Z \r \t ==> escape to one char
+	switch ch {
+	case 'n':
+		ch = '\n'
+	case '0':
+		ch = 0
+	case 'b':
+		ch = 8
+	case 'Z':
+		ch = 26
+	case 'r':
+		ch = '\r'
+	case 't':
+		ch = '\t'
+	}
+	return ch
+}