Initial commit

vendor/github.com/tsuna/gohbase/region/BUILD.bazel

@@ -0,0 +1,34 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "client.go",
+        "info.go",
+        "multi.go",
+        "new.go",
+    ],
+    importmap = "go-common/vendor/github.com/tsuna/gohbase/region",
+    importpath = "github.com/tsuna/gohbase/region",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/pb:go_default_library",
+        "@com_github_golang_protobuf//proto:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/tsuna/gohbase/region/client.go

@@ -0,0 +1,566 @@
+// Copyright (C) 2015  The GoHBase Authors.  All rights reserved.
+// This file is part of GoHBase.
+// Use of this source code is governed by the Apache License 2.0
+// that can be found in the COPYING file.
+
+package region
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/tsuna/gohbase/hrpc"
+	"github.com/tsuna/gohbase/pb"
+)
+
+// ClientType is a type alias to represent the type of this region client
+type ClientType string
+
+type canDeserializeCellBlocks interface {
+	// DeserializeCellBlocks populates passed protobuf message with results
+	// deserialized from the reader and returns number of bytes read or error.
+	DeserializeCellBlocks(proto.Message, []byte) (uint32, error)
+}
+
+var (
+	// ErrMissingCallID is used when HBase sends us a response message for a
+	// request that we didn't send
+	ErrMissingCallID = errors.New("got a response with a nonsensical call ID")
+
+	// ErrClientDead is returned to rpcs when Close() is called or when client
+	// died because of failed send or receive
+	ErrClientDead = UnrecoverableError{errors.New("client is dead")}
+
+	// javaRetryableExceptions is a map where all Java exceptions that signify
+	// the RPC should be sent again are listed (as keys). If a Java exception
+	// listed here is returned by HBase, the client should attempt to resend
+	// the RPC message, potentially via a different region client.
+	javaRetryableExceptions = map[string]struct{}{
+		"org.apache.hadoop.hbase.CallQueueTooBigException":          struct{}{},
+		"org.apache.hadoop.hbase.NotServingRegionException":         struct{}{},
+		"org.apache.hadoop.hbase.exceptions.RegionMovedException":   struct{}{},
+		"org.apache.hadoop.hbase.exceptions.RegionOpeningException": struct{}{},
+		"org.apache.hadoop.hbase.ipc.ServerNotRunningYetException":  struct{}{},
+		"org.apache.hadoop.hbase.quotas.RpcThrottlingException":     struct{}{},
+		"org.apache.hadoop.hbase.RetryImmediatelyException":         struct{}{},
+	}
+
+	// javaUnrecoverableExceptions is a map where all Java exceptions that signify
+	// the RPC should be sent again are listed (as keys). If a Java exception
+	// listed here is returned by HBase, the RegionClient will be closed and a new
+	// one should be established.
+	javaUnrecoverableExceptions = map[string]struct{}{
+		"org.apache.hadoop.hbase.regionserver.RegionServerAbortedException": struct{}{},
+		"org.apache.hadoop.hbase.regionserver.RegionServerStoppedException": struct{}{},
+	}
+)
+
+const (
+	//DefaultLookupTimeout is the default region lookup timeout
+	DefaultLookupTimeout = 30 * time.Second
+	//DefaultReadTimeout is the default region read timeout
+	DefaultReadTimeout = 30 * time.Second
+	// RegionClient is a ClientType that means this will be a normal client
+	RegionClient = ClientType("ClientService")
+
+	// MasterClient is a ClientType that means this client will talk to the
+	// master server
+	MasterClient = ClientType("MasterService")
+)
+
+var bufferPool = sync.Pool{
+	New: func() interface{} {
+		var b []byte
+		return b
+	},
+}
+
+func newBuffer(size int) []byte {
+	b := bufferPool.Get().([]byte)
+	if cap(b) < size {
+		doublecap := 2 * cap(b)
+		if doublecap > size {
+			return make([]byte, size, doublecap)
+		}
+		return make([]byte, size)
+	}
+	return b[:size]
+}
+
+func freeBuffer(b []byte) {
+	bufferPool.Put(b[:0])
+}
+
+// UnrecoverableError is an error that this region.Client can't recover from.
+// The connection to the RegionServer has to be closed and all queued and
+// outstanding RPCs will be failed / retried.
+type UnrecoverableError struct {
+	error
+}
+
+func (e UnrecoverableError) Error() string {
+	return e.error.Error()
+}
+
+// RetryableError is an error that indicates the RPC should be retried because
+// the error is transient (e.g. a region being momentarily unavailable).
+type RetryableError struct {
+	error
+}
+
+func (e RetryableError) Error() string {
+	return e.error.Error()
+}
+
+// client manages a connection to a RegionServer.
+type client struct {
+	conn net.Conn
+
+	// Address of the RegionServer.
+	addr string
+
+	// once used for concurrent calls to fail
+	once sync.Once
+
+	rpcs chan hrpc.Call
+	done chan struct{}
+
+	// sent contains the mapping of sent call IDs to RPC calls, so that when
+	// a response is received it can be tied to the correct RPC
+	sentM sync.Mutex // protects sent
+	sent  map[uint32]hrpc.Call
+
+	// inFlight is number of rpcs sent to regionserver awaiting response
+	inFlightM sync.Mutex // protects inFlight and SetReadDeadline
+	inFlight  uint32
+
+	id uint32
+
+	rpcQueueSize  int
+	flushInterval time.Duration
+
+	effectiveUser string
+
+	// readTimeout is the maximum amount of time to wait for regionserver reply
+	readTimeout time.Duration
+}
+
+// QueueRPC will add an rpc call to the queue for processing by the writer goroutine
+func (c *client) QueueRPC(rpc hrpc.Call) {
+	if b, ok := rpc.(hrpc.Batchable); ok && c.rpcQueueSize > 1 && !b.SkipBatch() {
+		// queue up the rpc
+		select {
+		case <-rpc.Context().Done():
+			// rpc timed out before being processed
+		case <-c.done:
+			returnResult(rpc, nil, ErrClientDead)
+		case c.rpcs <- rpc:
+		}
+	} else {
+		if err := c.trySend(rpc); err != nil {
+			returnResult(rpc, nil, err)
+		}
+	}
+}
+
+// Close asks this region.Client to close its connection to the RegionServer.
+// All queued and outstanding RPCs, if any, will be failed as if a connection
+// error had happened.
+func (c *client) Close() {
+	c.fail(ErrClientDead)
+}
+
+// Addr returns address of the region server the client is connected to
+func (c *client) Addr() string {
+	return c.addr
+}
+
+// String returns a string represintation of the current region client
+func (c *client) String() string {
+	return fmt.Sprintf("RegionClient{Addr: %s}", c.addr)
+}
+
+func (c *client) inFlightUp() {
+	c.inFlightM.Lock()
+	c.inFlight++
+	// we expect that at least the last request can be completed within readTimeout
+	c.conn.SetReadDeadline(time.Now().Add(c.readTimeout))
+	c.inFlightM.Unlock()
+}
+
+func (c *client) inFlightDown() {
+	c.inFlightM.Lock()
+	c.inFlight--
+	// reset read timeout if we are not waiting for any responses
+	// in order to prevent from closing this client if there are no request
+	if c.inFlight == 0 {
+		c.conn.SetReadDeadline(time.Time{})
+	}
+	c.inFlightM.Unlock()
+}
+
+func (c *client) fail(err error) {
+	c.once.Do(func() {
+		log.WithFields(log.Fields{
+			"client": c,
+			"err":    err,
+		}).Error("error occured, closing region client")
+
+		// we don't close c.rpcs channel to make it block in select of QueueRPC
+		// and avoid dealing with synchronization of closing it while someone
+		// might be sending to it. Go's GC will take care of it.
+
+		// tell goroutines to stop
+		close(c.done)
+
+		// close connection to the regionserver
+		// to let it know that we can't receive anymore
+		// and fail all the rpcs being sent
+		c.conn.Close()
+
+		c.failSentRPCs()
+	})
+}
+
+func (c *client) failSentRPCs() {
+	// channel is closed, clean up awaiting rpcs
+	c.sentM.Lock()
+	sent := c.sent
+	c.sent = make(map[uint32]hrpc.Call)
+	c.sentM.Unlock()
+
+	log.WithFields(log.Fields{
+		"client": c,
+		"count":  len(sent),
+	}).Debug("failing awaiting RPCs")
+
+	// send error to awaiting rpcs
+	for _, rpc := range sent {
+		returnResult(rpc, nil, ErrClientDead)
+	}
+}
+
+func (c *client) registerRPC(rpc hrpc.Call) uint32 {
+	currID := atomic.AddUint32(&c.id, 1)
+	c.sentM.Lock()
+	c.sent[currID] = rpc
+	c.sentM.Unlock()
+	return currID
+}
+
+func (c *client) unregisterRPC(id uint32) hrpc.Call {
+	c.sentM.Lock()
+	rpc := c.sent[id]
+	delete(c.sent, id)
+	c.sentM.Unlock()
+	return rpc
+}
+
+func (c *client) processRPCs() {
+	// TODO: flush when the size is too large
+	// TODO: if multi has only one call, send that call instead
+	m := newMulti(c.rpcQueueSize)
+	defer func() {
+		m.returnResults(nil, ErrClientDead)
+	}()
+
+	flush := func() {
+		if log.GetLevel() == log.DebugLevel {
+			log.WithFields(log.Fields{
+				"len":  m.len(),
+				"addr": c.Addr(),
+			}).Debug("flushing MultiRequest")
+		}
+		if err := c.trySend(m); err != nil {
+			m.returnResults(nil, err)
+		}
+		m = newMulti(c.rpcQueueSize)
+	}
+
+	for {
+		// first loop is to accomodate request heavy workload
+		// it will batch as long as conccurent writers are sending
+		// new rpcs or until multi is filled up
+		for {
+			select {
+			case <-c.done:
+				return
+			case rpc := <-c.rpcs:
+				// have things queued up, batch them
+				if !m.add(rpc) {
+					// can still put more rpcs into batch
+					continue
+				}
+			default:
+				// no more rpcs queued up
+			}
+			break
+		}
+
+		if l := m.len(); l == 0 {
+			// wait for the next batch
+			select {
+			case <-c.done:
+				return
+			case rpc := <-c.rpcs:
+				m.add(rpc)
+			}
+			continue
+		} else if l == c.rpcQueueSize || c.flushInterval == 0 {
+			// batch is full, flush
+			flush()
+			continue
+		}
+
+		// second loop is to accomodate less frequent callers
+		// that would like to maximize their batches at the expense
+		// of waiting for flushInteval
+		timer := time.NewTimer(c.flushInterval)
+		for {
+			select {
+			case <-c.done:
+				return
+			case <-timer.C:
+				// time to flush
+			case rpc := <-c.rpcs:
+				if !m.add(rpc) {
+					// can still put more rpcs into batch
+					continue
+				}
+				// batch is full
+				if !timer.Stop() {
+					<-timer.C
+				}
+			}
+			break
+		}
+		flush()
+	}
+}
+
+func returnResult(c hrpc.Call, msg proto.Message, err error) {
+	if m, ok := c.(*multi); ok {
+		m.returnResults(msg, err)
+	} else {
+		c.ResultChan() <- hrpc.RPCResult{Msg: msg, Error: err}
+	}
+}
+
+func (c *client) trySend(rpc hrpc.Call) error {
+	select {
+	case <-c.done:
+		// An unrecoverable error has occured,
+		// region client has been stopped,
+		// don't send rpcs
+		return ErrClientDead
+	case <-rpc.Context().Done():
+		// If the deadline has been exceeded, don't bother sending the
+		// request. The function that placed the RPC in our queue should
+		// stop waiting for a result and return an error.
+		return nil
+	default:
+		if id, err := c.send(rpc); err != nil {
+			if _, ok := err.(UnrecoverableError); ok {
+				c.fail(err)
+			}
+			if r := c.unregisterRPC(id); r != nil {
+				// we are the ones to unregister the rpc,
+				// return err to notify client of it
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func (c *client) receiveRPCs() {
+	for {
+		select {
+		case <-c.done:
+			return
+		default:
+			if err := c.receive(); err != nil {
+				switch err.(type) {
+				case UnrecoverableError:
+					c.fail(err)
+					return
+				case RetryableError:
+					continue
+				}
+			}
+		}
+	}
+}
+
+func (c *client) receive() (err error) {
+	var (
+		sz       [4]byte
+		header   pb.ResponseHeader
+		response proto.Message
+	)
+
+	err = c.readFully(sz[:])
+	if err != nil {
+		return UnrecoverableError{err}
+	}
+
+	size := binary.BigEndian.Uint32(sz[:])
+	b := make([]byte, size)
+
+	err = c.readFully(b)
+	if err != nil {
+		return UnrecoverableError{err}
+	}
+
+	buf := proto.NewBuffer(b)
+
+	if err = buf.DecodeMessage(&header); err != nil {
+		return fmt.Errorf("failed to decode the response header: %s", err)
+	}
+	if header.CallId == nil {
+		return ErrMissingCallID
+	}
+
+	callID := *header.CallId
+	rpc := c.unregisterRPC(callID)
+	if rpc == nil {
+		return fmt.Errorf("got a response with an unexpected call ID: %d", callID)
+	}
+	c.inFlightDown()
+
+	select {
+	case <-rpc.Context().Done():
+		// context has expired, don't bother deserializing
+		return
+	default:
+	}
+
+	// Here we know for sure that we got a response for rpc we asked.
+	// It's our responsibility to deliver the response or error to the
+	// caller as we unregistered the rpc.
+	defer func() { returnResult(rpc, response, err) }()
+
+	if header.Exception == nil {
+		response = rpc.NewResponse()
+		if err = buf.DecodeMessage(response); err != nil {
+			err = fmt.Errorf("failed to decode the response: %s", err)
+			return
+		}
+		var cellsLen uint32
+		if header.CellBlockMeta != nil {
+			cellsLen = header.CellBlockMeta.GetLength()
+		}
+		if d, ok := rpc.(canDeserializeCellBlocks); cellsLen > 0 && ok {
+			b := buf.Bytes()[size-cellsLen:]
+			var nread uint32
+			nread, err = d.DeserializeCellBlocks(response, b)
+			if err != nil {
+				err = fmt.Errorf("failed to decode the response: %s", err)
+				return
+			}
+			if int(nread) < len(b) {
+				err = fmt.Errorf("short read: buffer len %d, read %d", len(b), nread)
+				return
+			}
+		}
+	} else {
+		err = exceptionToError(*header.Exception.ExceptionClassName, *header.Exception.StackTrace)
+	}
+	return
+}
+
+func exceptionToError(class, stack string) error {
+	err := fmt.Errorf("HBase Java exception %s:\n%s", class, stack)
+	if _, ok := javaRetryableExceptions[class]; ok {
+		return RetryableError{err}
+	} else if _, ok := javaUnrecoverableExceptions[class]; ok {
+		return UnrecoverableError{err}
+	}
+	return err
+}
+
+// write sends the given buffer to the RegionServer.
+func (c *client) write(buf []byte) error {
+	_, err := c.conn.Write(buf)
+	return err
+}
+
+// Tries to read enough data to fully fill up the given buffer.
+func (c *client) readFully(buf []byte) error {
+	_, err := io.ReadFull(c.conn, buf)
+	return err
+}
+
+// sendHello sends the "hello" message needed when opening a new connection.
+func (c *client) sendHello(ctype ClientType) error {
+	connHeader := &pb.ConnectionHeader{
+		UserInfo: &pb.UserInformation{
+			EffectiveUser: proto.String(c.effectiveUser),
+		},
+		ServiceName:         proto.String(string(ctype)),
+		CellBlockCodecClass: proto.String("org.apache.hadoop.hbase.codec.KeyValueCodec"),
+	}
+	data, err := proto.Marshal(connHeader)
+	if err != nil {
+		return fmt.Errorf("failed to marshal connection header: %s", err)
+	}
+
+	const header = "HBas\x00\x50" // \x50 = Simple Auth.
+	buf := make([]byte, 0, len(header)+4+len(data))
+	buf = append(buf, header...)
+	buf = buf[:len(header)+4]
+	binary.BigEndian.PutUint32(buf[6:], uint32(len(data)))
+	buf = append(buf, data...)
+	return c.write(buf)
+}
+
+// send sends an RPC out to the wire.
+// Returns the response (for now, as the call is synchronous).
+func (c *client) send(rpc hrpc.Call) (uint32, error) {
+	b := newBuffer(4)
+	defer func() { freeBuffer(b) }()
+
+	buf := proto.NewBuffer(b[4:])
+	buf.Reset()
+
+	request := rpc.ToProto()
+
+	// we have to register rpc after we marhsal because
+	// registered rpc can fail before it was even sent
+	// in all the cases where c.fail() is called.
+	// If that happens, client can retry sending the rpc
+	// again potentially changing it's contents.
+	id := c.registerRPC(rpc)
+
+	header := &pb.RequestHeader{
+		CallId:       &id,
+		MethodName:   proto.String(rpc.Name()),
+		RequestParam: proto.Bool(true),
+	}
+	if err := buf.EncodeMessage(header); err != nil {
+		return id, fmt.Errorf("failed to marshal request header: %s", err)
+	}
+
+	if err := buf.EncodeMessage(request); err != nil {
+		return id, fmt.Errorf("failed to marshal request: %s", err)
+	}
+
+	payload := buf.Bytes()
+	binary.BigEndian.PutUint32(b, uint32(len(payload)))
+	b = append(b[:4], payload...)
+
+	if err := c.write(b); err != nil {
+		return id, UnrecoverableError{err}
+	}
+	c.inFlightUp()
+	return id, nil
+}

vendor/github.com/tsuna/gohbase/region/info.go

@@ -0,0 +1,349 @@
+// Copyright (C) 2015  The GoHBase Authors.  All rights reserved.
+// This file is part of GoHBase.
+// Use of this source code is governed by the Apache License 2.0
+// that can be found in the COPYING file.
+
+// Package region contains data structures to represent HBase regions.
+package region
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"sync"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/tsuna/gohbase/hrpc"
+	"github.com/tsuna/gohbase/pb"
+)
+
+var defaultNamespace = []byte("default")
+
+// OfflineRegionError is returned if region is offline
+type OfflineRegionError struct {
+	n string
+}
+
+func (e OfflineRegionError) Error() string {
+	return fmt.Sprintf("region %s is offline", e.n)
+}
+
+// info describes a region.
+type info struct {
+	id        uint64 // A timestamp when the region is created
+	namespace []byte
+	table     []byte
+	name      []byte
+	startKey  []byte
+	stopKey   []byte
+	ctx       context.Context
+	cancel    context.CancelFunc
+
+	// The attributes before this mutex are supposed to be immutable.
+	// The attributes defined below can be changed and accesses must
+	// be protected with this mutex.
+	m sync.RWMutex
+
+	client hrpc.RegionClient
+
+	// Once a region becomes unreachable, this channel is created, and any
+	// functions that wish to be notified when the region becomes available
+	// again can read from this channel, which will be closed when the region
+	// is available again
+	available chan struct{}
+}
+
+// NewInfo creates a new region info
+func NewInfo(id uint64, namespace, table, name, startKey, stopKey []byte) hrpc.RegionInfo {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &info{
+		id:        id,
+		ctx:       ctx,
+		cancel:    cancel,
+		namespace: namespace,
+		table:     table,
+		name:      name,
+		startKey:  startKey,
+		stopKey:   stopKey,
+	}
+}
+
+// infoFromCell parses a KeyValue from the meta table and creates the
+// corresponding Info object.
+func infoFromCell(cell *hrpc.Cell) (hrpc.RegionInfo, error) {
+	value := cell.Value
+	if len(value) == 0 {
+		return nil, fmt.Errorf("empty value in %q", cell)
+	} else if value[0] != 'P' {
+		return nil, fmt.Errorf("unsupported region info version %d in %q", value[0], cell)
+	}
+	const pbufMagic = 1346524486 // 4 bytes: "PBUF"
+	magic := binary.BigEndian.Uint32(value[:4])
+	if magic != pbufMagic {
+		return nil, fmt.Errorf("invalid magic number in %q", cell)
+	}
+	var regInfo pb.RegionInfo
+	err := proto.UnmarshalMerge(value[4:], &regInfo)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode %q: %s", cell, err)
+	}
+	if regInfo.GetOffline() {
+		return nil, OfflineRegionError{n: string(cell.Row)}
+	}
+	var namespace []byte
+	if !bytes.Equal(regInfo.TableName.Namespace, defaultNamespace) {
+		// if default namespace, pretend there's no namespace
+		namespace = regInfo.TableName.Namespace
+	}
+
+	return NewInfo(
+		regInfo.GetRegionId(),
+		namespace,
+		regInfo.TableName.Qualifier,
+		cell.Row,
+		regInfo.StartKey,
+		regInfo.EndKey,
+	), nil
+}
+
+// ParseRegionInfo parses the contents of a row from the meta table.
+// It's guaranteed to return a region info and a host:port OR return an error.
+func ParseRegionInfo(metaRow *hrpc.Result) (hrpc.RegionInfo, string, error) {
+	var reg hrpc.RegionInfo
+	var addr string
+
+	for _, cell := range metaRow.Cells {
+		switch string(cell.Qualifier) {
+		case "regioninfo":
+			var err error
+			reg, err = infoFromCell(cell)
+			if err != nil {
+				return nil, "", err
+			}
+		case "server":
+			value := cell.Value
+			if len(value) == 0 {
+				continue // Empty during NSRE.
+			}
+			addr = string(value)
+		default:
+			// Other kinds of qualifiers: ignore them.
+			// TODO: If this is the parent of a split region, there are two other
+			// KVs that could be useful: `info:splitA' and `info:splitB'.
+			// Need to investigate whether we can use those as a hint to update our
+			// regions_cache with the daughter regions of the split.
+		}
+	}
+
+	if reg == nil {
+		// There was no region in the row in meta, this is really not expected.
+		return nil, "", fmt.Errorf("meta seems to be broken, there was no region in %v", metaRow)
+	}
+	if len(addr) == 0 {
+		return nil, "", fmt.Errorf("meta doesn't have a server location in %v", metaRow)
+	}
+	return reg, addr, nil
+}
+
+// IsUnavailable returns true if this region has been marked as unavailable.
+func (i *info) IsUnavailable() bool {
+	i.m.RLock()
+	res := i.available != nil
+	i.m.RUnlock()
+	return res
+}
+
+// AvailabilityChan returns a channel that can be used to wait on for
+// notification that a connection to this region has been reestablished.
+// If this region is not marked as unavailable, nil will be returned.
+func (i *info) AvailabilityChan() <-chan struct{} {
+	i.m.RLock()
+	ch := i.available
+	i.m.RUnlock()
+	return ch
+}
+
+// MarkUnavailable will mark this region as unavailable, by creating the struct
+// returned by AvailabilityChan. If this region was marked as available
+// before this, true will be returned.
+func (i *info) MarkUnavailable() bool {
+	created := false
+	i.m.Lock()
+	if i.available == nil {
+		i.available = make(chan struct{})
+		created = true
+	}
+	i.m.Unlock()
+	return created
+}
+
+// MarkAvailable will mark this region as available again, by closing the struct
+// returned by AvailabilityChan
+func (i *info) MarkAvailable() {
+	i.m.Lock()
+	ch := i.available
+	i.available = nil
+	close(ch)
+	i.m.Unlock()
+}
+
+// MarkDead will mark this region as not useful anymore to notify everyone
+// who's trying to use it that there's no point
+func (i *info) MarkDead() {
+	i.cancel()
+}
+
+// Context to check if the region is dead
+func (i *info) Context() context.Context {
+	return i.ctx
+}
+
+func (i *info) String() string {
+	return fmt.Sprintf(
+		"RegionInfo{Name: %q, ID: %d, Namespace: %q, Table: %q, StartKey: %q, StopKey: %q}",
+		i.name, i.id, i.namespace, i.table, i.startKey, i.stopKey)
+}
+
+// ID returns region's age
+func (i *info) ID() uint64 {
+	return i.id
+}
+
+// Name returns region name
+func (i *info) Name() []byte {
+	return i.name
+}
+
+// StopKey return region stop key
+func (i *info) StopKey() []byte {
+	return i.stopKey
+}
+
+// StartKey return region start key
+func (i *info) StartKey() []byte {
+	return i.startKey
+}
+
+// Namespace returns region table
+func (i *info) Namespace() []byte {
+	return i.namespace
+}
+
+// Table returns region table
+func (i *info) Table() []byte {
+	return i.table
+}
+
+// Client returns region client
+func (i *info) Client() hrpc.RegionClient {
+	i.m.RLock()
+	c := i.client
+	i.m.RUnlock()
+	return c
+}
+
+// SetClient sets region client
+func (i *info) SetClient(c hrpc.RegionClient) {
+	i.m.Lock()
+	i.client = c
+	i.m.Unlock()
+}
+
+// CompareGeneric is the same thing as Compare but for interface{}.
+func CompareGeneric(a, b interface{}) int {
+	return Compare(a.([]byte), b.([]byte))
+}
+
+// Compare compares two region names.
+// We can't just use bytes.Compare() because it doesn't play nicely
+// with the way META keys are built as the first region has an empty start
+// key.  Let's assume we know about those 2 regions in our cache:
+//   .META.,,1
+//   tableA,,1273018455182
+// We're given an RPC to execute on "tableA", row "\x00" (1 byte row key
+// containing a 0).  If we use Compare() to sort the entries in the cache,
+// when we search for the entry right before "tableA,\000,:"
+// we'll erroneously find ".META.,,1" instead of the entry for first
+// region of "tableA".
+//
+// Since this scheme breaks natural ordering, we need this comparator to
+// implement a special version of comparison to handle this scenario.
+func Compare(a, b []byte) int {
+	var length int
+	if la, lb := len(a), len(b); la < lb {
+		length = la
+	} else {
+		length = lb
+	}
+	// Reminder: region names are of the form:
+	//   table_name,start_key,timestamp[.MD5.]
+	// First compare the table names.
+	var i int
+	for i = 0; i < length; i++ {
+		ai := a[i]    // Saves one pointer deference every iteration.
+		bi := b[i]    // Saves one pointer deference every iteration.
+		if ai != bi { // The name of the tables differ.
+			if ai == ',' {
+				return -1001 // `a' has a smaller table name.  a < b
+			} else if bi == ',' {
+				return 1001 // `b' has a smaller table name.  a > b
+			}
+			return int(ai) - int(bi)
+		}
+		if ai == ',' { // Remember: at this point ai == bi.
+			break // We're done comparing the table names.  They're equal.
+		}
+	}
+
+	// Now find the last comma in both `a' and `b'.  We need to start the
+	// search from the end as the row key could have an arbitrary number of
+	// commas and we don't know its length.
+	aComma := findCommaFromEnd(a, i)
+	bComma := findCommaFromEnd(b, i)
+	// If either `a' or `b' is followed immediately by another comma, then
+	// they are the first region (it's the empty start key).
+	i++ // No need to check against `length', there MUST be more bytes.
+
+	// Compare keys.
+	var firstComma int
+	if aComma < bComma {
+		firstComma = aComma
+	} else {
+		firstComma = bComma
+	}
+	for ; i < firstComma; i++ {
+		ai := a[i]
+		bi := b[i]
+		if ai != bi { // The keys differ.
+			return int(ai) - int(bi)
+		}
+	}
+	if aComma < bComma {
+		return -1002 // `a' has a shorter key.  a < b
+	} else if bComma < aComma {
+		return 1002 // `b' has a shorter key.  a > b
+	}
+
+	// Keys have the same length and have compared identical.  Compare the
+	// rest, which essentially means: use start code as a tie breaker.
+	for ; /*nothing*/ i < length; i++ {
+		ai := a[i]
+		bi := b[i]
+		if ai != bi { // The start codes differ.
+			return int(ai) - int(bi)
+		}
+	}
+
+	return len(a) - len(b)
+}
+
+// Because there is no `LastIndexByte()' in the standard `bytes' package.
+func findCommaFromEnd(b []byte, offset int) int {
+	for i := len(b) - 1; i > offset; i-- {
+		if b[i] == ',' {
+			return i
+		}
+	}
+	panic(fmt.Errorf("no comma found in %q after offset %d", b, offset))
+}

vendor/github.com/tsuna/gohbase/region/multi.go

@@ -0,0 +1,278 @@
+// Copyright (C) 2017  The GoHBase Authors.  All rights reserved.
+// This file is part of GoHBase.
+// Use of this source code is governed by the Apache License 2.0
+// that can be found in the COPYING file.
+
+package region
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/tsuna/gohbase/hrpc"
+	"github.com/tsuna/gohbase/pb"
+)
+
+var multiPool = sync.Pool{
+	New: func() interface{} {
+		return &multi{}
+	},
+}
+
+func freeMulti(m *multi) {
+	m.calls = m.calls[:0]
+	m.regions = m.regions[:0]
+	m.size = 0
+	multiPool.Put(m)
+}
+
+type multi struct {
+	size  int
+	calls []hrpc.Call
+	// regions preserves the order of regions to match against RegionActionResults
+	regions []hrpc.RegionInfo
+}
+
+func newMulti(queueSize int) *multi {
+	m := multiPool.Get().(*multi)
+	m.size = queueSize
+	return m
+}
+
+// Name returns the name of this RPC call.
+func (m *multi) Name() string {
+	return "Multi"
+}
+
+// ToProto converts all request in multi batch to a protobuf message.
+func (m *multi) ToProto() proto.Message {
+	// aggregate calls per region
+	actionsPerReg := map[hrpc.RegionInfo][]*pb.Action{}
+
+	for i, c := range m.calls {
+		select {
+		case <-c.Context().Done():
+			// context has expired, don't bother sending it
+			m.calls[i] = nil
+			continue
+		default:
+		}
+
+		msg := c.ToProto()
+
+		a := &pb.Action{
+			Index: proto.Uint32(uint32(i) + 1), // +1 because 0 index means there's no index
+		}
+
+		switch r := msg.(type) {
+		case *pb.GetRequest:
+			a.Get = r.Get
+		case *pb.MutateRequest:
+			a.Mutation = r.Mutation
+		default:
+			panic(fmt.Sprintf("unsupported call type for Multi: %T", c))
+		}
+
+		actionsPerReg[c.Region()] = append(actionsPerReg[c.Region()], a)
+	}
+
+	// construct the multi proto
+	ra := make([]*pb.RegionAction, len(actionsPerReg))
+	m.regions = make([]hrpc.RegionInfo, len(actionsPerReg))
+
+	i := 0
+	for r, as := range actionsPerReg {
+		ra[i] = &pb.RegionAction{
+			Region: &pb.RegionSpecifier{
+				Type:  pb.RegionSpecifier_REGION_NAME.Enum(),
+				Value: r.Name(),
+			},
+			Action: as,
+		}
+		// Track the order of RegionActions,
+		// so that we can handle whole region exceptions.
+		m.regions[i] = r
+		i++
+	}
+	return &pb.MultiRequest{RegionAction: ra}
+}
+
+// NewResponse creates an empty protobuf message to read the response of this RPC.
+func (m *multi) NewResponse() proto.Message {
+	return &pb.MultiResponse{}
+}
+
+// DeserializeCellBlocks deserializes action results from cell blocks.
+func (m *multi) DeserializeCellBlocks(msg proto.Message, b []byte) (uint32, error) {
+	mr := msg.(*pb.MultiResponse)
+
+	var nread uint32
+	for _, rar := range mr.GetRegionActionResult() {
+		if e := rar.GetException(); e != nil {
+			if l := len(rar.GetResultOrException()); l != 0 {
+				return 0, fmt.Errorf(
+					"got exception for region, but still have %d result(s) returned from it", l)
+			}
+			continue
+		}
+
+		for _, roe := range rar.GetResultOrException() {
+			e := roe.GetException()
+			r := roe.GetResult()
+			i := roe.GetIndex()
+
+			if i == 0 {
+				return 0, errors.New("no index for result in multi response")
+			} else if r == nil && e == nil {
+				return 0, errors.New("no result or exception for action in multi response")
+			} else if r != nil && e != nil {
+				return 0, errors.New("got result and exception for action in multi response")
+			} else if e != nil {
+				continue
+			}
+
+			c := m.get(i)                     // TODO: maybe return error if it's out-of-bounds
+			d := c.(canDeserializeCellBlocks) // let it panic, because then it's our bug
+
+			response := c.NewResponse()
+			switch rsp := response.(type) {
+			case *pb.GetResponse:
+				rsp.Result = r
+			case *pb.MutateResponse:
+				rsp.Result = r
+			default:
+				panic(fmt.Sprintf("unsupported response type for Multi: %T", response))
+			}
+
+			// TODO: don't bother deserializing if the call's context has already expired
+			n, err := d.DeserializeCellBlocks(response, b[nread:])
+			if err != nil {
+				return 0, fmt.Errorf(
+					"error deserializing cellblocks for %q call as part of MultiResponse: %v",
+					c.Name(), err)
+			}
+			nread += n
+		}
+	}
+	return nread, nil
+}
+
+func (m *multi) returnResults(msg proto.Message, err error) {
+	defer freeMulti(m)
+
+	if err != nil {
+		for _, c := range m.calls {
+			if c == nil {
+				continue
+			}
+			c.ResultChan() <- hrpc.RPCResult{Error: err}
+		}
+		return
+	}
+
+	mr := msg.(*pb.MultiResponse)
+
+	// Here we can assume that everything has been deserialized correctly.
+	// Dispatch results to appropriate calls.
+	for i, rar := range mr.GetRegionActionResult() {
+		if e := rar.GetException(); e != nil {
+			// Got an exception for the whole region,
+			// fail all the calls for that region.
+			reg := m.regions[i]
+
+			err := exceptionToError(*e.Name, string(e.Value))
+			for _, c := range m.calls {
+				if c == nil {
+					continue
+				}
+				if c.Region() == reg {
+					c.ResultChan() <- hrpc.RPCResult{Error: err}
+				}
+			}
+			continue
+		}
+
+		for _, roe := range rar.GetResultOrException() {
+			i := roe.GetIndex()
+			e := roe.GetException()
+			r := roe.GetResult()
+
+			c := m.get(i)
+
+			// TODO: don't bother if the call's context has already expired
+
+			if e != nil {
+				c.ResultChan() <- hrpc.RPCResult{
+					Error: exceptionToError(*e.Name, string(e.Value)),
+				}
+				continue
+			}
+
+			response := c.NewResponse()
+			switch rsp := response.(type) {
+			case *pb.GetResponse:
+				rsp.Result = r
+			case *pb.MutateResponse:
+				rsp.Result = r
+			default:
+				panic(fmt.Sprintf("unsupported response type for Multi: %T", response))
+			}
+
+			c.ResultChan() <- hrpc.RPCResult{Msg: response}
+		}
+	}
+}
+
+// add adds the call and returns wether the batch is full.
+func (m *multi) add(call hrpc.Call) bool {
+	m.calls = append(m.calls, call)
+	return len(m.calls) == m.size
+}
+
+// len returns number of batched calls.
+func (m *multi) len() int {
+	return len(m.calls)
+}
+
+// get retruns an rpc at index. Indicies start from 1 since 0 means that
+// region server didn't set an index for the action result.
+func (m *multi) get(i uint32) hrpc.Call {
+	if i == 0 {
+		panic("index cannot be 0")
+	}
+	return m.calls[i-1]
+}
+
+// Table is not supported for Multi.
+func (m *multi) Table() []byte {
+	panic("'Table' is not supported for 'Multi'")
+}
+
+// Reqion is not supported for Multi.
+func (m *multi) Region() hrpc.RegionInfo {
+	panic("'Region' is not supported for 'Multi'")
+}
+
+// SetRegion is not supported for Multi.
+func (m *multi) SetRegion(r hrpc.RegionInfo) {
+	panic("'SetRegion' is not supported for 'Multi'")
+}
+
+// ResultChan is not supported for Multi.
+func (m *multi) ResultChan() chan hrpc.RPCResult {
+	panic("'ResultChan' is not supported for 'Multi'")
+}
+
+// Context is not supported for Multi.
+func (m *multi) Context() context.Context {
+	// TODO: maybe pick the one with the longest deadline and use a context that has that deadline?
+	return context.Background()
+}
+
+// Key is not supported for Multi RPC.
+func (m *multi) Key() []byte {
+	panic("'Key' is not supported for 'Multi'")
+}

vendor/github.com/tsuna/gohbase/region/new.go

@@ -0,0 +1,56 @@
+// Copyright (C) 2016  The GoHBase Authors.  All rights reserved.
+// This file is part of GoHBase.
+// Use of this source code is governed by the Apache License 2.0
+// that can be found in the COPYING file.
+
+// +build !testing
+
+package region
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+// NewClient creates a new RegionClient.
+func NewClient(ctx context.Context, addr string, ctype ClientType,
+	queueSize int, flushInterval time.Duration, effectiveUser string,
+	readTimeout time.Duration) (hrpc.RegionClient, error) {
+	var d net.Dialer
+	conn, err := d.DialContext(ctx, "tcp", addr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to the RegionServer at %s: %s", addr, err)
+	}
+	c := &client{
+		addr:          addr,
+		conn:          conn,
+		rpcs:          make(chan hrpc.Call),
+		done:          make(chan struct{}),
+		sent:          make(map[uint32]hrpc.Call),
+		rpcQueueSize:  queueSize,
+		flushInterval: flushInterval,
+		effectiveUser: effectiveUser,
+		readTimeout:   readTimeout,
+	}
+	// time out send hello if it take long
+	// TODO: do we even need to bother, we are going to retry anyway?
+	if deadline, ok := ctx.Deadline(); ok {
+		conn.SetWriteDeadline(deadline)
+	}
+	if err := c.sendHello(ctype); err != nil {
+		conn.Close()
+		return nil, fmt.Errorf("failed to send hello to the RegionServer at %s: %s", addr, err)
+	}
+	// reset write deadline
+	conn.SetWriteDeadline(time.Time{})
+
+	if ctype == RegionClient {
+		go c.processRPCs() // Batching goroutine
+	}
+	go c.receiveRPCs() // Reader goroutine
+	return c, nil
+}

vendor/github.com/tsuna/gohbase/region/test_new.go

@@ -0,0 +1,284 @@
+// Copyright (C) 2016  The GoHBase Authors.  All rights reserved.
+// This file is part of GoHBase.
+// Use of this source code is governed by the Apache License 2.0
+// that can be found in the COPYING file.
+
+// +build testing
+
+package region
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/tsuna/gohbase/hrpc"
+	"github.com/tsuna/gohbase/pb"
+)
+
+type testClient struct {
+	addr    string
+	numNSRE int32
+}
+
+var nsreRegion = &pb.Result{Cell: []*pb.Cell{
+	&pb.Cell{
+		Row:       []byte("nsre,,1434573235908.56f833d5569a27c7a43fbf547b4924a4."),
+		Family:    []byte("info"),
+		Qualifier: []byte("regioninfo"),
+		Value: []byte("PBUF\b\xc4\xcd\xe9\x99\xe0)\x12\x0f\n\adefault\x12\x04nsre" +
+			"\x1a\x00\"\x00(\x000\x008\x00"),
+	},
+	&pb.Cell{
+		Row:       []byte("nsre,,1434573235908.56f833d5569a27c7a43fbf547b4924a4."),
+		Family:    []byte("info"),
+		Qualifier: []byte("seqnumDuringOpen"),
+		Value:     []byte("\x00\x00\x00\x00\x00\x00\x00\x02"),
+	},
+	&pb.Cell{
+		Row:       []byte("nsre,,1434573235908.56f833d5569a27c7a43fbf547b4924a4."),
+		Family:    []byte("info"),
+		Qualifier: []byte("server"),
+		Value:     []byte("regionserver:1"),
+	},
+	&pb.Cell{
+		Row:       []byte("nsre,,1434573235908.56f833d5569a27c7a43fbf547b4924a4."),
+		Family:    []byte("info"),
+		Qualifier: []byte("serverstartcode"),
+		Value:     []byte("\x00\x00\x01N\x02\x92R\xb1"),
+	},
+}}
+
+// makeRegionResult returns a region that spans the whole table
+// and uses name of the table as the hostname of the regionserver
+func makeRegionResult(key []byte) *pb.ScanResponse {
+	s := bytes.SplitN(key, []byte(","), 2)
+	fqtable := s[0]
+
+	row := append(fqtable, []byte(",,1434573235908.56f833d5569a27c7a43fbf547b4924a4.")...)
+	t := bytes.SplitN(fqtable, []byte{':'}, 2)
+	var namespace, table []byte
+	if len(t) == 2 {
+		namespace = t[0]
+		table = t[1]
+	} else {
+		namespace = []byte("default")
+		table = fqtable
+	}
+	regionInfo := &pb.RegionInfo{
+		RegionId: proto.Uint64(1434573235908),
+		TableName: &pb.TableName{
+			Namespace: namespace,
+			Qualifier: table,
+		},
+		Offline: proto.Bool(false),
+	}
+	regionInfoValue, err := proto.Marshal(regionInfo)
+	if err != nil {
+		panic(err)
+	}
+	regionInfoValue = append([]byte("PBUF"), regionInfoValue...)
+
+	return &pb.ScanResponse{Results: []*pb.Result{
+		&pb.Result{Cell: []*pb.Cell{
+			&pb.Cell{
+				Row:       row,
+				Family:    []byte("info"),
+				Qualifier: []byte("regioninfo"),
+				Value:     regionInfoValue,
+			},
+			&pb.Cell{
+				Row:       row,
+				Family:    []byte("info"),
+				Qualifier: []byte("seqnumDuringOpen"),
+				Value:     []byte("\x00\x00\x00\x00\x00\x00\x00\x02"),
+			},
+			&pb.Cell{
+				Row:       row,
+				Family:    []byte("info"),
+				Qualifier: []byte("server"),
+				Value:     fqtable,
+			},
+			&pb.Cell{
+				Row:       row,
+				Family:    []byte("info"),
+				Qualifier: []byte("serverstartcode"),
+				Value:     []byte("\x00\x00\x01N\x02\x92R\xb1"),
+			},
+		}}}}
+}
+
+var metaRow = &pb.Result{Cell: []*pb.Cell{
+	&pb.Cell{
+		Row:       []byte("test,,1434573235908.56f833d5569a27c7a43fbf547b4924a4."),
+		Family:    []byte("info"),
+		Qualifier: []byte("regioninfo"),
+		Value: []byte("PBUF\b\xc4\xcd\xe9\x99\xe0)\x12\x0f\n\adefault\x12\x04test" +
+			"\x1a\x00\"\x00(\x000\x008\x00"),
+	},
+	&pb.Cell{
+		Row:       []byte("test,,1434573235908.56f833d5569a27c7a43fbf547b4924a4."),
+		Family:    []byte("info"),
+		Qualifier: []byte("seqnumDuringOpen"),
+		Value:     []byte("\x00\x00\x00\x00\x00\x00\x00\x02"),
+	},
+	&pb.Cell{
+		Row:       []byte("test,,1434573235908.56f833d5569a27c7a43fbf547b4924a4."),
+		Family:    []byte("info"),
+		Qualifier: []byte("server"),
+		Value:     []byte("regionserver:2"),
+	},
+	&pb.Cell{
+		Row:       []byte("test,,1434573235908.56f833d5569a27c7a43fbf547b4924a4."),
+		Family:    []byte("info"),
+		Qualifier: []byte("serverstartcode"),
+		Value:     []byte("\x00\x00\x01N\x02\x92R\xb1"),
+	},
+}}
+
+var test1SplitA = &pb.Result{Cell: []*pb.Cell{
+	&pb.Cell{
+		Row:       []byte("test1,,1480547738107.825c5c7e480c76b73d6d2bad5d3f7bb8."),
+		Family:    []byte("info"),
+		Qualifier: []byte("regioninfo"),
+		Value: []byte("PBUF\b\xfbÖ\xbc\x8b+\x12\x10\n\adefault\x12\x05" +
+			"test1\x1a\x00\"\x03baz(\x000\x008\x00"),
+	},
+	&pb.Cell{
+		Row:       []byte("test1,,1480547738107.825c5c7e480c76b73d6d2bad5d3f7bb8."),
+		Family:    []byte("info"),
+		Qualifier: []byte("seqnumDuringOpen"),
+		Value:     []byte("\x00\x00\x00\x00\x00\x00\x00\v"),
+	},
+	&pb.Cell{
+		Row:       []byte("test1,,1480547738107.825c5c7e480c76b73d6d2bad5d3f7bb8."),
+		Family:    []byte("info"),
+		Qualifier: []byte("server"),
+		Value:     []byte("regionserver:1"),
+	},
+	&pb.Cell{
+		Row:       []byte("test1,,1480547738107.825c5c7e480c76b73d6d2bad5d3f7bb8."),
+		Family:    []byte("info"),
+		Qualifier: []byte("serverstartcode"),
+		Value:     []byte("\x00\x00\x01X\xb6\x83^3"),
+	},
+}}
+
+var test1SplitB = &pb.Result{Cell: []*pb.Cell{
+	&pb.Cell{
+		Row:       []byte("test1,baz,1480547738107.3f2483f5618e1b791f58f83a8ebba6a9."),
+		Family:    []byte("info"),
+		Qualifier: []byte("regioninfo"),
+		Value: []byte("PBUF\b\xfbÖ\xbc\x8b+\x12\x10\n\adefault\x12\x05" +
+			"test1\x1a\x03baz\"\x00(\x000\x008\x00"),
+	},
+	&pb.Cell{
+		Row:       []byte("test1,baz,1480547738107.3f2483f5618e1b791f58f83a8ebba6a9."),
+		Family:    []byte("info"),
+		Qualifier: []byte("seqnumDuringOpen"),
+		Value:     []byte("\x00\x00\x00\x00\x00\x00\x00\f"),
+	},
+	&pb.Cell{
+		Row:       []byte("test1,baz,1480547738107.3f2483f5618e1b791f58f83a8ebba6a9."),
+		Family:    []byte("info"),
+		Qualifier: []byte("server"),
+		Value:     []byte("regionserver:3"),
+	},
+	&pb.Cell{
+		Row:       []byte("test1,baz,1480547738107.3f2483f5618e1b791f58f83a8ebba6a9."),
+		Family:    []byte("info"),
+		Qualifier: []byte("serverstartcode"),
+		Value:     []byte("\x00\x00\x01X\xb6\x83^3"),
+	},
+}}
+
+var m sync.RWMutex
+var clients map[string]uint32
+
+func init() {
+	clients = make(map[string]uint32)
+}
+
+// NewClient creates a new test region client.
+func NewClient(ctx context.Context, addr string, ctype ClientType,
+	queueSize int, flushInterval time.Duration, effectiveUser string,
+	readTimeout time.Duration) (hrpc.RegionClient, error) {
+	m.Lock()
+	clients[addr]++
+	m.Unlock()
+	return &testClient{addr: addr}, nil
+}
+
+func (c *testClient) Addr() string {
+	return c.addr
+}
+
+func (c *testClient) String() string {
+	return fmt.Sprintf("RegionClient{Addr: %s}", c.addr)
+}
+
+func (c *testClient) QueueRPC(call hrpc.Call) {
+	// ignore timed out rpcs to mock the region client
+	select {
+	case <-call.Context().Done():
+		return
+	default:
+	}
+	if !bytes.Equal(call.Table(), []byte("hbase:meta")) {
+		_, ok := call.(*hrpc.Get)
+		if !ok || !bytes.HasSuffix(call.Key(), bytes.Repeat([]byte{0}, 17)) {
+			// not a get and not a region probe
+			// just return as the mock call should just populate the ResultChan in test
+			return
+		}
+		// region probe, fail for the nsre region 3 times to force retry
+		if bytes.Equal(call.Table(), []byte("nsre")) {
+			i := atomic.AddInt32(&c.numNSRE, 1)
+			if i <= 3 {
+				call.ResultChan() <- hrpc.RPCResult{Error: RetryableError{}}
+				return
+			}
+		}
+		m.RLock()
+		i := clients[c.addr]
+		m.RUnlock()
+
+		// if we are connected to this client the first time,
+		// pretend it's down to fail the probe and start a reconnect
+		if bytes.Equal(call.Table(), []byte("down")) {
+			if i <= 1 {
+				call.ResultChan() <- hrpc.RPCResult{Error: UnrecoverableError{}}
+			} else {
+				// otherwise, the region is fine
+				call.ResultChan() <- hrpc.RPCResult{}
+			}
+			return
+		}
+	}
+	if bytes.HasSuffix(call.Key(), bytes.Repeat([]byte{0}, 17)) {
+		// meta region probe, return empty to signify that region is online
+		call.ResultChan() <- hrpc.RPCResult{}
+	} else if bytes.HasPrefix(call.Key(), []byte("test,")) {
+		call.ResultChan() <- hrpc.RPCResult{Msg: &pb.ScanResponse{
+			Results: []*pb.Result{metaRow}}}
+	} else if bytes.HasPrefix(call.Key(), []byte("test1,,")) {
+		call.ResultChan() <- hrpc.RPCResult{Msg: &pb.ScanResponse{
+			Results: []*pb.Result{test1SplitA}}}
+	} else if bytes.HasPrefix(call.Key(), []byte("nsre,,")) {
+		call.ResultChan() <- hrpc.RPCResult{Msg: &pb.ScanResponse{
+			Results: []*pb.Result{nsreRegion}}}
+	} else if bytes.HasPrefix(call.Key(), []byte("tablenotfound,")) {
+		call.ResultChan() <- hrpc.RPCResult{Msg: &pb.ScanResponse{
+			Results:     []*pb.Result{},
+			MoreResults: proto.Bool(false),
+		}}
+	} else {
+		call.ResultChan() <- hrpc.RPCResult{Msg: makeRegionResult(call.Key())}
+	}
+}
+
+func (c *testClient) Close() {}