Initial commit

vendor/github.com/wvanbergen/kazoo-go/BUILD.bazel

@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "consumergroup.go",
+        "kazoo.go",
+        "topic_metadata.go",
+    ],
+    importmap = "go-common/vendor/github.com/wvanbergen/kazoo-go",
+    importpath = "github.com/wvanbergen/kazoo-go",
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/github.com/samuel/go-zookeeper/zk:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/wvanbergen/kazoo-go/MIT-LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Willem van Bergen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/wvanbergen/kazoo-go/Makefile

@@ -0,0 +1,83 @@
+.PHONY: confluent/kafka/* confluent/zookeeper/* confluent/registry/* confluent/start confluent/stop fmt vet errcheck test test/create_kafka_topics dependencies dependencies/*
+
+
+default: fmt vet errcheck test
+
+
+# Confluent platform tasks
+
+confluent/start: confluent/rest/start
+
+confluent/stop: confluent/rest/stop confluent/registry/stop confluent/kafka/stop confluent/zookeeper/stop
+
+# Download & extract tasks
+
+confluent/confluent.tgz:
+	mkdir -p confluent && wget http://packages.confluent.io/archive/1.0/confluent-1.0-2.10.4.tar.gz -O confluent/confluent.tgz
+
+confluent/EXTRACTED: confluent/confluent.tgz
+	tar xzf confluent/confluent.tgz -C confluent --strip-components 1 && mkdir confluent/logs && touch confluent/EXTRACTED
+
+# Zookeeper tasks
+
+confluent/zookeeper/start: confluent/EXTRACTED
+	nohup confluent/bin/zookeeper-server-start confluent/etc/kafka/zookeeper.properties 2> confluent/logs/zookeeper.err > confluent/logs/zookeeper.out < /dev/null &
+	while ! nc localhost 2181 </dev/null; do echo "Waiting for zookeeper..."; sleep 1; done
+
+confluent/zookeeper/stop: confluent/EXTRACTED
+	confluent/bin/zookeeper-server-stop
+
+# Kafka tasks
+
+confluent/kafka/start: confluent/zookeeper/start confluent/EXTRACTED
+	nohup confluent/bin/kafka-server-start confluent/etc/kafka/server.properties 2> confluent/logs/kafka.err > confluent/logs/kafka.out < /dev/null &
+	while ! nc localhost 9092 </dev/null; do echo "Waiting for Kafka..."; sleep 1; done
+
+confluent/kafka/stop: confluent/EXTRACTED
+	confluent/bin/kafka-server-stop
+
+# schema-registry tasks
+
+confluent/registry/start: confluent/kafka/start confluent/EXTRACTED
+	nohup confluent/bin/schema-registry-start confluent/etc/schema-registry/schema-registry.properties 2> confluent/logs/schema-registry.err > confluent/logs/schema-registry.out < /dev/null &
+	while ! nc localhost 8081 </dev/null; do echo "Waiting for schema registry..."; sleep 1; done
+
+confluent/registry/stop: confluent/EXTRACTED
+	confluent/bin/kafka-server-stop
+
+# REST proxy tasks
+
+confluent/rest/start: confluent/registry/start confluent/EXTRACTED
+	nohup confluent/bin/kafka-rest-start confluent/etc/kafka-rest/kafka-rest.properties 2> confluent/logs/kafka-rest.err > confluent/logs/kafka-rest.out < /dev/null &
+	while ! nc localhost 8082 </dev/null; do echo "Waiting for REST proxy..."; sleep 1; done
+
+confluent/rest/stop: confluent/EXTRACTED
+	confluent/bin/kafka-rest-stop
+
+
+# CI tasks
+
+test:
+	go test -v -race ./...
+
+vet:
+	go vet ./...
+
+errcheck:
+	errcheck ./...
+
+fmt:
+	@if [ -n "$$(go fmt ./...)" ]; then echo 'Please run go fmt on your code.' && exit 1; fi
+
+dependencies: dependencies/errcheck dependencies/get
+
+dependencies/errcheck:
+	go get github.com/kisielk/errcheck
+
+dependencies/get:
+	go get -t ./...
+
+test/create_kafka_topics: confluent/kafka/start
+	confluent/bin/kafka-topics --create --partitions  1 --replication-factor 1 --topic test.1  --zookeeper localhost:2181
+	confluent/bin/kafka-topics --create --partitions  4 --replication-factor 1 --topic test.4  --zookeeper localhost:2181 --config retention.ms=604800000
+	confluent/bin/kafka-topics --create --partitions 64 --replication-factor 1 --topic test.64 --zookeeper localhost:2181

vendor/github.com/wvanbergen/kazoo-go/README.md

@@ -0,0 +1,6 @@
+# Kazoo
+
+Kazoo is a library to interact with the Kafka metadata that lives in Zookeeper.
+It provides discovery of the cluster's brokers, topic metadata, and consumer groups.
+
+API documentation can be found on [godoc.org](http://godoc.org/github.com/wvanbergen/kazoo-go).

vendor/github.com/wvanbergen/kazoo-go/consumergroup.go

@@ -0,0 +1,514 @@
+package kazoo
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/samuel/go-zookeeper/zk"
+)
+
+var (
+	ErrRunningInstances          = errors.New("Cannot deregister a consumergroup with running instances")
+	ErrInstanceAlreadyRegistered = errors.New("Cannot register consumer instance because it already is registered")
+	ErrInstanceNotRegistered     = errors.New("Cannot deregister consumer instance because it not registered")
+	ErrPartitionClaimedByOther   = errors.New("Cannot claim partition: it is already claimed by another instance")
+	ErrPartitionNotClaimed       = errors.New("Cannot release partition: it is not claimed by this instance")
+)
+
+// Consumergroup represents a high-level consumer that is registered in Zookeeper,
+type Consumergroup struct {
+	kz   *Kazoo
+	Name string
+}
+
+// ConsumergroupInstance represents an instance of a Consumergroup.
+type ConsumergroupInstance struct {
+	cg *Consumergroup
+	ID string
+}
+
+// ConsumergroupList implements the sortable interface on top of a consumer group list
+type ConsumergroupList []*Consumergroup
+
+// ConsumergroupInstanceList implements the sortable interface on top of a consumer instance list
+type ConsumergroupInstanceList []*ConsumergroupInstance
+
+type Registration struct {
+	Pattern      RegPattern     `json:"pattern"`
+	Subscription map[string]int `json:"subscription"`
+	Timestamp    int64          `json:"timestamp"`
+	Version      RegVersion     `json:"version"`
+}
+
+type RegPattern string
+
+const (
+	RegPatternStatic    RegPattern = "static"
+	RegPatternWhiteList RegPattern = "white_list"
+	RegPatternBlackList RegPattern = "black_list"
+)
+
+type RegVersion int
+
+const (
+	RegDefaultVersion RegVersion = 1
+)
+
+// Consumergroups returns all the registered consumergroups
+func (kz *Kazoo) Consumergroups() (ConsumergroupList, error) {
+	root := fmt.Sprintf("%s/consumers", kz.conf.Chroot)
+	cgs, _, err := kz.conn.Children(root)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(ConsumergroupList, 0, len(cgs))
+	for _, cg := range cgs {
+		result = append(result, kz.Consumergroup(cg))
+	}
+	return result, nil
+}
+
+// Consumergroup instantiates a new consumergroup.
+func (kz *Kazoo) Consumergroup(name string) *Consumergroup {
+	return &Consumergroup{Name: name, kz: kz}
+}
+
+// Exists checks whether the consumergroup has been registered in Zookeeper
+func (cg *Consumergroup) Exists() (bool, error) {
+	return cg.kz.exists(fmt.Sprintf("%s/consumers/%s", cg.kz.conf.Chroot, cg.Name))
+}
+
+// Create registers the consumergroup in zookeeper
+func (cg *Consumergroup) Create() error {
+	return cg.kz.mkdirRecursive(fmt.Sprintf("%s/consumers/%s", cg.kz.conf.Chroot, cg.Name))
+}
+
+// Delete removes the consumergroup from zookeeper
+func (cg *Consumergroup) Delete() error {
+	if instances, err := cg.Instances(); err != nil {
+		return err
+	} else if len(instances) > 0 {
+		return ErrRunningInstances
+	}
+
+	return cg.kz.deleteRecursive(fmt.Sprintf("%s/consumers/%s", cg.kz.conf.Chroot, cg.Name))
+}
+
+// Instances returns a map of all running instances inside this consumergroup.
+func (cg *Consumergroup) Instances() (ConsumergroupInstanceList, error) {
+	root := fmt.Sprintf("%s/consumers/%s/ids", cg.kz.conf.Chroot, cg.Name)
+	if exists, err := cg.kz.exists(root); err != nil {
+		return nil, err
+	} else if exists {
+		cgis, _, err := cg.kz.conn.Children(root)
+		if err != nil {
+			return nil, err
+		}
+
+		result := make(ConsumergroupInstanceList, 0, len(cgis))
+		for _, cgi := range cgis {
+			result = append(result, cg.Instance(cgi))
+		}
+		return result, nil
+	} else {
+		result := make(ConsumergroupInstanceList, 0)
+		return result, nil
+	}
+}
+
+// WatchInstances returns a ConsumergroupInstanceList, and a channel that will be closed
+// as soon the instance list changes.
+func (cg *Consumergroup) WatchInstances() (ConsumergroupInstanceList, <-chan zk.Event, error) {
+	node := fmt.Sprintf("%s/consumers/%s/ids", cg.kz.conf.Chroot, cg.Name)
+	if exists, err := cg.kz.exists(node); err != nil {
+		return nil, nil, err
+	} else if !exists {
+		if err := cg.kz.mkdirRecursive(node); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	cgis, _, c, err := cg.kz.conn.ChildrenW(node)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	result := make(ConsumergroupInstanceList, 0, len(cgis))
+	for _, cgi := range cgis {
+		result = append(result, cg.Instance(cgi))
+	}
+
+	return result, c, nil
+}
+
+// NewInstance instantiates a new ConsumergroupInstance inside this consumer group,
+// using a newly generated ID.
+func (cg *Consumergroup) NewInstance() *ConsumergroupInstance {
+	id, err := generateConsumerInstanceID()
+	if err != nil {
+		panic(err)
+	}
+	return cg.Instance(id)
+}
+
+// Instance instantiates a new ConsumergroupInstance inside this consumer group,
+// using an existing ID.
+func (cg *Consumergroup) Instance(id string) *ConsumergroupInstance {
+	return &ConsumergroupInstance{cg: cg, ID: id}
+}
+
+// PartitionOwner returns the ConsumergroupInstance that has claimed the given partition.
+// This can be nil if nobody has claimed it yet.
+func (cg *Consumergroup) PartitionOwner(topic string, partition int32) (*ConsumergroupInstance, error) {
+	node := fmt.Sprintf("%s/consumers/%s/owners/%s/%d", cg.kz.conf.Chroot, cg.Name, topic, partition)
+	val, _, err := cg.kz.conn.Get(node)
+
+	// If the node does not exists, nobody has claimed it.
+	switch err {
+	case nil:
+		return &ConsumergroupInstance{cg: cg, ID: string(val)}, nil
+	case zk.ErrNoNode:
+		return nil, nil
+	default:
+		return nil, err
+	}
+}
+
+// WatchPartitionOwner retrieves what instance is currently owning the partition, and sets a
+// Zookeeper watch to be notified of changes. If the partition currently does not have an owner,
+// the function returns nil for every return value. In this case is should be safe to claim
+// the partition for an instance.
+func (cg *Consumergroup) WatchPartitionOwner(topic string, partition int32) (*ConsumergroupInstance, <-chan zk.Event, error) {
+	node := fmt.Sprintf("%s/consumers/%s/owners/%s/%d", cg.kz.conf.Chroot, cg.Name, topic, partition)
+	instanceID, _, changed, err := cg.kz.conn.GetW(node)
+
+	switch err {
+	case nil:
+		return &ConsumergroupInstance{cg: cg, ID: string(instanceID)}, changed, nil
+
+	case zk.ErrNoNode:
+		return nil, nil, nil
+
+	default:
+		return nil, nil, err
+	}
+}
+
+// Registered checks whether the consumergroup instance is registered in Zookeeper.
+func (cgi *ConsumergroupInstance) Registered() (bool, error) {
+	node := fmt.Sprintf("%s/consumers/%s/ids/%s", cgi.cg.kz.conf.Chroot, cgi.cg.Name, cgi.ID)
+	return cgi.cg.kz.exists(node)
+}
+
+// Registered returns current registration of the consumer group instance.
+func (cgi *ConsumergroupInstance) Registration() (*Registration, error) {
+	node := fmt.Sprintf("%s/consumers/%s/ids/%s", cgi.cg.kz.conf.Chroot, cgi.cg.Name, cgi.ID)
+	val, _, err := cgi.cg.kz.conn.Get(node)
+	if err != nil {
+		return nil, err
+	}
+
+	reg := &Registration{}
+	if err := json.Unmarshal(val, reg); err != nil {
+		return nil, err
+	}
+	return reg, nil
+}
+
+// RegisterSubscription registers the consumer instance in Zookeeper, with its subscription.
+func (cgi *ConsumergroupInstance) RegisterWithSubscription(subscriptionJSON []byte) error {
+	if exists, err := cgi.Registered(); err != nil {
+		return err
+	} else if exists {
+		return ErrInstanceAlreadyRegistered
+	}
+
+	// Create an ephemeral node for the the consumergroup instance.
+	node := fmt.Sprintf("%s/consumers/%s/ids/%s", cgi.cg.kz.conf.Chroot, cgi.cg.Name, cgi.ID)
+	return cgi.cg.kz.create(node, subscriptionJSON, true)
+}
+
+// Register registers the consumergroup instance in Zookeeper.
+func (cgi *ConsumergroupInstance) Register(topics []string) error {
+	subscription := make(map[string]int)
+	for _, topic := range topics {
+		subscription[topic] = 1
+	}
+
+	data, err := json.Marshal(&Registration{
+		Pattern:      RegPatternStatic,
+		Subscription: subscription,
+		Timestamp:    time.Now().Unix(),
+		Version:      RegDefaultVersion,
+	})
+	if err != nil {
+		return err
+	}
+
+	return cgi.RegisterWithSubscription(data)
+}
+
+// Deregister removes the registration of the instance from zookeeper.
+func (cgi *ConsumergroupInstance) Deregister() error {
+	node := fmt.Sprintf("%s/consumers/%s/ids/%s", cgi.cg.kz.conf.Chroot, cgi.cg.Name, cgi.ID)
+	exists, stat, err := cgi.cg.kz.conn.Exists(node)
+	if err != nil {
+		return err
+	} else if !exists {
+		return ErrInstanceNotRegistered
+	}
+
+	return cgi.cg.kz.conn.Delete(node, stat.Version)
+}
+
+// Claim claims a topic/partition ownership for a consumer ID within a group. If the
+// partition is already claimed by another running instance, it will return ErrAlreadyClaimed.
+func (cgi *ConsumergroupInstance) ClaimPartition(topic string, partition int32) error {
+	root := fmt.Sprintf("%s/consumers/%s/owners/%s", cgi.cg.kz.conf.Chroot, cgi.cg.Name, topic)
+	if err := cgi.cg.kz.mkdirRecursive(root); err != nil {
+		return err
+	}
+
+	// Create an ephemeral node for the partition to claim the partition for this instance
+	node := fmt.Sprintf("%s/%d", root, partition)
+	err := cgi.cg.kz.create(node, []byte(cgi.ID), true)
+	switch err {
+	case zk.ErrNodeExists:
+		data, _, err := cgi.cg.kz.conn.Get(node)
+		if err != nil {
+			return err
+		}
+		if string(data) != cgi.ID {
+			// Return a separate error for this, to allow for implementing a retry mechanism.
+			return ErrPartitionClaimedByOther
+		}
+		return nil
+	default:
+		return err
+	}
+}
+
+// ReleasePartition releases a claim to a partition.
+func (cgi *ConsumergroupInstance) ReleasePartition(topic string, partition int32) error {
+	owner, err := cgi.cg.PartitionOwner(topic, partition)
+	if err != nil {
+		return err
+	}
+	if owner == nil || owner.ID != cgi.ID {
+		return ErrPartitionNotClaimed
+	}
+
+	node := fmt.Sprintf("%s/consumers/%s/owners/%s/%d", cgi.cg.kz.conf.Chroot, cgi.cg.Name, topic, partition)
+	return cgi.cg.kz.conn.Delete(node, 0)
+}
+
+// Topics retrieves the list of topics the consumergroup has claimed ownership of at some point.
+func (cg *Consumergroup) Topics() (TopicList, error) {
+	root := fmt.Sprintf("%s/consumers/%s/owners", cg.kz.conf.Chroot, cg.Name)
+	children, _, err := cg.kz.conn.Children(root)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(TopicList, 0, len(children))
+	for _, name := range children {
+		result = append(result, cg.kz.Topic(name))
+	}
+	return result, nil
+}
+
+// CommitOffset commits an offset to a group/topic/partition
+func (cg *Consumergroup) CommitOffset(topic string, partition int32, offset int64) error {
+	node := fmt.Sprintf("%s/consumers/%s/offsets/%s/%d", cg.kz.conf.Chroot, cg.Name, topic, partition)
+	data := []byte(fmt.Sprintf("%d", offset))
+
+	_, stat, err := cg.kz.conn.Get(node)
+	switch err {
+	case zk.ErrNoNode: // Create a new node
+		return cg.kz.create(node, data, false)
+
+	case nil: // Update the existing node
+		_, err := cg.kz.conn.Set(node, data, stat.Version)
+		return err
+
+	default:
+		return err
+	}
+}
+
+// FetchOffset retrieves an offset to a group/topic/partition
+func (cg *Consumergroup) FetchOffset(topic string, partition int32) (int64, error) {
+	node := fmt.Sprintf("%s/consumers/%s/offsets/%s/%d", cg.kz.conf.Chroot, cg.Name, topic, partition)
+	val, _, err := cg.kz.conn.Get(node)
+	if err == zk.ErrNoNode {
+		return -1, nil
+	} else if err != nil {
+		return -1, err
+	}
+	return strconv.ParseInt(string(val), 10, 64)
+}
+
+// FetchOffset retrieves all the commmitted offsets for a group
+func (cg *Consumergroup) FetchAllOffsets() (map[string]map[int32]int64, error) {
+	result := make(map[string]map[int32]int64)
+
+	offsetsNode := fmt.Sprintf("%s/consumers/%s/offsets", cg.kz.conf.Chroot, cg.Name)
+	topics, _, err := cg.kz.conn.Children(offsetsNode)
+	if err == zk.ErrNoNode {
+		return result, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	for _, topic := range topics {
+		result[topic] = make(map[int32]int64)
+		topicNode := fmt.Sprintf("%s/consumers/%s/offsets/%s", cg.kz.conf.Chroot, cg.Name, topic)
+		partitions, _, err := cg.kz.conn.Children(topicNode)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, partition := range partitions {
+			partitionNode := fmt.Sprintf("%s/consumers/%s/offsets/%s/%s", cg.kz.conf.Chroot, cg.Name, topic, partition)
+			val, _, err := cg.kz.conn.Get(partitionNode)
+			if err != nil {
+				return nil, err
+			}
+
+			partition, err := strconv.ParseInt(partition, 10, 32)
+			if err != nil {
+				return nil, err
+			}
+
+			offset, err := strconv.ParseInt(string(val), 10, 64)
+			if err != nil {
+				return nil, err
+			}
+
+			result[topic][int32(partition)] = offset
+		}
+	}
+
+	return result, nil
+}
+
+func (cg *Consumergroup) ResetOffsets() error {
+	offsetsNode := fmt.Sprintf("%s/consumers/%s/offsets", cg.kz.conf.Chroot, cg.Name)
+	topics, _, err := cg.kz.conn.Children(offsetsNode)
+	if err == zk.ErrNoNode {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	for _, topic := range topics {
+		topicNode := fmt.Sprintf("%s/consumers/%s/offsets/%s", cg.kz.conf.Chroot, cg.Name, topic)
+		partitions, stat, err := cg.kz.conn.Children(topicNode)
+		if err != nil {
+			return err
+		}
+
+		for _, partition := range partitions {
+			partitionNode := fmt.Sprintf("%s/consumers/%s/offsets/%s/%s", cg.kz.conf.Chroot, cg.Name, topic, partition)
+			exists, stat, err := cg.kz.conn.Exists(partitionNode)
+			if exists {
+				if err = cg.kz.conn.Delete(partitionNode, stat.Version); err != nil {
+					if err != zk.ErrNoNode {
+						return err
+					}
+				}
+			}
+		}
+
+		if err := cg.kz.conn.Delete(topicNode, stat.Version); err != nil {
+			if err != zk.ErrNoNode {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// generateUUID Generates a UUIDv4.
+func generateUUID() (string, error) {
+	uuid := make([]byte, 16)
+	n, err := io.ReadFull(rand.Reader, uuid)
+	if n != len(uuid) || err != nil {
+		return "", err
+	}
+	// variant bits; see section 4.1.1
+	uuid[8] = uuid[8]&^0xc0 | 0x80
+	// version 4 (pseudo-random); see section 4.1.3
+	uuid[6] = uuid[6]&^0xf0 | 0x40
+	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
+}
+
+// generateConsumerInstanceID generates a consumergroup Instance ID
+// that is almost certain to be unique.
+func generateConsumerInstanceID() (string, error) {
+	uuid, err := generateUUID()
+	if err != nil {
+		return "", err
+	}
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%s:%s", hostname, uuid), nil
+}
+
+// Find returns the consumergroup with the given name if it exists in the list.
+// Otherwise it will return `nil`.
+func (cgl ConsumergroupList) Find(name string) *Consumergroup {
+	for _, cg := range cgl {
+		if cg.Name == name {
+			return cg
+		}
+	}
+	return nil
+}
+
+func (cgl ConsumergroupList) Len() int {
+	return len(cgl)
+}
+
+func (cgl ConsumergroupList) Less(i, j int) bool {
+	return cgl[i].Name < cgl[j].Name
+}
+
+func (cgl ConsumergroupList) Swap(i, j int) {
+	cgl[i], cgl[j] = cgl[j], cgl[i]
+}
+
+// Find returns the consumergroup instance with the given ID if it exists in the list.
+// Otherwise it will return `nil`.
+func (cgil ConsumergroupInstanceList) Find(id string) *ConsumergroupInstance {
+	for _, cgi := range cgil {
+		if cgi.ID == id {
+			return cgi
+		}
+	}
+	return nil
+}
+
+func (cgil ConsumergroupInstanceList) Len() int {
+	return len(cgil)
+}
+
+func (cgil ConsumergroupInstanceList) Less(i, j int) bool {
+	return cgil[i].ID < cgil[j].ID
+}
+
+func (cgil ConsumergroupInstanceList) Swap(i, j int) {
+	cgil[i], cgil[j] = cgil[j], cgil[i]
+}

vendor/github.com/wvanbergen/kazoo-go/kazoo.go

@@ -0,0 +1,224 @@
+package kazoo
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/samuel/go-zookeeper/zk"
+)
+
+var (
+	FailedToClaimPartition = errors.New("Failed to claim partition for this consumer instance. Do you have a rogue consumer running?")
+)
+
+// ParseConnectionString parses a zookeeper connection string in the form of
+// host1:2181,host2:2181/chroot and returns the list of servers, and the chroot.
+func ParseConnectionString(zookeeper string) (nodes []string, chroot string) {
+	nodesAndChroot := strings.SplitN(zookeeper, "/", 2)
+	if len(nodesAndChroot) == 2 {
+		chroot = fmt.Sprintf("/%s", nodesAndChroot[1])
+	}
+	nodes = strings.Split(nodesAndChroot[0], ",")
+	return
+}
+
+// BuildConnectionString builds a Zookeeper connection string for a list of nodes.
+// Returns a string like "zk1:2181,zk2:2181,zk3:2181"
+func BuildConnectionString(nodes []string) string {
+	return strings.Join(nodes, ",")
+}
+
+// ConnectionStringWithChroot builds a Zookeeper connection string for a list
+// of nodes and a chroot. The chroot should start with "/".
+// Returns a string like "zk1:2181,zk2:2181,zk3:2181/chroot"
+func BuildConnectionStringWithChroot(nodes []string, chroot string) string {
+	return fmt.Sprintf("%s%s", strings.Join(nodes, ","), chroot)
+}
+
+// Kazoo interacts with the Kafka metadata in Zookeeper
+type Kazoo struct {
+	conn *zk.Conn
+	conf *Config
+}
+
+// Config holds configuration values f.
+type Config struct {
+	// The chroot the Kafka installation is registerde under. Defaults to "".
+	Chroot string
+
+	// The amount of time the Zookeeper client can be disconnected from the Zookeeper cluster
+	// before the cluster will get rid of watches and ephemeral nodes. Defaults to 1 second.
+	Timeout time.Duration
+}
+
+// NewConfig instantiates a new Config struct with sane defaults.
+func NewConfig() *Config {
+	return &Config{Timeout: 1 * time.Second}
+}
+
+// NewKazoo creates a new connection instance
+func NewKazoo(servers []string, conf *Config) (*Kazoo, error) {
+	if conf == nil {
+		conf = NewConfig()
+	}
+
+	conn, _, err := zk.Connect(servers, conf.Timeout)
+	if err != nil {
+		return nil, err
+	}
+	return &Kazoo{conn, conf}, nil
+}
+
+// NewKazooFromConnectionString creates a new connection instance
+// based on a zookeeer connection string that can include a chroot.
+func NewKazooFromConnectionString(connectionString string, conf *Config) (*Kazoo, error) {
+	if conf == nil {
+		conf = NewConfig()
+	}
+
+	nodes, chroot := ParseConnectionString(connectionString)
+	conf.Chroot = chroot
+	return NewKazoo(nodes, conf)
+}
+
+// Brokers returns a map of all the brokers that make part of the
+// Kafka cluster that is registered in Zookeeper.
+func (kz *Kazoo) Brokers() (map[int32]string, error) {
+	root := fmt.Sprintf("%s/brokers/ids", kz.conf.Chroot)
+	children, _, err := kz.conn.Children(root)
+	if err != nil {
+		return nil, err
+	}
+
+	type brokerEntry struct {
+		Host string `json:"host"`
+		Port int    `json:"port"`
+	}
+
+	result := make(map[int32]string)
+	for _, child := range children {
+		brokerID, err := strconv.ParseInt(child, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+
+		value, _, err := kz.conn.Get(path.Join(root, child))
+		if err != nil {
+			return nil, err
+		}
+
+		var brokerNode brokerEntry
+		if err := json.Unmarshal(value, &brokerNode); err != nil {
+			return nil, err
+		}
+
+		result[int32(brokerID)] = fmt.Sprintf("%s:%d", brokerNode.Host, brokerNode.Port)
+	}
+
+	return result, nil
+}
+
+// BrokerList returns a slice of broker addresses that can be used to connect to
+// the Kafka cluster, e.g. using `sarama.NewAsyncProducer()`.
+func (kz *Kazoo) BrokerList() ([]string, error) {
+	brokers, err := kz.Brokers()
+	if err != nil {
+		return nil, err
+	}
+
+	result := make([]string, 0, len(brokers))
+	for _, broker := range brokers {
+		result = append(result, broker)
+	}
+
+	return result, nil
+}
+
+// Controller returns what broker is currently acting as controller of the Kafka cluster
+func (kz *Kazoo) Controller() (int32, error) {
+	type controllerEntry struct {
+		BrokerID int32 `json:"brokerid"`
+	}
+
+	node := fmt.Sprintf("%s/controller", kz.conf.Chroot)
+	data, _, err := kz.conn.Get(node)
+	if err != nil {
+		return -1, err
+	}
+
+	var controllerNode controllerEntry
+	if err := json.Unmarshal(data, &controllerNode); err != nil {
+		return -1, err
+	}
+
+	return controllerNode.BrokerID, nil
+}
+
+// Close closes the connection with the Zookeeper cluster
+func (kz *Kazoo) Close() error {
+	kz.conn.Close()
+	return nil
+}
+
+////////////////////////////////////////////////////////////////////////
+// Util methods
+////////////////////////////////////////////////////////////////////////
+
+// Exists checks existence of a node
+func (kz *Kazoo) exists(node string) (ok bool, err error) {
+	ok, _, err = kz.conn.Exists(node)
+	return
+}
+
+// DeleteAll deletes a node recursively
+func (kz *Kazoo) deleteRecursive(node string) (err error) {
+	children, stat, err := kz.conn.Children(node)
+	if err == zk.ErrNoNode {
+		return nil
+	} else if err != nil {
+		return
+	}
+
+	for _, child := range children {
+		if err = kz.deleteRecursive(path.Join(node, child)); err != nil {
+			return
+		}
+	}
+
+	return kz.conn.Delete(node, stat.Version)
+}
+
+// MkdirAll creates a directory recursively
+func (kz *Kazoo) mkdirRecursive(node string) (err error) {
+	parent := path.Dir(node)
+	if parent != "/" {
+		if err = kz.mkdirRecursive(parent); err != nil {
+			return
+		}
+	}
+
+	_, err = kz.conn.Create(node, nil, 0, zk.WorldACL(zk.PermAll))
+	if err == zk.ErrNodeExists {
+		err = nil
+	}
+	return
+}
+
+// Create stores a new value at node. Fails if already set.
+func (kz *Kazoo) create(node string, value []byte, ephemeral bool) (err error) {
+	if err = kz.mkdirRecursive(path.Dir(node)); err != nil {
+		return
+	}
+
+	flags := int32(0)
+	if ephemeral {
+		flags = zk.FlagEphemeral
+	}
+	_, err = kz.conn.Create(node, value, flags, zk.WorldACL(zk.PermAll))
+	return
+}

vendor/github.com/wvanbergen/kazoo-go/topic_metadata.go

@@ -0,0 +1,255 @@
+package kazoo
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/samuel/go-zookeeper/zk"
+)
+
+// Topic interacts with Kafka's topic metadata in Zookeeper.
+type Topic struct {
+	Name string
+	kz   *Kazoo
+}
+
+// TopicList is a type that implements the sortable interface for a list of Topic instances.
+type TopicList []*Topic
+
+// Partition interacts with Kafka's partition metadata in Zookeeper.
+type Partition struct {
+	topic    *Topic
+	ID       int32
+	Replicas []int32
+}
+
+// PartitionList is a type that implements the sortable interface for a list of Partition instances
+type PartitionList []*Partition
+
+// Topics returns a list of all registered Kafka topics.
+func (kz *Kazoo) Topics() (TopicList, error) {
+	root := fmt.Sprintf("%s/brokers/topics", kz.conf.Chroot)
+	children, _, err := kz.conn.Children(root)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(TopicList, 0, len(children))
+	for _, name := range children {
+		result = append(result, kz.Topic(name))
+	}
+	return result, nil
+}
+
+// WatchTopics returns a list of all registered Kafka topics, and
+// watches that list for changes.
+func (kz *Kazoo) WatchTopics() (TopicList, <-chan zk.Event, error) {
+	root := fmt.Sprintf("%s/brokers/topics", kz.conf.Chroot)
+	children, _, c, err := kz.conn.ChildrenW(root)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	result := make(TopicList, 0, len(children))
+	for _, name := range children {
+		result = append(result, kz.Topic(name))
+	}
+	return result, c, nil
+}
+
+// Topic returns a Topic instance for a given topic name
+func (kz *Kazoo) Topic(topic string) *Topic {
+	return &Topic{Name: topic, kz: kz}
+}
+
+// Exists returns true if the topic exists on the Kafka cluster.
+func (t *Topic) Exists() (bool, error) {
+	return t.kz.exists(fmt.Sprintf("%s/brokers/topics/%s", t.kz.conf.Chroot, t.Name))
+}
+
+// Partitions returns a list of all partitions for the topic.
+func (t *Topic) Partitions() (PartitionList, error) {
+	node := fmt.Sprintf("%s/brokers/topics/%s", t.kz.conf.Chroot, t.Name)
+	value, _, err := t.kz.conn.Get(node)
+	if err != nil {
+		return nil, err
+	}
+
+	return t.parsePartitions(value)
+}
+
+// WatchPartitions returns a list of all partitions for the topic, and watches the topic for changes.
+func (t *Topic) WatchPartitions() (PartitionList, <-chan zk.Event, error) {
+	node := fmt.Sprintf("%s/brokers/topics/%s", t.kz.conf.Chroot, t.Name)
+	value, _, c, err := t.kz.conn.GetW(node)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	list, err := t.parsePartitions(value)
+	return list, c, err
+}
+
+// parsePartitions pases the JSON representation of the partitions
+// that is stored as data on the topic node in Zookeeper.
+func (t *Topic) parsePartitions(value []byte) (PartitionList, error) {
+	type topicMetadata struct {
+		Partitions map[string][]int32 `json:"partitions"`
+	}
+
+	var tm topicMetadata
+	if err := json.Unmarshal(value, &tm); err != nil {
+		return nil, err
+	}
+
+	result := make(PartitionList, len(tm.Partitions))
+	for partitionNumber, replicas := range tm.Partitions {
+		partitionID, err := strconv.ParseInt(partitionNumber, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+
+		replicaIDs := make([]int32, 0, len(replicas))
+		for _, r := range replicas {
+			replicaIDs = append(replicaIDs, int32(r))
+		}
+		result[partitionID] = t.Partition(int32(partitionID), replicaIDs)
+	}
+
+	return result, nil
+}
+
+// Partition returns a Partition instance for the topic.
+func (t *Topic) Partition(id int32, replicas []int32) *Partition {
+	return &Partition{ID: id, Replicas: replicas, topic: t}
+}
+
+// Config returns topic-level configuration settings as a map.
+func (t *Topic) Config() (map[string]string, error) {
+	value, _, err := t.kz.conn.Get(fmt.Sprintf("%s/config/topics/%s", t.kz.conf.Chroot, t.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	var topicConfig struct {
+		ConfigMap map[string]string `json:"config"`
+	}
+
+	if err := json.Unmarshal(value, &topicConfig); err != nil {
+		return nil, err
+	}
+
+	return topicConfig.ConfigMap, nil
+}
+
+// Topic returns the Topic of this partition.
+func (p *Partition) Topic() *Topic {
+	return p.topic
+}
+
+// Key returns a unique identifier for the partition, using the form "topic/partition".
+func (p *Partition) Key() string {
+	return fmt.Sprintf("%s/%d", p.topic.Name, p.ID)
+}
+
+// PreferredReplica returns the preferred replica for this partition.
+func (p *Partition) PreferredReplica() int32 {
+	if len(p.Replicas) > 0 {
+		return p.Replicas[0]
+	} else {
+		return -1
+	}
+}
+
+// Leader returns the broker ID of the broker that is currently the leader for the partition.
+func (p *Partition) Leader() (int32, error) {
+	if state, err := p.state(); err != nil {
+		return -1, err
+	} else {
+		return state.Leader, nil
+	}
+}
+
+// ISR returns the broker IDs of the current in-sync replica set for the partition
+func (p *Partition) ISR() ([]int32, error) {
+	if state, err := p.state(); err != nil {
+		return nil, err
+	} else {
+		return state.ISR, nil
+	}
+}
+
+func (p *Partition) UnderReplicated() (bool, error) {
+	if state, err := p.state(); err != nil {
+		return false, err
+	} else {
+		return len(state.ISR) < len(p.Replicas), nil
+	}
+}
+
+func (p *Partition) UsesPreferredReplica() (bool, error) {
+	if state, err := p.state(); err != nil {
+		return false, err
+	} else {
+		return len(state.ISR) > 0 && state.ISR[0] == p.Replicas[0], nil
+	}
+}
+
+// partitionState represents the partition state as it is stored as JSON
+// in Zookeeper on the partition's state node.
+type partitionState struct {
+	Leader int32   `json:"leader"`
+	ISR    []int32 `json:"isr"`
+}
+
+// state retrieves and parses the partition State
+func (p *Partition) state() (partitionState, error) {
+	var state partitionState
+	node := fmt.Sprintf("%s/brokers/topics/%s/partitions/%d/state", p.topic.kz.conf.Chroot, p.topic.Name, p.ID)
+	value, _, err := p.topic.kz.conn.Get(node)
+	if err != nil {
+		return state, err
+	}
+
+	if err := json.Unmarshal(value, &state); err != nil {
+		return state, err
+	}
+
+	return state, nil
+}
+
+// Find returns the topic with the given name if it exists in the topic list,
+// and will return `nil` otherwise.
+func (tl TopicList) Find(name string) *Topic {
+	for _, topic := range tl {
+		if topic.Name == name {
+			return topic
+		}
+	}
+	return nil
+}
+
+func (tl TopicList) Len() int {
+	return len(tl)
+}
+
+func (tl TopicList) Less(i, j int) bool {
+	return tl[i].Name < tl[j].Name
+}
+
+func (tl TopicList) Swap(i, j int) {
+	tl[i], tl[j] = tl[j], tl[i]
+}
+
+func (pl PartitionList) Len() int {
+	return len(pl)
+}
+
+func (pl PartitionList) Less(i, j int) bool {
+	return pl[i].topic.Name < pl[j].topic.Name || (pl[i].topic.Name == pl[j].topic.Name && pl[i].ID < pl[j].ID)
+}
+
+func (pl PartitionList) Swap(i, j int) {
+	pl[i], pl[j] = pl[j], pl[i]
+}