Initial commit

vendor/k8s.io/test-infra/prow/pod-utils/clone/BUILD.bazel

@@ -0,0 +1,30 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "clone.go",
+        "format.go",
+        "types.go",
+    ],
+    importpath = "k8s.io/test-infra/prow/pod-utils/clone",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/kube:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/test-infra/prow/pod-utils/clone/clone.go

@@ -0,0 +1,146 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clone
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/test-infra/prow/kube"
+)
+
+// Run clones the refs under the prescribed directory and optionally
+// configures the git username and email in the repository as well.
+func Run(refs kube.Refs, dir, gitUserName, gitUserEmail, cookiePath string, env []string) Record {
+	logrus.WithFields(logrus.Fields{"refs": refs}).Info("Cloning refs")
+	record := Record{Refs: refs}
+	for _, command := range commandsForRefs(refs, dir, gitUserName, gitUserEmail, cookiePath, env) {
+		formattedCommand, output, err := command.run()
+		logrus.WithFields(logrus.Fields{"command": formattedCommand, "output": output, "error": err}).Info("Ran command")
+		message := ""
+		if err != nil {
+			message = err.Error()
+			record.Failed = true
+		}
+		record.Commands = append(record.Commands, Command{Command: formattedCommand, Output: output, Error: message})
+		if err != nil {
+			break
+		}
+	}
+
+	return record
+}
+
+// PathForRefs determines the full path to where
+// refs should be cloned
+func PathForRefs(baseDir string, refs kube.Refs) string {
+	var clonePath string
+	if refs.PathAlias != "" {
+		clonePath = refs.PathAlias
+	} else {
+		clonePath = fmt.Sprintf("github.com/%s/%s", refs.Org, refs.Repo)
+	}
+	return fmt.Sprintf("%s/src/%s", baseDir, clonePath)
+}
+
+func commandsForRefs(refs kube.Refs, dir, gitUserName, gitUserEmail, cookiePath string, env []string) []cloneCommand {
+	repositoryURI := fmt.Sprintf("https://github.com/%s/%s.git", refs.Org, refs.Repo)
+	if refs.CloneURI != "" {
+		repositoryURI = refs.CloneURI
+	}
+	cloneDir := PathForRefs(dir, refs)
+
+	commands := []cloneCommand{{"/", env, "mkdir", []string{"-p", cloneDir}}}
+
+	gitCommand := func(args ...string) cloneCommand {
+		return cloneCommand{dir: cloneDir, env: env, command: "git", args: args}
+	}
+	commands = append(commands, gitCommand("init"))
+	if gitUserName != "" {
+		commands = append(commands, gitCommand("config", "user.name", gitUserName))
+	}
+	if gitUserEmail != "" {
+		commands = append(commands, gitCommand("config", "user.email", gitUserEmail))
+	}
+	if cookiePath != "" {
+		commands = append(commands, gitCommand("config", "http.cookiefile", cookiePath))
+	}
+	commands = append(commands, gitCommand("fetch", repositoryURI, "--tags", "--prune"))
+	commands = append(commands, gitCommand("fetch", repositoryURI, refs.BaseRef))
+
+	// unless the user specifically asks us not to, init submodules
+	if !refs.SkipSubmodules {
+		commands = append(commands, gitCommand("submodule", "update", "--init", "--recursive"))
+	}
+
+	var target string
+	if refs.BaseSHA != "" {
+		target = refs.BaseSHA
+	} else {
+		target = "FETCH_HEAD"
+	}
+	// we need to be "on" the target branch after the sync
+	// so we need to set the branch to point to the base ref,
+	// but we cannot update a branch we are on, so in case we
+	// are on the branch we are syncing, we check out the SHA
+	// first and reset the branch second, then check out the
+	// branch we just reset to be in the correct final state
+	commands = append(commands, gitCommand("checkout", target))
+	commands = append(commands, gitCommand("branch", "--force", refs.BaseRef, target))
+	commands = append(commands, gitCommand("checkout", refs.BaseRef))
+
+	for _, prRef := range refs.Pulls {
+		ref := fmt.Sprintf("pull/%d/head", prRef.Number)
+		if prRef.Ref != "" {
+			ref = prRef.Ref
+		}
+		commands = append(commands, gitCommand("fetch", repositoryURI, ref))
+		var prCheckout string
+		if prRef.SHA != "" {
+			prCheckout = prRef.SHA
+		} else {
+			prCheckout = "FETCH_HEAD"
+		}
+		commands = append(commands, gitCommand("merge", prCheckout))
+	}
+	return commands
+}
+
+type cloneCommand struct {
+	dir     string
+	env     []string
+	command string
+	args    []string
+}
+
+func (c *cloneCommand) run() (string, string, error) {
+	output := bytes.Buffer{}
+	cmd := exec.Command(c.command, c.args...)
+	cmd.Dir = c.dir
+	cmd.Env = append(cmd.Env, c.env...)
+	cmd.Stdout = &output
+	cmd.Stderr = &output
+	err := cmd.Run()
+	return strings.Join(append([]string{c.command}, c.args...), " "), output.String(), err
+}
+
+func (c *cloneCommand) String() string {
+	return fmt.Sprintf("PWD=%s %s %s %s", c.dir, strings.Join(c.env, " "), c.command, strings.Join(c.env, " "))
+}

vendor/k8s.io/test-infra/prow/pod-utils/clone/format.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clone
+
+import (
+	"bytes"
+	"fmt"
+)
+
+// FormatRecord describes the record in a human-readable
+// manner for inclusion into build logs
+func FormatRecord(record Record) string {
+	output := bytes.Buffer{}
+	if record.Failed {
+		fmt.Fprintln(&output, "# FAILED!")
+	}
+	fmt.Fprintf(&output, "# Cloning %s/%s at %s", record.Refs.Org, record.Refs.Repo, record.Refs.BaseRef)
+	if record.Refs.BaseSHA != "" {
+		fmt.Fprintf(&output, "(%s)", record.Refs.BaseSHA)
+	}
+	output.WriteString("\n")
+	if len(record.Refs.Pulls) > 0 {
+		output.WriteString("# Checking out pulls:\n")
+		for _, pull := range record.Refs.Pulls {
+			fmt.Fprintf(&output, "#\t%d", pull.Number)
+			if pull.SHA != "" {
+				fmt.Fprintf(&output, "(%s)", pull.SHA)
+			}
+			fmt.Fprint(&output, "\n")
+		}
+	}
+	for _, command := range record.Commands {
+		fmt.Fprintf(&output, "$ %s\n", command.Command)
+		fmt.Fprint(&output, command.Output)
+		if command.Error != "" {
+			fmt.Fprintf(&output, "# Error: %s\n", command.Error)
+		}
+	}
+
+	return output.String()
+}

vendor/k8s.io/test-infra/prow/pod-utils/clone/types.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clone
+
+import (
+	"k8s.io/test-infra/prow/kube"
+)
+
+// Record is a trace of what the desired
+// git state was, what steps we took to get there,
+// and whether or not we were successful.
+type Record struct {
+	Refs     kube.Refs `json:"refs"`
+	Commands []Command `json:"commands"`
+	Failed   bool      `json:"failed"`
+}
+
+// Command is a trace of a command executed
+// while achieving the desired git state.
+type Command struct {
+	Command string `json:"command"`
+	Output  string `json:"output,omitempty"`
+	Error   string `json:"error,omitempty"`
+}

vendor/k8s.io/test-infra/prow/pod-utils/decorate/BUILD.bazel

@@ -0,0 +1,40 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "podspec.go",
+    ],
+    importpath = "k8s.io/test-infra/prow/pod-utils/decorate",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/clonerefs:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/entrypoint:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/gcsupload:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/initupload:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/kube:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/pod-utils/clone:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/pod-utils/downwardapi:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/pod-utils/wrapper:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/sidecar:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/test-infra/prow/pod-utils/decorate/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package decorate is a library for adding to a user-provided PodSpec
+// in order to create a full Pod that will fulfill a test job
+package decorate

vendor/k8s.io/test-infra/prow/pod-utils/decorate/podspec.go

@@ -0,0 +1,512 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package decorate
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/validation"
+
+	"k8s.io/test-infra/prow/clonerefs"
+	"k8s.io/test-infra/prow/entrypoint"
+	"k8s.io/test-infra/prow/gcsupload"
+	"k8s.io/test-infra/prow/initupload"
+	"k8s.io/test-infra/prow/kube"
+	"k8s.io/test-infra/prow/pod-utils/clone"
+	"k8s.io/test-infra/prow/pod-utils/downwardapi"
+	"k8s.io/test-infra/prow/pod-utils/wrapper"
+	"k8s.io/test-infra/prow/sidecar"
+)
+
+const (
+	logMountName            = "logs"
+	logMountPath            = "/logs"
+	artifactsEnv            = "ARTIFACTS"
+	artifactsPath           = logMountPath + "/artifacts"
+	codeMountName           = "code"
+	codeMountPath           = "/home/prow/go"
+	gopathEnv               = "GOPATH"
+	toolsMountName          = "tools"
+	toolsMountPath          = "/tools"
+	gcsCredentialsMountName = "gcs-credentials"
+	gcsCredentialsMountPath = "/secrets/gcs"
+)
+
+// Labels returns a string slice with label consts from kube.
+func Labels() []string {
+	return []string{kube.ProwJobTypeLabel, kube.CreatedByProw, kube.ProwJobIDLabel}
+}
+
+// VolumeMounts returns a string slice with *MountName consts in it.
+func VolumeMounts() []string {
+	return []string{logMountName, codeMountName, toolsMountName, gcsCredentialsMountName}
+}
+
+// VolumeMountPaths returns a string slice with *MountPath consts in it.
+func VolumeMountPaths() []string {
+	return []string{logMountPath, codeMountPath, toolsMountPath, gcsCredentialsMountPath}
+}
+
+// LabelsAndAnnotationsForSpec returns a minimal set of labels to add to prowjobs or its owned resources.
+//
+// User-provided extraLabels and extraAnnotations values will take precedence over auto-provided values.
+func LabelsAndAnnotationsForSpec(spec kube.ProwJobSpec, extraLabels, extraAnnotations map[string]string) (map[string]string, map[string]string) {
+	jobNameForLabel := spec.Job
+	if len(jobNameForLabel) > validation.LabelValueMaxLength {
+		// TODO(fejta): consider truncating middle rather than end.
+		jobNameForLabel = strings.TrimRight(spec.Job[:validation.LabelValueMaxLength], "-")
+		logrus.Warnf("Cannot use full job name '%s' for '%s' label, will be truncated to '%s'",
+			spec.Job,
+			kube.ProwJobAnnotation,
+			jobNameForLabel,
+		)
+	}
+	labels := map[string]string{
+		kube.CreatedByProw:     "true",
+		kube.ProwJobTypeLabel:  string(spec.Type),
+		kube.ProwJobAnnotation: jobNameForLabel,
+	}
+	if spec.Type != kube.PeriodicJob && spec.Refs != nil {
+		labels[kube.OrgLabel] = spec.Refs.Org
+		labels[kube.RepoLabel] = spec.Refs.Repo
+		if len(spec.Refs.Pulls) > 0 {
+			labels[kube.PullLabel] = strconv.Itoa(spec.Refs.Pulls[0].Number)
+		}
+	}
+
+	for k, v := range extraLabels {
+		labels[k] = v
+	}
+
+	// let's validate labels
+	for key, value := range labels {
+		if errs := validation.IsValidLabelValue(value); len(errs) > 0 {
+			// try to use basename of a path, if path contains invalid //
+			base := filepath.Base(value)
+			if errs := validation.IsValidLabelValue(base); len(errs) == 0 {
+				labels[key] = base
+				continue
+			}
+			logrus.Warnf("Removing invalid label: key - %s, value - %s, error: %s", key, value, errs)
+			delete(labels, key)
+		}
+	}
+
+	annotations := map[string]string{
+		kube.ProwJobAnnotation: spec.Job,
+	}
+	for k, v := range extraAnnotations {
+		annotations[k] = v
+	}
+
+	return labels, annotations
+}
+
+// LabelsAndAnnotationsForJob returns a standard set of labels to add to pod/build/etc resources.
+func LabelsAndAnnotationsForJob(pj kube.ProwJob) (map[string]string, map[string]string) {
+	var extraLabels map[string]string
+	if extraLabels = pj.ObjectMeta.Labels; extraLabels == nil {
+		extraLabels = map[string]string{}
+	}
+	extraLabels[kube.ProwJobIDLabel] = pj.ObjectMeta.Name
+	return LabelsAndAnnotationsForSpec(pj.Spec, extraLabels, nil)
+}
+
+// ProwJobToPod converts a ProwJob to a Pod that will run the tests.
+func ProwJobToPod(pj kube.ProwJob, buildID string) (*v1.Pod, error) {
+	if pj.Spec.PodSpec == nil {
+		return nil, fmt.Errorf("prowjob %q lacks a pod spec", pj.Name)
+	}
+
+	rawEnv, err := downwardapi.EnvForSpec(downwardapi.NewJobSpec(pj.Spec, buildID, pj.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	spec := pj.Spec.PodSpec.DeepCopy()
+	spec.RestartPolicy = "Never"
+	spec.Containers[0].Name = kube.TestContainerName
+
+	// we treat this as false if unset, while kubernetes treats it as true if
+	// unset because it was added in v1.6
+	if spec.AutomountServiceAccountToken == nil {
+		myFalse := false
+		spec.AutomountServiceAccountToken = &myFalse
+	}
+
+	if pj.Spec.DecorationConfig == nil {
+		spec.Containers[0].Env = append(spec.Containers[0].Env, kubeEnv(rawEnv)...)
+	} else {
+		if err := decorate(spec, &pj, rawEnv); err != nil {
+			return nil, fmt.Errorf("error decorating podspec: %v", err)
+		}
+	}
+
+	podLabels, annotations := LabelsAndAnnotationsForJob(pj)
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        pj.ObjectMeta.Name,
+			Labels:      podLabels,
+			Annotations: annotations,
+		},
+		Spec: *spec,
+	}, nil
+}
+
+const cloneLogPath = "clone.json"
+
+// CloneLogPath returns the path to the clone log file in the volume mount.
+func CloneLogPath(logMount kube.VolumeMount) string {
+	return filepath.Join(logMount.MountPath, cloneLogPath)
+}
+
+// Exposed for testing
+const (
+	cloneRefsName    = "clonerefs"
+	cloneRefsCommand = "/clonerefs"
+)
+
+// cloneEnv encodes clonerefs Options into json and puts it into an environment variable
+func cloneEnv(opt clonerefs.Options) ([]v1.EnvVar, error) {
+	// TODO(fejta): use flags
+	cloneConfigEnv, err := clonerefs.Encode(opt)
+	if err != nil {
+		return nil, err
+	}
+	return kubeEnv(map[string]string{clonerefs.JSONConfigEnvVar: cloneConfigEnv}), nil
+}
+
+// sshVolume converts a secret holding ssh keys into the corresponding volume and mount.
+//
+// This is used by CloneRefs to attach the mount to the clonerefs container.
+func sshVolume(secret string) (kube.Volume, kube.VolumeMount) {
+	var sshKeyMode int32 = 0400 // this is octal, so symbolic ref is `u+r`
+	name := strings.Join([]string{"ssh-keys", secret}, "-")
+	mountPath := path.Join("/secrets/ssh", secret)
+	v := kube.Volume{
+		Name: name,
+		VolumeSource: kube.VolumeSource{
+			Secret: &kube.SecretSource{
+				SecretName:  secret,
+				DefaultMode: &sshKeyMode,
+			},
+		},
+	}
+
+	vm := kube.VolumeMount{
+		Name:      name,
+		MountPath: mountPath,
+		ReadOnly:  true,
+	}
+
+	return v, vm
+}
+
+// cookiefileVolumes converts a secret holding cookies into the corresponding volume and mount.
+//
+// Secret can be of the form secret-name/base-name or just secret-name.
+// Here secret-name refers to the kubernetes secret volume to mount, and base-name refers to the key in the secret
+// where the cookies are stored. The secret-name pattern is equivalent to secret-name/secret-name.
+//
+// This is used by CloneRefs to attach the mount to the clonerefs container.
+// The returned string value is the path to the cookiefile for use with --cookiefile.
+func cookiefileVolume(secret string) (kube.Volume, kube.VolumeMount, string) {
+	// Separate secret-name/key-in-secret
+	parts := strings.SplitN(secret, "/", 2)
+	cookieSecret := parts[0]
+	var base string
+	if len(parts) == 1 {
+		base = parts[0] // Assume key-in-secret == secret-name
+	} else {
+		base = parts[1]
+	}
+	var cookiefileMode int32 = 0400 // u+r
+	vol := kube.Volume{
+		Name: "cookiefile",
+		VolumeSource: kube.VolumeSource{
+			Secret: &kube.SecretSource{
+				SecretName:  cookieSecret,
+				DefaultMode: &cookiefileMode,
+			},
+		},
+	}
+	mount := kube.VolumeMount{
+		Name:      vol.Name,
+		MountPath: "/secrets/cookiefile", // append base to flag
+		ReadOnly:  true,
+	}
+	return vol, mount, path.Join(mount.MountPath, base)
+}
+
+// CloneRefs constructs the container and volumes necessary to clone the refs requested by the ProwJob.
+//
+// The container checks out repositories specified by the ProwJob Refs to `codeMount`.
+// A log of what it checked out is written to `clone.json` in `logMount`.
+//
+// The container may need to mount SSH keys and/or cookiefiles in order to access private refs.
+// CloneRefs returns a list of volumes containing these secrets required by the container.
+func CloneRefs(pj kube.ProwJob, codeMount, logMount kube.VolumeMount) (*kube.Container, []kube.Refs, []kube.Volume, error) {
+	if pj.Spec.DecorationConfig == nil {
+		return nil, nil, nil, nil
+	}
+	if skip := pj.Spec.DecorationConfig.SkipCloning; skip != nil && *skip {
+		return nil, nil, nil, nil
+	}
+	var cloneVolumes []kube.Volume
+	var refs []kube.Refs // Do not return []*kube.Refs which we do not own
+	if pj.Spec.Refs != nil {
+		refs = append(refs, *pj.Spec.Refs)
+	}
+	for _, r := range pj.Spec.ExtraRefs {
+		refs = append(refs, r)
+	}
+	if len(refs) == 0 { // nothing to clone
+		return nil, nil, nil, nil
+	}
+	if codeMount.Name == "" || codeMount.MountPath == "" {
+		return nil, nil, nil, fmt.Errorf("codeMount must set Name and MountPath")
+	}
+	if logMount.Name == "" || logMount.MountPath == "" {
+		return nil, nil, nil, fmt.Errorf("logMount must set Name and MountPath")
+	}
+
+	var cloneMounts []kube.VolumeMount
+	var sshKeyPaths []string
+	for _, secret := range pj.Spec.DecorationConfig.SSHKeySecrets {
+		volume, mount := sshVolume(secret)
+		cloneMounts = append(cloneMounts, mount)
+		sshKeyPaths = append(sshKeyPaths, mount.MountPath)
+		cloneVolumes = append(cloneVolumes, volume)
+	}
+
+	var cloneArgs []string
+	var cookiefilePath string
+
+	if cp := pj.Spec.DecorationConfig.CookiefileSecret; cp != "" {
+		v, vm, vp := cookiefileVolume(cp)
+		cloneMounts = append(cloneMounts, vm)
+		cloneVolumes = append(cloneVolumes, v)
+		cookiefilePath = vp
+		cloneArgs = append(cloneArgs, "--cookiefile="+cookiefilePath)
+	}
+
+	env, err := cloneEnv(clonerefs.Options{
+		CookiePath:       cookiefilePath,
+		GitRefs:          refs,
+		GitUserEmail:     clonerefs.DefaultGitUserEmail,
+		GitUserName:      clonerefs.DefaultGitUserName,
+		HostFingerprints: pj.Spec.DecorationConfig.SSHHostFingerprints,
+		KeyFiles:         sshKeyPaths,
+		Log:              CloneLogPath(logMount),
+		SrcRoot:          codeMount.MountPath,
+	})
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("clone env: %v", err)
+	}
+
+	container := kube.Container{
+		Name:         cloneRefsName,
+		Image:        pj.Spec.DecorationConfig.UtilityImages.CloneRefs,
+		Command:      []string{cloneRefsCommand},
+		Args:         cloneArgs,
+		Env:          env,
+		VolumeMounts: append([]kube.VolumeMount{logMount, codeMount}, cloneMounts...),
+	}
+	return &container, refs, cloneVolumes, nil
+}
+
+func decorate(spec *kube.PodSpec, pj *kube.ProwJob, rawEnv map[string]string) error {
+	rawEnv[artifactsEnv] = artifactsPath
+	rawEnv[gopathEnv] = codeMountPath
+	logMount := kube.VolumeMount{
+		Name:      logMountName,
+		MountPath: logMountPath,
+	}
+	logVolume := kube.Volume{
+		Name: logMountName,
+		VolumeSource: kube.VolumeSource{
+			EmptyDir: &kube.EmptyDirVolumeSource{},
+		},
+	}
+
+	codeMount := kube.VolumeMount{
+		Name:      codeMountName,
+		MountPath: codeMountPath,
+	}
+	codeVolume := kube.Volume{
+		Name: codeMountName,
+		VolumeSource: kube.VolumeSource{
+			EmptyDir: &kube.EmptyDirVolumeSource{},
+		},
+	}
+
+	toolsMount := kube.VolumeMount{
+		Name:      toolsMountName,
+		MountPath: toolsMountPath,
+	}
+	toolsVolume := kube.Volume{
+		Name: toolsMountName,
+		VolumeSource: kube.VolumeSource{
+			EmptyDir: &kube.EmptyDirVolumeSource{},
+		},
+	}
+
+	gcsCredentialsMount := kube.VolumeMount{
+		Name:      gcsCredentialsMountName,
+		MountPath: gcsCredentialsMountPath,
+	}
+	gcsCredentialsVolume := kube.Volume{
+		Name: gcsCredentialsMountName,
+		VolumeSource: kube.VolumeSource{
+			Secret: &kube.SecretSource{
+				SecretName: pj.Spec.DecorationConfig.GCSCredentialsSecret,
+			},
+		},
+	}
+
+	cloner, refs, cloneVolumes, err := CloneRefs(*pj, codeMount, logMount)
+	if err != nil {
+		return fmt.Errorf("could not create clonerefs container: %v", err)
+	}
+	if cloner != nil {
+		spec.InitContainers = append([]kube.Container{*cloner}, spec.InitContainers...)
+	}
+
+	gcsOptions := gcsupload.Options{
+		// TODO: pass the artifact dir here too once we figure that out
+		GCSConfiguration:   pj.Spec.DecorationConfig.GCSConfiguration,
+		GcsCredentialsFile: fmt.Sprintf("%s/service-account.json", gcsCredentialsMountPath),
+		DryRun:             false,
+	}
+
+	initUploadOptions := initupload.Options{
+		Options: &gcsOptions,
+	}
+	if cloner != nil {
+		initUploadOptions.Log = CloneLogPath(logMount)
+	}
+
+	// TODO(fejta): use flags
+	initUploadConfigEnv, err := initupload.Encode(initUploadOptions)
+	if err != nil {
+		return fmt.Errorf("could not encode initupload configuration as JSON: %v", err)
+	}
+
+	entrypointLocation := fmt.Sprintf("%s/entrypoint", toolsMountPath)
+
+	spec.InitContainers = append(spec.InitContainers,
+		kube.Container{
+			Name:    "initupload",
+			Image:   pj.Spec.DecorationConfig.UtilityImages.InitUpload,
+			Command: []string{"/initupload"},
+			Env: kubeEnv(map[string]string{
+				initupload.JSONConfigEnvVar: initUploadConfigEnv,
+				downwardapi.JobSpecEnv:      rawEnv[downwardapi.JobSpecEnv], // TODO: shouldn't need this?
+			}),
+			VolumeMounts: []kube.VolumeMount{logMount, gcsCredentialsMount},
+		},
+		kube.Container{
+			Name:         "place-tools",
+			Image:        pj.Spec.DecorationConfig.UtilityImages.Entrypoint,
+			Command:      []string{"/bin/cp"},
+			Args:         []string{"/entrypoint", entrypointLocation},
+			VolumeMounts: []kube.VolumeMount{toolsMount},
+		},
+	)
+
+	wrapperOptions := wrapper.Options{
+		ProcessLog: fmt.Sprintf("%s/process-log.txt", logMountPath),
+		MarkerFile: fmt.Sprintf("%s/marker-file.txt", logMountPath),
+	}
+	// TODO(fejta): use flags
+	entrypointConfigEnv, err := entrypoint.Encode(entrypoint.Options{
+		Args:        append(spec.Containers[0].Command, spec.Containers[0].Args...),
+		Options:     &wrapperOptions,
+		Timeout:     pj.Spec.DecorationConfig.Timeout,
+		GracePeriod: pj.Spec.DecorationConfig.GracePeriod,
+		ArtifactDir: artifactsPath,
+	})
+	if err != nil {
+		return fmt.Errorf("could not encode entrypoint configuration as JSON: %v", err)
+	}
+	allEnv := rawEnv
+	allEnv[entrypoint.JSONConfigEnvVar] = entrypointConfigEnv
+
+	spec.Containers[0].Command = []string{entrypointLocation}
+	spec.Containers[0].Args = []string{}
+	spec.Containers[0].Env = append(spec.Containers[0].Env, kubeEnv(allEnv)...)
+	spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, logMount, toolsMount)
+
+	gcsOptions.Items = append(gcsOptions.Items, artifactsPath)
+	// TODO(fejta): use flags
+	sidecarConfigEnv, err := sidecar.Encode(sidecar.Options{
+		GcsOptions:     &gcsOptions,
+		WrapperOptions: &wrapperOptions,
+	})
+	if err != nil {
+		return fmt.Errorf("could not encode sidecar configuration as JSON: %v", err)
+	}
+
+	spec.Containers = append(spec.Containers, kube.Container{
+		Name:    "sidecar",
+		Image:   pj.Spec.DecorationConfig.UtilityImages.Sidecar,
+		Command: []string{"/sidecar"},
+		Env: kubeEnv(map[string]string{
+			sidecar.JSONConfigEnvVar: sidecarConfigEnv,
+			downwardapi.JobSpecEnv:   rawEnv[downwardapi.JobSpecEnv], // TODO: shouldn't need this?
+		}),
+		VolumeMounts: []kube.VolumeMount{logMount, gcsCredentialsMount},
+	})
+	spec.Volumes = append(spec.Volumes, logVolume, toolsVolume, gcsCredentialsVolume)
+
+	if len(refs) > 0 {
+		spec.Containers[0].WorkingDir = clone.PathForRefs(codeMount.MountPath, refs[0])
+		spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, codeMount)
+		spec.Volumes = append(spec.Volumes, append(cloneVolumes, codeVolume)...)
+	}
+
+	return nil
+}
+
+// kubeEnv transforms a mapping of environment variables
+// into their serialized form for a PodSpec, sorting by
+// the name of the env vars
+func kubeEnv(environment map[string]string) []v1.EnvVar {
+	var keys []string
+	for key := range environment {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	var kubeEnvironment []v1.EnvVar
+	for _, key := range keys {
+		kubeEnvironment = append(kubeEnvironment, v1.EnvVar{
+			Name:  key,
+			Value: environment[key],
+		})
+	}
+
+	return kubeEnvironment
+}

vendor/k8s.io/test-infra/prow/pod-utils/downwardapi/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "jobspec.go",
+    ],
+    importpath = "k8s.io/test-infra/prow/pod-utils/downwardapi",
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/k8s.io/test-infra/prow/kube:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/test-infra/prow/pod-utils/downwardapi/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package downwardapi declares the types used to expose
+// job configuration to the jobs themselves
+package downwardapi

vendor/k8s.io/test-infra/prow/pod-utils/downwardapi/jobspec.go

@@ -0,0 +1,156 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package downwardapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+
+	"k8s.io/test-infra/prow/kube"
+)
+
+// JobSpec is the full downward API that we expose to
+// jobs that realize a ProwJob. We will provide this
+// data to jobs with environment variables in two ways:
+//  - the full spec, in serialized JSON in one variable
+//  - individual fields of the spec in their own variables
+type JobSpec struct {
+	Type      kube.ProwJobType `json:"type,omitempty"`
+	Job       string           `json:"job,omitempty"`
+	BuildID   string           `json:"buildid,omitempty"`
+	ProwJobID string           `json:"prowjobid,omitempty"`
+
+	Refs kube.Refs `json:"refs,omitempty"`
+
+	// we need to keep track of the agent until we
+	// migrate everyone away from using the $BUILD_NUMBER
+	// environment variable
+	agent kube.ProwJobAgent
+}
+
+// NewJobSpec converts a kube.ProwJobSpec invocation into a JobSpec
+func NewJobSpec(spec kube.ProwJobSpec, buildID, prowJobID string) JobSpec {
+	refs := kube.Refs{}
+	if spec.Refs != nil {
+		refs = *spec.Refs
+	}
+
+	return JobSpec{
+		Type:      spec.Type,
+		Job:       spec.Job,
+		BuildID:   buildID,
+		ProwJobID: prowJobID,
+		Refs:      refs,
+		agent:     spec.Agent,
+	}
+}
+
+// ResolveSpecFromEnv will determine the Refs being
+// tested in by parsing Prow environment variable contents
+func ResolveSpecFromEnv() (*JobSpec, error) {
+	specEnv, ok := os.LookupEnv(JobSpecEnv)
+	if !ok {
+		return nil, fmt.Errorf("$%s unset", JobSpecEnv)
+	}
+
+	spec := &JobSpec{}
+	if err := json.Unmarshal([]byte(specEnv), spec); err != nil {
+		return nil, fmt.Errorf("malformed $%s: %v", JobSpecEnv, err)
+	}
+
+	return spec, nil
+}
+
+const (
+	// JobSpecEnv is the name that contains JobSpec marshaled into a string.
+	JobSpecEnv = "JOB_SPEC"
+
+	jobNameEnv   = "JOB_NAME"
+	jobTypeEnv   = "JOB_TYPE"
+	prowJobIDEnv = "PROW_JOB_ID"
+
+	buildIDEnv     = "BUILD_ID"
+	prowBuildIDEnv = "BUILD_NUMBER" // Deprecated, will be removed in the future.
+
+	repoOwnerEnv   = "REPO_OWNER"
+	repoNameEnv    = "REPO_NAME"
+	pullBaseRefEnv = "PULL_BASE_REF"
+	pullBaseShaEnv = "PULL_BASE_SHA"
+	pullRefsEnv    = "PULL_REFS"
+	pullNumberEnv  = "PULL_NUMBER"
+	pullPullShaEnv = "PULL_PULL_SHA"
+)
+
+// EnvForSpec returns a mapping of environment variables
+// to their values that should be available for a job spec
+func EnvForSpec(spec JobSpec) (map[string]string, error) {
+	env := map[string]string{
+		jobNameEnv:   spec.Job,
+		buildIDEnv:   spec.BuildID,
+		prowJobIDEnv: spec.ProwJobID,
+		jobTypeEnv:   string(spec.Type),
+	}
+
+	// for backwards compatibility, we provide the build ID
+	// in both $BUILD_ID and $BUILD_NUMBER for Prow agents
+	// and in both $buildId and $BUILD_NUMBER for Jenkins
+	if spec.agent == kube.KubernetesAgent {
+		env[prowBuildIDEnv] = spec.BuildID
+	}
+
+	raw, err := json.Marshal(spec)
+	if err != nil {
+		return env, fmt.Errorf("failed to marshal job spec: %v", err)
+	}
+	env[JobSpecEnv] = string(raw)
+
+	if spec.Type == kube.PeriodicJob {
+		return env, nil
+	}
+	env[repoOwnerEnv] = spec.Refs.Org
+	env[repoNameEnv] = spec.Refs.Repo
+	env[pullBaseRefEnv] = spec.Refs.BaseRef
+	env[pullBaseShaEnv] = spec.Refs.BaseSHA
+	env[pullRefsEnv] = spec.Refs.String()
+
+	if spec.Type == kube.PostsubmitJob || spec.Type == kube.BatchJob {
+		return env, nil
+	}
+	env[pullNumberEnv] = strconv.Itoa(spec.Refs.Pulls[0].Number)
+	env[pullPullShaEnv] = spec.Refs.Pulls[0].SHA
+	return env, nil
+}
+
+// EnvForType returns the slice of environment variables to export for jobType
+func EnvForType(jobType kube.ProwJobType) []string {
+	baseEnv := []string{jobNameEnv, JobSpecEnv, jobTypeEnv, prowJobIDEnv, buildIDEnv, prowBuildIDEnv}
+	refsEnv := []string{repoOwnerEnv, repoNameEnv, pullBaseRefEnv, pullBaseShaEnv, pullRefsEnv}
+	pullEnv := []string{pullNumberEnv, pullPullShaEnv}
+
+	switch jobType {
+	case kube.PeriodicJob:
+		return baseEnv
+	case kube.PostsubmitJob, kube.BatchJob:
+		return append(baseEnv, refsEnv...)
+	case kube.PresubmitJob:
+		return append(append(baseEnv, refsEnv...), pullEnv...)
+	default:
+		return []string{}
+	}
+}

vendor/k8s.io/test-infra/prow/pod-utils/gcs/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "target.go",
+        "upload.go",
+    ],
+    importpath = "k8s.io/test-infra/prow/pod-utils/gcs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/cloud.google.com/go/storage:go_default_library",
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/errorutil:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/kube:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/pod-utils/downwardapi:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/test-infra/prow/pod-utils/gcs/doc.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package gcs handles uploading files and raw data
+// to GCS and determines where in the GCS
+// bucket data should go given a specific
+// job specification
+package gcs

vendor/k8s.io/test-infra/prow/pod-utils/gcs/target.go

@@ -0,0 +1,139 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcs
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	"k8s.io/test-infra/prow/kube"
+	"k8s.io/test-infra/prow/pod-utils/downwardapi"
+)
+
+// PathForSpec determines the GCS path prefix for files uploaded
+// for a specific job spec
+func PathForSpec(spec *downwardapi.JobSpec, pathSegment RepoPathBuilder) string {
+	switch spec.Type {
+	case kube.PeriodicJob, kube.PostsubmitJob:
+		return path.Join("logs", spec.Job, spec.BuildID)
+	case kube.PresubmitJob:
+		return path.Join("pr-logs", "pull", pathSegment(spec.Refs.Org, spec.Refs.Repo), strconv.Itoa(spec.Refs.Pulls[0].Number), spec.Job, spec.BuildID)
+	case kube.BatchJob:
+		return path.Join("pr-logs", "pull", "batch", spec.Job, spec.BuildID)
+	default:
+		logrus.Fatalf("unknown job spec type: %v", spec.Type)
+	}
+	return ""
+}
+
+// AliasForSpec determines the GCS path aliases for a job spec
+func AliasForSpec(spec *downwardapi.JobSpec) string {
+	switch spec.Type {
+	case kube.PeriodicJob, kube.PostsubmitJob, kube.BatchJob:
+		return ""
+	case kube.PresubmitJob:
+		return path.Join("pr-logs", "directory", spec.Job, fmt.Sprintf("%s.txt", spec.BuildID))
+	default:
+		logrus.Fatalf("unknown job spec type: %v", spec.Type)
+	}
+	return ""
+}
+
+// LatestBuildForSpec determines the GCS path for storing the latest
+// build id for a job. pathSegment can be nil so callers of this
+// helper are not required to choose a path strategy but can still
+// get back a result.
+func LatestBuildForSpec(spec *downwardapi.JobSpec, pathSegment RepoPathBuilder) []string {
+	var latestBuilds []string
+	switch spec.Type {
+	case kube.PeriodicJob, kube.PostsubmitJob:
+		latestBuilds = append(latestBuilds, path.Join("logs", spec.Job, "latest-build.txt"))
+	case kube.PresubmitJob:
+		latestBuilds = append(latestBuilds, path.Join("pr-logs", "directory", spec.Job, "latest-build.txt"))
+		// Gubernator expects presubmit tests to upload latest-build.txt
+		// under the PR-specific directory too.
+		if pathSegment != nil {
+			latestBuilds = append(latestBuilds, path.Join("pr-logs", "pull", pathSegment(spec.Refs.Org, spec.Refs.Repo), strconv.Itoa(spec.Refs.Pulls[0].Number), spec.Job, "latest-build.txt"))
+		}
+	case kube.BatchJob:
+		latestBuilds = append(latestBuilds, path.Join("pr-logs", "directory", spec.Job, "latest-build.txt"))
+	default:
+		logrus.Errorf("unknown job spec type: %v", spec.Type)
+		return nil
+	}
+	return latestBuilds
+}
+
+// RootForSpec determines the root GCS path for storing artifacts about
+// the provided job.
+func RootForSpec(spec *downwardapi.JobSpec) string {
+	switch spec.Type {
+	case kube.PeriodicJob, kube.PostsubmitJob:
+		return path.Join("logs", spec.Job)
+	case kube.PresubmitJob, kube.BatchJob:
+		return path.Join("pr-logs", "directory", spec.Job)
+	default:
+		logrus.Errorf("unknown job spec type: %v", spec.Type)
+	}
+	return ""
+}
+
+// RepoPathBuilder builds GCS path segments and embeds defaulting behavior
+type RepoPathBuilder func(org, repo string) string
+
+// NewLegacyRepoPathBuilder returns a builder that handles the legacy path
+// encoding where a path will only contain an org or repo if they are non-default
+func NewLegacyRepoPathBuilder(defaultOrg, defaultRepo string) RepoPathBuilder {
+	return func(org, repo string) string {
+		if org == defaultOrg {
+			if repo == defaultRepo {
+				return ""
+			}
+			return repo
+		}
+		// handle gerrit repo
+		repo = strings.Replace(repo, "/", "_", -1)
+		return fmt.Sprintf("%s_%s", org, repo)
+	}
+}
+
+// NewSingleDefaultRepoPathBuilder returns a builder that handles the legacy path
+// encoding where a path will contain org and repo for all but one default repo
+func NewSingleDefaultRepoPathBuilder(defaultOrg, defaultRepo string) RepoPathBuilder {
+	return func(org, repo string) string {
+		if org == defaultOrg && repo == defaultRepo {
+			return ""
+		}
+		// handle gerrit repo
+		repo = strings.Replace(repo, "/", "_", -1)
+		return fmt.Sprintf("%s_%s", org, repo)
+	}
+}
+
+// NewExplicitRepoPathBuilder returns a builder that handles the path encoding
+// where a path will always have an explicit "org_repo" path segment
+func NewExplicitRepoPathBuilder() RepoPathBuilder {
+	return func(org, repo string) string {
+		// handle gerrit repo
+		repo = strings.Replace(repo, "/", "_", -1)
+		return fmt.Sprintf("%s_%s", org, repo)
+	}
+}

vendor/k8s.io/test-infra/prow/pod-utils/gcs/upload.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+
+	"cloud.google.com/go/storage"
+	"github.com/sirupsen/logrus"
+
+	"k8s.io/test-infra/prow/errorutil"
+)
+
+// UploadFunc knows how to upload into an object
+type UploadFunc func(obj *storage.ObjectHandle) error
+
+// Upload uploads all of the data in the
+// uploadTargets map to GCS in parallel. The map is
+// keyed on GCS path under the bucket
+func Upload(bucket *storage.BucketHandle, uploadTargets map[string]UploadFunc) error {
+	errCh := make(chan error, len(uploadTargets))
+	group := &sync.WaitGroup{}
+	group.Add(len(uploadTargets))
+	for dest, upload := range uploadTargets {
+		obj := bucket.Object(dest)
+		logrus.WithField("dest", dest).Info("Queued for upload")
+		go func(f UploadFunc, obj *storage.ObjectHandle, name string) {
+			defer group.Done()
+			if err := f(obj); err != nil {
+				errCh <- err
+			}
+			logrus.WithField("dest", name).Info("Finished upload")
+		}(upload, obj, dest)
+	}
+	group.Wait()
+	close(errCh)
+	if len(errCh) != 0 {
+		var uploadErrors []error
+		for err := range errCh {
+			uploadErrors = append(uploadErrors, err)
+		}
+		return fmt.Errorf("encountered errors during upload: %v", uploadErrors)
+	}
+
+	return nil
+}
+
+// FileUpload returns an UploadFunc which copies all
+// data from the file on disk to the GCS object
+func FileUpload(file string) UploadFunc {
+	return func(obj *storage.ObjectHandle) error {
+		reader, err := os.Open(file)
+		if err != nil {
+			return err
+		}
+
+		uploadErr := DataUpload(reader)(obj)
+		closeErr := reader.Close()
+
+		return errorutil.NewAggregate(uploadErr, closeErr)
+	}
+}
+
+// DataUpload returns an UploadFunc which copies all
+// data from src reader into GCS
+func DataUpload(src io.Reader) UploadFunc {
+	return func(obj *storage.ObjectHandle) error {
+		writer := obj.NewWriter(context.Background())
+		_, copyErr := io.Copy(writer, src)
+		closeErr := writer.Close()
+
+		return errorutil.NewAggregate(copyErr, closeErr)
+	}
+}

vendor/k8s.io/test-infra/prow/pod-utils/wrapper/BUILD.bazel

@@ -0,0 +1,25 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "options.go",
+    ],
+    importpath = "k8s.io/test-infra/prow/pod-utils/wrapper",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/test-infra/prow/pod-utils/wrapper/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package wrapper contains utilities for the processes that
+// wrap the test execution in a ProwJob test container
+package wrapper

vendor/k8s.io/test-infra/prow/pod-utils/wrapper/options.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package wrapper
+
+import (
+	"errors"
+	"flag"
+)
+
+// Options exposes the configuration options
+// used when wrapping test execution
+type Options struct {
+	// ProcessLog will contain std{out,err} from the
+	// wrapped test process
+	ProcessLog string `json:"process_log"`
+
+	// MarkerFile will be written with the exit code
+	// of the test process or an internal error code
+	// if the entrypoint fails.
+	MarkerFile string `json:"marker_file"`
+}
+
+// AddFlags adds flags to the FlagSet that populate
+// the wrapper options struct provided.
+func (o *Options) AddFlags(fs *flag.FlagSet) {
+	fs.StringVar(&o.ProcessLog, "process-log", "", "path to the log where stdout and stderr are streamed for the process we execute")
+	fs.StringVar(&o.MarkerFile, "marker-file", "", "file we write the return code of the process we execute once it has finished running")
+}
+
+// Validate ensures that the set of options are
+// self-consistent and valid
+func (o *Options) Validate() error {
+	if o.ProcessLog == "" {
+		return errors.New("no log file specified with --process-log")
+	}
+
+	if o.MarkerFile == "" {
+		return errors.New("no marker file specified with --marker-file")
+	}
+
+	return nil
+}