Initial commit

vendor/k8s.io/test-infra/prow/pod-utils/gcs/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "target.go",
+        "upload.go",
+    ],
+    importpath = "k8s.io/test-infra/prow/pod-utils/gcs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/cloud.google.com/go/storage:go_default_library",
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/errorutil:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/kube:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/pod-utils/downwardapi:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/test-infra/prow/pod-utils/gcs/doc.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package gcs handles uploading files and raw data
+// to GCS and determines where in the GCS
+// bucket data should go given a specific
+// job specification
+package gcs

vendor/k8s.io/test-infra/prow/pod-utils/gcs/target.go

@@ -0,0 +1,139 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcs
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	"k8s.io/test-infra/prow/kube"
+	"k8s.io/test-infra/prow/pod-utils/downwardapi"
+)
+
+// PathForSpec determines the GCS path prefix for files uploaded
+// for a specific job spec
+func PathForSpec(spec *downwardapi.JobSpec, pathSegment RepoPathBuilder) string {
+	switch spec.Type {
+	case kube.PeriodicJob, kube.PostsubmitJob:
+		return path.Join("logs", spec.Job, spec.BuildID)
+	case kube.PresubmitJob:
+		return path.Join("pr-logs", "pull", pathSegment(spec.Refs.Org, spec.Refs.Repo), strconv.Itoa(spec.Refs.Pulls[0].Number), spec.Job, spec.BuildID)
+	case kube.BatchJob:
+		return path.Join("pr-logs", "pull", "batch", spec.Job, spec.BuildID)
+	default:
+		logrus.Fatalf("unknown job spec type: %v", spec.Type)
+	}
+	return ""
+}
+
+// AliasForSpec determines the GCS path aliases for a job spec
+func AliasForSpec(spec *downwardapi.JobSpec) string {
+	switch spec.Type {
+	case kube.PeriodicJob, kube.PostsubmitJob, kube.BatchJob:
+		return ""
+	case kube.PresubmitJob:
+		return path.Join("pr-logs", "directory", spec.Job, fmt.Sprintf("%s.txt", spec.BuildID))
+	default:
+		logrus.Fatalf("unknown job spec type: %v", spec.Type)
+	}
+	return ""
+}
+
+// LatestBuildForSpec determines the GCS path for storing the latest
+// build id for a job. pathSegment can be nil so callers of this
+// helper are not required to choose a path strategy but can still
+// get back a result.
+func LatestBuildForSpec(spec *downwardapi.JobSpec, pathSegment RepoPathBuilder) []string {
+	var latestBuilds []string
+	switch spec.Type {
+	case kube.PeriodicJob, kube.PostsubmitJob:
+		latestBuilds = append(latestBuilds, path.Join("logs", spec.Job, "latest-build.txt"))
+	case kube.PresubmitJob:
+		latestBuilds = append(latestBuilds, path.Join("pr-logs", "directory", spec.Job, "latest-build.txt"))
+		// Gubernator expects presubmit tests to upload latest-build.txt
+		// under the PR-specific directory too.
+		if pathSegment != nil {
+			latestBuilds = append(latestBuilds, path.Join("pr-logs", "pull", pathSegment(spec.Refs.Org, spec.Refs.Repo), strconv.Itoa(spec.Refs.Pulls[0].Number), spec.Job, "latest-build.txt"))
+		}
+	case kube.BatchJob:
+		latestBuilds = append(latestBuilds, path.Join("pr-logs", "directory", spec.Job, "latest-build.txt"))
+	default:
+		logrus.Errorf("unknown job spec type: %v", spec.Type)
+		return nil
+	}
+	return latestBuilds
+}
+
+// RootForSpec determines the root GCS path for storing artifacts about
+// the provided job.
+func RootForSpec(spec *downwardapi.JobSpec) string {
+	switch spec.Type {
+	case kube.PeriodicJob, kube.PostsubmitJob:
+		return path.Join("logs", spec.Job)
+	case kube.PresubmitJob, kube.BatchJob:
+		return path.Join("pr-logs", "directory", spec.Job)
+	default:
+		logrus.Errorf("unknown job spec type: %v", spec.Type)
+	}
+	return ""
+}
+
+// RepoPathBuilder builds GCS path segments and embeds defaulting behavior
+type RepoPathBuilder func(org, repo string) string
+
+// NewLegacyRepoPathBuilder returns a builder that handles the legacy path
+// encoding where a path will only contain an org or repo if they are non-default
+func NewLegacyRepoPathBuilder(defaultOrg, defaultRepo string) RepoPathBuilder {
+	return func(org, repo string) string {
+		if org == defaultOrg {
+			if repo == defaultRepo {
+				return ""
+			}
+			return repo
+		}
+		// handle gerrit repo
+		repo = strings.Replace(repo, "/", "_", -1)
+		return fmt.Sprintf("%s_%s", org, repo)
+	}
+}
+
+// NewSingleDefaultRepoPathBuilder returns a builder that handles the legacy path
+// encoding where a path will contain org and repo for all but one default repo
+func NewSingleDefaultRepoPathBuilder(defaultOrg, defaultRepo string) RepoPathBuilder {
+	return func(org, repo string) string {
+		if org == defaultOrg && repo == defaultRepo {
+			return ""
+		}
+		// handle gerrit repo
+		repo = strings.Replace(repo, "/", "_", -1)
+		return fmt.Sprintf("%s_%s", org, repo)
+	}
+}
+
+// NewExplicitRepoPathBuilder returns a builder that handles the path encoding
+// where a path will always have an explicit "org_repo" path segment
+func NewExplicitRepoPathBuilder() RepoPathBuilder {
+	return func(org, repo string) string {
+		// handle gerrit repo
+		repo = strings.Replace(repo, "/", "_", -1)
+		return fmt.Sprintf("%s_%s", org, repo)
+	}
+}

vendor/k8s.io/test-infra/prow/pod-utils/gcs/upload.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+
+	"cloud.google.com/go/storage"
+	"github.com/sirupsen/logrus"
+
+	"k8s.io/test-infra/prow/errorutil"
+)
+
+// UploadFunc knows how to upload into an object
+type UploadFunc func(obj *storage.ObjectHandle) error
+
+// Upload uploads all of the data in the
+// uploadTargets map to GCS in parallel. The map is
+// keyed on GCS path under the bucket
+func Upload(bucket *storage.BucketHandle, uploadTargets map[string]UploadFunc) error {
+	errCh := make(chan error, len(uploadTargets))
+	group := &sync.WaitGroup{}
+	group.Add(len(uploadTargets))
+	for dest, upload := range uploadTargets {
+		obj := bucket.Object(dest)
+		logrus.WithField("dest", dest).Info("Queued for upload")
+		go func(f UploadFunc, obj *storage.ObjectHandle, name string) {
+			defer group.Done()
+			if err := f(obj); err != nil {
+				errCh <- err
+			}
+			logrus.WithField("dest", name).Info("Finished upload")
+		}(upload, obj, dest)
+	}
+	group.Wait()
+	close(errCh)
+	if len(errCh) != 0 {
+		var uploadErrors []error
+		for err := range errCh {
+			uploadErrors = append(uploadErrors, err)
+		}
+		return fmt.Errorf("encountered errors during upload: %v", uploadErrors)
+	}
+
+	return nil
+}
+
+// FileUpload returns an UploadFunc which copies all
+// data from the file on disk to the GCS object
+func FileUpload(file string) UploadFunc {
+	return func(obj *storage.ObjectHandle) error {
+		reader, err := os.Open(file)
+		if err != nil {
+			return err
+		}
+
+		uploadErr := DataUpload(reader)(obj)
+		closeErr := reader.Close()
+
+		return errorutil.NewAggregate(uploadErr, closeErr)
+	}
+}
+
+// DataUpload returns an UploadFunc which copies all
+// data from src reader into GCS
+func DataUpload(src io.Reader) UploadFunc {
+	return func(obj *storage.ObjectHandle) error {
+		writer := obj.NewWriter(context.Background())
+		_, copyErr := io.Copy(writer, src)
+		closeErr := writer.Close()
+
+		return errorutil.NewAggregate(copyErr, closeErr)
+	}
+}