Initial commit

vendor/k8s.io/test-infra/prow/sidecar/BUILD.bazel

@@ -0,0 +1,34 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "options.go",
+        "run.go",
+    ],
+    importpath = "k8s.io/test-infra/prow/sidecar",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/fsnotify/fsnotify:go_default_library",
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/gcsupload:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/pod-utils/downwardapi:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/pod-utils/gcs:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/pod-utils/wrapper:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/test-infra/prow/sidecar/doc.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package sidecar is a library that knows how to report on the
+// output of a process that writes its output and exit code to
+// disk
+package sidecar

vendor/k8s.io/test-infra/prow/sidecar/options.go

@@ -0,0 +1,87 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sidecar
+
+import (
+	"encoding/json"
+	"flag"
+
+	"k8s.io/test-infra/prow/gcsupload"
+	"k8s.io/test-infra/prow/pod-utils/wrapper"
+)
+
+// NewOptions returns an empty Options with no nil fields
+func NewOptions() *Options {
+	return &Options{
+		GcsOptions:     gcsupload.NewOptions(),
+		WrapperOptions: &wrapper.Options{},
+	}
+}
+
+// Options exposes the configuration necessary
+// for defining the process being watched and
+// where in GCS an upload will land.
+type Options struct {
+	GcsOptions     *gcsupload.Options `json:"gcs_options"`
+	WrapperOptions *wrapper.Options   `json:"wrapper_options"`
+}
+
+// Validate ensures that the set of options are
+// self-consistent and valid
+func (o *Options) Validate() error {
+	if err := o.WrapperOptions.Validate(); err != nil {
+		return err
+	}
+
+	return o.GcsOptions.Validate()
+}
+
+const (
+	// JSONConfigEnvVar is the environment variable that
+	// utilities expect to find a full JSON configuration
+	// in when run.
+	JSONConfigEnvVar = "SIDECAR_OPTIONS"
+)
+
+// ConfigVar exposese the environment variable used
+// to store serialized configuration
+func (o *Options) ConfigVar() string {
+	return JSONConfigEnvVar
+}
+
+// LoadConfig loads options from serialized config
+func (o *Options) LoadConfig(config string) error {
+	return json.Unmarshal([]byte(config), o)
+}
+
+// AddFlags binds flags to options
+func (o *Options) AddFlags(flags *flag.FlagSet) {
+	o.GcsOptions.AddFlags(flags)
+	o.WrapperOptions.AddFlags(flags)
+}
+
+// Complete internalizes command line arguments
+func (o *Options) Complete(args []string) {
+	o.GcsOptions.Complete(args)
+}
+
+// Encode will encode the set of options in the format that
+// is expected for the configuration environment variable
+func Encode(options Options) (string, error) {
+	encoded, err := json.Marshal(options)
+	return string(encoded), err
+}

vendor/k8s.io/test-infra/prow/sidecar/run.go

@@ -0,0 +1,162 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sidecar
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/sirupsen/logrus"
+	"k8s.io/test-infra/prow/pod-utils/downwardapi"
+
+	"k8s.io/test-infra/prow/pod-utils/gcs"
+)
+
+// Run will watch for the process being wrapped to exit
+// and then post the status of that process and any artifacts
+// to cloud storage.
+func (o Options) Run() error {
+	spec, err := downwardapi.ResolveSpecFromEnv()
+	if err != nil {
+		return fmt.Errorf("could not resolve job spec: %v", err)
+	}
+
+	// If we are being asked to terminate by the kubelet but we have
+	// NOT seen the test process exit cleanly, we need a to start
+	// uploading artifacts to GCS immediately. If we notice the process
+	// exit while doing this best-effort upload, we can race with the
+	// second upload but we can tolerate this as we'd rather get SOME
+	// data into GCS than attempt to cancel these uploads and get none.
+	interrupt := make(chan os.Signal)
+	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		select {
+		case s := <-interrupt:
+			logrus.Errorf("Received an interrupt: %s", s)
+			o.doUpload(spec, false, true)
+		}
+	}()
+
+	// Only start watching file events if the file doesn't exist
+	// If the file exists, it means the main process already completed.
+	if _, err := os.Stat(o.WrapperOptions.MarkerFile); os.IsNotExist(err) {
+		watcher, err := fsnotify.NewWatcher()
+		if err != nil {
+			return fmt.Errorf("could not begin fsnotify watch: %v", err)
+		}
+		defer watcher.Close()
+
+		ticker := time.NewTicker(30 * time.Second)
+		group := sync.WaitGroup{}
+		group.Add(1)
+		go func() {
+			defer group.Done()
+			for {
+				select {
+				case event := <-watcher.Events:
+					if event.Name == o.WrapperOptions.MarkerFile && event.Op&fsnotify.Create == fsnotify.Create {
+						return
+					}
+				case err := <-watcher.Errors:
+					logrus.WithError(err).Info("Encountered an error during fsnotify watch")
+				case <-ticker.C:
+					if _, err := os.Stat(o.WrapperOptions.MarkerFile); err == nil {
+						return
+					}
+				}
+			}
+		}()
+
+		dir := filepath.Dir(o.WrapperOptions.MarkerFile)
+		if err := watcher.Add(dir); err != nil {
+			return fmt.Errorf("could not add to fsnotify watch: %v", err)
+		}
+		group.Wait()
+		ticker.Stop()
+	}
+
+	// If we are being asked to terminate by the kubelet but we have
+	// seen the test process exit cleanly, we need a chance to upload
+	// artifacts to GCS. The only valid way for this program to exit
+	// after a SIGINT or SIGTERM in this situation is to finish]
+	// uploading, so we ignore the signals.
+	signal.Ignore(os.Interrupt, syscall.SIGTERM)
+
+	passed := false
+	aborted := false
+	returnCodeData, err := ioutil.ReadFile(o.WrapperOptions.MarkerFile)
+	if err != nil {
+		logrus.WithError(err).Warn("Could not read return code from marker file")
+	} else {
+		returnCode, err := strconv.Atoi(strings.TrimSpace(string(returnCodeData)))
+		if err != nil {
+			logrus.WithError(err).Warn("Failed to parse process return code")
+		}
+		passed = returnCode == 0 && err == nil
+		aborted = returnCode == 130
+	}
+
+	return o.doUpload(spec, passed, aborted)
+}
+
+func (o Options) doUpload(spec *downwardapi.JobSpec, passed, aborted bool) error {
+	uploadTargets := map[string]gcs.UploadFunc{
+		"build-log.txt": gcs.FileUpload(o.WrapperOptions.ProcessLog),
+	}
+	var result string
+	switch {
+	case passed:
+		result = "SUCCESS"
+	case aborted:
+		result = "ABORTED"
+	default:
+		result = "FAILURE"
+	}
+
+	finished := struct {
+		Timestamp int64  `json:"timestamp"`
+		Passed    bool   `json:"passed"`
+		Result    string `json:"result"`
+	}{
+		Timestamp: time.Now().Unix(),
+		Passed:    passed,
+		Result:    result,
+	}
+	finishedData, err := json.Marshal(&finished)
+	if err != nil {
+		logrus.WithError(err).Warn("Could not marshal finishing data")
+	} else {
+		uploadTargets["finished.json"] = gcs.DataUpload(bytes.NewBuffer(finishedData))
+	}
+
+	if err := o.GcsOptions.Run(spec, uploadTargets); err != nil {
+		return fmt.Errorf("failed to upload to GCS: %v", err)
+	}
+
+	return nil
+}