feat(818): add metrics

Docker/run.sh

@@ -2,8 +2,18 @@
 
 # Get push gateway url and container image from env variable
 if ([ ! -z "$PUSHGATEWAY_URL" ] && [ ! -z "$CONTAINER_IMAGE" ] && [ ! -z "$SD_PIPELINE_ID" ]); then
+  ts=`date "+%s"`
   echo "push build image metrics to prometheus"
-  echo "sd_build_image{image_name=\"$CONTAINER_IMAGE\", pipeline_id=\"$SD_PIPELINE_ID\", node=\"$NODE_ID\"} 1" | curl -s -m 10 --data-binary @- "$PUSHGATEWAY_URL/metrics/job/containerd/instance/$5" &>/dev/null &
+  launcherstartts=$(cat /workspace/metrics | grep launcher_start_ts | awk -F':' '{print $2}')
+  [ -z "$launcherstartts" ] && launcherstartts=$ts
+  launcherendts=$(cat /workspace/metrics | grep launcher_end_ts | awk -F':' '{print $2}')
+  [ -z "$launcherendts" ] && launcherendts=$ts
+  export SD_LAUNCHER_END_TS=$launcherendts
+  duration=$(($ts - $launcherendts))
+  launcherduration=$(($launcherendts - $launcherstartts))
+  echo "sd_build_scheduled{image_name=\"$CONTAINER_IMAGE\", pipeline_id=\"$SD_PIPELINE_ID\", node=\"$NODE_ID\"} 1" | curl -s -m 10 --data-binary @- "$PUSHGATEWAY_URL/metrics/job/containerd/instance/$5" &>/dev/null &
+  echo "sd_build_image_pull_duration_secs{image_name=\"$CONTAINER_IMAGE\", pipeline_id=\"$SD_PIPELINE_ID\", node=\"$NODE_ID\"} $duration" | curl -s -m 10 --data-binary @- "$PUSHGATEWAY_URL/metrics/job/containerd/instance/$5" &>/dev/null &
+  echo "sd_build_launcher_duration_secs{image_name=\"$CONTAINER_IMAGE\", pipeline_id=\"$SD_PIPELINE_ID\", node=\"$NODE_ID\"} $launcherduration" | curl -s -m 10 --data-binary @- "$PUSHGATEWAY_URL/metrics/job/containerd/instance/$5" &>/dev/null &
 fi
 
 echo "run launch"

launch.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -15,10 +16,11 @@ import (
 	"time"
 
 	"github.com/peterbourgon/mergemap"
-	"github.com/screwdriver-cd/launcher/executor"
-	"github.com/screwdriver-cd/launcher/screwdriver"
 	"github.com/urfave/cli"
 	"gopkg.in/fatih/color.v1"
+
+	"github.com/screwdriver-cd/launcher/executor"
+	"github.com/screwdriver-cd/launcher/screwdriver"
 )
 
 // These variables get set by the build script via the LDFLAGS
@@ -41,13 +43,71 @@ var marshal = json.Marshal
 var unmarshal = json.Unmarshal
 var cyanFprintf = color.New(color.FgCyan).Add(color.Underline).FprintfFunc()
 var blackSprint = color.New(color.FgHiBlack).SprintFunc()
+var pushgatewayUrlTimeout = 10
 
 var cleanExit = func() {
 	os.Exit(0)
 }
 
+var client *http.Client
+
 const DefaultTimeout = 90 // 90 minutes
 
+type scmPath struct {
+	Host    string
+	Org     string
+	Repo    string
+	Branch  string
+	RootDir string
+}
+
+/* push metrics to prometheus
+metrics - sd_build_completed, sd_build_run_duration_secs
+status => sd build status
+buildID => sd build id
+*/
+func pushMetrics(status string, buildID int) error {
+	// push metrics if pushgateway url is available
+	if strings.TrimSpace(os.Getenv("PUSHGATEWAY_URL")) != "" && buildID > 0 {
+		timeout := time.Duration(pushgatewayUrlTimeout) * time.Second
+		client.Timeout = timeout
+		url := "http://" + os.Getenv("PUSHGATEWAY_URL") + "/metrics/job/containerd/instance/" + strconv.Itoa(buildID)
+		defer client.CloseIdleConnections()
+		image := os.Getenv("CONTAINER_IMAGE")
+		pipelineId := os.Getenv("SD_PIPELINE_ID")
+		node := os.Getenv("NODE_ID")
+		jobId := os.Getenv("SD_JOB_ID")
+		jobName := os.Getenv("SD_JOB_NAME")
+		scmUrl := os.Getenv("SCM_URL")
+		ts := time.Now().Unix()
+		launcherEndTS := ts
+		if strings.TrimSpace(os.Getenv("SD_LAUNCHER_END_TS")) != "" {
+			launcherEndTS, _ = strconv.ParseInt(os.Getenv("SD_LAUNCHER_END_TS"), 10, 64)
+		}
+		durationSecs := ts - launcherEndTS
+
+		// data need to be specified in this format for pushgateway
+		data := `sd_build_completed{image_name="` + image + `",pipeline_id="` + pipelineId + `",node="` + node + `",job_id="` + jobId + `",job_name="` + jobName + `",scm_url="` + scmUrl + `",status="` + status + `"} 1
+sd_build_run_duration_secs{image_name="` + image + `",pipeline_id="` + pipelineId + `",node="` + node + `",job_id="` + jobId + `",job_name="` + jobName + `",scm_url="` + scmUrl + `",status="` + status + `"} ` + strconv.FormatInt(durationSecs, 10) + `
+`
+		body := strings.NewReader(data)
+		res, err := client.Post(url, "", body)
+		if res != nil {
+			defer res.Body.Close()
+		}
+		if err != nil {
+			log.Printf("pushMetrics: failed to push metrics to [%v], buildId:[%v], error:[%v]", url, buildID, err)
+			return nil
+		}
+		if res.StatusCode/100 != 2 {
+			log.Printf("pushMetrics: failed to push metrics to[%v], buildId:[%v], respose status code:[%v]", url, buildID, res.StatusCode)
+			return nil
+		}
+		log.Printf("pushMetrics: successfully pushed metrics for build:[%v]", buildID)
+	}
+	return nil
+}
+
 // exit sets the build status and exits successfully
 func exit(status screwdriver.BuildStatus, buildID int, api screwdriver.API, metaSpace string) {
 	if api != nil {
@@ -70,17 +130,10 @@ func exit(status screwdriver.BuildStatus, buildID int, api screwdriver.API, meta
 			log.Printf("Failed updating the build status: %v", err)
 		}
 	}
+	pushMetrics(status.String(), buildID)
 	cleanExit()
 }
 
-type scmPath struct {
-	Host    string
-	Org     string
-	Repo    string
-	Branch  string
-	RootDir string
-}
-
 // e.g. scmUri: "github:123456:master", scmName: "screwdriver-cd/launcher"
 func parseScmURI(scmURI, scmName string) (scmPath, error) {
 	uri := strings.Split(scmURI, ":")

launch_test.go

@@ -4,11 +4,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
 	"os"
 	"path"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/screwdriver-cd/launcher/executor"
 	"github.com/screwdriver-cd/launcher/screwdriver"
@@ -1381,3 +1386,97 @@ func TestFetchEventMetaMarshalError(t *testing.T) {
 		t.Errorf("Error is wrong, got '%v', expected '%v'", err, expected)
 	}
 }
+
+func fakeHttpClient() *http.Client {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		data := strings.Split(r.URL.String(), "&")
+		code, _ := strconv.Atoi(data[1])
+		body := data[1]
+		timeout, _ := strconv.Atoi(data[2])
+		timeoutDuration := time.Duration(timeout) * time.Second
+		w.WriteHeader(code)
+		if timeout > 0 {
+			time.Sleep(timeoutDuration)
+		} else {
+			w.Header().Set("Content-Type", "application/json")
+			w.Write([]byte(body))
+		}
+	}))
+
+	transport := &http.Transport{
+		Proxy: func(req *http.Request) (*url.URL, error) {
+			return url.Parse(server.URL)
+		},
+	}
+
+	return &http.Client{Transport: transport}
+}
+
+func TestPushMetrics(t *testing.T) {
+	httpTest := fakeHttpClient()
+	client = httpTest
+	ts := time.Now().Unix() - 2000
+
+	// PUSHGATEWAY_URL null
+	os.Setenv("PUSHGATEWAY_URL", "")
+	err := pushMetrics("success", 1)
+	if err != nil {
+		t.Errorf("Push metrics expect to return [nil] but got [%v]", err)
+	}
+
+	// build id 0
+	os.Setenv("PUSHGATEWAY_URL", "http://fake.pushgateway.url&200&0")
+	err = pushMetrics("success", 0)
+	if err != nil {
+		t.Errorf("Push metrics expect to return [nil] but got [%v]", err)
+	}
+
+	// 200 success
+	os.Setenv("PUSHGATEWAY_URL", "http://fake.pushgateway.url&200&0")
+	os.Setenv("SD_LAUNCHER_END_TS", strconv.FormatInt(ts, 10))
+	err = pushMetrics("success", 1)
+	if err != nil {
+		t.Errorf("Push metrics expect to return [nil] but got [%v]", err)
+	}
+
+	// 200 success, launcher end timestamp null / blank
+	os.Setenv("PUSHGATEWAY_URL", "http://fake.pushgateway.url&200&0")
+	os.Setenv("SD_LAUNCHER_END_TS", "")
+	err = pushMetrics("success", 1)
+	if err != nil {
+		t.Errorf("Push metrics expect to return [nil] but got [%v]", err)
+	}
+
+	// 400
+	os.Setenv("PUSHGATEWAY_URL", "http://fake.pushgateway.url&400&0")
+	os.Setenv("SD_LAUNCHER_END_TS", strconv.FormatInt(ts, 10))
+	err = pushMetrics("success", 1)
+	if err != nil {
+		t.Errorf("Push metrics expect to return [nil] but got [%v]", err)
+	}
+
+	// 200 success
+	os.Setenv("PUSHGATEWAY_URL", "http://fake.pushgateway.url&200&0")
+	os.Setenv("SD_LAUNCHER_END_TS", strconv.FormatInt(ts, 10))
+	err = pushMetrics("failed", 1)
+	if err != nil {
+		t.Errorf("Push metrics expect to return [nil] but got [%v]", err)
+	}
+
+	// 500 success
+	os.Setenv("PUSHGATEWAY_URL", "http://fake.pushgateway.url&500&0")
+	os.Setenv("SD_LAUNCHER_END_TS", strconv.FormatInt(ts, 10))
+	err = pushMetrics("failed", 1)
+	if err != nil {
+		t.Errorf("Push metrics expect to return [nil] but got [%v]", err)
+	}
+
+	// 504
+	pushgatewayUrlTimeout = 1
+	os.Setenv("PUSHGATEWAY_URL", "http://fake.pushgateway.url&504&3")
+	os.Setenv("SD_LAUNCHER_END_TS", strconv.FormatInt(ts, 10))
+	err = pushMetrics("success", 1)
+	if err != nil {
+		t.Errorf("Push metrics expect to return [nil] but got [%v]", err)
+	}
+}