Added two new metrics to collect all the past commit history accordin…

.gitignore

@@ -0,0 +1 @@
+github-exporter

config/config.go

@@ -17,6 +17,7 @@ type Config struct {
 	*cfg.BaseConfig
 	APIURL        string
 	Repositories  string
+	Branch        string
 	Organisations string
 	Users         string
 	APITokenEnv   string
@@ -31,12 +32,13 @@ func Init() Config {
 	ac := cfg.Init()
 	url := cfg.GetEnv("API_URL", "https://api.github.com")
 	repos := os.Getenv("REPOS")
+	branch := os.Getenv("BRANCH")
 	orgs := os.Getenv("ORGS")
 	users := os.Getenv("USERS")
 	tokenEnv := os.Getenv("GITHUB_TOKEN")
 	tokenFile := os.Getenv("GITHUB_TOKEN_FILE")
 	token, err := getAuth(tokenEnv, tokenFile)
-	scraped, err := getScrapeURLs(url, repos, orgs, users)
+	scraped, err := getScrapeURLs(url, repos, branch, orgs, users)
 
 	if err != nil {
 		log.Errorf("Error initialising Configuration, Error: %v", err)
@@ -46,6 +48,7 @@ func Init() Config {
 		&ac,
 		url,
 		repos,
+		branch,
 		orgs,
 		users,
 		tokenEnv,
@@ -59,7 +62,7 @@ func Init() Config {
 
 // Init populates the Config struct based on environmental runtime configuration
 // All URL's are added to the TargetURL's string array
-func getScrapeURLs(apiURL, repos, orgs, users string) ([]string, error) {
+func getScrapeURLs(apiURL, repos, branch, orgs, users string) ([]string, error) {
 
 	urls := []string{}
 
@@ -76,6 +79,9 @@ func getScrapeURLs(apiURL, repos, orgs, users string) ([]string, error) {
 		for _, x := range rs {
 			y := fmt.Sprintf("%s/repos/%s%s", apiURL, x, opts)
 			urls = append(urls, y)
+			// Append commits history to the array
+			z := fmt.Sprintf("%s/repos/%s/commits%s&sha=%s", apiURL, x, opts, branch)
+			urls = append(urls, z)
 		}
 	}
 

docker-compose.yml

@@ -10,5 +10,8 @@ services:
       - 9171:9171
     image: infinityworks/github-exporter:latest 
     environment:
-      - REPOS=<REPOS you want to monitor>
+      - REPOS=<REPOS you want to monitor separated by a comma>
+      - BRANCH=<BRANCH you want to look at for commit history>
       - GITHUB_TOKEN=<your github api token>
+      - LOG_LEVEL=<log level - default is DEBUG>
+

exporter/gather.go

@@ -1,36 +1,65 @@
 package exporter
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"strconv"
+	"strings"
 
 	log "github.com/sirupsen/logrus"
 )
 
 // gatherData - Collects the data from the API and stores into struct
-func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) {
+func (e *Exporter) gatherData() ([]*Datum, []*CommitDatum, *RateLimits, error) {
 
 	data := []*Datum{}
+	commitData := []*CommitDatum{}
 
 	responses, err := asyncHTTPGets(e.TargetURLs, e.APIToken)
 
 	if err != nil {
-		return data, nil, err
+		return data, commitData, nil, err
 	}
 
+	opts := "?&per_page=100"
 	for _, response := range responses {
 
 		// Github can at times present an array, or an object for the same data set.
 		// This code checks handles this variation.
 		if isArray(response.body) {
-			ds := []*Datum{}
-			json.Unmarshal(response.body, &ds)
-			data = append(data, ds...)
+			if isCommitData(response.body) {
+				cds := []*CommitDatum{}
+				json.Unmarshal(response.body, &cds)
+				commitData = append(commitData, cds...)
+				for len(commitData[len(commitData)-1].Parents) != 0 {
+					apiURL := strings.Split(commitData[len(commitData)-1].URL, "/commits/")[0]
+					urls := []string{fmt.Sprintf("%s/commits%s&sha=%s", apiURL, opts, commitData[len(commitData)-1].CommitHash)}
+					responsesNext, err := asyncHTTPGets(urls, e.APIToken)
+					if err != nil {
+						break
+					}
+					for _, r := range responsesNext {
+						cds = []*CommitDatum{}
+						json.Unmarshal(r.body, &cds)
+						commitData = append(commitData, cds...)
+					}
+				}
+			} else {
+				ds := []*Datum{}
+				json.Unmarshal(response.body, &ds)
+				data = append(data, ds...)
+			}
 		} else {
-			d := new(Datum)
-			json.Unmarshal(response.body, &d)
-			data = append(data, d)
+			if isCommitData(response.body) {
+				cd := new(CommitDatum)
+				json.Unmarshal(response.body, &cd)
+				commitData = append(commitData, cd)
+			} else {
+				d := new(Datum)
+				json.Unmarshal(response.body, &d)
+				data = append(data, d)
+			}
 		}
 
 		log.Infof("API data fetched for repository: %s", response.url)
@@ -43,8 +72,8 @@ func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) {
 		log.Errorf("Unable to obtain rate limit data from API, Error: %s", err)
 	}
 
-	//return data, rates, err
-	return data, rates, nil
+	//return data, commitData, rates, err
+	return data, commitData, rates, nil
 
 }
 
@@ -108,3 +137,15 @@ func isArray(body []byte) bool {
 	return isArray
 
 }
+
+func isCommitData(body []byte) bool {
+
+	isCommitData := false
+
+	data := body[:10]
+	if bytes.Contains(data, []byte(`"sha":`)) {
+		isCommitData = true
+	}
+
+	return isCommitData
+}

exporter/metrics.go

@@ -1,7 +1,11 @@
 package exporter
 
-import "github.com/prometheus/client_golang/prometheus"
-import "strconv"
+import (
+	"strconv"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
 
 // AddMetrics - Add's all of the metrics to a map of strings, returns the map.
 func AddMetrics() map[string]*prometheus.Desc {
@@ -33,6 +37,16 @@ func AddMetrics() map[string]*prometheus.Desc {
 		"Size in KB for given repository",
 		[]string{"repo", "user", "private", "fork", "archived", "license", "language"}, nil,
 	)
+	APIMetrics["CommitsHistory"] = prometheus.NewDesc(
+		prometheus.BuildFQName("github", "commit", "count"),
+		"Total number of commits for given repository and given branch",
+		[]string{"repo", "branch", "author"}, nil,
+	)
+	APIMetrics["LatestCommit"] = prometheus.NewDesc(
+		prometheus.BuildFQName("github", "commit", "latest"),
+		"Latest Commit for a given repository and given branch",
+		[]string{"repo", "branch", "author", "date", "commithash"}, nil,
+	)
 	APIMetrics["Limit"] = prometheus.NewDesc(
 		prometheus.BuildFQName("github", "rate", "limit"),
 		"Number of API queries allowed in a 60 minute window",
@@ -53,7 +67,7 @@ func AddMetrics() map[string]*prometheus.Desc {
 }
 
 // processMetrics - processes the response data and sets the metrics using it as a source
-func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- prometheus.Metric) error {
+func (e *Exporter) processMetrics(data []*Datum, commitData []*CommitDatum, rates *RateLimits, ch chan<- prometheus.Metric) error {
 
 	// APIMetrics - range through the data slice
 	for _, x := range data {
@@ -62,7 +76,31 @@ func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- pr
 		ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, x.OpenIssues, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language)
 		ch <- prometheus.MustNewConstMetric(e.APIMetrics["Watchers"], prometheus.GaugeValue, x.Watchers, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language)
 		ch <- prometheus.MustNewConstMetric(e.APIMetrics["Size"], prometheus.GaugeValue, x.Size, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language)
+	}
 
+	branch := e.Config.Branch
+	latestCommits := make(map[string]*LatestCommitHistory)
+	totalCommits := make(map[string]*CommitHistory)
+	for _, x := range commitData {
+		shortenedRepo := strings.Replace(x.URL, "https://api.github.com/repos/", "", -1)
+		repo := shortenedRepo[:strings.Index(shortenedRepo, "/commits")]
+		author := x.Commit.Author.Name
+		if _, ok := latestCommits[repo]; !ok {
+			date := strings.Split(x.Commit.Author.Date, "T")[0]
+			hash := x.CommitHash
+			latestCommits[repo] = &LatestCommitHistory{author, repo, date, hash}
+		}
+		if _, ok := totalCommits[author+repo]; ok {
+			totalCommits[author+repo].Count++
+		} else {
+			totalCommits[author+repo] = &CommitHistory{author, repo, 1.0}
+		}
+	}
+	for _, val := range totalCommits {
+		ch <- prometheus.MustNewConstMetric(e.APIMetrics["CommitsHistory"], prometheus.GaugeValue, val.Count, val.Repo, branch, val.Author)
+	}
+	for _, val := range latestCommits {
+		ch <- prometheus.MustNewConstMetric(e.APIMetrics["LatestCommit"], prometheus.GaugeValue, 1.0, val.Repo, branch, val.Author, val.Date, val.Hash)
 	}
 
 	// Set Rate limit stats

exporter/prometheus.go

@@ -18,16 +18,16 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
 // This function is called when a scrape is peformed on the /metrics page
 func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
 
-	// Scrape the Data from Github
-	var data, rates, err = e.gatherData()
+	// Scrape the Data and CommitData from Github
+	var data, commitData, rates, err = e.gatherData()
 
 	if err != nil {
 		log.Errorf("Error gathering Data from remote API: %v", err)
 		return
 	}
 
 	// Set prometheus gauge metrics using the data gathered
-	err = e.processMetrics(data, rates, ch)
+	err = e.processMetrics(data, commitData, rates, ch)
 
 	if err != nil {
 		log.Error("Error Processing Metrics", err)

exporter/structs.go

@@ -39,6 +39,37 @@ type Datum struct {
 	Size       float64 `json:"size"`
 }
 
+// CommitData is used to store an array of CommitDatum
+type CommitData []CommitDatum
+
+// CommitDatum is used to store commit historical data
+type CommitDatum struct {
+	CommitHash string `json:"sha"`
+	Commit     struct {
+		Author struct {
+			Name string `json:"name"`
+			Date string `json:"date"`
+		} `json:"author"`
+	} `json:"commit"`
+	URL     string `json:"url"`
+	Parents []struct {
+		CommitHash string `json:"sha"`
+	} `json:"parents"`
+}
+
+type CommitHistory struct {
+	Author string
+	Repo   string
+	Count  float64
+}
+
+type LatestCommitHistory struct {
+	Author string
+	Repo   string
+	Date   string
+	Hash   string
+}
+
 // RateLimits is used to store rate limit data into a struct
 // This data is later represented as a metric, captured at the end of a scrape
 type RateLimits struct {