Merge branch 'master' into dave-j/support-docs-update

.github/CODEOWNERS

@@ -0,0 +1,7 @@
+# Backup-Utils owned by lifecycle AOR
+* @github/ghes-lifecycle
+# Actions related backups and restores
+# /share/github-backup-utils/*-actions @github/ghes-lifecycle @github/<TBD>
+# Git related backups and restores
+# /share/github-backup-utils/*-repositories @github/ghes-lifecycle @github/<TBD>
+# /share/github-backup-utils/*-git-hooks @github/ghes-lifecycle @github/<TBD>

.github/actions/proxy-janky-build/action.yml

@@ -0,0 +1,33 @@
+name: 'Trigger a CI Job on Janky'
+description: 'Action to trigger and poll a Janky CI job'
+inputs:
+  janky-token: 
+    description: 'Token for making request to Janky'
+    required: true
+  job-name:
+    description: 'The name of the job to run'
+    required: true
+  branch-name:
+    description: 'The name of the branch to use'
+    required: true
+  force:
+    description: 'Force the job to run even if it is already passed'
+    required: false
+  envVars:
+    description: 'Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3'
+    required: false
+runs:
+  using: 'composite'
+  steps:
+      - uses: actions/setup-go@a3d889c34c5d4e071b33595c5fe8edfcaaad8260
+        with:
+          go-version: '1.21'
+      - run: |
+          go run main.go \
+          -token ${{ inputs.janky-token }} \
+          -job ${{ inputs.job-name }} \
+          -branch ${{ inputs.branch-name }} \
+          -force ${{ inputs.force }} \
+          -envVars ${{ inputs.envVars }}
+        shell: bash
+        working-directory: .github/actions/proxy-janky-build

.github/actions/proxy-janky-build/go.mod

@@ -0,0 +1,7 @@
+module github.com/github/enterprise2/actions/proxy-janky-build
+
+go 1.21
+
+require github.com/hashicorp/go-retryablehttp v0.7.2
+
+require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect

.github/actions/proxy-janky-build/go.sum

@@ -0,0 +1,9 @@
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
+github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

.github/actions/proxy-janky-build/main.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/hashicorp/go-retryablehttp"
+)
+
+// Define our Janky Response Structs
+type JankyBuildStruct struct {
+	Result string
+	Url    string
+}
+type JankyStatusStruct struct {
+	Id            string
+	Green         bool
+	Completed     bool
+	StartedAt     string
+	CompletedAt   string
+	Sha           string
+	BuildableName string
+}
+
+const (
+	pollWaitTime      = 10 * time.Second
+	jankyPollTimeout  = 5 * time.Hour
+	jankyHttpRetryMax = 5
+	jankyUrl          = "https://janky.githubapp.com"
+)
+
+func main() {
+	// Parse command-line arguments
+	job := flag.String("job", "", "Name of the Janky job")
+	token := flag.String("token", "", "Name of the Janky token")
+	branch := flag.String("branch", "", "Name of the Git branch")
+	force := flag.String("force", "false", "Force a build even if one is already passed")
+	envVars := flag.String("envVars", "", "Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3")
+	flag.Parse()
+
+	// Validate command-line arguments
+	if *job == "" || *token == "" || *branch == "" {
+		log.Fatal("job, token and branch flags must be specified")
+	}
+
+	// Set up the token + request payload
+	authToken := base64.StdEncoding.EncodeToString([]byte(":" + *token))
+	type buildRequestObject struct {
+		BuildableName string            `json:"buildable_name"`
+		BranchName    string            `json:"branch_name"`
+		Force         string            `json:"force"`
+		EnvVars       map[string]string `json:"env_vars"`
+	}
+
+	requestBody := buildRequestObject{
+		BuildableName: *job,
+		BranchName:    *branch,
+		Force:         *force,
+	}
+
+	// Parse the envVars flag into a map and add to the request payload
+	fmt.Println("Environment Variables:")
+	fmt.Println(*envVars)
+	if *envVars != "" {
+		envVarsMap := make(map[string]string)
+		for _, envVar := range strings.Split(*envVars, ",") {
+			envVarSplit := strings.Split(envVar, "=")
+			envVarsMap[envVarSplit[0]] = envVarSplit[1]
+		}
+		requestBody.EnvVars = envVarsMap
+	}
+
+	payloadBytes, err := json.Marshal(requestBody)
+	if err != nil {
+		log.Fatal("Failed to marshal the JSON payload!\n" + err.Error())
+	}
+
+	// Send build request to Janky
+	buildRequest, err := http.NewRequest("POST", jankyUrl+"/api/builds", bytes.NewBuffer(payloadBytes))
+	if err != nil {
+		log.Fatal("Failed to create build request!\n" + err.Error())
+	}
+	buildRequest.Header.Set("Content-Type", "application/json")
+	buildRequest.Header.Set("Authorization", "Basic "+authToken)
+	retryClient := retryablehttp.NewClient() //nolint:all
+	retryClient.RetryMax = jankyHttpRetryMax
+	retryClient.Logger = nil               // disable debug logging
+	client := retryClient.StandardClient() // uses *http.Client
+	resp, err := client.Do(buildRequest)
+	if err != nil {
+		log.Fatal("Failed to send build request!\n" + err.Error())
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatal("Error reading build response!\n" + err.Error())
+	}
+
+	// Check if the build was triggered successfully
+	if resp.StatusCode == 404 {
+		log.Fatal("Failed to trigger build! Either " + *job + " is not the name of a Janky job or " + *branch + " is not a branch for the repository that job belongs to.")
+	}
+	if resp.StatusCode != 201 {
+		log.Fatal("Failed to trigger build! Got exception: " + string(body))
+	}
+
+	// Parse the build request response
+	var buildResponse JankyBuildStruct
+	json.Unmarshal(body, &buildResponse)
+	log.Println("Succesfully triggered janky!\n" + buildResponse.Result)
+
+	// Parse the request response for the buildId
+	r, err := regexp.Compile("/[0-9]+/")
+	if err != nil {
+		log.Fatal("Failed to trigger build!\n" + err.Error())
+	}
+	buildId := strings.Trim(r.FindString(buildResponse.Result), "/")
+
+	// Setup our second HTTP client for reuse in during status polling
+	jankyStatusUrl := jankyUrl + "/api/" + buildId + "/status"
+	statusRequest, err := http.NewRequest("GET", jankyStatusUrl, nil)
+	if err != nil {
+		log.Fatal("Failed to create status request!\n" + err.Error())
+	}
+	statusRequest.Header.Set("Content-Type", "application/json")
+	statusRequest.Header.Set("Authorization", "Basic "+authToken)
+	retryClient2 := retryablehttp.NewClient() //nolint:all
+	retryClient2.RetryMax = jankyHttpRetryMax
+	retryClient2.Logger = nil                // disable debug logging
+	client2 := retryClient2.StandardClient() // uses *http.Client
+
+	// Wait for a completed status from Janky or break the loop after a certain amount of time
+	timeout := time.NewTimer(jankyPollTimeout)
+	poll := time.NewTicker(pollWaitTime)
+
+jobLoop:
+	for {
+		select {
+		case <-timeout.C:
+			log.Fatal("Failed to poll for build status after " + jankyPollTimeout.String() + "hours")
+		case <-poll.C:
+			// Send build status request to Janky
+			statusResponse, err := client2.Do(statusRequest)
+			if err != nil {
+				log.Fatal("Failed to send status request!\n" + err.Error())
+			}
+			defer statusResponse.Body.Close()
+			statusBody, err := io.ReadAll(statusResponse.Body)
+			if err != nil {
+				log.Fatal("Error reading status response!\n" + err.Error())
+			}
+
+			// Parse the status response for a green completed build
+			var jankyStatusResponse JankyStatusStruct
+			json.Unmarshal(statusBody, &jankyStatusResponse)
+			//fmt.Println("Janky Status Response:")
+			//fmt.Println(string(statusBody))
+			if jankyStatusResponse.Completed && jankyStatusResponse.Green {
+				log.Println("Janky build Succeeded!")
+				break jobLoop
+			}
+			if jankyStatusResponse.Completed && !jankyStatusResponse.Green {
+				log.Fatal("Build failed, see Janky for more info: " + buildResponse.Url)
+			}
+
+			// wait for a bit and try again
+			log.Println("Build still in progress, will poll for status again in [" + pollWaitTime.String() + "]")
+			continue
+		}
+	}
+}

.github/linters/.yaml-lint.yml

@@ -0,0 +1,53 @@
+---
+###########################################
+# These are the rules used for            #
+# linting all the yaml files in the stack #
+# NOTE:                                   #
+# You can disable line with:              #
+# # yamllint disable-line                 #
+###########################################
+rules:
+  braces:
+    level: warning
+    min-spaces-inside: 0
+    max-spaces-inside: 0
+    min-spaces-inside-empty: 1
+    max-spaces-inside-empty: 5
+  brackets:
+    level: warning
+    min-spaces-inside: 0
+    max-spaces-inside: 0
+    min-spaces-inside-empty: 1
+    max-spaces-inside-empty: 5
+  colons:
+    level: warning
+    max-spaces-before: 0
+    max-spaces-after: 1
+  commas:
+    level: warning
+    max-spaces-before: 0
+    min-spaces-after: 1
+    max-spaces-after: 1
+  comments: disable
+  comments-indentation: disable
+  document-end: disable
+  document-start: disable
+  empty-lines:
+    level: warning
+    max: 2
+    max-start: 0
+    max-end: 0
+  hyphens:
+    level: warning
+    max-spaces-after: 1
+  indentation:
+    level: warning
+    spaces: consistent
+    indent-sequences: true
+    check-multi-line-strings: false
+  key-duplicates: enable
+  line-length: disable
+  new-line-at-end-of-file: disable
+  new-lines:
+    type: unix
+  trailing-spaces: disable

.github/pull_request_template.md

@@ -0,0 +1,40 @@
+<!-- Welcome to backup-utils-private repo and Thanks for contributing!
+
+Note: Merging to the master branch will include your change in a future (unreleased) version of backup-utils. If the change needs to be shipped to the current release versions it will need to be backported. For more information, see the backport guide https://github.com/github/enterprise-releases/blob/master/docs/backport-an-existing-pr.md
+
+If you have any questions we can be found in the #ghes-backup-utils Slack channel.
+-->
+
+<!--
+Additional notes regarding CI:
+- All required CIs needs to be pass before merging PR
+- Integration test will run against enterprise2 repo with environment variable, do not re-run directly from janky or Github CI, please use Actions to re-run the failed tests
+- If you are making changes impacts cluster, please add `cluster` label or `[cluster]` in your PR title so it will trigger optional cluster integration test. Those tests will take about 3 hours so relax and come back later to check the results. ;)
+-->
+
+# PR Details
+
+## Description
+<!--
+[Please fill out a brief description of the change being made]
+-->
+## Testing
+<!--
+[Please add testing done as part of this change.] 
+-->
+<!-- Keep in mind that for backup-utils the following applies:
+- Backup-util [current version] will support
+   - GHES [current version]
+   - GHES [current version -1]
+   - GHES [current version -2]
+- Any changes that are made to backup-utils will also need to be supported on those GHES versions above (n-2)
+- Please make sure those versions are tested against for this change 
+-->
+
+## Ownership
+<!-- [Add any relevants owners for this change]
+-->
+
+## Related Links
+<!-- [Please add any related links/issues to this PR]
+-->

.github/workflows/backup.yml

@@ -47,7 +47,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         repository: github/backup-utils-private
         token: "${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}"
@@ -62,7 +62,7 @@ jobs:
     needs: build
     runs-on:
       group: larger-hosted-public-runners
-      labels: ubuntu-latest-xl
+      labels: ubuntu-latest
     env:
         SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }}
     steps:
@@ -71,7 +71,7 @@ jobs:
         name: backup-utils
     - name: Load docker container
       run: docker load -i backup-utils.tar
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Create backup directory
       run: mkdir "$HOME/ghe-backup-data"
     - name: set up ssh SSH_KEY
@@ -98,7 +98,7 @@ jobs:
         sudo tar -czvf "${{ inputs.backup-name }}.tar.gz" -C "$HOME/ghe-backup-data/$current" .
 
     - name: Login to Azure
-      if: ${{ inputs.backup-name }} != ""
+      if: "${{ inputs.backup-name != '' }}"
       run: |
         az login \
           --service-principal \
@@ -108,11 +108,11 @@ jobs:
         az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
 
     - name: Upload backup to Azure
-      if: ${{ inputs.backup-name }} != ""
+      if: "${{ inputs.backup-name != '' }}"
       run: |
         az storage blob upload \
         --account-name "${{ secrets.AZURE_ACCOUNT_NAME }}" \
         --container-name "${{ secrets.AZURE_CONTAINER_NAME }}" \
         --name "${{ inputs.backup-name }}.tar.gz" \
         --file "${{ inputs.backup-name }}.tar.gz" \
-        --connection-string "${{ secrets.CONNECTIONSTRING }}"
+        --connection-string "${{ secrets.CONNECTIONSTRING }}"