Merge branch 'master' into refactor/fix-minor-issues

.github/workflows/codeql-analysis.yml

@@ -38,11 +38,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +53,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -67,4 +67,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3

.github/workflows/test.yml

@@ -11,10 +11,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
         with:
           go-version: "1.21"
           check-latest: true

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21 as builder
+FROM golang:1.22 as builder
 WORKDIR /go/src/app
 
 COPY . .

controller/controller.go

@@ -8,12 +8,14 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/dynamodb"
+	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
 	"github.com/brave/go-update/extension"
 	"github.com/getsentry/sentry-go"
 	"github.com/go-chi/chi"
@@ -43,8 +45,11 @@ func IsJSONRequest(contentType string) bool {
 }
 
 func initExtensionUpdatesFromDynamoDB() {
-	sess, err := session.NewSession(&aws.Config{})
-
+	awsConfig := &aws.Config{}
+	if endpoint := os.Getenv("DYNAMODB_ENDPOINT"); endpoint != "" {
+		awsConfig.Endpoint = aws.String(endpoint)
+	}
+	sess, err := session.NewSession(awsConfig)
 	if err != nil {
 		log.Printf("failed to connect to new session %v\n", err)
 		sentry.CaptureException(err)
@@ -70,13 +75,27 @@ func initExtensionUpdatesFromDynamoDB() {
 	// Update the extensions map
 	for _, item := range result.Items {
 		id := *item["ID"].S
-		AllExtensionsMap.Store(id, extension.Extension{
+
+		ext := extension.Extension{
 			ID:          id,
 			Blacklisted: *item["Disabled"].BOOL,
 			SHA256:      *item["SHA256"].S,
 			Title:       *item["Title"].S,
 			Version:     *item["Version"].S,
-		})
+		}
+
+		if plist := item["PatchList"]; plist != nil {
+			var pinfo map[string]*extension.PatchInfo
+			if err := dynamodbattribute.UnmarshalMap(plist.M, &pinfo); err != nil {
+				log.Printf("failed to parse PatchList %v\n", err)
+				sentry.CaptureException(err)
+			} else {
+				ext.PatchList = pinfo
+			}
+		}
+
+		AllExtensionsMap.Store(id, ext)
+
 	}
 }
 

extension/extension.go

@@ -7,16 +7,24 @@ import (
 	"sync"
 )
 
+type PatchInfo struct {
+	Hashdiff string `json:"hashdiff"`
+	Namediff string `json:"namediff"`
+	Sizediff int    `json:"sizediff"`
+}
+
 // Extension represents an extension which is both used in update checks
 // and responses.
 type Extension struct {
 	ID          string
+	FP          string
 	Version     string
 	SHA256      string
 	Title       string
 	URL         string
 	Blacklisted bool
 	Status      string
+	PatchList   map[string]*PatchInfo
 }
 
 // Extensions is type for a slice of Extension.
@@ -83,6 +91,9 @@ func (updateRequest *UpdateRequest) FilterForUpdates(allExtensionsMap *Extension
 				if status == 0 {
 					foundExtension.Status = "noupdate"
 				}
+
+				foundExtension.FP = extensionBeingChecked.FP
+
 				filteredExtensions = append(filteredExtensions, foundExtension)
 			}
 		}

extension/json.go

@@ -9,15 +9,20 @@ import (
 // MarshalJSON encodes the extension list into response JSON
 func (updateResponse *UpdateResponse) MarshalJSON() ([]byte, error) {
 	type URL struct {
-		Codebase string `json:"codebase"`
+		Codebase     string `json:"codebase,omitempty"`
+		CodebaseDiff string `json:"codebasediff,omitempty"`
 	}
 	type URLs struct {
 		URLs []URL `json:"url"`
 	}
 	type Package struct {
-		Name     string `json:"name"`
-		SHA256   string `json:"hash_sha256"`
-		Required bool   `json:"required"`
+		Name       string `json:"name"`
+		NameDiff   string `json:"namediff,omitempty"`
+		SizeDiff   int    `json:"sizediff,omitempty"`
+		FP         string `json:"fp"`
+		SHA256     string `json:"hash_sha256"`
+		DiffSHA256 string `json:"hashdiff_sha256,omitempty"`
+		Required   bool   `json:"required"`
 	}
 	type Packages struct {
 		Package []Package `json:"package"`
@@ -50,9 +55,11 @@ func (updateResponse *UpdateResponse) MarshalJSON() ([]byte, error) {
 	response.Server = "prod"
 	for _, extension := range *updateResponse {
 		app := App{AppID: extension.ID, Status: "ok"}
+		patchInfo, pInfoFound := extension.PatchList[extension.FP]
 		app.UpdateCheck = UpdateCheck{Status: GetUpdateStatus(extension)}
 		extensionName := "extension_" + strings.Replace(extension.Version, ".", "_", -1) + ".crx"
 		url := "https://" + GetS3ExtensionBucketHost(extension.ID) + "/release/" + extension.ID + "/" + extensionName
+		diffUrl := "https://" + GetS3ExtensionBucketHost(extension.ID) + "/release/" + extension.ID + "/patches/" + extension.SHA256 + "/"
 		if app.UpdateCheck.Status == "ok" {
 			if app.UpdateCheck.URLs == nil {
 				app.UpdateCheck.URLs = &URLs{
@@ -62,15 +69,27 @@ func (updateResponse *UpdateResponse) MarshalJSON() ([]byte, error) {
 			app.UpdateCheck.URLs.URLs = append(app.UpdateCheck.URLs.URLs, URL{
 				Codebase: url,
 			})
+
 			app.UpdateCheck.Manifest = &Manifest{
 				Version: extension.Version,
 			}
 
 			pkg := Package{
 				Name:     extensionName,
 				SHA256:   extension.SHA256,
+				FP:       extension.SHA256,
 				Required: true,
 			}
+
+			if pInfoFound {
+				app.UpdateCheck.URLs.URLs = append(app.UpdateCheck.URLs.URLs, URL{
+					CodebaseDiff: diffUrl,
+				})
+				pkg.NameDiff = patchInfo.Namediff
+				pkg.DiffSHA256 = patchInfo.Hashdiff
+				pkg.SizeDiff = patchInfo.Sizediff
+			}
+
 			app.UpdateCheck.Manifest.Packages.Package = append(app.UpdateCheck.Manifest.Packages.Package, pkg)
 		}
 
@@ -129,9 +148,17 @@ func (updateResponse *WebStoreUpdateResponse) MarshalJSON() ([]byte, error) {
 
 // UnmarshalJSON decodes the update server request JSON data for a list of extensions
 func (updateRequest *UpdateRequest) UnmarshalJSON(b []byte) error {
+	type Package struct {
+		FP string `json:"fp"`
+	}
+	type Packages struct {
+		Package []Package `json:"package"`
+	}
 	type App struct {
-		AppID   string `json:"appid"`
-		Version string `json:"version"`
+		AppID    string   `json:"appid"`
+		FP       string   `json:"fp"`
+		Version  string   `json:"version"`
+		Packages Packages `json:"packages"`
 	}
 	type Request struct {
 		OS       string `json:"@os"`
@@ -151,8 +178,16 @@ func (updateRequest *UpdateRequest) UnmarshalJSON(b []byte) error {
 
 	*updateRequest = UpdateRequest{}
 	for _, app := range request.Request.App {
+		fp := app.FP
+		// spec discrepancy: FP might be set within a "package" object (v3) instead of the "app" object (v3.1)
+		// https://github.com/google/omaha/blob/main/doc/ServerProtocolV3.md#package-request
+		// https://chromium.googlesource.com/chromium/src.git/+/master/docs/updater/protocol_3_1.md#update-checks-body-update-check-request-objects-update-check-request-3
+		if fp == "" && len(app.Packages.Package) > 0 {
+			fp = app.Packages.Package[0].FP
+		}
 		*updateRequest = append(*updateRequest, Extension{
 			ID:      app.AppID,
+			FP:      fp,
 			Version: app.Version,
 		})
 	}

extension/json_test.go

@@ -25,7 +25,7 @@ func TestUpdateResponseMarshalJSON(t *testing.T) {
 	updateResponse = []Extension{darkThemeExtension}
 	jsonData, err = json.Marshal(&updateResponse)
 	assert.Nil(t, err)
-	expectedOutput = `{"response":{"protocol":"3.1","server":"prod","app":[{"appid":"bfdgpgibhagkpdlnjonhkabjoijopoge","status":"ok","updatecheck":{"status":"ok","urls":{"url":[{"codebase":"https://` + GetS3ExtensionBucketHost(darkThemeExtension.ID) + `/release/bfdgpgibhagkpdlnjonhkabjoijopoge/extension_1_0_0.crx"}]},"manifest":{"version":"1.0.0","packages":{"package":[{"name":"extension_1_0_0.crx","hash_sha256":"ae517d6273a4fc126961cb026e02946db4f9dbb58e3d9bc29f5e1270e3ce9834","required":true}]}}}}]}}`
+	expectedOutput = `{"response":{"protocol":"3.1","server":"prod","app":[{"appid":"bfdgpgibhagkpdlnjonhkabjoijopoge","status":"ok","updatecheck":{"status":"ok","urls":{"url":[{"codebase":"https://` + GetS3ExtensionBucketHost(darkThemeExtension.ID) + `/release/bfdgpgibhagkpdlnjonhkabjoijopoge/extension_1_0_0.crx"}]},"manifest":{"version":"1.0.0","packages":{"package":[{"name":"extension_1_0_0.crx","fp":"ae517d6273a4fc126961cb026e02946db4f9dbb58e3d9bc29f5e1270e3ce9834","hash_sha256":"ae517d6273a4fc126961cb026e02946db4f9dbb58e3d9bc29f5e1270e3ce9834","required":true}]}}}}]}}`
 	assert.Equal(t, expectedOutput, string(jsonData))
 
 	// Multiple extensions returns a multiple extension JSON update
@@ -36,7 +36,7 @@ func TestUpdateResponseMarshalJSON(t *testing.T) {
 	updateResponse = []Extension{lightThemeExtension, darkThemeExtension}
 	jsonData, err = json.Marshal(&updateResponse)
 	assert.Nil(t, err)
-	expectedOutput = `{"response":{"protocol":"3.1","server":"prod","app":[{"appid":"ldimlcelhnjgpjjemdjokpgeeikdinbm","status":"ok","updatecheck":{"status":"ok","urls":{"url":[{"codebase":"https://` + GetS3ExtensionBucketHost(lightThemeExtension.ID) + `/release/ldimlcelhnjgpjjemdjokpgeeikdinbm/extension_1_0_0.crx"}]},"manifest":{"version":"1.0.0","packages":{"package":[{"name":"extension_1_0_0.crx","hash_sha256":"1c714fadd4208c63f74b707e4c12b81b3ad0153c37de1348fa810dd47cfc5618","required":true}]}}}},{"appid":"bfdgpgibhagkpdlnjonhkabjoijopoge","status":"ok","updatecheck":{"status":"ok","urls":{"url":[{"codebase":"https://` + GetS3ExtensionBucketHost(darkThemeExtension.ID) + `/release/bfdgpgibhagkpdlnjonhkabjoijopoge/extension_1_0_0.crx"}]},"manifest":{"version":"1.0.0","packages":{"package":[{"name":"extension_1_0_0.crx","hash_sha256":"ae517d6273a4fc126961cb026e02946db4f9dbb58e3d9bc29f5e1270e3ce9834","required":true}]}}}}]}}`
+	expectedOutput = `{"response":{"protocol":"3.1","server":"prod","app":[{"appid":"ldimlcelhnjgpjjemdjokpgeeikdinbm","status":"ok","updatecheck":{"status":"ok","urls":{"url":[{"codebase":"https://` + GetS3ExtensionBucketHost(lightThemeExtension.ID) + `/release/ldimlcelhnjgpjjemdjokpgeeikdinbm/extension_1_0_0.crx"}]},"manifest":{"version":"1.0.0","packages":{"package":[{"name":"extension_1_0_0.crx","fp":"1c714fadd4208c63f74b707e4c12b81b3ad0153c37de1348fa810dd47cfc5618","hash_sha256":"1c714fadd4208c63f74b707e4c12b81b3ad0153c37de1348fa810dd47cfc5618","required":true}]}}}},{"appid":"bfdgpgibhagkpdlnjonhkabjoijopoge","status":"ok","updatecheck":{"status":"ok","urls":{"url":[{"codebase":"https://` + GetS3ExtensionBucketHost(darkThemeExtension.ID) + `/release/bfdgpgibhagkpdlnjonhkabjoijopoge/extension_1_0_0.crx"}]},"manifest":{"version":"1.0.0","packages":{"package":[{"name":"extension_1_0_0.crx","fp":"ae517d6273a4fc126961cb026e02946db4f9dbb58e3d9bc29f5e1270e3ce9834","hash_sha256":"ae517d6273a4fc126961cb026e02946db4f9dbb58e3d9bc29f5e1270e3ce9834","required":true}]}}}}]}}`
 	assert.Equal(t, expectedOutput, string(jsonData))
 }
 

go.mod

@@ -3,13 +3,13 @@ module github.com/brave/go-update
 go 1.21
 
 require (
-	github.com/aws/aws-sdk-go v1.45.2
+	github.com/aws/aws-sdk-go v1.52.1
 	github.com/brave-intl/bat-go v0.1.0
-	github.com/getsentry/sentry-go v0.23.0
+	github.com/getsentry/sentry-go v0.27.0
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/pressly/lg v1.1.1
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 )
 
 require (
@@ -28,7 +28,7 @@ require (
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.0 // indirect
 	golang.org/x/sys v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )