Skip to content

Commit

Permalink
Merge pull request #1162 from jpts/fix-private-armory-github-auth
Browse files Browse the repository at this point in the history 
Add armory support for private Github APIs
  • Loading branch information
@moloch--
moloch-- authored Mar 26, 2023
2 parents 6aeab60 + c40f59b commit 796a0db
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 123 deletions.
2 changes: 1 addition & 1 deletion client/command/armory/armory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -484,7 +484,7 @@ func fetchPackageSignature(wg *sync.WaitGroup, armoryConfig *assets.ArmoryConfig

var sig *minisign.Signature
if pkgParser, ok := pkgParsers[repoURL.Hostname()]; ok {
sig, _, err = pkgParser(armoryPkg, true, clientConfig)
sig, _, err = pkgParser(armoryConfig, armoryPkg, true, clientConfig)
} else {
sig, _, err = DefaultArmoryPkgParser(armoryConfig, armoryPkg, true, clientConfig)
}
Expand Down
4 changes: 2 additions & 2 deletions client/command/armory/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ func installAliasPackageByName(name string, clientConfig ArmoryHTTPConfig, con *
var sig *minisign.Signature
var tarGz []byte
if pkgParser, ok := pkgParsers[repoURL.Hostname()]; ok {
sig, tarGz, err = pkgParser(&entry.Pkg, false, clientConfig)
sig, tarGz, err = pkgParser(entry.ArmoryConfig, &entry.Pkg, false, clientConfig)
} else {
sig, tarGz, err = DefaultArmoryPkgParser(entry.ArmoryConfig, &entry.Pkg, false, clientConfig)
}
Expand Down Expand Up @@ -261,7 +261,7 @@ func installExtensionPackageByName(name string, clientConfig ArmoryHTTPConfig, c
var sig *minisign.Signature
var tarGz []byte
if pkgParser, ok := pkgParsers[repoURL.Hostname()]; ok {
sig, tarGz, err = pkgParser(&entry.Pkg, false, clientConfig)
sig, tarGz, err = pkgParser(entry.ArmoryConfig, &entry.Pkg, false, clientConfig)
} else {
sig, tarGz, err = DefaultArmoryPkgParser(entry.ArmoryConfig, &entry.Pkg, false, clientConfig)
}
Expand Down
184 changes: 64 additions & 120 deletions client/command/armory/parsers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ import (
type ArmoryIndexParser func(*assets.ArmoryConfig, ArmoryHTTPConfig) (*ArmoryIndex, error)

// ArmoryPackageParser - Generic interface to fetch armory package manifests
type ArmoryPackageParser func(*ArmoryPackage, bool, ArmoryHTTPConfig) (*minisign.Signature, []byte, error)
type ArmoryPackageParser func(*assets.ArmoryConfig, *ArmoryPackage, bool, ArmoryHTTPConfig) (*minisign.Signature, []byte, error)

var (
indexParsers = map[string]ArmoryIndexParser{
Expand Down Expand Up @@ -79,28 +79,7 @@ func DefaultArmoryIndexParser(armoryConfig *assets.ArmoryConfig, clientConfig Ar
return nil, err
}

client := httpClient(clientConfig)
repoURL, err := url.Parse(armoryConfig.RepoURL)
if err != nil {
return nil, err
}
req := &http.Request{
Method: http.MethodGet,
URL: repoURL,
Header: map[string][]string{},
}
if armoryConfig.Authorization != "" {
req.Header.Set("Authorization", armoryConfig.Authorization)
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
resp, body, err := httpRequest(clientConfig, armoryConfig.RepoURL, armoryConfig, http.Header{})
if resp.StatusCode != http.StatusOK {
return nil, errors.New("api returned non-200 status code")
}
Expand All @@ -125,9 +104,8 @@ func DefaultArmoryIndexParser(armoryConfig *assets.ArmoryConfig, clientConfig Ar
return nil, errors.New("index has invalid signature")
}

armoryIndex := &ArmoryIndex{ArmoryConfig: armoryConfig}
if err != nil {
return nil, err
armoryIndex := &ArmoryIndex{
ArmoryConfig: armoryConfig,
}
err = json.Unmarshal(armoryIndexData, armoryIndex)
if err != nil {
Expand All @@ -144,25 +122,7 @@ func DefaultArmoryPkgParser(armoryConfig *assets.ArmoryConfig, armoryPkg *Armory
return nil, nil, err
}

client := httpClient(clientConfig)
repoURL, err := url.Parse(armoryConfig.RepoURL)
if err != nil {
return nil, nil, err
}
req := &http.Request{
Method: http.MethodGet,
URL: repoURL,
Header: map[string][]string{},
}
if armoryConfig.Authorization != "" {
req.Header.Set("Authorization", armoryConfig.Authorization)
}
resp, err := client.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
resp, body, err := httpRequest(clientConfig, armoryConfig.RepoURL, armoryConfig, http.Header{})
if err != nil {
return nil, nil, err
}
Expand All @@ -187,18 +147,10 @@ func DefaultArmoryPkgParser(armoryConfig *assets.ArmoryConfig, armoryPkg *Armory
if tarGzURL.Scheme != "https" && tarGzURL.Scheme != "http" {
return nil, nil, errors.New("invalid url scheme")
}
resp, err := client.Get(tarGzURL.String())
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
tarGz, err = ioutil.ReadAll(resp.Body)
tarGz, err = downloadRequest(clientConfig, tarGzURL.String(), armoryConfig)
if err != nil {
return nil, nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, nil, errors.New("api returned non-200 status code")
}
}
return sig, tarGz, nil
}
Expand Down Expand Up @@ -239,21 +191,15 @@ func GithubAPIArmoryIndexParser(armoryConfig *assets.ArmoryConfig, clientConfig
return nil, err
}

client := httpClient(clientConfig)
resp, err := client.Get(armoryConfig.RepoURL)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
resp, body, err := httpRequest(clientConfig, armoryConfig.RepoURL, armoryConfig, http.Header{})
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusForbidden {
return nil, errors.New("you hit the github api rate limit (60 req/hr), try later")
}
return nil, errors.New("api returned non-200 status code")
return nil, fmt.Errorf("api returned non-200 status code: %d", resp.StatusCode)
}

releases := []GithubRelease{}
Expand All @@ -270,23 +216,13 @@ func GithubAPIArmoryIndexParser(armoryConfig *assets.ArmoryConfig, clientConfig
var sigData []byte
for _, asset := range release.Assets {
if asset.Name == armoryIndexFileName {
resp, err := client.Get(asset.BrowserDownloadURL)
if err != nil {
return nil, err
}
defer resp.Body.Close()
armoryIndexData, err = ioutil.ReadAll(resp.Body)
armoryIndexData, err = downloadRequest(clientConfig, asset.URL, armoryConfig)
if err != nil {
return nil, err
}
}
if asset.Name == armoryIndexSigFileName {
resp, err := client.Get(asset.BrowserDownloadURL)
if err != nil {
return nil, err
}
defer resp.Body.Close()
sigData, err = ioutil.ReadAll(resp.Body)
sigData, err = downloadRequest(clientConfig, asset.URL, armoryConfig)
if err != nil {
return nil, err
}
Expand All @@ -299,7 +235,9 @@ func GithubAPIArmoryIndexParser(armoryConfig *assets.ArmoryConfig, clientConfig
return nil, errors.New("invalid signature")
}

armoryIndex := &ArmoryIndex{}
armoryIndex := &ArmoryIndex{
ArmoryConfig: armoryConfig,
}
err = json.Unmarshal(armoryIndexData, armoryIndex)
if err != nil {
return nil, err
Expand All @@ -308,28 +246,22 @@ func GithubAPIArmoryIndexParser(armoryConfig *assets.ArmoryConfig, clientConfig
}

// GithubAPIArmoryPackageParser - Retrieve the minisig and tar.gz for an armory package from a GitHub release
func GithubAPIArmoryPackageParser(armoryPkg *ArmoryPackage, sigOnly bool, clientConfig ArmoryHTTPConfig) (*minisign.Signature, []byte, error) {
func GithubAPIArmoryPackageParser(armoryConfig *assets.ArmoryConfig, armoryPkg *ArmoryPackage, sigOnly bool, clientConfig ArmoryHTTPConfig) (*minisign.Signature, []byte, error) {
var publicKey minisign.PublicKey
err := publicKey.UnmarshalText([]byte(armoryPkg.PublicKey))
if err != nil {
return nil, nil, err
}

client := httpClient(clientConfig)
resp, err := client.Get(armoryPkg.RepoURL)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
resp, body, err := httpRequest(clientConfig, armoryPkg.RepoURL, armoryConfig, http.Header{})
if err != nil {
return nil, nil, err
}
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusForbidden {
return nil, nil, errors.New("you hit the github api rate limit (60 req/hr), try later")
}
return nil, nil, errors.New("api returned non-200 status code")
return nil, nil, fmt.Errorf("api returned non-200 status code: %d", resp.StatusCode)
}

releases := []GithubRelease{}
Expand All @@ -343,14 +275,7 @@ func GithubAPIArmoryPackageParser(armoryPkg *ArmoryPackage, sigOnly bool, client
var tarGz []byte
for _, asset := range release.Assets {
if asset.Name == fmt.Sprintf("%s.minisig", armoryPkg.CommandName) {
var resp *http.Response
resp, err = client.Get(asset.BrowserDownloadURL)
if err != nil {
break
}
defer resp.Body.Close()
var body []byte
body, err = ioutil.ReadAll(resp.Body)
body, err := downloadRequest(clientConfig, asset.URL, armoryConfig)
if err != nil {
break
}
Expand All @@ -360,13 +285,7 @@ func GithubAPIArmoryPackageParser(armoryPkg *ArmoryPackage, sigOnly bool, client
}
}
if asset.Name == fmt.Sprintf("%s.tar.gz", armoryPkg.CommandName) && !sigOnly {
var resp *http.Response
resp, err = client.Get(asset.BrowserDownloadURL)
if err != nil {
break
}
defer resp.Body.Close()
tarGz, err = ioutil.ReadAll(resp.Body)
tarGz, err = downloadRequest(clientConfig, asset.URL, armoryConfig)
if err != nil {
break
}
Expand All @@ -380,31 +299,27 @@ func GithubAPIArmoryPackageParser(armoryPkg *ArmoryPackage, sigOnly bool, client
//

// GithubArmoryPackageParser - Uses github.com instead of api.github.com to download packages
func GithubArmoryPackageParser(armoryPkg *ArmoryPackage, sigOnly bool, clientConfig ArmoryHTTPConfig) (*minisign.Signature, []byte, error) {
func GithubArmoryPackageParser(_ *assets.ArmoryConfig, armoryPkg *ArmoryPackage, sigOnly bool, clientConfig ArmoryHTTPConfig) (*minisign.Signature, []byte, error) {
latestTag, err := githubLatestTagParser(armoryPkg, clientConfig)
if err != nil {
return nil, nil, err
}
client := httpClient(clientConfig)

sigURL, err := url.Parse(armoryPkg.RepoURL)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse armory pkg url '%s': %s", armoryPkg.RepoURL, err)
}
sigURL.Path = path.Join(sigURL.Path, "releases", "download", latestTag, fmt.Sprintf("%s.minisig", armoryPkg.CommandName))
sigResp, err := client.Get(sigURL.String())
if err != nil {
return nil, nil, err
}
defer sigResp.Body.Close()
if sigResp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("failed to get signature for armory pkg '%s': %s", armoryPkg.RepoURL, sigResp.Status)

// Setup dummy auth here as the non-api endpoints don't support the Authorization header
noAuth := &assets.ArmoryConfig{
Authorization: "",
}
var body []byte
body, err = ioutil.ReadAll(sigResp.Body)
body, err := downloadRequest(clientConfig, sigURL.String(), noAuth)
if err != nil {
return nil, nil, fmt.Errorf("failed to read resp body '%s': %s", armoryPkg.RepoURL, err)
return nil, nil, err
}

sig, err := parsePkgMinsig(body)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse pkg sig '%s': %s", armoryPkg.RepoURL, err)
Expand All @@ -417,18 +332,10 @@ func GithubArmoryPackageParser(armoryPkg *ArmoryPackage, sigOnly bool, clientCon
return nil, nil, fmt.Errorf("failed to parse armory pkg url '%s': %s", armoryPkg.RepoURL, err)
}
tarGzURL.Path = path.Join(tarGzURL.Path, "releases", "download", latestTag, fmt.Sprintf("%s.tar.gz", armoryPkg.CommandName))
tarGzResp, err := client.Get(tarGzURL.String())
tarGz, err = downloadRequest(clientConfig, tarGzURL.String(), noAuth)
if err != nil {
return nil, nil, err
}
defer tarGzResp.Body.Close()
if tarGzResp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("failed to get tar.gz for armory pkg '%s': %s", armoryPkg.RepoURL, tarGzResp.Status)
}
tarGz, err = ioutil.ReadAll(tarGzResp.Body)
if err != nil {
return nil, nil, fmt.Errorf("failed to read tar.gz body '%s': %s", armoryPkg.RepoURL, err)
}
}

return sig, tarGz, nil
Expand Down Expand Up @@ -498,3 +405,40 @@ func httpClient(config ArmoryHTTPConfig) *http.Client {
},
}
}

func httpRequest(clientConfig ArmoryHTTPConfig, reqURL string, armoryConfig *assets.ArmoryConfig, extraHeaders http.Header) (*http.Response, []byte, error) {
client := httpClient(clientConfig)
req, err := http.NewRequest(http.MethodGet, reqURL, http.NoBody)
if err != nil {
return nil, nil, err
}

if len(extraHeaders) > 0 {
for key := range extraHeaders {
req.Header.Add(key, strings.Join(extraHeaders[key], ","))
}
}
if armoryConfig.Authorization != "" {
req.Header.Set("Authorization", armoryConfig.Authorization)
}
resp, err := client.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()

body, err := ioutil.ReadAll(resp.Body)
return resp, body, err
}

func downloadRequest(clientConfig ArmoryHTTPConfig, reqURL string, armoryConfig *assets.ArmoryConfig) ([]byte, error) {
downloadHdr := http.Header{
"Accept": {"application/octet-stream"},
}
resp, body, err := httpRequest(clientConfig, reqURL, armoryConfig, downloadHdr)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusFound {
return nil, fmt.Errorf("Error downloading asset: http %d", resp.StatusCode)
}

return body, err
}

0 comments on commit 796a0db

Please sign in to comment.