bug/pypi bq api change (#215)

* Use a JSON encoder instead of sprintf-ing JSON

* More useful error messages

* "downloads" field is now a structure

internal/backends/python/gen_pypi_map/download_stats.go

@@ -62,7 +62,7 @@ func GetPypiStats(projectID string) (map[string]int, error) {
  ctx := context.Background()
  client, err := bigquery.NewClient(ctx, projectID)
  if err != nil {
- return map[string]int{}, fmt.Errorf("Failed to connect to bigquery: %v", err)
+ return map[string]int{}, fmt.Errorf("Failed to connect to bigquery [%s]: %v", projectID, err)
  }
 
  q := client.Query(
@@ -88,7 +88,7 @@ func GetPypiStats(projectID string) (map[string]int, error) {
  if err != nil {
  return packages, err
  }
- packages[info.Name] = info.Downloads
+ packages[info.Name] = info.Downloads.LastMonth
  }
 
  return packages, nil

internal/backends/python/gen_pypi_map/test_modules.go

@@ -55,7 +55,7 @@ func TestModules(packages PackageIndex, bigqueryFile string, cacheDir string, pk
  packageInfo, err := ProcessPackage(packageName, cache, cacheDir, distMods, force)
  packageInfo.Name = packageName
  if err != nil {
- fmt.Fprintf(os.Stderr, "Failed to process package: %s\n", err.Error())
+ fmt.Fprintf(os.Stderr, "Failed to process package [%v]: %v\n", packageName, err)
  packageInfo.Error = err.Error()
  }
  resultQueue <- packageInfo

internal/backends/python/gen_pypi_map/types.go

@@ -1,14 +1,23 @@
 package main
 
-import "fmt"
+import (
+ "encoding/json"
+ "fmt"
+)
 
 type PackageCache = map[string]PackageInfo
 
+type DownloadsInfo struct {
+ LastDay int `json:"last_day"`
+ LastWeek int `json:"last_week"`
+ LastMonth int `json:"last_month"`
+}
+
 type PackageInfo struct {
- Name string `json:"name,omitempty"`
- Downloads int `json:"downloads,string,omitempty"`
- Version string `json:"version,omitempty"`
- RequiresDist []string `json:"requires_dist,omitempty"`
+ Name string  `json:"name,omitempty"`
+ Downloads DownloadsInfo `json:"downloads,omitempty"`
+ Version string  `json:"version,omitempty"`
+ RequiresDist []string  `json:"requires_dist,omitempty"`
 
  // Specific to the dist we use to get modules from
  Modules []string `json:"modules,omitempty"`
@@ -31,7 +40,7 @@ type PackageURL struct {
 }
 
 type PackageData struct {
- Info PackageInfo
+ Info PackageInfo `json:"info"`
  Releases map[string][]PackageURL
 }
 
@@ -60,5 +69,16 @@ func (e PypiError) Error() string {
  "Failed to download: " + e.Info,
  "Failed to install: " + e.Info,
  }[e.Class]
- return fmt.Sprintf("{\"type\": %v, \"message\": \"%v\"}", e.Class, message)
+ jsonError := map[string]interface{}{
+ "type": e.Class,
+ "message": message,
+ }
+ if e.WrappedError != nil {
+ jsonError["wrapped"] = e.WrappedError.Error()
+ }
+ encodedError, err := json.Marshal(jsonError)
+ if err != nil {
+ return fmt.Sprintf("{\"error\": \"%v\"}", err)
+ }
+ return string(encodedError)
 }