implement download-src version of packages to directory

cmd/cleanPkgdb.go

@@ -18,6 +18,7 @@ import (
 	"strings"
 
 	"github.com/metrumresearchgroup/pkgr/configlib"
+	"github.com/metrumresearchgroup/pkgr/cran"
 
 	"github.com/metrumresearchgroup/pkgr/rcmd"
 	log "github.com/sirupsen/logrus"
@@ -84,7 +85,7 @@ func removePackageDatabases(pkgdbsToClear []string, cfg configlib.PkgrConfig) er
 
 	rs := rcmd.NewRSettings(cfg.RPath)
 
-	pkgNexus, _, _ := planInstall(rs.Version, false)
+	pkgNexus, _, _ := planInstall(rs.Version, cran.DefaultType(), false)
 	repoDatabases := pkgNexus.Db
 
 	for _, dbToClear := range pkgdbsToClear {

cmd/download.go

@@ -0,0 +1,95 @@
+// Copyright © 2019 Metrum Research Group
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/metrumresearchgroup/pkgr/configlib"
+	"github.com/metrumresearchgroup/pkgr/cran"
+	"github.com/metrumresearchgroup/pkgr/rcmd"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// installCmd represents the R CMD install command
+var downloadCmd = &cobra.Command{
+	Use:   "download-src",
+	Short: "download all packages",
+	Long: `
+	download src version of packages in the plan into a single folder.
+	download works by converting the package type to source, and downloading
+	the src version from the appropriate repo. It does not do version mismatch
+	reconciliation or other more sophisticated analysis to confirm that 
+	the src version and binary version are consistent for platforms that
+	support binary downloads
+
+	This will download packages to the folder 'src' in the cache folder
+`,
+	RunE: rDownload,
+}
+
+func init() {
+	downloadCmd.PersistentFlags().String("dir", "", "directory to download")
+	viper.BindPFlag("dir", downloadCmd.PersistentFlags().Lookup("dir"))
+	RootCmd.AddCommand(downloadCmd)
+}
+
+func rDownload(cmd *cobra.Command, args []string) error {
+	dir := viper.GetString("dir")
+	if dir != "" {
+		cfg.Cache = dir
+	}
+	// set Type to source for everything to make sure we get source tarballs
+	// this will brak the abstraction if src and binary version are different for the platforms
+	if cfg.Customizations.Packages == nil {
+		cfg.Customizations.Packages = make(map[string]configlib.PkgConfig)
+	}
+	for _, p := range cfg.Packages {
+		//lets set every package as having a customization of type source
+		pkg, _ := cfg.Customizations.Packages[p]
+		pkg.Type = "source"
+		cfg.Customizations.Packages[p] = pkg
+	}
+	// Initialize log and start time.
+	rSettings := rcmd.NewRSettings(cfg.RPath)
+	rVersion := rcmd.GetRVersion(&rSettings)
+	log.Infoln("R Version " + rVersion.ToFullString())
+	// most people should know what platform they are on
+	log.Debugln("OS Platform " + rSettings.Platform)
+
+	// Get master object containing the packages available in each repository (pkgNexus),
+	//  as well as a master install plan to guide our process.
+	_, installPlan, _ := planInstall(rVersion, cran.Source, true)
+	for _, pdl := range installPlan.PackageDownloads {
+		pkg, repo := pdl.PkgAndRepoNames()
+		fmt.Println(pkg, repo, pdl.Package.Version, pdl.Config.Type)
+	}
+	// Retrieve a cache to store any packages we need to download for the install.
+	packageCache := rcmd.NewPackageCache(userCache(cfg.Cache), false)
+	dlPath := filepath.Join(packageCache.BaseDir, "src")
+	err := fs.MkdirAll(dlPath, 0777)
+	if err != nil {
+		log.Fatal("error creating download cache dir at: ", dlPath)
+	}
+	// //Create a pkgMap object, which helps us with parallel downloads (?)
+	_, err = cran.DownloadPackages(fs, installPlan.PackageDownloads, packageCache.BaseDir, rVersion, false)
+	if err != nil {
+		log.Fatalf("error downloading packages: %s", err)
+	}
+	return nil
+}

cmd/inspect.go

@@ -17,6 +17,7 @@ package cmd
 import (
 	"fmt"
 
+	"github.com/metrumresearchgroup/pkgr/cran"
 	"github.com/metrumresearchgroup/pkgr/logger"
 
 	"github.com/metrumresearchgroup/pkgr/gpsr"
@@ -68,7 +69,7 @@ func inspect(cmd *cobra.Command, args []string) error {
 
 	rs := rcmd.NewRSettings(cfg.RPath)
 	rVersion := rcmd.GetRVersion(&rs)
-	_, ip, _ := planInstall(rVersion, true)
+	_, ip, _ := planInstall(rVersion, cran.DefaultType(), true)
 	if showDeps {
 		var allDeps map[string][]string
 		keepDeps := make(map[string][]string)

cmd/install.go

@@ -15,11 +15,12 @@
 package cmd
 
 import (
-	"github.com/metrumresearchgroup/pkgr/pacman"
-	"github.com/metrumresearchgroup/pkgr/rollback"
 	"path/filepath"
 	"time"
 
+	"github.com/metrumresearchgroup/pkgr/pacman"
+	"github.com/metrumresearchgroup/pkgr/rollback"
+
 	"github.com/spf13/viper"
 
 	"github.com/metrumresearchgroup/pkgr/configlib"
@@ -57,7 +58,7 @@ func rInstall(cmd *cobra.Command, args []string) error {
 
 	// Get master object containing the packages available in each repository (pkgNexus),
 	//  as well as a master install plan to guide our process.
-	_, installPlan, rollbackPlan := planInstall(rVersion, true)
+	_, installPlan, rollbackPlan := planInstall(rVersion, cran.DefaultType(), true)
 
 	if viper.GetBool("update") {
 		log.Info("update argument passed. staging packages for update...")
@@ -71,7 +72,7 @@ func rInstall(cmd *cobra.Command, args []string) error {
 	packageCache := rcmd.NewPackageCache(userCache(cfg.Cache), false)
 
 	//Create a pkgMap object, which helps us with parallel downloads (?)
-	pkgMap, err := cran.DownloadPackages(fs, installPlan.PackageDownloads, packageCache.BaseDir, rVersion)
+	pkgMap, err := cran.DownloadPackages(fs, installPlan.PackageDownloads, packageCache.BaseDir, rVersion, true)
 	if err != nil {
 		log.Fatalf("error downloading packages: %s", err)
 	}
@@ -99,9 +100,7 @@ func rInstall(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		errRollback := rollback.RollbackPackageEnvironment(fs, rollbackPlan)
 		if errRollback != nil {
-			log.WithFields(log.Fields{
-
-			}).Error("failed to reset package environment after bad installation. Your package Library will be in a corrupt state. It is recommended you delete your Library and reinstall all packages.")
+			log.WithFields(log.Fields{}).Error("failed to reset package environment after bad installation. Your package Library will be in a corrupt state. It is recommended you delete your Library and reinstall all packages.")
 		}
 	} else {
 		pacman.CleanUpdateBackups(fs, rollbackPlan.UpdateRollbacks)
@@ -116,7 +115,6 @@ func rInstall(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-
 func initInstallLog() {
 	//Init install-specific log, if one has been set. This overwrites the default log.
 	if cfg.Logging.Install != "" {

cmd/plan.go

@@ -61,7 +61,7 @@ func plan(cmd *cobra.Command, args []string) error {
 	rVersion := rcmd.GetRVersion(&rs)
 	log.Infoln("R Version " + rVersion.ToFullString())
 	log.Debugln("OS Platform " + rs.Platform)
-	_, ip, _ := planInstall(rVersion, true)
+	_, ip, _ := planInstall(rVersion, cran.DefaultType(), true)
 	if viper.GetBool("show-deps") {
 		for pkg, deps := range ip.DepDb {
 			fmt.Println("-----------  ", pkg, "   ------------")
@@ -71,7 +71,7 @@ func plan(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func planInstall(rv cran.RVersion, exitOnMissing bool) (*cran.PkgNexus, gpsr.InstallPlan, rollback.RollbackPlan) {
+func planInstall(rv cran.RVersion, st cran.SourceType, exitOnMissing bool) (*cran.PkgNexus, gpsr.InstallPlan, rollback.RollbackPlan) {
 	startTime := time.Now()
 
 	installedPackages := pacman.GetPriorInstalledPackages(fs, cfg.Library)
@@ -92,7 +92,6 @@ func planInstall(rv cran.RVersion, exitOnMissing bool) (*cran.PkgNexus, gpsr.Ins
 			repos = append(repos, cran.RepoURL{Name: nm, URL: url})
 		}
 	}
-	st := cran.DefaultType()
 	cic := cran.NewInstallConfig()
 	for rn, val := range cfg.Customizations.Repos {
 		if strings.EqualFold(val.Type, "binary") {

cran/download-package.go

@@ -49,20 +49,22 @@ func getRepos(ds []PkgDl) map[string]RepoURL {
 }
 
 // DownloadPackages downloads a set of packages concurrently
-func DownloadPackages(fs afero.Fs, ds []PkgDl, baseDir string, rv RVersion) (*PkgMap, error) {
+func DownloadPackages(fs afero.Fs, ds []PkgDl, baseDir string, rv RVersion, hash bool) (*PkgMap, error) {
 	startTime := time.Now()
 	result := NewPkgMap()
 	sem := make(chan struct{}, 10)
 	wg := sync.WaitGroup{}
 	rpm := getRepos(ds)
-	for _, r := range rpm {
-		urlHash := RepoURLHash(r)
-		for _, pt := range []string{"src", filepath.Join("binary", rv.ToString())} {
-			pkgdir := filepath.Join(baseDir, urlHash, pt)
-			err := fs.MkdirAll(pkgdir, 0777)
-			if err != nil {
-				log.WithField("dir", pkgdir).WithField("error", err.Error()).Fatal("error creating package directory ")
-				return result, err
+	if hash {
+		for _, r := range rpm {
+			urlHash := RepoURLHash(r)
+			for _, pt := range []string{"src", filepath.Join("binary", rv.ToString())} {
+				pkgdir := filepath.Join(baseDir, urlHash, pt)
+				err := fs.MkdirAll(pkgdir, 0777)
+				if err != nil {
+					log.WithField("dir", pkgdir).WithField("error", err.Error()).Fatal("error creating package directory ")
+					return result, err
+				}
 			}
 		}
 	}
@@ -91,7 +93,17 @@ func DownloadPackages(fs afero.Fs, ds []PkgDl, baseDir string, rv RVersion) (*Pk
 			// but would want to do this outside the goroutine that downloads\
 			// the package so didn't get invoked multiple times
 			urlHash := RepoURLHash(d.Config.Repo)
-			pkgdir := filepath.Join(baseDir, urlHash, pkgType)
+
+			// the idea here is hashing should be used when maintaining the package versions for purposes
+			// of longer term caching, however we also now have the case of just wanting to download
+			// all files to a single directory for uses such as easier archival or
+			// for preparing for R CMD checks
+			var pkgdir string
+			if hash {
+				pkgdir = filepath.Join(baseDir, urlHash, pkgType)
+			} else {
+				pkgdir = filepath.Join(baseDir, pkgType)
+			}
 			var pkgFile string
 			if d.Config.Type == Binary {
 				pkgFile = filepath.Join(pkgdir, rv.ToString(), binaryName(d.Package.Package, d.Package.Version))
@@ -188,6 +200,7 @@ func DownloadPackage(fs afero.Fs, d PkgDl, dest string, rv RVersion) (Download,
 				"path":    pkgdl,
 			}).Fatal("missing package")
 		}
+		defer from.Close()
 	}
 
 	file, err := fs.Create(dest)

go.mod

@@ -21,3 +21,5 @@ require (
 	pault.ag/go/debian v0.0.0-20180722221659-90aeb542bd40
 	pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a // indirect
 )
+
+go 1.13