diff --git a/cmd/cleanPkgdb.go b/cmd/cleanPkgdb.go index 288e3968..029dfe45 100644 --- a/cmd/cleanPkgdb.go +++ b/cmd/cleanPkgdb.go @@ -18,6 +18,7 @@ import ( "strings" "github.com/metrumresearchgroup/pkgr/configlib" + "github.com/metrumresearchgroup/pkgr/cran" "github.com/metrumresearchgroup/pkgr/rcmd" log "github.com/sirupsen/logrus" @@ -84,7 +85,7 @@ func removePackageDatabases(pkgdbsToClear []string, cfg configlib.PkgrConfig) er rs := rcmd.NewRSettings(cfg.RPath) - pkgNexus, _, _ := planInstall(rs.Version, false) + pkgNexus, _, _ := planInstall(rs.Version, cran.DefaultType(), false) repoDatabases := pkgNexus.Db for _, dbToClear := range pkgdbsToClear { diff --git a/cmd/download.go b/cmd/download.go new file mode 100644 index 00000000..302312df --- /dev/null +++ b/cmd/download.go @@ -0,0 +1,95 @@ +// Copyright © 2019 Metrum Research Group +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/metrumresearchgroup/pkgr/configlib" + "github.com/metrumresearchgroup/pkgr/cran" + "github.com/metrumresearchgroup/pkgr/rcmd" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// installCmd represents the R CMD install command +var downloadCmd = &cobra.Command{ + Use: "download-src", + Short: "download all packages", + Long: ` + download src version of packages in the plan into a single folder. + download works by converting the package type to source, and downloading + the src version from the appropriate repo. It does not do version mismatch + reconciliation or other more sophisticated analysis to confirm that + the src version and binary version are consistent for platforms that + support binary downloads + + This will download packages to the folder 'src' in the cache folder +`, + RunE: rDownload, +} + +func init() { + downloadCmd.PersistentFlags().String("dir", "", "directory to download") + viper.BindPFlag("dir", downloadCmd.PersistentFlags().Lookup("dir")) + RootCmd.AddCommand(downloadCmd) +} + +func rDownload(cmd *cobra.Command, args []string) error { + dir := viper.GetString("dir") + if dir != "" { + cfg.Cache = dir + } + // set Type to source for everything to make sure we get source tarballs + // this will brak the abstraction if src and binary version are different for the platforms + if cfg.Customizations.Packages == nil { + cfg.Customizations.Packages = make(map[string]configlib.PkgConfig) + } + for _, p := range cfg.Packages { + //lets set every package as having a customization of type source + pkg, _ := cfg.Customizations.Packages[p] + pkg.Type = "source" + cfg.Customizations.Packages[p] = pkg + } + // Initialize log and start time. + rSettings := rcmd.NewRSettings(cfg.RPath) + rVersion := rcmd.GetRVersion(&rSettings) + log.Infoln("R Version " + rVersion.ToFullString()) + // most people should know what platform they are on + log.Debugln("OS Platform " + rSettings.Platform) + + // Get master object containing the packages available in each repository (pkgNexus), + // as well as a master install plan to guide our process. + _, installPlan, _ := planInstall(rVersion, cran.Source, true) + for _, pdl := range installPlan.PackageDownloads { + pkg, repo := pdl.PkgAndRepoNames() + fmt.Println(pkg, repo, pdl.Package.Version, pdl.Config.Type) + } + // Retrieve a cache to store any packages we need to download for the install. + packageCache := rcmd.NewPackageCache(userCache(cfg.Cache), false) + dlPath := filepath.Join(packageCache.BaseDir, "src") + err := fs.MkdirAll(dlPath, 0777) + if err != nil { + log.Fatal("error creating download cache dir at: ", dlPath) + } + // //Create a pkgMap object, which helps us with parallel downloads (?) + _, err = cran.DownloadPackages(fs, installPlan.PackageDownloads, packageCache.BaseDir, rVersion, false) + if err != nil { + log.Fatalf("error downloading packages: %s", err) + } + return nil +} diff --git a/cmd/inspect.go b/cmd/inspect.go index f1151705..e307cebd 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -17,6 +17,7 @@ package cmd import ( "fmt" + "github.com/metrumresearchgroup/pkgr/cran" "github.com/metrumresearchgroup/pkgr/logger" "github.com/metrumresearchgroup/pkgr/gpsr" @@ -68,7 +69,7 @@ func inspect(cmd *cobra.Command, args []string) error { rs := rcmd.NewRSettings(cfg.RPath) rVersion := rcmd.GetRVersion(&rs) - _, ip, _ := planInstall(rVersion, true) + _, ip, _ := planInstall(rVersion, cran.DefaultType(), true) if showDeps { var allDeps map[string][]string keepDeps := make(map[string][]string) diff --git a/cmd/install.go b/cmd/install.go index 6d5aa27f..d90d5bd8 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -15,11 +15,12 @@ package cmd import ( - "github.com/metrumresearchgroup/pkgr/pacman" - "github.com/metrumresearchgroup/pkgr/rollback" "path/filepath" "time" + "github.com/metrumresearchgroup/pkgr/pacman" + "github.com/metrumresearchgroup/pkgr/rollback" + "github.com/spf13/viper" "github.com/metrumresearchgroup/pkgr/configlib" @@ -57,7 +58,7 @@ func rInstall(cmd *cobra.Command, args []string) error { // Get master object containing the packages available in each repository (pkgNexus), // as well as a master install plan to guide our process. - _, installPlan, rollbackPlan := planInstall(rVersion, true) + _, installPlan, rollbackPlan := planInstall(rVersion, cran.DefaultType(), true) if viper.GetBool("update") { log.Info("update argument passed. staging packages for update...") @@ -71,7 +72,7 @@ func rInstall(cmd *cobra.Command, args []string) error { packageCache := rcmd.NewPackageCache(userCache(cfg.Cache), false) //Create a pkgMap object, which helps us with parallel downloads (?) - pkgMap, err := cran.DownloadPackages(fs, installPlan.PackageDownloads, packageCache.BaseDir, rVersion) + pkgMap, err := cran.DownloadPackages(fs, installPlan.PackageDownloads, packageCache.BaseDir, rVersion, true) if err != nil { log.Fatalf("error downloading packages: %s", err) } @@ -99,9 +100,7 @@ func rInstall(cmd *cobra.Command, args []string) error { if err != nil { errRollback := rollback.RollbackPackageEnvironment(fs, rollbackPlan) if errRollback != nil { - log.WithFields(log.Fields{ - - }).Error("failed to reset package environment after bad installation. Your package Library will be in a corrupt state. It is recommended you delete your Library and reinstall all packages.") + log.WithFields(log.Fields{}).Error("failed to reset package environment after bad installation. Your package Library will be in a corrupt state. It is recommended you delete your Library and reinstall all packages.") } } else { pacman.CleanUpdateBackups(fs, rollbackPlan.UpdateRollbacks) @@ -116,7 +115,6 @@ func rInstall(cmd *cobra.Command, args []string) error { return nil } - func initInstallLog() { //Init install-specific log, if one has been set. This overwrites the default log. if cfg.Logging.Install != "" { diff --git a/cmd/plan.go b/cmd/plan.go index de0e7774..8ade79c0 100644 --- a/cmd/plan.go +++ b/cmd/plan.go @@ -61,7 +61,7 @@ func plan(cmd *cobra.Command, args []string) error { rVersion := rcmd.GetRVersion(&rs) log.Infoln("R Version " + rVersion.ToFullString()) log.Debugln("OS Platform " + rs.Platform) - _, ip, _ := planInstall(rVersion, true) + _, ip, _ := planInstall(rVersion, cran.DefaultType(), true) if viper.GetBool("show-deps") { for pkg, deps := range ip.DepDb { fmt.Println("----------- ", pkg, " ------------") @@ -71,7 +71,7 @@ func plan(cmd *cobra.Command, args []string) error { return nil } -func planInstall(rv cran.RVersion, exitOnMissing bool) (*cran.PkgNexus, gpsr.InstallPlan, rollback.RollbackPlan) { +func planInstall(rv cran.RVersion, st cran.SourceType, exitOnMissing bool) (*cran.PkgNexus, gpsr.InstallPlan, rollback.RollbackPlan) { startTime := time.Now() installedPackages := pacman.GetPriorInstalledPackages(fs, cfg.Library) @@ -92,7 +92,6 @@ func planInstall(rv cran.RVersion, exitOnMissing bool) (*cran.PkgNexus, gpsr.Ins repos = append(repos, cran.RepoURL{Name: nm, URL: url}) } } - st := cran.DefaultType() cic := cran.NewInstallConfig() for rn, val := range cfg.Customizations.Repos { if strings.EqualFold(val.Type, "binary") { diff --git a/cran/download-package.go b/cran/download-package.go index 4535495e..e495c688 100644 --- a/cran/download-package.go +++ b/cran/download-package.go @@ -49,20 +49,22 @@ func getRepos(ds []PkgDl) map[string]RepoURL { } // DownloadPackages downloads a set of packages concurrently -func DownloadPackages(fs afero.Fs, ds []PkgDl, baseDir string, rv RVersion) (*PkgMap, error) { +func DownloadPackages(fs afero.Fs, ds []PkgDl, baseDir string, rv RVersion, hash bool) (*PkgMap, error) { startTime := time.Now() result := NewPkgMap() sem := make(chan struct{}, 10) wg := sync.WaitGroup{} rpm := getRepos(ds) - for _, r := range rpm { - urlHash := RepoURLHash(r) - for _, pt := range []string{"src", filepath.Join("binary", rv.ToString())} { - pkgdir := filepath.Join(baseDir, urlHash, pt) - err := fs.MkdirAll(pkgdir, 0777) - if err != nil { - log.WithField("dir", pkgdir).WithField("error", err.Error()).Fatal("error creating package directory ") - return result, err + if hash { + for _, r := range rpm { + urlHash := RepoURLHash(r) + for _, pt := range []string{"src", filepath.Join("binary", rv.ToString())} { + pkgdir := filepath.Join(baseDir, urlHash, pt) + err := fs.MkdirAll(pkgdir, 0777) + if err != nil { + log.WithField("dir", pkgdir).WithField("error", err.Error()).Fatal("error creating package directory ") + return result, err + } } } } @@ -91,7 +93,17 @@ func DownloadPackages(fs afero.Fs, ds []PkgDl, baseDir string, rv RVersion) (*Pk // but would want to do this outside the goroutine that downloads\ // the package so didn't get invoked multiple times urlHash := RepoURLHash(d.Config.Repo) - pkgdir := filepath.Join(baseDir, urlHash, pkgType) + + // the idea here is hashing should be used when maintaining the package versions for purposes + // of longer term caching, however we also now have the case of just wanting to download + // all files to a single directory for uses such as easier archival or + // for preparing for R CMD checks + var pkgdir string + if hash { + pkgdir = filepath.Join(baseDir, urlHash, pkgType) + } else { + pkgdir = filepath.Join(baseDir, pkgType) + } var pkgFile string if d.Config.Type == Binary { pkgFile = filepath.Join(pkgdir, rv.ToString(), binaryName(d.Package.Package, d.Package.Version)) @@ -188,6 +200,7 @@ func DownloadPackage(fs afero.Fs, d PkgDl, dest string, rv RVersion) (Download, "path": pkgdl, }).Fatal("missing package") } + defer from.Close() } file, err := fs.Create(dest) diff --git a/go.mod b/go.mod index 69dd6c1c..3bfcf125 100644 --- a/go.mod +++ b/go.mod @@ -21,3 +21,5 @@ require ( pault.ag/go/debian v0.0.0-20180722221659-90aeb542bd40 pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a // indirect ) + +go 1.13 diff --git a/pkgr_ws.code-workspace b/pkgr_ws.code-workspace deleted file mode 100644 index 362d7c25..00000000 --- a/pkgr_ws.code-workspace +++ /dev/null @@ -1,7 +0,0 @@ -{ - "folders": [ - { - "path": "." - } - ] -} \ No newline at end of file