diff --git a/.github/newt_upgrade/allowed-ignored/expected.txt b/.github/newt_upgrade/allowed-ignored/expected.txt new file mode 100644 index 000000000..683660be0 --- /dev/null +++ b/.github/newt_upgrade/allowed-ignored/expected.txt @@ -0,0 +1,7 @@ +apache-mynewt-core +apache-mynewt-mcumgr +apache-mynewt-nimble +arm-CMSIS_5 +mbedtls +mcuboot +nordic-nrfx diff --git a/.github/newt_upgrade/allowed-ignored/project.yml b/.github/newt_upgrade/allowed-ignored/project.yml new file mode 100644 index 000000000..a19d0874b --- /dev/null +++ b/.github/newt_upgrade/allowed-ignored/project.yml @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +project.repositories: + - apache-mynewt-core + +repository.apache-mynewt-core: + type: github + vers: 0-dev + user: apache + repo: mynewt-core + +repository.tinyusb: + type: github + vers: 0.0.0 + user: hathach + repo: tinyusb + +project.repositories.allowed: + - apache-mynewt-core + - apache-mynewt-nimble + - apache-mynewt-mcumgr + - mcuboot + - arm-CMSIS_5 + - nordic-nrfx + - mbedtls + - stm-cmsis_device_f3 + - stm-stm32f3xx_hal_driver + +project.repositories.ignored: + - stm-cmsis_device_f3 + - stm-stm32f3xx_hal_driver diff --git a/.github/newt_upgrade/fail/core-ignored/expected.txt b/.github/newt_upgrade/fail/core-ignored/expected.txt new file mode 100644 index 000000000..af5781a97 --- /dev/null +++ b/.github/newt_upgrade/fail/core-ignored/expected.txt @@ -0,0 +1 @@ +Error: apache-mynewt-core repository must be allowed. Please add it to the allowed list and/or remove it from the ignored list. diff --git a/.github/newt_upgrade/fail/core-ignored/project.yml b/.github/newt_upgrade/fail/core-ignored/project.yml new file mode 100644 index 000000000..98b4a5322 --- /dev/null +++ b/.github/newt_upgrade/fail/core-ignored/project.yml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +project.repositories: + - apache-mynewt-core + +repository.apache-mynewt-core: + type: github + vers: 0-dev + user: apache + repo: mynewt-core + +project.repositories.ignored: + - apache-mynewt-core diff --git a/.github/newt_upgrade/fail/core-not-allowed/expected.txt b/.github/newt_upgrade/fail/core-not-allowed/expected.txt new file mode 100644 index 000000000..af5781a97 --- /dev/null +++ b/.github/newt_upgrade/fail/core-not-allowed/expected.txt @@ -0,0 +1 @@ +Error: apache-mynewt-core repository must be allowed. Please add it to the allowed list and/or remove it from the ignored list. diff --git a/.github/newt_upgrade/fail/core-not-allowed/project.yml b/.github/newt_upgrade/fail/core-not-allowed/project.yml new file mode 100644 index 000000000..588caf339 --- /dev/null +++ b/.github/newt_upgrade/fail/core-not-allowed/project.yml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +project.repositories: + - apache-mynewt-core + +repository.apache-mynewt-core: + type: github + vers: 0-dev + user: apache + repo: mynewt-core + +project.repositories.allowed: + - apache-mynewt-nimble + - apache-mynewt-mcumgr diff --git a/.github/workflows/test_upgrade.yml b/.github/workflows/test_upgrade.yml index 9d2dc93c7..8d20cf7f2 100644 --- a/.github/workflows/test_upgrade.yml +++ b/.github/workflows/test_upgrade.yml @@ -100,3 +100,31 @@ jobs: echo "Checking target ${{ matrix.targets }}" ! newt upgrade &> tmp.txt cat tmp.txt | tail -n `wc -l < expected.txt` | diff -w expected.txt - + + test_upgrade_filters: + name: newt upgrade (filters) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 'stable' + - name: Build newt + working-directory: newt + shell: bash + run: | + go version + go build + echo ${GITHUB_WORKSPACE}/newt >> $GITHUB_PATH + git config --global url."https://github.com/".insteadOf "git@github.com:" + - name: Test upgrade allowed-ignored + shell: bash + working-directory: .github/newt_upgrade/allowed-ignored + run: | + echo "Test upgrade" + newt upgrade + ls repos | diff -w expected.txt - diff --git a/newt/deprepo/deprepo.go b/newt/deprepo/deprepo.go index 159bac0cb..5d7faa5ad 100644 --- a/newt/deprepo/deprepo.go +++ b/newt/deprepo/deprepo.go @@ -136,6 +136,10 @@ func BuildDepGraph(repos RepoMap, rootReqs RequirementMap) (DepGraph, error) { reqMap := RequirementMap{} for _, d := range deps { depRepo := repos[d.Name] + // This might be nil if d.Name is not on allowed repositories list + if depRepo == nil { + continue + } verReqs, err := depRepo.NormalizeVerReq(d.VerReqs) if err != nil { return nil, err diff --git a/newt/install/install.go b/newt/install/install.go index 81c3d2b8e..a70f46bdd 100644 --- a/newt/install/install.go +++ b/newt/install/install.go @@ -199,6 +199,10 @@ func (inst *Installer) ensureDepsInList(repos []*repo.Repo, } for _, d := range deps { depRepo := inst.repos[d.Name] + // This might be nil if d.Name is not on allowed repositories list + if depRepo == nil { + continue + } result = append(result, recurse(depRepo)...) } diff --git a/newt/project/project.go b/newt/project/project.go index e4f4bae91..6fb9f7954 100644 --- a/newt/project/project.go +++ b/newt/project/project.go @@ -70,7 +70,12 @@ type Project struct { // read. repos deprepo.RepoMap - // Contains names of repositories that will not be upgraded/downloaded. + // Contains names of repositories that will be upgraded. + // If it's empty all repos are allowed. + reposAllowed []string + + // Contains names of repositories that will be excluded from upgrade. + // Can override repositories from reposAllowed. reposIgnored []string // The local repository at the top-level of the project. This repo is @@ -176,9 +181,16 @@ func NewProject(dir string, download bool) (*Project, error) { return proj, nil } -func (proj *Project) isRepoAdded(r *repo.Repo) bool { +func (proj *Project) isRepoAllowed(repoName string) bool { + if (len(proj.reposAllowed) == 0) || (util.SliceContains(proj.reposAllowed, repoName)) { + return !util.SliceContains(proj.reposIgnored, repoName) + } + return false +} + +func (proj *Project) isRepoAdded(repoName string) bool { for _, pr := range proj.repos { - if pr.Name() == r.Name() { + if pr.Name() == repoName { return true } } @@ -191,7 +203,7 @@ func (proj *Project) GetPkgRepos() error { if pkg.PkgConfig().HasKey("repository") { for k, _ := range pkg.PkgConfig().AllSettings() { repoName := strings.TrimPrefix(k, "repository.") - if repoName != k && !util.SliceContains(proj.reposIgnored, repoName) { + if repoName != k { fields, err := pkg.PkgConfig().GetValStringMapString(k, nil) util.OneTimeWarningError(err) @@ -203,6 +215,10 @@ func (proj *Project) GetPkgRepos() error { return err } } + if r == nil { + continue + } + verReq, err := newtutil.ParseRepoVersion(fields["vers"]) if err != nil { return util.FmtNewtError( @@ -212,7 +228,7 @@ func (proj *Project) GetPkgRepos() error { } r.SetPkgName(pkg.Name()) - if !proj.isRepoAdded(r) { + if !proj.isRepoAdded(r.Name()) { if err := proj.addRepo(r, true); err != nil { return err } @@ -405,7 +421,7 @@ func (proj *Project) InfoIf(predicate func(r *repo.Repo) bool, // @param name The name of the repo to read. // @param fields Fields containing the basic repo description. // -// @return *Repo The fully-read repo on success; nil on failure. +// @return *Repo The fully-read repo on success; nil on failure or when repo is not allowed // @return error Error on failure. func (proj *Project) loadRepo(name string, fields map[string]string) ( *repo.Repo, error) { @@ -427,12 +443,16 @@ func (proj *Project) loadRepo(name string, fields map[string]string) ( return nil, err } + if !proj.isRepoAllowed(r.Name()) { + return nil, nil + } + for _, ignDir := range ignoreSearchDirs { r.AddIgnoreDir(ignDir) } // Read the full repo definition from its `repository.yml` file. - if err := r.Read(proj.reposIgnored); err != nil { + if err := r.Read(); err != nil { return r, err } @@ -500,6 +520,9 @@ func (proj *Project) loadRepoDeps(download bool) error { return nil, err } } + if depRepo == nil { + continue + } if err := proj.addRepo(depRepo, download); err != nil { return nil, err } @@ -507,7 +530,7 @@ func (proj *Project) loadRepoDeps(download bool) error { newRepos = append(newRepos, depRepo) if download { - if _, err := depRepo.UpdateDesc(proj.reposIgnored); err != nil { + if _, err := depRepo.UpdateDesc(); err != nil { return nil, err } } @@ -542,7 +565,7 @@ func (proj *Project) downloadRepositoryYmlFiles() error { // specified in the `project.yml` file). for _, r := range proj.repos.Sorted() { if !r.IsLocal() && !r.IsExternal(r.Path()) { - if _, err := r.UpdateDesc(proj.reposIgnored); err != nil { + if _, err := r.UpdateDesc(); err != nil { return err } } @@ -631,13 +654,17 @@ func (proj *Project) loadConfig(download bool) error { proj.name, err = yc.GetValString("project.name", nil) util.OneTimeWarningError(err) + proj.reposAllowed = make([]string, 0) + proj.reposAllowed, err = yc.GetValStringSlice("project.repositories.allowed", nil) + util.OneTimeWarningError(err) + proj.reposIgnored = make([]string, 0) proj.reposIgnored, err = yc.GetValStringSlice("project.repositories.ignored", nil) util.OneTimeWarningError(err) - if util.SliceContains(proj.reposIgnored, "apache-mynewt-core") { - return util.NewNewtError("apache-mynewt-core repository can't be ignored. " + - "Please remove it from the ignored repositories list.") + if !proj.isRepoAllowed("apache-mynewt-core") { + return util.NewNewtError("apache-mynewt-core repository must be allowed. " + + "Please add it to the allowed list and/or remove it from the ignored list.") } // Local repository always included in initialization @@ -668,6 +695,10 @@ func (proj *Project) loadConfig(download bool) error { return err } } + if r == nil { + continue + } + verReq, err := newtutil.ParseRepoVersion(fields["vers"]) if err != nil { return util.FmtNewtError( diff --git a/newt/repo/repo.go b/newt/repo/repo.go index b5f6c1c09..791656714 100644 --- a/newt/repo/repo.go +++ b/newt/repo/repo.go @@ -397,7 +397,7 @@ func (r *Repo) Upgrade(ver newtutil.RepoVersion) error { // from master. The repo object is then populated with the contents of the // downladed file. If this repo has already had its descriptor updated, this // function is a no-op. -func (r *Repo) UpdateDesc(reposIgnored []string) (bool, error) { +func (r *Repo) UpdateDesc() (bool, error) { if r.updated { return false, nil } @@ -417,7 +417,7 @@ func (r *Repo) UpdateDesc(reposIgnored []string) (bool, error) { } // Read `repository.yml` and populate this repo object. - if err := r.Read(reposIgnored); err != nil { + if err := r.Read(); err != nil { return false, err } @@ -555,14 +555,11 @@ func parseRepoDepMap(depName string, return result, nil } -func (r *Repo) readDepRepos(yc ycfg.YCfg, reposIgnored []string) error { +func (r *Repo) readDepRepos(yc ycfg.YCfg) error { depMap, err := yc.GetValStringMap("repo.deps", nil) util.OneTimeWarningError(err) for depName, repoMapYml := range depMap { - if util.SliceContains(reposIgnored, depName) { - continue - } rdm, err := parseRepoDepMap(depName, repoMapYml) if err != nil { return util.FmtNewtError( @@ -580,7 +577,7 @@ func (r *Repo) readDepRepos(yc ycfg.YCfg, reposIgnored []string) error { // Reads a `repository.yml` file and populates the receiver repo with its // contents. -func (r *Repo) Read(reposIgnored []string) error { +func (r *Repo) Read() error { r.Init(r.Name(), r.downloader) yc, err := config.ReadFile(r.repoFilePath() + "/" + REPO_FILE_NAME) @@ -607,7 +604,7 @@ func (r *Repo) Read(reposIgnored []string) error { r.vers[vers] = commit } - if err := r.readDepRepos(yc, reposIgnored); err != nil { + if err := r.readDepRepos(yc); err != nil { return err }