From bc267cec6d035be8b742e7ba9de922d88ce8ca7f Mon Sep 17 00:00:00 2001 From: Michal Gorecki Date: Thu, 27 Jun 2024 14:35:34 +0200 Subject: [PATCH 1/2] Add allowed repositories list This in addition to ignored repositories list allows to specify a list of repositories that will be upgraded. The list should be placed inside project.yml and look for example like this: project.repositories.allowed: - apache-mynewt-core - apache-mynewt-nimble - apache-mynewt-mcumgr - mcuboot - arm-CMSIS_5 - nordic-nrfx - mbedtls If the same repository will be placed on both lists, it will be ignored by newt. IMPORTANT NOTE: Ignored repositories will be ignored even if they were previously downloaded. So if repo was downloaded by newt upgrade and then added to ignored list, targets that depend on this repo won't build. --- newt/deprepo/deprepo.go | 4 +++ newt/install/install.go | 4 +++ newt/project/project.go | 55 ++++++++++++++++++++++++++++++++--------- newt/repo/repo.go | 13 ++++------ 4 files changed, 56 insertions(+), 20 deletions(-) diff --git a/newt/deprepo/deprepo.go b/newt/deprepo/deprepo.go index 159bac0cb..5d7faa5ad 100644 --- a/newt/deprepo/deprepo.go +++ b/newt/deprepo/deprepo.go @@ -136,6 +136,10 @@ func BuildDepGraph(repos RepoMap, rootReqs RequirementMap) (DepGraph, error) { reqMap := RequirementMap{} for _, d := range deps { depRepo := repos[d.Name] + // This might be nil if d.Name is not on allowed repositories list + if depRepo == nil { + continue + } verReqs, err := depRepo.NormalizeVerReq(d.VerReqs) if err != nil { return nil, err diff --git a/newt/install/install.go b/newt/install/install.go index 81c3d2b8e..a70f46bdd 100644 --- a/newt/install/install.go +++ b/newt/install/install.go @@ -199,6 +199,10 @@ func (inst *Installer) ensureDepsInList(repos []*repo.Repo, } for _, d := range deps { depRepo := inst.repos[d.Name] + // This might be nil if d.Name is not on allowed repositories list + if depRepo == nil { + continue + } result = append(result, recurse(depRepo)...) } diff --git a/newt/project/project.go b/newt/project/project.go index e4f4bae91..6fb9f7954 100644 --- a/newt/project/project.go +++ b/newt/project/project.go @@ -70,7 +70,12 @@ type Project struct { // read. repos deprepo.RepoMap - // Contains names of repositories that will not be upgraded/downloaded. + // Contains names of repositories that will be upgraded. + // If it's empty all repos are allowed. + reposAllowed []string + + // Contains names of repositories that will be excluded from upgrade. + // Can override repositories from reposAllowed. reposIgnored []string // The local repository at the top-level of the project. This repo is @@ -176,9 +181,16 @@ func NewProject(dir string, download bool) (*Project, error) { return proj, nil } -func (proj *Project) isRepoAdded(r *repo.Repo) bool { +func (proj *Project) isRepoAllowed(repoName string) bool { + if (len(proj.reposAllowed) == 0) || (util.SliceContains(proj.reposAllowed, repoName)) { + return !util.SliceContains(proj.reposIgnored, repoName) + } + return false +} + +func (proj *Project) isRepoAdded(repoName string) bool { for _, pr := range proj.repos { - if pr.Name() == r.Name() { + if pr.Name() == repoName { return true } } @@ -191,7 +203,7 @@ func (proj *Project) GetPkgRepos() error { if pkg.PkgConfig().HasKey("repository") { for k, _ := range pkg.PkgConfig().AllSettings() { repoName := strings.TrimPrefix(k, "repository.") - if repoName != k && !util.SliceContains(proj.reposIgnored, repoName) { + if repoName != k { fields, err := pkg.PkgConfig().GetValStringMapString(k, nil) util.OneTimeWarningError(err) @@ -203,6 +215,10 @@ func (proj *Project) GetPkgRepos() error { return err } } + if r == nil { + continue + } + verReq, err := newtutil.ParseRepoVersion(fields["vers"]) if err != nil { return util.FmtNewtError( @@ -212,7 +228,7 @@ func (proj *Project) GetPkgRepos() error { } r.SetPkgName(pkg.Name()) - if !proj.isRepoAdded(r) { + if !proj.isRepoAdded(r.Name()) { if err := proj.addRepo(r, true); err != nil { return err } @@ -405,7 +421,7 @@ func (proj *Project) InfoIf(predicate func(r *repo.Repo) bool, // @param name The name of the repo to read. // @param fields Fields containing the basic repo description. // -// @return *Repo The fully-read repo on success; nil on failure. +// @return *Repo The fully-read repo on success; nil on failure or when repo is not allowed // @return error Error on failure. func (proj *Project) loadRepo(name string, fields map[string]string) ( *repo.Repo, error) { @@ -427,12 +443,16 @@ func (proj *Project) loadRepo(name string, fields map[string]string) ( return nil, err } + if !proj.isRepoAllowed(r.Name()) { + return nil, nil + } + for _, ignDir := range ignoreSearchDirs { r.AddIgnoreDir(ignDir) } // Read the full repo definition from its `repository.yml` file. - if err := r.Read(proj.reposIgnored); err != nil { + if err := r.Read(); err != nil { return r, err } @@ -500,6 +520,9 @@ func (proj *Project) loadRepoDeps(download bool) error { return nil, err } } + if depRepo == nil { + continue + } if err := proj.addRepo(depRepo, download); err != nil { return nil, err } @@ -507,7 +530,7 @@ func (proj *Project) loadRepoDeps(download bool) error { newRepos = append(newRepos, depRepo) if download { - if _, err := depRepo.UpdateDesc(proj.reposIgnored); err != nil { + if _, err := depRepo.UpdateDesc(); err != nil { return nil, err } } @@ -542,7 +565,7 @@ func (proj *Project) downloadRepositoryYmlFiles() error { // specified in the `project.yml` file). for _, r := range proj.repos.Sorted() { if !r.IsLocal() && !r.IsExternal(r.Path()) { - if _, err := r.UpdateDesc(proj.reposIgnored); err != nil { + if _, err := r.UpdateDesc(); err != nil { return err } } @@ -631,13 +654,17 @@ func (proj *Project) loadConfig(download bool) error { proj.name, err = yc.GetValString("project.name", nil) util.OneTimeWarningError(err) + proj.reposAllowed = make([]string, 0) + proj.reposAllowed, err = yc.GetValStringSlice("project.repositories.allowed", nil) + util.OneTimeWarningError(err) + proj.reposIgnored = make([]string, 0) proj.reposIgnored, err = yc.GetValStringSlice("project.repositories.ignored", nil) util.OneTimeWarningError(err) - if util.SliceContains(proj.reposIgnored, "apache-mynewt-core") { - return util.NewNewtError("apache-mynewt-core repository can't be ignored. " + - "Please remove it from the ignored repositories list.") + if !proj.isRepoAllowed("apache-mynewt-core") { + return util.NewNewtError("apache-mynewt-core repository must be allowed. " + + "Please add it to the allowed list and/or remove it from the ignored list.") } // Local repository always included in initialization @@ -668,6 +695,10 @@ func (proj *Project) loadConfig(download bool) error { return err } } + if r == nil { + continue + } + verReq, err := newtutil.ParseRepoVersion(fields["vers"]) if err != nil { return util.FmtNewtError( diff --git a/newt/repo/repo.go b/newt/repo/repo.go index b5f6c1c09..791656714 100644 --- a/newt/repo/repo.go +++ b/newt/repo/repo.go @@ -397,7 +397,7 @@ func (r *Repo) Upgrade(ver newtutil.RepoVersion) error { // from master. The repo object is then populated with the contents of the // downladed file. If this repo has already had its descriptor updated, this // function is a no-op. -func (r *Repo) UpdateDesc(reposIgnored []string) (bool, error) { +func (r *Repo) UpdateDesc() (bool, error) { if r.updated { return false, nil } @@ -417,7 +417,7 @@ func (r *Repo) UpdateDesc(reposIgnored []string) (bool, error) { } // Read `repository.yml` and populate this repo object. - if err := r.Read(reposIgnored); err != nil { + if err := r.Read(); err != nil { return false, err } @@ -555,14 +555,11 @@ func parseRepoDepMap(depName string, return result, nil } -func (r *Repo) readDepRepos(yc ycfg.YCfg, reposIgnored []string) error { +func (r *Repo) readDepRepos(yc ycfg.YCfg) error { depMap, err := yc.GetValStringMap("repo.deps", nil) util.OneTimeWarningError(err) for depName, repoMapYml := range depMap { - if util.SliceContains(reposIgnored, depName) { - continue - } rdm, err := parseRepoDepMap(depName, repoMapYml) if err != nil { return util.FmtNewtError( @@ -580,7 +577,7 @@ func (r *Repo) readDepRepos(yc ycfg.YCfg, reposIgnored []string) error { // Reads a `repository.yml` file and populates the receiver repo with its // contents. -func (r *Repo) Read(reposIgnored []string) error { +func (r *Repo) Read() error { r.Init(r.Name(), r.downloader) yc, err := config.ReadFile(r.repoFilePath() + "/" + REPO_FILE_NAME) @@ -607,7 +604,7 @@ func (r *Repo) Read(reposIgnored []string) error { r.vers[vers] = commit } - if err := r.readDepRepos(yc, reposIgnored); err != nil { + if err := r.readDepRepos(yc); err != nil { return err } From 1bcadf40fa29ca23c17c57e792ba40b922937839 Mon Sep 17 00:00:00 2001 From: Michal Gorecki Date: Thu, 11 Jul 2024 14:58:12 +0200 Subject: [PATCH 2/2] Add tests for allowed and ignored repos lists --- .../newt_upgrade/allowed-ignored/expected.txt | 7 +++ .../newt_upgrade/allowed-ignored/project.yml | 48 +++++++++++++++++++ .../fail/core-ignored/expected.txt | 1 + .../fail/core-ignored/project.yml | 30 ++++++++++++ .../fail/core-not-allowed/expected.txt | 1 + .../fail/core-not-allowed/project.yml | 31 ++++++++++++ .github/workflows/test_upgrade.yml | 28 +++++++++++ 7 files changed, 146 insertions(+) create mode 100644 .github/newt_upgrade/allowed-ignored/expected.txt create mode 100644 .github/newt_upgrade/allowed-ignored/project.yml create mode 100644 .github/newt_upgrade/fail/core-ignored/expected.txt create mode 100644 .github/newt_upgrade/fail/core-ignored/project.yml create mode 100644 .github/newt_upgrade/fail/core-not-allowed/expected.txt create mode 100644 .github/newt_upgrade/fail/core-not-allowed/project.yml diff --git a/.github/newt_upgrade/allowed-ignored/expected.txt b/.github/newt_upgrade/allowed-ignored/expected.txt new file mode 100644 index 000000000..683660be0 --- /dev/null +++ b/.github/newt_upgrade/allowed-ignored/expected.txt @@ -0,0 +1,7 @@ +apache-mynewt-core +apache-mynewt-mcumgr +apache-mynewt-nimble +arm-CMSIS_5 +mbedtls +mcuboot +nordic-nrfx diff --git a/.github/newt_upgrade/allowed-ignored/project.yml b/.github/newt_upgrade/allowed-ignored/project.yml new file mode 100644 index 000000000..a19d0874b --- /dev/null +++ b/.github/newt_upgrade/allowed-ignored/project.yml @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +project.repositories: + - apache-mynewt-core + +repository.apache-mynewt-core: + type: github + vers: 0-dev + user: apache + repo: mynewt-core + +repository.tinyusb: + type: github + vers: 0.0.0 + user: hathach + repo: tinyusb + +project.repositories.allowed: + - apache-mynewt-core + - apache-mynewt-nimble + - apache-mynewt-mcumgr + - mcuboot + - arm-CMSIS_5 + - nordic-nrfx + - mbedtls + - stm-cmsis_device_f3 + - stm-stm32f3xx_hal_driver + +project.repositories.ignored: + - stm-cmsis_device_f3 + - stm-stm32f3xx_hal_driver diff --git a/.github/newt_upgrade/fail/core-ignored/expected.txt b/.github/newt_upgrade/fail/core-ignored/expected.txt new file mode 100644 index 000000000..af5781a97 --- /dev/null +++ b/.github/newt_upgrade/fail/core-ignored/expected.txt @@ -0,0 +1 @@ +Error: apache-mynewt-core repository must be allowed. Please add it to the allowed list and/or remove it from the ignored list. diff --git a/.github/newt_upgrade/fail/core-ignored/project.yml b/.github/newt_upgrade/fail/core-ignored/project.yml new file mode 100644 index 000000000..98b4a5322 --- /dev/null +++ b/.github/newt_upgrade/fail/core-ignored/project.yml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +project.repositories: + - apache-mynewt-core + +repository.apache-mynewt-core: + type: github + vers: 0-dev + user: apache + repo: mynewt-core + +project.repositories.ignored: + - apache-mynewt-core diff --git a/.github/newt_upgrade/fail/core-not-allowed/expected.txt b/.github/newt_upgrade/fail/core-not-allowed/expected.txt new file mode 100644 index 000000000..af5781a97 --- /dev/null +++ b/.github/newt_upgrade/fail/core-not-allowed/expected.txt @@ -0,0 +1 @@ +Error: apache-mynewt-core repository must be allowed. Please add it to the allowed list and/or remove it from the ignored list. diff --git a/.github/newt_upgrade/fail/core-not-allowed/project.yml b/.github/newt_upgrade/fail/core-not-allowed/project.yml new file mode 100644 index 000000000..588caf339 --- /dev/null +++ b/.github/newt_upgrade/fail/core-not-allowed/project.yml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +project.repositories: + - apache-mynewt-core + +repository.apache-mynewt-core: + type: github + vers: 0-dev + user: apache + repo: mynewt-core + +project.repositories.allowed: + - apache-mynewt-nimble + - apache-mynewt-mcumgr diff --git a/.github/workflows/test_upgrade.yml b/.github/workflows/test_upgrade.yml index 9d2dc93c7..8d20cf7f2 100644 --- a/.github/workflows/test_upgrade.yml +++ b/.github/workflows/test_upgrade.yml @@ -100,3 +100,31 @@ jobs: echo "Checking target ${{ matrix.targets }}" ! newt upgrade &> tmp.txt cat tmp.txt | tail -n `wc -l < expected.txt` | diff -w expected.txt - + + test_upgrade_filters: + name: newt upgrade (filters) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 'stable' + - name: Build newt + working-directory: newt + shell: bash + run: | + go version + go build + echo ${GITHUB_WORKSPACE}/newt >> $GITHUB_PATH + git config --global url."https://github.com/".insteadOf "git@github.com:" + - name: Test upgrade allowed-ignored + shell: bash + working-directory: .github/newt_upgrade/allowed-ignored + run: | + echo "Test upgrade" + newt upgrade + ls repos | diff -w expected.txt -