Add allowed repositories list

.github/newt_upgrade/allowed-ignored/expected.txt

@@ -0,0 +1,7 @@
+apache-mynewt-core
+apache-mynewt-mcumgr
+apache-mynewt-nimble
+arm-CMSIS_5
+mbedtls
+mcuboot
+nordic-nrfx

.github/newt_upgrade/allowed-ignored/project.yml

@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+project.repositories:
+   - apache-mynewt-core
+
+repository.apache-mynewt-core:
+   type: github
+   vers: 0-dev
+   user: apache
+   repo: mynewt-core
+
+repository.tinyusb:
+   type: github
+   vers: 0.0.0
+   user: hathach
+   repo: tinyusb
+
+project.repositories.allowed:
+   - apache-mynewt-core
+   - apache-mynewt-nimble
+   - apache-mynewt-mcumgr
+   - mcuboot
+   - arm-CMSIS_5
+   - nordic-nrfx
+   - mbedtls
+   - stm-cmsis_device_f3
+   - stm-stm32f3xx_hal_driver
+
+project.repositories.ignored:
+   - stm-cmsis_device_f3
+   - stm-stm32f3xx_hal_driver

.github/newt_upgrade/fail/core-ignored/expected.txt

@@ -0,0 +1 @@
+Error: apache-mynewt-core repository must be allowed. Please add it to the allowed list and/or remove it from the ignored list.

.github/newt_upgrade/fail/core-ignored/project.yml

@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+project.repositories:
+   - apache-mynewt-core
+
+repository.apache-mynewt-core:
+   type: github
+   vers: 0-dev
+   user: apache
+   repo: mynewt-core
+
+project.repositories.ignored:
+   - apache-mynewt-core

.github/newt_upgrade/fail/core-not-allowed/expected.txt

@@ -0,0 +1 @@
+Error: apache-mynewt-core repository must be allowed. Please add it to the allowed list and/or remove it from the ignored list.

.github/newt_upgrade/fail/core-not-allowed/project.yml

@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+project.repositories:
+   - apache-mynewt-core
+
+repository.apache-mynewt-core:
+   type: github
+   vers: 0-dev
+   user: apache
+   repo: mynewt-core
+
+project.repositories.allowed:
+   - apache-mynewt-nimble
+   - apache-mynewt-mcumgr

.github/workflows/test_upgrade.yml

@@ -100,3 +100,31 @@ jobs:
              echo "Checking target ${{ matrix.targets }}"
              ! newt upgrade &> tmp.txt
              cat tmp.txt | tail -n `wc -l < expected.txt` | diff -w expected.txt -
+
+  test_upgrade_filters:
+    name: newt upgrade (filters)
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 'stable'
+      - name: Build newt
+        working-directory: newt
+        shell: bash
+        run: |
+          go version
+          go build
+          echo  ${GITHUB_WORKSPACE}/newt >> $GITHUB_PATH
+          git config --global url."https://github.com/".insteadOf "git@github.com:"
+      - name: Test upgrade allowed-ignored
+        shell: bash
+        working-directory: .github/newt_upgrade/allowed-ignored
+        run: |
+             echo "Test upgrade"
+             newt upgrade
+             ls repos | diff -w expected.txt -

newt/deprepo/deprepo.go

@@ -136,6 +136,10 @@ func BuildDepGraph(repos RepoMap, rootReqs RequirementMap) (DepGraph, error) {
 			reqMap := RequirementMap{}
 			for _, d := range deps {
 				depRepo := repos[d.Name]
+				// This might be nil if d.Name is not on allowed repositories list
+				if depRepo == nil {
+					continue
+				}
 				verReqs, err := depRepo.NormalizeVerReq(d.VerReqs)
 				if err != nil {
 					return nil, err

newt/install/install.go

@@ -199,6 +199,10 @@ func (inst *Installer) ensureDepsInList(repos []*repo.Repo,
 		}
 		for _, d := range deps {
 			depRepo := inst.repos[d.Name]
+			// This might be nil if d.Name is not on allowed repositories list
+			if depRepo == nil {
+				continue
+			}
 			result = append(result, recurse(depRepo)...)
 		}
 

newt/project/project.go

@@ -70,7 +70,12 @@ type Project struct {
 	//                read.
 	repos deprepo.RepoMap
 
-	// Contains names of repositories that will not be upgraded/downloaded.
+	// Contains names of repositories that will be upgraded.
+	// If it's empty all repos are allowed.
+	reposAllowed []string
+
+	// Contains names of repositories that will be excluded from upgrade.
+	// Can override repositories from reposAllowed.
 	reposIgnored []string
 
 	// The local repository at the top-level of the project.  This repo is
@@ -176,9 +181,16 @@ func NewProject(dir string, download bool) (*Project, error) {
 	return proj, nil
 }
 
-func (proj *Project) isRepoAdded(r *repo.Repo) bool {
+func (proj *Project) isRepoAllowed(repoName string) bool {
+	if (len(proj.reposAllowed) == 0) || (util.SliceContains(proj.reposAllowed, repoName)) {
+		return !util.SliceContains(proj.reposIgnored, repoName)
+	}
+	return false
+}
+
+func (proj *Project) isRepoAdded(repoName string) bool {
 	for _, pr := range proj.repos {
-		if pr.Name() == r.Name() {
+		if pr.Name() == repoName {
 			return true
 		}
 	}
@@ -191,7 +203,7 @@ func (proj *Project) GetPkgRepos() error {
 			if pkg.PkgConfig().HasKey("repository") {
 				for k, _ := range pkg.PkgConfig().AllSettings() {
 					repoName := strings.TrimPrefix(k, "repository.")
-					if repoName != k && !util.SliceContains(proj.reposIgnored, repoName) {
+					if repoName != k {
 						fields, err := pkg.PkgConfig().GetValStringMapString(k, nil)
 						util.OneTimeWarningError(err)
 
@@ -203,6 +215,10 @@ func (proj *Project) GetPkgRepos() error {
 								return err
 							}
 						}
+						if r == nil {
+							continue
+						}
+
 						verReq, err := newtutil.ParseRepoVersion(fields["vers"])
 						if err != nil {
 							return util.FmtNewtError(
@@ -212,7 +228,7 @@ func (proj *Project) GetPkgRepos() error {
 						}
 						r.SetPkgName(pkg.Name())
 
-						if !proj.isRepoAdded(r) {
+						if !proj.isRepoAdded(r.Name()) {
 							if err := proj.addRepo(r, true); err != nil {
 								return err
 							}
@@ -405,7 +421,7 @@ func (proj *Project) InfoIf(predicate func(r *repo.Repo) bool,
 // @param name                  The name of the repo to read.
 // @param fields                Fields containing the basic repo description.
 //
-// @return *Repo                The fully-read repo on success; nil on failure.
+// @return *Repo                The fully-read repo on success; nil on failure or when repo is not allowed
 // @return error                Error on failure.
 func (proj *Project) loadRepo(name string, fields map[string]string) (
 	*repo.Repo, error) {
@@ -427,12 +443,16 @@ func (proj *Project) loadRepo(name string, fields map[string]string) (
 		return nil, err
 	}
 
+	if !proj.isRepoAllowed(r.Name()) {
+		return nil, nil
+	}
+
 	for _, ignDir := range ignoreSearchDirs {
 		r.AddIgnoreDir(ignDir)
 	}
 
 	// Read the full repo definition from its `repository.yml` file.
-	if err := r.Read(proj.reposIgnored); err != nil {
+	if err := r.Read(); err != nil {
 		return r, err
 	}
 
@@ -500,14 +520,17 @@ func (proj *Project) loadRepoDeps(download bool) error {
 								return nil, err
 							}
 						}
+						if depRepo == nil {
+							continue
+						}
 						if err := proj.addRepo(depRepo, download); err != nil {
 							return nil, err
 						}
 					}
 					newRepos = append(newRepos, depRepo)
 
 					if download {
-						if _, err := depRepo.UpdateDesc(proj.reposIgnored); err != nil {
+						if _, err := depRepo.UpdateDesc(); err != nil {
 							return nil, err
 						}
 					}
@@ -542,7 +565,7 @@ func (proj *Project) downloadRepositoryYmlFiles() error {
 	// specified in the `project.yml` file).
 	for _, r := range proj.repos.Sorted() {
 		if !r.IsLocal() && !r.IsExternal(r.Path()) {
-			if _, err := r.UpdateDesc(proj.reposIgnored); err != nil {
+			if _, err := r.UpdateDesc(); err != nil {
 				return err
 			}
 		}
@@ -631,13 +654,17 @@ func (proj *Project) loadConfig(download bool) error {
 	proj.name, err = yc.GetValString("project.name", nil)
 	util.OneTimeWarningError(err)
 
+	proj.reposAllowed = make([]string, 0)
+	proj.reposAllowed, err = yc.GetValStringSlice("project.repositories.allowed", nil)
+	util.OneTimeWarningError(err)
+
 	proj.reposIgnored = make([]string, 0)
 	proj.reposIgnored, err = yc.GetValStringSlice("project.repositories.ignored", nil)
 	util.OneTimeWarningError(err)
 
-	if util.SliceContains(proj.reposIgnored, "apache-mynewt-core") {
-		return util.NewNewtError("apache-mynewt-core repository can't be ignored. " +
-			"Please remove it from the ignored repositories list.")
+	if !proj.isRepoAllowed("apache-mynewt-core") {
+		return util.NewNewtError("apache-mynewt-core repository must be allowed. " +
+			"Please add it to the allowed list and/or remove it from the ignored list.")
 	}
 
 	// Local repository always included in initialization
@@ -668,6 +695,10 @@ func (proj *Project) loadConfig(download bool) error {
 					return err
 				}
 			}
+			if r == nil {
+				continue
+			}
+
 			verReq, err := newtutil.ParseRepoVersion(fields["vers"])
 			if err != nil {
 				return util.FmtNewtError(

newt/repo/repo.go

@@ -397,7 +397,7 @@ func (r *Repo) Upgrade(ver newtutil.RepoVersion) error {
 // from master.  The repo object is then populated with the contents of the
 // downladed file.  If this repo has already had its descriptor updated, this
 // function is a no-op.
-func (r *Repo) UpdateDesc(reposIgnored []string) (bool, error) {
+func (r *Repo) UpdateDesc() (bool, error) {
 	if r.updated {
 		return false, nil
 	}
@@ -417,7 +417,7 @@ func (r *Repo) UpdateDesc(reposIgnored []string) (bool, error) {
 	}
 
 	// Read `repository.yml` and populate this repo object.
-	if err := r.Read(reposIgnored); err != nil {
+	if err := r.Read(); err != nil {
 		return false, err
 	}
 
@@ -555,14 +555,11 @@ func parseRepoDepMap(depName string,
 	return result, nil
 }
 
-func (r *Repo) readDepRepos(yc ycfg.YCfg, reposIgnored []string) error {
+func (r *Repo) readDepRepos(yc ycfg.YCfg) error {
 	depMap, err := yc.GetValStringMap("repo.deps", nil)
 	util.OneTimeWarningError(err)
 
 	for depName, repoMapYml := range depMap {
-		if util.SliceContains(reposIgnored, depName) {
-			continue
-		}
 		rdm, err := parseRepoDepMap(depName, repoMapYml)
 		if err != nil {
 			return util.FmtNewtError(
@@ -580,7 +577,7 @@ func (r *Repo) readDepRepos(yc ycfg.YCfg, reposIgnored []string) error {
 
 // Reads a `repository.yml` file and populates the receiver repo with its
 // contents.
-func (r *Repo) Read(reposIgnored []string) error {
+func (r *Repo) Read() error {
 	r.Init(r.Name(), r.downloader)
 
 	yc, err := config.ReadFile(r.repoFilePath() + "/" + REPO_FILE_NAME)
@@ -607,7 +604,7 @@ func (r *Repo) Read(reposIgnored []string) error {
 		r.vers[vers] = commit
 	}
 
-	if err := r.readDepRepos(yc, reposIgnored); err != nil {
+	if err := r.readDepRepos(yc); err != nil {
 		return err
 	}