Merge branch 'master' into pkg_whole_archive

.github/newt_sysinit/expected.txt

@@ -0,0 +1,18 @@
+Brief sysinit config for targets/nordic_pca10056_btshell:
+ STAGE   | PACKAGE                                           | FUNCTION                | SETTING                     
+---------+---------------------------------------------------+-------------------------+------------------------------
+ 0       | @apache-mynewt-core/kernel/os                     | os_pkg_init             | OS_SYSINIT_STAGE            
+ 9       | @apache-mynewt-core/sys/flash_map                 | flash_map_init          | FLASH_MAP_SYSINIT_STAGE     
+ 10      | @apache-mynewt-core/sys/stats/full                | stats_module_init       | STATS_SYSINIT_STAGE         
+ 20      | @apache-mynewt-core/sys/console/full              | console_pkg_init        | CONSOLE_SYSINIT_STAGE       
+ 100     | @apache-mynewt-core/sys/log/full                  | log_init                | LOG_SYSINIT_STAGE_MAIN      
+ 100     | @apache-mynewt-core/sys/log/modlog                | modlog_init             | MODLOG_SYSINIT_STAGE        
+ 250     | @apache-mynewt-nimble/nimble/transport            | ble_transport_init      |
+         | @apache-mynewt-nimble/nimble/controller           | ble_ll_init             |
+ 251     | @apache-mynewt-nimble/nimble/transport            | ble_transport_hs_init   |
+ 301     | @apache-mynewt-nimble/nimble/host/services/gap    | ble_svc_gap_init        | BLE_SVC_GAP_SYSINIT_STAGE   
+ 302     | @apache-mynewt-nimble/nimble/host/services/gatt   | ble_svc_gatt_init       | BLE_SVC_GATT_SYSINIT_STAGE  
+ 303     | @apache-mynewt-nimble/nimble/host/services/ans    | ble_svc_ans_init        | BLE_SVC_ANS_SYSINIT_STAGE   
+ 500     | @apache-mynewt-nimble/nimble/host/store/config    | ble_store_config_init   | BLE_STORE_SYSINIT_STAGE     
+ 500     | @apache-mynewt-core/sys/shell                     | shell_init              | SHELL_SYSINIT_STAGE         
+         | @apache-mynewt-nimble/nimble/transport            | ble_transport_ll_init   |

.github/workflows/test_sysinit.yml

@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Test sysinit
+
+on: [push, pull_request]
+
+jobs:
+  test_sysinit:
+    name: other
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 'stable'
+      - uses: carlosperate/arm-none-eabi-gcc-action@48db4484a55750df7a0ccca63347fcdea6534d78
+        with:
+          release: '12.2.Rel1'
+      - name: Install Dependencies
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+             sudo apt-get update
+             sudo apt-get install -y gcc-multilib
+      - name: Build newt
+        working-directory: newt
+        shell: bash
+        run: |
+             go version
+             go build
+             echo  ${GITHUB_WORKSPACE}/newt >> $GITHUB_PATH
+      - name: Test_sysinit
+        shell: bash
+        run: |
+             newt new project
+             cp -r .github/targets/nordic_pca10056_btshell project/targets
+             cd project/
+             newt upgrade -v --escape=false apache-mynewt-core apache-mynewt-nimble
+             newt info
+             newt target sysinit brief nordic_pca10056_btshell > tmp.txt
+             diff -w tmp.txt ../.github/newt_sysinit/expected.txt

NOTICE

@@ -1,5 +1,5 @@
 Apache Mynewt
-Copyright 2015-2022 The Apache Software Foundation
+Copyright 2015-2024 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

RELEASE_NOTES.md

@@ -1,6 +1,6 @@
 # RELEASE NOTES
 
-20 April 2022 - Apache Newt v1.10.0
+27 March 2024 - Apache Newt v1.12.0
 
 For full release notes, please visit the
 [Apache Mynewt Wiki](https://cwiki.apache.org/confluence/display/MYNEWT/Release+Notes).

go.mod

@@ -3,7 +3,7 @@ module mynewt.apache.org/newt
 go 1.13
 
 require (
-	github.com/apache/mynewt-artifact v0.0.24
+	github.com/apache/mynewt-artifact v0.0.25-0.20230515081815-85acad353839
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
@@ -16,5 +16,4 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	golang.org/x/crypto v0.7.0 // indirect
 )

go.sum

@@ -2,6 +2,8 @@ github.com/NickBall/go-aes-key-wrap v0.0.0-20170929221519-1c3aa3e4dfc5 h1:5BIUS5
 github.com/NickBall/go-aes-key-wrap v0.0.0-20170929221519-1c3aa3e4dfc5/go.mod h1:w5D10RxC0NmPYxmQ438CC1S07zaC1zpvuNW7s5sUk2Q=
 github.com/apache/mynewt-artifact v0.0.24 h1:haCZWFX4ftGSuXZj98QLRyS329xLLxa6arvmgqYZnJc=
 github.com/apache/mynewt-artifact v0.0.24/go.mod h1:vFUd47t74KPQMzSBhQ2qp5Hc7D29OU/Tl3xHtFwN3k8=
+github.com/apache/mynewt-artifact v0.0.25-0.20230515081815-85acad353839 h1:BTeK7lp8NUk66UftXyfsLrxAnlCbaMtMudh6Xe0F1lY=
+github.com/apache/mynewt-artifact v0.0.25-0.20230515081815-85acad353839/go.mod h1:8FsD0U8mLRSQ1I+zYnh8KZ5vKKtYUQQbj+j9+rhVst0=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -69,6 +71,8 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
+golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -77,6 +81,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -92,16 +97,23 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

newt/builder/build.go

@@ -57,6 +57,7 @@ type Builder struct {
 	buildName        string
 	linkElf          string
 	injectedSettings map[string]string
+	modifiedExtRepos []string
 }
 
 func NewBuilder(
@@ -149,6 +150,10 @@ func (b *Builder) addPackage(rpkg *resolve.ResolvePackage) (
 	return bpkg, nil
 }
 
+func (b *Builder) GetAutogeneratedLinkerIncludeDir() (string, error) {
+	return b.targetBuilder.BspPkg().GetAutogeneratedLinkerIncludePath()
+}
+
 func pkgTypeConflictErr(p1 *BuildPackage, p2 *BuildPackage) error {
 	return util.FmtNewtError("Two %s packages in build: %s, %s",
 		pkg.PackageTypeNames[p1.rpkg.Lpkg.Type()],
@@ -475,7 +480,13 @@ func (b *Builder) link(elfName string, linkerScripts []string,
 	}
 
 	c.LinkerScripts = linkerScripts
+	c.AutogeneratedLinkerIncludeDir, err = b.GetAutogeneratedLinkerIncludeDir()
+	if err != nil {
+		return err
+	}
+
 	err = c.CompileElf(elfName, staticLibs, keepSymbols, b.linkElf)
+
 	if err != nil {
 		return err
 	}
@@ -641,12 +652,40 @@ func buildWorker(
 	}
 }
 
+func (b *Builder) appendAppCflags(bpkgs []*BuildPackage) error {
+	for _, bpkg := range bpkgs {
+		settings := b.cfg.AllSettingsForLpkg(bpkg.rpkg.Lpkg)
+		globalAppCflags, err := bpkg.rpkg.Lpkg.PkgY.Get("app.cflags", settings)
+		if err != nil {
+			return err
+		}
+		for _, f := range globalAppCflags {
+			if itfVals, ok := f.Value.([]interface{}); ok {
+				for _, itfVal := range itfVals {
+					if strVal, ok := itfVal.(string); ok {
+						b.compilerInfo.Cflags = append(b.compilerInfo.Cflags, strVal)
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 func (b *Builder) Build() error {
+	var err error
+
 	b.CleanArtifacts()
 
 	// Build the packages alphabetically to ensure a consistent order.
 	bpkgs := b.sortedBuildPackages()
 
+	err = b.appendAppCflags(bpkgs)
+	if err != nil {
+		return err
+	}
+
 	// Calculate the list of jobs.  Each record represents a single file that
 	// needs to be compiled.
 	entries := []toolchain.CompilerJob{}
@@ -658,6 +697,8 @@ func (b *Builder) Build() error {
 		}
 		entries = append(entries, subEntries...)
 
+		b.modifiedExtRepos = append(b.modifiedExtRepos, bpkg.getModifiedReposNames()...)
+
 		if len(subEntries) > 0 {
 			bpkgCompilerMap[bpkg] = subEntries[0].Compiler
 		}
@@ -681,7 +722,6 @@ func (b *Builder) Build() error {
 		go buildWorker(i, jobs, stop, errors)
 	}
 
-	var err error
 	for i := 0; i < newtutil.NewtNumJobs; i++ {
 		subErr := <-errors
 		if err == nil && subErr != nil {
@@ -899,3 +939,24 @@ func (b *Builder) CleanArtifacts() {
 		os.Remove(p)
 	}
 }
+
+func Contains(elements []string, val string) bool {
+	for _, s := range elements {
+		if val == s {
+			return true
+		}
+	}
+	return false
+}
+
+func (b *Builder) AppendModifiedRepos(modifiedRepos []string) {
+	for _, repo := range modifiedRepos {
+		if !Contains(b.modifiedExtRepos, repo) {
+			b.modifiedExtRepos = append(b.modifiedExtRepos, repo)
+		}
+	}
+}
+
+func (b *Builder) GetModifiedRepos() []string {
+	return b.modifiedExtRepos
+}

newt/builder/buildpackage.go

@@ -20,6 +20,8 @@
 package builder
 
 import (
+	"mynewt.apache.org/newt/newt/downloader"
+	"mynewt.apache.org/newt/newt/repo"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -352,3 +354,46 @@ func (bpkg *BuildPackage) privateIncludeDirs(b *Builder) []string {
 
 	return incls
 }
+
+func (bpkg *BuildPackage) getModifiedReposNames() []string {
+	var modifiedRepos []string
+
+	settings := bpkg.rpkg.Lpkg.PkgY.AllSettings()
+	for settingName, setting := range settings {
+		if strings.HasPrefix(settingName, "repository") {
+			var version string
+
+			dl := downloader.NewGitDownloader()
+			rName := strings.TrimPrefix(settingName, "repository.")
+			r, _ := repo.NewRepo(rName, dl)
+
+			if util.NodeNotExist(r.Path()) {
+				modifiedRepos = append(modifiedRepos, r.Name())
+				continue
+			}
+
+			currentHash, _ := dl.HashFor(r.Path(), "HEAD")
+
+			aSetting, ok := setting.(map[interface{}]interface{})
+			if ok {
+				for field, value := range aSetting {
+					if field == "vers" {
+						aValue, ok := value.(string)
+						if ok {
+							version = strings.TrimSuffix(aValue, "-commit")
+						}
+					}
+				}
+			}
+
+			expectedHash, _ := dl.HashFor(r.Path(), version)
+			dirtyState, _ := r.DirtyState()
+
+			if currentHash != expectedHash || dirtyState != "" {
+				modifiedRepos = append(modifiedRepos, r.Name())
+			}
+		}
+	}
+
+	return modifiedRepos
+}

newt/builder/buildutil.go

@@ -259,6 +259,8 @@ type UserEnvParams struct {
 	UserSrcDir   string // "" if none
 	UserIncDir   string // "" if none
 	WorkDir      string
+	GeneratedDir string
+	TergetBinDir string
 }
 
 // UserEnvVars calculates the set of environment variables required by external
@@ -285,6 +287,8 @@ func UserEnvVars(params UserEnvParams) map[string]string {
 	}
 
 	m["MYNEWT_BUILD_PROFILE"] = params.BuildProfile
+	m["MYNEWT_BUILD_GENERATED_DIR"] = params.GeneratedDir
+	m["MYNEWT_BIN_DIR"] = params.TergetBinDir
 
 	return m
 }
@@ -324,6 +328,8 @@ func (b *Builder) EnvVars(imageSlot int) (map[string]string, error) {
 
 	env["MYNEWT_INCLUDE_PATH"] = strings.Join(b.compilerInfo.Includes, ":")
 	env["MYNEWT_CFLAGS"] = strings.Join(b.compilerInfo.Cflags, " ")
+	env["MYNEWT_TARGET_PATH"] = b.targetPkg.rpkg.Lpkg.FullName()
+	env["MYNEWT_APP_PATH"] = b.appPkg.rpkg.Lpkg.FullName()
 
 	pkgNames := []string{}
 	for _, p := range b.PkgMap {