Merge branch 'master' into pkg_whole_archive

.github/newt_sysinit/expected.txt

@@ -0,0 +1,18 @@
+Brief sysinit config for targets/nordic_pca10056_btshell:
+ STAGE | PACKAGE | FUNCTION | SETTING 
+---------+---------------------------------------------------+-------------------------+------------------------------
+ 0 | @apache-mynewt-core/kernel/os | os_pkg_init | OS_SYSINIT_STAGE 
+ 9 | @apache-mynewt-core/sys/flash_map | flash_map_init | FLASH_MAP_SYSINIT_STAGE 
+ 10 | @apache-mynewt-core/sys/stats/full | stats_module_init | STATS_SYSINIT_STAGE 
+ 20 | @apache-mynewt-core/sys/console/full | console_pkg_init | CONSOLE_SYSINIT_STAGE 
+ 100 | @apache-mynewt-core/sys/log/full | log_init | LOG_SYSINIT_STAGE_MAIN 
+ 100 | @apache-mynewt-core/sys/log/modlog | modlog_init | MODLOG_SYSINIT_STAGE 
+ 250 | @apache-mynewt-nimble/nimble/transport | ble_transport_init |
+ | @apache-mynewt-nimble/nimble/controller | ble_ll_init |
+ 251 | @apache-mynewt-nimble/nimble/transport | ble_transport_hs_init |
+ 301 | @apache-mynewt-nimble/nimble/host/services/gap | ble_svc_gap_init | BLE_SVC_GAP_SYSINIT_STAGE 
+ 302 | @apache-mynewt-nimble/nimble/host/services/gatt | ble_svc_gatt_init | BLE_SVC_GATT_SYSINIT_STAGE 
+ 303 | @apache-mynewt-nimble/nimble/host/services/ans | ble_svc_ans_init | BLE_SVC_ANS_SYSINIT_STAGE 
+ 500 | @apache-mynewt-nimble/nimble/host/store/config | ble_store_config_init | BLE_STORE_SYSINIT_STAGE 
+ 500 | @apache-mynewt-core/sys/shell | shell_init | SHELL_SYSINIT_STAGE 
+ | @apache-mynewt-nimble/nimble/transport | ble_transport_ll_init |

.github/workflows/test_sysinit.yml

@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Test sysinit
+
+on: [push, pull_request]
+
+jobs:
+ test_sysinit:
+ name: other
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest, macos-latest]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-go@v3
+ with:
+ go-version: 'stable'
+ - uses: carlosperate/arm-none-eabi-gcc-action@48db4484a55750df7a0ccca63347fcdea6534d78
+ with:
+ release: '12.2.Rel1'
+ - name: Install Dependencies
+ if: matrix.os == 'ubuntu-latest'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y gcc-multilib
+ - name: Build newt
+ working-directory: newt
+ shell: bash
+ run: |
+ go version
+ go build
+ echo ${GITHUB_WORKSPACE}/newt >> $GITHUB_PATH
+ - name: Test_sysinit
+ shell: bash
+ run: |
+ newt new project
+ cp -r .github/targets/nordic_pca10056_btshell project/targets
+ cd project/
+ newt upgrade -v --escape=false apache-mynewt-core apache-mynewt-nimble
+ newt info
+ newt target sysinit brief nordic_pca10056_btshell > tmp.txt
+ diff -w tmp.txt ../.github/newt_sysinit/expected.txt

NOTICE

@@ -1,5 +1,5 @@
 Apache Mynewt
-Copyright 2015-2022 The Apache Software Foundation
+Copyright 2015-2024 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

RELEASE_NOTES.md

@@ -1,6 +1,6 @@
 # RELEASE NOTES
 
-20 April 2022 - Apache Newt v1.10.0
+27 March 2024 - Apache Newt v1.12.0
 
 For full release notes, please visit the
 [Apache Mynewt Wiki](https://cwiki.apache.org/confluence/display/MYNEWT/Release+Notes).

go.mod

@@ -3,7 +3,7 @@ module mynewt.apache.org/newt
 go 1.13
 
 require (
- github.com/apache/mynewt-artifact v0.0.24
+ github.com/apache/mynewt-artifact v0.0.25-0.20230515081815-85acad353839
  github.com/inconshreveable/mousetrap v1.1.0 // indirect
  github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
  github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
@@ -16,5 +16,4 @@ require (
  github.com/spf13/pflag v1.0.5
  github.com/tklauser/go-sysconf v0.3.11 // indirect
  github.com/yusufpapurcu/wmi v1.2.2 // indirect
- golang.org/x/crypto v0.7.0 // indirect
 )

go.sum

@@ -2,6 +2,8 @@ github.com/NickBall/go-aes-key-wrap v0.0.0-20170929221519-1c3aa3e4dfc5 h1:5BIUS5
 github.com/NickBall/go-aes-key-wrap v0.0.0-20170929221519-1c3aa3e4dfc5/go.mod h1:w5D10RxC0NmPYxmQ438CC1S07zaC1zpvuNW7s5sUk2Q=
 github.com/apache/mynewt-artifact v0.0.24 h1:haCZWFX4ftGSuXZj98QLRyS329xLLxa6arvmgqYZnJc=
 github.com/apache/mynewt-artifact v0.0.24/go.mod h1:vFUd47t74KPQMzSBhQ2qp5Hc7D29OU/Tl3xHtFwN3k8=
+github.com/apache/mynewt-artifact v0.0.25-0.20230515081815-85acad353839 h1:BTeK7lp8NUk66UftXyfsLrxAnlCbaMtMudh6Xe0F1lY=
+github.com/apache/mynewt-artifact v0.0.25-0.20230515081815-85acad353839/go.mod h1:8FsD0U8mLRSQ1I+zYnh8KZ5vKKtYUQQbj+j9+rhVst0=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -69,6 +71,8 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
+golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -77,6 +81,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -92,16 +97,23 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

newt/builder/build.go

@@ -57,6 +57,7 @@ type Builder struct {
  buildName string
  linkElf string
  injectedSettings map[string]string
+ modifiedExtRepos []string
 }
 
 func NewBuilder(
@@ -149,6 +150,10 @@ func (b *Builder) addPackage(rpkg *resolve.ResolvePackage) (
  return bpkg, nil
 }
 
+func (b *Builder) GetAutogeneratedLinkerIncludeDir() (string, error) {
+ return b.targetBuilder.BspPkg().GetAutogeneratedLinkerIncludePath()
+}
+
 func pkgTypeConflictErr(p1 *BuildPackage, p2 *BuildPackage) error {
  return util.FmtNewtError("Two %s packages in build: %s, %s",
  pkg.PackageTypeNames[p1.rpkg.Lpkg.Type()],
@@ -475,7 +480,13 @@ func (b *Builder) link(elfName string, linkerScripts []string,
  }
 
  c.LinkerScripts = linkerScripts
+ c.AutogeneratedLinkerIncludeDir, err = b.GetAutogeneratedLinkerIncludeDir()
+ if err != nil {
+ return err
+ }
+
  err = c.CompileElf(elfName, staticLibs, keepSymbols, b.linkElf)
+
  if err != nil {
  return err
  }
@@ -641,12 +652,40 @@ func buildWorker(
  }
 }
 
+func (b *Builder) appendAppCflags(bpkgs []*BuildPackage) error {
+ for _, bpkg := range bpkgs {
+ settings := b.cfg.AllSettingsForLpkg(bpkg.rpkg.Lpkg)
+ globalAppCflags, err := bpkg.rpkg.Lpkg.PkgY.Get("app.cflags", settings)
+ if err != nil {
+ return err
+ }
+ for _, f := range globalAppCflags {
+ if itfVals, ok := f.Value.([]interface{}); ok {
+ for _, itfVal := range itfVals {
+ if strVal, ok := itfVal.(string); ok {
+ b.compilerInfo.Cflags = append(b.compilerInfo.Cflags, strVal)
+ }
+ }
+ }
+ }
+ }
+
+ return nil
+}
+
 func (b *Builder) Build() error {
+ var err error
+
  b.CleanArtifacts()
 
  // Build the packages alphabetically to ensure a consistent order.
  bpkgs := b.sortedBuildPackages()
 
+ err = b.appendAppCflags(bpkgs)
+ if err != nil {
+ return err
+ }
+
  // Calculate the list of jobs. Each record represents a single file that
  // needs to be compiled.
  entries := []toolchain.CompilerJob{}
@@ -658,6 +697,8 @@ func (b *Builder) Build() error {
  }
  entries = append(entries, subEntries...)
 
+ b.modifiedExtRepos = append(b.modifiedExtRepos, bpkg.getModifiedReposNames()...)
+
  if len(subEntries) > 0 {
  bpkgCompilerMap[bpkg] = subEntries[0].Compiler
  }
@@ -681,7 +722,6 @@ func (b *Builder) Build() error {
  go buildWorker(i, jobs, stop, errors)
  }
 
- var err error
  for i := 0; i < newtutil.NewtNumJobs; i++ {
  subErr := <-errors
  if err == nil && subErr != nil {
@@ -899,3 +939,24 @@ func (b *Builder) CleanArtifacts() {
  os.Remove(p)
  }
 }
+
+func Contains(elements []string, val string) bool {
+ for _, s := range elements {
+ if val == s {
+ return true
+ }
+ }
+ return false
+}
+
+func (b *Builder) AppendModifiedRepos(modifiedRepos []string) {
+ for _, repo := range modifiedRepos {
+ if !Contains(b.modifiedExtRepos, repo) {
+ b.modifiedExtRepos = append(b.modifiedExtRepos, repo)
+ }
+ }
+}
+
+func (b *Builder) GetModifiedRepos() []string {
+ return b.modifiedExtRepos
+}

newt/builder/buildpackage.go

@@ -20,6 +20,8 @@
 package builder
 
 import (
+ "mynewt.apache.org/newt/newt/downloader"
+ "mynewt.apache.org/newt/newt/repo"
  "os"
  "path/filepath"
  "regexp"
@@ -352,3 +354,46 @@ func (bpkg *BuildPackage) privateIncludeDirs(b *Builder) []string {
 
  return incls
 }
+
+func (bpkg *BuildPackage) getModifiedReposNames() []string {
+ var modifiedRepos []string
+
+ settings := bpkg.rpkg.Lpkg.PkgY.AllSettings()
+ for settingName, setting := range settings {
+ if strings.HasPrefix(settingName, "repository") {
+ var version string
+
+ dl := downloader.NewGitDownloader()
+ rName := strings.TrimPrefix(settingName, "repository.")
+ r, _ := repo.NewRepo(rName, dl)
+
+ if util.NodeNotExist(r.Path()) {
+ modifiedRepos = append(modifiedRepos, r.Name())
+ continue
+ }
+
+ currentHash, _ := dl.HashFor(r.Path(), "HEAD")
+
+ aSetting, ok := setting.(map[interface{}]interface{})
+ if ok {
+ for field, value := range aSetting {
+ if field == "vers" {
+ aValue, ok := value.(string)
+ if ok {
+ version = strings.TrimSuffix(aValue, "-commit")
+ }
+ }
+ }
+ }
+
+ expectedHash, _ := dl.HashFor(r.Path(), version)
+ dirtyState, _ := r.DirtyState()
+
+ if currentHash != expectedHash || dirtyState != "" {
+ modifiedRepos = append(modifiedRepos, r.Name())
+ }
+ }
+ }
+
+ return modifiedRepos
+}

newt/builder/buildutil.go

@@ -259,6 +259,8 @@ type UserEnvParams struct {
  UserSrcDir string // "" if none
  UserIncDir string // "" if none
  WorkDir string
+ GeneratedDir string
+ TergetBinDir string
 }
 
 // UserEnvVars calculates the set of environment variables required by external
@@ -285,6 +287,8 @@ func UserEnvVars(params UserEnvParams) map[string]string {
  }
 
  m["MYNEWT_BUILD_PROFILE"] = params.BuildProfile
+ m["MYNEWT_BUILD_GENERATED_DIR"] = params.GeneratedDir
+ m["MYNEWT_BIN_DIR"] = params.TergetBinDir
 
  return m
 }
@@ -324,6 +328,8 @@ func (b *Builder) EnvVars(imageSlot int) (map[string]string, error) {
 
  env["MYNEWT_INCLUDE_PATH"] = strings.Join(b.compilerInfo.Includes, ":")
  env["MYNEWT_CFLAGS"] = strings.Join(b.compilerInfo.Cflags, " ")
+ env["MYNEWT_TARGET_PATH"] = b.targetPkg.rpkg.Lpkg.FullName()
+ env["MYNEWT_APP_PATH"] = b.appPkg.rpkg.Lpkg.FullName()
 
  pkgNames := []string{}
  for _, p := range b.PkgMap {