diff --git a/.github/newt_sysinit/expected.txt b/.github/newt_sysinit/expected.txt new file mode 100644 index 000000000..e3f5790eb --- /dev/null +++ b/.github/newt_sysinit/expected.txt @@ -0,0 +1,18 @@ +Brief sysinit config for targets/nordic_pca10056_btshell: + STAGE | PACKAGE | FUNCTION | SETTING +---------+---------------------------------------------------+-------------------------+------------------------------ + 0 | @apache-mynewt-core/kernel/os | os_pkg_init | OS_SYSINIT_STAGE + 9 | @apache-mynewt-core/sys/flash_map | flash_map_init | FLASH_MAP_SYSINIT_STAGE + 10 | @apache-mynewt-core/sys/stats/full | stats_module_init | STATS_SYSINIT_STAGE + 20 | @apache-mynewt-core/sys/console/full | console_pkg_init | CONSOLE_SYSINIT_STAGE + 100 | @apache-mynewt-core/sys/log/full | log_init | LOG_SYSINIT_STAGE_MAIN + 100 | @apache-mynewt-core/sys/log/modlog | modlog_init | MODLOG_SYSINIT_STAGE + 250 | @apache-mynewt-nimble/nimble/transport | ble_transport_init | + | @apache-mynewt-nimble/nimble/controller | ble_ll_init | + 251 | @apache-mynewt-nimble/nimble/transport | ble_transport_hs_init | + 301 | @apache-mynewt-nimble/nimble/host/services/gap | ble_svc_gap_init | BLE_SVC_GAP_SYSINIT_STAGE + 302 | @apache-mynewt-nimble/nimble/host/services/gatt | ble_svc_gatt_init | BLE_SVC_GATT_SYSINIT_STAGE + 303 | @apache-mynewt-nimble/nimble/host/services/ans | ble_svc_ans_init | BLE_SVC_ANS_SYSINIT_STAGE + 500 | @apache-mynewt-nimble/nimble/host/store/config | ble_store_config_init | BLE_STORE_SYSINIT_STAGE + 500 | @apache-mynewt-core/sys/shell | shell_init | SHELL_SYSINIT_STAGE + | @apache-mynewt-nimble/nimble/transport | ble_transport_ll_init | diff --git a/.github/workflows/test_sysinit.yml b/.github/workflows/test_sysinit.yml new file mode 100644 index 000000000..e0875c6f6 --- /dev/null +++ b/.github/workflows/test_sysinit.yml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Test sysinit + +on: [push, pull_request] + +jobs: + test_sysinit: + name: other + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 'stable' + - uses: carlosperate/arm-none-eabi-gcc-action@48db4484a55750df7a0ccca63347fcdea6534d78 + with: + release: '12.2.Rel1' + - name: Install Dependencies + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install -y gcc-multilib + - name: Build newt + working-directory: newt + shell: bash + run: | + go version + go build + echo ${GITHUB_WORKSPACE}/newt >> $GITHUB_PATH + - name: Test_sysinit + shell: bash + run: | + newt new project + cp -r .github/targets/nordic_pca10056_btshell project/targets + cd project/ + newt upgrade -v --escape=false apache-mynewt-core apache-mynewt-nimble + newt info + newt target sysinit brief nordic_pca10056_btshell > tmp.txt + diff -w tmp.txt ../.github/newt_sysinit/expected.txt diff --git a/NOTICE b/NOTICE index d2c51bad9..b2b78e7b9 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Mynewt -Copyright 2015-2022 The Apache Software Foundation +Copyright 2015-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d74a75ddf..ebe63bfe2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,6 @@ # RELEASE NOTES -20 April 2022 - Apache Newt v1.10.0 +27 March 2024 - Apache Newt v1.12.0 For full release notes, please visit the [Apache Mynewt Wiki](https://cwiki.apache.org/confluence/display/MYNEWT/Release+Notes). diff --git a/go.mod b/go.mod index 868e871f4..3c417e77f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module mynewt.apache.org/newt go 1.13 require ( - github.com/apache/mynewt-artifact v0.0.24 + github.com/apache/mynewt-artifact v0.0.25-0.20230515081815-85acad353839 github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 @@ -16,5 +16,4 @@ require ( github.com/spf13/pflag v1.0.5 github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/crypto v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index 5e3f0d3af..4c4237ecb 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/NickBall/go-aes-key-wrap v0.0.0-20170929221519-1c3aa3e4dfc5 h1:5BIUS5 github.com/NickBall/go-aes-key-wrap v0.0.0-20170929221519-1c3aa3e4dfc5/go.mod h1:w5D10RxC0NmPYxmQ438CC1S07zaC1zpvuNW7s5sUk2Q= github.com/apache/mynewt-artifact v0.0.24 h1:haCZWFX4ftGSuXZj98QLRyS329xLLxa6arvmgqYZnJc= github.com/apache/mynewt-artifact v0.0.24/go.mod h1:vFUd47t74KPQMzSBhQ2qp5Hc7D29OU/Tl3xHtFwN3k8= +github.com/apache/mynewt-artifact v0.0.25-0.20230515081815-85acad353839 h1:BTeK7lp8NUk66UftXyfsLrxAnlCbaMtMudh6Xe0F1lY= +github.com/apache/mynewt-artifact v0.0.25-0.20230515081815-85acad353839/go.mod h1:8FsD0U8mLRSQ1I+zYnh8KZ5vKKtYUQQbj+j9+rhVst0= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -69,6 +71,8 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -77,6 +81,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -92,16 +97,23 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/newt/builder/build.go b/newt/builder/build.go index a9e487c46..b611df3af 100644 --- a/newt/builder/build.go +++ b/newt/builder/build.go @@ -57,6 +57,7 @@ type Builder struct { buildName string linkElf string injectedSettings map[string]string + modifiedExtRepos []string } func NewBuilder( @@ -149,6 +150,10 @@ func (b *Builder) addPackage(rpkg *resolve.ResolvePackage) ( return bpkg, nil } +func (b *Builder) GetAutogeneratedLinkerIncludeDir() (string, error) { + return b.targetBuilder.BspPkg().GetAutogeneratedLinkerIncludePath() +} + func pkgTypeConflictErr(p1 *BuildPackage, p2 *BuildPackage) error { return util.FmtNewtError("Two %s packages in build: %s, %s", pkg.PackageTypeNames[p1.rpkg.Lpkg.Type()], @@ -475,7 +480,13 @@ func (b *Builder) link(elfName string, linkerScripts []string, } c.LinkerScripts = linkerScripts + c.AutogeneratedLinkerIncludeDir, err = b.GetAutogeneratedLinkerIncludeDir() + if err != nil { + return err + } + err = c.CompileElf(elfName, staticLibs, keepSymbols, b.linkElf) + if err != nil { return err } @@ -641,12 +652,40 @@ func buildWorker( } } +func (b *Builder) appendAppCflags(bpkgs []*BuildPackage) error { + for _, bpkg := range bpkgs { + settings := b.cfg.AllSettingsForLpkg(bpkg.rpkg.Lpkg) + globalAppCflags, err := bpkg.rpkg.Lpkg.PkgY.Get("app.cflags", settings) + if err != nil { + return err + } + for _, f := range globalAppCflags { + if itfVals, ok := f.Value.([]interface{}); ok { + for _, itfVal := range itfVals { + if strVal, ok := itfVal.(string); ok { + b.compilerInfo.Cflags = append(b.compilerInfo.Cflags, strVal) + } + } + } + } + } + + return nil +} + func (b *Builder) Build() error { + var err error + b.CleanArtifacts() // Build the packages alphabetically to ensure a consistent order. bpkgs := b.sortedBuildPackages() + err = b.appendAppCflags(bpkgs) + if err != nil { + return err + } + // Calculate the list of jobs. Each record represents a single file that // needs to be compiled. entries := []toolchain.CompilerJob{} @@ -658,6 +697,8 @@ func (b *Builder) Build() error { } entries = append(entries, subEntries...) + b.modifiedExtRepos = append(b.modifiedExtRepos, bpkg.getModifiedReposNames()...) + if len(subEntries) > 0 { bpkgCompilerMap[bpkg] = subEntries[0].Compiler } @@ -681,7 +722,6 @@ func (b *Builder) Build() error { go buildWorker(i, jobs, stop, errors) } - var err error for i := 0; i < newtutil.NewtNumJobs; i++ { subErr := <-errors if err == nil && subErr != nil { @@ -899,3 +939,24 @@ func (b *Builder) CleanArtifacts() { os.Remove(p) } } + +func Contains(elements []string, val string) bool { + for _, s := range elements { + if val == s { + return true + } + } + return false +} + +func (b *Builder) AppendModifiedRepos(modifiedRepos []string) { + for _, repo := range modifiedRepos { + if !Contains(b.modifiedExtRepos, repo) { + b.modifiedExtRepos = append(b.modifiedExtRepos, repo) + } + } +} + +func (b *Builder) GetModifiedRepos() []string { + return b.modifiedExtRepos +} diff --git a/newt/builder/buildpackage.go b/newt/builder/buildpackage.go index 37d99ce95..7bcd835d8 100644 --- a/newt/builder/buildpackage.go +++ b/newt/builder/buildpackage.go @@ -20,6 +20,8 @@ package builder import ( + "mynewt.apache.org/newt/newt/downloader" + "mynewt.apache.org/newt/newt/repo" "os" "path/filepath" "regexp" @@ -352,3 +354,46 @@ func (bpkg *BuildPackage) privateIncludeDirs(b *Builder) []string { return incls } + +func (bpkg *BuildPackage) getModifiedReposNames() []string { + var modifiedRepos []string + + settings := bpkg.rpkg.Lpkg.PkgY.AllSettings() + for settingName, setting := range settings { + if strings.HasPrefix(settingName, "repository") { + var version string + + dl := downloader.NewGitDownloader() + rName := strings.TrimPrefix(settingName, "repository.") + r, _ := repo.NewRepo(rName, dl) + + if util.NodeNotExist(r.Path()) { + modifiedRepos = append(modifiedRepos, r.Name()) + continue + } + + currentHash, _ := dl.HashFor(r.Path(), "HEAD") + + aSetting, ok := setting.(map[interface{}]interface{}) + if ok { + for field, value := range aSetting { + if field == "vers" { + aValue, ok := value.(string) + if ok { + version = strings.TrimSuffix(aValue, "-commit") + } + } + } + } + + expectedHash, _ := dl.HashFor(r.Path(), version) + dirtyState, _ := r.DirtyState() + + if currentHash != expectedHash || dirtyState != "" { + modifiedRepos = append(modifiedRepos, r.Name()) + } + } + } + + return modifiedRepos +} diff --git a/newt/builder/buildutil.go b/newt/builder/buildutil.go index 6212fad0d..323e716ce 100644 --- a/newt/builder/buildutil.go +++ b/newt/builder/buildutil.go @@ -259,6 +259,8 @@ type UserEnvParams struct { UserSrcDir string // "" if none UserIncDir string // "" if none WorkDir string + GeneratedDir string + TergetBinDir string } // UserEnvVars calculates the set of environment variables required by external @@ -285,6 +287,8 @@ func UserEnvVars(params UserEnvParams) map[string]string { } m["MYNEWT_BUILD_PROFILE"] = params.BuildProfile + m["MYNEWT_BUILD_GENERATED_DIR"] = params.GeneratedDir + m["MYNEWT_BIN_DIR"] = params.TergetBinDir return m } @@ -324,6 +328,8 @@ func (b *Builder) EnvVars(imageSlot int) (map[string]string, error) { env["MYNEWT_INCLUDE_PATH"] = strings.Join(b.compilerInfo.Includes, ":") env["MYNEWT_CFLAGS"] = strings.Join(b.compilerInfo.Cflags, " ") + env["MYNEWT_TARGET_PATH"] = b.targetPkg.rpkg.Lpkg.FullName() + env["MYNEWT_APP_PATH"] = b.appPkg.rpkg.Lpkg.FullName() pkgNames := []string{} for _, p := range b.PkgMap { diff --git a/newt/builder/cmake.go b/newt/builder/cmake.go index a287ec3e3..67d44f6fb 100644 --- a/newt/builder/cmake.go +++ b/newt/builder/cmake.go @@ -212,6 +212,11 @@ func (b *Builder) CMakeTargetWrite(w io.Writer, targetCompiler *toolchain.Compil var linkFlags []string var libraries []string + err := b.appendAppCflags(bpkgs) + if err != nil { + return err + } + c := targetCompiler c.AddInfo(b.GetCompilerInfo()) @@ -279,6 +284,8 @@ func (b *Builder) CMakeTargetWrite(w io.Writer, targetCompiler *toolchain.Compil lFlags = append(lFlags, "-T"+ld) } + lFlags = append(lFlags, "-L"+c.AutogeneratedLinkerIncludeDir) + var cFlags []string cFlags = append(cFlags, c.GetCompilerInfo().Cflags...) cFlags = append(cFlags, c.GetLocalCompilerInfo().Cflags...) @@ -360,7 +367,8 @@ func CmakeCompilerInfoWrite(w io.Writer, archiveFile string, bpkg *BuildPackage, } func (t *TargetBuilder) CMakeTargetBuilderWrite(w io.Writer, targetCompiler *toolchain.Compiler) error { - if err := t.PrepBuild(); err != nil { + var err error + if err = t.PrepBuild(); err != nil { return err } @@ -376,12 +384,15 @@ func (t *TargetBuilder) CMakeTargetBuilderWrite(w io.Writer, targetCompiler *too project.ResetDeps(t.AppList) targetCompiler.LinkerScripts = t.bspPkg.LinkerScripts + if targetCompiler.AutogeneratedLinkerIncludeDir, err = t.bspPkg.GetAutogeneratedLinkerIncludePath(); err != nil { + return err + } - if err := t.bspPkg.Reload(t.AppBuilder.cfg.SettingValues()); err != nil { + if err = t.bspPkg.Reload(t.AppBuilder.cfg.SettingValues()); err != nil { return err } - if err := t.AppBuilder.CMakeTargetWrite(w, targetCompiler); err != nil { + if err = t.AppBuilder.CMakeTargetWrite(w, targetCompiler); err != nil { return err } diff --git a/newt/builder/extcmd.go b/newt/builder/extcmd.go index 3424cc02b..e75530f8b 100644 --- a/newt/builder/extcmd.go +++ b/newt/builder/extcmd.go @@ -120,6 +120,8 @@ func (t *TargetBuilder) envVarsForCmd(sf stage.StageFunc, userSrcDir string, UserSrcDir: userSrcDir, UserIncDir: userIncDir, WorkDir: workDir, + GeneratedDir: GeneratedBaseDir(t.target.FullName()), + TergetBinDir: TargetBinDir(t.target.FullName()), } uenv := UserEnvVars(p) for k, v := range uenv { @@ -182,6 +184,47 @@ func (t *TargetBuilder) execExtCmds(sf stage.StageFunc, userSrcDir string, return nil } +func getLinkTableEntry(name string) string { + indent := " " + + entry := indent + "__" + name + "_start__ = .;\n" + + indent + "KEEP(*(." + name + "))\n" + + indent + "__" + name + "_end__ = .;\n\n" + + return entry +} + +func (t *TargetBuilder) generateLinkTables() { + var s []string + + dir := GeneratedBaseDir(t.target.FullName()) + "/link/include" + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + log.Error("Generate link tables error:\n", err) + return + } + + linkHeader, err := os.Create(dir + "/link_tables.ld.h") + if err != nil { + log.Error("Generate link tables error:\n", err) + return + } + + for _, pkg := range t.res.LpkgRpkgMap { + s = append(s, pkg.Lpkg.LinkTables()...) + } + + if len(s) == 0 { + return + } + + for _, linkTable := range s { + linkHeader.WriteString(getLinkTableEntry(linkTable)) + } + +} + +//link tables // execPreBuildCmds runs the target's set of pre-build user commands. It is an // error if any command fails (exits with a nonzero status). func (t *TargetBuilder) execPreBuildCmds(workDir string) error { diff --git a/newt/builder/targetbuild.go b/newt/builder/targetbuild.go index 1eefd8006..c7efc03aa 100644 --- a/newt/builder/targetbuild.go +++ b/newt/builder/targetbuild.go @@ -284,7 +284,11 @@ func (t *TargetBuilder) validateAndWriteCfg() error { log.Warn(line) } - for _, line := range t.res.ExperimentalWarning() { + for _, line := range t.res.CfgExperimentalWarning() { + log.Warn(line) + } + + for _, line := range t.res.PkgExperimentalWarning() { log.Warn(line) } @@ -292,7 +296,11 @@ func (t *TargetBuilder) validateAndWriteCfg() error { srcDir := GeneratedSrcDir(t.target.FullName()) lpkgs := resolve.RpkgSliceToLpkgSlice(t.res.MasterSet.Rpkgs) - if err := syscfg.EnsureWritten(t.res.Cfg, incDir, lpkgs); err != nil { + apis := []string{} + for api := range t.res.ApiMap { + apis = append(apis, api) + } + if err := syscfg.EnsureWritten(t.res.Cfg, incDir, lpkgs, apis); err != nil { return err } @@ -577,6 +585,8 @@ func (t *TargetBuilder) Build() error { os.RemoveAll(workDir) }() + t.generateLinkTables() + // Execute the set of pre-build user scripts. if err := t.execPreBuildCmds(workDir); err != nil { return err diff --git a/newt/cli/build_cmds.go b/newt/cli/build_cmds.go index 6902e098b..ce1611536 100644 --- a/newt/cli/build_cmds.go +++ b/newt/cli/build_cmds.go @@ -161,6 +161,13 @@ func buildRunCmd(cmd *cobra.Command, args []string, printShellCmds bool, execute } if err := b.Build(); err != nil { + if b.AppBuilder != nil { + if b.AppBuilder.GetModifiedRepos() != nil { + util.ErrorMessage(util.VERBOSITY_DEFAULT, + "Warning: Following external repos are modified or missing, which might be causing build errors:\n%v\n", + b.AppBuilder.GetModifiedRepos()) + } + } NewtUsage(nil, err) } diff --git a/newt/cli/project_cmds.go b/newt/cli/project_cmds.go index 2d5f57f87..9a742462e 100644 --- a/newt/cli/project_cmds.go +++ b/newt/cli/project_cmds.go @@ -72,14 +72,20 @@ func newRunCmd(cmd *cobra.Command, args []string) { NewtUsage(nil, err) } - commit, err := dl.LatestRc(tmpdir, newtutil.NewtBlinkyTag) - if err != nil { - NewtUsage(nil, err) - } - + commit := newtutil.NewtBlinkyTag err = dl.Checkout(tmpdir, commit) + + /* If checkout with final tag fails, try to find latest rc tag */ if err != nil { - NewtUsage(nil, err) + commit, err = dl.LatestRc(tmpdir, newtutil.NewtBlinkyTag) + if err != nil { + NewtUsage(nil, err) + } + + err = dl.Checkout(tmpdir, commit) + if err != nil { + NewtUsage(nil, err) + } } util.StatusMessage(util.VERBOSITY_DEFAULT, "Installing "+ @@ -125,6 +131,7 @@ func upgradeRunCmd(cmd *cobra.Command, args []string) { interfaces.SetProject(proj) proj.GetPkgRepos() + proj.SetGitEnvVariables() pred := makeRepoPredicate(args) if err := proj.UpgradeIf( diff --git a/newt/cli/target_cmds.go b/newt/cli/target_cmds.go index a4fa89f84..b3a13e543 100644 --- a/newt/cli/target_cmds.go +++ b/newt/cli/target_cmds.go @@ -23,6 +23,7 @@ import ( "bytes" "fmt" "io/ioutil" + "mynewt.apache.org/newt/newt/ycfg" "os" "sort" "strings" @@ -249,6 +250,50 @@ func targetShowCmd(cmd *cobra.Command, args []string) { } } +func printCflags(appCflags []ycfg.YCfgEntry) { + for _, f := range appCflags { + if itfVals, ok := f.Value.([]interface{}); ok { + for _, itfVal := range itfVals { + if strVal, ok := itfVal.(string); ok { + fmt.Println(strVal) + } + } + } + } +} + +func targetInfoCmd(cmd *cobra.Command, args []string) { + if len(args) < 1 { + NewtUsage(cmd, + util.NewNewtError("Must specify target name")) + } + + TryGetProject() + + b, err := TargetBuilderForTargetOrUnittest(args[0]) + if err != nil { + NewtUsage(cmd, err) + } + + if err := b.PrepBuild(); err != nil { + NewtUsage(nil, err) + } + + fmt.Println("Packages containing app.cflags:") + + for _, rpkg := range b.AppBuilder.SortedRpkgs() { + appCflags, err := rpkg.Lpkg.PkgY.Get("app.cflags", nil) + if err != nil { + NewtUsage(nil, err) + } + if appCflags != nil { + fmt.Println(rpkg.Lpkg.Name()) + printCflags(appCflags) + fmt.Println("") + } + } +} + func targetListCmd(cmd *cobra.Command, args []string) { TryGetProject() targetNames := []string{} @@ -937,6 +982,21 @@ func AddTargetCommands(cmd *cobra.Command) { return append(targetList(), unittestList()...) }) + infoHelpText := "Shows which packages contain app cflags in the target specified " + + "by ." + infoHelpEx := " newt target info \n" + infoHelpEx += " newt target info my_target1" + + infoCmd := &cobra.Command{ + Use: "info", + Short: "Show packages with global cflags", + Long: infoHelpText, + Example: infoHelpEx, + Run: targetInfoCmd, + } + targetCmd.AddCommand(infoCmd) + AddTabCompleteFn(infoCmd, targetList) + for _, cmd := range targetCfgCmdAll() { targetCmd.AddCommand(cmd) } diff --git a/newt/downloader/downloader.go b/newt/downloader/downloader.go index 8ed2927b6..2be323a14 100644 --- a/newt/downloader/downloader.go +++ b/newt/downloader/downloader.go @@ -61,6 +61,9 @@ type Downloader interface { // Fetches all remotes. Fetch(path string) error + // Fetches specific commit + FetchCommit(path string, commit string) error + // Checks out the specified commit (hash, tag, or branch). Always puts the // repo in a "detached head" state. Checkout(path string, commit string) error @@ -94,6 +97,10 @@ type Downloader interface { // If such a commit exists, it is returned. Otherwise, "" is returned. LatestRc(path string, base string) (string, error) + // Applies patches provided inside "patches" directory. + // If no patch is provided function does nothing + ApplyPatches(path string, patches []string) error + // Returns the branch that contains the YAML control files; this option // allows implementers to override "master" as the main branch. MainBranch() string @@ -448,6 +455,26 @@ func (gd *GenericDownloader) Checkout(repoDir string, commit string) error { return err } +func (gd *GenericDownloader) ApplyPatches(repoDir string, patches []string) error { + cmd := []string{ + "am", + } + cmd = append(cmd, patches...) + + _, err := executeGitCommand(repoDir, cmd, true) + if err != nil { + // Abort git am if applying patches failed + cmd = []string{ + "am", + "--abort", + } + executeGitCommand(repoDir, cmd, true) + + return err + } + return nil +} + // Update one submodule tree in a repo (under path) func (gd *GenericDownloader) UpdateSubmodule(path string, submodule string) error { cmd := []string{ @@ -774,6 +801,11 @@ func (gd *GithubDownloader) Fetch(repoDir string) error { }) } +func (gd *GithubDownloader) FetchCommit(repoDir string, commit string) error { + _, err := executeGitCommand(repoDir, []string{"fetch", "--depth=1", "origin", commit}, true) + return err +} + func (gd *GithubDownloader) password() string { if gd.Password != "" { return gd.Password @@ -943,6 +975,11 @@ func (gd *GitDownloader) Fetch(repoDir string) error { }) } +func (gd *GitDownloader) FetchCommit(repoDir string, commit string) error { + _, err := executeGitCommand(repoDir, []string{"fetch", "--depth=1", "origin", commit}, true) + return err +} + func (gd *GitDownloader) FetchFile( commit string, path string, filename string, dstDir string) error { @@ -1043,6 +1080,11 @@ func (ld *LocalDownloader) Fetch(path string) error { return ld.Clone(ld.MainBranch(), path) } +func (ld *LocalDownloader) FetchCommit(path string, commit string) error { + _, err := executeGitCommand(path, []string{"fetch", "--depth=1", "origin", commit}, true) + return err +} + func (ld *LocalDownloader) Checkout(path string, commit string) error { _, err := executeGitCommand(path, []string{"checkout", commit}, true) return err diff --git a/newt/flashmap/flashmap.go b/newt/flashmap/flashmap.go index 517ad720c..d0c0446c3 100644 --- a/newt/flashmap/flashmap.go +++ b/newt/flashmap/flashmap.go @@ -245,12 +245,15 @@ func writeFlashMapHeader(w io.Writer, fm FlashMap) { fmt.Fprintf(w, "#ifndef H_MYNEWT_SYSFLASH_\n") fmt.Fprintf(w, "#define H_MYNEWT_SYSFLASH_\n") fmt.Fprintf(w, "\n") - fmt.Fprintf(w, "#include \"flash_map/flash_map.h\"\n") - fmt.Fprintf(w, "\n") fmt.Fprintf(w, "#define FLASH_AREA_COUNT %d\n", len(fm.Areas)) fmt.Fprintf(w, "\n") + fmt.Fprintf(w, "/* MYNEWT_SYSFLASH_ONLY_CONST is defined when this header is used for linker script generation */\n") + fmt.Fprintf(w, "#ifndef MYNEWT_SYSFLASH_ONLY_CONST\n") + fmt.Fprintf(w, "#include \"flash_map/flash_map.h\"\n") + fmt.Fprintf(w, "\n") fmt.Fprintf(w, "%s", C_VAR_COMMENT) fmt.Fprintf(w, "extern %s;\n", flashMapVarDecl(fm)) + fmt.Fprintf(w, "#endif\n") fmt.Fprintf(w, "\n") fmt.Fprintf(w, "/* Flash map was defined in %s */\n", fm.PkgName) fmt.Fprintf(w, "\n") diff --git a/newt/install/install.go b/newt/install/install.go index f3380e01e..81c3d2b8e 100644 --- a/newt/install/install.go +++ b/newt/install/install.go @@ -241,6 +241,10 @@ func (inst *Installer) shouldUpgradeRepo( return true, nil } + if r.IsUpgradedFromProjectYml() { + return false, nil + } + if !r.VersionsEqual(*curVer, destVer) { return true, nil } @@ -338,7 +342,7 @@ func (inst *Installer) installPrompt(vm deprepo.VersionMap, op installOp, } util.StatusMessage(util.VERBOSITY_DEFAULT, - "Making the following changes to the project:\n") + "Trying to make the following changes to the project:\n") names := vm.SortedNames() for _, name := range names { @@ -484,46 +488,6 @@ func (inst *Installer) calcVersionMap(candidates []*repo.Repo) ( return vm, nil } -// Checks if any repos in the specified slice are in a dirty state. If any -// repos are dirty and `force` is *not* enabled, an error is returned. If any -// repos are dirty and `force` is enabled, a warning is displayed. -func verifyRepoDirtyState(repos []*repo.Repo, force bool) error { - // [repo] => dirty-state. - var m map[*repo.Repo]string - - // Collect all dirty repos and insert them into m. - for _, r := range repos { - dirtyState, err := r.DirtyState() - if err != nil { - return err - } - - if dirtyState != "" { - if m == nil { - m = make(map[*repo.Repo]string) - m[r] = dirtyState - } - } - } - - if len(m) > 0 { - s := "some repos are in a dirty state:\n" - for r, d := range m { - s += fmt.Sprintf(" %s: contains %s\n", r.Name(), d) - } - - if !force { - s += "Specify the `-f` (force) switch to attempt anyway" - return util.NewNewtError(s) - } else { - util.OneTimeWarning("%s", s) - } - } - - return nil - -} - func verifyNewtCompat(repos []*repo.Repo, vm deprepo.VersionMap) error { var errors []string @@ -550,10 +514,6 @@ func verifyNewtCompat(repos []*repo.Repo, vm deprepo.VersionMap) error { func (inst *Installer) Upgrade(candidates []*repo.Repo, force bool, ask bool) error { - if err := verifyRepoDirtyState(candidates, force); err != nil { - return err - } - vm, err := inst.calcVersionMap(candidates) if err != nil { return err @@ -587,14 +547,43 @@ func (inst *Installer) Upgrade(candidates []*repo.Repo, force bool, // Upgrade each repo in the version map. for _, r := range repos { destVer := vm[r.Name()] + dirtyState, err := r.DirtyState() + if err != nil { + return err + } + + if dirtyState != "" { + if !force { + util.StatusMessage(util.VERBOSITY_DEFAULT, + "%s is in dirty state, it won't be upgraded\n", + r.Name()) + continue + } else { + util.OneTimeWarning("forced update of repo in dirty state: %s", r.Name()) + } + } + if err := r.Upgrade(destVer); err != nil { return err } + if r.IsFromProjectYml() { + r.SetIsUpgradedFromProjectYml() + } util.StatusMessage(util.VERBOSITY_DEFAULT, "%s successfully upgraded to version %s\n", r.Name(), destVer.String()) } + for _, r := range candidates { + err = r.ApplyPatches() + if err != nil { + util.StatusMessage(util.VERBOSITY_DEFAULT, + "Applying patches in repository %s failed\n", r.Name()) + + return err + } + } + return nil } diff --git a/newt/newtutil/newtutil.go b/newt/newtutil/newtutil.go index 751152dab..9e129e67c 100644 --- a/newt/newtutil/newtutil.go +++ b/newt/newtutil/newtutil.go @@ -30,8 +30,8 @@ import ( "mynewt.apache.org/newt/util" ) -var NewtVersion = Version{1, 10, 9900} -var NewtVersionStr = "1.11.0-dev" +var NewtVersion = Version{1, 12, 9900} +var NewtVersionStr = "1.13.0-dev" var NewtGitHash = "unknown" var NewtDate = "unknown" diff --git a/newt/pkg/bsp_package.go b/newt/pkg/bsp_package.go index 158ebec2d..46ecb7fb1 100644 --- a/newt/pkg/bsp_package.go +++ b/newt/pkg/bsp_package.go @@ -82,6 +82,26 @@ func (bsp *BspPackage) resolvePathSetting( return path, nil } +func (bsp *BspPackage) getAutogeneratedLinkerScriptPath() (string, error) { + defaultLinkerScriptPath := "bin/" + bsp.yov.Pkg.Name() + "/generated/link/mynewt.ld" + proj := interfaces.GetProject() + path, err := proj.ResolvePath(proj.Path(), defaultLinkerScriptPath) + if err != nil { + return "", err + } + return path, nil +} + +func (bsp *BspPackage) GetAutogeneratedLinkerIncludePath() (string, error) { + defaultLinkerScriptPath := "bin/" + bsp.yov.Pkg.Name() + "/generated/link/include" + proj := interfaces.GetProject() + path, err := proj.ResolvePath(proj.Path(), defaultLinkerScriptPath) + if err != nil { + return "", err + } + return path, nil +} + // Interprets a setting as either a single linker script or a list of linker // scripts. func (bsp *BspPackage) resolveLinkerScriptSetting( @@ -111,6 +131,22 @@ func (bsp *BspPackage) resolveLinkerScriptSetting( // Read each linker script from the list. for _, val := range vals { + if val == "autogenerated" { + if len(vals) > 1 { + return nil, util.NewNewtError("Both autogenerated and custom linker scripts cannot be used. " + + "Newt handles either autogenerated linker script or a list of custom linker scripts.") + } else { + path, err := bsp.getAutogeneratedLinkerScriptPath() + if err != nil { + return nil, util.PreNewtError(err, + "Could not resolve autogenerated linker script path for target \"%s\"", + bsp.yov.Pkg.Name()) + } + paths = append(paths, path) + continue + } + } + path, err := proj.ResolvePath(ypkg.Repo().Path(), val) if err != nil { return nil, util.PreNewtError(err, @@ -226,7 +262,6 @@ func NewBspPackage(lpkg *LocalPackage, yov *BspYCfgOverride) (*BspPackage, error lpkg.Load() bsp.LocalPackage = lpkg bsp.BspV = ycfg.NewYCfg(bsp.BspYamlPath()) - err := bsp.Reload(nil) return bsp, err diff --git a/newt/pkg/localpackage.go b/newt/pkg/localpackage.go index 344633be9..44d3208ba 100644 --- a/newt/pkg/localpackage.go +++ b/newt/pkg/localpackage.go @@ -59,6 +59,8 @@ type LocalPackage struct { // General information about the package desc *PackageDesc + linkTables []string + // Extra package-specific settings that don't come from syscfg. For // example, SELFTEST gets set when the newt test command is used. injectedSettings *cfgv.Settings @@ -145,6 +147,10 @@ func (pkg *LocalPackage) Desc() *PackageDesc { return pkg.desc } +func (pkg *LocalPackage) LinkTables() []string { + return pkg.linkTables +} + func (pkg *LocalPackage) SetName(name string) { pkg.name = name } @@ -197,6 +203,15 @@ func (pkg *LocalPackage) readDesc(yc ycfg.YCfg) (*PackageDesc, error) { return pdesc, nil } +func (pkg *LocalPackage) readLinkTables(yc ycfg.YCfg) ([]string, error) { + var err error + + sections, err := yc.GetValStringSlice("pkg.link_tables", nil) + util.OneTimeWarningError(err) + + return sections, nil +} + func (pkg *LocalPackage) sequenceString(key string) string { var buffer bytes.Buffer @@ -361,6 +376,8 @@ func (pkg *LocalPackage) Load() error { return err } + pkg.linkTables, err = pkg.readLinkTables(pkg.PkgY) + // Load syscfg settings. pkg.SyscfgY, err = config.ReadFile(pkg.SyscfgYamlPath()) if err != nil && !util.IsNotExist(err) { diff --git a/newt/project/project.go b/newt/project/project.go index 45ecaf327..b285e6644 100644 --- a/newt/project/project.go +++ b/newt/project/project.go @@ -44,6 +44,7 @@ import ( var globalProject *Project = nil const PROJECT_FILE_NAME = "project.yml" +const PATCHES_DIR = "patches" var ignoreSearchDirs []string = []string{ "bin", @@ -92,6 +93,15 @@ func initProject(dir string, download bool) error { if err != nil { return err } + + if download { + err = globalProject.UpgradeIf(newtutil.NewtForce, newtutil.NewtAsk, + func(r *repo.Repo) bool { return !r.IsExternal(r.Path()) }) + if err != nil { + return err + } + } + if err := globalProject.loadPackageList(); err != nil { return err } @@ -163,8 +173,16 @@ func NewProject(dir string, download bool) (*Project, error) { return proj, nil } -func (proj *Project) GetPkgRepos() error { +func (proj *Project) isRepoAdded(r *repo.Repo) bool { + for _, pr := range proj.repos { + if pr.Name() == r.Name() { + return true + } + } + return false +} +func (proj *Project) GetPkgRepos() error { for _, pkgList := range proj.packages { for _, pkg := range *pkgList { if pkg.PkgConfig().HasKey("repository") { @@ -190,10 +208,28 @@ func (proj *Project) GetPkgRepos() error { repoName, fields["vers"], err.Error()) } r.SetPkgName(pkg.Name()) - if err := proj.addRepo(r, true); err != nil { - return err + + if !proj.isRepoAdded(r) { + if err := proj.addRepo(r, true); err != nil { + return err + } + proj.rootRepoReqs[repoName] = verReq + } + + if _, err := os.Stat(pkg.BasePath() + "/" + PATCHES_DIR + "/" + r.Name()); os.IsNotExist(err) { + continue + } else { + dirEntries, err := os.ReadDir(pkg.BasePath() + "/" + PATCHES_DIR + "/" + r.Name()) + if err != nil { + return err + } + + for _, e := range dirEntries { + if strings.HasSuffix(e.Name(), ".patch") { + r.AddPatch(pkg.BasePath() + "/" + PATCHES_DIR + "/" + r.Name() + "/" + e.Name()) + } + } } - proj.rootRepoReqs[repoName] = verReq } } } @@ -202,6 +238,19 @@ func (proj *Project) GetPkgRepos() error { return nil } +func (proj *Project) SetGitEnvVariables() error { + err := os.Setenv("GIT_COMMITTER_NAME", "newt") + if err != nil { + return err + } + + err = os.Setenv("GIT_COMMITTER_EMAIL", "dev@mynewt.apache.org") + if err != nil { + return err + } + return nil +} + func (proj *Project) Path() string { return proj.BasePath } @@ -614,7 +663,7 @@ func (proj *Project) loadConfig(download bool) error { "%s (%s)", repoName, fields["vers"], err.Error()) } - + r.SetIsFromProjectYml() if err := proj.addRepo(r, download); err != nil { return err } diff --git a/newt/repo/repo.go b/newt/repo/repo.go index 9e624b736..791656714 100644 --- a/newt/repo/repo.go +++ b/newt/repo/repo.go @@ -44,6 +44,7 @@ const REPO_DEFAULT_PERMS = 0755 const REPO_FILE_NAME = "repository.yml" const REPOS_DIR = "repos" +const PATCHES_DIR = "patches" type Repo struct { name string @@ -66,8 +67,20 @@ type Repo struct { // version => commit vers map[newtutil.RepoVersion]string + // Since we are calling upgrade twice - first time for repos from project.yml + // and second time for repos from packages, we need to keep track on status + // of each repo. If repo is defined in project.yml we want to upgrade it only + // once so the version from project.yml stays valid. These flags are used for + // this purpose. + isFromProjectYml bool + isUpgradedFromProjectYml bool + hasSubmodules bool submodules []string + + // Used with external repos. If package that adds external repository provides patches for it, + // the paths to them are going to be stored here. + patches []string } type RepoDependency struct { @@ -116,6 +129,19 @@ func (r *Repo) Downloader() downloader.Downloader { return r.downloader } +func (r *Repo) AddPatch(path string) { + r.patches = append(r.patches, path) +} + +func (r *Repo) ApplyPatches() error { + if len(r.patches) == 0 { + return nil + } + + err := r.Downloader().ApplyPatches(r.Path(), r.patches) + return err +} + func (repo *Repo) FilteredSearchList( curPath string, searchedMap map[string]struct{}) ([]string, error) { @@ -201,6 +227,22 @@ func (r *Repo) patchesFilePath() string { "/.patches/" } +func (r *Repo) IsUpgradedFromProjectYml() bool { + return r.isUpgradedFromProjectYml +} + +func (r *Repo) SetIsUpgradedFromProjectYml() { + r.isUpgradedFromProjectYml = true +} + +func (r *Repo) IsFromProjectYml() bool { + return r.isFromProjectYml +} + +func (r *Repo) SetIsFromProjectYml() { + r.isFromProjectYml = true +} + // Checks for repository.yml file presence in specified repo folder. // If there is no such file, the repo in specified folder is external. func (r *Repo) IsExternal(dir string) bool { diff --git a/newt/repo/version.go b/newt/repo/version.go index 5f0329be0..b4aa09c5b 100644 --- a/newt/repo/version.go +++ b/newt/repo/version.go @@ -126,6 +126,15 @@ func (r *Repo) VersionIsValid(ver newtutil.RepoVersion) bool { } cs, _ := r.downloader.CommitsFor(r.Path(), ver.Commit) + + // Try to fetch commit if it was not found + if len(cs) <= 0 { + err := r.Downloader().FetchCommit(r.Path(), ver.Commit) + if err != nil { + return false + } + cs, _ = r.downloader.CommitsFor(r.Path(), ver.Commit) + } return len(cs) > 0 } diff --git a/newt/resolve/resolve.go b/newt/resolve/resolve.go index c04d33364..ef883de52 100644 --- a/newt/resolve/resolve.go +++ b/newt/resolve/resolve.go @@ -1358,10 +1358,27 @@ func (res *Resolution) DeprecatedWarning() []string { return res.Cfg.DeprecatedWarning() } -func (res *Resolution) ExperimentalWarning() []string { +func (res *Resolution) CfgExperimentalWarning() []string { return res.Cfg.ExperimentalWarning() } +func (res *Resolution) PkgExperimentalWarning() []string { + lines := []string{} + + for pkg := range res.LpkgRpkgMap { + experimental, err := pkg.PkgY.GetValBool("pkg.experimental", nil) + if err != nil { + log.Errorf("Internal error; Could not read package %s yml file", pkg.Name()) + } + if experimental { + lines = append(lines, + fmt.Sprintf("Use of experimental package %s", pkg.Name())) + } + } + + return lines +} + func LogTransientWarning(lpkg *pkg.LocalPackage) { if lpkg.Type() == pkg.PACKAGE_TYPE_TRANSIENT { log.Warnf("Transient package %s used, update configuration "+ diff --git a/newt/syscfg/syscfg.go b/newt/syscfg/syscfg.go index a9906685a..e7f62ff09 100644 --- a/newt/syscfg/syscfg.go +++ b/newt/syscfg/syscfg.go @@ -1599,7 +1599,19 @@ func writePackages(lpkgs []*pkg.LocalPackage, w io.Writer) { } } -func write(cfg Cfg, lpkgs []*pkg.LocalPackage, w io.Writer) { +func writeApis(apis []string, w io.Writer) { + for i, api := range apis { + apis[i] = cfgPkgIllegalChar.ReplaceAllLiteralString(api, "_") + } + sort.Strings(apis) + + fmt.Fprintf(w, "/*** Included APIs */\n") + for _, name := range apis { + fmt.Fprintf(w, "#define MYNEWT_API_%s 1\n", name) + } +} + +func write(cfg Cfg, lpkgs []*pkg.LocalPackage, apis []string, w io.Writer) { fmt.Fprintf(w, newtutil.GeneratedPreamble()) fmt.Fprintf(w, "#ifndef H_MYNEWT_SYSCFG_\n") @@ -1619,10 +1631,13 @@ func write(cfg Cfg, lpkgs []*pkg.LocalPackage, w io.Writer) { writePackages(lpkgs, w) fmt.Fprintf(w, "\n") + writeApis(apis, w) + fmt.Fprintf(w, "\n") + fmt.Fprintf(w, "#endif\n") } -func EnsureWritten(cfg Cfg, includeDir string, lpkgs []*pkg.LocalPackage) error { +func EnsureWritten(cfg Cfg, includeDir string, lpkgs []*pkg.LocalPackage, apis []string) error { // XXX: Detect these problems at error text generation time. if err := calcPriorities(cfg, CFG_SETTING_TYPE_TASK_PRIO, SYSCFG_TASK_PRIO_MAX, false); err != nil { @@ -1631,7 +1646,7 @@ func EnsureWritten(cfg Cfg, includeDir string, lpkgs []*pkg.LocalPackage) error } buf := bytes.Buffer{} - write(cfg, lpkgs, &buf) + write(cfg, lpkgs, apis, &buf) path := includeDir + "/" + HEADER_PATH diff --git a/newt/sysinit/sysinit.go b/newt/sysinit/sysinit.go index efa226aea..ee1caf3cf 100644 --- a/newt/sysinit/sysinit.go +++ b/newt/sysinit/sysinit.go @@ -133,12 +133,43 @@ func ResolveStageFuncsOrder(sfs []stage.StageFunc) ([]stage.StageFunc, error) { if len(sfRef.Stage.Befores) == 0 && len(sfRef.Stage.Afters) == 0 { stage, _ := sfRef.Stage.IntVal() nodesByStage[stage] = append(nodesByStage[stage], sfRef) - nodes = append(nodes, sfRef) } else { nodesQ = append(nodesQ, sfRef) } } + var stages []int + for stage := range nodesByStage { + stages = append(stages, stage) + } + sort.Ints(stages) + + // Lexicographically sort nodes in each stage, then build the nodes stack. + // This helps ensure that sysinit order is reproducable and deterministic + for stageIndex, _ := range stages { + lsfs := nodesByStage[stages[stageIndex]] + sort.Slice(lsfs, func(i int, j int) bool { + a := lsfs[i] + b := lsfs[j] + + if strings.Compare(a.Name, b.Name) == -1 { + return false + } + return true + }) + nodes = append(nodes, nodesByStage[stages[stageIndex]]...) + } + + sort.Slice(nodesQ, func(i int, j int) bool { + a := nodesQ[i] + b := nodesQ[j] + + if strings.Compare(a.Name, b.Name) == -1 { + return false + } + return true + }) + // Put nodes without stages first, so they are resolved and put to // stack first - we do not want them to precede all nodes with stages. // While technically correct, it's better not to start sysinit with @@ -146,12 +177,6 @@ func ResolveStageFuncsOrder(sfs []stage.StageFunc) ([]stage.StageFunc, error) { // before os init packages. nodes = append(nodesQ, nodes...) - var stages []int - for stage := range nodesByStage { - stages = append(stages, stage) - } - sort.Ints(stages) - // Add implicit dependencies for nodes with stages. We need to add // direct dependencies between each node of stage X to each node of // stage Y to make sure they can be resolved properly and reordered diff --git a/newt/toolchain/compiler.go b/newt/toolchain/compiler.go index c1a75c9b1..f84e16ed6 100644 --- a/newt/toolchain/compiler.go +++ b/newt/toolchain/compiler.go @@ -23,6 +23,7 @@ import ( "fmt" "io/ioutil" "mynewt.apache.org/newt/newt/cfgv" + "mynewt.apache.org/newt/newt/pkg" "os" "path" "path/filepath" @@ -70,8 +71,9 @@ type CompileCommand struct { } type Compiler struct { - objPathList map[string]bool - LinkerScripts []string + objPathList map[string]bool + LinkerScripts []string + AutogeneratedLinkerIncludeDir string // Needs to be locked whenever a mutable field in this struct is accessed // during a build. Currently, objPathList is the only such member. @@ -317,12 +319,50 @@ func loadFlags(yc ycfg.YCfg, settings *cfgv.Settings, key string, cfg *cfgv.Sett return flags } +func getConfigMap(compilerDir string) (map[string]string, error) { + dstMap := make(map[string]string) + + scfg, err := config.ReadFile(compilerDir + "/" + pkg.SYSCFG_YAML_FILENAME) + if err != nil { + return nil, err + } + + for _, setting := range scfg.AllSettings() { + aSetting, ok := setting.(map[interface{}]interface{}) + if ok { + for settingName, settingFields := range aSetting { + aSettingName := settingName.(string) + aSettingFields, ok := settingFields.(map[interface{}]interface{}) + if ok { + for settingField, fieldValue := range aSettingFields { + if settingField == "value" { + aFieldValue, ok := fieldValue.(string) + if ok { + dstMap[aSettingName] = aFieldValue + } + } + } + } + } + } + } + + return dstMap, nil +} + func (c *Compiler) load(compilerDir string, buildProfile string, cfg *cfgv.Settings) error { yc, err := config.ReadFile(compilerDir + "/" + COMPILER_FILENAME) if err != nil { return err } + if cfg == nil { + cfgMap, err := getConfigMap(compilerDir) + if err == nil { + cfg = cfgv.NewSettingsFromMap(cfgMap) + } + } + settings := cfgv.NewSettingsFromMap(map[string]string{ buildProfile: "1", strings.ToUpper(runtime.GOOS): "1", @@ -434,6 +474,15 @@ func (c *Compiler) includesStrings() []string { func (c *Compiler) cflagsStrings() []string { cflags := util.SortFields(c.info.Cflags...) + + for _, lclinfo_flag := range util.SortFields(c.lclInfo.Cflags...) { + for _, info_flag := range cflags { + if lclinfo_flag == info_flag { + continue + } + } + cflags = append(cflags, lclinfo_flag) + } return cflags } @@ -1012,6 +1061,9 @@ func (c *Compiler) CompileBinaryCmd(dstFile string, options map[string]bool, cmd = append(cmd, "-T") cmd = append(cmd, ls) } + cmd = append(cmd, "-L") + cmd = append(cmd, c.AutogeneratedLinkerIncludeDir) + if options["mapFile"] { cmd = append(cmd, "-Wl,-Map="+dstFile+".map") } @@ -1082,6 +1134,8 @@ func (c *Compiler) generateExtras(elfFilename string, ".bss.core", "-R", ".bss.core.nz", + "-R", + ".image_header", "-O", "binary", elfFilename,