Merge branch 'sonic-net:master' into reserved_grpc_port

.azure-pipelines/azure-pipelines-Official.yml

@@ -14,7 +14,7 @@ schedules:
     - 202012
   always: true
 
-pool: sonicbld
+pool: sonicbld-1es
 
 stages:
 - stage: Build

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -26,28 +26,50 @@ resources:
     ref: master
     endpoint: sonic-net
 
-pool: sonicbld
+pool: sonicbld-1es
 
 parameters:
 - name: 'jobFilters'
   type: object
   default:
   - vs
-  - barefoot
   - broadcom
   - centec
   - centec-arm64
   - generic
   - marvell-armhf
+  - marvell-arm64
   - mellanox
+  - pensando
+  - nvidia-bluefield
 
 stages:
+- stage: Prepare
+  jobs:
+  - job: Prepare
+    steps:
+    - script: |
+        DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
+        DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp)
+        DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp)
+        echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_TIMESTAMP;isOutput=true]$DEBIAN_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_SECURITY_TIMESTAMP;isOutput=true]$DEBIAN_SECURITY_TIMESTAMP"
+      name: SetVersions
+      displayName: 'Set snapshot versions'
 - stage: Build
+  dependsOn: Prepare
   variables:
     - name: CACHE_MODE
       value: none
     - name: VERSION_CONTROL_OPTIONS
       value: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - name: SKIP_CHECKOUT
+      value: true
+    - name: DEBIAN_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_TIMESTAMP'] ]
+    - name: DEBIAN_SECURITY_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_SECURITY_TIMESTAMP'] ]
     - template: .azure-pipelines/template-variables.yml@buildimage
   jobs:
   - template: azure-pipelines-build.yml
@@ -56,6 +78,21 @@ stages:
       buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       preSteps:
       - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
+      - checkout: self
+        submodules: recursive
+        fetchDepth: 0
+        path: s
+        displayName: 'Checkout code'
+      - script: |
+          echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+          if [ "$MIRROR_SNAPSHOT" == y ]; then
+            mkdir -p target/versions/default/
+            echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror
+            echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror
+            cat target/versions/default/versions-mirror
+          fi
+        displayName: 'Set snapshot versions'
+
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions
@@ -81,7 +118,7 @@ stages:
         echo "artifacts$artifacts"
         cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/target/versions target/
         make freeze FREEZE_VERSION_OPTIONS=-r
-        find files/build/versions 
+        find files/build/versions
         ordered_artifacts=$(echo "$artifacts" | grep -v -E "arm64|armhf" && echo "$artifacts" | grep -E "arm64|armhf")
         for artifact in $ordered_artifacts
         do
@@ -126,8 +163,8 @@ stages:
         git branch -u remote/$BRANCH_NAME
 
         echo $GIT_PASSWORD | gh auth login --with-token
-        TITLE="Upgrade SONiC Versions"
-        BODY="Upgrade SONiC Versions"
+        TITLE="[${SOURCE_BRANCH#refs/heads/}] Upgrade SONiC package Versions"
+        BODY=$TITLE
         RET=0
         if ! gh pr create -t "$TITLE" -b "$BODY" -B $(Build.SourceBranch) -R $(Build.Repository.Name) > pr.log 2>&1; then
           if ! grep -q "already exists" pr.log; then
@@ -143,4 +180,4 @@ stages:
     - publish: $(System.DefaultWorkingDirectory)/files/build/versions
       artifact: 'sonic-buildimage.versions'
       displayName: 'Archive SONiC versions'
-    
+

.azure-pipelines/azure-pipelines-build-ubuntu-2004.yml

@@ -0,0 +1,67 @@
+# C/C++ with GCC
+# Build your C/C++ project with GCC using make.
+# Add steps that publish test results, save build artifacts, deploy, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/apps/c-cpp/gcc
+pr: none
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+    - 202???
+
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily build
+  branches:
+    include:
+      - master
+      - 202???
+  always: true
+
+stages:
+- stage: Build
+
+  jobs:
+  - job:
+    displayName: "amd64/ubuntu-20.04"
+    pool:
+      vmImage: 'ubuntu-20.04'
+
+    steps:
+    - checkout: self
+      submodules: true
+    - script: |
+        sudo apt-get update
+        sudo apt-get install -y make wget libtool m4 autoconf dh-exec libdebhelper-perl=13.6ubuntu1~bpo20.04.1 debhelper=13.6ubuntu1~bpo20.04.1 \
+                         cmake pkg-config python3-pip python cmake libgtest-dev libgmock-dev libyang-dev \
+                         debhelper-compat dh-elpa dh-sequence-python3 python3-all \
+                         libpython3-all-dev python3-six xmlto unzip rake-compiler gem2deb pkg-php-tools \
+                         ant default-jdk maven-repo-helper libguava-java \
+                         libboost-all-dev libgtest-dev build-essential swig4.0 swig
+        sudo pip3 install pytest click
+        wget http://ftp.us.debian.org/debian/pool/main/libg/libgoogle-gson-java/libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
+        sudo dpkg -i libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
+        mkdir -p /tmp/artifacts
+      displayName: "Install dependencies"
+    - script: |
+        BLDENV=bullseye SONIC_CONFIG_MAKE_JOBS=$(nproc) CONFIGURED_ARCH=amd64 DEST=/tmp/artifacts make -f ../rules/protobuf.mk -f protobuf/Makefile
+      workingDirectory: src
+      displayName: "Build protobuf"
+    - script: |
+        sudo dpkg -i protobuf-compiler_3.21.12-3_amd64.deb libprotoc32_3.21.12-3_amd64.deb \
+                      libprotobuf32_3.21.12-3_amd64.deb libprotobuf-dev_3.21.12-3_amd64.deb \
+                      libprotobuf-lite32_3.21.12-3_amd64.deb
+      workingDirectory: /tmp/artifacts
+      displayName: "Install protobuf"
+    - script: |
+        dpkg-buildpackage -rfakeroot -b -us -uc
+      workingDirectory: src/sonic-dash-api
+      displayName: "Build sonic-dash-api"
+    - script: |
+        cp *.deb /tmp/artifacts
+      workingDirectory: src
+    - publish: /tmp/artifacts
+      artifact: sonic-buildimage.amd64.ubuntu20_04
+      displayName: "Archive sonic-buildimage debian packages for ubuntu20.04"

.azure-pipelines/azure-pipelines-build.yml

@@ -35,7 +35,6 @@ jobs:
       dbg_image: no
       asan_image: no
       swi_image: no
-      raw_image: no
       docker_syncd_rpc_image: no
       syncd_rpc_image: no
       platform_rpc: no
@@ -62,7 +61,6 @@ jobs:
           variables:
             dbg_image: yes
             swi_image: yes
-            raw_image: yes
             docker_syncd_rpc_image: yes
             platform_rpc: brcm
 
@@ -92,8 +90,19 @@ jobs:
             pool: sonicbld-armhf
           timeoutInMinutes: 2880
           variables:
+            PLATFORM_NAME: marvell
             PLATFORM_ARCH: armhf
 
+        - name: marvell-arm64
+          ${{ if not(parameters.qemuOrCrossBuild) }}:
+            pool: sonicbld-arm64
+          timeoutInMinutes: 2880
+          variables:
+            PLATFORM_NAME: marvell
+            PLATFORM_ARCH: arm64
+
+        - name: marvell
+
         - name: mellanox
           variables:
             dbg_image: yes
@@ -107,11 +116,24 @@ jobs:
             docker_syncd_rpc_image: yes
             platform_rpc: nephos
 
+        - name: pensando
+          pool: sonicbld-arm64
+          variables:
+            PLATFORM_ARCH: arm64
+
+        - name: nvidia-bluefield
+          pool: sonicbld-arm64
+          variables:
+            PLATFORM_ARCH: arm64
+
     buildSteps:
-      - template: template-skipvstest.yml
+      - template: .azure-pipelines/template-skipvstest.yml@buildimage
+      - template: .azure-pipelines/template-daemon.yml@buildimage
       - bash: |
           set -ex
-          if [ $(GROUP_NAME) == vs ]; then
+          if [ $(GROUP_NAME) == pensando ]; then
+            make $BUILD_OPTIONS target/sonic-pensando.tar
+          elif [ $(GROUP_NAME) == vs ]; then
             if [ $(dbg_image) == yes ]; then
               make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz
               mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
@@ -120,11 +142,19 @@ jobs:
               make $BUILD_OPTIONS ENABLE_ASAN=y target/docker-sonic-vs.gz
               mv target/docker-sonic-vs.gz target/docker-sonic-vs-asan.gz
             fi
+            if [ "$(K8S_MASTER_CHANGED)" == 'YES' ]; then
+              make $BUILD_OPTIONS INCLUDE_KUBERNETES_MASTER=y target/sonic-vs.img.gz
+              gzip -d target/sonic-vs.img.gz
+              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make $BUILD_OPTIONS sonic-slave-run
+              gzip target/sonic-vs.vhdx
+              mv target/sonic-vs.vhdx.gz target/sonic-vs-k8s.vhdx.gz
+              rm target/sonic-vs.img
+            fi
             make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
             make $BUILD_OPTIONS target/docker-ptf-sai.gz
             if [ $(Build.Reason) != 'PullRequest' ];then
               gzip -kd target/sonic-vs.img.gz
-              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make sonic-slave-run
+              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make $BUILD_OPTIONS sonic-slave-run
               rm target/sonic-vs.img
             fi
           else
@@ -135,9 +165,6 @@ jobs:
             if [ $(swi_image) == yes ]; then
               make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
             fi
-            if [ $(raw_image) == yes ]; then
-              make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
-            fi
             if [ $(docker_syncd_rpc_image) == yes ]; then
               # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
@@ -148,10 +175,6 @@ jobs:
                 make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-dnx-rpc.gz
                 pushd ./src/sonic-sairedis/SAI
                 git stash
-                popd 
-                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
-                pushd ./src/sonic-sairedis/SAI
-                git stash
                 popd
               fi
             fi
@@ -162,4 +185,4 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
-      - template: check-dirty-version.yml
+      - template: .azure-pipelines/check-dirty-version.yml@buildimage

.azure-pipelines/azure-pipelines-image-template.yml

@@ -25,14 +25,17 @@ jobs:
     jobFilters: ${{ parameters.jobFilters }}
     jobVariables: ${{ parameters.jobVariables }}
     preSteps:
-      - template: cleanup.yml
+      - template: .azure-pipelines/cleanup.yml@buildimage
       - ${{ parameters.preSteps }}
       - script: |
-          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then
+          [ -n "$OVERRIDE_BUILD_OPTIONS" ] && OVERRIDE_BUILD_OPTIONS=$(OVERRIDE_BUILD_OPTIONS)
+          BUILD_OPTIONS="$(BUILD_OPTIONS) $OVERRIDE_BUILD_OPTIONS"
+          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf|marvell-arm64)$"; then
             CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
-            BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
-            echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
+            BUILD_OPTIONS="$BUILD_OPTIONS $CACHE_OPTIONS"
           fi
+          echo $BUILD_OPTIONS
+          echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
         displayName: "Set cache options"
       - checkout: self
         submodules: recursive
@@ -46,7 +49,8 @@ jobs:
           sudo modprobe overlay
           sudo apt-get install -y acl
           sudo bash -c "echo 1 > /proc/sys/vm/compact_memory"
-          ENABLE_DOCKER_BASE_PULL=y make PLATFORM=$(PLATFORM_AZP) PLATFORM_ARCH=$(PLATFORM_ARCH) $(BUILD_OPTIONS) configure
+          [ -z "$PLATFORM_NAME" ] && PLATFORM_NAME=$(PLATFORM_AZP)
+          ENABLE_DOCKER_BASE_PULL=y make PLATFORM=$PLATFORM_NAME PLATFORM_ARCH=$(PLATFORM_ARCH) $(BUILD_OPTIONS) configure
         displayName: 'Make configure'
     postSteps:
       - script: |
@@ -67,6 +71,6 @@ jobs:
             artifactName: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
             publishPrefix: '$(Build.DefinitionName)/$(Build.SourceBranchName)/$(GROUP_NAME)'
       - ${{ parameters.postSteps }}
-      - template: cleanup.yml
+      - template: .azure-pipelines/cleanup.yml@buildimage
     jobGroups: ${{ parameters.jobGroups }}
     buildSteps: ${{ parameters.buildSteps }}

.azure-pipelines/azure-pipelines-repd-build-variables.yml

@@ -1,7 +1,2 @@
 variables:
-  ${{ if eq(variables['Build.Reason'],'PullRequest') }}:
-    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=$([[ "$(System.PullRequest.TargetBranch)" =~ ^20[2-9][0-9]{3}$ ]] && echo deb,py2,py3,web,git,docker)'
-  ${{ elseif ne(variables['Build.SourceBranchName'],'master') }}:
-    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
-  ${{ else }}:
-    VERSION_CONTROL_OPTIONS: ''
+  VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'

.azure-pipelines/build-template.yml

@@ -35,9 +35,9 @@ parameters:
 - name: pool
   type: string
   values:
-  - sonicbld
+  - sonicbld-1es
   - sonicbld_8c
-  default: sonicbld
+  default: sonicbld-1es
 
 - name: dbg_image
   type: boolean
@@ -101,9 +101,6 @@ jobs:
         if [ ${{ parameters.swi_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-${{ parameters.platform }}.swi
         fi
-        if [ ${{ parameters.raw_image }} == true ]; then
-          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.raw
-        fi
         if [ ${{ parameters.sync_rpc_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
           # workaround for issue in rules/sairedis.dep,  git ls-files will list un-exist files for cache
@@ -116,6 +113,12 @@ jobs:
             git stash
             popd 
           fi
+          if [ ${{ parameters.platform }} == barefoot ]; then
+            make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) SAITHRIFT_V2=y  ENABLE_SYNCD_RPC=y  target/docker-saiserverv2-bfn.gz
+            pushd ./src/sonic-sairedis/SAI
+            git stash
+            popd 
+          fi
         fi
 
         make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin

.azure-pipelines/cleanup.yml

@@ -1,5 +1,11 @@
 steps:
 - script: |
+    set -x
+    # kill daemon process
+    ps $(cat /tmp/azp_daemon_kill_docker_pid)
+    sudo kill $(cat /tmp/azp_daemon_kill_docker_pid)
+    rm /tmp/azp_daemon_kill_docker_pid
+
     if sudo [ -f /var/run/march/docker.pid ] ; then
         pid=`sudo cat /var/run/march/docker.pid` ; sudo kill $pid
     fi
@@ -11,4 +17,5 @@ steps:
         pid=`sudo cat dockerfs/var/run/docker.pid` ; sudo kill $pid
     fi
     sudo rm -rf $(ls -A1)
+  condition: always()
   displayName: "Clean Workspace"