Merged PR 6004325: Manual merge github master to internal

Related work items: #49, #58, #107, sonic-net#247, sonic-net#249, sonic-net#277, sonic-net#593, sonic-net#597, sonic-net#1035, sonic-net#2130, sonic-net#2150, sonic-net#2165, sonic-net#2169, sonic-net#2178, sonic-net#2179, sonic-net#2187, sonic-net#2188, sonic-net#2191, sonic-net#2195, sonic-net#2197, sonic-net#2198, sonic-net#2200, sonic-net#2202, sonic-net#2206, sonic-net#2209, sonic-net#2211, sonic-net#2216, sonic-net#7909, sonic-net#8927, sonic-net#9681, sonic-net#9733, sonic-net#9746, sonic-net#9850, sonic-net#9967, sonic-net#10104, sonic-net#10152, sonic-net#10168, sonic-net#10228, sonic-net#10266, sonic-net#10288, sonic-net#10294, sonic-net#10313, sonic-net#10394, sonic-net#10403, sonic-net#10404, sonic-net#10421, sonic-net#10431, sonic-net#10437, sonic-net#10445, sonic-net#10457, sonic-net#10458, sonic-net#10465, sonic-net#10467, sonic-net#10469, sonic-net#10470, sonic-net#10474, sonic-net#10477, sonic-net#10478, sonic-net#10482, sonic-net#10485, sonic-net#10488, sonic-net#10489, sonic-net#10492, sonic-net#10494, sonic-net#10498, sonic-net#10501, sonic-net#10509, sonic-net#10512, sonic-net#10514, sonic-net#10516, sonic-net#10517, sonic-net#10523, sonic-net#10525, sonic-net#10531, sonic-net#10532, sonic-net#10538, sonic-net#10555, sonic-net#10557, sonic-net#10559, sonic-net#10561, sonic-net#10565, sonic-net#10572, sonic-net#10574, sonic-net#10576, sonic-net#10578, sonic-net#10581, sonic-net#10585, sonic-net#10587, sonic-net#10599, sonic-net#10607, sonic-net#10611, sonic-net#10616, sonic-net#10618, sonic-net#10619, sonic-net#10623, sonic-net#10624, sonic-net#10633, sonic-net#10646, sonic-net#10655, sonic-net#10660, sonic-net#10664, sonic-net#10680, sonic-net#10683

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -38,7 +38,6 @@ parameters:
   - centec
   - centec-arm64
   - generic
-  - innovium
   - marvell-armhf
   - mellanox
 

.azure-pipelines/azure-pipelines-build.yml

@@ -50,6 +50,7 @@ jobs:
             swi_image: yes
 
         - name: broadcom
+          timeoutInMinutes: 1440
           variables:
             dbg_image: yes
             swi_image: yes
@@ -131,3 +132,4 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
+      - template: check-dirty-version.yml

.azure-pipelines/azure-pipelines-download-certificate.yml

@@ -0,0 +1,33 @@
+parameters:
+- name: connectionName
+  type: string
+  default: sonic-dev-connection
+- name: kevaultName
+  type: string
+  default: sonic-kv
+- name: certificateName
+  type: string
+  default: sonic-secure-boot
+
+steps:
+- task: AzureKeyVault@2
+  inputs:
+    connectedServiceName: ${{ parameters.connectionName }}
+    keyVaultName: ${{ parameters.kevaultName }}
+    secretsFilter: ${{ parameters.certificateName }}
+
+- script: |
+    set -e
+    TMP_FILE=$(mktemp)
+    echo "$CERTIFICATE" | base64 -d > $TMP_FILE
+    sudo mkdir -p /etc/certificates
+    mkdir -p $(Build.StagingDirectory)/target
+    # Save the public key
+    openssl pkcs12 -in $TMP_FILE -clcerts --nokeys -nodes -passin pass: | sed -z -e "s/.*\(-----BEGIN CERTIFICATE\)/\1/" > $(SIGNING_CERT)
+    # Save the private key
+    openssl pkcs12 -in $TMP_FILE -nocerts -nodes -passin pass: | sed -z -e "s/.*\(-----BEGIN PRIVATE KEY\)/\1/" | sudo tee $(SIGNING_KEY) 1>/dev/null
+    ls -lt $(SIGNING_CERT) $(SIGNING_KEY)
+    rm $TMP_FILE
+  env:
+    CERTIFICATE: $(${{ parameters.certificateName }})
+  displayName: "Save certificate"

.azure-pipelines/check-dirty-version.yml

@@ -0,0 +1,16 @@
+steps:
+- script: |
+    . functions.sh
+    SONIC_VERSION=$(sonic_get_version)
+    echo "SONIC_VERSION=$SONIC_VERSION"
+    if [[ "$SONIC_VERSION" == *dirty* ]]; then
+      # Print the detail dirty info
+      git status --untracked-files=no -s --ignore-submodules
+
+      # Exit with error, if it is a PR build
+      if [ "$(Build.Reason)" == "PullRequest" ]; then
+        echo "Build failed for the dirty version: $SONIC_VERSION" 1>&2
+        exit 1
+      fi
+    fi
+  displayName: "Check the dirty version"

.azure-pipelines/docker-sonic-slave-template.yml

@@ -101,10 +101,13 @@ jobs:
       containerRegistry: ${{ parameters.registry_conn }}
       repository: $(VARIABLE_SLAVE_BASE_IMAGE)
       command: push
-      tags: |
-        $(VARIABLE_SLAVE_BASE_TAG)
-        ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
+      ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
+        tags: |
+          $(VARIABLE_SLAVE_BASE_TAG)
           latest
+      ${{ else }}:
+        tags: |
+          $(VARIABLE_SLAVE_BASE_TAG)
   - ${{ if ne(parameters.arch, 'amd64') }}:
     - task: Docker@2
       condition: ne(variables['Build.Reason'], 'PullRequest')

.azure-pipelines/docker-sonic-slave.yml

@@ -60,7 +60,7 @@ stages:
   - ${{ each dist in parameters.dists }}:
     - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
       - ${{ each arch in parameters.arches }}:
-        - ${{ if eq(variables['System.PullRequest.TargetBranch'], 'master') }}:
+        - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
           - template: docker-sonic-slave-template.yml
             parameters:
               pool: sonicbld

.azure-pipelines/official-build-cisco-8000.yml

@@ -22,10 +22,17 @@ resources:
     name: Cisco-8000-sonic/platform-cisco-8000
     endpoint: cisco-connection
 
+
 variables:
 - group: SONIC-AKV-STROAGE-1
 - name: StorageSASKey
   value: $(sonicstorage-SasToken)
+- name: SONIC_ENABLE_SECUREBOOT_SIGNATURE
+  value: y
+- name: SIGNING_KEY
+  value: /etc/certificates/sonic-secure-boot-private.pem
+- name: SIGNING_CERT
+  value: $(Build.StagingDirectory)/target/sonic-secure-boot-public.pem
 
 stages:
 - stage: Build
@@ -41,6 +48,7 @@ stages:
     parameters:
       buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) ${{ variables.VERSION_CONTROL_OPTIONS }}'
       preSteps:
+      - template: azure-pipelines-download-certificate.yml
       - checkout: self
         submodules: recursive
         path: s
@@ -90,5 +98,10 @@ stages:
           StorageSASKey: $(StorageSASKey)
         condition: ne(variables['Build.Reason'], 'PullRequest')
         displayName: "Override cisco sai packages"
+      - script: |
+          echo "SONIC_ENABLE_SECUREBOOT_SIGNATURE := y" >> rules/config.user
+          echo "SIGNING_KEY := $(SIGNING_KEY)" >> rules/config.user
+          echo "SIGNING_CERT := $(SIGNING_CERT)" >> rules/config.user
+        displayName: "Enable secure boot signature"
       jobGroups:
       - name: cisco-8000

.azure-pipelines/template-commonlib.yml

@@ -2,33 +2,39 @@ jobs:
 - job: Build
   timeoutInMinutes: 120
   pool: sonicbld
+  variables:
+  - template: template-variables.yml
   steps:
   - checkout: self
     clean: true
     submodules: recursive
   - script: |
       set -ex
-      case $(Build.SourceBranchName) in
-        202012 | 202106)
-          bldenv=buster
-          ;;
-        *)
-          bldenv=bullseye
-          ;;
-      esac
-      BLDENV=$bldenv make -f Makefile.work configure PLATFORM=vs ENABLE_DOCKER_BASE_PULL=y
-      echo "##vso[task.setvariable variable=bldenv;]$bldenv"
+      branch=$(Build.SourceBranchName)
+      # DIST_MASTER is set in variable.
+      BRANCH=DIST_${branch^^}
+      bldenvs=${!BRANCH}
+      [ "$bldenvs" == "" ] && bldenvs="$(COMMON_LIB_BUILD_ENVS)"
+      for bldenv in $bldenvs
+      do
+        BLDENV=$bldenv make -f Makefile.work configure PLATFORM=vs ENABLE_DOCKER_BASE_PULL=y
+      done
+      set +x
+      echo "##vso[task.setvariable variable=bldenvs;]$bldenvs"
     displayName: Make configure
   - script: |
       set -ex
-      LIBNL3_VERSION_BASE=$(grep "LIBNL3_VERSION_BASE =" rules/libnl3.mk | awk '{print$3}')
-      LIBNL3_VERSION=$(grep "LIBNL3_VERSION =" rules/libnl3.mk | awk '{print$3}' | sed -e "s/(//" -e "s/)//" -e "s/\\$//" -e "s/LIBNL3_VERSION_BASE/$LIBNL3_VERSION_BASE/")
-      BLDENV=$(bldenv) make -f Makefile.work target/debs/$(bldenv)/libnl-3-200_${LIBNL3_VERSION}_amd64.deb ENABLE_DOCKER_BASE_PULL=y
+      for bldenv in $(bldenvs)
+      do
+        LIBNL3_VERSION_BASE=$(grep "LIBNL3_VERSION_BASE =" rules/libnl3.mk | awk '{print$3}')
+        LIBNL3_VERSION=$(grep "LIBNL3_VERSION =" rules/libnl3.mk | awk '{print$3}' | sed -e "s/(//" -e "s/)//" -e "s/\\$//" -e "s/LIBNL3_VERSION_BASE/$LIBNL3_VERSION_BASE/")
+        SONIC_BUILD_JOBS=$(nproc) BLDENV=$bldenv make -f Makefile.work target/debs/$bldenv/libnl-3-200_${LIBNL3_VERSION}_amd64.deb ENABLE_DOCKER_BASE_PULL=y
 
-      LIBYANG_VERSION_BASE=$(grep "LIBYANG_VERSION_BASE =" rules/libyang.mk | awk '{print$3}')
-      LIBYANG_VERSION=$(grep "LIBYANG_VERSION =" rules/libyang.mk | awk '{print$3}' | sed -e "s/\\$//" -e "s/(//" -e "s/)//" -e "s/LIBYANG_VERSION_BASE/$LIBYANG_VERSION_BASE/")
-      BLDENV=$(bldenv) make -f Makefile.work target/debs/$(bldenv)/libyang_${LIBYANG_VERSION}_amd64.deb
-      find target -name *.deb | xargs -i cp {} $(Build.ArtifactStagingDirectory)
+        LIBYANG_VERSION_BASE=$(grep "LIBYANG_VERSION_BASE =" rules/libyang.mk | awk '{print$3}')
+        LIBYANG_VERSION=$(grep "LIBYANG_VERSION =" rules/libyang.mk | awk '{print$3}' | sed -e "s/\\$//" -e "s/(//" -e "s/)//" -e "s/LIBYANG_VERSION_BASE/$LIBYANG_VERSION_BASE/")
+        SONIC_BUILD_JOBS=$(nproc) BLDENV=$bldenv make -f Makefile.work target/debs/$bldenv/libyang_${LIBYANG_VERSION}_amd64.deb
+      done
+      mv target $(Build.ArtifactStagingDirectory)
     displayName: Make common lib packages
   - publish: $(Build.ArtifactStagingDirectory)
     artifact: common-lib

.azure-pipelines/template-variables.yml

@@ -1,2 +1,3 @@
 variables:
   DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io'
+  COMMON_LIB_BUILD_ENVS: 'bullseye'

.gitmodules

@@ -62,6 +62,9 @@
 [submodule "src/redis-dump-load"]
 	path = src/redis-dump-load
 	url = https://github.com/p/redis-dump-load.git
+[submodule "src/scapy"]
+	path = src/scapy
+	url = https://github.com/secdev/scapy.git
 [submodule "platform/mellanox/mlnx-sai/SAI-Implementation"]
 	path = platform/mellanox/mlnx-sai/SAI-Implementation
 	url = https://github.com/Mellanox/SAI-Implementation
@@ -103,6 +106,6 @@
 [submodule "src/sonic-p4rt/sonic-pins"]
 	path = src/sonic-p4rt/sonic-pins
 	url = https://github.com/Azure/sonic-pins.git
-[submodule "src/thrift_0_14_1/thrift"]
-	path = src/thrift_0_14_1/thrift
-	url = https://github.com/apache/thrift.git
+[submodule "src/ptf-py3"]
+	path = src/ptf-py3
+	url = https://github.com/p4lang/ptf.git

Makefile.work

@@ -188,6 +188,17 @@ ifneq ($(SONIC_DPKG_CACHE_SOURCE),)
 	DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache:rw"
 endif
 
+ifeq ($(SONIC_ENABLE_SECUREBOOT_SIGNATURE), y)
+ifneq ($(SIGNING_KEY),)
+	DOCKER_SIGNING_SOURCE := $(shell dirname $(SIGNING_KEY))
+	DOCKER_RUN += -v "$(DOCKER_SIGNING_SOURCE):$(DOCKER_SIGNING_SOURCE):ro"
+endif
+ifneq ($(SIGNING_CERT),)
+	DOCKER_SIGNING_SOURCE := $(shell dirname $(SIGNING_CERT))
+	DOCKER_RUN += -v "$(DOCKER_SIGNING_SOURCE):$(DOCKER_SIGNING_SOURCE):ro"
+endif
+endif
+
 ifeq ($(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD), y)
 ifneq ($(MULTIARCH_QEMU_ENVIRON), y)
     DOCKER_RUN += -v /var/run/docker.sock:/var/run/docker.sock
@@ -297,6 +308,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \
                            SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
+                           SONIC_ENABLE_SECUREBOOT_SIGNATURE=$(SONIC_ENABLE_SECUREBOOT_SIGNATURE) \
                            SONIC_DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
                            ENABLE_HOST_SERVICE_ON_START=$(ENABLE_HOST_SERVICE_ON_START) \
                            SLAVE_DIR=$(SLAVE_DIR) \

build_debian.sh

@@ -113,6 +113,12 @@ sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-language
 ## Note: set lang to prevent locale warnings in your chroot
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y update
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y upgrade
+
+echo '[INFO] Install and setup eatmydata'
+sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install eatmydata
+sudo LANG=C chroot $FILESYSTEM_ROOT ln -s /usr/bin/eatmydata /usr/local/bin/dpkg
+echo 'Dir::Bin::dpkg "/usr/local/bin/dpkg";' | sudo tee $FILESYSTEM_ROOT/etc/apt/apt.conf.d/00image-install-eatmydata > /dev/null
+
 echo '[INFO] Install packages for building image'
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install makedev psmisc
 
@@ -143,6 +149,23 @@ if [[ $CONFIGURED_ARCH == amd64 ]]; then
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install dmidecode hdparm
 fi
 
+## Sign the Linux kernel
+if [ "$SONIC_ENABLE_SECUREBOOT_SIGNATURE" = "y" ]; then
+    if [ ! -f $SIGNING_KEY ]; then
+       echo "Error: SONiC linux kernel signing key missing"
+       exit 1
+    fi
+    if [ ! -f $SIGNING_CERT ]; then
+       echo "Error: SONiC linux kernel signing certificate missing"
+       exit 1
+    fi
+
+    echo '[INFO] Signing SONiC linux kernel image'
+    K=$FILESYSTEM_ROOT/boot/vmlinuz-${LINUX_KERNEL_VERSION}-amd64
+    sbsign --key $SIGNING_KEY --cert $SIGNING_CERT --output /tmp/${K##*/} ${K}
+    sudo cp -f /tmp/${K##*/} ${K}
+fi
+
 ## Update initramfs for booting with squashfs+overlay
 cat files/initramfs-tools/modules | sudo tee -a $FILESYSTEM_ROOT/etc/initramfs-tools/modules > /dev/null
 
@@ -397,7 +420,8 @@ fi
 ## Remove sshd host keys, and will regenerate on first sshd start
 sudo rm -f $FILESYSTEM_ROOT/etc/ssh/ssh_host_*_key*
 sudo cp files/sshd/host-ssh-keygen.sh $FILESYSTEM_ROOT/usr/local/bin/
-sudo cp -f files/sshd/sshd.service $FILESYSTEM_ROOT/lib/systemd/system/ssh.service
+sudo mkdir $FILESYSTEM_ROOT/etc/systemd/system/ssh.service.d
+sudo cp files/sshd/override.conf $FILESYSTEM_ROOT/etc/systemd/system/ssh.service.d/override.conf
 # Config sshd
 # 1. Set 'UseDNS' to 'no'
 # 2. Configure sshd to close all SSH connetions after 15 minutes of inactivity
@@ -460,7 +484,7 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'setup
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'wheel==0.35.1'
 
 # docker Python API package is needed by Ansible docker module as well as some SONiC applications
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'docker==4.3.1'
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'docker==5.0.3'
 
 # Install scapy
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'scapy==2.4.4'
@@ -586,6 +610,10 @@ scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
 # Remove GCC
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove gcc
 
+# Remove eatmydata
+sudo rm $FILESYSTEM_ROOT/etc/apt/apt.conf.d/00image-install-eatmydata $FILESYSTEM_ROOT/usr/local/bin/dpkg
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove eatmydata
+
 ## Clean up apt
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y autoremove
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get autoclean

device/accton/x86_64-accton_as9716_32d-r0/pddf/pddf-device.json

@@ -53,7 +53,8 @@
         ],
         "custom_kos":
         [
-            "pddf_custom_psu"
+            "pddf_custom_psu",
+            "accton_as9716_32d_ioport"
         ]
     },
 

device/arista/x86_64-arista_7050cx3_32s/Arista-7050CX3-32S-C32/td3-a7050cx3-32s-32x100G.config.bcm

@@ -6,6 +6,7 @@ sai_trap_group_priority=1000
 use_all_splithorizon_groups=1
 riot_enable=1
 sai_tunnel_support=1
+sai_tunnel_underlay_route_mode=1
 riot_overlay_l3_intf_mem_size=4096
 riot_overlay_l3_egress_mem_size=32768
 l3_ecmp_levels=2
@@ -52,7 +53,7 @@ stable_size=0x5500000
 tdma_timeout_usec=15000000
 tslam_timeout_usec=15000000
 sai_optimized_mmu=1
-mmu_init_config="TD3-MSFT-T0-100G"
+mmu_init_config="TD3-MSFT-CUSTOM"
 buf.map.egress_pool0.ingress_pool=0
 buf.map.egress_pool1.ingress_pool=0
 buf.map.egress_pool2.ingress_pool=1