Merge remote-tracking branch 'upstream/master'

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -161,8 +161,8 @@ stages:
         git branch -u remote/$BRANCH_NAME
 
         echo $GIT_PASSWORD | gh auth login --with-token
-        TITLE="Upgrade SONiC Versions"
-        BODY="Upgrade SONiC Versions"
+        TITLE="[${SOURCE_BRANCH#refs/heads/}] Upgrade SONiC package Versions"
+        BODY=$TITLE
         RET=0
         if ! gh pr create -t "$TITLE" -b "$BODY" -B $(Build.SourceBranch) -R $(Build.Repository.Name) > pr.log 2>&1; then
           if ! grep -q "already exists" pr.log; then

.azure-pipelines/azure-pipelines-build.yml

@@ -35,7 +35,6 @@ jobs:
       dbg_image: no
       asan_image: no
       swi_image: no
-      raw_image: no
       docker_syncd_rpc_image: no
       syncd_rpc_image: no
       platform_rpc: no
@@ -62,7 +61,6 @@ jobs:
           variables:
             dbg_image: yes
             swi_image: yes
-            raw_image: yes
             docker_syncd_rpc_image: yes
             platform_rpc: brcm
 
@@ -149,9 +147,6 @@ jobs:
             if [ $(swi_image) == yes ]; then
               make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
             fi
-            if [ $(raw_image) == yes ]; then
-              make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
-            fi
             if [ $(docker_syncd_rpc_image) == yes ]; then
               # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz

.azure-pipelines/build-template.yml

@@ -101,9 +101,6 @@ jobs:
         if [ ${{ parameters.swi_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-${{ parameters.platform }}.swi
         fi
-        if [ ${{ parameters.raw_image }} == true ]; then
-          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.raw
-        fi
         if [ ${{ parameters.sync_rpc_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
           # workaround for issue in rules/sairedis.dep,  git ls-files will list un-exist files for cache

.azure-pipelines/official-build-cache.yml

@@ -39,7 +39,6 @@ stages:
         - name: broadcom
           variables:
             swi_image: yes
-            raw_image: yes
             docker_syncd_rpc_image: yes
             platform_rpc: brcm
         - name: mellanox

.github/workflows/automerge_scan.yml

@@ -16,58 +16,74 @@ jobs:
         set -e
 
         echo ${TOKEN} | gh auth login --with-token
-        gh pr list -R sonic-net/sonic-buildimage -A mssonicbld --json additions,assignees,author,baseRefName,body,changedFiles,closed,closedAt,comments,commits,createdAt,deletions,files,headRefName,headRepository,headRepositoryOwner,id,isCrossRepository,isDraft,labels,latestReviews,maintainerCanModify,mergeCommit,mergeStateStatus,mergeable,mergedAt,mergedBy,milestone,number,potentialMergeCommit,projectCards,reactionGroups,reviewDecision,reviewRequests,reviews,state,statusCheckRollup,title,updatedAt,url > prs.log
+        gh pr list -R sonic-net/sonic-buildimage -A mssonicbld -L 100 -S "label:automerge" --json url,body,title,createdAt,labels,statusCheckRollup > prs.log
         cat prs.log | jq
     - name: Main
       run: |
         set -e
 
+        # PR merge run per 2 hours
+        # Other operation run per day.
+        # Cherry pick PR:
+        # more than 3 days, comment @author to check
+        # more than 10 days, stop comment.
+        # more than 28 days, comment @author PR will be closed
+        # more than 30 days, close PR
+        date_3d_ago=$(date --date "3 day ago" -u +"%FT%TZ")
+        date_10d_ago=$(date --date "10 day ago" -u +"%FT%TZ")
+        date_28d_ago=$(date --date "28 day ago" -u +"%FT%TZ")
+        date_30d_ago=$(date --date "30 day ago" -u +"%FT%TZ")
+        date_now=$(date -u +"%T")
+        operate=false
+        [[ "$date_now" < "02:00:00" ]] && operate=true
+
         count=$(cat prs.log | jq 'length')
         for ((i=0;i<$count;i++))
         do
           url=$(cat prs.log | jq -r ".[$i].url")
           body=$(cat prs.log | jq -r ".[$i].body")
           title=$(cat prs.log | jq -r ".[$i].title")
+          origin_pr_id=$(echo $title | grep -Eo "\[action\] \[PR:[0-9]*\]" | grep -Eo [0-9]* || true)
           created_at=$(cat prs.log | jq -r ".[$i].createdAt")
-          echo PR: $(($i+1))/$count, URL: $url, createdAt: $created_at, now: $(date -u +"%FT%TZ")
+          echo PR: $(($i+1))/$count, URL: $url, origin PR: $origin_pr_id, createdAt: $created_at, operate: $operate
           [[ "$url" == "" ]] && continue
           [[ $created_at > $(date --date "1 hour ago" -u +"%FT%TZ") ]] && continue
-          # only check automerge PR.
-          cat prs.log | jq -r ".[$i].labels[].name" | grep automerge || continue
 
           checks=$(cat prs.log | jq ".[$i].statusCheckRollup")
           checks_count=$(echo $checks | jq 'length')
-          echo Checks count: $checks_count
+          pr_success=true
           for ((j=0;j<$checks_count;j++))
           do
             check=$(echo $checks | jq ".[$j]")
-            state=$(echo $check | jq -r '.state')
+            status=$(echo $check | jq -r '.status')
             conclusion=$(echo $check | jq -r '.conclusion')
             name=$(echo $check | jq -r '.name')
 
             # EasyCLA success flag:  state=SUCCESS
             # Others success flag:   conclusion in SUCCESS,NEUTRAL
-            # Ignore Azure.sonic-buildimage stage check result. It may be set continueOnError
-            echo "$name" | grep "Azure.sonic-buildimage (" && continue
-            # rerun Azure.sonic-buildimage per day
-            if [[ "$name" == "Azure.sonic-buildimage" ]] && [[ "$conclusion" == "FAILURE" ]];then
-              completedAt=$(echo $check | jq -r '.completedAt')
-              [[ "$completedAt" < $(date --date "2 hour ago" -u +"%FT%TZ") ]] && [[ $(date -u +"%T") < "02:00:00" ]] && gh pr comment $url --body "/azp run Azure.sonic-buildimage"
-            fi
-            # Ignore Semgrep, it has issues.
-            [[ "$name" == "Semgrep" ]] && continue
-            if [[ "$state" == "SUCCESS" ]];then
-              # check pass
-              continue
-            elif [[ "$conclusion" == "SUCCESS" ]] || [[ "$conclusion" == "NEUTRAL" ]];then
-              # check pass
-              continue
-            else
-              echo "$url Check failed!!!"
-              echo $check | jq
-              continue 2
-            fi
+            # only check Azure.sonic-buildimage currently
+            echo "$name" | grep -v "Azure.sonic-buildimage" > /dev/null && continue
+            [[ "$status" != "COMPLETED" ]] && echo "$name: $status" && continue 2
+
+            success=true
+            ( [[ "$conclusion" == "FAILURE" ]] || [[ "$conclusion" == "CANCELLED" ]] ) && success=false && pr_success=false
+            ! $success && echo "FAIL: $name"
           done
+
+          # rerun Azure.sonic-buildimage per day
+          ! $pr_success && $operate && gh pr comment $url --body "/azp run Azure.sonic-buildimage"
+
+          # If auto cherry pick PRs failed, comment in original PR and close cherry pick PR
+          if [ -n "$origin_pr_id" ] && [[ $created_at < $date_3d_ago ]] && ! $pr_success;then
+            origin_pr_url=https://github.com/sonic-net/sonic-buildimage/pull/$origin_pr_id
+            author=$(gh pr view $origin_pr_url --json author | jq .author.login -r)
+            echo "Original author will check."
+            $operate && [[ $created_at > $date_10d_ago ]] && gh pr comment $origin_pr_url --body "@$author cherry pick PR didn't pass PR checker. Please check!!!<br>$url"
+            $operate && [[ $created_at < $date_28d_ago ]] && gh pr comment $origin_pr_url --body "@$author cherry pick PR didn't pass PR checker. Please check!!! Auto cherry pick PR will be closed in 2 days.<br>$url"
+            $operate && [[ $created_at < $date_30d_ago ]] && echo "$url Closed" && gh pr close $url
+          fi
+
+          ! $pr_success && continue
           # merge the PR
           echo ========Merging  PR========
           if echo $title | grep "^\[submodule\]";then
@@ -77,3 +93,4 @@ jobs:
           fi
           echo ========Finished PR========
         done
+

.gitignore

@@ -105,3 +105,6 @@ htmlcov/
 # Debian mirror Sources
 sources.list.*
 !sources.list*.j2
+
+# Generated mirror configs
+apt-retries-count

Makefile.cache

@@ -564,10 +564,12 @@ ALL_DEP_FILES_LIST += $(foreach pkg,$(2), $($(filter none,$($(1)_CACHE_MODE)), \
                       $(addsuffix .$(3).sha,$(addprefix $(pkg)/, $(1)))))
 $(foreach docker, $(filter $(SONIC_DOCKER_IMAGES), $(1)), \
 	$(eval $(docker)_DEP_FILES+=$(wildcard files/build/versions/default/*) \
-	$(wildcard files/build/versions/dockers/$(basename $(docker))/*)))
+	$(wildcard files/build/versions/dockers/$(basename $(docker))/*) \
+	$(foreach docker_file, $($(docker)_FILES), $(addprefix $(if $($(docker_file)_PATH), $($(docker_file)_PATH), $(FILES_PATH))/, $(docker_file))) ))
 $(foreach docker, $(filter $(SONIC_DOCKER_DBG_IMAGES), $(1)), \
 	$(eval $(docker)_DEP_FILES+=$(wildcard files/build/versions/default/*) \
-	$(wildcard files/build/versions/dockers/$(patsubst %-$(DBG_IMAGE_MARK).gz,%,$(docker))/*)))
+	$(wildcard files/build/versions/dockers/$(patsubst %-$(DBG_IMAGE_MARK).gz,%,$(docker))/*) \
+	$(foreach docker_file, $($(docker)_FILES), $(addprefix $(if $($(docker_file)_PATH), $($(docker_file)_PATH), $(FILES_PATH))/, $(docker_file))) ))
 $(addsuffix .$(3),$(addprefix $(2)/, $(1))) : $(2)/%.$(3) : \
 	$(2)/%.flags $$$$($$$$*_DEP_FILES) $$$$(if $$$$($$$$*_SMDEP_FILES), $(2)/%.smdep)
 	@$$(eval $$*_DEP_FILES_MODIFIED := $$? )

Makefile.work

@@ -537,6 +537,7 @@ SONIC_BUILD_INSTRUCTION :=  $(MAKE) \
                            SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD=$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) \
                            SONIC_INCLUDE_SYSTEM_TELEMETRY=$(INCLUDE_SYSTEM_TELEMETRY) \
                            INCLUDE_DHCP_RELAY=$(INCLUDE_DHCP_RELAY) \
+                           INCLUDE_DHCP_SERVER=$(INCLUDE_DHCP_SERVER) \
                            INCLUDE_MACSEC=$(INCLUDE_MACSEC) \
                            SONIC_INCLUDE_RESTAPI=$(INCLUDE_RESTAPI) \
                            SONIC_INCLUDE_MUX=$(INCLUDE_MUX) \

README.md

@@ -353,6 +353,10 @@ on the SAI version for each SONiC release.
     [managing, configuring and monitoring SONiC](https://github.com/sonic-net/sonic-mgmt)
     (gzip tar archive)
 
+## SONiC Image Azure Pipelines
+
+All SONiC project build pipeline could be found at [Download Portal for SONiC Images](https://sonic-build.azurewebsites.net)
+
 ## Contribution Guide
 
 All contributors must sign a contribution license agreement before contributions

azure-pipelines.yml

@@ -116,12 +116,16 @@ stages:
         artifact: sonic-buildimage.vs
       displayName: "Download sonic-buildimage.vs artifact"
 
+    - script: |
+        sudo src/sonic-swss-common/.azure-pipelines/build_and_install_module.sh
+      displayName: "Install kernel modules"
+
     - script: |
         sudo apt-get update
         sudo apt-get install -y make libtool m4 autoconf dh-exec debhelper cmake pkg-config \
-                         libhiredis-dev libnl-3-dev libnl-genl-3-dev libnl-route-3-dev libnl-nf-3-dev swig3.0 \
+                         libhiredis-dev libnl-3-dev libnl-genl-3-dev libnl-route-3-dev libnl-nf-3-dev swig \
                          libpython2.7-dev libboost-dev libboost-serialization-dev uuid-dev libzmq5 libzmq3-dev  python3-pip \
-                         cmake libgtest-dev libgmock-dev libyang-dev
+                         cmake libgtest-dev libgmock-dev libyang-dev nlohmann-json3-dev
         sudo pip3 install pytest
         cd src/sonic-swss-common
         ./autogen.sh

build_debian.sh

@@ -33,7 +33,7 @@ CONFIGURED_ARCH=$([ -f .arch ] && cat .arch || echo amd64)
 ## docker engine version (with platform)
 DOCKER_VERSION=5:24.0.2-1~debian.11~$IMAGE_DISTRO
 CONTAINERD_IO_VERSION=1.6.21-1
-LINUX_KERNEL_VERSION=5.10.0-18-2
+LINUX_KERNEL_VERSION=5.10.0-23-2
 
 ## Working directory to prepare the file system
 FILESYSTEM_ROOT=./fsroot
@@ -113,7 +113,8 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount
 ## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates
 scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO
 sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list
-sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until,apt-multiple-retries} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
+sudo cp files/apt/apt-retries-count $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
+sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
 
 ## Note: set lang to prevent locale warnings in your chroot
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y update
@@ -296,7 +297,7 @@ then
     install_kubernetes ${MASTER_KUBERNETES_VERSION}
 
     sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update
-    sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install hyperv-daemons gnupg xmlstarlet
+    sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install hyperv-daemons gnupg xmlstarlet parted
     sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove gnupg
     sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/cri-dockerd.deb -fsSL \
         https://github.com/Mirantis/cri-dockerd/releases/download/v${MASTER_CRI_DOCKERD}/cri-dockerd_${MASTER_CRI_DOCKERD}.3-0.debian-${IMAGE_DISTRO}_amd64.deb
@@ -407,6 +408,10 @@ LogsDirectory=audit
 LogsDirectoryMode=0750
 EOF
 
+# latest tcpdump control resource access with AppArmor.
+# override tcpdump profile to allow tcpdump access TACACS config file.
+sudo cp files/apparmor/usr.bin.tcpdump $FILESYSTEM_ROOT/etc/apparmor.d/local/usr.bin.tcpdump
+
 if [[ $CONFIGURED_ARCH == amd64 ]]; then
 ## Pre-install the fundamental packages for amd64 (x86)
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install      \

build_image.sh

@@ -139,23 +139,19 @@ elif [ "$IMAGE_TYPE" = "raw" ]; then
     ## Run the installer
     ## The 'build' install mode of the installer is used to generate this dump.
     sudo chmod a+x $tmp_output_onie_image
-    sudo ./$tmp_output_onie_image
+    sudo ./$tmp_output_onie_image || {
+        ## Failure during 'build' install mode of the installer results in an incomplete raw image.
+        ## Delete the incomplete raw image.
+        sudo rm -f $OUTPUT_RAW_IMAGE
+    }
     rm $tmp_output_onie_image
 
     [ -r $OUTPUT_RAW_IMAGE ] || {
         echo "Error : $OUTPUT_RAW_IMAGE not generated!"
         exit 1
     }
 
-    $GZ_COMPRESS_PROGRAM $OUTPUT_RAW_IMAGE
-
-    [ -r $OUTPUT_RAW_IMAGE.gz ] || {
-        echo "Error : $GZ_COMPRESS_PROGRAM $OUTPUT_RAW_IMAGE failed!"
-        exit 1
-    }
-
-    mv $OUTPUT_RAW_IMAGE.gz $OUTPUT_RAW_IMAGE
-    echo "The compressed raw image is in $OUTPUT_RAW_IMAGE"
+    echo "The raw image is in $OUTPUT_RAW_IMAGE"
 
 elif [ "$IMAGE_TYPE" = "kvm" ]; then
 

device/arista/x86_64-arista_7060dx5_32/Arista-7060DX5-32-200Gx48-100Gx32/hwsku.json

@@ -0,0 +1,104 @@
+{
+    "interfaces": {
+        "Ethernet0": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet8": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet16": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet24": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet32": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet40": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet48": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet56": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet64": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet72": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet80": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet88": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet96": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet104": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet112": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet120": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet128": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet136": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet144": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet152": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet160": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet168": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet176": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet184": {
+            "default_brkout_mode": "2x200G[100G,50G,40G,25G,10G]"
+        },
+        "Ethernet192": {
+            "default_brkout_mode": "4x100G[50G,40G,25G,10G]"
+        },
+        "Ethernet200": {
+            "default_brkout_mode": "4x100G[50G,40G,25G,10G]"
+        },
+        "Ethernet208": {
+            "default_brkout_mode": "4x100G[50G,40G,25G,10G]"
+        },
+        "Ethernet216": {
+            "default_brkout_mode": "4x100G[50G,40G,25G,10G]"
+        },
+        "Ethernet224": {
+            "default_brkout_mode": "4x100G[50G,40G,25G,10G]"
+        },
+        "Ethernet232": {
+            "default_brkout_mode": "4x100G[50G,40G,25G,10G]"
+        },
+        "Ethernet240": {
+            "default_brkout_mode": "4x100G[50G,40G,25G,10G]"
+        },
+        "Ethernet248": {
+            "default_brkout_mode": "4x100G[50G,40G,25G,10G]"
+        },
+        "Ethernet256": {
+            "default_brkout_mode": "1x10G",
+            "fec": "none"
+        }
+    }
+}

device/arista/x86_64-arista_7060dx5_32/Arista-7060DX5-32-200Gx48-100Gx32/sai.profile

@@ -0,0 +1 @@
+SAI_INIT_CONFIG_FILE=/usr/share/sonic/hwsku/th4-a7060dx5-32-200Gx48-100Gx32.config.bcm