Package Size Test [clean, No.5, @latest] #63
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Package Size Test | |
| run-name: Package Size Test [${{ inputs.CONFIG_BASE }}, No.${{ inputs.device_choice }}, @${{ inputs.commit_sha }}] | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| device_choice: | |
| description: '选择型号(device-env.sh中的编号)' | |
| required: false | |
| default: 5 | |
| type: number | |
| CONFIG_BASE: | |
| type: choice | |
| required: true | |
| description: '选择基准配置版本' | |
| default: 'clean' | |
| options: | |
| - clean | |
| - basic | |
| - func | |
| - test | |
| COMMIT_SHA: | |
| description: '使用指定的commit(可选)' | |
| required: false | |
| default: 'latest' | |
| use_cache: | |
| description: '尝试使用缓存的tool和toolchain' | |
| required: false | |
| default: true | |
| type: boolean | |
| TEST_PER_JOB_MAX: | |
| description: '单作业最大测试数(免费用户并行作业数上限为20)' | |
| required: false | |
| default: '10' | |
| env: | |
| # REPO_URL: https://github.com/openwrt/openwrt | |
| # REPO_BRANCH: master | |
| FEEDS_CONF: feeds.conf.default | |
| CONFIG_FILE: .config | |
| COMMIT_SHA: ${{ inputs.commit_sha }} | |
| DIY_P1_SH: diy-part1.sh | |
| # DIY_P2_SH: ${{ inputs.DIY_P2_FILE }} | |
| TZ: Asia/Shanghai | |
| # SEQ_FILE: ${{ inputs.SEQ_FILE }} | |
| jobs: | |
| init: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| DEVICE_TAG: ${{steps.job_tagging.outputs.DEVICE_TAG}} | |
| steps: | |
| - uses: actions/checkout@main | |
| - name: Load Environment Variable | |
| run: | | |
| chmod +x $GITHUB_WORKSPACE/device-env.sh | |
| $GITHUB_WORKSPACE/device-env.sh ${{ inputs.device_choice }} | |
| - name: job_tagging | |
| id: job_tagging | |
| run: | | |
| # notifications | |
| # echo "::set-output name=DEVICE_TAG::${DEVICE_TAG}" | |
| echo "DEVICE_TAG=${DEVICE_TAG}" >> $GITHUB_OUTPUT | |
| echo "::notice title=Target::[No.${{ inputs.device_choice }}]${DEVICE_TAG} with ${{ inputs.config_tag }}.config" | |
| # 若指定了 commit 提个醒 | |
| if [ "$USE_COMMIT_SHA" == "latest" ]; then | |
| LATEST_SHA=$(curl "https://api.github.com/repos/$REPO_USE/commits/$REPO_BRANCH" | grep sha | head -n1 | cut -d\" -f4) | |
| echo "::notice title=Using commit::[latest] https://github.com/$REPO_USE/tree/$LATEST_SHA" | |
| else | |
| echo "::notice title=Using commit::[specific] https://github.com/$REPO_USE/tree/$USE_COMMIT_SHA" | |
| fi | |
| Factory: | |
| runs-on: ubuntu-22.04 | |
| needs: init | |
| name: ${{needs.init.outputs.DEVICE_TAG}} | |
| outputs: | |
| matrix: ${{ steps.setup_worker_matrix.outputs.json_content }} | |
| TimeStamp: ${{ steps.info.outputs.TimeStamp }} | |
| steps: | |
| - uses: actions/checkout@main | |
| - name: Load Environment Variable | |
| id: load_env | |
| run: | | |
| chmod +x $GITHUB_WORKSPACE/device-env.sh | |
| $GITHUB_WORKSPACE/device-env.sh ${{ inputs.device_choice }} | |
| - name: List Test Sequence | |
| id: info | |
| run: | | |
| # cd $GITHUB_WORKSPACE | |
| # 时间戳，组成Artifact的文件名 | |
| date +"%Y%m%d%H%M" | |
| # echo "::set-output name=TimeStamp::$(date +"%Y%m%d-%H%M")" | |
| echo "TimeStamp=$(date +"%Y%m%d-%H%M")" >> $GITHUB_OUTPUT | |
| # 打印序列信息和目录 | |
| echo '====================序列内容====================' | |
| cat "${SEQ_FILE}" -An | |
| echo '====================测试目录====================' | |
| grep '@' "${SEQ_FILE}" -n | cat -n | |
| # 从DIY_P2_SH中截取型号信息 | |
| grep '^CONFIG_TARGET.*DEVICE.*=y' "${DIY_P2_SH}" | sed -r 's/.*DEVICE_(.*)=y/\1/' > DEVICE_NAME | |
| [ -s DEVICE_NAME ] && echo "::notice title=Device@.config:: $(cat DEVICE_NAME)" && echo "DEVICE_NAME=$(cat DEVICE_NAME)" >> $GITHUB_ENV | |
| # 分任务 | |
| - name: Assign Jobs(Setup Worker Matrix) | |
| id: setup_worker_matrix | |
| run: | | |
| # cd $GITHUB_WORKSPACE | |
| # 分析列表 | |
| lines_num=$(wc -l "${SEQ_FILE}" | cut -f1 -d' ') | |
| tests_num=$(grep '@' "${SEQ_FILE}" -c) | |
| title_line_num_list=($(grep '@' "${SEQ_FILE}" -n | cut -d: -f1)) | |
| # 输出分析结果 | |
| echo "测试序列：共${lines_num}行，包含${tests_num}条测试任务" | |
| echo '====================标题清单====================' | |
| echo '（序列格式： [测试序号]标题行行编号）' | |
| for i in $(seq ${tests_num}); do | |
| echo -n "[$i]${title_line_num_list[i-1]}p " | |
| done | |
| echo | |
| # 根据单作业最大测试数，计算总作业数及每个作业分配的测试数 | |
| TEST_PER_JOB_MAX=${{ inputs.TEST_PER_JOB_MAX }} | |
| if [ $(( tests_num % TEST_PER_JOB_MAX )) -eq 0 ]; then | |
| # 可平均分配 | |
| job_num=$((tests_num/TEST_PER_JOB_MAX)) | |
| tests_per_job=${TEST_PER_JOB_MAX} | |
| else | |
| # 非平均分配 | |
| job_num=$((tests_num/TEST_PER_JOB_MAX+1)) | |
| if [ $(( tests_num % job_num )) -eq 0 ]; then | |
| tests_per_job=$((tests_num/job_num)) | |
| else | |
| tests_per_job=$((tests_num/job_num+1)) | |
| fi | |
| fi | |
| spliter_seq=($(seq 1 ${tests_per_job} ${tests_num})) # job的起始test编号 | |
| spliter_seq[job_num]=$((tests_num+1)) # 为循环补充一个开点 | |
| # 生成job matrix，配置格式为json | 同时打印日志 | |
| # { | |
| # "include":[ | |
| # { | |
| # "Worker_ID":"1", | |
| # "Sed_Script":"1,11p", | |
| # "First_Job_No":"1" | |
| # },... | |
| # ] | |
| # } | |
| json_content='{"include":[' | |
| echo "${tests_num} 条测试任务分 ${job_num} 个作业并行，分工依次为：" | |
| for i in $(seq 1 ${job_num}); do | |
| job_no_from=${spliter_seq[i-1]} | |
| job_no_to=$((spliter_seq[i]-1)) | |
| seq_line_from=${title_line_num_list[job_no_from-1]} | |
| seq_line_to=$((title_line_num_list[job_no_to]-1)) | |
| [[ $seq_line_to -eq -1 ]] && seq_line_to=${lines_num} # 越界返回0，打个补丁 | |
| Sed_Script="${seq_line_from},${seq_line_to}p" | |
| echo "[Job $i]Test ${job_no_from}-${job_no_to}" "|" "行号：${Sed_Script}" | |
| json_content="${json_content}{\"Worker_ID\":\"${i}\",\"Sed_Script\":\"${Sed_Script}\",\"First_Job_No\":\"${job_no_from}\"}," | |
| done | |
| # 去掉最后一个逗号，然后封底 | |
| json_content=${json_content%?}']}' | |
| echo "[json_content] ${json_content}" | |
| # 输出到output作为matrix | |
| # echo "::set-output name=json_content::${json_content}" | |
| echo "json_content=${json_content}" >> $GITHUB_OUTPUT | |
| Workers: | |
| runs-on: ubuntu-22.04 | |
| needs: Factory | |
| strategy: | |
| matrix: ${{fromJSON(needs.Factory.outputs.matrix)}} | |
| env: | |
| Worker_ID: ${{ matrix.Worker_ID }} | |
| Sed_Script: ${{ matrix.Sed_Script }} | |
| First_Job_No: ${{ matrix.First_Job_No }} | |
| steps: | |
| - uses: actions/checkout@main | |
| - name: Load Environment Variable | |
| id: load_env | |
| run: | | |
| chmod +x $GITHUB_WORKSPACE/device-env.sh | |
| $GITHUB_WORKSPACE/device-env.sh ${{ inputs.device_choice }} | |
| - name: Read Job Sequence | |
| run: | | |
| # 依据作业分配到的行号，截取对应序列输出到文件 | |
| sed -n ${Sed_Script} "${SEQ_FILE}" > JobTest.list | |
| # 打印序列信息和目录 | |
| echo "====================作业${Worker_ID}测试序列(${Sed_Script})====================" | |
| cat JobTest.list -An | |
| echo "====================测试目录(计数器初始值为${First_Job_No})====================" | |
| grep '@' JobTest.list -n | cat -n | |
| - name: Initialize Environment | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| run: | | |
| sudo rm -rf /etc/apt/sources.list.d/* /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| sudo -E apt-get -qq update | |
| sudo -E apt-get -qq install $DEPENDS | |
| sudo -E apt-get -qq autoremove --purge | |
| sudo -E apt-get -qq clean | |
| # docker image prune -a -f | |
| sudo timedatectl set-timezone "$TZ" | |
| sudo mkdir -p /workdir | |
| sudo chown $USER:$GROUPS /workdir | |
| - name: Clone Source Code | |
| working-directory: /workdir | |
| run: | | |
| git clone https://github.com/$REPO_USE -b $REPO_BRANCH openwrt | |
| ln -sf /workdir/openwrt $GITHUB_WORKSPACE/openwrt | |
| # 若指定了 commit | |
| if [ "$USE_COMMIT_SHA" != "latest" ]; then | |
| cd openwrt | |
| git checkout "$USE_COMMIT_SHA" | |
| cd .. | |
| fi | |
| echo "[Check disk usage]" | |
| df -hT . | |
| echo "[Check space usage] $PWD" | |
| du -h --max-depth=2 | |
| - name: Cache | |
| uses: klever1988/cachewrtbuild@main | |
| if: ${{ inputs.use_cache }} | |
| with: | |
| ccache: 'false' | |
| prefix: ${{ github.workspace }}/openwrt | |
| - name: Load Custom Feeds | |
| run: | | |
| [ -e $FEEDS_CONF ] && mv $FEEDS_CONF openwrt/feeds.conf.default | |
| chmod +x $DIY_P1_SH | |
| cd openwrt | |
| $GITHUB_WORKSPACE/$DIY_P1_SH | |
| - name: Update & Install Feeds | |
| working-directory: /workdir/openwrt | |
| run: | | |
| ./scripts/feeds update -a | |
| ./scripts/feeds install -a | |
| echo "[Check disk usage]" | |
| df -hT . | |
| echo "[Check space usage] $PWD" | |
| du -h --max-depth=1 | |
| - name: Load Custom Configuration and Download Packages | |
| id: download | |
| run: | | |
| [ -e files ] && mv files openwrt/files | |
| # [ -e $CONFIG_FILE ] && mv $CONFIG_FILE openwrt/.config | |
| chmod +x "${DIY_P2_SH}" | |
| cd openwrt | |
| $GITHUB_WORKSPACE/"${DIY_P2_SH}" ${{ inputs.CONFIG_BASE }} # 生成配置基准(.config baseline) | |
| cp -v .config clean.config # 保留副本，每次编译都以此为起点 | |
| sed -e '/[@#]/d' $GITHUB_WORKSPACE/JobTest.list >> .config # 去掉标题(@开头)/不勾选(#开头)的行，增补到配置文件以下载全部内容 | |
| make defconfig | |
| # 下载包，如若出错生成警告 | |
| make download -j8 | tee make_download.log | |
| grep 'ERROR:' make_download.log | xargs -i echo "::warning:: [Download Try 1]" {} | |
| find dl -size -1024c -exec ls -l {} \; | |
| find dl -size -1024c -exec rm -f {} \; | |
| # 二次尝试 | |
| grep 'ERROR:' -q make_download.log && make download -j1 V=s | tee make_download.log | |
| grep 'ERROR:' make_download.log | xargs -i echo "::warning:: [Download Try 2]" {} | |
| find dl -size -1024c -exec ls -l {} \; | |
| find dl -size -1024c -exec rm -f {} \; | |
| echo "[Check disk usage]" | |
| df -hT . | |
| echo "[Check space usage] $PWD" | |
| du -h --max-depth=1 | |
| - name: Compile & Log Filesize | |
| id: compile | |
| working-directory: /workdir/openwrt | |
| run: | | |
| # 测试序列预处理，去除win换行符中的CR(\r)，否则上传artifacts时会出错 | |
| sed -i 's/\r//' "${GITHUB_WORKSPACE}/JobTest.list" | |
| # 按标题识别位点'@'切分测试序列，组合成一系列配置文件 | |
| (mkdir config_seq; cd config_seq; csplit --prefix=patch "${GITHUB_WORKSPACE}/JobTest.list" /@/ {*} -sz) | |
| # 创建产物目录 | |
| ARTIFACTS_DIR=$(mkdir ARTIFACTS_DIR; cd ARTIFACTS_DIR; pwd) | |
| echo "[ARTIFACTS_DIR]${ARTIFACTS_DIR}" | |
| # echo "::set-output name=ARTIFACTS_DIR::${ARTIFACTS_DIR}" | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> $GITHUB_OUTPUT | |
| # 后台记录CPU占用 | |
| mkdir ${ARTIFACTS_DIR}/vmstat_log | |
| vmstat 10 >> ${ARTIFACTS_DIR}/vmstat_log/worker_${Worker_ID}.log & | |
| # 依次执行任务列表 | |
| test_no=$((First_Job_No-1)) # 轮次计数器初始化，因为一开始就++所以先-1 | |
| for patch_file in config_seq/patch* | |
| do | |
| # 计时起点，启动打点计时器 | |
| start_time=$(date +%s) | |
| ( while true; do echo "...Timer.$[ $(date +%s)-${start_time} ]s" >&2 && sleep 20; done ) & | |
| TIMER_PID=$! | |
| title=$(head "${patch_file}" -n1) # 原汁原味的标题，保留所有特殊符号 | |
| echo "[Test No.$((++test_no))] $title, start at $(date +"%Y-%m-%d %H:%M")" | |
| cp -vf clean.config .config # 起点，重置为基准配置 | |
| echo "============[Test $test_no] Patch Content============" | |
| sed '/@/d' "${patch_file}" | tee -a .config # 去掉标题行，增写配置 | |
| echo "===============[Test $test_no] .config===============" | |
| bash "$GITHUB_WORKSPACE/conflict-clamer.sh" # 尝试解决一些已知的冲突项 | |
| cat .config | |
| echo "===============================================" | |
| # 给测试起个名：测试序号+标题，例：1@ Clean | |
| TEST_NAME="${test_no}${title}" | |
| # 测试成品存放目录 | |
| TEST_RESULT_DIR="${ARTIFACTS_DIR}/${TEST_NAME}" | |
| mkdir "${TEST_RESULT_DIR}" | |
| echo "[TEST_RESULT_DIR] ${TEST_RESULT_DIR}" | |
| # 测试日志文件 | |
| TEST_LOG_FILE="${TEST_RESULT_DIR}/${TEST_NAME}_compile.log" | |
| SIZE_LOG_FILE="${TEST_RESULT_DIR}/${TEST_NAME}_size.log" | |
| cp -v "$patch_file" "${TEST_RESULT_DIR}/config.patch" | |
| echo "[TEST_LOG_FILE] ${TEST_LOG_FILE}" | |
| # 补全默认值 | |
| make defconfig | |
| # 分析配置文件是否包含补丁 | |
| subset_check=$(comm -23 <(sed '/@/d' "${patch_file}" | sort | uniq) <(sort .config | uniq) | tee .notfound) | |
| if [[ -z $subset_check ]]; then | |
| echo 补丁是 .conifg 的子集 | |
| else | |
| echo 补丁不是 .conifg 的子集！以下 patch 行未在 .config 中找到 | |
| cat .notfound | |
| echo "=====================EOF=======================" | |
| echo "::warning title=Test ${test_no} ${title}:: Patch is not a subset of .config, please check..." | |
| fi | |
| # 执行编译 | |
| echo "[Test ${test_no}] Try $(($(nproc)+1)) thread compile first" | tee "$TEST_LOG_FILE" | |
| # try | |
| ( make -j $(($(nproc)+1)) || make -j1 || make -j1 V=s ) 2>&1 | tee -a "$TEST_LOG_FILE" | |
| # 根据第一节管道返回值判断编译结果 | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| # Exception | |
| echo "::error title=[${test_no} ${title}] Test Failed:: Error(s) occurred during compile, please check..." | |
| touch "${TEST_RESULT_DIR}/test_failed" | |
| # 保留配置文件 | |
| mv -v .config "${TEST_RESULT_DIR}/" | |
| else | |
| # no Exception | |
| echo "[Test ${test_no}] Compile success..." | |
| echo 'in path ' ./bin/targets/*/* | |
| ls -lah ./bin/targets/*/* | |
| echo "===============================================" | |
| # 截取bytes换算为MB（精度-两位小数），记录到文件 | |
| echo "[Test ${test_no}] Logging filesize..." | |
| du -ab --max-depth=1 ./bin/targets/*/* | awk '{printf "%8.2fM",$1/2^20; $1=""; print}' | tee "${SIZE_LOG_FILE}" | |
| # 复制保留有用文件 | |
| # ./bin/targets/ramips/mt7621/packages | |
| mv -v .config ./bin/targets/*/*/*.{buildinfo,manifest} "${TEST_RESULT_DIR}/" | |
| cp -urv --parents ./bin/targets/*/*/packages "${ARTIFACTS_DIR}" # 不移走会被覆盖/删除掉 | |
| # bin太大不留了，连带其校验码文件删除 | |
| echo "[Test ${test_no}] Cleaning for the next test..." | |
| rm -v ./bin/targets/*/*/*.bin ./bin/targets/*/*/sha256sums | |
| echo "[Test $test_no] --Finished--" $(date +"%Y-%m-%d %H:%M") | |
| fi | |
| # 计时终点，关闭打点计时器 | |
| kill -15 $TIMER_PID | |
| end_time=$(date +%s) | |
| cost_time=$[ ${end_time}-${start_time} ] | |
| echo "===============================================" | |
| echo -ne "Start Time:\t${start_time}\nEnd Time:\t${end_time}\nCost Time:\t${cost_time}\n" | tee "${TEST_RESULT_DIR}/Timer.log" | |
| echo "===============================================" | |
| done | |
| find ./bin/targets/*/* -maxdepth 1 -type f -exec rm -v {} \; | |
| # 保留所有的ipk | |
| cp -uvr ./bin "${ARTIFACTS_DIR}/" | |
| # 保留测试序列和DIY脚本2 | |
| cp -v "${GITHUB_WORKSPACE}/${SEQ_FILE}" ${GITHUB_WORKSPACE}/"${DIY_P2_SH}" "${ARTIFACTS_DIR}/" | |
| echo "[Check disk usage]" | |
| df -hT . | |
| echo "[Check space usage] $PWD" | |
| du -h --max-depth=1 | |
| echo "------------------------End of Job------------------------" | |
| - name: Upload Artifact Directory | |
| uses: actions/upload-artifact@main | |
| # if: steps.compile.conclusion == 'success' | |
| with: | |
| name: SizeTest_${{ inputs.CONFIG_BASE }}_${{ needs.Factory.outputs.TimeStamp }} | |
| path: ${{ steps.compile.outputs.ARTIFACTS_DIR }} | |
| - name: SSH Debug | |
| uses: mxschmitt/action-tmate@v3 | |
| with: | |
| limit-access-to-actor: false | |
| if: ${{ failure() }} | |
| delete-workflow-run: | |
| runs-on: ubuntu-latest | |
| needs: Workers | |
| if: ${{ always() }} | |
| steps: | |
| - uses: Mattraks/delete-workflow-runs@main | |
| with: | |
| retain_days: 30 | |
| keep_minimum_runs: 5 |