-
Notifications
You must be signed in to change notification settings - Fork 5
/
package-diff
executable file
·232 lines (217 loc) · 9.68 KB
/
package-diff
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
#!/bin/bash
if [ $# -lt 2 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
echo "Usage: $0 FLATCAR_VERSION_A FLATCAR_VERSION_B"
echo "Shows the ebuild package changes between two Flatcar versions"
echo "Environment variables:"
echo "Set FROM_(A|B)=(release|bincache|file://.../) to select an other image location than the bucket (note that file://.../ should be the directory where VERSION_. is appended, you may use '.' for the version)"
echo "Set BOARD_(A|B)=arm64-usr to select an arm64 build (ignored if FROM_.=file...)"
echo "Set CHANNEL_(A|B)=(alpha|beta|lts|developer) to select a build for another channel than stable (ignored for FROM_.=bincache|file...)"
echo "Set FILE=(flatcar_production_image_contents.txt|flatcar_production_image_initrd_contents.txt|flatcar_developer_container_packages.txt|flatcar_developer_container_contents.txt|flatcar_production_image_kernel_config.txt)"
echo " to show image contents or developer container packages instead of flatcar_production_image_packages.txt"
echo " NOTE: for {flatcar_production_image,flatcar_developer_container}_contents.txt and flatcar_production_image_initd_contents.txt"
echo " a wtd will be tried out first ({flatcar_production_image,flatcar_developer_container}_contents_wtd.txt and flatcar_production_image_initd_contents_wtd.txt)"
echo " if WITHNEWSTUFF env var is set to 1 (the default)"
echo "Set MODE_(A|B)=/developer/ to select a developer build (only for FROM_.=bucket)"
echo "Set FILESONLY=1 to reduce the flatcar_production_image_contents.txt file to contain only path information"
echo "Set CUTKERNEL=1 to reduce the flatcar_production_image_contents.txt file to contain no kernel version in paths but just 'a.b.c-flatcar'"
echo "Alternatively, set CALCSIZE=1 to sum up the file sizes from flatcar_production_image_contents.txt (/boot and /usr, excluding symlinks and directories)"
echo " If WITHNEWSTUFF is set to 1 (the default), this will try fetching the disk usage reports from the server instead"
echo " If this fails, falls back to the old method."
exit 1
fi
set -euo pipefail
FROM_A="${FROM_A-bucket}"
FROM_B="${FROM_B-bucket}"
CHANNEL_A="${CHANNEL_A-stable}"
CHANNEL_B="${CHANNEL_B-stable}"
BOARD_A="${BOARD_A-amd64-usr}"
BOARD_B="${BOARD_B-amd64-usr}"
MODE_A="${MODE_A-/}"
MODE_B="${MODE_B-/}"
FILE="${FILE-flatcar_production_image_packages.txt}"
VERSION_A="$1"
VERSION_B="$2"
FILESONLY="${FILESONLY-0}"
CUTKERNEL="${CUTKERNEL-0}"
CALCSIZE="${CALCSIZE-0}"
WITHNEWSTUFF="${WITHNEWSTUFF-1}"
WTD=0
NEWCALCSIZE=0
A="$(mktemp "/tmp/$VERSION_A-XXXXXX")"
B="$(mktemp "/tmp/$VERSION_B-XXXXXX")"
trap "rm -f \"$A\" \"$B\"" EXIT
function download {
local file="${1}"; shift
if [ "$FROM_A" = "release" ]; then
URL_A="https://${CHANNEL_A}.release.flatcar-linux.net/${BOARD_A}/${VERSION_A}/${file}"
elif [ "$FROM_A" = "bincache" ]; then
URL_A="https://bincache.flatcar-linux.net/images/${BOARD_A/-usr/}/${VERSION_A}/${file}"
elif echo "$FROM_A" | grep -q '^file'; then
URL_A="${FROM_A}/${VERSION_A}/${file}"
else
URL_A="https://bucket.release.flatcar-linux.net/flatcar-jenkins${MODE_A}${CHANNEL_A}/boards/${BOARD_A}/${VERSION_A}/${file}"
fi
if [ "$FROM_B" = "release" ]; then
URL_B="https://${CHANNEL_B}.release.flatcar-linux.net/${BOARD_B}/${VERSION_B}/${file}"
elif [ "$FROM_B" = "bincache" ]; then
URL_B="https://bincache.flatcar-linux.net/images/${BOARD_B/-usr/}/${VERSION_B}/${file}"
elif echo "$FROM_B" | grep -q '^file'; then
URL_B="${FROM_B}/${VERSION_B}/${file}"
else
URL_B="https://bucket.release.flatcar-linux.net/flatcar-jenkins${MODE_B}${CHANNEL_B}/boards/${BOARD_B}/${VERSION_B}/${file}"
fi
curl --location --silent -S -o "$A" "$URL_A"
curl --location --silent -S -o "$B" "$URL_B"
if [[ ! -s "${A}" ]] || [[ ! -s "${B}" ]]; then
return 1
fi
return 0
}
if [[ "${WITHNEWSTUFF}" = '1' ]]; then
if [[ "${FILE}" = *_contents.txt ]] && \
[[ "${CALCSIZE}" = 1 ]]; then
if download "${FILE%_*}_disk_usage.txt"; then
NEWCALCSIZE=1
else
echo "Disk usage files missing, falling back to the contents files"
fi
fi
if [[ "${NEWCALCSIZE}" = '0' ]]; then
if [[ "${FILE}" = *_contents.txt ]]; then
if download "${FILE%.*}_wtd.txt"; then
WTD=1
else
echo "WTD variants of the contents files missing, falling back to old files"
fi
fi
fi
fi
if [[ "${NEWCALCSIZE}" = '0' ]] && [[ "${WTD}" = '0' ]]; then
if ! download "${FILE}"; then
echo "Failed to download contents files"
exit 1
fi
fi
function no_hardlinks {
local f="${1}"; shift
# cache contains entries in form of either <hardlink_count>:<size>
# or <device_id>:<inode>, if such a key exists in cache then the
# line should be ignored for size accounting
local -A cache
cache=()
# ignore directories and symlinks
#
# for each non-ignored line generate a key for the cache; if the
# key is not in cache, we print the original line and insert the
# key into cache, otherwise we ignore the line - each hardlink is
# accounted for just once
#
# hardlink count of 1 is a special case - we always print the line
# and don't bother with adding it to cache
#
# handy, since git seems to be a hardlink farm - it has around 140
# binaries being actually hardlinks to a single 3mb file
#
# the form of the cache key depends on the file we process, if
# it's the one with device ids and inodes, then the key is
# <device_id>:<inode>, otherwise it's heuristical
# <hardlink_count>:<size>
if [[ "${WTD}" = '1' ]]; then
# awk fields:
# 0 - whole line
# 1 - permissions
# 2 - device id
# 3 - inode
# 4 - hardlink count
# 5 - size
# 6 - path
local -a lines
local line tuple clear_line device_id pair inode hardlink key
mapfile -t lines < <(grep -v '^[dl]' "${f}" | awk '{print $2 "-" $3 ":" $4 "@" $0}')
for line in "${lines[@]}"; do
tuple="${line%%@*}"
clear_line="${line#*@}"
device_id="${tuple%-*}"
pair="${tuple#*-}"
inode="${pair%:*}"
hardlink="${pair#*:}"
if [[ "${hardlink}" -eq 1 ]]; then
echo "${clear_line}"
continue
fi
key="${device_id}:${inode}"
if [[ -n "${cache[${key}]:+isset}" ]]; then
continue
fi
cache[${key}]=x
echo "${clear_line}"
done
else
# awk fields:
# 0 - whole line
# 1 - permissions
# 2 - hardlink count
# 3 - user
# 4 - greoup
# 5 - size
# 6 - path
local -a lines
local line pair clear_line hardlink size key
mapfile -t lines < <(grep -v '^[dl]' "${f}" | awk '{print $2 ":" $5 "@" $0}')
for line in "${lines[@]}"; do
pair="${line%%@*}"
clear_line="${line#*@}"
hardlink="${pair%:*}"
if [[ "${hardlink}" -eq 1 ]]; then
echo "${clear_line}"
continue
fi
size="${pair#*:}"
key="${hardlink}:${size}"
if [[ -n "${cache[${key}]:+isset}" ]]; then
continue
fi
cache[${key}]=x
echo "${clear_line}"
done
fi
}
if [[ "${NEWCALCSIZE}" = '1' ]]; then
: # nothing to do, A and B are already prepared
elif [[ "$FILE" = *_contents.txt ]]; then
if [[ "${WTD}" = '0' ]]; then
# Cut date and time noise away
sed -i 's/....-..-.. ..:.. //g' "$A" "$B"
fi
# Sort by path
sort -t / -k 2 --output "$A" "$A"
sort -t / -k 2 --output "$B" "$B"
if [ "$FILESONLY" = 1 ]; then
cut -d . -f 2- "$A" > "$A.cut"
mv "$A.cut" "$A"
cut -d . -f 2- "$B" > "$B.cut"
mv "$B.cut" "$B"
fi
if [ "$CUTKERNEL" = 1 ]; then
sed -i -E 's#[0-9]+\.[0-9]+\.[0-9]+-flatcar#a.b.c-flatcar#g' "$A" "$B"
fi
if [ "$CALCSIZE" = 1 ]; then
A_SUM=$(($(no_hardlinks "$A" | tr -c '[:graph:][:space:]' '?' | rev | cut -d ' ' -f 2 | rev | paste -sd+ -)))
A_BOOT=$(($(no_hardlinks "$A" | grep " [\.]*/boot" | tr -c '[:graph:][:space:]' '?' | rev | cut -d ' ' -f 2 | rev | paste -sd+ -)))
A_USR=$(($(no_hardlinks "$A" | grep " [\.]*/usr" | tr -c '[:graph:][:space:]' '?' | rev | cut -d ' ' -f 2 | rev | paste -sd+ -)))
A_ROOT=$(($(no_hardlinks "$A" | grep -v " [\.]*/usr" | grep -v " [\.]*/boot" | tr -c '[:graph:][:space:]' '?' | rev | cut -d ' ' -f 2 | rev | paste -sd+ -)))
echo "Sum: $((${A_SUM}/1024/1024)) MiB" > "$A"
echo "Boot: $((${A_BOOT}/1024/1024)) MiB (must be < 60 MiB or updates will break)" >> "$A"
echo "Usr: $((${A_USR}/1024/1024)) MiB (inc. sparse files)" >> "$A"
echo "Rootfs: $((${A_ROOT}/1024/1024)) MiB" >> "$A"
B_SUM=$(($(no_hardlinks "$B" | tr -c '[:graph:][:space:]' '?' | rev | cut -d ' ' -f 2 | rev | paste -sd+ -)))
B_BOOT=$(($(no_hardlinks "$B" | grep " [\.]*/boot" | tr -c '[:graph:][:space:]' '?' | rev | cut -d ' ' -f 2 | rev | paste -sd+ -)))
B_USR=$(($(no_hardlinks "$B" | grep " [\.]*/usr" | tr -c '[:graph:][:space:]' '?' | rev | cut -d ' ' -f 2 | rev | paste -sd+ -)))
B_ROOT=$(($(no_hardlinks "$B" | grep -v " [\.]*/usr" | grep -v " [\.]*/boot" | tr -c '[:graph:][:space:]' '?' | rev | cut -d ' ' -f 2 | rev | paste -sd+ -)))
echo "Sum: $((${B_SUM}/1024/1024)) MiB" > "$B"
echo "Boot: $((${B_BOOT}/1024/1024)) MiB (must be < 60 MiB or updates will break)" >> "$B"
echo "Usr: $((${B_USR}/1024/1024)) MiB (inc. sparse files)" >> "$B"
echo "Rootfs: $((${B_ROOT}/1024/1024)) MiB" >> "$B"
fi
fi
git diff --no-index -- "$A" "$B"