scripts: add size_compare.sh

As package size changes are a continuous topic on the mailing list this
scripts helps developers to compare their local package modifications
against latest upstream.

The script downloads the latest package indexes based on env variables
or the `.config` file. The script compares the actual installed size
(data.tar.gz) or the IPK package size.

An example output is found below:

```
user@dawn:~/src/openwrt/openwrt$ ./scripts/size_compare.sh
Compare packages of ath79/tiny/mips_24kc:
dropbear busybox iw ubus

Checking configuration difference
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   554  100   554    0     0    336      0  0:00:01  0:00:01 --:--:--   336

 --- start config diff ---
--- /tmp/config.DDjwVh-LOCAL	2020-11-23 09:08:28.913203068 -1000
+++ /tmp/config.DDjwVh-UPSTREAM	2020-11-23 09:08:36.369240887 -1000
@@ -1,5 +1,9 @@
+CONFIG_ALL_KMODS=y
+CONFIG_ALL_NONSHARED=y
 CONFIG_AUTOREBUILD=y
+CONFIG_AUTOREMOVE=y
--- 8< ---
 CONFIG_BINARY_FOLDER=""
+CONFIG_BUILDBOT=y
+CONFIG_TARGET_ALL_PROFILES=y
 CONFIG_TARGET_ROOTFS_DIR=""
 CONFIG_USE_SSTRIP=y
 CONFIG_USE_UCLIBCXX=y
 --- end config diff ---

Checking installed size

Fetching latest package indexes...
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 80634  100 80634    0     0  33499      0  0:00:02  0:00:02 --:--:-- 33485
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 54082  100 54082    0     0  24252      0  0:00:02  0:00:02 --:--:-- 24252

Comparing package sizes...
Change 	Local	Remote 	Package
+271	51386	51115	base-files
+123	705241	705118	bnx2-firmware
+86	17209	17123	fstools
+22	47989	47967	procd
+21	208311	208290	busybox
+19	67181	67162	netifd

```

I plan to integrate this script into the CI so we have a summary how
sizes change over different architectures.

Signed-off-by: Paul Spooren <[email protected]>

scripts/size_compare.sh

@@ -0,0 +1,151 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Copyright (C) 2020 Paul Spooren <[email protected]>
+#
+###
+### size_compare - compare size of OpenWrt packages against upstream
+###
+### The script compares locally compiled package with the package indexes
+### available upstream. This way the storage impact of optimizations or
+### feature modifiactions is easy to see.
+###
+### If no environmental variables are set the scritp reads the current
+### .config file. The evaluated env variables are the following:
+###
+###   TARGET SUBTARGET ARCH PACKAGES BIN_DIR BASE_URL CHECK_INSTALLED
+###
+### Usage:
+###   ./scripts/size_compare.sh
+###
+### Options:
+###   -p --package-size 	Check IPK package size and not installed size
+###   -h --help 		This message
+
+CONFIG_TARGET=$(sed -n 's/^CONFIG_TARGET_BOARD="\(.*\)"$/\1/p' .config)
+CONFIG_SUBTARGET=$(sed -n 's/^CONFIG_TARGET_SUBTARGET="\(.*\)"$/\1/p' .config)
+CONFIG_ARCH=$(sed -n 's/^CONFIG_TARGET_ARCH_PACKAGES="\(.*\)"$/\1/p' .config)
+CONFIG_PACKAGES=$(sed -n 's/^CONFIG_PACKAGE_\(.*\)=y$/\1/p' .config | tr '\n' ' ')
+CONFIG_BIN_DIR=$(sed -n 's/^CONFIG_BINARY_DIR="\(.*\)"$/\1/p' .config)
+
+TARGET=${TARGET:-$CONFIG_TARGET}
+SUBTARGET=${SUBTARGET:-$CONFIG_SUBTARGET}
+ARCH=${ARCH:-$CONFIG_ARCH}
+PACKAGES=${PACKAGES:-$CONFIG_PACKAGES}
+BIN_DIR=${CONFIG_BIN_DIR:-./bin}
+BASE_URL="${BASE_URL:-https://downloads.openwrt.org/snapshots}"
+CHECK_INSTALLED="${CHECK_INSTALLED:-y}"
+
+TARGET_URL="$BASE_URL/targets/$TARGET/$SUBTARGET/packages/Packages.gz"
+CONFIG_URL="$BASE_URL/targets/$TARGET/$SUBTARGET/config.buildinfo"
+PACKAGES_URL="$BASE_URL/packages/$ARCH/base/Packages.gz"
+
+if command -v curl > /dev/null; then
+	DOWNLOAD_METHOD="curl"
+else
+	DOWNLOAD_METHOD="wget --output-document=-"
+fi
+
+help() {
+    sed -rn 's/^### ?//;T;p' "$0"
+}
+
+package_size () {
+	FOUND_PACKAGE=
+	if [ -z "$CHECK_INSTALLED" ]; then
+		SEARCH_PATTERN="Size"
+	else
+		SEARCH_PATTERN="Installed-Size"
+	fi
+	while IFS= read -r line; do
+		if [ "$line" = "Package: $2" ]; then
+			FOUND_PACKAGE=y
+		fi
+		if [ -n "$FOUND_PACKAGE" ]; then
+			case $line in
+				"$SEARCH_PATTERN"*)
+					echo "$line" | cut -d ' ' -f 2
+					break
+					;;
+			esac
+		fi
+	done < "$1"
+}
+
+compare_sizes () {
+	for PACKAGE in $PACKAGES; do
+		if [ "$PACKAGE" = "libc" ]; then
+			continue
+		fi
+		PACKAGE_FILE=$(find "$BIN_DIR/packages/$ARCH/" \
+			"$BIN_DIR/targets/$TARGET/$SUBTARGET/" \
+			-name "${PACKAGE}_*.ipk" 2>/dev/null | head -n1)
+
+		if [ -z "$PACKAGE_FILE" ]; then
+			continue
+		fi
+		if [ -z "$CHECK_INSTALLED" ]; then
+			SIZE_LOCAL=$(stat -c '%s' "$PACKAGE_FILE")
+		else
+			SIZE_LOCAL=$(tar tzvf "$PACKAGE_FILE" ./data.tar.gz | awk '{ print $3 }')
+		fi
+		SIZE_UPSTREAM=$(package_size "$TMP_INDEX" "$PACKAGE")
+		SIZE_DIFF="$((SIZE_LOCAL-SIZE_UPSTREAM))"
+		if [ "$SIZE_DIFF" -gt 0 ]; then
+			SIZE_DIFF="+$SIZE_DIFF"
+		fi
+		if [ -n "$SIZE_UPSTREAM" ]; then
+			echo "${SIZE_DIFF}	${SIZE_LOCAL}	${SIZE_UPSTREAM}	$PACKAGE"
+		else
+			echo "$PACKAGE is missing upstream"
+		fi
+	done
+}
+
+if [ "$1" = "-h" ]; then
+    help
+    exit 0
+fi
+
+if [ "$1" = "-p" ]; then
+    CHECK_INSTALLED=
+fi
+
+echo "Compare packages of $TARGET/$SUBTARGET/$ARCH":
+echo "$PACKAGES"
+echo
+
+echo "Checking configuration difference"
+TMP_CONFIG=$(mktemp /tmp/config.XXXXXX)
+sed -n 's/^	\+config \(.*\)/\1/p' config/Config-build.in config/Config-devel.in > "${TMP_CONFIG}-FOCUS"
+sort .config | grep -f "${TMP_CONFIG}-FOCUS" | grep -v "^#" | sort > "${TMP_CONFIG}-LOCAL"
+mv .config .config.bak
+"$DOWNLOAD_METHOD" "$CONFIG_URL" > .config
+make defconfig > /dev/null 2> /dev/null
+grep -f "${TMP_CONFIG}-FOCUS" .config | grep -v "^#" | sort > "${TMP_CONFIG}-UPSTREAM"
+mv .config.bak .config
+
+echo
+echo " --- start config diff ---"
+diff -u "${TMP_CONFIG}-LOCAL" "${TMP_CONFIG}-UPSTREAM"
+echo " --- end config diff ---"
+rm "${TMP_CONFIG}-FOCUS" "${TMP_CONFIG}-UPSTREAM" "${TMP_CONFIG}-LOCAL"
+echo
+
+if [ -z "$CHECK_INSTALLED" ]; then
+	echo "Checking IPK package size"
+else
+	echo "Checking installed size"
+fi
+echo
+
+echo "Fetching latest package indexes..."
+TMP_INDEX=$(mktemp /tmp/size_compare_package_index.XXXXXX)
+"$DOWNLOAD_METHOD" "$TARGET_URL" | gzip -d > "$TMP_INDEX" || exit 1
+"$DOWNLOAD_METHOD" "$PACKAGES_URL" | gzip -d >> "$TMP_INDEX" || exit 1
+echo
+
+echo "Comparing package sizes..."
+echo "Change 	Local	Remote 	Package"
+compare_sizes | sort -g -r
+
+rm "$TMP_INDEX"