[RFC PATCH 2/2] deb-dl-dir: optimize caching of source packages using apt natively

[Cedric Hombourger]

source package are downloaded by entering the target rootfs and run
apt there. For foreign architectures, this results in apt being
executed under QEMU and leads to poor performance. By using the
recently introduced rootfs_native_cmd command wrapper, apt will be
executed natively against the target rootfs and without elevated
privileges. For our test work-load, caching was reduced from more
than 10 hours to an hour. Performance is also more consistent as
it will no longer depend as to when bitbake kicks caching of
source packages for foreign architecture rootfs vs rootfs for the
host (in multiconfig builds).

Signed-off-by: Cedric Hombourger <cedric.h...@siemens.com>
---
meta/classes/deb-dl-dir.bbclass | 36 ++++++++-------------------------
1 file changed, 8 insertions(+), 28 deletions(-)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index e75e6be5..a8e8261b 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -5,25 +5,6 @@

inherit repository

-debsrc_do_mounts() {
- sudo -s <<EOSUDO
- set -e
- mkdir -p "${1}/deb-src"
- mountpoint -q "${1}/deb-src" || \
- mount -o bind,private "${DEBSRCDIR}" "${1}/deb-src"
-EOSUDO
-}
-
-debsrc_undo_mounts() {
- sudo -s <<EOSUDO
- set -e
- mkdir -p "${1}/deb-src"
- mountpoint -q "${1}/deb-src" && \
- umount "${1}/deb-src"
- rm -rf "${1}/deb-src"
-EOSUDO
-}
-
debsrc_source_version_filter() {
# Filter the input to only consider Package, Version and Source lines
#
@@ -51,11 +32,6 @@ debsrc_download() {
export rootfs_distro="$2"
mkdir -p "${DEBSRCDIR}"/"${rootfs_distro}"

- debsrc_do_mounts "${rootfs}"
-
- trap 'exit 1' INT HUP QUIT TERM ALRM USR1
- trap 'debsrc_undo_mounts "${rootfs}"' EXIT
-
( flock 9
set -e
printenv | grep -q BB_VERBOSE_LOGS && set -x
@@ -90,13 +66,17 @@ debsrc_download() {
dscname="${src}_${version#*:}.dsc"
[ -f "${DEBSRCDIR}"/"${rootfs_distro}"/"${src}"/"${dscname}" ] || {
# use apt-get source to download sources in DEBSRCDIR
- sudo -E chroot --userspec=$( id -u ):$( id -g ) ${rootfs} \
- sh -c ' mkdir -p "/deb-src/${1}/${2}" && cd "/deb-src/${1}/${2}" && apt-get -y --download-only --only-source source "$2"="$3" ' download-src "${rootfs_distro}" "${src}" "${version}"
+ mkdir -p "${DEBSRCDIR}/${rootfs_distro}"/"${src}"
+ rootfs_native_cmd \
+ --bind "${DEBSRCDIR}" "/deb-src" \
+ --chdir "/deb-src/${rootfs_distro}/${src}" \
+ "${rootfs}" -- \
+ apt-get -o APT::Architecture=${DISTRO_ARCH} \
+ -o Dir="${rootfs}" -y --download-only \
+ --only-source source "${src}=${version}"
}
done
) 9>"${DEBSRCDIR}/${rootfs_distro}.lock"
-
- debsrc_undo_mounts "${rootfs}"
}

dbg_pkgs_download() {
--
2.39.5