[PATCH 2/2] dpkg: directly deploy via sstate cache

[Felix Moessbauer]

By that, we can get rid of the custom sstate logic, which should be
avoided if not needed. Further, the deploy process is now cleaner, as we
deploy from a dedicated dir instead of the WORKDIR.

Signed-off-by: Felix Moessbauer <felix.mo...@siemens.com>
---
doc/technical_overview.md | 2 +-
meta/classes-recipe/dpkg-base.bbclass | 37 +++++++++------------------
2 files changed, 13 insertions(+), 26 deletions(-)

diff --git a/doc/technical_overview.md b/doc/technical_overview.md
index f7e00f7b..fe427cae 100644
--- a/doc/technical_overview.md
+++ b/doc/technical_overview.md
@@ -217,7 +217,7 @@ Both consist of the following steps:
finally unmount again (`dpkg_undo_mounts`).

6. Task `do_deploy_deb`: add successfully built packages
- `${WORKDIR}/*.deb` to the isar-apt repository
+ `${DEPLOYDIR}/*.deb` to the isar-apt repository
`${REPO_ISAR_DIR}/${DISTRO}`

## 3.6 Populate Target Filesystem
diff --git a/meta/classes-recipe/dpkg-base.bbclass b/meta/classes-recipe/dpkg-base.bbclass
index db912e90..e2ed4dfb 100644
--- a/meta/classes-recipe/dpkg-base.bbclass
+++ b/meta/classes-recipe/dpkg-base.bbclass
@@ -12,6 +12,8 @@ inherit repository
inherit deb-dl-dir
inherit essential

+DEPLOYDIR = "${WORKDIR}/deploy"
+
DEPENDS ?= ""
RPROVIDES ?= "${PROVIDES}"

@@ -186,6 +188,10 @@ dpkg_prepare_unshare_ccache() {
setfacl -m u:${UNSHARE_SUBUID_BASE}:rwX -m u:${@int(d.getVar('UNSHARE_SUBUID_BASE')) + 999}:rwx "${CCACHE_DIR}"
}

+dpkg_collect_debs() {
+ find ${WORKDIR} -maxdepth 1 -name "*.deb" -exec mv {} ${DEPLOYDIR} \;
+}
+
python do_dpkg_build() {
bb.build.exec_func('dpkg_chroot_prepare', d)
try:
@@ -193,45 +199,26 @@ python do_dpkg_build() {
finally:
bb.build.exec_func('dpkg_chroot_finalize', d)
}
+do_dpkg_build[cleandirs] = "${DEPLOYDIR}"
+do_dpkg_build[sstate-plaindirs] = "${DEPLOYDIR}"
do_dpkg_build[network] = "${TASK_USE_NETWORK_AND_SUDO}"
+do_dpkg_build[depends] = "${SCHROOT_DEP}"
+do_dpkg_build[postfuncs] += "dpkg_collect_debs"

addtask dpkg_build

SSTATETASKS += "do_dpkg_build"
-SSTATECREATEFUNCS += "dpkg_build_sstate_prepare"
-SSTATEPOSTINSTFUNCS += "dpkg_build_sstate_finalize"
-
-dpkg_build_sstate_prepare() {
- [ "${SSTATE_CURRTASK}" = "dpkg_build" ] || return 0
-
- # this runs in SSTATE_BUILDDIR, which will be deleted automatically
- if [ -n "$(find ${WORKDIR} -maxdepth 1 -name '*.deb' -print -quit)" ]; then
- cp -f ${WORKDIR}/*.deb -t .
- fi
-}
-
-dpkg_build_sstate_finalize() {
- [ "${SSTATE_CURRTASK}" = "dpkg_build" ] || return 0
-
- # this runs in SSTATE_INSTDIR
- if [ -n "$(find . -maxdepth 1 -name '*.deb' -print -quit)" ]; then
- mv -f ./*.deb -t ${WORKDIR}/
- fi
-}

python do_dpkg_build_setscene() {
sstate_setscene(d)
}

addtask dpkg_build_setscene
-do_dpkg_build_setscene[dirs] += "${S}/.."
-
-do_dpkg_build[depends] = "${SCHROOT_DEP}"

CLEANFUNCS += "deb_clean"

deb_clean() {
- DEBS=$( find ${WORKDIR} -maxdepth 1 -name "*.deb" || [ ! -d ${S} ] )
+ DEBS=$( find ${DEPLOYDIR} -maxdepth 1 -name "*.deb" || [ ! -d ${S} ] )
if [ -n "${DEBS}" ]; then
for d in ${DEBS}; do
repo_del_package "${REPO_ISAR_DIR}"/"${DISTRO}" \
@@ -246,7 +233,7 @@ do_clean[network] = "${TASK_USE_SUDO}"
do_deploy_deb() {
deb_clean
repo_add_packages "${REPO_ISAR_DIR}"/"${DISTRO}" \
- "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" ${WORKDIR}/*.deb
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" ${DEPLOYDIR}/*.deb
}

addtask deploy_deb after do_dpkg_build before do_build
--
2.53.0

[Felix Moessbauer]

To simplify the sstate handling (including cleanup), we deploy via
sstate-outputdirs.

Signed-off-by: Felix Moessbauer <felix.mo...@siemens.com>
---

meta/classes-recipe/bootstrap.bbclass | 5 ---
.../isar-mmdebstrap/isar-mmdebstrap-host.bb | 2 +-
.../isar-mmdebstrap/isar-mmdebstrap-target.bb | 2 +-
.../isar-mmdebstrap/isar-mmdebstrap.inc | 34 ++++---------------
4 files changed, 8 insertions(+), 35 deletions(-)

diff --git a/meta/classes-recipe/bootstrap.bbclass b/meta/classes-recipe/bootstrap.bbclass
index 51cd2517..c1a59fd2 100644
--- a/meta/classes-recipe/bootstrap.bbclass
+++ b/meta/classes-recipe/bootstrap.bbclass
@@ -226,8 +226,3 @@ python do_apt_config_prepare() {
aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
}
addtask apt_config_prepare before do_bootstrap after do_unpack
-
-CLEANFUNCS = "clean_deploy"
-clean_deploy() {
- rm -f "${DEPLOY_ISAR_BOOTSTRAP}"
-}
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
index 6de9c21a..fa4b76a6 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
@@ -7,7 +7,7 @@

Description = "Minimal host Debian root file system"

-DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${HOST_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
+DEPLOY_ISAR_BOOTSTRAP = "${HOST_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"

PROVIDES += "bootstrap-host"

diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
index a9695d44..9b51ab88 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
@@ -7,7 +7,7 @@

Description = "Minimal target Debian root file system"

-DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${DISTRO}-${DISTRO_ARCH}"
+DEPLOY_ISAR_BOOTSTRAP = "${DISTRO}-${DISTRO_ARCH}"
MMOPTS ?= "${DISTRO_MM_OPTS}"

PROVIDES += "bootstrap-target"
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index a2e15627..994da174 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -13,6 +13,7 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES += "locales apt"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = " ca-certificates"

BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"

+DEPLOYDIR = "${WORKDIR}/deploy"

# Fix for /var/lib/apt/available while maybe-jessie-or-older hook do not work
MM_HOOK_JESSIE = "/usr/share/mmdebstrap/hooks/jessie-or-older"
@@ -129,7 +130,10 @@ do_bootstrap[vardeps] += " \
${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
bootstrap_cleanup \
"
-do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR} ${WORKDIR}/trusted.gpg.d ${WORKDIR}/sources.list.d"
+do_bootstrap[cleandirs] = "${DEPLOYDIR}"
+do_bootstrap[sstate-inputdirs] = "${DEPLOYDIR}"
+do_bootstrap[sstate-outputdirs] = "${DEPLOY_DIR_BOOTSTRAP}"
+do_bootstrap[dirs] = "${BOOTSTRAP_TMPDIR} ${WORKDIR}/trusted.gpg.d ${WORKDIR}/sources.list.d"
do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"

@@ -259,10 +263,7 @@ do_bootstrap() {
${@get_distro_components_argument(d)} \
"${@get_distro_suite(d)}" \
- \
- "$bootstrap_list" > ${WORKDIR}/rootfs.tar.zst
-
- # Finalize bootstrap by setting the link in deploy
- ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
+ "$bootstrap_list" > ${DEPLOYDIR}/${DEPLOY_ISAR_BOOTSTRAP}.tar.zst

if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
@@ -273,31 +274,8 @@ do_bootstrap() {
addtask bootstrap before do_build after do_generate_keyrings

SSTATETASKS += "do_bootstrap"
-SSTATECREATEFUNCS += "bootstrap_sstate_prepare"
-SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
-
-bootstrap_sstate_prepare() {
- [ "${SSTATE_CURRTASK}" = "bootstrap" ] || return 0

-
- # this runs in SSTATE_BUILDDIR, which will be deleted automatically

- cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
-}
-
-bootstrap_sstate_finalize() {
- [ "${SSTATE_CURRTASK}" = "bootstrap" ] || return 0

-
- # this runs in SSTATE_INSTDIR

- # we should restore symlinks after using tar
- if [ -f bootstrap.tar.zst ]; then
- mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
- ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
- "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
- fi
-}

python do_bootstrap_setscene() {
sstate_setscene(d)
}
-
addtask do_bootstrap_setscene
-do_bootstrap_setscene[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
--
2.53.0

[MOESSBAUER, Felix]

On Mon, 2026-07-20 at 10:54 +0200, Felix Moessbauer wrote:
> By that, we can get rid of the custom sstate logic, which should be
> avoided if not needed. Further, the deploy process is now cleaner, as we
> deploy from a dedicated dir instead of the WORKDIR.

I just noticed, that this breaks the barebox.bbclass, as this looks for
the .deb in ${WORKDIR} (but now it is placed in DEPLOYDIR). The fix is
trivial.

Will send a v2.

Felix

[Felix Moessbauer]

To simplify the sstate handling (including cleanup), we deploy via
sstate-outputdirs.

Signed-off-by: Felix Moessbauer <felix.mo...@siemens.com>
---

+DEPLOYDIR = "${WORKDIR}/deploy"

- [ "${SSTATE_CURRTASK}" = "bootstrap" ] || return 0

-
- # this runs in SSTATE_BUILDDIR, which will be deleted automatically

- cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
-}
-
-bootstrap_sstate_finalize() {

- [ "${SSTATE_CURRTASK}" = "bootstrap" ] || return 0

-
- # this runs in SSTATE_INSTDIR

[Felix Moessbauer]

By that, we can get rid of the custom sstate logic, which should be
avoided if not needed. Further, the deploy process is now cleaner, as we
deploy from a dedicated dir instead of the WORKDIR.

Signed-off-by: Felix Moessbauer <felix.mo...@siemens.com>
---

Changes since v1:

- adapt barebox.bbclass to deploy from DEPLOYDIR instead of WORKDIR

doc/technical_overview.md | 2 +-
meta/classes-recipe/barebox.bbclass | 4 +--
meta/classes-recipe/dpkg-base.bbclass | 37 +++++++++------------------
3 files changed, 15 insertions(+), 28 deletions(-)

diff --git a/doc/technical_overview.md b/doc/technical_overview.md
index f7e00f7b..fe427cae 100644
--- a/doc/technical_overview.md
+++ b/doc/technical_overview.md
@@ -217,7 +217,7 @@ Both consist of the following steps:
finally unmount again (`dpkg_undo_mounts`).

6. Task `do_deploy_deb`: add successfully built packages
- `${WORKDIR}/*.deb` to the isar-apt repository
+ `${DEPLOYDIR}/*.deb` to the isar-apt repository
`${REPO_ISAR_DIR}/${DISTRO}`

## 3.6 Populate Target Filesystem

diff --git a/meta/classes-recipe/barebox.bbclass b/meta/classes-recipe/barebox.bbclass
index 45d77e3c..c9a767d5 100644
--- a/meta/classes-recipe/barebox.bbclass
+++ b/meta/classes-recipe/barebox.bbclass
@@ -47,12 +47,12 @@ BAREBOX_BASE_BIN ?= "barebox"

do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
do_deploy() {
- dpkg --fsys-tarfile ${WORKDIR}/${PN}_${CHANGELOG_V}_${DISTRO_ARCH}.deb | \
+ dpkg --fsys-tarfile ${DEPLOYDIR}/${PN}_${CHANGELOG_V}_${DISTRO_ARCH}.deb | \
tar xOf - ./usr/lib/barebox/barebox.bin \
> "${DEPLOY_DIR_IMAGE}/${BAREBOX_BASE_BIN}.img"
ln -sf ${BAREBOX_BASE_BIN}.img ${DEPLOY_DIR_IMAGE}/barebox.bin

- dpkg --fsys-tarfile ${WORKDIR}/${PN}_${CHANGELOG_V}_${DISTRO_ARCH}.deb | \
+ dpkg --fsys-tarfile ${DEPLOYDIR}/${PN}_${CHANGELOG_V}_${DISTRO_ARCH}.deb | \
tar xOf - ./usr/lib/barebox/barebox.config \
> "${DEPLOY_DIR_IMAGE}/${BAREBOX_BASE_BIN}.config"
ln -sf ${BAREBOX_BASE_BIN}.config ${DEPLOY_DIR_IMAGE}/barebox.config

diff --git a/meta/classes-recipe/dpkg-base.bbclass b/meta/classes-recipe/dpkg-base.bbclass
index db912e90..e2ed4dfb 100644
--- a/meta/classes-recipe/dpkg-base.bbclass
+++ b/meta/classes-recipe/dpkg-base.bbclass
@@ -12,6 +12,8 @@ inherit repository
inherit deb-dl-dir
inherit essential

+DEPLOYDIR = "${WORKDIR}/deploy"

- [ "${SSTATE_CURRTASK}" = "dpkg_build" ] || return 0

-
- # this runs in SSTATE_BUILDDIR, which will be deleted automatically

- if [ -n "$(find ${WORKDIR} -maxdepth 1 -name '*.deb' -print -quit)" ]; then
- cp -f ${WORKDIR}/*.deb -t .
- fi
-}
-
-dpkg_build_sstate_finalize() {

- [ "${SSTATE_CURRTASK}" = "dpkg_build" ] || return 0

-
- # this runs in SSTATE_INSTDIR

[MOESSBAUER, Felix]

Hi,

I'm currently playing around with using a hash equivalence server in
the testsuite (which brings major speedups on repeated runs and also
reduces the storage size needed). However, when do_dpkg_build is served
from the sstate cache, that WORKDIR-internal plaindir is not guaranteed
to be materialized (e.g. when the task is covered via hash equivalence
without a setscene restore). do_deploy_deb then runs against an empty
deploy dir, breaking the build.

I already have fix for that which substitutes this patch. However,
depending on the timeline (and that this patch works in the current
mode), I'm wondering if I should send that as a replacement or as a
later improvement, together with a ton of sstate fixes that currently
remain unnoticed.

Felix

[Zhihang Wei]

Hi Felix,

I would say send it as a later improvement because we are about to merge
this patch.

Zhihang