[PATCHv4 05/26] base-apt: add deb-src packages as well

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

So far the deb-src packages downloaded via "apt://" where not cached.
This commit fixes that and includes those packages into the cache.

Signed-off-by: Henning Schild <henning...@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 7 +++++++
meta/classes/image-cache-extension.bbclass | 4 ++++
meta/classes/repository.bbclass | 14 ++++++++++++++
3 files changed, 25 insertions(+)

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 1f08c703..db25adca 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -52,4 +52,11 @@ populate_base_apt() {
"${BASE_DISTRO_CODENAME}" \
"${package}"
done
+
+ find $search_dir -name '*.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
}
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 30db5493..f8b9ba34 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -32,5 +32,9 @@ do_cache_base_repo() {
if [ -d ${BUILDCHROOT_TARGET_DIR}/var/cache/apt ]; then
populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
fi
+
+ if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
+ populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
+ fi
}
addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index 880ddefc..dc4c1dcd 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -35,6 +35,20 @@ EOF
fi
}

+repo_add_srcpackage() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main -P source \
+ includedsc "${codename}" \
+ "$@"
+}
+
repo_add_packages() {
local dir="$1"
local dbdir="$2"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Now that base-apt supports deb-src we can include hello in a standard
build.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta-isar/conf/local.conf.sample | 4 +---
scripts/ci_build.sh | 1 -
2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 280e0072..0e55da19 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -166,11 +166,9 @@ CONF_VERSION = "1"

#
# The default list of extra packages to be installed.
-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello"
## "cache_base_repo" relies on the cache ...
#IMAGE_INSTALL += "isar-disable-apt-cache"
-## uses apt:// SRC_URI, which does not work with the cache
-#IMAGE_INSTALL += "hello"

#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7cd5a2fc..f50c9e42 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -169,7 +169,6 @@ if [ -n "$REPRO_BUILD" ]; then
fi

sed -i -e 's/#IMAGE_INSTALL += "isar-disable-apt-cache"/IMAGE_INSTALL += "isar-disable-apt-cache"/g' conf/local.conf
-sed -i -e 's/#IMAGE_INSTALL += "hello"/IMAGE_INSTALL += "hello"/g' conf/local.conf

echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Factor out all the "reprepro" code into a common class. There are slight
functional changes since the two copies of the code got out of sync.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/base-apt-helper.bbclass | 28 ++++----
meta/classes/dpkg-base.bbclass | 24 ++-----
meta/classes/repository.bbclass | 69 +++++++++++++++++++
meta/recipes-devtools/base-apt/base-apt.bb | 37 +++-------
.../base-apt/files/distributions.in | 3 -
.../isar-apt/files/distributions.in | 3 -
meta/recipes-devtools/isar-apt/isar-apt.bb | 26 ++-----
7 files changed, 102 insertions(+), 88 deletions(-)
create mode 100644 meta/classes/repository.bbclass
delete mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
delete mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 90b2cfcb..1f08c703 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -1,5 +1,10 @@
# This software is a part of ISAR.
# Copyright (C) 2018 ilbers GmbH
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+inherit repository

compare_pkg_md5sums() {
pkg1=$1
@@ -14,10 +19,6 @@ compare_pkg_md5sums() {
populate_base_apt() {
search_dir=$1

- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
find $search_dir -name '*.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In
@@ -40,18 +41,15 @@ populate_base_apt() {
compare_pkg_md5sums "$package" "$base_apt_p" && continue

# md5sum differs, so remove the package from base-apt
- name=$(echo $base_name | cut -d '_' -f 1)
- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main -A ${DISTRO_ARCH} \
- remove ${BASE_DISTRO_CODENAME} \
- $name
+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \

+ "${base_apt_p}"
fi

- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main \
- includedeb ${BASE_DISTRO_CODENAME} \
- $package
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"

done
}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index af9c9840..f38168c7 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -7,6 +7,7 @@
inherit buildchroot
inherit debianize
inherit terminal
+inherit repository

DEPENDS ?= ""

@@ -140,31 +141,16 @@ repo_clean() {
DEBS=$( find ${S}/.. -maxdepth 1 -name "*.deb" || [ ! -d ${S} ] )
if [ -n "${DEBS}" ]; then
for d in ${DEBS}; do
- p=$( dpkg-deb --show --showformat '${Package}' ${d} )
- a=$( dpkg-deb --show --showformat '${Architecture}' ${d} )
- # removing "all" means no arch
- aarg="-A ${a}"
- [ "${a}" = "all" ] && aarg=""
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main ${aarg} \
- remove ${DEBDISTRONAME} \
- ${p}
+ repo_del_package "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" "${d}"
done
fi
}

-# Install package to Isar-apt
do_deploy_deb() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
repo_clean
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main \
- includedeb ${DEBDISTRONAME} \
- ${S}/../*.deb
+ repo_add_packages "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" ${S}/../*.deb
}

addtask deploy_deb after do_dpkg_build before do_build
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
new file mode 100644
index 00000000..880ddefc
--- /dev/null
+++ b/meta/classes/repository.bbclass
@@ -0,0 +1,69 @@
+# This software is a part of ISAR.
+# Copyright (C) 2017-2019 Siemens AG
+# Copyright (C) 2019 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+repo_create() {

+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"

+ local keyfiles="$4"

+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+

+ if [ ! -f "${dir}"/conf/distributions ]; then
+ mkdir -p "${dir}"/conf/
+ cat <<EOF > "${dir}"/conf/distributions
+Codename: ${codename}
+Architectures: i386 armhf arm64 amd64 mipsel source
+Components: main
+EOF
+ if [ -n "${keyfiles}" ] ; then
+ local option=""
+ for key in ${keyfiles}; do
+ keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
+ option="${option}${keyid} "
+ done
+ echo "SignWith: ${option}" >> "${dir}"/conf/distributions
+ fi
+ fi
+ if [ ! -d "${dbdir}" ]; then
+ reprepro -b "${dir}" --dbdir "${dbdir}" export "${codename}"
+ fi
+}
+
+repo_add_packages() {

+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi

+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main \
+ includedeb "${codename}" \
+ "$@"
+}
+
+repo_del_package() {

+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"

+ local file="$4"

+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi

+ local p=$( dpkg-deb --show --showformat '${Package}' "${file}" )
+ local a=$( dpkg-deb --show --showformat '${Architecture}' "${file}" )
+ # removing "all" means no arch
+ local aarg="-A ${a}"
+ [ "${a}" = "all" ] && aarg=""
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main ${aarg} \
+ remove "${codename}" \
+ "${p}"
+}
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 9a0f7c8b..c9a7dad1 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -1,45 +1,24 @@
# This software is a part of ISAR.
# Copyright (C) 2018 ilbers GmbH
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT

-SRC_URI = "file://distributions.in"
+inherit repository

BASE_REPO_KEY ?= ""
KEYFILES ?= ""

-CACHE_CONF_DIR = "${REPO_BASE_DIR}/${BASE_DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"

# Generate reprepro config for current distro if it doesn't exist. Once it's
# generated, this task should do nothing.
repo_config() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- if [ -n "${KEYFILES}" ]; then
- option=""
- for key in ${KEYFILES}; do
- keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
- option="${option}${keyid} "
- done
- # To generate Release.gpg
- echo "SignWith: ${option}" >> ${CACHE_CONF_DIR}/distributions
- fi
- fi
-
- path_cache="${REPO_BASE_DIR}/${BASE_DISTRO}"
- path_databases="${REPO_BASE_DB_DIR}/${BASE_DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${BASE_DISTRO_CODENAME}
- fi
+ repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \

+ "${KEYFILES}"
}

python do_cache_config() {
diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
deleted file mode 100644
index fd7de0b7..00000000
--- a/meta/recipes-devtools/base-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {CODENAME}
-Architectures: i386 armhf arm64 amd64 mipsel source
-Components: main
diff --git a/meta/recipes-devtools/isar-apt/files/distributions.in b/meta/recipes-devtools/isar-apt/files/distributions.in
deleted file mode 100644
index a24a7000..00000000
--- a/meta/recipes-devtools/isar-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {DISTRO_NAME}
-Architectures: i386 armhf arm64 amd64 mipsel source
-Components: main
diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index e6f1753d..0227c1b0 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -1,32 +1,20 @@
# This software is a part of ISAR.
# Copyright (C) 2015-2017 ilbers GmbH
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT

-SRC_URI = "file://distributions.in"
+inherit repository

-CACHE_CONF_DIR = "${REPO_ISAR_DIR}/${DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"

# Generate reprepro config for current distro if it doesn't exist. Once it's
# generated, this task should do nothing.
do_cache_config() {
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{DISTRO_NAME}#"${DEBDISTRONAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- fi
-
- path_cache="${REPO_ISAR_DIR}/${DISTRO}"
- path_databases="${REPO_ISAR_DB_DIR}/${DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${DEBDISTRONAME}
- fi
+ repo_create "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" \
+ "${DEBDISTRONAME}"
}

addtask cache_config after do_unpack before do_build
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

changes since v2:

Rebased/merged onto current next and added p26. The custom kernel caching
that was missing in v2 is not required on next anymore.

This one was sent as v3 by Vijai Kumar K
<vijaikumar....@gmail.com>. That just contained a fix in case
the patches would not be rebased, can be ignored.

---

That one had to wait for some time. I finally managed to have a look at
it again and came up with a v2. Sorry that one is really big and a lot
of things happened since v1.

The most significant change to last time it p23, which switches the way
the offline cache works. Unfortunately this is turned out pretty big,
splitting it into readably and working chunks was not reasonable.

Everything up to p19 was already sent once, at least roughly ...

I hope a v3 will come faster and we will have better tracking of
changes between 2 and 3.

changes since v1:

- pull the base-apt building to the front of the build chain
- performance optimze all that copying
- add import/export of bootstrap step
- rebased on current next
- exclude isar-apt in deb_dl_dir_export
- use find exec + for deb_dl_dir_import
- fixed "==" operator in deps.sh in p8
- fixed typo in p2 message
- fixed variable scope to local in repository functions


Henning Schild (26):
repository: new class to deal with repos
dpkg-base: add download caching of apt:// downloads
meta: dpkg-base: convert "exit" into "return 0"
base-apt: change the sources.list to also offer deb-src
base-apt: add deb-src packages as well
base-apt: do not skip gpg check when it is signed
ci: conf: add "hello" to the sample config and every build
meta: split all apt-get invocations into download and execution
meta: create DL_DIR support for all apt-get downloaded .debs
meta: import DL_DIR debs before apt-get download steps
meta: include DL_DIR deb import/export into bootstrap
base-apt: populate from DEBDIR as well
base-apt: drop the "apt_cache" feature
base-apt: do not copy debs directly out of rootfss anymore
base-apt: rework base-apt population
base-apt: move class "base-apt-helper" into only user
CI: include "isar-disable-apt-cache" into all CI images
CI: include "cowsay" into default build to test dpkg-gbp
CI: set BB_NO_NETWORK for cached rebuild
meta: repository: implement repo_contains_package and use it in
base_apt
meta: repository: simplify the matching of packages
meta: deb-dl-dir: do not cache debs from isar-apt
base-apt: pull base-apt population to the front of the build chain
meta: base-apt: remove isar-apt check from population
meta: deb-dl-dir: only export newly downloaded files
CI: test a custom kernel build in the base-apt offline rebuild

doc/user_manual.md | 9 +-
meta-isar/conf/local.conf.sample | 10 +-
meta/classes/base-apt-helper.bbclass | 57 ----------
meta/classes/deb-dl-dir.bbclass | 38 +++++++
meta/classes/dpkg-base.bbclass | 47 ++++----
meta/classes/dpkg-gbp.bbclass | 5 +
meta/classes/dpkg.bbclass | 7 +-
meta/classes/image-cache-extension.bbclass | 36 -------
meta/classes/image-locales-extension.bbclass | 2 +-
meta/classes/image-tools-extension.bbclass | 7 ++
meta/classes/image.bbclass | 3 +-
meta/classes/repository.bbclass | 100 ++++++++++++++++++
meta/classes/rootfs.bbclass | 30 +++---
meta/conf/bitbake.conf | 6 +-
.../isar-bootstrap/files/base-apt-sources | 1 -
.../isar-bootstrap/isar-bootstrap.inc | 21 ++--
meta/recipes-devtools/base-apt/base-apt.bb | 92 ++++++++++------
.../base-apt/files/distributions.in | 3 -
.../buildchroot/files/deps.sh | 37 ++++---

.../isar-apt/files/distributions.in | 3 -
meta/recipes-devtools/isar-apt/isar-apt.bb | 26 ++---

scripts/ci_build.sh | 15 ++-
22 files changed, 322 insertions(+), 233 deletions(-)
delete mode 100644 meta/classes/base-apt-helper.bbclass
create mode 100644 meta/classes/deb-dl-dir.bbclass
delete mode 100644 meta/classes/image-cache-extension.bbclass
create mode 100644 meta/classes/repository.bbclass
delete mode 100644 meta/recipes-core/isar-bootstrap/files/base-apt-sources

delete mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
delete mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

The repository is ready for it already. For the moment there will be
nothing there.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/recipes-core/isar-bootstrap/files/base-apt-sources | 1 +
1 file changed, 1 insertion(+)

diff --git a/meta/recipes-core/isar-bootstrap/files/base-apt-sources b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
index ddc45093..2a4ab2c4 100644
--- a/meta/recipes-core/isar-bootstrap/files/base-apt-sources
+++ b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
@@ -1 +1,2 @@
deb [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
+deb-src [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Bitbake seems to execute all the functions in subshells, but in fact we
mean to return here so correct the style.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/dpkg-base.bbclass | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 7a97d91e..aa11660e 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -53,7 +53,7 @@ SRC_APT ?= ""

do_apt_fetch() {
if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
- exit
+ return 0
fi
rm -rf ${S}
dpkg_do_mounts
@@ -80,7 +80,7 @@ addtask cleanall_apt before do_cleanall
do_cleanall_apt[nostamp] = "1"
do_cleanall_apt() {
if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
- exit
+ return 0
fi
for uri in "${SRC_APT}"; do
rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Hook in between the "apt-get --download-only"s and the "apt-get"s and
copy out all the debs for later.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 15 +++++++++++++++
meta/classes/dpkg-base.bbclass | 1 +
meta/classes/dpkg-gbp.bbclass | 1 +
meta/classes/dpkg.bbclass | 1 +
meta/classes/image-locales-extension.bbclass | 2 +-
meta/classes/image-tools-extension.bbclass | 1 +
meta/classes/rootfs.bbclass | 12 ++++++++++--
meta/conf/bitbake.conf | 1 +
8 files changed, 31 insertions(+), 3 deletions(-)
create mode 100644 meta/classes/deb-dl-dir.bbclass

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
new file mode 100644
index 00000000..e41e9813
--- /dev/null
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -0,0 +1,15 @@

+# This software is a part of ISAR.

+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+

+deb_dl_dir_export() {
+ export pc="${DEBDIR}/${DISTRO}"
+ export rootfs="${1}"
+ mkdir -p "${pc}"
+ flock "${pc}".lock -c ' \
+ sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' \
+ -exec cp -f '{}' "${pc}" \;
+ sudo chown -R $(id -u):$(id -g) "${pc}"
+ '
+}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index aa11660e..fd0b07fe 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -8,6 +8,7 @@ inherit buildchroot
inherit debianize
inherit terminal
inherit repository
+inherit deb-dl-dir

DEPENDS ?= ""

diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 2afe9db4..7e908eba 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -15,6 +15,7 @@ do_install_builddeps_append() {
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends --download-only ${GBP_DEPENDS}
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index b26c6456..7e2c86d6 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -9,6 +9,7 @@ do_install_builddeps() {
E="${@ isar_export_proxies(d)}"
sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${DISTRO_ARCH}
dpkg_undo_mounts
diff --git a/meta/classes/image-locales-extension.bbclass b/meta/classes/image-locales-extension.bbclass
index 3c0758ff..0f0d0ca4 100644
--- a/meta/classes/image-locales-extension.bbclass
+++ b/meta/classes/image-locales-extension.bbclass
@@ -25,7 +25,7 @@ def get_nopurge(d):
j.split()[0].split(".")[0],
j.split()[0]))))

-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN += "image_install_localepurge_download"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download"
image_install_localepurge_download[weight] = "40"
image_install_localepurge_download() {
sudo -E chroot '${ROOTFSDIR}' \
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index b8531cb4..ec67d940 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -34,6 +34,7 @@ do_install_imager_deps() {
--allow-unauthenticated --allow-downgrades --download-only install \
${IMAGER_INSTALL}'

+ deb_dl_dir_export ${BUILDCHROOT_DIR}
sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 64eaef70..25af4688 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -1,6 +1,8 @@

# This software is a part of ISAR.

# Copyright (c) Siemens AG, 2019

+inherit deb-dl-dir
+
ROOTFS_ARCH ?= "${DISTRO_ARCH}"
ROOTFS_DISTRO ?= "${DISTRO}"
ROOTFS_PACKAGES ?= ""
@@ -119,8 +121,14 @@ rootfs_install_pkgs_download() {
/usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
}

-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN ??= ""
-ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN}"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= ""
+ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT}"
+
+ROOTFS_INSTALL_COMMAND += "rootfs_export_package_cache"
+rootfs_export_package_cache[weight] = "5"
+rootfs_export_package_cache() {
+ deb_dl_dir_export ${ROOTFSDIR}
+}

ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
rootfs_install_clean_files[weight] = "2"
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index ab608da1..d86c5b92 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
FILESEXTRAPATHS ?= "__default:"
GITDIR = "${DL_DIR}/git"
+DEBDIR = "${DL_DIR}/deb"
DEBSRCDIR = "${DL_DIR}/deb-src"
P = "${PN}-${PV}"
PF = "${PN}-${PV}-${PR}"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Just call that function once and make it find the debs and dscs in one
run.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/base-apt-helper.bbclass | 6 ++----
meta/classes/image-cache-extension.bbclass | 8 +-------
2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index db25adca..18d43aa3 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -17,9 +17,7 @@ compare_pkg_md5sums() {
}

populate_base_apt() {
- search_dir=$1
-

- find $search_dir -name '*.deb' | while read package; do

+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do

# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In

# addition, md5sums are compared to ensure that the package is the
@@ -53,7 +51,7 @@ populate_base_apt() {
"${package}"
done

- find $search_dir -name '*.dsc' | while read package; do
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \

"${BASE_DISTRO_CODENAME}" \
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index cff2a8ba..7945e8bc 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,12 +21,6 @@ do_cache_base_repo() {
"Try it without cross-build."
fi

- if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
- populate_base_apt "${DEBDIR}"/"${DISTRO}"
- fi
-
- if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
- populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
- fi
+ populate_base_apt

}
addtask cache_base_repo after do_rootfs do_install_imager_deps

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

The earlier commits make sure we already have all .debs in the repo, no
need to get them from these locations. In fact these locations are not
reliable anyways, see hook in "isar-disable-apt-cache".

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 8 --------
1 file changed, 8 deletions(-)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 19c9c593..cff2a8ba 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,14 +21,6 @@ do_cache_base_repo() {

"Try it without cross-build."
fi

- if [ -d ${BUILDCHROOT_HOST_DIR}/var/cache/apt ]; then
- populate_base_apt ${BUILDCHROOT_HOST_DIR}/var/cache/apt
- fi
-
- if [ -d ${BUILDCHROOT_TARGET_DIR}/var/cache/apt ]; then
- populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
- fi

-
if [ -d "${DEBDIR}"/"${DISTRO}" ]; then

populate_base_apt "${DEBDIR}"/"${DISTRO}"

fi
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta-isar/conf/local.conf.sample | 2 +-
scripts/ci_build.sh | 2 --
2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index dee12505..9237bb35 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -166,7 +166,7 @@ CONF_VERSION = "1"

#
# The default list of extra packages to be installed.

-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache cowsay"

#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh

index bfaf1f6e..d440c248 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -168,8 +168,6 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
fi

-echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf
-
# Start cross build for the defined set of configurations
sed -i -e 's/ISAR_CROSS_COMPILE ?= "0"/ISAR_CROSS_COMPILE ?= "1"/g' conf/local.conf
bitbake $BB_ARGS $CROSS_TARGETS_SET
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

The bootstrap step also contains downloading of debs, so it should be
surrounded with an import/export as well.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 4 ++++
1 file changed, 4 insertions(+)

diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index cf3df407..2078115b 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -28,6 +28,8 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES_append_gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support = ",apt-transport-https,ca-certificates"
HOST_DISTRO_APT_SOURCES += "conf/distro/${HOST_DISTRO}.list"

+inherit deb-dl-dir
+
python () {
distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split()

@@ -219,6 +221,7 @@ do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"

isar_bootstrap() {
+ deb_dl_dir_import "${ROOTFSDIR}"
IS_HOST=""
while true; do
case "$1" in
@@ -335,6 +338,7 @@ isar_bootstrap() {
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
fi
EOSUDO
+ deb_dl_dir_export "${ROOTFSDIR}"
}

CLEANFUNCS = "clean_deploy"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

We now have all debs in DEBDIR anyways, no need to conditionally copy
them out of the rootfss we create.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 4 ----
meta/classes/image.bbclass | 2 +-
meta/classes/rootfs.bbclass | 10 ----------
3 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 07e9e7d0..19c9c593 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,10 +21,6 @@ do_cache_base_repo() {

"Try it without cross-build."
fi

- if [ -d ${WORKDIR}/apt_cache ]; then
- populate_base_apt ${WORKDIR}/apt_cache
- fi
-
if [ -d ${BUILDCHROOT_HOST_DIR}/var/cache/apt ]; then
populate_base_apt ${BUILDCHROOT_HOST_DIR}/var/cache/apt
fi
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 725bc04f..b687fcfb 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -60,7 +60,7 @@ image_do_mounts() {
}

ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
+ROOTFS_FEATURES += "clean-package-cache finalize-rootfs generate-manifest"
ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"

diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 4e145400..557756c7 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -9,7 +9,6 @@ ROOTFS_PACKAGES ?= ""

# Features of the rootfs creation:
# available features are:
-# 'copy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
# 'clean-package-cache' - delete package cache from rootfs
# 'generate-manifest' - generate a package manifest of the rootfs into ${ROOTFS_MANIFEST_DEPLOY_DIR}
# 'finalize-rootfs' - delete files needed to chroot into the rootfs
@@ -186,15 +185,6 @@ python do_rootfs_install() {
}
addtask rootfs_install before do_rootfs_postprocess after do_unpack

-ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache', 'rootfs_postprocess_copy_package_cache', '', d)}"
-rootfs_postprocess_copy_package_cache() {
- mkdir -p '${WORKDIR}/apt_cache'
- sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
- -maxdepth 1 -name '*.deb' -execdir /bin/mv -t '${WORKDIR}/apt_cache' '{}' '+'
- me="$(id -u):$(id -g)"
- sudo chown -R "$me" '${WORKDIR}/apt_cache'
-}
-
ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', 'rootfs_postprocess_clean_package_cache', '', d)}"
rootfs_postprocess_clean_package_cache() {

sudo -E chroot '${ROOTFSDIR}' \

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

We have all the .debs cached in DEBDIR now, take them from there because
the rootfss might not actually have all of them.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 4 ++++

1 file changed, 4 insertions(+)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index f8b9ba34..07e9e7d0 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -33,6 +33,10 @@ do_cache_base_repo() {

populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
fi

+ if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
+ populate_base_apt "${DEBDIR}"/"${DISTRO}"
+ fi
+
if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
fi
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Store the results of "apt-get source" in the DL_DIR. This means we
really only need to fetch once on incremental rebuilds or if multiple
recipes use the same SRC_URI.
We can also later collect the results and put them in a repository.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/dpkg-base.bbclass | 20 ++++++++++++++++++--
meta/conf/bitbake.conf | 1 +
2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index f38168c7..7a97d91e 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -63,14 +63,30 @@ do_apt_fetch() {
-o Dir::Etc::SourceParts="-" \
-o APT::Get::List-Cleanup="0"

- sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
- sh -c 'cd ${PP} && apt-get -y --only-source source ${SRC_APT}'
+ for uri in "${SRC_APT}"; do
+ sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${DISTRO}" "${uri}"
+ sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+ sh -c 'cp /downloads/deb-src/"$1"/"$2"/* ${PP} && cd ${PP} && apt-get -y --only-source source "$2"' my_script "${DISTRO}" "${uri}"
+ done
+
dpkg_undo_mounts
}

addtask apt_fetch after do_unpack before do_patch
do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"

+addtask cleanall_apt before do_cleanall
+do_cleanall_apt[nostamp] = "1"
+do_cleanall_apt() {
+ if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
+ exit
+ fi
+ for uri in "${SRC_APT}"; do
+ rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
+ done
+}
+
def get_package_srcdir(d):
s = os.path.abspath(d.getVar("S", True))
workdir = os.path.abspath(d.getVar("WORKDIR", True))
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 0eb3b544..ab608da1 100644

--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
FILESEXTRAPATHS ?= "__default:"
GITDIR = "${DL_DIR}/git"

+DEBSRCDIR = "${DL_DIR}/deb-src"

P = "${PN}-${PV}"
PF = "${PN}-${PV}-${PR}"

PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

This commit does not contain a functional change. We just split all
calls of "apt-get" into a download and an execution phase, so we can
later copy out the downloaded files.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/dpkg-gbp.bbclass | 3 ++
meta/classes/dpkg.bbclass | 5 ++-
meta/classes/image-tools-extension.bbclass | 5 +++
.../buildchroot/files/deps.sh | 37 +++++++++++--------
4 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 0533a52f..2afe9db4 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,6 +12,9 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"

do_install_builddeps_append() {
dpkg_do_mounts
+ sudo -E chroot ${BUILDCHROOT_DIR} \
+ apt-get install -y -o Debug::pkgProblemResolver=yes \
+ --no-install-recommends --download-only ${GBP_DEPENDS}

sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass

index 180a3a89..b26c6456 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -7,7 +7,10 @@ inherit dpkg-base
do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"
- sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh ${PP}/${PPS} ${DISTRO_ARCH}
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH}
dpkg_undo_mounts
}

diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index d4f216da..b8531cb4 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -30,6 +30,11 @@ do_install_imager_deps() {
-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \

-o Dir::Etc::SourceParts="-" \
-o APT::Get::List-Cleanup="0"

+ apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
+ --allow-unauthenticated --allow-downgrades --download-only install \
+ ${IMAGER_INSTALL}'
+
+ sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \

apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \

${IMAGER_INSTALL}'
diff --git a/meta/recipes-devtools/buildchroot/files/deps.sh b/meta/recipes-devtools/buildchroot/files/deps.sh
index 002085b1..0f11d78e 100644
--- a/meta/recipes-devtools/buildchroot/files/deps.sh
+++ b/meta/recipes-devtools/buildchroot/files/deps.sh
@@ -12,27 +12,34 @@ source /isar/common.sh
# 2) we add -y to go non-interactive
# 3) downgrades shall be allowed in case a package recipe was changed
install_cmd="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends \
- -y --allow-downgrades"
+ -y --allow-downgrades $3"

-# Make sure that we have latest isar-apt content.
-# Options meaning:
-# Dir::Etc::SourceList - specifies which source to be used
-# Dir::Etc::SourceParts - disables looking for the other sources
-# APT::Get::List-Cleanup - do not erase obsolete packages list for
-# upstream in '/var/lib/apt/lists'
-apt-get update \
- -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
- -o Dir::Etc::SourceParts="-" \

- -o APT::Get::List-Cleanup="0"

+if [ "$3" != "-download-only" ]; then
+ # Make sure that we have latest isar-apt content.
+ # Options meaning:
+ # Dir::Etc::SourceList - specifies which source to be used
+ # Dir::Etc::SourceParts - disables looking for the other sources
+ # APT::Get::List-Cleanup - do not erase obsolete packages list for
+ # upstream in '/var/lib/apt/lists'
+ apt-get update \
+ -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
+ -o Dir::Etc::SourceParts="-" \
+ -o APT::Get::List-Cleanup="0"
+fi

# Do not set an architecture when building only 'all' (generic) packages.
# This can avoid unneeded cross-build issues.
if ! grep "^Architecture:" debian/control | grep -qv "all"; then
- set_arch=""
+ set_arch=""
fi

# Install all build deps
-mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control
+if [ "$3" = "--download-only" ]; then
+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control 2>&1 \
+ | grep "mk-build-deps: Unable to install all build-dep packages"
+else
+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control

-# Upgrade any already installed packages in case we are partially rebuilding
-apt-get upgrade -y --allow-downgrades
+ # Upgrade any already installed packages in case we are partially rebuilding
+ apt-get upgrade -y --allow-downgrades
+fi
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

We used to have "trusted=yes" hardcoded even for a signed base-apt. Make
that flag depend on whether it is signed and generate that file with two
echos, it is simple enough.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

.../isar-bootstrap/files/base-apt-sources | 2 --
.../isar-bootstrap/isar-bootstrap.inc | 16 ++++++++--------
2 files changed, 8 insertions(+), 10 deletions(-)
delete mode 100644 meta/recipes-core/isar-bootstrap/files/base-apt-sources

diff --git a/meta/recipes-core/isar-bootstrap/files/base-apt-sources b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
deleted file mode 100644
index 2a4ab2c4..00000000
--- a/meta/recipes-core/isar-bootstrap/files/base-apt-sources
+++ /dev/null
@@ -1,2 +0,0 @@
-deb [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
-deb-src [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 16b4395b..cf3df407 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -12,8 +12,7 @@ SRC_URI = " \
file://isar-apt.conf \
file://isar-apt-fallback.conf \
file://locale \
- file://chroot-setup.sh \
- file://base-apt-sources"
+ file://chroot-setup.sh"
PV = "1.0"

DEBOOTSTRAP ?= "qemu-debootstrap"
@@ -21,7 +20,6 @@ ROOTFSDIR = "${WORKDIR}/rootfs"
APTPREFS = "${WORKDIR}/apt-preferences"
APTSRCS = "${WORKDIR}/apt-sources"
APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
-BASEAPTSRCS = "${WORKDIR}/base-apt-sources"
DISTRO_BOOTSTRAP_KEYFILES = ""
THIRD_PARTY_APT_KEYFILES = ""
DEPLOY_ISAR_BOOTSTRAP ?= ""
@@ -266,13 +264,15 @@ isar_bootstrap() {
"${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- sed -e "s#{BASE_DISTRO}#"${BASE_DISTRO}"#g" \
- -e "s#{BASE_DISTRO_CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- -i ${BASEAPTSRCS}
+ line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+ echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+
mkdir -p ${ROOTFSDIR}/base-apt
mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- install -v -m644 "${BASEAPTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
else
install -v -m644 "${APTSRCS}" \
"${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Make the lookup and the checksum comparison one function of a
repository. That cleans things up and allows for easier reuse.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 31 ++++++----------------
meta/classes/repository.bbclass | 19 +++++++++++++
2 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index c3ee7b83..4123326e 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -7,16 +7,6 @@

inherit repository

-compare_pkg_md5sums() {
- pkg1=$1
- pkg2=$2
-
- md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
- md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
-
- [ "$md1" = "$md2" ]
-}
-
populate_base_apt() {

find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can

@@ -25,21 +15,16 @@ populate_base_apt() {
# same and should not be overwritten. This method is easier and more
# robust than querying reprepro by name.

- # Check if this package is taken from Isar-apt, if so - ignore it.
- base_name=${package##*/}
- isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
- if [ -n "$isar_apt_p" ]; then
- # Check if MD5 sums are identical. This helps to avoid the case
- # when packages is overridden from another repo.
- compare_pkg_md5sums "$package" "$isar_apt_p" && continue
- fi
+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue

# Check if this package is already in base-apt
- base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
- if [ -n "$base_apt_p" ]; then
- compare_pkg_md5sums "$package" "$base_apt_p" && continue
-
- # md5sum differs, so remove the package from base-apt
+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ ret=$?
+ [ "${ret}" = "0" ] && continue
+ if [ "${ret}" = "1" ]; then
repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \

diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index dc4c1dcd..e7acf591 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -81,3 +81,22 @@ repo_del_package() {
remove "${codename}" \
"${p}"
}
+
+repo_contains_package() {
+ local dir="$1"
+ local file="$2"
+ local package
+
+ package=$(find ${dir} -name ${file##*/})
+ if [ -n "$package" ]; then
+ local md1=$(md5sum "$package" | cut -d ' ' -f 1)
+ local md2=$(sudo md5sum "$file" | cut -d ' ' -f 1)
+
+ # yes
+ [ "${md1}" = "${md2}" ] && return 0
+ # yes but not the exact same file
+ return 1
+ fi
+ # no
+ return 2
+}
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

If we ever downloaded a file there is no need to do that again. We can
not only use that DL_DIR to later construct a repository but also to
speed up our rebuilds.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 10 ++++++++++

meta/classes/dpkg-gbp.bbclass | 1 +
meta/classes/dpkg.bbclass | 1 +

meta/classes/image-tools-extension.bbclass | 1 +
meta/classes/rootfs.bbclass | 6 ++++++
5 files changed, 19 insertions(+)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index e41e9813..ab4b1f09 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -3,6 +3,16 @@
#
# SPDX-License-Identifier: MIT

+deb_dl_dir_import() {

+ export pc="${DEBDIR}/${DISTRO}"
+ export rootfs="${1}"

+ [ ! -d "${pc}" ] && return 0
+ flock -s "${pc}".lock -c ' \
+ sudo find "${pc}" -type f -iname '*\.deb' -exec \
+ cp -f --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
+ '
+}
+
deb_dl_dir_export() {

export pc="${DEBDIR}/${DISTRO}"

export rootfs="${1}"
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 7e908eba..afa1e19b 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,6 +12,7 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"

do_install_builddeps_append() {
dpkg_do_mounts
+ deb_dl_dir_import "${BUILDCHROOT_DIR}"

sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \

--no-install-recommends --download-only ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 7e2c86d6..585365ee 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -7,6 +7,7 @@ inherit dpkg-base

do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"

+ deb_dl_dir_import "${BUILDCHROOT_DIR}"

sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \

${PP}/${PPS} ${DISTRO_ARCH} --download-only
deb_dl_dir_export "${BUILDCHROOT_DIR}"
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index ec67d940..6590ee7a 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -25,6 +25,7 @@ do_install_imager_deps() {
buildchroot_do_mounts

E="${@ isar_export_proxies(d)}"
+ deb_dl_dir_import ${BUILDCHROOT_DIR}

sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \

apt-get update \

-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \

diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 25af4688..4e145400 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -113,6 +113,12 @@ rootfs_install_resolvconf() {
fi
}

+ROOTFS_INSTALL_COMMAND += "rootfs_import_package_cache"
+rootfs_import_package_cache[weight] = "5"
+rootfs_import_package_cache() {
+ deb_dl_dir_import ${ROOTFSDIR}
+}
+
ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
rootfs_install_pkgs_download[weight] = "600"
rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Now that we populate just from the DL_DIR and filter out the isar-apt
packages when populating this, we can skip that test since it will never
match.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/recipes-devtools/base-apt/base-apt.bb | 4 ----
1 file changed, 4 deletions(-)

diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 2dba7794..cfc26edc 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -17,10 +17,6 @@ populate_base_apt() {

# same and should not be overwritten. This method is easier and more
# robust than querying reprepro by name.

- # Check if this package is taken from Isar-apt, if so - ingore it.
- repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
- continue
-
# Check if this package is already in base-apt
ret=0

repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Users had to call a special task before switching to using base-apt.
With this commit you flip one switch and the repo will be created as one
of the first steps.

The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
"processed" files and not just a download-cache.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

doc/user_manual.md | 8 +--
meta/classes/image-cache-extension.bbclass | 64 ------------------
meta/classes/image.bbclass | 1 -
meta/classes/rootfs.bbclass | 2 +-
meta/conf/bitbake.conf | 4 +-
.../isar-bootstrap/isar-bootstrap.inc | 1 +
meta/recipes-devtools/base-apt/base-apt.bb | 67 ++++++++++++++++---
scripts/ci_build.sh | 4 +-
8 files changed, 69 insertions(+), 82 deletions(-)
delete mode 100644 meta/classes/image-cache-extension.bbclass

diff --git a/doc/user_manual.md b/doc/user_manual.md
index c67c1778..ebf57940 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -823,10 +823,10 @@ path to the public key in `conf/local.conf`, e.g.:
BASE_REPO_KEY = "file://<absolute_path_to_your_pub_key_file>"'
```

- - Trigger creation of local apt caching Debian packages during image generation.
+ - Trigger the download and caching of all required files by doing a warm-up build.

```
-bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
+bitbake mc:qemuarm-buster:isar-image-base
```

- Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
@@ -836,14 +836,14 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
#ISAR_USE_CACHED_BASE_REPO ?= "1"
#BB_NO_NETWORK ?= "1"
```
- - Remove build artifacts to use only local base-apt:
+ - Remove build artifacts to use only local base-apt, in fact toggling ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is just the way to be extra sure that only the download cache is used.

```
sudo rm -rf tmp

```

- - Trigger again generation of image (now using local caching repo):
+ - Trigger the generation of your image again (now a local repo will be created out of the download cache from the last run):

```
bitbake mc:qemuarm-buster:isar-image-base
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
deleted file mode 100644
index 4123326e..00000000
--- a/meta/classes/image-cache-extension.bbclass
+++ /dev/null
@@ -1,64 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) Siemens AG, 2019
-#
-# SPDX-License-Identifier: MIT
-#
-# This class extends the image.bbclass to supply the creation of cache repositories
-
-inherit repository
-
-populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the
- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.

-
- # Check if this package is taken from Isar-apt, if so - ingore it.
- repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
- continue
-
- # Check if this package is already in base-apt

- ret=0
- repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
- ret=$?
- [ "${ret}" = "0" ] && continue
- if [ "${ret}" = "1" ]; then
- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
- fi
-
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-}
-
-do_cache_base_repo[depends] = "base-apt:do_cache_config"
-do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
-do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}"
-do_cache_base_repo() {
- if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
- [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
- # We would need two separate repository paths for that.
- # Otherwise packages (especially the 'all' arch ones) from one
- # distribution can influence the package versions of the other
- # distribution.
- bbfatal "Different host and target distributions are currently not supported." \
- "Try it without cross-build."
- fi
-
- populate_base_apt
-}
-addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index b687fcfb..cfd617af 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"

inherit rootfs
inherit image-sdk-extension
-inherit image-cache-extension
inherit image-tools-extension
inherit image-postproc-extension
inherit image-locales-extension
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 557756c7..54b5e5c9 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {

do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
-do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config"
+do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache"
do_rootfs_install[deptask] = "do_deploy_deb"
python do_rootfs_install() {
configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or "").split()
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index d86c5b92..e46142da 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db"
THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"

# Base apt repository paths
-REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
-REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
+REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
+REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"

BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH BBSERVER DL_DIR \
SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 2078115b..a93c4e3b 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -219,6 +219,7 @@ def get_host_release():

do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"

+do_bootstrap[depends] = "base-apt:do_cache"

isar_bootstrap() {
deb_dl_dir_import "${ROOTFSDIR}"
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index c9a7dad1..2dba7794 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -9,26 +9,77 @@ inherit repository
BASE_REPO_KEY ?= ""
KEYFILES ?= ""

-do_cache_config[stamp-extra-info] = "${DISTRO}"
-do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+populate_base_apt() {
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ # NOTE: due to packages stored by reprepro are not modified, we can
+ # use search by filename to check if package is already in repo. In
+ # addition, md5sums are compared to ensure that the package is the
+ # same and should not be overwritten. This method is easier and more
+ # robust than querying reprepro by name.

-# Generate reprepro config for current distro if it doesn't exist. Once it's
-# generated, this task should do nothing.
-repo_config() {
+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue
+
+ # Check if this package is already in base-apt

+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ ret=$?
+ [ "${ret}" = "0" ] && continue
+ if [ "${ret}" = "1" ]; then

+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"

+ fi
+

+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"

+ done
+

+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do

+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"

+ done
+}
+
+do_cache[stamp-extra-info] = "${DISTRO}"
+do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+
+repo() {
repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \

"${KEYFILES}"
+
+ if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
+ [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
+ # We would need two separate repository paths for that.
+ # Otherwise packages (especially the 'all' arch ones) from one
+ # distribution can influence the package versions of the other
+ # distribution.
+ bbfatal "Different host and target distributions are currently not supported." \
+ "Try it without cross-build."
+ fi
+
+ populate_base_apt
}

-python do_cache_config() {
+python do_cache() {
+ if not bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
+ return 0
+
for key in d.getVar('BASE_REPO_KEY').split():
d.appendVar("SRC_URI", " %s" % key)
fetcher = bb.fetch2.Fetch([key], d)
filename = fetcher.localpath(key)
d.appendVar("KEYFILES", " %s" % filename)

- bb.build.exec_func('repo_config', d)
+ bb.build.exec_func('repo', d)
}

-addtask cache_config after do_unpack before do_build
+addtask cache after do_unpack before do_build
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7da3ee91..733ba86e 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then

# Enable use of signed cached base repository
echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\" >> conf/local.conf
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED
+ bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
@@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf

# Enable use of unsigned cached base repository
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
+ bitbake $BB_ARGS $REPRO_TARGETS_SET
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

During review of the original series it was found that the offline build
did not work for kernels. That has been solved by the new way we build
kernels.

Still having the test and increased coverage is a good idea.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

scripts/ci_build.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 733ba86e..adc403be 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -50,7 +50,7 @@ CROSS_TARGETS_SET="\
mc:rpi-stretch:isar-image-base"

REPRO_TARGETS_SET_SIGNED="\
- mc:qemuarm-stretch:isar-image-base \
+ mc:de0-nano-soc-stretch:isar-image-base \
mc:qemuarm64-stretch:isar-image-base"

REPRO_TARGETS_SET="\
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Packages from isar-apt are not downloaded from the outside and should
not be cached.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index ab4b1f09..f9699603 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -3,8 +3,10 @@
#
# SPDX-License-Identifier: MIT

+inherit repository
+
deb_dl_dir_import() {
- export pc="${DEBDIR}/${DISTRO}"
+ export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"

[ ! -d "${pc}" ] && return 0

flock -s "${pc}".lock -c ' \

@@ -14,12 +16,16 @@ deb_dl_dir_import() {
}

deb_dl_dir_export() {
- export pc="${DEBDIR}/${DISTRO}"
+ export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"
mkdir -p "${pc}"

flock "${pc}".lock -c ' \

- sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' \
- -exec cp -f '{}' "${pc}" \;
+ find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' |\
+ while read p; do
+ repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" && \
+ continue
+ sudo cp -f "${p}" "${pc}"
+ done

sudo chown -R $(id -u):$(id -g) "${pc}"

'
}
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

The base-apt does not rely on the rootfs-cache anymore, so we can test
our cleanup package in all images.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta-isar/conf/local.conf.sample | 4 +---
scripts/ci_build.sh | 2 --
2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 0e55da19..dee12505 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -166,9 +166,7 @@ CONF_VERSION = "1"

#
# The default list of extra packages to be installed.
-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello"

-## "cache_base_repo" relies on the cache ...
-#IMAGE_INSTALL += "isar-disable-apt-cache"

+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache"

#
# Enable cross-compilation support

diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index f50c9e42..bfaf1f6e 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh

@@ -168,8 +168,6 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
fi

-sed -i -e 's/#IMAGE_INSTALL += "isar-disable-apt-cache"/IMAGE_INSTALL += "isar-disable-apt-cache"/g' conf/local.conf
-

echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf

# Start cross build for the defined set of configurations

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Just call cmp once instead of comparing the md5 sums.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/repository.bbclass | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index e7acf591..de1fa5eb 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -89,11 +89,9 @@ repo_contains_package() {

package=$(find ${dir} -name ${file##*/})

if [ -n "$package" ]; then
- local md1=$(md5sum "$package" | cut -d ' ' -f 1)
- local md2=$(sudo md5sum "$file" | cut -d ' ' -f 1)
-
# yes
- [ "${md1}" = "${md2}" ] && return 0
+ cmp --silent "$package" "$file" && return 0

+
# yes but not the exact same file

return 1
fi
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

There is only one place we use that code, move it there instead of
having an extra class.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/base-apt-helper.bbclass | 60 ----------------------
meta/classes/image-cache-extension.bbclass | 55 +++++++++++++++++++-
2 files changed, 54 insertions(+), 61 deletions(-)
delete mode 100644 meta/classes/base-apt-helper.bbclass

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
deleted file mode 100644
index 18d43aa3..00000000
--- a/meta/classes/base-apt-helper.bbclass
+++ /dev/null
@@ -1,60 +0,0 @@

-# This software is a part of ISAR.

-# Copyright (C) 2018 ilbers GmbH
-# Copyright (C) 2019 Siemens AG
-#
-# SPDX-License-Identifier: MIT
-
-inherit repository
-

-compare_pkg_md5sums() {
- pkg1=$1
- pkg2=$2
-

- md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
- md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
-

- [ "$md1" = "$md2" ]
-}

-
-populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the
- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.
-

- # Check if this package is taken from Isar-apt, if so - ignore it.
- base_name=${package##*/}
- isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
- if [ -n "$isar_apt_p" ]; then
- # Check if MD5 sums are identical. This helps to avoid the case
- # when packages is overridden from another repo.
- compare_pkg_md5sums "$package" "$isar_apt_p" && continue
- fi

-
- # Check if this package is already in base-apt

- base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
- if [ -n "$base_apt_p" ]; then
- compare_pkg_md5sums "$package" "$base_apt_p" && continue
-
- # md5sum differs, so remove the package from base-apt

- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
- fi
-
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-}

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 7945e8bc..c3ee7b83 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -5,7 +5,60 @@
#

# This class extends the image.bbclass to supply the creation of cache repositories

-inherit base-apt-helper
+inherit repository
+
+compare_pkg_md5sums() {
+ pkg1=$1
+ pkg2=$2
+
+ md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
+ md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
+
+ [ "$md1" = "$md2" ]
+}
+

+populate_base_apt() {
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ # NOTE: due to packages stored by reprepro are not modified, we can
+ # use search by filename to check if package is already in repo. In
+ # addition, md5sums are compared to ensure that the package is the
+ # same and should not be overwritten. This method is easier and more
+ # robust than querying reprepro by name.

+
+ # Check if this package is taken from Isar-apt, if so - ignore it.
+ base_name=${package##*/}
+ isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
+ if [ -n "$isar_apt_p" ]; then
+ # Check if MD5 sums are identical. This helps to avoid the case
+ # when packages is overridden from another repo.
+ compare_pkg_md5sums "$package" "$isar_apt_p" && continue
+ fi

+
+ # Check if this package is already in base-apt

+ base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
+ if [ -n "$base_apt_p" ]; then
+ compare_pkg_md5sums "$package" "$base_apt_p" && continue
+
+ # md5sum differs, so remove the package from base-apt

+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
+ fi
+
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+}

do_cache_base_repo[depends] = "base-apt:do_cache_config"

do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Remember the time we imported the previously downloaded files, and only
copy out new ones when exporting again. That saved pointless copies and
time.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index f9699603..e65fc518 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -8,24 +8,31 @@ inherit repository
deb_dl_dir_import() {

export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"

+ rm -f "${T}"/deb_dl_dir_import.stamp
+ touch "${T}"/deb_dl_dir_import.stamp

[ ! -d "${pc}" ] && return 0

+ [ ! -d "${rootfs}"/var/cache/apt/archives/ ] && return 0

flock -s "${pc}".lock -c ' \

sudo find "${pc}" -type f -iname '*\.deb' -exec \
- cp -f --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
+ cp -n --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
'
}

deb_dl_dir_export() {

export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"

+ export T

mkdir -p "${pc}"
flock "${pc}".lock -c ' \

- find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' |\
+ find "${rootfs}"/var/cache/apt/archives/ \
+ -cnewer "${T}"/deb_dl_dir_import.stamp \
+ -type f -iname '*\.deb' |\
while read p; do

repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" && \

continue
- sudo cp -f "${p}" "${pc}"
+ sudo cp -n "${p}" "${pc}"

done
sudo chown -R $(id -u):$(id -g) "${pc}"
'

+ rm -f "${T}"/deb_dl_dir_import.stamp
}
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Also add it to our sample config and comment on its use in conjuction
with base-apt.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

doc/user_manual.md | 1 +
meta-isar/conf/local.conf.sample | 4 ++++
scripts/ci_build.sh | 4 ++++
3 files changed, 9 insertions(+)

diff --git a/doc/user_manual.md b/doc/user_manual.md
index d501a706..c67c1778 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -834,6 +834,7 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
```
# Uncomment this to enable use of cached base repository
#ISAR_USE_CACHED_BASE_REPO ?= "1"
+#BB_NO_NETWORK ?= "1"
```

- Remove build artifacts to use only local base-apt:

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 9237bb35..8355a40c 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -176,6 +176,10 @@ ISAR_CROSS_COMPILE ?= "0"
#
# Uncomment this to enable use of cached base repository
#ISAR_USE_CACHED_BASE_REPO ?= "1"
+#
+# You probably want to uncomment this as well to make sure the build
+# does not access the network
+#BB_NO_NETWORK ?= "1"

# Set root password to 'root'
# Password was encrypted using following command:
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index d440c248..7da3ee91 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -149,11 +149,13 @@ if [ -n "$REPRO_BUILD" ]; then

while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf

+ sed -i -e 's/^#BB_NO_NETWORK/BB_NO_NETWORK/g' conf/local.conf

bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
while [ -e bitbake.sock ]; do sleep 1; done

# Cleanup and disable use of signed cached base repository
sudo rm -rf tmp

sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf

+ sed -i -e 's/^BB_NO_NETWORK/#BB_NO_NETWORK/g' conf/local.conf

sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf

# Enable use of unsigned cached base repository

@@ -161,11 +163,13 @@ if [ -n "$REPRO_BUILD" ]; then

while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf

+ sed -i -e 's/^#BB_NO_NETWORK/BB_NO_NETWORK/g' conf/local.conf

bitbake $BB_ARGS $REPRO_TARGETS_SET
while [ -e bitbake.sock ]; do sleep 1; done

# Cleanup and disable use of unsigned cached base repository
sudo rm -rf tmp

sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf

+ sed -i -e 's/^BB_NO_NETWORK/#BB_NO_NETWORK/g' conf/local.conf
fi

[Henning Schild]

https://github.com/henning-schild-work/isar/tree/henning/staging4

[Jan Kiszka]

repo_contains_package may not be found inside the flock shell context,
as Bao Cheng noticed out. He suggests the pattern

(flock 8
...
) 8>${LOCKFILE}

Bao Check, maybe you can describe how you noticed.

Jan

> + continue
> + sudo cp -f "${p}" "${pc}"
> + done
> sudo chown -R $(id -u):$(id -g) "${pc}"
> '
> }
>

--

Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

[Henning Schild]

Am Mon, 3 Feb 2020 18:20:11 +0100
schrieb Jan Kiszka <jan.k...@siemens.com>:

That is possible indeed. I remember that i struggled re-using the
function since i did not want to code it twice.

But to be honest i do not understand the problem with the given
description. Bao Cheng please go into more detail.

Maybe the filtering indeed does not work, i will try that on a simple
test now. And it might be a good idea to make sure that code gets a
"set -e".

Henning

[Henning Schild]

Am Mon, 3 Feb 2020 19:27:01 +0100
schrieb "[ext] Henning Schild" <henning...@siemens.com>:

This is a hot code-path and if it is _very_ broken that would be very
visible.

Just tried a build and am looking at the download cache and isar-apt

find tmp/deploy/isar-apt/ -iname *hello*
tmp/deploy/isar-apt/apt/debian-buster/pool/main/libh/libhello
tmp/deploy/isar-apt/apt/debian-buster/pool/main/libh/libhello/libhello-dbgsym_0.1_amd64.deb
tmp/deploy/isar-apt/apt/debian-buster/pool/main/libh/libhello/libhello-dev_0.1_amd64.deb
tmp/deploy/isar-apt/apt/debian-buster/pool/main/libh/libhello/libhello_0.1_amd64.deb
....
tmp/deploy/isar-apt/apt/debian-buster/pool/main/h/hello-isar/hello-isar_0.3_amd64.deb

Ok so we have the hello stuff in isar-apt, the own package as well as
the rebuild upstream (also own).

And nothing in the download cache on hello -> expected for the image
find downloads/deb/ -iname *hello*
<nothing>

Meaning the filtering worked in my case.

And the caching of all required outside stuff works as well, since the
offline rebuild works.

Caching and filtering are the two main aspects of that code.

Please make sure to give feedback for sure, i do not want such a
unclear description slowing down the merge even more.

Henning

[Su, Bao Cheng]

after building, grep "command not found" or "repo_contains_package" in the log.do_* file.

I met this problem on an early version of v4 round patch, not sure if still exists on the latest version. will run a test build against the latest patch to check.

Sorry for mailing via phone, due to coronavirus, not convenient for me to use laptop emails.

With best regards

---

发件人： Schild, Henning (CT RDA IOT SES-DE)
发送时间： 2020年2月4日星期二 02:59
收件人： Kiszka, Jan (CT RDA IOT SES-DE)
抄送： isar-...@googlegroups.com; Su, Bao Cheng (RC-CN DI FA R&D SW); Vijai Kumar K
主题： Re: [PATCHv4 22/26] meta: deb-dl-dir: do not cache debs from isar-apt

[Su, Bao Cheng]

Still exists.

Under `build/tmp/work/`:

$ grep "command not found" . -r --include="log.do_*"
./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash: line 5: repo_contains_package: command not found
./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash: line 5: repo_contains_package: command not found
./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash: line 5: repo_contains_package: command not found
./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash: line 5: repo_contains_package: command not found

$ grep "command not found" . -r --include="log.do_*" | wc -l
1596

This is only a half-way build, for a full build, there are more occurrences.

With best regards

---

发件人： Su, Bao Cheng (RC-CN DI FA R&D SW)
发送时间： 2020年2月4日星期二 10:11
收件人： Schild, Henning (CT RDA IOT SES-DE); Kiszka, Jan (CT RDA IOT SES-DE)
抄送： isar-...@googlegroups.com; Vijai Kumar K

[Jan Kiszka]

On 04.02.20 06:10, Su, Bao Cheng (RC-CN DI FA R&D SW) wrote:
> Still exists.
>
> Under `build/tmp/work/`:
>
> $ grep "command not found" . -r --include="log.do_*"
> ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> line 5: repo_contains_package: command not found
> ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> line 5: repo_contains_package: command not found
> ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> line 5: repo_contains_package: command not found
> ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> line 5: repo_contains_package: command not found
>
> $ grep "command not found" . -r --include="log.do_*" | wc -l
> 1596
>
> This is only a half-way build, for a full build, there are more occurrences.

And if I read the code correctly, the issue only prevents an
optimization, not the correct functioning of the code. Thus, when fixing
this, building should be even faster :).

Thanks,
Jan

>
> With best regards
> ------------------------------------------------------------------------
> *发件人：* Su, Bao Cheng (RC-CN DI FA R&D SW)
> *发送时间：* 2020年2月4日星期二 10:11
> *收件人：* Schild, Henning (CT RDA IOT SES-DE); Kiszka, Jan (CT RDA IOT
> SES-DE)
> *抄送：* isar-...@googlegroups.com; Vijai Kumar K
> *主题：* Re: [PATCHv4 22/26] meta: deb-dl-dir: do not cache debs from

> isar-apt
>
> after building, grep "command not found" or "repo_contains_package" in
> the log.do_* file.
>
> I met this problem on an early version of v4 round patch, not sure if
> still exists on the latest version. will run a test build against the
> latest patch to check.
>
> Sorry for mailing via phone, due to coronavirus, not convenient for me
> to use laptop emails.
>
> With best regards

> ------------------------------------------------------------------------
> *发件人：* Schild, Henning (CT RDA IOT SES-DE)
> *发送时间：* 2020年2月4日星期二 02:59
> *收件人：* Kiszka, Jan (CT RDA IOT SES-DE)
> *抄送：* isar-...@googlegroups.com; Su, Bao Cheng (RC-CN DI FA R&D
> SW); Vijai Kumar K
> *主题：* Re: [PATCHv4 22/26] meta: deb-dl-dir: do not cache debs from

[Su, Bao Cheng]

I think so, the build will complete with this issue.

With best regards

---

发件人： Kiszka, Jan (CT RDA IOT SES-DE)
发送时间： 2020年2月4日星期二 13:14
收件人： Su, Bao Cheng (RC-CN DI FA R&D SW); Schild, Henning (CT RDA IOT SES-DE)

抄送： isar-...@googlegroups.com; Vijai Kumar K

主题： Re: [PATCHv4 22/26] meta: deb-dl-dir: do not cache debs from isar-apt

[Henning Schild]

I can not reproduce this issue. Meaning we have different build
environments or different isar-versions.

Since i know you ported to q onto some unknown-to-me state of isar and
i do not know your build environment, i would blame of the two.

I suggest you try your build env on
https://github.com/henning-schild-work/isar/tree/henning/staging4

If the error occurs is could be connected to the build env. I am using
a docker based workflow with "kasproject/kas-isar:latest" where latest
is pretty new.

If the error does not occur i would blame your isar-version or your
layer.

And with blame i mean we have to look for differences there, nothing
more.

Henning

[Henning Schild]

On Tue, 4 Feb 2020 06:14:00 +0100
Jan Kiszka <jan.k...@siemens.com> wrote:

> On 04.02.20 06:10, Su, Bao Cheng (RC-CN DI FA R&D SW) wrote:
> > Still exists.
> >
> > Under `build/tmp/work/`:
> >
> > $ grep "command not found" . -r --include="log.do_*"
> > ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> > line 5: repo_contains_package: command not found
> > ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> > line 5: repo_contains_package: command not found
> > ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> > line 5: repo_contains_package: command not found
> > ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> > line 5: repo_contains_package: command not found
> >
> > $ grep "command not found" . -r --include="log.do_*" | wc -l
> > 1596
> >
> > This is only a half-way build, for a full build, there are more
> > occurrences.
>
> And if I read the code correctly, the issue only prevents an
> optimization, not the correct functioning of the code. Thus, when
> fixing this, building should be even faster :).

No, it is putting packages we built into the dl-cache where they really
do not belong. Having them in there is harmful because it will most
likely cause issues when rebuilding, checksum changes on
pre-downloaded files ... something that can not happen with real
downloads. But something that is probably common in Isar, when people
do not bump PV on every change.

Henning

[Henning Schild]

On Tue, 4 Feb 2020 05:10:05 +0000
"Su, Bao Cheng (RC-CN DI FA R&D SW)" <baoch...@siemens.com> wrote:

> Still exists.
>
> Under `build/tmp/work/`:
>
> $ grep "command not found" . -r --include="log.do_*"
> ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> line 5: repo_contains_package: command not found
> ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> line 5: repo_contains_package: command not found
> ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> line 5: repo_contains_package: command not found
> ./isar-arm64/isar-bootstrap-target/1.0-r0/temp/log.do_bootstrap.329:/bin/bash:
> line 5: repo_contains_package: command not found
>
> $ grep "command not found" . -r --include="log.do_*" | wc -l
> 1596
>
> This is only a half-way build, for a full build, there are more
> occurrences.

Ok i can reproduce the issue and am investigating, will probably take
the other flock notation.

Funny thing is that despite our efforts to make the build env
reproducible we are using "dash" in a plain Isar build and "bash" in a
kas based build.

That is what dash has to say ..
/bin/sh: 9: repo_contains_package: not found

So the grep did not find it.

What i still do not understand is how the download cache seems to be or
is exactly in the state i would expect. You probably did not patch
because of the "command not found" but because of a result of some sort.

Have packages been missing in the dl cache? What kind of packages? Did
this break your offline rebuild, or did it show up in another context
(maybe clearing).

Thanks,
Henning

[Su, Bao Cheng]

Glad of hearing you have spotted the issue.

It seemed to be ok with this "command not found" log and I remember it's ok to rebuild offline even with the command not found, but I am not very sure now because I only have dim memories for it.

I only remember I noticed it while investigating another issue of ours, by accidently looking into the build log.

[Baurzhan Ismagulov]

Hello Henning,

On Fri, Jan 31, 2020 at 03:29:34PM +0100, Henning Schild wrote:
> Rebased/merged onto current next and added p26. The custom kernel caching
> that was missing in v2 is not required on next anymore.

Thanks for the series. Took some time, since I had CI failures before the
machine was made public, sorry for the delay.

Two issues:


1. [PATCHv4 01/26] repository: new class to deal with repos

On Mon, Nov 25, 2019 at 01:03:50PM +0100, Henning Schild wrote:
> > I suggest to keep modifying distributions.in and avoid hard-coding
> > architectures and components that are specific to the current Debian.
>
> Same comment as before. As far as i know this is not a visible
> interface and was never meant to be changed by i.e. a layer.
> Distributing the code over multiple files, with just one place that
> uses it, is a layer of abstraction that just makes the code less
> readable.

Adding riscv64 should not require editing a bbclass. Currently, the code and
the configuration are separated. I don't see why we should break that and
suggest to keep it that way. I'll send a reworked patch.


2. [PATCHv4 08/26] meta: split all apt-get invocations into download and execution

On Fri, Jan 31, 2020 at 03:29:42PM +0100, Henning Schild wrote:
...
> diff --git a/meta/recipes-devtools/buildchroot/files/deps.sh b/meta/recipes-devtools/buildchroot/files/deps.sh
...

> +if [ "$3" != "-download-only" ]; then

...

> +if [ "$3" = "--download-only" ]; then

In v1, both had two dashes. I assume "--download-only" is the intended one?


What is your plan regarding "command not found"?


With kind regards,
Baurzhan.

[Baurzhan Ismagulov]

From: Henning Schild <henning...@siemens.com>

Factor out all the "reprepro" code into a common class. There are slight
functional changes since the two copies of the code got out of sync.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/base-apt-helper.bbclass | 28 ++++----
meta/classes/dpkg-base.bbclass | 24 ++-----
meta/classes/repository.bbclass | 67 +++++++++++++++++++
meta/recipes-devtools/base-apt/base-apt.bb | 38 +++--------
.../isar-apt/files/distributions.in | 2 +-
meta/recipes-devtools/isar-apt/isar-apt.bb | 27 +++-----
6 files changed, 105 insertions(+), 81 deletions(-)
create mode 100644 meta/classes/repository.bbclass

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 90b2cfc..1f08c70 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -1,5 +1,10 @@

# This software is a part of ISAR.

# Copyright (C) 2018 ilbers GmbH

+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+inherit repository

compare_pkg_md5sums() {
pkg1=$1
@@ -14,10 +19,6 @@ compare_pkg_md5sums() {
populate_base_apt() {
search_dir=$1

- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
find $search_dir -name '*.deb' | while read package; do

# NOTE: due to packages stored by reprepro are not modified, we can

# use search by filename to check if package is already in repo. In

@@ -40,18 +41,15 @@ populate_base_apt() {

compare_pkg_md5sums "$package" "$base_apt_p" && continue

# md5sum differs, so remove the package from base-apt

- name=$(echo $base_name | cut -d '_' -f 1)
- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main -A ${DISTRO_ARCH} \
- remove ${BASE_DISTRO_CODENAME} \
- $name

+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"

fi

- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main \
- includedeb ${BASE_DISTRO_CODENAME} \
- $package

+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"

done
}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 1decf58..3d0a905 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -7,6 +7,7 @@
inherit buildchroot
inherit debianize
inherit terminal
+inherit repository

DEPENDS ?= ""

@@ -142,31 +143,16 @@ repo_clean() {
DEBS=$( find ${S}/.. -maxdepth 1 -name "*.deb" || [ ! -d ${S} ] )
if [ -n "${DEBS}" ]; then
for d in ${DEBS}; do
- p=$( dpkg-deb --show --showformat '${Package}' ${d} )
- a=$( dpkg-deb --show --showformat '${Architecture}' ${d} )
- # removing "all" means no arch
- aarg="-A ${a}"
- [ "${a}" = "all" ] && aarg=""
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main ${aarg} \
- remove ${DEBDISTRONAME} \
- ${p}
+ repo_del_package "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" "${d}"
done
fi
}

-# Install package to Isar-apt
do_deploy_deb() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
repo_clean
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main \
- includedeb ${DEBDISTRONAME} \
- ${S}/../*.deb
+ repo_add_packages "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" ${S}/../*.deb
}

addtask deploy_deb after do_dpkg_build before do_build
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
new file mode 100644
index 0000000..04a6454
--- /dev/null
+++ b/meta/classes/repository.bbclass
@@ -0,0 +1,67 @@
+# This software is a part of ISAR.
+# Copyright (C) 2017-2019 Siemens AG
+# Copyright (C) 2019 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+repo_create() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ local reprepro_in="$4"
+ local keyfiles="$5"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+
+ if [ ! -f "${dir}"/conf/distributions ]; then
+ mkdir -p "${dir}"/conf/
+ sed -e "s#{CODENAME}#${codename}#g" ${reprepro_in} \
+ >"${dir}"/conf/distributions
+ if [ -n "${keyfiles}" ] ; then
+ local option=""
+ for key in ${keyfiles}; do
+ keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
+ option="${option}${keyid} "
+ done
+ echo "SignWith: ${option}" >> "${dir}"/conf/distributions
+ fi
+ fi
+ if [ ! -d "${dbdir}" ]; then
+ reprepro -b "${dir}" --dbdir "${dbdir}" export "${codename}"
+ fi
+}
+
+repo_add_packages() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main \
+ includedeb "${codename}" \
+ "$@"
+}
+
+repo_del_package() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ local file="$4"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ local p=$( dpkg-deb --show --showformat '${Package}' "${file}" )
+ local a=$( dpkg-deb --show --showformat '${Architecture}' "${file}" )
+ # removing "all" means no arch
+ local aarg="-A ${a}"
+ [ "${a}" = "all" ] && aarg=""
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main ${aarg} \
+ remove "${codename}" \
+ "${p}"
+}
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 9a0f7c8..8bb169c 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -1,45 +1,27 @@

# This software is a part of ISAR.

# Copyright (C) 2018 ilbers GmbH

+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+inherit repository

SRC_URI = "file://distributions.in"

BASE_REPO_KEY ?= ""
KEYFILES ?= ""

-CACHE_CONF_DIR = "${REPO_BASE_DIR}/${BASE_DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"

# Generate reprepro config for current distro if it doesn't exist. Once it's

# generated, this task should do nothing.

repo_config() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- if [ -n "${KEYFILES}" ]; then
- option=""
- for key in ${KEYFILES}; do
- keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
- option="${option}${keyid} "
- done
- # To generate Release.gpg
- echo "SignWith: ${option}" >> ${CACHE_CONF_DIR}/distributions
- fi
- fi
-
- path_cache="${REPO_BASE_DIR}/${BASE_DISTRO}"
- path_databases="${REPO_BASE_DB_DIR}/${BASE_DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${BASE_DISTRO_CODENAME}
- fi
+ repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \

+ "${WORKDIR}/distributions.in" \
+ "${KEYFILES}"
}

python do_cache_config() {
diff --git a/meta/recipes-devtools/isar-apt/files/distributions.in b/meta/recipes-devtools/isar-apt/files/distributions.in
index 6471736..3cf7ea5 100644
--- a/meta/recipes-devtools/isar-apt/files/distributions.in
+++ b/meta/recipes-devtools/isar-apt/files/distributions.in
@@ -1,3 +1,3 @@
-Codename: {DISTRO_NAME}
+Codename: {CODENAME}
Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
Components: main
diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index e6f1753..c052934 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -1,32 +1,23 @@

# This software is a part of ISAR.

# Copyright (C) 2015-2017 ilbers GmbH
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+inherit repository

SRC_URI = "file://distributions.in"

-CACHE_CONF_DIR = "${REPO_ISAR_DIR}/${DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"

# Generate reprepro config for current distro if it doesn't exist. Once it's

# generated, this task should do nothing.

do_cache_config() {
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{DISTRO_NAME}#"${DEBDISTRONAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- fi
-
- path_cache="${REPO_ISAR_DIR}/${DISTRO}"
- path_databases="${REPO_ISAR_DB_DIR}/${DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${DEBDISTRONAME}
- fi
+ repo_create "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" \
+ "${DEBDISTRONAME}" \
+ "${WORKDIR}/distributions.in"

}

addtask cache_config after do_unpack before do_build

--
2.20.1

[Baurzhan Ismagulov]

From: Henning Schild <henning...@siemens.com>

This commit does not contain a functional change. We just split all
calls of "apt-get" into a download and an execution phase, so we can
later copy out the downloaded files.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/dpkg-gbp.bbclass | 3 ++
meta/classes/dpkg.bbclass | 5 ++-
meta/classes/image-tools-extension.bbclass | 5 +++
.../buildchroot/files/deps.sh | 37 +++++++++++--------
4 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 0533a52..2afe9db 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,6 +12,9 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"

do_install_builddeps_append() {
dpkg_do_mounts
+ sudo -E chroot ${BUILDCHROOT_DIR} \
+ apt-get install -y -o Debug::pkgProblemResolver=yes \
+ --no-install-recommends --download-only ${GBP_DEPENDS}

sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \

--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 180a3a8..b26c645 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -7,7 +7,10 @@ inherit dpkg-base

do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"

- sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh ${PP}/${PPS} ${DISTRO_ARCH}
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH}
dpkg_undo_mounts
}

diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index d4f216d..b8531cb 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -30,6 +30,11 @@ do_install_imager_deps() {

-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \

-o Dir::Etc::SourceParts="-" \
-o APT::Get::List-Cleanup="0"
+ apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
+ --allow-unauthenticated --allow-downgrades --download-only install \
+ ${IMAGER_INSTALL}'
+
+ sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \
${IMAGER_INSTALL}'
diff --git a/meta/recipes-devtools/buildchroot/files/deps.sh b/meta/recipes-devtools/buildchroot/files/deps.sh
index 002085b..6048305 100644
--- a/meta/recipes-devtools/buildchroot/files/deps.sh
+++ b/meta/recipes-devtools/buildchroot/files/deps.sh
@@ -12,27 +12,34 @@ source /isar/common.sh
# 2) we add -y to go non-interactive
# 3) downgrades shall be allowed in case a package recipe was changed
install_cmd="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends \
- -y --allow-downgrades"
+ -y --allow-downgrades $3"

-# Make sure that we have latest isar-apt content.
-# Options meaning:
-# Dir::Etc::SourceList - specifies which source to be used
-# Dir::Etc::SourceParts - disables looking for the other sources
-# APT::Get::List-Cleanup - do not erase obsolete packages list for
-# upstream in '/var/lib/apt/lists'
-apt-get update \
- -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
- -o Dir::Etc::SourceParts="-" \
- -o APT::Get::List-Cleanup="0"
+if [ "$3" != "--download-only" ]; then
+ # Make sure that we have latest isar-apt content.
+ # Options meaning:
+ # Dir::Etc::SourceList - specifies which source to be used
+ # Dir::Etc::SourceParts - disables looking for the other sources
+ # APT::Get::List-Cleanup - do not erase obsolete packages list for
+ # upstream in '/var/lib/apt/lists'
+ apt-get update \
+ -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
+ -o Dir::Etc::SourceParts="-" \
+ -o APT::Get::List-Cleanup="0"
+fi

# Do not set an architecture when building only 'all' (generic) packages.
# This can avoid unneeded cross-build issues.
if ! grep "^Architecture:" debian/control | grep -qv "all"; then
- set_arch=""
+ set_arch=""
fi

# Install all build deps
-mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control

+if [ "$3" = "--download-only" ]; then

+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control 2>&1 \
+ | grep "mk-build-deps: Unable to install all build-dep packages"
+else
+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control

-# Upgrade any already installed packages in case we are partially rebuilding
-apt-get upgrade -y --allow-downgrades
+ # Upgrade any already installed packages in case we are partially rebuilding
+ apt-get upgrade -y --allow-downgrades
+fi
--
2.20.1

[Henning Schild]

On Thu, 5 Mar 2020 00:25:37 +0100
Baurzhan Ismagulov <i...@radix50.net> wrote:

> Hello Henning,
>
> On Fri, Jan 31, 2020 at 03:29:34PM +0100, Henning Schild wrote:
> > Rebased/merged onto current next and added p26. The custom kernel
> > caching that was missing in v2 is not required on next anymore.
>
> Thanks for the series. Took some time, since I had CI failures before
> the machine was made public, sorry for the delay.
>
> Two issues:

Thanks for the review, i will look into folding those into a v5. I
think i will add your signed-offs to the ones you touched as well.

>
> 1. [PATCHv4 01/26] repository: new class to deal with repos
>
> On Mon, Nov 25, 2019 at 01:03:50PM +0100, Henning Schild wrote:
> > > I suggest to keep modifying distributions.in and avoid
> > > hard-coding architectures and components that are specific to
> > > the current Debian.
> >
> > Same comment as before. As far as i know this is not a visible
> > interface and was never meant to be changed by i.e. a layer.
> > Distributing the code over multiple files, with just one place
> > that uses it, is a layer of abstraction that just makes the code
> > less readable.
>
> Adding riscv64 should not require editing a bbclass. Currently,
> the code and the configuration are separated. I don't see why we
> should break that and suggest to keep it that way. I'll send a
> reworked patch.

I have a rebased version here, but fair enough, i will proceed with
yours.

>
> 2. [PATCHv4 08/26] meta: split all apt-get invocations into download
> and execution
>
> On Fri, Jan 31, 2020 at 03:29:42PM +0100, Henning Schild wrote:
> ...
> > diff --git a/meta/recipes-devtools/buildchroot/files/deps.sh
> > b/meta/recipes-devtools/buildchroot/files/deps.sh
> ...
> > +if [ "$3" != "-download-only" ]; then
> ...
> > +if [ "$3" = "--download-only" ]; then
>
> In v1, both had two dashes. I assume "--download-only" is the
> intended one?

Good catch.

>
> What is your plan regarding "command not found"?
>

I already have a fix for that since weeks. That showed some weird
errors in CI ... sometimes. I have come to the conclusion that those
must be the network connectivity of the machine i was on.

I hope i will send a v5 today or tomorrow.

Henning

> With kind regards,
> Baurzhan.
>

[Henning Schild]

I will in fact ignore this one again. Since this is not my code anymore
and i would need to compare the patch to my original patch.

My suggestion would be to restore the split of code and config on top
again, if that is ok with you. Maybe directly by you, with the merge.
If you want to fold it, please become a co-author of the patch, with a
signed-off.
My next version will have 2020 copyright headers.

Henning

[Baurzhan Ismagulov]

On Tue, Mar 10, 2020 at 02:14:37PM +0100, Henning Schild wrote:
> I will in fact ignore this one again. Since this is not my code anymore
> and i would need to compare the patch to my original patch.
>
> My suggestion would be to restore the split of code and config on top
> again, if that is ok with you. Maybe directly by you, with the merge.

Fine with me, will apply the split on top.

With kind regards,
Baurzhan.

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Factor out all the "reprepro" code into a common class. There are slight
functional changes since the two copies of the code got out of sync.

Signed-off-by: Henning Schild <henning...@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 28 ++++----
meta/classes/dpkg-base.bbclass | 24 ++-----

meta/classes/repository.bbclass | 69 +++++++++++++++++++
meta/recipes-devtools/base-apt/base-apt.bb | 37 +++-------
.../base-apt/files/distributions.in | 3 -
.../isar-apt/files/distributions.in | 3 -
meta/recipes-devtools/isar-apt/isar-apt.bb | 26 ++-----
7 files changed, 102 insertions(+), 88 deletions(-)
create mode 100644 meta/classes/repository.bbclass

delete mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
delete mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 90b2cfcb..6fc57303 100644

--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -1,5 +1,10 @@
# This software is a part of ISAR.
# Copyright (C) 2018 ilbers GmbH

+# Copyright (C) 2020 Siemens AG

index 5d2c161c..4702e52b 100644

index 00000000..cf0fb5a2
--- /dev/null
+++ b/meta/classes/repository.bbclass
@@ -0,0 +1,69 @@

+# This software is a part of ISAR.

+# Copyright (C) 2017-2020 Siemens AG

+# Copyright (C) 2019 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+repo_create() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"

+ local keyfiles="$4"

+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+

+ if [ ! -f "${dir}"/conf/distributions ]; then
+ mkdir -p "${dir}"/conf/

+ cat <<EOF > "${dir}"/conf/distributions
+Codename: ${codename}
+Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
+Components: main
+EOF

index 9a0f7c8b..7c891fa0 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -1,45 +1,24 @@

# This software is a part of ISAR.
# Copyright (C) 2018 ilbers GmbH

+# Copyright (C) 2020 Siemens AG
+#
+# SPDX-License-Identifier: MIT

-SRC_URI = "file://distributions.in"
+inherit repository

+ "${KEYFILES}"
}

python do_cache_config() {

diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
deleted file mode 100644
index 3cf7ea55..00000000
--- a/meta/recipes-devtools/base-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {CODENAME}
-Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
-Components: main
diff --git a/meta/recipes-devtools/isar-apt/files/distributions.in b/meta/recipes-devtools/isar-apt/files/distributions.in
deleted file mode 100644
index 64717364..00000000
--- a/meta/recipes-devtools/isar-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {DISTRO_NAME}
-Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
-Components: main
diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index e6f1753d..c53b01f7 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -1,32 +1,20 @@

# This software is a part of ISAR.
# Copyright (C) 2015-2017 ilbers GmbH

+# Copyright (C) 2020 Siemens AG
+#
+# SPDX-License-Identifier: MIT

-SRC_URI = "file://distributions.in"
+inherit repository

addtask cache_config after do_unpack before do_build

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Bitbake seems to execute all the functions in subshells, but in fact we
mean to return here so correct the style.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/dpkg-base.bbclass | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 1000ef37..5dd334d0 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -55,7 +55,7 @@ SRC_APT ?= ""

do_apt_fetch() {
if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
- exit
+ return 0
fi
rm -rf ${S}
dpkg_do_mounts
@@ -82,7 +82,7 @@ addtask cleanall_apt before do_cleanall
do_cleanall_apt[nostamp] = "1"
do_cleanall_apt() {
if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
- exit
+ return 0
fi
for uri in "${SRC_APT}"; do
rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

The repository is ready for it already. For the moment there will be
nothing there.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/recipes-core/isar-bootstrap/files/base-apt-sources | 1 +
1 file changed, 1 insertion(+)

diff --git a/meta/recipes-core/isar-bootstrap/files/base-apt-sources b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
index ddc45093..2a4ab2c4 100644
--- a/meta/recipes-core/isar-bootstrap/files/base-apt-sources
+++ b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
@@ -1 +1,2 @@
deb [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
+deb-src [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

changes since v4:

- add missing - in p8 download-only arg matching, found by Baurzhan
- adding patch 27 and 28 to deal with shell code reuse issues pointed out
by Su, Bao Cheng ... "command not found"
- bump copyright headers from 2019 to 2020
- adding p29, RECIPE-API-CHANGELOG

p8 and p1 appeared on the list as v5, but not from me, the p8 thing is
in here the p1 change will probably be added by Baurzhan on a merge, not
calling this v6 because of that repost

changes since v2:

Rebased/merged onto current next and added p26. The custom kernel caching
that was missing in v2 is not required on next anymore.

This one was sent as v3 by Vijai Kumar K
<vijaikumar....@gmail.com>. That just contained a fix in case
the patches would not be rebased, can be ignored.

Henning Schild (29):

repository: new class to deal with repos

dpkg-base: add download caching of apt:// downloads
meta: dpkg-base: convert "exit" into "return 0"
base-apt: change the sources.list to also offer deb-src
base-apt: add deb-src packages as well
base-apt: do not skip gpg check when it is signed
ci: conf: add "hello" to the sample config and every build

meta: split all apt-get invocations into download and execution

meta: create DL_DIR support for all apt-get downloaded .debs
meta: import DL_DIR debs before apt-get download steps
meta: include DL_DIR deb import/export into bootstrap
base-apt: populate from DEBDIR as well
base-apt: drop the "apt_cache" feature
base-apt: do not copy debs directly out of rootfss anymore
base-apt: rework base-apt population
base-apt: move class "base-apt-helper" into only user
CI: include "isar-disable-apt-cache" into all CI images
CI: include "cowsay" into default build to test dpkg-gbp
CI: set BB_NO_NETWORK for cached rebuild
meta: repository: implement repo_contains_package and use it in
base_apt
meta: repository: simplify the matching of packages

meta: deb-dl-dir: do not cache debs from isar-apt

base-apt: pull base-apt population to the front of the build chain
meta: base-apt: remove isar-apt check from population
meta: deb-dl-dir: only export newly downloaded files
CI: test a custom kernel build in the base-apt offline rebuild

meta/deb-dl-dir: make subshell verbose and fail on error
meta: deb-dl-dir: fix "command not found" error when excluding pkgs
RECIPE-API-CHANGELOG: add a few bits on the base-apt caching changes

RECIPE-API-CHANGELOG.md | 5 +

doc/user_manual.md | 9 +-
meta-isar/conf/local.conf.sample | 10 +-
meta/classes/base-apt-helper.bbclass | 57 ----------

meta/classes/deb-dl-dir.bbclass | 44 ++++++++

meta/classes/dpkg-base.bbclass | 47 ++++----
meta/classes/dpkg-gbp.bbclass | 5 +
meta/classes/dpkg.bbclass | 7 +-
meta/classes/image-cache-extension.bbclass | 36 -------
meta/classes/image-locales-extension.bbclass | 2 +-
meta/classes/image-tools-extension.bbclass | 7 ++
meta/classes/image.bbclass | 3 +-
meta/classes/repository.bbclass | 100 ++++++++++++++++++

meta/classes/rootfs.bbclass | 32 +++---

meta/conf/bitbake.conf | 6 +-
.../isar-bootstrap/files/base-apt-sources | 1 -
.../isar-bootstrap/isar-bootstrap.inc | 21 ++--
meta/recipes-devtools/base-apt/base-apt.bb | 92 ++++++++++------
.../base-apt/files/distributions.in | 3 -
.../buildchroot/files/deps.sh | 37 ++++---

.../isar-apt/files/distributions.in | 3 -
meta/recipes-devtools/isar-apt/isar-apt.bb | 26 ++---

scripts/ci_build.sh | 15 ++-
23 files changed, 334 insertions(+), 234 deletions(-)

delete mode 100644 meta/classes/base-apt-helper.bbclass
create mode 100644 meta/classes/deb-dl-dir.bbclass
delete mode 100644 meta/classes/image-cache-extension.bbclass

create mode 100644 meta/classes/repository.bbclass
delete mode 100644 meta/recipes-core/isar-bootstrap/files/base-apt-sources

delete mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
delete mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

So far the deb-src packages downloaded via "apt://" where not cached.
This commit fixes that and includes those packages into the cache.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/base-apt-helper.bbclass | 7 +++++++
meta/classes/image-cache-extension.bbclass | 4 ++++
meta/classes/repository.bbclass | 14 ++++++++++++++
3 files changed, 25 insertions(+)

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 6fc57303..4be7ad27 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -52,4 +52,11 @@ populate_base_apt() {
"${BASE_DISTRO_CODENAME}" \
"${package}"
done
+
+ find $search_dir -name '*.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"

+ done
}
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 30db5493..f8b9ba34 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -32,5 +32,9 @@ do_cache_base_repo() {
if [ -d ${BUILDCHROOT_TARGET_DIR}/var/cache/apt ]; then
populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
fi
+
+ if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
+ populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
+ fi
}
addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index cf0fb5a2..591ee04a 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -35,6 +35,20 @@ EOF
fi
}

+repo_add_srcpackage() {

+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi

+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main -P source \
+ includedsc "${codename}" \
+ "$@"
+}
+
repo_add_packages() {
local dir="$1"
local dbdir="$2"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

We used to have "trusted=yes" hardcoded even for a signed base-apt. Make
that flag depend on whether it is signed and generate that file with two
echos, it is simple enough.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

.../isar-bootstrap/files/base-apt-sources | 2 --
.../isar-bootstrap/isar-bootstrap.inc | 16 ++++++++--------
2 files changed, 8 insertions(+), 10 deletions(-)
delete mode 100644 meta/recipes-core/isar-bootstrap/files/base-apt-sources

diff --git a/meta/recipes-core/isar-bootstrap/files/base-apt-sources b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
deleted file mode 100644
index 2a4ab2c4..00000000
--- a/meta/recipes-core/isar-bootstrap/files/base-apt-sources
+++ /dev/null
@@ -1,2 +0,0 @@
-deb [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
-deb-src [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 16b4395b..cf3df407 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -12,8 +12,7 @@ SRC_URI = " \
file://isar-apt.conf \
file://isar-apt-fallback.conf \
file://locale \
- file://chroot-setup.sh \
- file://base-apt-sources"
+ file://chroot-setup.sh"
PV = "1.0"

DEBOOTSTRAP ?= "qemu-debootstrap"
@@ -21,7 +20,6 @@ ROOTFSDIR = "${WORKDIR}/rootfs"
APTPREFS = "${WORKDIR}/apt-preferences"
APTSRCS = "${WORKDIR}/apt-sources"
APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
-BASEAPTSRCS = "${WORKDIR}/base-apt-sources"
DISTRO_BOOTSTRAP_KEYFILES = ""
THIRD_PARTY_APT_KEYFILES = ""
DEPLOY_ISAR_BOOTSTRAP ?= ""
@@ -266,13 +264,15 @@ isar_bootstrap() {
"${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- sed -e "s#{BASE_DISTRO}#"${BASE_DISTRO}"#g" \
- -e "s#{BASE_DISTRO_CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- -i ${BASEAPTSRCS}
+ line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+ echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+
mkdir -p ${ROOTFSDIR}/base-apt
mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- install -v -m644 "${BASEAPTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
else
install -v -m644 "${APTSRCS}" \
"${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Hook in between the "apt-get --download-only"s and the "apt-get"s and
copy out all the debs for later.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 15 +++++++++++++++
meta/classes/dpkg-base.bbclass | 1 +
meta/classes/dpkg-gbp.bbclass | 1 +
meta/classes/dpkg.bbclass | 1 +
meta/classes/image-locales-extension.bbclass | 2 +-
meta/classes/image-tools-extension.bbclass | 1 +
meta/classes/rootfs.bbclass | 14 +++++++++++---
meta/conf/bitbake.conf | 1 +
8 files changed, 32 insertions(+), 4 deletions(-)
create mode 100644 meta/classes/deb-dl-dir.bbclass

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
new file mode 100644
index 00000000..066ce1a3
--- /dev/null
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -0,0 +1,15 @@

+# This software is a part of ISAR.

+# Copyright (C) 2020 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+

+deb_dl_dir_export() {

+ export pc="${DEBDIR}/${DISTRO}"

+ export rootfs="${1}"
+ mkdir -p "${pc}"
+ flock "${pc}".lock -c ' \
+ sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' \
+ -exec cp -f '{}' "${pc}" \;
+ sudo chown -R $(id -u):$(id -g) "${pc}"
+ '
+}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 5dd334d0..9aa2d546 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -8,6 +8,7 @@ inherit buildchroot
inherit debianize
inherit terminal
inherit repository
+inherit deb-dl-dir

DEPENDS ?= ""

diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 2afe9db4..7e908eba 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -15,6 +15,7 @@ do_install_builddeps_append() {

sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \

--no-install-recommends --download-only ${GBP_DEPENDS}
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"

sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass

index b26c6456..7e2c86d6 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -9,6 +9,7 @@ do_install_builddeps() {
E="${@ isar_export_proxies(d)}"

sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \

${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"

sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${DISTRO_ARCH}

dpkg_undo_mounts
diff --git a/meta/classes/image-locales-extension.bbclass b/meta/classes/image-locales-extension.bbclass
index 3c0758ff..0f0d0ca4 100644
--- a/meta/classes/image-locales-extension.bbclass
+++ b/meta/classes/image-locales-extension.bbclass
@@ -25,7 +25,7 @@ def get_nopurge(d):
j.split()[0].split(".")[0],
j.split()[0]))))

-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN += "image_install_localepurge_download"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download"
image_install_localepurge_download[weight] = "40"
image_install_localepurge_download() {
sudo -E chroot '${ROOTFSDIR}' \
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index b8531cb4..ec67d940 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -34,6 +34,7 @@ do_install_imager_deps() {
--allow-unauthenticated --allow-downgrades --download-only install \
${IMAGER_INSTALL}'

+ deb_dl_dir_export ${BUILDCHROOT_DIR}

sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \

diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 64eaef70..706b43a7 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -1,5 +1,7 @@

# This software is a part of ISAR.

-# Copyright (c) Siemens AG, 2019
+# Copyright (c) Siemens AG, 2020
+
+inherit deb-dl-dir

ROOTFS_ARCH ?= "${DISTRO_ARCH}"
ROOTFS_DISTRO ?= "${DISTRO}"
@@ -119,8 +121,14 @@ rootfs_install_pkgs_download() {
/usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
}

-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN ??= ""
-ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN}"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= ""
+ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT}"
+
+ROOTFS_INSTALL_COMMAND += "rootfs_export_package_cache"
+rootfs_export_package_cache[weight] = "5"
+rootfs_export_package_cache() {
+ deb_dl_dir_export ${ROOTFSDIR}
+}

ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
rootfs_install_clean_files[weight] = "2"
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 6b05e91b..b7c0a8f5 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
FILESEXTRAPATHS ?= "__default:"
GITDIR = "${DL_DIR}/git"
+DEBDIR = "${DL_DIR}/deb"
DEBSRCDIR = "${DL_DIR}/deb-src"
P = "${PN}-${PV}"
PF = "${PN}-${PV}-${PR}"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

This commit does not contain a functional change. We just split all
calls of "apt-get" into a download and an execution phase, so we can
later copy out the downloaded files.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/dpkg-gbp.bbclass | 3 ++
meta/classes/dpkg.bbclass | 5 ++-
meta/classes/image-tools-extension.bbclass | 5 +++
.../buildchroot/files/deps.sh | 37 +++++++++++--------
4 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 0533a52f..2afe9db4 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass

@@ -12,6 +12,9 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"

do_install_builddeps_append() {
dpkg_do_mounts
+ sudo -E chroot ${BUILDCHROOT_DIR} \
+ apt-get install -y -o Debug::pkgProblemResolver=yes \
+ --no-install-recommends --download-only ${GBP_DEPENDS}

sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass

index 180a3a89..b26c6456 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass

@@ -7,7 +7,10 @@ inherit dpkg-base
do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"

- sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh ${PP}/${PPS} ${DISTRO_ARCH}
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH}
dpkg_undo_mounts
}

diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index d4f216da..b8531cb4 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass

@@ -30,6 +30,11 @@ do_install_imager_deps() {
-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
-o Dir::Etc::SourceParts="-" \
-o APT::Get::List-Cleanup="0"
+ apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
+ --allow-unauthenticated --allow-downgrades --download-only install \
+ ${IMAGER_INSTALL}'
+

+ sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \

apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \

${IMAGER_INSTALL}'
diff --git a/meta/recipes-devtools/buildchroot/files/deps.sh b/meta/recipes-devtools/buildchroot/files/deps.sh

index 002085b1..6048305a 100644

+if [ "$3" = "--download-only" ]; then

+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control 2>&1 \
+ | grep "mk-build-deps: Unable to install all build-dep packages"
+else
+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control

-# Upgrade any already installed packages in case we are partially rebuilding
-apt-get upgrade -y --allow-downgrades
+ # Upgrade any already installed packages in case we are partially rebuilding
+ apt-get upgrade -y --allow-downgrades
+fi
--

2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Now that base-apt supports deb-src we can include hello in a standard
build.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta-isar/conf/local.conf.sample | 4 +---
scripts/ci_build.sh | 1 -
2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 81a04261..ebc277b1 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -169,11 +169,9 @@ CONF_VERSION = "1"

#
# The default list of extra packages to be installed.
-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello"
## "cache_base_repo" relies on the cache ...
#IMAGE_INSTALL += "isar-disable-apt-cache"
-## uses apt:// SRC_URI, which does not work with the cache
-#IMAGE_INSTALL += "hello"

#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7cd5a2fc..f50c9e42 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -169,7 +169,6 @@ if [ -n "$REPRO_BUILD" ]; then
fi

sed -i -e 's/#IMAGE_INSTALL += "isar-disable-apt-cache"/IMAGE_INSTALL += "isar-disable-apt-cache"/g' conf/local.conf
-sed -i -e 's/#IMAGE_INSTALL += "hello"/IMAGE_INSTALL += "hello"/g' conf/local.conf

echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

The bootstrap step also contains downloading of debs, so it should be
surrounded with an import/export as well.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 4 ++++
1 file changed, 4 insertions(+)

diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index cf3df407..2078115b 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -28,6 +28,8 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES_append_gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support = ",apt-transport-https,ca-certificates"
HOST_DISTRO_APT_SOURCES += "conf/distro/${HOST_DISTRO}.list"

+inherit deb-dl-dir
+
python () {
distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split()

@@ -219,6 +221,7 @@ do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"

isar_bootstrap() {
+ deb_dl_dir_import "${ROOTFSDIR}"
IS_HOST=""
while true; do
case "$1" in
@@ -335,6 +338,7 @@ isar_bootstrap() {
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
fi
EOSUDO
+ deb_dl_dir_export "${ROOTFSDIR}"
}

CLEANFUNCS = "clean_deploy"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

We have all the .debs cached in DEBDIR now, take them from there because
the rootfss might not actually have all of them.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 4 ++++

1 file changed, 4 insertions(+)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index f8b9ba34..07e9e7d0 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -33,6 +33,10 @@ do_cache_base_repo() {

populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
fi

+ if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
+ populate_base_apt "${DEBDIR}"/"${DISTRO}"
+ fi

+
if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then

populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"

fi
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

If we ever downloaded a file there is no need to do that again. We can
not only use that DL_DIR to later construct a repository but also to
speed up our rebuilds.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 10 ++++++++++

meta/classes/dpkg-gbp.bbclass | 1 +
meta/classes/dpkg.bbclass | 1 +

meta/classes/image-tools-extension.bbclass | 1 +
meta/classes/rootfs.bbclass | 6 ++++++
5 files changed, 19 insertions(+)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 066ce1a3..2858fea2 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -3,6 +3,16 @@
#
# SPDX-License-Identifier: MIT

+deb_dl_dir_import() {

+ export pc="${DEBDIR}/${DISTRO}"
+ export rootfs="${1}"

+ [ ! -d "${pc}" ] && return 0
+ flock -s "${pc}".lock -c ' \
+ sudo find "${pc}" -type f -iname '*\.deb' -exec \
+ cp -f --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
+ '
+}
+
deb_dl_dir_export() {

export pc="${DEBDIR}/${DISTRO}"

export rootfs="${1}"
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 7e908eba..afa1e19b 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,6 +12,7 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"

do_install_builddeps_append() {
dpkg_do_mounts
+ deb_dl_dir_import "${BUILDCHROOT_DIR}"

sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \

--no-install-recommends --download-only ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 7e2c86d6..585365ee 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -7,6 +7,7 @@ inherit dpkg-base

do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"

+ deb_dl_dir_import "${BUILDCHROOT_DIR}"

sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \

${PP}/${PPS} ${DISTRO_ARCH} --download-only
deb_dl_dir_export "${BUILDCHROOT_DIR}"
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index ec67d940..6590ee7a 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -25,6 +25,7 @@ do_install_imager_deps() {
buildchroot_do_mounts

E="${@ isar_export_proxies(d)}"
+ deb_dl_dir_import ${BUILDCHROOT_DIR}

sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \

apt-get update \

-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \

diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 706b43a7..81759ebc 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -113,6 +113,12 @@ rootfs_install_resolvconf() {
fi
}

+ROOTFS_INSTALL_COMMAND += "rootfs_import_package_cache"
+rootfs_import_package_cache[weight] = "5"
+rootfs_import_package_cache() {
+ deb_dl_dir_import ${ROOTFSDIR}
+}
+
ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
rootfs_install_pkgs_download[weight] = "600"
rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

We now have all debs in DEBDIR anyways, no need to conditionally copy
them out of the rootfss we create.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 4 ----
meta/classes/image.bbclass | 2 +-
meta/classes/rootfs.bbclass | 10 ----------
3 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 07e9e7d0..19c9c593 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,10 +21,6 @@ do_cache_base_repo() {
"Try it without cross-build."
fi

- if [ -d ${WORKDIR}/apt_cache ]; then
- populate_base_apt ${WORKDIR}/apt_cache
- fi
-
if [ -d ${BUILDCHROOT_HOST_DIR}/var/cache/apt ]; then
populate_base_apt ${BUILDCHROOT_HOST_DIR}/var/cache/apt
fi
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 6bcc6cd2..adaca5db 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -60,7 +60,7 @@ image_do_mounts() {
}

ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
+ROOTFS_FEATURES += "clean-package-cache finalize-rootfs generate-manifest"
ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"

diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 81759ebc..be101890 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -9,7 +9,6 @@ ROOTFS_PACKAGES ?= ""

# Features of the rootfs creation:
# available features are:
-# 'copy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
# 'clean-package-cache' - delete package cache from rootfs
# 'generate-manifest' - generate a package manifest of the rootfs into ${ROOTFS_MANIFEST_DEPLOY_DIR}
# 'finalize-rootfs' - delete files needed to chroot into the rootfs
@@ -186,15 +185,6 @@ python do_rootfs_install() {
}
addtask rootfs_install before do_rootfs_postprocess after do_unpack

-ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache', 'rootfs_postprocess_copy_package_cache', '', d)}"
-rootfs_postprocess_copy_package_cache() {
- mkdir -p '${WORKDIR}/apt_cache'
- sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
- -maxdepth 1 -name '*.deb' -execdir /bin/mv -t '${WORKDIR}/apt_cache' '{}' '+'
- me="$(id -u):$(id -g)"
- sudo chown -R "$me" '${WORKDIR}/apt_cache'
-}
-
ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', 'rootfs_postprocess_clean_package_cache', '', d)}"
rootfs_postprocess_clean_package_cache() {

sudo -E chroot '${ROOTFSDIR}' \

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

The earlier commits make sure we already have all .debs in the repo, no
need to get them from these locations. In fact these locations are not
reliable anyways, see hook in "isar-disable-apt-cache".

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 8 --------
1 file changed, 8 deletions(-)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 19c9c593..cff2a8ba 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,14 +21,6 @@ do_cache_base_repo() {

"Try it without cross-build."
fi

- if [ -d ${BUILDCHROOT_HOST_DIR}/var/cache/apt ]; then
- populate_base_apt ${BUILDCHROOT_HOST_DIR}/var/cache/apt
- fi
-
- if [ -d ${BUILDCHROOT_TARGET_DIR}/var/cache/apt ]; then
- populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
- fi
-

if [ -d "${DEBDIR}"/"${DISTRO}" ]; then

populate_base_apt "${DEBDIR}"/"${DISTRO}"

fi
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Just call that function once and make it find the debs and dscs in one
run.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/base-apt-helper.bbclass | 6 ++----
meta/classes/image-cache-extension.bbclass | 8 +-------
2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 4be7ad27..7ed28595 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -17,9 +17,7 @@ compare_pkg_md5sums() {
}

populate_base_apt() {
- search_dir=$1
-
- find $search_dir -name '*.deb' | while read package; do
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do

# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In

# addition, md5sums are compared to ensure that the package is the
@@ -53,7 +51,7 @@ populate_base_apt() {
"${package}"
done

- find $search_dir -name '*.dsc' | while read package; do
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do

repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \

"${BASE_DISTRO_CODENAME}" \
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index cff2a8ba..7945e8bc 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,12 +21,6 @@ do_cache_base_repo() {

"Try it without cross-build."
fi

- if [ -d "${DEBDIR}"/"${DISTRO}" ]; then

- populate_base_apt "${DEBDIR}"/"${DISTRO}"
- fi
-
- if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
- populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
- fi
+ populate_base_apt

}
addtask cache_base_repo after do_rootfs do_install_imager_deps

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

There is only one place we use that code, move it there instead of
having an extra class.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/base-apt-helper.bbclass | 60 ----------------------
meta/classes/image-cache-extension.bbclass | 55 +++++++++++++++++++-
2 files changed, 54 insertions(+), 61 deletions(-)
delete mode 100644 meta/classes/base-apt-helper.bbclass

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
deleted file mode 100644
index 7ed28595..00000000
--- a/meta/classes/base-apt-helper.bbclass
+++ /dev/null
@@ -1,60 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) 2018 ilbers GmbH
-# Copyright (C) 2020 Siemens AG
-#
-# SPDX-License-Identifier: MIT
-
-inherit repository
-
-compare_pkg_md5sums() {
- pkg1=$1
- pkg2=$2
-
- md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
- md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
-
- [ "$md1" = "$md2" ]
-}
-
-populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the
- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.
-
- # Check if this package is taken from Isar-apt, if so - ignore it.
- base_name=${package##*/}
- isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
- if [ -n "$isar_apt_p" ]; then
- # Check if MD5 sums are identical. This helps to avoid the case
- # when packages is overridden from another repo.
- compare_pkg_md5sums "$package" "$isar_apt_p" && continue
- fi
-
- # Check if this package is already in base-apt
- base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
- if [ -n "$base_apt_p" ]; then
- compare_pkg_md5sums "$package" "$base_apt_p" && continue
-
- # md5sum differs, so remove the package from base-apt
- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
- fi
-
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-}
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 7945e8bc..c3ee7b83 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -5,7 +5,60 @@
#
# This class extends the image.bbclass to supply the creation of cache repositories

-inherit base-apt-helper
+inherit repository
+
+compare_pkg_md5sums() {
+ pkg1=$1
+ pkg2=$2
+
+ md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
+ md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
+
+ [ "$md1" = "$md2" ]
+}
+
+populate_base_apt() {

+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do

+ # NOTE: due to packages stored by reprepro are not modified, we can
+ # use search by filename to check if package is already in repo. In
+ # addition, md5sums are compared to ensure that the package is the
+ # same and should not be overwritten. This method is easier and more
+ # robust than querying reprepro by name.
+
+ # Check if this package is taken from Isar-apt, if so - ignore it.
+ base_name=${package##*/}
+ isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
+ if [ -n "$isar_apt_p" ]; then
+ # Check if MD5 sums are identical. This helps to avoid the case
+ # when packages is overridden from another repo.
+ compare_pkg_md5sums "$package" "$isar_apt_p" && continue
+ fi
+
+ # Check if this package is already in base-apt
+ base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
+ if [ -n "$base_apt_p" ]; then
+ compare_pkg_md5sums "$package" "$base_apt_p" && continue
+
+ # md5sum differs, so remove the package from base-apt
+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \

+ "${base_apt_p}"
+ fi
+
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done

+

+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do

+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done

+}

do_cache_base_repo[depends] = "base-apt:do_cache_config"
do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

The base-apt does not rely on the rootfs-cache anymore, so we can test
our cleanup package in all images.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta-isar/conf/local.conf.sample | 4 +---
scripts/ci_build.sh | 2 --
2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index ebc277b1..1cf9519f 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -169,9 +169,7 @@ CONF_VERSION = "1"

#
# The default list of extra packages to be installed.

-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello"
-## "cache_base_repo" relies on the cache ...
-#IMAGE_INSTALL += "isar-disable-apt-cache"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache"

#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh

index f50c9e42..bfaf1f6e 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -168,8 +168,6 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
fi

-sed -i -e 's/#IMAGE_INSTALL += "isar-disable-apt-cache"/IMAGE_INSTALL += "isar-disable-apt-cache"/g' conf/local.conf
-

echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf

# Start cross build for the defined set of configurations
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Also add it to our sample config and comment on its use in conjuction
with base-apt.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

doc/user_manual.md | 1 +
meta-isar/conf/local.conf.sample | 4 ++++
scripts/ci_build.sh | 4 ++++
3 files changed, 9 insertions(+)

diff --git a/doc/user_manual.md b/doc/user_manual.md
index 0582ac74..8ba92eed 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -847,6 +847,7 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
```
# Uncomment this to enable use of cached base repository
#ISAR_USE_CACHED_BASE_REPO ?= "1"
+#BB_NO_NETWORK ?= "1"
```
- Remove build artifacts to use only local base-apt:

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index e8da548f..274cdaea 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -179,6 +179,10 @@ ISAR_CROSS_COMPILE ?= "0"
#
# Uncomment this to enable use of cached base repository
#ISAR_USE_CACHED_BASE_REPO ?= "1"
+#
+# You probably want to uncomment this as well to make sure the build
+# does not access the network
+#BB_NO_NETWORK ?= "1"

# Set root password to 'root'
# Password was encrypted using following command:
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index d440c248..7da3ee91 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -149,11 +149,13 @@ if [ -n "$REPRO_BUILD" ]; then
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
+ sed -i -e 's/^#BB_NO_NETWORK/BB_NO_NETWORK/g' conf/local.conf
bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
while [ -e bitbake.sock ]; do sleep 1; done
# Cleanup and disable use of signed cached base repository
sudo rm -rf tmp

sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf

+ sed -i -e 's/^BB_NO_NETWORK/#BB_NO_NETWORK/g' conf/local.conf
sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf

# Enable use of unsigned cached base repository
@@ -161,11 +163,13 @@ if [ -n "$REPRO_BUILD" ]; then
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
+ sed -i -e 's/^#BB_NO_NETWORK/BB_NO_NETWORK/g' conf/local.conf
bitbake $BB_ARGS $REPRO_TARGETS_SET
while [ -e bitbake.sock ]; do sleep 1; done
# Cleanup and disable use of unsigned cached base repository
sudo rm -rf tmp

sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf

+ sed -i -e 's/^BB_NO_NETWORK/#BB_NO_NETWORK/g' conf/local.conf
fi

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta-isar/conf/local.conf.sample | 2 +-
scripts/ci_build.sh | 2 --
2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 1cf9519f..e8da548f 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -169,7 +169,7 @@ CONF_VERSION = "1"

#
# The default list of extra packages to be installed.

-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache cowsay"

#
# Enable cross-compilation support

diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index bfaf1f6e..d440c248 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -168,8 +168,6 @@ if [ -n "$REPRO_BUILD" ]; then

sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf

fi

-echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf
-

# Start cross build for the defined set of configurations

sed -i -e 's/ISAR_CROSS_COMPILE ?= "0"/ISAR_CROSS_COMPILE ?= "1"/g' conf/local.conf
bitbake $BB_ARGS $CROSS_TARGETS_SET
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Make the lookup and the checksum comparison one function of a
repository. That cleans things up and allows for easier reuse.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 31 ++++++----------------
meta/classes/repository.bbclass | 19 +++++++++++++
2 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index c3ee7b83..4123326e 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -7,16 +7,6 @@

inherit repository

-compare_pkg_md5sums() {
- pkg1=$1
- pkg2=$2
-
- md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
- md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
-
- [ "$md1" = "$md2" ]
-}
-

populate_base_apt() {

find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do

# NOTE: due to packages stored by reprepro are not modified, we can

@@ -25,21 +15,16 @@ populate_base_apt() {

# same and should not be overwritten. This method is easier and more

# robust than querying reprepro by name.

- # Check if this package is taken from Isar-apt, if so - ignore it.
- base_name=${package##*/}
- isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
- if [ -n "$isar_apt_p" ]; then
- # Check if MD5 sums are identical. This helps to avoid the case
- # when packages is overridden from another repo.
- compare_pkg_md5sums "$package" "$isar_apt_p" && continue
- fi

+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue

# Check if this package is already in base-apt
- base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
- if [ -n "$base_apt_p" ]; then
- compare_pkg_md5sums "$package" "$base_apt_p" && continue
-
- # md5sum differs, so remove the package from base-apt

+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ ret=$?
+ [ "${ret}" = "0" ] && continue
+ if [ "${ret}" = "1" ]; then

repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \

"${BASE_DISTRO_CODENAME}" \
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index 591ee04a..18f020bd 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -81,3 +81,22 @@ repo_del_package() {
remove "${codename}" \
"${p}"
}
+
+repo_contains_package() {
+ local dir="$1"
+ local file="$2"
+ local package
+
+ package=$(find ${dir} -name ${file##*/})
+ if [ -n "$package" ]; then
+ local md1=$(md5sum "$package" | cut -d ' ' -f 1)
+ local md2=$(sudo md5sum "$file" | cut -d ' ' -f 1)
+
+ # yes
+ [ "${md1}" = "${md2}" ] && return 0
+ # yes but not the exact same file
+ return 1
+ fi
+ # no
+ return 2
+}
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Just call cmp once instead of comparing the md5 sums.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/repository.bbclass | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index 18f020bd..80502dcf 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -89,11 +89,9 @@ repo_contains_package() {

package=$(find ${dir} -name ${file##*/})

if [ -n "$package" ]; then
- local md1=$(md5sum "$package" | cut -d ' ' -f 1)
- local md2=$(sudo md5sum "$file" | cut -d ' ' -f 1)
-
# yes
- [ "${md1}" = "${md2}" ] && return 0
+ cmp --silent "$package" "$file" && return 0

+
# yes but not the exact same file

return 1
fi
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Packages from isar-apt are not downloaded from the outside and should
not be cached.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 2858fea2..011dd717 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -3,8 +3,10 @@
#
# SPDX-License-Identifier: MIT

+inherit repository
+
deb_dl_dir_import() {
- export pc="${DEBDIR}/${DISTRO}"
+ export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"

[ ! -d "${pc}" ] && return 0

flock -s "${pc}".lock -c ' \

@@ -14,12 +16,16 @@ deb_dl_dir_import() {
}

deb_dl_dir_export() {
- export pc="${DEBDIR}/${DISTRO}"
+ export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"
mkdir -p "${pc}"

flock "${pc}".lock -c ' \

- sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' \
- -exec cp -f '{}' "${pc}" \;
+ find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' |\
+ while read p; do
+ repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" && \
+ continue
+ sudo cp -f "${p}" "${pc}"
+ done

sudo chown -R $(id -u):$(id -g) "${pc}"

'
}
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Now that we populate just from the DL_DIR and filter out the isar-apt
packages when populating this, we can skip that test since it will never
match.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/recipes-devtools/base-apt/base-apt.bb | 4 ----
1 file changed, 4 deletions(-)

diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 16fc5f22..d047b83f 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -17,10 +17,6 @@ populate_base_apt() {

# same and should not be overwritten. This method is easier and more
# robust than querying reprepro by name.

- # Check if this package is taken from Isar-apt, if so - ingore it.
- repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
- continue
-
# Check if this package is already in base-apt
ret=0

repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Users had to call a special task before switching to using base-apt.
With this commit you flip one switch and the repo will be created as one
of the first steps.

The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
"processed" files and not just a download-cache.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

doc/user_manual.md | 8 +--
meta/classes/image-cache-extension.bbclass | 64 ------------------
meta/classes/image.bbclass | 1 -
meta/classes/rootfs.bbclass | 2 +-
meta/conf/bitbake.conf | 4 +-
.../isar-bootstrap/isar-bootstrap.inc | 1 +
meta/recipes-devtools/base-apt/base-apt.bb | 67 ++++++++++++++++---
scripts/ci_build.sh | 4 +-
8 files changed, 69 insertions(+), 82 deletions(-)
delete mode 100644 meta/classes/image-cache-extension.bbclass

diff --git a/doc/user_manual.md b/doc/user_manual.md
index 8ba92eed..d13a74e9 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -836,10 +836,10 @@ path to the public key in `conf/local.conf`, e.g.:
BASE_REPO_KEY = "file://<absolute_path_to_your_pub_key_file>"'
```

- - Trigger creation of local apt caching Debian packages during image generation.
+ - Trigger the download and caching of all required files by doing a warm-up build.

```
-bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
+bitbake mc:qemuarm-buster:isar-image-base
```

- Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
@@ -849,14 +849,14 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
#ISAR_USE_CACHED_BASE_REPO ?= "1"
#BB_NO_NETWORK ?= "1"
```
- - Remove build artifacts to use only local base-apt:
+ - Remove build artifacts to use only local base-apt, in fact toggling ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is just the way to be extra sure that only the download cache is used.

```
sudo rm -rf tmp

```

- - Trigger again generation of image (now using local caching repo):
+ - Trigger the generation of your image again (now a local repo will be created out of the download cache from the last run):

```
bitbake mc:qemuarm-buster:isar-image-base
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
deleted file mode 100644
index 4123326e..00000000
--- a/meta/classes/image-cache-extension.bbclass
+++ /dev/null
@@ -1,64 +0,0 @@

-# This software is a part of ISAR.

-# Copyright (C) Siemens AG, 2019
-#
-# SPDX-License-Identifier: MIT
-#
-# This class extends the image.bbclass to supply the creation of cache repositories
-
-inherit repository

-
-populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the

- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.

-
- # Check if this package is taken from Isar-apt, if so - ingore it.
- repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
- continue
-
- # Check if this package is already in base-apt

- ret=0
- repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
- ret=$?
- [ "${ret}" = "0" ] && continue
- if [ "${ret}" = "1" ]; then

- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
- fi
-
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do

- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-}
-

-do_cache_base_repo[depends] = "base-apt:do_cache_config"
-do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
-do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}"
-do_cache_base_repo() {
- if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
- [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
- # We would need two separate repository paths for that.
- # Otherwise packages (especially the 'all' arch ones) from one
- # distribution can influence the package versions of the other
- # distribution.
- bbfatal "Different host and target distributions are currently not supported." \
- "Try it without cross-build."
- fi
-
- populate_base_apt
-}
-addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index adaca5db..96ba863e 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"

inherit rootfs
inherit image-sdk-extension
-inherit image-cache-extension
inherit image-tools-extension
inherit image-postproc-extension
inherit image-locales-extension
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index be101890..806e824c 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {

do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
-do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config"
+do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache"
do_rootfs_install[deptask] = "do_deploy_deb"
python do_rootfs_install() {
configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or "").split()
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index b7c0a8f5..848e085a 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -82,8 +82,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db"
THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"

# Base apt repository paths
-REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
-REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
+REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
+REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"

BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH BBSERVER DL_DIR \
SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 2078115b..a93c4e3b 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -219,6 +219,7 @@ def get_host_release():

do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"

+do_bootstrap[depends] = "base-apt:do_cache"

isar_bootstrap() {
deb_dl_dir_import "${ROOTFSDIR}"
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 7c891fa0..16fc5f22 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -9,26 +9,77 @@ inherit repository
BASE_REPO_KEY ?= ""
KEYFILES ?= ""

-do_cache_config[stamp-extra-info] = "${DISTRO}"
-do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"

+populate_base_apt() {
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ # NOTE: due to packages stored by reprepro are not modified, we can
+ # use search by filename to check if package is already in repo. In
+ # addition, md5sums are compared to ensure that the package is the

+ # same and should not be overwritten. This method is easier and more
+ # robust than querying reprepro by name.

-# Generate reprepro config for current distro if it doesn't exist. Once it's
-# generated, this task should do nothing.
-repo_config() {
+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue
+
+ # Check if this package is already in base-apt

+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ ret=$?
+ [ "${ret}" = "0" ] && continue
+ if [ "${ret}" = "1" ]; then

+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
+ fi
+
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do

+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+}
+

+do_cache[stamp-extra-info] = "${DISTRO}"
+do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+
+repo() {
repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \

"${KEYFILES}"
+
+ if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
+ [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
+ # We would need two separate repository paths for that.
+ # Otherwise packages (especially the 'all' arch ones) from one
+ # distribution can influence the package versions of the other
+ # distribution.
+ bbfatal "Different host and target distributions are currently not supported." \
+ "Try it without cross-build."
+ fi
+
+ populate_base_apt
}

-python do_cache_config() {
+python do_cache() {
+ if not bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
+ return 0
+
for key in d.getVar('BASE_REPO_KEY').split():
d.appendVar("SRC_URI", " %s" % key)
fetcher = bb.fetch2.Fetch([key], d)
filename = fetcher.localpath(key)
d.appendVar("KEYFILES", " %s" % filename)

- bb.build.exec_func('repo_config', d)
+ bb.build.exec_func('repo', d)
}

-addtask cache_config after do_unpack before do_build
+addtask cache after do_unpack before do_build
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7da3ee91..733ba86e 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then

# Enable use of signed cached base repository
echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\" >> conf/local.conf
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED
+ bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED

while [ -e bitbake.sock ]; do sleep 1; done

sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf

@@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then

sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf

# Enable use of unsigned cached base repository

- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
+ bitbake $BB_ARGS $REPRO_TARGETS_SET

while [ -e bitbake.sock ]; do sleep 1; done

sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

During review of the original series it was found that the offline build
did not work for kernels. That has been solved by the new way we build
kernels.

Still having the test and increased coverage is a good idea.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

scripts/ci_build.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 733ba86e..adc403be 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -50,7 +50,7 @@ CROSS_TARGETS_SET="\
mc:rpi-stretch:isar-image-base"

REPRO_TARGETS_SET_SIGNED="\
- mc:qemuarm-stretch:isar-image-base \
+ mc:de0-nano-soc-stretch:isar-image-base \
mc:qemuarm64-stretch:isar-image-base"

REPRO_TARGETS_SET="\
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Remember the time we imported the previously downloaded files, and only
copy out new ones when exporting again. That saved pointless copies and
time.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 011dd717..55c4a839 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -8,24 +8,31 @@ inherit repository
deb_dl_dir_import() {

export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"

+ rm -f "${T}"/deb_dl_dir_import.stamp
+ touch "${T}"/deb_dl_dir_import.stamp

[ ! -d "${pc}" ] && return 0

+ [ ! -d "${rootfs}"/var/cache/apt/archives/ ] && return 0

flock -s "${pc}".lock -c ' \

sudo find "${pc}" -type f -iname '*\.deb' -exec \
- cp -f --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
+ cp -n --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
'
}

deb_dl_dir_export() {

export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"

+ export T

mkdir -p "${pc}"
flock "${pc}".lock -c ' \

- find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' |\
+ find "${rootfs}"/var/cache/apt/archives/ \
+ -cnewer "${T}"/deb_dl_dir_import.stamp \
+ -type f -iname '*\.deb' |\
while read p; do

repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" && \

continue
- sudo cp -f "${p}" "${pc}"
+ sudo cp -n "${p}" "${pc}"

done
sudo chown -R $(id -u):$(id -g) "${pc}"
'

+ rm -f "${T}"/deb_dl_dir_import.stamp
}
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Signed-off-by: Henning Schild <henning...@siemens.com>
---

RECIPE-API-CHANGELOG.md | 5 +++++
1 file changed, 5 insertions(+)

diff --git a/RECIPE-API-CHANGELOG.md b/RECIPE-API-CHANGELOG.md
index fb04af99..c0e5d332 100644
--- a/RECIPE-API-CHANGELOG.md
+++ b/RECIPE-API-CHANGELOG.md
@@ -224,3 +224,8 @@ In order to get a LOCALVERSION appendix into both the kernel config and the
version information of the self-built packages, the LINUX_VERSION_EXTENSION is
now available. It remains empty by default unless a recipe sets it. The
appended version usually starts with a "-".
+
+### Image task `cache_base_repo` was removed
+
+That task used to be at the end of a cache-warming build, a follow-up build with `ISAR_USE_CACHED_BASE_REPO` did use that.
+Now we cache all downloads anyways, if `ISAR_USE_CACHED_BASE_REPO` is set a build will use all the downloads from previous builds for the cache.
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Fix the logic that excludes packages from isar-apt from the export. The
subshell can not reuse our function and runs into "command not found".

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index bcac7dbd..3aeefad7 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -30,10 +30,12 @@ deb_dl_dir_export() {
-maxdepth 1 -cnewer "${T}"/deb_dl_dir_import.stamp \

-type f -iname '*\.deb' | \
while read p; do

- ret=0
- repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" || \

- ret=$?
- [ "${ret}" = "0" ] && continue

+ # can not reuse bitbake function here, this is basically
+ # "repo_contains_package"
+ package=$(find "${REPO_ISAR_DIR}"/"${DISTRO}" -name ${p##*/})

+ if [ -n "$package" ]; then

+ cmp --silent "$package" "$p" && continue
+ fi

sudo cp -n "${p}" "${pc}"
done
sudo chown -R $(id -u):$(id -g) "${pc}"

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 55c4a839..bcac7dbd 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -23,13 +23,17 @@ deb_dl_dir_export() {
export rootfs="${1}"

export T
mkdir -p "${pc}"

- flock "${pc}".lock -c ' \
+ flock "${pc}".lock -c '
+ set -e
+ printenv | grep -q BB_VERBOSE_LOGS && set -x
find "${rootfs}"/var/cache/apt/archives/ \
- -cnewer "${T}"/deb_dl_dir_import.stamp \
- -type f -iname '*\.deb' |\
+ -maxdepth 1 -cnewer "${T}"/deb_dl_dir_import.stamp \
+ -type f -iname '*\.deb' | \
while read p; do
- repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" && \
- continue
+ ret=0
+ repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" || \
+ ret=$?
+ [ "${ret}" = "0" ] && continue

[Baurzhan Ismagulov]

Hello Henning,

On Tue, Mar 10, 2020 at 10:56:02PM +0100, Henning Schild wrote:
> changes since v4:
>
> - add missing - in p8 download-only arg matching, found by Baurzhan
> - adding patch 27 and 28 to deal with shell code reuse issues pointed out
> by Su, Bao Cheng ... "command not found"
> - bump copyright headers from 2019 to 2020
> - adding p29, RECIPE-API-CHANGELOG
>
> p8 and p1 appeared on the list as v5, but not from me, the p8 thing is
> in here the p1 change will probably be added by Baurzhan on a merge, not
> calling this v6 because of that repost

Thanks for the updated series. I've adjusted whitespace in p20, 23, 24, 28
(trailing whitespace, tabs to spaces) -- I can send the patches if you like.
I'll send the code-config split patch.

Regarding CI: From seven runs, five have failed, in two ways:

1. Four of them can't find packages in base-apt. I've checked the workspaces
where available, the packages in question are not in base-apt.

Example:

http://ci.isar-build.org:8080/job/isar_ibr_devel_fast/22/consoleFull
http://ci.isar-build.org:8080/job/isar_ibr_devel_fast/ws/*zip*/22.zip

2. One can't chroot into the isar-bootstrap-host rootfs. Debootstrap failed
with:

Processing triggers for libc-bin (2.28-10) ...
Terminated

I've checked the logs and haven't seen e.g. an OOM. Why could a process be
terminated?

http://ci.isar-build.org:8080/job/isar_ibr_devel_2_fast/3/consoleFull
http://ci.isar-build.org:8080/job/isar_ibr_devel_2_fast/ws/*zip*/3.zip

I suggest to check base-apt populating (add debug output which packages are
copied?). Even if this is due to download errors (which I haven't seen any
signs of), we should have failed on downloading -- but that succeeds.

With kind regards,
Baurzhan.

[Baurzhan Ismagulov]

From: Baurzhan Ismagulov <i...@ilbers.de>

Signed-off-by: Baurzhan Ismagulov <i...@ilbers.de>
---
meta/classes/repository.bbclass | 10 ++++------
meta/recipes-devtools/base-apt/base-apt.bb | 3 +++
meta/recipes-devtools/base-apt/files/distributions.in | 3 +++
meta/recipes-devtools/isar-apt/files/distributions.in | 3 +++
meta/recipes-devtools/isar-apt/isar-apt.bb | 5 ++++-
5 files changed, 17 insertions(+), 7 deletions(-)
create mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
create mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in

diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index 80502dc..c70138f 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -8,7 +8,8 @@ repo_create() {
local dir="$1"
local dbdir="$2"
local codename="$3"
- local keyfiles="$4"
+ local distros_in="$4"
+ local keyfiles="$5"

if [ -n "${GNUPGHOME}" ]; then
export GNUPGHOME="${GNUPGHOME}"
@@ -16,11 +17,8 @@ repo_create() {

if [ ! -f "${dir}"/conf/distributions ]; then
mkdir -p "${dir}"/conf/
- cat <<EOF > "${dir}"/conf/distributions
-Codename: ${codename}
-Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
-Components: main
-EOF
+ sed -e "s#{CODENAME}#${codename}#g" ${distros_in} \
+ >"${dir}"/conf/distributions
if [ -n "${keyfiles}" ] ; then
local option=""
for key in ${keyfiles}; do
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 408dd63..da3e32e 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -6,6 +6,8 @@

inherit repository

+SRC_URI = "file://distributions.in"
+
BASE_REPO_KEY ?= ""
KEYFILES ?= ""

@@ -50,6 +52,7 @@ repo() {

repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \

+ "${WORKDIR}/distributions.in" \
"${KEYFILES}"

if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&

diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
new file mode 100644
index 0000000..3cf7ea5
--- /dev/null
+++ b/meta/recipes-devtools/base-apt/files/distributions.in
@@ -0,0 +1,3 @@
+Codename: {CODENAME}
+Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
+Components: main
diff --git a/meta/recipes-devtools/isar-apt/files/distributions.in b/meta/recipes-devtools/isar-apt/files/distributions.in
new file mode 100644
index 0000000..3cf7ea5
--- /dev/null
+++ b/meta/recipes-devtools/isar-apt/files/distributions.in
@@ -0,0 +1,3 @@
+Codename: {CODENAME}
+Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
+Components: main
diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index c53b01f..0458bac 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -6,6 +6,8 @@

inherit repository

+SRC_URI = "file://distributions.in"
+
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"

@@ -14,7 +16,8 @@ do_cache_config[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
do_cache_config() {
repo_create "${REPO_ISAR_DIR}"/"${DISTRO}" \
"${REPO_ISAR_DB_DIR}"/"${DISTRO}" \
- "${DEBDISTRONAME}"
+ "${DEBDISTRONAME}" \
+ "${WORKDIR}/distributions.in"

}

addtask cache_config after do_unpack before do_build

--
2.20.1

[Henning Schild]

On Sun, 15 Mar 2020 17:58:52 +0100
Baurzhan Ismagulov <i...@radix50.net> wrote:

> Hello Henning,
>
> On Tue, Mar 10, 2020 at 10:56:02PM +0100, Henning Schild wrote:
> > changes since v4:
> >
> > - add missing - in p8 download-only arg matching, found by Baurzhan
> > - adding patch 27 and 28 to deal with shell code reuse issues
> > pointed out by Su, Bao Cheng ... "command not found"
> > - bump copyright headers from 2019 to 2020
> > - adding p29, RECIPE-API-CHANGELOG
> >
> > p8 and p1 appeared on the list as v5, but not from me, the p8 thing
> > is in here the p1 change will probably be added by Baurzhan on a
> > merge, not calling this v6 because of that repost
>
> Thanks for the updated series. I've adjusted whitespace in p20, 23,
> 24, 28 (trailing whitespace, tabs to spaces) -- I can send the
> patches if you like. I'll send the code-config split patch.

If they are whitespace only please send them so i can exchange them in
the next post. If there is anything else, i would prefer a review so i
can make the changes myself.

> Regarding CI: From seven runs, five have failed, in two ways:

Maybe what i saw before. I thought it must have been network issues on
the CI runners or the missing dash in the download-only, because that
one ran fine on two CI runners.

I will look into what is going on, could not reproduce it locally to
have decent debugging.

Could you please send the links to all four failures?

Henning

[Henning Schild]

On Tue, 10 Mar 2020 22:56:27 +0100
Henning Schild <henning...@siemens.com> wrote:

> From: Henning Schild <henning...@siemens.com>
>
> Remember the time we imported the previously downloaded files, and
> only copy out new ones when exporting again. That saved pointless
> copies and time.
>
> Signed-off-by: Henning Schild <henning...@siemens.com>
> ---
> meta/classes/deb-dl-dir.bbclass | 13 ++++++++++---
> 1 file changed, 10 insertions(+), 3 deletions(-)
>
> diff --git a/meta/classes/deb-dl-dir.bbclass
> b/meta/classes/deb-dl-dir.bbclass index 011dd717..55c4a839 100644
> --- a/meta/classes/deb-dl-dir.bbclass
> +++ b/meta/classes/deb-dl-dir.bbclass
> @@ -8,24 +8,31 @@ inherit repository
> deb_dl_dir_import() {
> export pc="${DEBDIR}/${DISTRO}/"
> export rootfs="${1}"
> + rm -f "${T}"/deb_dl_dir_import.stamp
> + touch "${T}"/deb_dl_dir_import.stamp

Creating the stamp outside the flock is a race. We could create a new
stamp and get stuck on the lock while a reader would skip new packages
that have not been imported yet.

Henning

[Henning Schild]

Hi,

i think i have a working replacement for this one now. In order to
prepare a new q i still lack the whitespace fixes or the review on what
needed fixing.

Henning

[Baurzhan Ismagulov]

From: Henning Schild <henning...@siemens.com>

Make the lookup and the checksum comparison one function of a
repository. That cleans things up and allows for easier reuse.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 31 ++++++----------------
meta/classes/repository.bbclass | 19 +++++++++++++
2 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass

index c3ee7b8..8df06f3 100644

--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -7,16 +7,6 @@

inherit repository

-compare_pkg_md5sums() {
- pkg1=$1
- pkg2=$2
-

- md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
- md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
-

- [ "$md1" = "$md2" ]
-}
-
populate_base_apt() {

find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do

# NOTE: due to packages stored by reprepro are not modified, we can

@@ -25,21 +15,16 @@ populate_base_apt() {

# same and should not be overwritten. This method is easier and more

# robust than querying reprepro by name.

- # Check if this package is taken from Isar-apt, if so - ignore it.
- base_name=${package##*/}
- isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
- if [ -n "$isar_apt_p" ]; then
- # Check if MD5 sums are identical. This helps to avoid the case
- # when packages is overridden from another repo.
- compare_pkg_md5sums "$package" "$isar_apt_p" && continue
- fi

+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue

# Check if this package is already in base-apt

- base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
- if [ -n "$base_apt_p" ]; then
- compare_pkg_md5sums "$package" "$base_apt_p" && continue
-
- # md5sum differs, so remove the package from base-apt

+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||

+ ret=$?
+ [ "${ret}" = "0" ] && continue

+ if [ "${ret}" = "1" ]; then

repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \

diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index 591ee04..18f020b 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass

@@ -81,3 +81,22 @@ repo_del_package() {
remove "${codename}" \
"${p}"
}
+
+repo_contains_package() {
+ local dir="$1"
+ local file="$2"
+ local package
+

+ package=$(find ${dir} -name ${file##*/})

+ if [ -n "$package" ]; then

+ local md1=$(md5sum "$package" | cut -d ' ' -f 1)
+ local md2=$(sudo md5sum "$file" | cut -d ' ' -f 1)
+
+ # yes

+ [ "${md1}" = "${md2}" ] && return 0

+ # yes but not the exact same file

+ return 1
+ fi
+ # no
+ return 2
+}
--

2.20.1

[Baurzhan Ismagulov]

From: Henning Schild <henning...@siemens.com>

Users had to call a special task before switching to using base-apt.
With this commit you flip one switch and the repo will be created as one
of the first steps.

The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
"processed" files and not just a download-cache.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

doc/user_manual.md | 8 +--
meta/classes/image-cache-extension.bbclass | 64 ------------------
meta/classes/image.bbclass | 1 -
meta/classes/rootfs.bbclass | 2 +-
meta/conf/bitbake.conf | 4 +-
.../isar-bootstrap/isar-bootstrap.inc | 1 +
meta/recipes-devtools/base-apt/base-apt.bb | 67 ++++++++++++++++---
scripts/ci_build.sh | 4 +-
8 files changed, 69 insertions(+), 82 deletions(-)
delete mode 100644 meta/classes/image-cache-extension.bbclass

diff --git a/doc/user_manual.md b/doc/user_manual.md

index 8ba92ee..d13a74e 100644

--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -836,10 +836,10 @@ path to the public key in `conf/local.conf`, e.g.:
BASE_REPO_KEY = "file://<absolute_path_to_your_pub_key_file>"'
```

- - Trigger creation of local apt caching Debian packages during image generation.
+ - Trigger the download and caching of all required files by doing a warm-up build.

```
-bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
+bitbake mc:qemuarm-buster:isar-image-base
```

- Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
@@ -849,14 +849,14 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
#ISAR_USE_CACHED_BASE_REPO ?= "1"
#BB_NO_NETWORK ?= "1"
```
- - Remove build artifacts to use only local base-apt:
+ - Remove build artifacts to use only local base-apt, in fact toggling ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is just the way to be extra sure that only the download cache is used.

```
sudo rm -rf tmp

```

- - Trigger again generation of image (now using local caching repo):
+ - Trigger the generation of your image again (now a local repo will be created out of the download cache from the last run):

```
bitbake mc:qemuarm-buster:isar-image-base
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
deleted file mode 100644

index 8df06f3..0000000

--- a/meta/classes/image-cache-extension.bbclass
+++ /dev/null
@@ -1,64 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) Siemens AG, 2019
-#
-# SPDX-License-Identifier: MIT
-#
-# This class extends the image.bbclass to supply the creation of cache repositories
-
-inherit repository
-
-populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the

- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.
-

index adaca5d..96ba863 100644

--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"

inherit rootfs
inherit image-sdk-extension
-inherit image-cache-extension
inherit image-tools-extension
inherit image-postproc-extension
inherit image-locales-extension
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass

index be10189..806e824 100644

--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {

do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
-do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config"
+do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache"
do_rootfs_install[deptask] = "do_deploy_deb"
python do_rootfs_install() {
configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or "").split()
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf

index b7c0a8f..848e085 100644

--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -82,8 +82,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db"
THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"

# Base apt repository paths
-REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
-REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
+REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
+REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"

BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH BBSERVER DL_DIR \
SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc

index 2078115..a93c4e3 100644

--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -219,6 +219,7 @@ def get_host_release():

do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
+do_bootstrap[depends] = "base-apt:do_cache"

isar_bootstrap() {
deb_dl_dir_import "${ROOTFSDIR}"

diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 7c891fa..9e42c04 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb

@@ -9,26 +9,77 @@ inherit repository
BASE_REPO_KEY ?= ""
KEYFILES ?= ""

-do_cache_config[stamp-extra-info] = "${DISTRO}"
-do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+populate_base_apt() {
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ # NOTE: due to packages stored by reprepro are not modified, we can
+ # use search by filename to check if package is already in repo. In
+ # addition, md5sums are compared to ensure that the package is the
+ # same and should not be overwritten. This method is easier and more
+ # robust than querying reprepro by name.

-# Generate reprepro config for current distro if it doesn't exist. Once it's
-# generated, this task should do nothing.
-repo_config() {

+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue

+
+ # Check if this package is already in base-apt

+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ ret=$?
+ [ "${ret}" = "0" ] && continue
+ if [ "${ret}" = "1" ]; then

+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
+ fi
+
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+}
+
+do_cache[stamp-extra-info] = "${DISTRO}"
+do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+
+repo() {

repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \

diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 71adea8..1061226 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh

@@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then

# Enable use of signed cached base repository
echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\" >> conf/local.conf
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED
+ bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
@@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf

# Enable use of unsigned cached base repository
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
+ bitbake $BB_ARGS $REPRO_TARGETS_SET
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
--

2.20.1

[Baurzhan Ismagulov]

From: Henning Schild <henning...@siemens.com>

Now that we populate just from the DL_DIR and filter out the isar-apt
packages when populating this, we can skip that test since it will never
match.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/recipes-devtools/base-apt/base-apt.bb | 4 ----
1 file changed, 4 deletions(-)

diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 9e42c04..408dd63 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -17,10 +17,6 @@ populate_base_apt() {

# same and should not be overwritten. This method is easier and more

# robust than querying reprepro by name.

- # Check if this package is taken from Isar-apt, if so - ingore it.
- repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
- continue
-

# Check if this package is already in base-apt

ret=0

repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||

--
2.20.1

[Baurzhan Ismagulov]

From: Henning Schild <henning...@siemens.com>

Fix the logic that excludes packages from isar-apt from the export. The
subshell can not reuse our function and runs into "command not found".

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index bcac7db..f9a6a11 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -30,10 +30,12 @@ deb_dl_dir_export() {

-maxdepth 1 -cnewer "${T}"/deb_dl_dir_import.stamp \

-type f -iname '*\.deb' | \
while read p; do

- ret=0
- repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" || \

- ret=$?
- [ "${ret}" = "0" ] && continue

+ # can not reuse bitbake function here, this is basically
+ # "repo_contains_package"

+ package=$(find "${REPO_ISAR_DIR}"/"${DISTRO}" -name ${p##*/})

+ if [ -n "$package" ]; then

+ cmp --silent "$package" "$p" && continue
+ fi

sudo cp -n "${p}" "${pc}"
done
sudo chown -R $(id -u):$(id -g) "${pc}"

--
2.20.1

[Baurzhan Ismagulov]

From: Henning Schild <henning...@siemens.com>

Signed-off-by: Henning Schild <henning...@siemens.com>
---

RECIPE-API-CHANGELOG.md | 7 +++++++
1 file changed, 7 insertions(+)

diff --git a/RECIPE-API-CHANGELOG.md b/RECIPE-API-CHANGELOG.md
index fb04af9..292b62a 100644
--- a/RECIPE-API-CHANGELOG.md
+++ b/RECIPE-API-CHANGELOG.md
@@ -224,3 +224,10 @@ In order to get a LOCALVERSION appendix into both the kernel config and the

version information of the self-built packages, the LINUX_VERSION_EXTENSION is
now available. It remains empty by default unless a recipe sets it. The
appended version usually starts with a "-".
+
+### Image task `cache_base_repo` was removed
+
+That task used to be at the end of a cache-warming build, a follow-up build

+with `ISAR_USE_CACHED_BASE_REPO` did use that. Now we cache all downloads
+anyway, if `ISAR_USE_CACHED_BASE_REPO` is set a build will use all the
+downloads from previous builds for the cache.
--
2.20.1

[Baurzhan Ismagulov]

Hello Henning,

On Fri, Mar 20, 2020 at 10:27:46AM +0100, Henning Schild wrote:
> i think i have a working replacement for this one now.

Great to hear that, looking forward to the patches.

> In order to prepare a new q i still lack the whitespace fixes or the review
> on what needed fixing.

Done, sorry for the delay. I've added line breaking at 79 in
RECIPE-API-CHANGELOG for 80x25 people like me.


With kind regards,
Baurzhan.

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

changes since v5:

- took whitespace and RECIPE-API-CHANGELOG modifications from Baurzhan
- fixed p25 to use another mechanism to speed up exports
- fixed p11 to actually not re-fetch on bootstrap, saves time and
bandwidth
- calling it v7 because some patches appeared as v6 on the list

changes since v4:

- add missing - in p8 download-only arg matching, found by Baurzhan
- adding patch 27 and 28 to deal with shell code reuse issues pointed out
by Su, Bao Cheng ... "command not found"
- bump copyright headers from 2019 to 2020
- adding p29, RECIPE-API-CHANGELOG

p8 and p1 appeared on the list as v5, but not from me, the p8 thing is
in here the p1 change will probably be added by Baurzhan on a merge, not
calling this v6 because of that repost

changes since v2:

Rebased/merged onto current next and added p26. The custom kernel caching
that was missing in v2 is not required on next anymore.

This one was sent as v3 by Vijai Kumar K
<vijaikumar....@gmail.com>. That just contained a fix in case
the patches would not be rebased, can be ignored.

Henning Schild (29):
repository: new class to deal with repos
dpkg-base: add download caching of apt:// downloads
meta: dpkg-base: convert "exit" into "return 0"
base-apt: change the sources.list to also offer deb-src
base-apt: add deb-src packages as well
base-apt: do not skip gpg check when it is signed
ci: conf: add "hello" to the sample config and every build
meta: split all apt-get invocations into download and execution
meta: create DL_DIR support for all apt-get downloaded .debs
meta: import DL_DIR debs before apt-get download steps
meta: include DL_DIR deb import/export into bootstrap
base-apt: populate from DEBDIR as well
base-apt: drop the "apt_cache" feature
base-apt: do not copy debs directly out of rootfss anymore
base-apt: rework base-apt population
base-apt: move class "base-apt-helper" into only user
CI: include "isar-disable-apt-cache" into all CI images
CI: include "cowsay" into default build to test dpkg-gbp
CI: set BB_NO_NETWORK for cached rebuild
meta: repository: implement repo_contains_package and use it in
base_apt
meta: repository: simplify the matching of packages
meta: deb-dl-dir: do not cache debs from isar-apt
base-apt: pull base-apt population to the front of the build chain
meta: base-apt: remove isar-apt check from population
meta: deb-dl-dir: only export newly downloaded files
CI: test a custom kernel build in the base-apt offline rebuild
meta/deb-dl-dir: make subshell verbose and fail on error
meta: deb-dl-dir: fix "command not found" error when excluding pkgs
RECIPE-API-CHANGELOG: add a few bits on the base-apt caching changes

RECIPE-API-CHANGELOG.md | 7 ++
doc/user_manual.md | 9 +-
meta-isar/conf/local.conf.sample | 10 +-
meta/classes/base-apt-helper.bbclass | 57 ----------
meta/classes/deb-dl-dir.bbclass | 45 ++++++++
meta/classes/dpkg-base.bbclass | 47 ++++----
meta/classes/dpkg-gbp.bbclass | 5 +
meta/classes/dpkg.bbclass | 7 +-
meta/classes/image-cache-extension.bbclass | 36 -------
meta/classes/image-locales-extension.bbclass | 2 +-
meta/classes/image-tools-extension.bbclass | 7 ++
meta/classes/image.bbclass | 3 +-
meta/classes/repository.bbclass | 100 ++++++++++++++++++
meta/classes/rootfs.bbclass | 32 +++---
meta/conf/bitbake.conf | 6 +-
.../isar-bootstrap/files/base-apt-sources | 1 -
.../isar-bootstrap/isar-bootstrap.inc | 31 +++---
meta/recipes-devtools/base-apt/base-apt.bb | 92 ++++++++++------
.../base-apt/files/distributions.in | 3 -
.../buildchroot/files/deps.sh | 37 ++++---
.../isar-apt/files/distributions.in | 3 -
meta/recipes-devtools/isar-apt/isar-apt.bb | 26 ++---
scripts/ci_build.sh | 15 ++-
23 files changed, 342 insertions(+), 239 deletions(-)
delete mode 100644 meta/classes/base-apt-helper.bbclass
create mode 100644 meta/classes/deb-dl-dir.bbclass
delete mode 100644 meta/classes/image-cache-extension.bbclass
create mode 100644 meta/classes/repository.bbclass
delete mode 100644 meta/recipes-core/isar-bootstrap/files/base-apt-sources
delete mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
delete mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Factor out all the "reprepro" code into a common class. There are slight
functional changes since the two copies of the code got out of sync.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/base-apt-helper.bbclass | 28 ++++----
meta/classes/dpkg-base.bbclass | 24 ++-----
meta/classes/repository.bbclass | 69 +++++++++++++++++++
meta/recipes-devtools/base-apt/base-apt.bb | 37 +++-------
.../base-apt/files/distributions.in | 3 -
.../isar-apt/files/distributions.in | 3 -
meta/recipes-devtools/isar-apt/isar-apt.bb | 26 ++-----
7 files changed, 102 insertions(+), 88 deletions(-)
create mode 100644 meta/classes/repository.bbclass

delete mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
delete mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 90b2cfcb..6fc57303 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -1,5 +1,10 @@

# This software is a part of ISAR.

# Copyright (C) 2018 ilbers GmbH
+# Copyright (C) 2020 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+inherit repository

compare_pkg_md5sums() {
pkg1=$1
@@ -14,10 +19,6 @@ compare_pkg_md5sums() {
populate_base_apt() {
search_dir=$1

- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
find $search_dir -name '*.deb' | while read package; do

# NOTE: due to packages stored by reprepro are not modified, we can

# use search by filename to check if package is already in repo. In

@@ -40,18 +41,15 @@ populate_base_apt() {

compare_pkg_md5sums "$package" "$base_apt_p" && continue

# md5sum differs, so remove the package from base-apt

- name=$(echo $base_name | cut -d '_' -f 1)
- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main -A ${DISTRO_ARCH} \
- remove ${BASE_DISTRO_CODENAME} \
- $name

+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"

fi

- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main \
- includedeb ${BASE_DISTRO_CODENAME} \
- $package

+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"

done
}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 5d2c161c..4702e52b 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -7,6 +7,7 @@
inherit buildchroot
inherit debianize
inherit terminal
+inherit repository

DEPENDS ?= ""

@@ -142,31 +143,16 @@ repo_clean() {
DEBS=$( find ${S}/.. -maxdepth 1 -name "*.deb" || [ ! -d ${S} ] )
if [ -n "${DEBS}" ]; then
for d in ${DEBS}; do
- p=$( dpkg-deb --show --showformat '${Package}' ${d} )
- a=$( dpkg-deb --show --showformat '${Architecture}' ${d} )
- # removing "all" means no arch
- aarg="-A ${a}"
- [ "${a}" = "all" ] && aarg=""
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main ${aarg} \
- remove ${DEBDISTRONAME} \
- ${p}
+ repo_del_package "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" "${d}"
done
fi
}

-# Install package to Isar-apt
do_deploy_deb() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
repo_clean
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main \
- includedeb ${DEBDISTRONAME} \
- ${S}/../*.deb
+ repo_add_packages "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" ${S}/../*.deb
}

addtask deploy_deb after do_dpkg_build before do_build
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
new file mode 100644
index 00000000..cf0fb5a2
--- /dev/null
+++ b/meta/classes/repository.bbclass
@@ -0,0 +1,69 @@
+# This software is a part of ISAR.
+# Copyright (C) 2017-2020 Siemens AG
+# Copyright (C) 2019 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+repo_create() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ local keyfiles="$4"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+
+ if [ ! -f "${dir}"/conf/distributions ]; then
+ mkdir -p "${dir}"/conf/
+ cat <<EOF > "${dir}"/conf/distributions
+Codename: ${codename}

+Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
+Components: main

+EOF
+ if [ -n "${keyfiles}" ] ; then
+ local option=""
+ for key in ${keyfiles}; do
+ keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
+ option="${option}${keyid} "
+ done
+ echo "SignWith: ${option}" >> "${dir}"/conf/distributions
+ fi
+ fi
+ if [ ! -d "${dbdir}" ]; then
+ reprepro -b "${dir}" --dbdir "${dbdir}" export "${codename}"
+ fi
+}
+
+repo_add_packages() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main \
+ includedeb "${codename}" \
+ "$@"
+}
+
+repo_del_package() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ local file="$4"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ local p=$( dpkg-deb --show --showformat '${Package}' "${file}" )
+ local a=$( dpkg-deb --show --showformat '${Architecture}' "${file}" )
+ # removing "all" means no arch
+ local aarg="-A ${a}"
+ [ "${a}" = "all" ] && aarg=""
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main ${aarg} \
+ remove "${codename}" \
+ "${p}"
+}
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 9a0f7c8b..7c891fa0 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -1,45 +1,24 @@

# This software is a part of ISAR.

# Copyright (C) 2018 ilbers GmbH
+# Copyright (C) 2020 Siemens AG
+#
+# SPDX-License-Identifier: MIT

-SRC_URI = "file://distributions.in"
+inherit repository

BASE_REPO_KEY ?= ""
KEYFILES ?= ""

-CACHE_CONF_DIR = "${REPO_BASE_DIR}/${BASE_DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"

do_cache_config[stamp-extra-info] = "${DISTRO}"

do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"

# Generate reprepro config for current distro if it doesn't exist. Once it's

# generated, this task should do nothing.

repo_config() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- if [ -n "${KEYFILES}" ]; then
- option=""
- for key in ${KEYFILES}; do
- keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
- option="${option}${keyid} "
- done
- # To generate Release.gpg
- echo "SignWith: ${option}" >> ${CACHE_CONF_DIR}/distributions
- fi
- fi
-
- path_cache="${REPO_BASE_DIR}/${BASE_DISTRO}"
- path_databases="${REPO_BASE_DB_DIR}/${BASE_DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${BASE_DISTRO_CODENAME}
- fi
+ repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \

+ "${KEYFILES}"
}

python do_cache_config() {
diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
deleted file mode 100644
index 3cf7ea55..00000000
--- a/meta/recipes-devtools/base-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {CODENAME}

-Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
-Components: main

diff --git a/meta/recipes-devtools/isar-apt/files/distributions.in b/meta/recipes-devtools/isar-apt/files/distributions.in
deleted file mode 100644
index 64717364..00000000
--- a/meta/recipes-devtools/isar-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {DISTRO_NAME}

-Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
-Components: main

diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index e6f1753d..c53b01f7 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -1,32 +1,20 @@

# This software is a part of ISAR.

# Copyright (C) 2015-2017 ilbers GmbH
+# Copyright (C) 2020 Siemens AG
+#
+# SPDX-License-Identifier: MIT

-SRC_URI = "file://distributions.in"
+inherit repository

-CACHE_CONF_DIR = "${REPO_ISAR_DIR}/${DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"

do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"

# Generate reprepro config for current distro if it doesn't exist. Once it's

# generated, this task should do nothing.

do_cache_config() {
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{DISTRO_NAME}#"${DEBDISTRONAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- fi
-
- path_cache="${REPO_ISAR_DIR}/${DISTRO}"
- path_databases="${REPO_ISAR_DB_DIR}/${DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${DEBDISTRONAME}
- fi
+ repo_create "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" \
+ "${DEBDISTRONAME}"

}

addtask cache_config after do_unpack before do_build

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Store the results of "apt-get source" in the DL_DIR. This means we
really only need to fetch once on incremental rebuilds or if multiple
recipes use the same SRC_URI.
We can also later collect the results and put them in a repository.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/dpkg-base.bbclass | 20 ++++++++++++++++++--
meta/conf/bitbake.conf | 1 +
2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 4702e52b..1000ef37 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -65,14 +65,30 @@ do_apt_fetch() {
-o Dir::Etc::SourceParts="-" \
-o APT::Get::List-Cleanup="0"

- sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
- sh -c 'cd ${PP} && apt-get -y --only-source source ${SRC_APT}'
+ for uri in "${SRC_APT}"; do
+ sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${DISTRO}" "${uri}"
+ sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+ sh -c 'cp /downloads/deb-src/"$1"/"$2"/* ${PP} && cd ${PP} && apt-get -y --only-source source "$2"' my_script "${DISTRO}" "${uri}"
+ done
+
dpkg_undo_mounts
}

addtask apt_fetch after do_unpack before do_patch
do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"

+addtask cleanall_apt before do_cleanall
+do_cleanall_apt[nostamp] = "1"
+do_cleanall_apt() {
+ if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
+ exit
+ fi
+ for uri in "${SRC_APT}"; do
+ rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
+ done
+}
+
def get_package_srcdir(d):
s = os.path.abspath(d.getVar("S", True))
workdir = os.path.abspath(d.getVar("WORKDIR", True))
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index ab954b1f..6b05e91b 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
FILESEXTRAPATHS ?= "__default:"
GITDIR = "${DL_DIR}/git"
+DEBSRCDIR = "${DL_DIR}/deb-src"
P = "${PN}-${PV}"
PF = "${PN}-${PV}-${PR}"
PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Bitbake seems to execute all the functions in subshells, but in fact we
mean to return here so correct the style.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/dpkg-base.bbclass | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 1000ef37..5dd334d0 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -55,7 +55,7 @@ SRC_APT ?= ""

do_apt_fetch() {

if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then

- exit
+ return 0
fi
rm -rf ${S}
dpkg_do_mounts
@@ -82,7 +82,7 @@ addtask cleanall_apt before do_cleanall
do_cleanall_apt[nostamp] = "1"
do_cleanall_apt() {

if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then

- exit
+ return 0
fi

for uri in "${SRC_APT}"; do

rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

So far the deb-src packages downloaded via "apt://" where not cached.
This commit fixes that and includes those packages into the cache.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/base-apt-helper.bbclass | 7 +++++++
meta/classes/image-cache-extension.bbclass | 4 ++++
meta/classes/repository.bbclass | 14 ++++++++++++++
3 files changed, 25 insertions(+)

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 6fc57303..4be7ad27 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -52,4 +52,11 @@ populate_base_apt() {
"${BASE_DISTRO_CODENAME}" \
"${package}"
done
+
+ find $search_dir -name '*.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"

+ done
}
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 30db5493..f8b9ba34 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -32,5 +32,9 @@ do_cache_base_repo() {
if [ -d ${BUILDCHROOT_TARGET_DIR}/var/cache/apt ]; then
populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
fi
+
+ if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
+ populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
+ fi

}
addtask cache_base_repo after do_rootfs do_install_imager_deps

diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index cf0fb5a2..591ee04a 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -35,6 +35,20 @@ EOF
fi
}

+repo_add_srcpackage() {

+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi

+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main -P source \
+ includedsc "${codename}" \
+ "$@"
+}
+
repo_add_packages() {

local dir="$1"
local dbdir="$2"

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

The repository is ready for it already. For the moment there will be
nothing there.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/recipes-core/isar-bootstrap/files/base-apt-sources | 1 +
1 file changed, 1 insertion(+)

diff --git a/meta/recipes-core/isar-bootstrap/files/base-apt-sources b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
index ddc45093..2a4ab2c4 100644
--- a/meta/recipes-core/isar-bootstrap/files/base-apt-sources
+++ b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
@@ -1 +1,2 @@
deb [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
+deb-src [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

We used to have "trusted=yes" hardcoded even for a signed base-apt. Make
that flag depend on whether it is signed and generate that file with two
echos, it is simple enough.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

.../isar-bootstrap/files/base-apt-sources | 2 --
.../isar-bootstrap/isar-bootstrap.inc | 16 ++++++++--------
2 files changed, 8 insertions(+), 10 deletions(-)
delete mode 100644 meta/recipes-core/isar-bootstrap/files/base-apt-sources

diff --git a/meta/recipes-core/isar-bootstrap/files/base-apt-sources b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
deleted file mode 100644
index 2a4ab2c4..00000000
--- a/meta/recipes-core/isar-bootstrap/files/base-apt-sources
+++ /dev/null
@@ -1,2 +0,0 @@
-deb [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
-deb-src [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 16b4395b..cf3df407 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -12,8 +12,7 @@ SRC_URI = " \
file://isar-apt.conf \
file://isar-apt-fallback.conf \
file://locale \
- file://chroot-setup.sh \
- file://base-apt-sources"
+ file://chroot-setup.sh"
PV = "1.0"

DEBOOTSTRAP ?= "qemu-debootstrap"
@@ -21,7 +20,6 @@ ROOTFSDIR = "${WORKDIR}/rootfs"
APTPREFS = "${WORKDIR}/apt-preferences"
APTSRCS = "${WORKDIR}/apt-sources"
APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
-BASEAPTSRCS = "${WORKDIR}/base-apt-sources"
DISTRO_BOOTSTRAP_KEYFILES = ""
THIRD_PARTY_APT_KEYFILES = ""
DEPLOY_ISAR_BOOTSTRAP ?= ""
@@ -266,13 +264,15 @@ isar_bootstrap() {
"${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- sed -e "s#{BASE_DISTRO}#"${BASE_DISTRO}"#g" \
- -e "s#{BASE_DISTRO_CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- -i ${BASEAPTSRCS}
+ line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+ echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+
mkdir -p ${ROOTFSDIR}/base-apt
mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- install -v -m644 "${BASEAPTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
else
install -v -m644 "${APTSRCS}" \
"${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Now that base-apt supports deb-src we can include hello in a standard
build.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta-isar/conf/local.conf.sample | 4 +---
scripts/ci_build.sh | 1 -
2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 81a04261..ebc277b1 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -169,11 +169,9 @@ CONF_VERSION = "1"

#
# The default list of extra packages to be installed.
-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello"
## "cache_base_repo" relies on the cache ...
#IMAGE_INSTALL += "isar-disable-apt-cache"
-## uses apt:// SRC_URI, which does not work with the cache
-#IMAGE_INSTALL += "hello"

#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7cd5a2fc..f50c9e42 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -169,7 +169,6 @@ if [ -n "$REPRO_BUILD" ]; then
fi

sed -i -e 's/#IMAGE_INSTALL += "isar-disable-apt-cache"/IMAGE_INSTALL += "isar-disable-apt-cache"/g' conf/local.conf
-sed -i -e 's/#IMAGE_INSTALL += "hello"/IMAGE_INSTALL += "hello"/g' conf/local.conf

echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

This commit does not contain a functional change. We just split all
calls of "apt-get" into a download and an execution phase, so we can
later copy out the downloaded files.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/dpkg-gbp.bbclass | 3 ++
meta/classes/dpkg.bbclass | 5 ++-
meta/classes/image-tools-extension.bbclass | 5 +++
.../buildchroot/files/deps.sh | 37 +++++++++++--------
4 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 0533a52f..2afe9db4 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,6 +12,9 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"

do_install_builddeps_append() {
dpkg_do_mounts
+ sudo -E chroot ${BUILDCHROOT_DIR} \
+ apt-get install -y -o Debug::pkgProblemResolver=yes \
+ --no-install-recommends --download-only ${GBP_DEPENDS}
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 180a3a89..b26c6456 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -7,7 +7,10 @@ inherit dpkg-base
do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"
- sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh ${PP}/${PPS} ${DISTRO_ARCH}
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH}
dpkg_undo_mounts
}

diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index d4f216da..b8531cb4 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -30,6 +30,11 @@ do_install_imager_deps() {
-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \

-o Dir::Etc::SourceParts="-" \
-o APT::Get::List-Cleanup="0"

+ apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
+ --allow-unauthenticated --allow-downgrades --download-only install \
+ ${IMAGER_INSTALL}'
+
+ sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \
${IMAGER_INSTALL}'
diff --git a/meta/recipes-devtools/buildchroot/files/deps.sh b/meta/recipes-devtools/buildchroot/files/deps.sh
index 002085b1..6048305a 100644
--- a/meta/recipes-devtools/buildchroot/files/deps.sh
+++ b/meta/recipes-devtools/buildchroot/files/deps.sh
@@ -12,27 +12,34 @@ source /isar/common.sh
# 2) we add -y to go non-interactive
# 3) downgrades shall be allowed in case a package recipe was changed
install_cmd="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends \
- -y --allow-downgrades"
+ -y --allow-downgrades $3"

-# Make sure that we have latest isar-apt content.
-# Options meaning:
-# Dir::Etc::SourceList - specifies which source to be used
-# Dir::Etc::SourceParts - disables looking for the other sources
-# APT::Get::List-Cleanup - do not erase obsolete packages list for
-# upstream in '/var/lib/apt/lists'
-apt-get update \
- -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
- -o Dir::Etc::SourceParts="-" \

- -o APT::Get::List-Cleanup="0"

+if [ "$3" != "--download-only" ]; then
+ # Make sure that we have latest isar-apt content.
+ # Options meaning:
+ # Dir::Etc::SourceList - specifies which source to be used
+ # Dir::Etc::SourceParts - disables looking for the other sources
+ # APT::Get::List-Cleanup - do not erase obsolete packages list for
+ # upstream in '/var/lib/apt/lists'
+ apt-get update \
+ -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
+ -o Dir::Etc::SourceParts="-" \
+ -o APT::Get::List-Cleanup="0"
+fi

# Do not set an architecture when building only 'all' (generic) packages.
# This can avoid unneeded cross-build issues.
if ! grep "^Architecture:" debian/control | grep -qv "all"; then
- set_arch=""
+ set_arch=""
fi

# Install all build deps
-mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control
+if [ "$3" = "--download-only" ]; then
+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control 2>&1 \
+ | grep "mk-build-deps: Unable to install all build-dep packages"
+else
+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control

-# Upgrade any already installed packages in case we are partially rebuilding
-apt-get upgrade -y --allow-downgrades
+ # Upgrade any already installed packages in case we are partially rebuilding
+ apt-get upgrade -y --allow-downgrades
+fi
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Hook in between the "apt-get --download-only"s and the "apt-get"s and
copy out all the debs for later.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 15 +++++++++++++++
meta/classes/dpkg-base.bbclass | 1 +
meta/classes/dpkg-gbp.bbclass | 1 +
meta/classes/dpkg.bbclass | 1 +
meta/classes/image-locales-extension.bbclass | 2 +-
meta/classes/image-tools-extension.bbclass | 1 +
meta/classes/rootfs.bbclass | 14 +++++++++++---
meta/conf/bitbake.conf | 1 +
8 files changed, 32 insertions(+), 4 deletions(-)
create mode 100644 meta/classes/deb-dl-dir.bbclass

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
new file mode 100644
index 00000000..066ce1a3
--- /dev/null
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -0,0 +1,15 @@

+# This software is a part of ISAR.

+# Copyright (C) 2020 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+

+deb_dl_dir_export() {
+ export pc="${DEBDIR}/${DISTRO}"
+ export rootfs="${1}"
+ mkdir -p "${pc}"
+ flock "${pc}".lock -c ' \
+ sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' \
+ -exec cp -f '{}' "${pc}" \;
+ sudo chown -R $(id -u):$(id -g) "${pc}"
+ '
+}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 5dd334d0..9aa2d546 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -8,6 +8,7 @@ inherit buildchroot
inherit debianize
inherit terminal
inherit repository
+inherit deb-dl-dir

DEPENDS ?= ""

diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 2afe9db4..7e908eba 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -15,6 +15,7 @@ do_install_builddeps_append() {

sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \

--no-install-recommends --download-only ${GBP_DEPENDS}
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"

sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass

index b26c6456..7e2c86d6 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -9,6 +9,7 @@ do_install_builddeps() {
E="${@ isar_export_proxies(d)}"

sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \

${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"

sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${DISTRO_ARCH}

dpkg_undo_mounts
diff --git a/meta/classes/image-locales-extension.bbclass b/meta/classes/image-locales-extension.bbclass
index 3c0758ff..0f0d0ca4 100644
--- a/meta/classes/image-locales-extension.bbclass
+++ b/meta/classes/image-locales-extension.bbclass
@@ -25,7 +25,7 @@ def get_nopurge(d):
j.split()[0].split(".")[0],
j.split()[0]))))

-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN += "image_install_localepurge_download"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download"
image_install_localepurge_download[weight] = "40"
image_install_localepurge_download() {
sudo -E chroot '${ROOTFSDIR}' \
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index b8531cb4..ec67d940 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -34,6 +34,7 @@ do_install_imager_deps() {
--allow-unauthenticated --allow-downgrades --download-only install \
${IMAGER_INSTALL}'

+ deb_dl_dir_export ${BUILDCHROOT_DIR}

sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \

diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 64eaef70..706b43a7 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -1,5 +1,7 @@

# This software is a part of ISAR.

-# Copyright (c) Siemens AG, 2019
+# Copyright (c) Siemens AG, 2020
+
+inherit deb-dl-dir

ROOTFS_ARCH ?= "${DISTRO_ARCH}"
ROOTFS_DISTRO ?= "${DISTRO}"
@@ -119,8 +121,14 @@ rootfs_install_pkgs_download() {
/usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
}

-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN ??= ""
-ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN}"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= ""
+ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT}"
+
+ROOTFS_INSTALL_COMMAND += "rootfs_export_package_cache"
+rootfs_export_package_cache[weight] = "5"
+rootfs_export_package_cache() {
+ deb_dl_dir_export ${ROOTFSDIR}
+}

ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
rootfs_install_clean_files[weight] = "2"
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 6b05e91b..b7c0a8f5 100644

--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
FILESEXTRAPATHS ?= "__default:"
GITDIR = "${DL_DIR}/git"

+DEBDIR = "${DL_DIR}/deb"

DEBSRCDIR = "${DL_DIR}/deb-src"
P = "${PN}-${PV}"
PF = "${PN}-${PV}-${PR}"

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

We have all the .debs cached in DEBDIR now, take them from there because
the rootfss might not actually have all of them.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 4 ++++
1 file changed, 4 insertions(+)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index f8b9ba34..07e9e7d0 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -33,6 +33,10 @@ do_cache_base_repo() {

populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
fi

+ if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
+ populate_base_apt "${DEBDIR}"/"${DISTRO}"
+ fi

+
if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then

populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"

fi
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

If we ever downloaded a file there is no need to do that again. We can
not only use that DL_DIR to later construct a repository but also to
speed up our rebuilds.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 10 ++++++++++

meta/classes/dpkg-gbp.bbclass | 1 +
meta/classes/dpkg.bbclass | 1 +

meta/classes/image-tools-extension.bbclass | 1 +
meta/classes/rootfs.bbclass | 6 ++++++
5 files changed, 19 insertions(+)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 066ce1a3..2858fea2 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -3,6 +3,16 @@
#
# SPDX-License-Identifier: MIT

+deb_dl_dir_import() {

+ export pc="${DEBDIR}/${DISTRO}"
+ export rootfs="${1}"

+ [ ! -d "${pc}" ] && return 0
+ flock -s "${pc}".lock -c ' \
+ sudo find "${pc}" -type f -iname '*\.deb' -exec \
+ cp -f --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
+ '
+}
+
deb_dl_dir_export() {

export pc="${DEBDIR}/${DISTRO}"

export rootfs="${1}"
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 7e908eba..afa1e19b 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,6 +12,7 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"

do_install_builddeps_append() {
dpkg_do_mounts
+ deb_dl_dir_import "${BUILDCHROOT_DIR}"

sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \

--no-install-recommends --download-only ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 7e2c86d6..585365ee 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -7,6 +7,7 @@ inherit dpkg-base

do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"

+ deb_dl_dir_import "${BUILDCHROOT_DIR}"

sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \

${PP}/${PPS} ${DISTRO_ARCH} --download-only
deb_dl_dir_export "${BUILDCHROOT_DIR}"
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index ec67d940..6590ee7a 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -25,6 +25,7 @@ do_install_imager_deps() {
buildchroot_do_mounts

E="${@ isar_export_proxies(d)}"
+ deb_dl_dir_import ${BUILDCHROOT_DIR}

sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \

apt-get update \

-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \

diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 706b43a7..81759ebc 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -113,6 +113,12 @@ rootfs_install_resolvconf() {
fi
}

+ROOTFS_INSTALL_COMMAND += "rootfs_import_package_cache"
+rootfs_import_package_cache[weight] = "5"
+rootfs_import_package_cache() {
+ deb_dl_dir_import ${ROOTFSDIR}
+}
+
ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
rootfs_install_pkgs_download[weight] = "600"
rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

The bootstrap step also contains downloading of debs, so it should be
surrounded with an import/export as well.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/deb-dl-dir.bbclass | 1 +
.../recipes-core/isar-bootstrap/isar-bootstrap.inc | 14 +++++++++-----
2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 2858fea2..4eef0134 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -7,6 +7,7 @@ deb_dl_dir_import() {

export pc="${DEBDIR}/${DISTRO}"
export rootfs="${1}"

[ ! -d "${pc}" ] && return 0

+ mkdir -p "${rootfs}"/var/cache/apt/archives/

flock -s "${pc}".lock -c ' \

sudo find "${pc}" -type f -iname '*\.deb' -exec \

cp -f --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +

diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index cf3df407..7c4c7d66 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -28,6 +28,8 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES_append_gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support = ",apt-transport-https,ca-certificates"
HOST_DISTRO_APT_SOURCES += "conf/distro/${HOST_DISTRO}.list"

+inherit deb-dl-dir
+
python () {
distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split()

@@ -237,11 +239,12 @@ isar_bootstrap() {
fi
E="${@ isar_export_proxies(d)}"
export IS_HOST debootstrap_args E
- sudo -E -s <<'EOSUDO'
- set -e
- if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
- rm -rf --one-file-system "${ROOTFSDIR}"
+ if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
+ sudo rm -rf --one-file-system "${ROOTFSDIR}"
+ deb_dl_dir_import "${ROOTFSDIR}"

+ sudo -E -s <<'EOSUDO'
+ set -e
if [ ${IS_HOST} ]; then
${DEBOOTSTRAP} $debootstrap_args \
${@get_distro_components_argument(d, True)} \
@@ -333,8 +336,9 @@ isar_bootstrap() {

# Finalize debootstrap by setting the link in deploy
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
- fi
EOSUDO
+ fi
+ deb_dl_dir_export "${ROOTFSDIR}"
}

CLEANFUNCS = "clean_deploy"
--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

We now have all debs in DEBDIR anyways, no need to conditionally copy
them out of the rootfss we create.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/image-cache-extension.bbclass | 4 ----
meta/classes/image.bbclass | 2 +-
meta/classes/rootfs.bbclass | 10 ----------
3 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 07e9e7d0..19c9c593 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,10 +21,6 @@ do_cache_base_repo() {
"Try it without cross-build."
fi

- if [ -d ${WORKDIR}/apt_cache ]; then
- populate_base_apt ${WORKDIR}/apt_cache
- fi
-
if [ -d ${BUILDCHROOT_HOST_DIR}/var/cache/apt ]; then
populate_base_apt ${BUILDCHROOT_HOST_DIR}/var/cache/apt
fi
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 6bcc6cd2..adaca5db 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -60,7 +60,7 @@ image_do_mounts() {
}

ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
+ROOTFS_FEATURES += "clean-package-cache finalize-rootfs generate-manifest"
ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"

diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 81759ebc..be101890 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -9,7 +9,6 @@ ROOTFS_PACKAGES ?= ""

# Features of the rootfs creation:
# available features are:
-# 'copy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
# 'clean-package-cache' - delete package cache from rootfs
# 'generate-manifest' - generate a package manifest of the rootfs into ${ROOTFS_MANIFEST_DEPLOY_DIR}
# 'finalize-rootfs' - delete files needed to chroot into the rootfs
@@ -186,15 +185,6 @@ python do_rootfs_install() {
}
addtask rootfs_install before do_rootfs_postprocess after do_unpack

-ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache', 'rootfs_postprocess_copy_package_cache', '', d)}"
-rootfs_postprocess_copy_package_cache() {
- mkdir -p '${WORKDIR}/apt_cache'
- sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
- -maxdepth 1 -name '*.deb' -execdir /bin/mv -t '${WORKDIR}/apt_cache' '{}' '+'
- me="$(id -u):$(id -g)"
- sudo chown -R "$me" '${WORKDIR}/apt_cache'
-}
-
ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', 'rootfs_postprocess_clean_package_cache', '', d)}"
rootfs_postprocess_clean_package_cache() {

sudo -E chroot '${ROOTFSDIR}' \

--
2.24.1

[Henning Schild]

From: Henning Schild <henning...@siemens.com>

Just call that function once and make it find the debs and dscs in one
run.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

meta/classes/base-apt-helper.bbclass | 6 ++----
meta/classes/image-cache-extension.bbclass | 8 +-------
2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 4be7ad27..7ed28595 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -17,9 +17,7 @@ compare_pkg_md5sums() {
}

populate_base_apt() {
- search_dir=$1
-
- find $search_dir -name '*.deb' | while read package; do
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do

# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In

# addition, md5sums are compared to ensure that the package is the

@@ -53,7 +51,7 @@ populate_base_apt() {
"${package}"
done

- find $search_dir -name '*.dsc' | while read package; do
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do

repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \

"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \

"${BASE_DISTRO_CODENAME}" \
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index cff2a8ba..7945e8bc 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,12 +21,6 @@ do_cache_base_repo() {

"Try it without cross-build."
fi

- if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
- populate_base_apt "${DEBDIR}"/"${DISTRO}"
- fi
-
- if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
- populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
- fi
+ populate_base_apt

}
addtask cache_base_repo after do_rootfs do_install_imager_deps

--
2.24.1