[PATCH v2 0/2] Isar installer copy installer-target direct to rootfs

[Quirin Gylstorff]

From: Quirin Gylstorff <quirin.g...@siemens.com>

This reduce the build time of the isar-installer image as we don't apply
gzip on a already compressed image. Also the sstate size is reduced as
we don't store a package with hundredths of MB.

Changes v2:
- split patches into rootfs-add-files and the installer specific
target

Quirin Gylstorff (2):
add new class to add files directly to the rootfs
isar-installer: use rootfs-add-files class instead of seperate package

.../classes/installer-add-rootfs.bbclass | 43 ++++++++++++++++++
.../images/isar-image-installer.bb | 2 +-
.../deploy-image/deploy-image_0.1.bb | 3 +-
.../store-target-image_0.1.bb | 25 -----------
meta/classes/rootfs-add-files.bbclass | 45 +++++++++++++++++++
5 files changed, 90 insertions(+), 28 deletions(-)
create mode 100644 meta-isar/classes/installer-add-rootfs.bbclass
delete mode 100644 meta-isar/recipes-installer/store-target-image/store-target-image_0.1.bb
create mode 100644 meta/classes/rootfs-add-files.bbclass

--
2.43.0

[Quirin Gylstorff]

From: Quirin Gylstorff <quirin.g...@siemens.com>

This reduces the sstate size and speed up the build.

Signed-off-by: Quirin Gylstorff <quirin.g...@siemens.com>
---
.../classes/installer-add-rootfs.bbclass | 43 +++++++++++++++++++

.../images/isar-image-installer.bb | 2 +-
.../deploy-image/deploy-image_0.1.bb | 3 +-
.../store-target-image_0.1.bb | 25 -----------

4 files changed, 45 insertions(+), 28 deletions(-)

create mode 100644 meta-isar/classes/installer-add-rootfs.bbclass
delete mode 100644 meta-isar/recipes-installer/store-target-image/store-target-image_0.1.bb

diff --git a/meta-isar/classes/installer-add-rootfs.bbclass b/meta-isar/classes/installer-add-rootfs.bbclass
new file mode 100644
index 00000000..765e10ac
--- /dev/null
+++ b/meta-isar/classes/installer-add-rootfs.bbclass
@@ -0,0 +1,43 @@
+# Copy the target image to the installer image
+#
+# This software is a part of ISAR.
+# Copyright (C) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+inherit rootfs-add-files
+
+INSTALLER_MC ??= "isar-installer"
+INSTALLER_TARGET_IMAGE ??= "isar-image-base"
+INSTALLER_TARGET_MC ??= "installer-target"
+INSTALLER_TARGET_DISTRO ??= "${DISTRO}"
+INSTALLER_TARGET_MACHINE ??= "${MACHINE}"
+
+IMAGE_DATA_FILE ??= "${INSTALLER_TARGET_IMAGE}-${INSTALLER_TARGET_DISTRO}-${INSTALLER_TARGET_MACHINE}"
+IMAGE_DATA_POSTFIX ??= "wic.zst"
+IMAGE_DATA_POSTFIX:buster ??= "wic.xz"
+IMAGE_DATA_POSTFIX:bullseye ??= "wic.xz"
+
+ROOTFS_ADDITIONAL_FILES ??= "installer-target"
+
+def get_installer_source(d):
+ installer_target_image = d.getVar('INSTALLER_TARGET_IMAGE') or ""
+ if not installer_target_image:
+ return ""
+ deploy_dir = d.getVar('DEPLOY_DIR_IMAGE')
+ image_data = d.getVar('IMAGE_DATA_FILE')
+ image_data_postfix = d.getVar('IMAGE_DATA_POSTFIX')
+ return f"{deploy_dir}/{image_data}.{image_data_postfix}"
+
+def get_installer_destination(d):
+ installer_target_image = d.getVar('INSTALLER_TARGET_IMAGE') or ""
+ if not installer_target_image:
+ return "/install/keep"
+ image_data = d.getVar('IMAGE_DATA_FILE')
+ image_data_postfix = d.getVar('IMAGE_DATA_POSTFIX')
+ return f"/install/{image_data}.{image_data_postfix}"
+
+ROOTFS_ADDITIONAL_FILE_installer-target[source] = "${@ get_installer_source(d)}"
+ROOTFS_ADDITIONAL_FILE_installer-target[destination] = "${@ get_installer_destination(d)}"
+
+do_rootfs_install[mcdepends] += "mc:${INSTALLER_MC}:${INSTALLER_TARGET_MC}:${INSTALLER_TARGET_IMAGE}:do_image_wic"
diff --git a/meta-isar/recipes-core/images/isar-image-installer.bb b/meta-isar/recipes-core/images/isar-image-installer.bb
index 27de03d5..34fb5daf 100644
--- a/meta-isar/recipes-core/images/isar-image-installer.bb
+++ b/meta-isar/recipes-core/images/isar-image-installer.bb
@@ -4,6 +4,7 @@
# SPDX-License-Identifier: MIT

inherit image
+inherit installer-add-rootfs
DESCRIPTION = "Example of a ISAR based Installer Image"

# Use variable to switch easily to another wks
@@ -11,7 +12,6 @@ INSTALLER_WKS_FILE ??= "installer-efi.wks.in"
WKS_FILE = "${INSTALLER_WKS_FILE}"
IMAGER_INSTALL:wic:append = " systemd-boot"

-IMAGE_INSTALL += "store-target-image"
IMAGE_INSTALL += "deploy-image"

IMAGE_INSTALL:remove = "expand-on-first-boot"
diff --git a/meta-isar/recipes-installer/deploy-image/deploy-image_0.1.bb b/meta-isar/recipes-installer/deploy-image/deploy-image_0.1.bb
index ea047849..0e61de03 100644
--- a/meta-isar/recipes-installer/deploy-image/deploy-image_0.1.bb
+++ b/meta-isar/recipes-installer/deploy-image/deploy-image_0.1.bb
@@ -11,8 +11,7 @@ inherit dpkg-raw
SRC_URI = "file://deploy-image-wic.sh \
file://install.override.conf \
"
-DEPENDS = "store-target-image"
-DEBIAN_DEPENDS = "store-target-image, bmap-tools, pv, dialog, util-linux, parted, fdisk, gdisk, pigz, xz-utils, pbzip2, zstd"
+DEBIAN_DEPENDS = "bmap-tools, pv, dialog, util-linux, parted, fdisk, gdisk, pigz, xz-utils, pbzip2, zstd"
do_install[cleandirs] = "${D}/usr/bin/ \
${D}/usr/lib/systemd/system/ge...@tty1.service.d/ \
${D}/usr/lib/systemd/system/serial...@ttyS0.service.d/"
diff --git a/meta-isar/recipes-installer/store-target-image/store-target-image_0.1.bb b/meta-isar/recipes-installer/store-target-image/store-target-image_0.1.bb
deleted file mode 100644
index 2f72412b..00000000
--- a/meta-isar/recipes-installer/store-target-image/store-target-image_0.1.bb
+++ /dev/null
@@ -1,25 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) Siemens AG, 2024
-#
-# SPDX-License-Identifier: MIT
-
-DESCRIPTION = "Add a given target image to rootfs"
-
-inherit dpkg-raw
-
-INSTALLER_TARGET_IMAGE ??= "isar-image-base"
-IMG_DATA_FILE ??= "${INSTALLER_TARGET_IMAGE}-${DISTRO}-${MACHINE}"
-IMG_DATA_POSTFIX ??= "wic.zst"
-IMG_DATA_POSTFIX:buster ??= "wic.xz"
-IMG_DATA_POSTFIX:bullseye ??= "wic.xz"
-do_install[mcdepends] = "${@ 'mc:isar-installer:installer-target:' + d.getVar('INSTALLER_TARGET_IMAGE') + ':do_image_wic' if d.getVar('INSTALLER_TARGET_IMAGE') else ''}"
-do_install[cleandirs] = "${D}/install/"
-do_install() {
- if [ -f ${DEPLOY_DIR_IMAGE}/${IMG_DATA_FILE}.${IMG_DATA_POSTFIX} ]; then
- install -m 0600 ${DEPLOY_DIR_IMAGE}/${IMG_DATA_FILE}.${IMG_DATA_POSTFIX} ${D}/install/
- install -m 0600 ${DEPLOY_DIR_IMAGE}/${IMG_DATA_FILE}.wic.bmap ${D}/install/
- else
- # mcopy cannot handle .keep or empty directory , therefore use visible file
- touch ${D}/install/keep
- fi
-}
--
2.43.0

[Quirin Gylstorff]

From: Quirin Gylstorff <quirin.g...@siemens.com>

This allows to add files like the installer target directly
to rootfs without compressing the target file. It also reduces
the sstate size.

Signed-off-by: Quirin Gylstorff <quirin.g...@siemens.com>
---

meta/classes/rootfs-add-files.bbclass | 45 +++++++++++++++++++++++++++
1 file changed, 45 insertions(+)
create mode 100644 meta/classes/rootfs-add-files.bbclass

diff --git a/meta/classes/rootfs-add-files.bbclass b/meta/classes/rootfs-add-files.bbclass
new file mode 100644
index 00000000..dfee440f
--- /dev/null
+++ b/meta/classes/rootfs-add-files.bbclass
@@ -0,0 +1,45 @@

+# This software is a part of ISAR.
+# Copyright (C) Siemens AG, 2024

+# This class allows to add a file directly to
+# the rootfs.
+# An example usecase would be an installer which
+# contains an complete rootfs added with multiconfig.
+ROOTFS_INSTALL_COMMAND =+ "rootfs_add_files"
+ROOTFS_ADDITIONAL_FILES ??= ""
+
+# ROOTFS_ADDITIONAL_FILES ??= "installer-target"
+#
+# ROOTFS_ADDITIONAL_FILE_installer-target[source] = \
+# "${DEPLOY_DIR_IMAGE}/${IMG_DATA_FILE}.${IMAGE_DATA_POSTFIX}"
+# ROOTFS_ADDITIONAL_FILE_installer-target[destination] = \
+# "/install/${IMG_DATA_FILE}.${IMAGE_DATA_POSTFIX}"
+
+
+python rootfs_add_files() {
+ import os
+ if d.getVar("SOURCE_DATE_EPOCH") != None:
+ os.environ["SOURCE_DATE_EPOCH"] = d.getVar("SOURCE_DATE_EPOCH")
+
+ postprocess_additional_files = d.getVar('ROOTFS_ADDITIONAL_FILES').split()
+ rootfsdir = d.getVar("ROOTFSDIR")
+
+ for entry in postprocess_additional_files:
+ additional_file_entry = f"ROOTFS_ADDITIONAL_FILE_{entry}"
+ destination = d.getVarFlag(additional_file_entry, "destination") or ""
+ source = d.getVarFlag(additional_file_entry, "source") or ""
+ if os.path.exists(f"{rootfsdir}/{destination}"):
+ bb.process.run([ "/usr/bin/rm", "-f", f"{destination}"])
+
+ # empty source creates only an empty destination file
+ if not source:
+ dest_dir = os.path.dirname(destination)
+ bb.process.run(["sudo", "-E", "/usr/bin/install", "-d", "-m", "600", f"{rootfsdir}/{dest_dir}" ])
+ bb.process.run(["sudo", "-E", "/usr/bin/touch", f"{rootfsdir}/{destination}" ])
+ return
+
+ if not os.path.exists(f"{source}"):
+ bb.error(f"{source} does not exists and cannot be copied to the rootfs!")
+ # no recursive copy only single files
+ bb.process.run(["sudo", "-E", "/usr/bin/install", "-D", "-m", "600", f"{source}", f"{rootfsdir}/{destination}" ])
+}
+ROOTFS_INSTALL_COMMAND += "rootfs_add_files"
--
2.43.0

[MOESSBAUER, Felix]

Hi Quirin,

we also need to copy the .bmap file (if available):

bmaptool: info: no bmap given, copy entire image to '/dev/sda'

Felix

> +}
> +ROOTFS_INSTALL_COMMAND += "rootfs_add_files"

--

Siemens AG, Technology
Linux Expert Center

[Quirin Gylstorff]

From: Quirin Gylstorff <quirin.g...@siemens.com>

This reduce the build time of the isar-installer image as we don't apply
gzip on a already compressed image. Also the sstate size is reduced as
we don't store a package with hundredths of MB.

Changes v3:
- add missing bmap file
- support initramfs, install -D -m is greed and sets the permissions
of the parent folder. Therefore the initramfs could not be
generated.

Changes v2:
- split patches into rootfs-add-files and the installer specific
target

Quirin Gylstorff (2):
add new class to add files directly to the rootfs
isar-installer: use rootfs-add-files class instead of seperate package

.../classes/installer-add-rootfs.bbclass | 43 ++++++++++++++++++

.../images/isar-image-installer.bb | 2 +-
.../deploy-image/deploy-image_0.1.bb | 3 +-
.../store-target-image_0.1.bb | 25 -----------

meta/classes/rootfs-add-files.bbclass | 45 +++++++++++++++++++
5 files changed, 90 insertions(+), 28 deletions(-)

create mode 100644 meta-isar/classes/installer-add-rootfs.bbclass
delete mode 100644 meta-isar/recipes-installer/store-target-image/store-target-image_0.1.bb

[Quirin Gylstorff]

From: Quirin Gylstorff <quirin.g...@siemens.com>

This allows to add files like the installer target directly
to rootfs without compressing the target file. It also reduces
the sstate size.

Signed-off-by: Quirin Gylstorff <quirin.g...@siemens.com>
---
meta/classes/rootfs-add-files.bbclass | 45 +++++++++++++++++++++++++++
1 file changed, 45 insertions(+)
create mode 100644 meta/classes/rootfs-add-files.bbclass

diff --git a/meta/classes/rootfs-add-files.bbclass b/meta/classes/rootfs-add-files.bbclass
new file mode 100644

index 00000000..4125caf5

+ dest_dir = os.path.dirname(destination)
+ bb.process.run(["sudo", "-E", "/usr/bin/mkdir", "-p", f"{rootfsdir}/{dest_dir}" ])
+ # empty source creates only an empty destination file
+ if not source:
+ bb.process.run(["sudo", "-E", "/usr/bin/touch", f"{rootfsdir}/{destination}" ])

+ return
+
+ if not os.path.exists(f"{source}"):
+ bb.error(f"{source} does not exists and cannot be copied to the rootfs!")
+ # no recursive copy only single files

+ bb.process.run(["sudo", "-E", "/usr/bin/cp", "-a", f"{source}", f"{rootfsdir}/{destination}" ])

+}
+ROOTFS_INSTALL_COMMAND += "rootfs_add_files"
--

2.43.0

[Quirin Gylstorff]

From: Quirin Gylstorff <quirin.g...@siemens.com>

This reduces the sstate size and speed up the build.

Signed-off-by: Quirin Gylstorff <quirin.g...@siemens.com>
---

.../classes/installer-add-rootfs.bbclass | 43 +++++++++++++++++++

.../images/isar-image-installer.bb | 2 +-
.../deploy-image/deploy-image_0.1.bb | 3 +-
.../store-target-image_0.1.bb | 25 -----------

4 files changed, 45 insertions(+), 28 deletions(-)

create mode 100644 meta-isar/classes/installer-add-rootfs.bbclass
delete mode 100644 meta-isar/recipes-installer/store-target-image/store-target-image_0.1.bb

diff --git a/meta-isar/classes/installer-add-rootfs.bbclass b/meta-isar/classes/installer-add-rootfs.bbclass
new file mode 100644
index 00000000..656761e7

--- /dev/null
+++ b/meta-isar/classes/installer-add-rootfs.bbclass
@@ -0,0 +1,43 @@
+# Copy the target image to the installer image
+#

+# This software is a part of ISAR.
+# Copyright (C) Siemens AG, 2024
+#

+# SPDX-License-Identifier: MIT
+
+inherit rootfs-add-files
+
+INSTALLER_MC ??= "isar-installer"
+INSTALLER_TARGET_IMAGE ??= "isar-image-base"
+INSTALLER_TARGET_MC ??= "installer-target"
+INSTALLER_TARGET_DISTRO ??= "${DISTRO}"
+INSTALLER_TARGET_MACHINE ??= "${MACHINE}"
+
+IMAGE_DATA_FILE ??= "${INSTALLER_TARGET_IMAGE}-${INSTALLER_TARGET_DISTRO}-${INSTALLER_TARGET_MACHINE}"
+IMAGE_DATA_POSTFIX ??= "wic.zst"
+IMAGE_DATA_POSTFIX:buster ??= "wic.xz"
+IMAGE_DATA_POSTFIX:bullseye ??= "wic.xz"
+

+ROOTFS_ADDITIONAL_FILES ??= "installer-target installer-target-bmap"
+
+def get_installer_source(d, suffix):

+ installer_target_image = d.getVar('INSTALLER_TARGET_IMAGE') or ""
+ if not installer_target_image:
+ return ""
+ deploy_dir = d.getVar('DEPLOY_DIR_IMAGE')
+ image_data = d.getVar('IMAGE_DATA_FILE')

+ return f"{deploy_dir}/{image_data}.{suffix}"
+
+def get_installer_destination(d, suffix):

+ installer_target_image = d.getVar('INSTALLER_TARGET_IMAGE') or ""
+ if not installer_target_image:
+ return "/install/keep"
+ image_data = d.getVar('IMAGE_DATA_FILE')

+ return f"/install/{image_data}.{suffix}"
+
+ROOTFS_ADDITIONAL_FILE_installer-target[source] = "${@ get_installer_source(d, d.getVar('IMAGE_DATA_POSTFIX'))}"
+ROOTFS_ADDITIONAL_FILE_installer-target[destination] = "${@ get_installer_destination(d, d.getVar('IMAGE_DATA_POSTFIX'))}"
+ROOTFS_ADDITIONAL_FILE_installer-target-bmap[source] = "${@ get_installer_source(d, "wic.bmap")}"
+ROOTFS_ADDITIONAL_FILE_installer-target-bmap[destination] = "${@ get_installer_destination(d, "wic.bmap")}"

[Uladzimir Bely]

On Mon, 2024-03-25 at 16:34 +0100, 'Quirin Gylstorff' via isar-users
wrote:

--
Applied to next, thanks.