[RFC 1/1] mark ci tests that need attention

[Felix Moessbauer]

We mark the tests that are either suboptimal from a performance
PoV or that otherwise need attention. In general, the following things
need to be improved:

- use smaller baseline images for feature tests
- avoid repeated builds of kernels (especially with fast tag)

No functional change.

Signed-off-by: Felix Moessbauer <felix.mo...@siemens.com>
---
testsuite/citest.py | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)

diff --git a/testsuite/citest.py b/testsuite/citest.py
index 27cc9ff6..23cd06e9 100755
--- a/testsuite/citest.py
+++ b/testsuite/citest.py
@@ -199,11 +199,14 @@ class CrossTest(CIBaseTest):
targets = [
'mc:qemuarm-buster:isar-image-ci',
'mc:qemuarm-bullseye:isar-image-ci',
+ # TODO: avoid kernel build (or at least build with ccache)
'mc:de0-nano-soc-bullseye:isar-image-base',
+ # TODO: avoid kernel build
'mc:stm32mp15x-bullseye:isar-image-base',
'mc:qemuarm-bookworm:isar-image-ci',
'mc:qemuarm64-focal:isar-image-base',
'mc:nanopi-neo-efi-bookworm:isar-image-base',
+ # TODO: avoid kernel build
'mc:phyboard-mira-bookworm:isar-image-base',
]

@@ -212,6 +215,7 @@ class CrossTest(CIBaseTest):

def test_cross_debsrc(self):
targets = [
+ # TODO: speedup task by removing prebuilt container added in 407a91322
'mc:qemuarm64-bookworm:isar-image-ci',
]

@@ -220,6 +224,7 @@ class CrossTest(CIBaseTest):

def test_cross_trixie(self):
targets = [
+ # TODO: avoid sbuild-chroot-target-db2m
'mc:qemuamd64-trixie:isar-image-base',
'mc:qemuarm64-trixie:isar-image-base',
]
@@ -271,6 +276,9 @@ class KernelTests(CIBaseTest):
"""
Tests associated with kernel builds and development.
:avocado: tags=kernel,full
+
+ TODO: as we have dedicated kernel tests here, can we avoid the
+ kernel builds in test_cross?
"""

def test_per_kernel(self):
@@ -306,6 +314,7 @@ class InitRdBaseTest(CIBaseTest):
def deploy_dir_image(self, machine):
return os.path.join(self.deploy_dir, 'images', machine)

+ # TODO: can we use a smaller baseline than isar-image-ci?
def dracut_in_image(self, targets):
machine = 'qemuamd64'
distro = 'bookworm'
@@ -326,6 +335,7 @@ class InitRdBaseTest(CIBaseTest):
cmd="systemctl is-active dracut-shutdown"
)

+ # TODO: use smaller baseline, currently we include containers
def build_image_with_dependent_initrd(self, image, initrd,
distro="debian-bookworm",
machine="qemuamd64",
@@ -365,6 +375,7 @@ class InitRdTest(InitRdBaseTest):

def test_dracut_in_image(self):
"""Test switch to dracut in an image recipe."""
+ # TODO: smaller baseline is sufficient to test this
self.dracut_in_image(['mc:qemuamd64-bookworm:isar-image-ci'])

def test_dracut_build_initrd(self):
@@ -377,6 +388,7 @@ class InitRdTest(InitRdBaseTest):
lines = InitRdBaseTest.DRACUT_CONF
lines.append("ROOTFS_INITRAMFS_GENERATOR_CMDLINE:append = ' --unknown-option'")
self.init()
+ # TODO: smaller baseline is sufficient to test this
self.perform_build_test('mc:qemuamd64-bookworm:isar-image-ci',
should_fail=True, lines=lines)

@@ -390,6 +402,7 @@ class InitRdTest(InitRdBaseTest):
f"INITRD_IMAGE = '{initrd}-{distro}-{machine}.initrd.img'",
f"do_image[depends] += '{initrd}:do_build'"
]
+ # TODO: either use smaller baseline, or better a parsing-only test

self.init()
self.build_image_with_dependent_initrd('isar-image-ci', initrd,
@@ -400,6 +413,7 @@ class InitRdTest(InitRdBaseTest):
initrd = 'isar-initramfs'
lines = [f"IMAGE_INITRD = '{initrd}'"]

+ # TODO: use smaller baseline
self.init()
self.build_image_with_dependent_initrd('isar-image-ci', initrd, lines=lines)

@@ -424,6 +438,7 @@ class InitRdTest(InitRdBaseTest):
f"IMAGE_INITRD = '{initrd}'",
"INITRD_IMAGE = 'not-a-valid-initrd-file'"
]
+ # TODO: smaller baseline is sufficient to test this
self.build_image_with_dependent_initrd('isar-image-ci', initrd, lines=lines)


@@ -435,6 +450,7 @@ class InitRdCrossTests(InitRdBaseTest):

def test_dracut_in_image(self):
"""Test switch to dracut in an image recipe."""
+ # TODO: can be tested with bare-minimal images
self.dracut_in_image([
'mc:qemuamd64-bookworm:isar-image-ci',
'mc:qemuamd64-trixie:isar-image-ci',
@@ -503,8 +519,11 @@ class NoCrossTest(CIBaseTest):
'mc:qemuamd64-buster:isar-image-ci',
'mc:qemuamd64-bullseye:isar-initramfs',
'mc:qemumipsel-bullseye:isar-image-base',
+ # TODO: avoid targets with kernel builds or use ccache
'mc:imx6-sabrelite-bullseye:isar-image-base',
+ # TODO: avoid targets with kernel builds or use ccache
'mc:phyboard-mira-bullseye:isar-image-base',
+ # TODO: avoid targets with kernel builds or use ccache
'mc:hikey-bullseye:isar-image-base',
'mc:virtualbox-bullseye:isar-image-base',
'mc:virtualbox-bookworm:isar-image-base',
@@ -517,6 +536,7 @@ class NoCrossTest(CIBaseTest):
'mc:qemuamd64-iso-bookworm:isar-image-ci',
'mc:qemui386-bookworm:isar-image-base',
'mc:qemumipsel-bookworm:isar-image-ci',
+ # TODO: avoid targets with kernel builds or use ccache
'mc:hikey-bookworm:isar-image-base',
'mc:beagleplay-bookworm:isar-image-base',
'mc:qemuarm64-noble:isar-image-base',
@@ -532,6 +552,7 @@ class NoCrossTest(CIBaseTest):
def test_nocross_debsrc(self):
targets = [
'mc:qemuarm-bookworm:isar-image-ci',
+ # TODO: avoid targets with kernel builds or use ccache
'mc:stm32mp15x-bullseye:isar-image-base',
'mc:de0-nano-soc-bookworm:isar-image-base',
]
@@ -565,6 +586,7 @@ class NoCrossTest(CIBaseTest):
'mc:qemuarm64-trixie:isar-image-base',
'mc:qemuarm-trixie:isar-image-base',
'mc:qemuriscv64-trixie:isar-image-base',
+ # TODO: avoid targets with kernel builds or use ccache
'mc:sifive-fu540-trixie:isar-image-base',
'mc:starfive-visionfive2-trixie:isar-image-base',
]
@@ -598,6 +620,7 @@ class ContainerImageTest(CIBaseTest):

@skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found')
def test_container_image(self):
+ # TODO: add trixie target
targets = [
'mc:container-amd64-buster:isar-image-base',
'mc:container-amd64-bullseye:isar-image-base',
@@ -618,6 +641,7 @@ class ContainerSdkTest(CIBaseTest):

@skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found')
def test_container_sdk(self):
+ # TODO: why is this only tested on bullseye? Consider moving to trixie
targets = ['mc:container-amd64-bullseye:isar-image-base']

self.init()
--
2.51.0

[Jan Kiszka]

Most arm boards, specifically 32-bit, still have issues with debian
kernels. So, if you want to test the image, you must build the kernel.
But using caches makes a lot of sense.

> ]
>
> @@ -212,6 +215,7 @@ class CrossTest(CIBaseTest):
>
> def test_cross_debsrc(self):
> targets = [
> + # TODO: speedup task by removing prebuilt container added in 407a91322
> 'mc:qemuarm64-bookworm:isar-image-ci',

...provided the prebuild container in a cross scenario is already tested
elsewhere. Provide reasoning, please, not just speed-up suggestions, or
we cannot execute them.

> ]
>
> @@ -220,6 +224,7 @@ class CrossTest(CIBaseTest):
>
> def test_cross_trixie(self):
> targets = [
> + # TODO: avoid sbuild-chroot-target-db2m

See above.

> 'mc:qemuamd64-trixie:isar-image-base',
> 'mc:qemuarm64-trixie:isar-image-base',
> ]
> @@ -271,6 +276,9 @@ class KernelTests(CIBaseTest):
> """
> Tests associated with kernel builds and development.
> :avocado: tags=kernel,full
> +
> + TODO: as we have dedicated kernel tests here, can we avoid the
> + kernel builds in test_cross?

Only if those other kernel builds do not need to generate bootable images.

A good share of you remarks point out that testcases are also aging and
need frequent maintenance to keep them focused, fresh and still
sufficiently broad. Tough work. I just spent >3 weeks on getting
isar-cip-core tests in a better shape again, and we are constantly
struggling how to avoid test matrix explosion in isar-cip-core or xenomai.

Jan

--
Siemens AG, Foundational Technologies
Linux Expert Center

[MOESSBAUER, Felix]

On Fri, 2025-12-12 at 17:42 +0100, Jan Kiszka wrote:
> On 12.12.25 17:34, Felix Moessbauer wrote:
> > We mark the tests that are either suboptimal from a performance
> > PoV or that otherwise need attention. In general, the following things
> > need to be improved:
> >
> > - use smaller baseline images for feature tests
> > - avoid repeated builds of kernels (especially with fast tag)
> >
> > No functional change.
> >
> > Signed-off-by: Felix Moessbauer <felix.mo...@siemens.com>
> >
>

> A good share of you remarks point out that testcases are also aging and
> need frequent maintenance to keep them focused, fresh and still
> sufficiently broad. 
>

I just noticed, that the test_cross_kselftest is not cross anymore,
because the kselftest class explicitly disables it via
ISAR_CROSS_COMPILE = "0". This is a prime example of the test aging
pattern.

Monitoring the CI execution time helps to spot slow time increases, as
well as spikes due to unintended changes (like this one).
This at least gives an indication.

Felix

--
Siemens AG
Linux Expert Center
Friedrich-Ludwig-Bauer-Str. 3
85748 Garching, Germany

[Anton Mikanovich]

Hello Felix,

12/12/2025 18:58, 'MOESSBAUER, Felix' via isar-users wrote:
> I just noticed, that the test_cross_kselftest is not cross anymore,
> because the kselftest class explicitly disables it via
> ISAR_CROSS_COMPILE = "0". This is a prime example of the test aging
> pattern.

I've just merged the commit:
[PATCH] meta/classes-recipe/linux-kselftest: remove forced
ISAR_CROSS_COMPILE = "0"
so this case is no more exists.