[PATCH 0/3] isar-sstate: improve cache debugging capabilities

[Felix Moessbauer]

We sometimes still could profit more from the sstate cache by avoiding
needless dependencies or shrinking artifact sizes.

To help identify cases where we trash the cache, we add two helper commands:

- isar-sstate show: list all artifacts of a PN
- isar-sstate delta: compare an artifact with others

The helpers work both with local and remote caches.

Best regards,
Felix Moessbauer
Siemens AG

Felix Moessbauer (3):
isar-sstate: add helpers to read signature data
isar-sstate: add show command to list all artifacts of a PN
isar-sstate: add delta command to compare artifact with others

scripts/isar-sstate | 171 ++++++++++++++++++++++++++++++++++++++++----
1 file changed, 159 insertions(+), 12 deletions(-)

--
2.55.0

[Felix Moessbauer]

For analyzing why some tasks are not cached efficiently (i.e. we get an
excessive amount of cache artifacts) it helps to just list all artifacts
of that PN.

This is implemented in the isar-sstate show command, which takes a PN and
lists all cache artifacts (along with age and size), grouped by
architecture, task name and DISTRO. As the distro is not encoded in the
artifact name, it is read from the signature data. Tasks that do not
depend on DISTRO are reported as 'unknown'.

Signed-off-by: Felix Moessbauer <felix.mo...@siemens.com>
---
scripts/isar-sstate | 71 ++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 67 insertions(+), 4 deletions(-)

diff --git a/scripts/isar-sstate b/scripts/isar-sstate
index b5eaabaa..b0d4033f 100755
--- a/scripts/isar-sstate
+++ b/scripts/isar-sstate
@@ -22,7 +22,7 @@ sstate cache:
`SSTATE_DIR`. To share them, you need to explicitly upload them to
the shared location, which is what isar-sstate is for.

-isar-sstate implements five commands (upload, clean, info, analyze, lint),
+isar-sstate implements six commands (upload, clean, info, show, analyze, lint),
and supports three remote backends (filesystem, http/webdav, AWS S3).

## Commands
@@ -54,6 +54,11 @@ than `max_age`.
The `info` command scans the remote cache and displays some basic statistics.
The argument `--verbose` increases the amount of information displayed.

+### show
+
+The `show` command lists all individual artifacts for a recipe (`PN`)
+in the remote cache, grouped by architecture, task name, `DISTRO`, and hash.
+
### analyze

The `analyze` command iterates over all artifacts in the local sstate cache,
@@ -592,6 +597,15 @@ def apply_filters(items, pn_filter=None, arch=None):
return items


+def format_size(size_bytes):
+ size = float(size_bytes)
+ for unit in ['B', 'KB', 'MB', 'GB']:
+ if size < 1024.0:
+ return f"{size:.0f} {unit}" if unit == 'B' else f"{size:.2f} {unit}"
+ size /= 1024.0
+ return f"{size:.2f} TB"
+
+
def load_sigdata(target, path):
sig_file = target.download(path)
try:
@@ -616,11 +630,11 @@ def arguments():
parser = argparse.ArgumentParser()
parser.add_argument(
'command', type=str, metavar='command',
- choices='info upload clean analyze lint'.split(),
- help="command to execute (info, upload, clean, analyze, lint)")
+ choices='info upload clean show analyze lint'.split(),
+ help="command to execute (info, upload, clean, show, analyze, lint)")
parser.add_argument(
'source', type=str, nargs='?',
- help="local sstate dir (for uploads or analysis)")
+ help="local sstate dir (for uploads or analysis), or PN (for show)")
parser.add_argument(
'target', type=str,
help="remote sstate location (a file://, http://, or s3:// URI)")
@@ -661,6 +675,9 @@ def arguments():
if args.command in 'upload analyze'.split() and args.source is None:
print(f"ERROR: '{args.command}' needs a source and target")
sys.exit(1)
+ elif args.command == 'show' and args.source is None:
+ print(f"ERROR: '{args.command}' needs a PN and target")
+ sys.exit(1)
elif args.command in 'info clean'.split() and args.source is not None:
print(f"ERROR: '{args.command}' must not have a source (only a target)")
sys.exit(1)
@@ -801,6 +818,52 @@ def sstate_info(target, verbose, filter, arch, **kwargs):
return 0


+def sstate_show(source, target, verbose, filter, arch, **kwargs):
+ pn = source
+ if not target.exists():
+ print(f"WARNING: cannot access target {target}. No info to show.")
+ return 0
+
+ print(f"INFO: scanning {target}")
+ all_files = target.list_all()
+ suffixes = ['tgz', 'tar.zst']
+ if verbose:
+ suffixes += ['tgz.siginfo', 'tar.zst.siginfo']
+ artifacts = [f for f in all_files if f.pn == pn and f.suffix in suffixes]
+ artifacts = apply_filters(artifacts, filter, arch)
+
+ # DISTRO is not part of the artifact name, it has to come from the siginfo
+ distro = {f.hash: get_distro(target, f.path) or 'unknown' for f in all_files
+ if f.pn == pn and f.suffix.endswith('.siginfo')}
+
+ archs = sorted(set([f.arch for f in artifacts]))
+ for a in archs:
+ print(f"{a}:")
+ arch_entries = [f for f in artifacts if f.arch == a]
+ tasks = sorted(set([f.task for f in arch_entries]))
+ for t in tasks:
+ print(f" {t}:")
+ task_entries = [f for f in arch_entries if f.task == t]
+ distros = sorted(set(distro.get(f.hash, 'unknown') for f in task_entries))
+ for d in distros:
+ print(f" {d}:")
+ distro_entries = [f for f in task_entries if distro.get(f.hash, 'unknown') == d]
+ # a hash covers both the archive and its siginfo, use the newest of them
+ hash_age = {}
+ for f in distro_entries:
+ hash_age[f.hash] = min(hash_age.get(f.hash, f.age), f.age)
+ for h in sorted(hash_age, key=lambda x: (hash_age[x], x)):
+ print(f" - {h}")
+ hash_entries = [f for f in distro_entries if f.hash == h]
+ for f in sorted(hash_entries, key=lambda x: x.suffix):
+ age_str = str(datetime.timedelta(seconds=f.age))
+ size_str = format_size(f.size)
+ if verbose:
+ print(f" {f.path}")
+ print(f" {age_str:>20}\t{size_str:>10}\t{f.suffix}")
+ return 0
+
+
def sstate_analyze(source, target, filter, arch, **kwargs):
if not os.path.isdir(source):
print(f"WARNING: source {source} does not exist. Nothing to analyze.")
--
2.55.0

[Felix Moessbauer]

The lint command reads and decodes the signature data inline. Upcoming
commands need the same data, so move the reading into a load_sigdata
helper and add get_distro on top of it, which extracts the DISTRO a
signature was generated for.

Next to deduplicating the code, this also releases the downloaded file
in case it cannot be decoded, which previously leaked a temporary file
on the http and s3 backends.

Signed-off-by: Felix Moessbauer <felix.mo...@siemens.com>
---

scripts/isar-sstate | 30 ++++++++++++++++++++++--------
1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/scripts/isar-sstate b/scripts/isar-sstate
index ada154e2..b5eaabaa 100755
--- a/scripts/isar-sstate
+++ b/scripts/isar-sstate
@@ -592,6 +592,26 @@ def apply_filters(items, pn_filter=None, arch=None):
return items


+def load_sigdata(target, path):
+ sig_file = target.download(path)
+ try:
+ with bb.compress.zstd.open(sig_file, "rt", encoding="utf-8", num_threads=1) as f:
+ sigdata = json.load(f, object_hook=bb.siggen.SetDecoder)
+ bb.siggen.handle_renames(sigdata)
+ return sigdata
+ except:
+ # invalid file or format, ignore to continue processing
+ return None
+ finally:
+ target.release(sig_file)
+
+
+def get_distro(target, path):
+ # only tasks that depend on DISTRO carry it in their signature
+ sigdata = load_sigdata(target, path)
+ return sigdata['varvals'].get('DISTRO') if sigdata else None
+
+

def arguments():
parser = argparse.ArgumentParser()
parser.add_argument(

@@ -883,13 +903,8 @@ def sstate_lint(target, verbose, sources_dir, build_dir, exit_code, pedantic, li
if any(fnmatchcase(sig.task, pattern) for pattern in ADDITIONAL_IGNORED_TASKS):
continue

- sig_file = target.download(sig.path)
- try:
- with bb.compress.zstd.open(sig_file, "rt", encoding="utf-8", num_threads=1) as f:
- sigdata = json.load(f, object_hook=bb.siggen.SetDecoder)
- bb.siggen.handle_renames(sigdata)
- except:
- # invalid file or format... never mind
+ sigdata = load_sigdata(target, sig.path)
+ if sigdata is None:
continue

pn_issues = []
@@ -937,7 +952,6 @@ def sstate_lint(target, verbose, sources_dir, build_dir, exit_code, pedantic, li
if len(pn_issues) > 0:
print(f"\033[1;33m==== issues found in {sig.arch}:{sig.pn}:{sig.task} ({sig.hash[:8]}) ====\033[0m")
print('\n'.join(pn_issues))
- target.release(sig_file)

sum_hits = hits_srcdir + hits_builddir
if sum_hits == 0:
--
2.55.0

[Felix Moessbauer]

For debugging cache efficiency, it helps to easily compare one artifact
to all other artifacts of the same arch / PN / task. For that, we
introduce the delta command, which behaves similar to the analyze
command but takes a hash as input.

The comparison candidates are ordered by age and are limited to the
DISTRO of the given hash, as artifacts of other distros always differ.
Use --verbose to compare against all distros.

Signed-off-by: Felix Moessbauer <felix.mo...@siemens.com>
---

scripts/isar-sstate | 78 ++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 74 insertions(+), 4 deletions(-)

diff --git a/scripts/isar-sstate b/scripts/isar-sstate
index b0d4033f..767661f6 100755
--- a/scripts/isar-sstate
+++ b/scripts/isar-sstate

@@ -22,7 +22,7 @@ sstate cache:
`SSTATE_DIR`. To share them, you need to explicitly upload them to
the shared location, which is what isar-sstate is for.

-isar-sstate implements six commands (upload, clean, info, show, analyze, lint),
+isar-sstate implements seven commands (upload, clean, info, show, delta, analyze, lint),

and supports three remote backends (filesystem, http/webdav, AWS S3).

## Commands

@@ -59,6 +59,13 @@ The argument `--verbose` increases the amount of information displayed.

The `show` command lists all individual artifacts for a recipe (`PN`)

in the remote cache, grouped by architecture, task name, `DISTRO`, and hash.

+### delta
+
+The `delta` command compares the signature of a specified hash against
+all other signatures in the cache matching the same architecture,
+recipe (`PN`), and task. Artifacts built for a different `DISTRO` are
+skipped, unless `--verbose` is given.

+
### analyze

The `analyze` command iterates over all artifacts in the local sstate cache,

@@ -630,11 +637,11 @@ def arguments():
parser = argparse.ArgumentParser()
parser.add_argument(

'command', type=str, metavar='command',

- choices='info upload clean show analyze lint'.split(),
- help="command to execute (info, upload, clean, show, analyze, lint)")
+ choices='info upload clean show delta analyze lint'.split(),
+ help="command to execute (info, upload, clean, show, delta, analyze, lint)")

parser.add_argument(
'source', type=str, nargs='?',

- help="local sstate dir (for uploads or analysis), or PN (for show)")
+ help="local sstate dir (for uploads or analysis), PN (for show), or hash (for delta)")

parser.add_argument(
'target', type=str,
help="remote sstate location (a file://, http://, or s3:// URI)")

@@ -678,6 +685,9 @@ def arguments():

elif args.command == 'show' and args.source is None:

print(f"ERROR: '{args.command}' needs a PN and target")

sys.exit(1)
+ elif args.command == 'delta' and args.source is None:
+ print(f"ERROR: '{args.command}' needs a hash and target")

+ sys.exit(1)
elif args.command in 'info clean'.split() and args.source is not None:
print(f"ERROR: '{args.command}' must not have a source (only a target)")
sys.exit(1)

@@ -864,6 +874,64 @@ def sstate_show(source, target, verbose, filter, arch, **kwargs):
return 0


+def sstate_delta(hash, target, verbose, filter, arch, **kwargs):
+ if not target.exists():
+ print(f"WARNING: {target} does not exist. Nothing to analyze.")
+ return 0
+
+ target.enable_cache()
+ sigs = {s.hash: s for s in target.list_all() if s.suffix.endswith('.siginfo')}
+
+ matches = [s for s in sigs.values() if s.hash.startswith(hash)]
+ if len(matches) == 0:
+ print(f"ERROR: hash '{hash}' not found in {target}")
+ return 1
+ if len(set(s.hash for s in matches)) > 1:
+ print(f"ERROR: hash prefix '{hash}' is ambiguous")
+ return 1
+ ref_sig = matches[0]
+
+ print(f"\033[1;33m==== checking item {ref_sig.arch}:{ref_sig.pn}:{ref_sig.task} ({ref_sig.hash[:8]}) ====\033[0m")
+ other_matches = apply_filters([
+ s for s in sigs.values()
+ if s.arch == ref_sig.arch and s.pn == ref_sig.pn and s.task == ref_sig.task and s.hash != ref_sig.hash
+ ], filter, arch)
+
+ ref_distro = None if verbose else get_distro(target, ref_sig.path)
+ if ref_distro:
+ other_matches = [s for s in other_matches
+ if get_distro(target, s.path) in (ref_distro, None)]
+
+ if len(other_matches) == 0:
+ print(" -> found no other matches for comparison")
+ return 0
+ print(f" -> found {len(other_matches)} potential matches")
+
+ def recursecb(key, hash_a, hash_b):
+ recout = []
+ if hash_a not in sigs or hash_b not in sigs:
+ recout.append(f"could not find signatures for job {key}")
+ return recout
+ out = compare_sigfiles(target.download(sigs[hash_a].path),
+ target.download(sigs[hash_b].path), recursecb, color=True)
+ for change in out:
+ recout.extend([' ' + line for line in change.splitlines()])
+ return recout
+
+ ref_file = target.download(ref_sig.path)
+ for t in sorted(other_matches, key=lambda x: (x.age, x.hash)):
+ age_str = str(datetime.timedelta(seconds=t.age))
+ print(f"\033[0;33m**** comparing to {t.hash[:8]} (age: {age_str}) ****\033[0m")
+ try:
+ out = compare_sigfiles(target.download(t.path), ref_file, recursecb, color=True)
+ except:
+ out = ["Failed to compare signatures."]
+ # shorten hashes from 64 to 8 characters for better readability
+ out = [re.sub(r'([0-9a-f]{8})[0-9a-f]{56}', r'\1', line) for line in out]
+ print('\n'.join(out))

+ return 0
+
+
def sstate_analyze(source, target, filter, arch, **kwargs):
if not os.path.isdir(source):
print(f"WARNING: source {source} does not exist. Nothing to analyze.")

@@ -1042,6 +1110,8 @@ def main():
return 1

args.target = target
+ if args.command == 'delta':
+ args.hash = args.source
return globals()[f'sstate_{args.command}'](**vars(args))


--
2.55.0