[meta-swupdate][PATCH] swupdate classes: fix hash mismatch on master
149 views
Skip to first unread message
Dan Walkes
Dec 1, 2024, 4:13:11 PM12/1/24
to swup...@googlegroups.com, Dan Walkes
The fix discussed in [1] resolves build issues on styhead but
introduces an issue which causes hash mismatches.
The first build succeeds to generate a working .swu, however
future builds fail with hash mismatch and a message like this:
```
[ERROR] : SWUPDATE failed [0] ERROR cpio_utils.c : __swupdate_copy : 670 : HASH mismatch : 5bb39965bfcedd87fc02bd51fac6636b924aed606d0984dcebae6a3250b01af6 <--> ff70a8dbc50a9ce2bad724554f5f14b8c933ad87fc544d054142755e8ef07f1e
[ERROR] : SWUPDATE failed [0] ERROR archive_handler.c : install_archive_image : 344 : Error copying extracted file
```
This happens because the modifications made to files in the
S directory only happen once. This means there's a single
conversion of `$swupdate_get_sha256(` to the file sha, and future
builds use the hash previously placed in the file in the S directory
which is no longer correct when the rootfs file hash changes.
It would probably be possible to make a simpler change here which
splits the UNPACKDIR and S and continues to make runtime
edits in S. However, it seems the intent of S and/or UNPACKDIR
is to contain unmodified source as discussed in [1].
This change instead:
1. Moves the sw-description file to WORKDIR for editing.
2. Continues to use S as source location for sw-description
and other source files which are not edited.
3. Makes all edits in WORKDIR.
4. When preparing the cpio, prepares from WORKDIR instead of
S. using any existing files in WORKDIR if they exist. If they
do not exist, symlinks the relevant files in S instead.
6. Adds the -L option to cpio to prevent adding symlinks to the
cpio file, and avoid the need to copy from S to WORKDIR when
preparing the cpio.
1: https://groups.google.com/g/swupdate/c/8K-9H7C9o5E/m/a9fqhIOjAAAJ
Signed-off-by: Dan Walkes <danw...@trellis-logic.com>
---
classes-recipe/swupdate-common.bbclass | 37 +++++++++++++++-----------
classes-recipe/swupdate-image.bbclass | 2 +-
2 files changed, 23 insertions(+), 16 deletions(-)
diff --git a/classes-recipe/swupdate-common.bbclass b/classes-recipe/swupdate-common.bbclass
index 7b49561..085a7b5 100644
--- a/classes-recipe/swupdate-common.bbclass
+++ b/classes-recipe/swupdate-common.bbclass
@@ -66,22 +66,22 @@ def swupdate_getdepends(d):
return depstr
-def swupdate_write_sha256(s):
+def swupdate_write_sha256(workdir):
import re
write_lines = []
- with open(os.path.join(s, "sw-description"), 'r') as f:
+ with open(os.path.join(workdir, "sw-description"), 'r') as f:
for line in f:
shastr = r"sha256.+=.+@(.+\")"
m = re.match(r"^(?P<before_placeholder>.+)(sha256|version).+[=:].*(?P<quote>[\'\"])@(?P<filename>.*)(?P=quote)", line)
if m:
filename = m.group('filename')
bb.warn("Syntax for sha256 changed, please use $swupdate_get_sha256(%s)" % filename)
- hash = swupdate_get_sha256(None, s, filename)
+ hash = swupdate_get_sha256(None, workdir, filename)
write_lines.append(line.replace("@%s" % (filename), hash))
else:
write_lines.append(line)
- with open(os.path.join(s, "sw-description"), 'w+') as f:
+ with open(os.path.join(workdir, "sw-description"), 'w+') as f:
for line in write_lines:
f.write(line)
@@ -97,7 +97,7 @@ def swupdate_exec_functions(d, s, write_lines):
write_lines[index] = line
-def swupdate_expand_bitbake_variables(d, s):
+def swupdate_expand_bitbake_variables(d, s, workdir):
write_lines = []
with open(os.path.join(s, "sw-description"), 'r') as f:
@@ -131,7 +131,7 @@ def swupdate_expand_bitbake_variables(d, s):
swupdate_exec_functions(d, s, write_lines)
- with open(os.path.join(s, "sw-description"), 'w+') as f:
+ with open(os.path.join(workdir, "sw-description"), 'w+') as f:
for line in write_lines:
f.write(line)
@@ -173,17 +173,18 @@ def prepare_sw_description(d):
import subprocess
s = d.getVar('S')
- swupdate_expand_bitbake_variables(d, s)
+ workdir = d.getVar('WORKDIR')
+ swupdate_expand_bitbake_variables(d, s, workdir)
- swupdate_write_sha256(s)
+ swupdate_write_sha256(workdir)
encrypt = d.getVar('SWUPDATE_ENCRYPT_SWDESC')
if encrypt:
bb.note("Encryption of sw-description")
- shutil.copyfile(os.path.join(s, 'sw-description'), os.path.join(s, 'sw-description.plain'))
+ shutil.copyfile(os.path.join(workdir, 'sw-description'), os.path.join(workdir, 'sw-description.plain'))
key,iv = swupdate_extract_keys(d.getVar('SWUPDATE_AES_FILE'))
- iv = swupdate_get_IV(d, s, 'sw-description')
- swupdate_encrypt_file(os.path.join(s, 'sw-description.plain'), os.path.join(s, 'sw-description'), key, iv)
+ iv = swupdate_get_IV(d, workdir, 'sw-description')
+ swupdate_encrypt_file(os.path.join(workdir, 'sw-description.plain'), os.path.join(workdir, 'sw-description'), key, iv)
signing = d.getVar('SWUPDATE_SIGNING')
if signing == "1":
@@ -191,8 +192,8 @@ def prepare_sw_description(d):
signing = "RSA"
if signing:
- sw_desc_sig = os.path.join(s, 'sw-description.sig')
- sw_desc = os.path.join(s, 'sw-description.plain' if encrypt else 'sw-description')
+ sw_desc_sig = os.path.join(workdir, 'sw-description.sig')
+ sw_desc = os.path.join(workdir, 'sw-description.plain' if encrypt else 'sw-description')
if signing == "CUSTOM":
signcmd = []
@@ -233,6 +234,7 @@ def swupdate_add_src_uri(d, list_for_cpio):
import shutil
s = d.getVar('S')
+ workdir = d.getVar('WORKDIR')
exclude = (d.getVar("SWUPDATE_SRC_URI_EXCLUDE") or "").split()
fetch = bb.fetch2.Fetch([], d)
@@ -310,9 +312,14 @@ def swupdate_add_artifacts(d, list_for_cpio):
def swupdate_create_cpio(d, swudeploydir, list_for_cpio):
s = d.getVar('S')
- os.chdir(s)
+ workdir = d.getVar('WORKDIR')
+ os.chdir(workdir)
+ for file in list_for_cpio:
+ if not os.path.exists(file):
+ os.symlink(os.path.join(s, file), file)
+
updateimage = d.getVar('IMAGE_NAME') + '.swu'
- line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible > ' + os.path.join(swudeploydir, updateimage)
+ line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible -L > ' + os.path.join(swudeploydir, updateimage)
os.system(line)
os.chdir(swudeploydir)
updateimage_link = d.getVar('IMAGE_LINK_NAME')
diff --git a/classes-recipe/swupdate-image.bbclass b/classes-recipe/swupdate-image.bbclass
index 1cd3eeb..bf8c5cc 100644
--- a/classes-recipe/swupdate-image.bbclass
+++ b/classes-recipe/swupdate-image.bbclass
@@ -40,7 +40,7 @@ python do_swupdate_copy_swdescription() {
import shutil
- workdir = d.getVar('S')
+ workdir = d.getVar('WORKDIR')
filespath = d.getVar('FILESPATH')
sw_desc_path = bb.utils.which(filespath, "sw-description")
shutil.copyfile(sw_desc_path, os.path.join(workdir, "sw-description"))
--
2.34.1
introduces an issue which causes hash mismatches.
The first build succeeds to generate a working .swu, however
future builds fail with hash mismatch and a message like this:
```
[ERROR] : SWUPDATE failed [0] ERROR cpio_utils.c : __swupdate_copy : 670 : HASH mismatch : 5bb39965bfcedd87fc02bd51fac6636b924aed606d0984dcebae6a3250b01af6 <--> ff70a8dbc50a9ce2bad724554f5f14b8c933ad87fc544d054142755e8ef07f1e
[ERROR] : SWUPDATE failed [0] ERROR archive_handler.c : install_archive_image : 344 : Error copying extracted file
```
This happens because the modifications made to files in the
S directory only happen once. This means there's a single
conversion of `$swupdate_get_sha256(` to the file sha, and future
builds use the hash previously placed in the file in the S directory
which is no longer correct when the rootfs file hash changes.
It would probably be possible to make a simpler change here which
splits the UNPACKDIR and S and continues to make runtime
edits in S. However, it seems the intent of S and/or UNPACKDIR
is to contain unmodified source as discussed in [1].
This change instead:
1. Moves the sw-description file to WORKDIR for editing.
2. Continues to use S as source location for sw-description
and other source files which are not edited.
3. Makes all edits in WORKDIR.
4. When preparing the cpio, prepares from WORKDIR instead of
S. using any existing files in WORKDIR if they exist. If they
do not exist, symlinks the relevant files in S instead.
6. Adds the -L option to cpio to prevent adding symlinks to the
cpio file, and avoid the need to copy from S to WORKDIR when
preparing the cpio.
1: https://groups.google.com/g/swupdate/c/8K-9H7C9o5E/m/a9fqhIOjAAAJ
Signed-off-by: Dan Walkes <danw...@trellis-logic.com>
---
classes-recipe/swupdate-common.bbclass | 37 +++++++++++++++-----------
classes-recipe/swupdate-image.bbclass | 2 +-
2 files changed, 23 insertions(+), 16 deletions(-)
diff --git a/classes-recipe/swupdate-common.bbclass b/classes-recipe/swupdate-common.bbclass
index 7b49561..085a7b5 100644
--- a/classes-recipe/swupdate-common.bbclass
+++ b/classes-recipe/swupdate-common.bbclass
@@ -66,22 +66,22 @@ def swupdate_getdepends(d):
return depstr
-def swupdate_write_sha256(s):
+def swupdate_write_sha256(workdir):
import re
write_lines = []
- with open(os.path.join(s, "sw-description"), 'r') as f:
+ with open(os.path.join(workdir, "sw-description"), 'r') as f:
for line in f:
shastr = r"sha256.+=.+@(.+\")"
m = re.match(r"^(?P<before_placeholder>.+)(sha256|version).+[=:].*(?P<quote>[\'\"])@(?P<filename>.*)(?P=quote)", line)
if m:
filename = m.group('filename')
bb.warn("Syntax for sha256 changed, please use $swupdate_get_sha256(%s)" % filename)
- hash = swupdate_get_sha256(None, s, filename)
+ hash = swupdate_get_sha256(None, workdir, filename)
write_lines.append(line.replace("@%s" % (filename), hash))
else:
write_lines.append(line)
- with open(os.path.join(s, "sw-description"), 'w+') as f:
+ with open(os.path.join(workdir, "sw-description"), 'w+') as f:
for line in write_lines:
f.write(line)
@@ -97,7 +97,7 @@ def swupdate_exec_functions(d, s, write_lines):
write_lines[index] = line
-def swupdate_expand_bitbake_variables(d, s):
+def swupdate_expand_bitbake_variables(d, s, workdir):
write_lines = []
with open(os.path.join(s, "sw-description"), 'r') as f:
@@ -131,7 +131,7 @@ def swupdate_expand_bitbake_variables(d, s):
swupdate_exec_functions(d, s, write_lines)
- with open(os.path.join(s, "sw-description"), 'w+') as f:
+ with open(os.path.join(workdir, "sw-description"), 'w+') as f:
for line in write_lines:
f.write(line)
@@ -173,17 +173,18 @@ def prepare_sw_description(d):
import subprocess
s = d.getVar('S')
- swupdate_expand_bitbake_variables(d, s)
+ workdir = d.getVar('WORKDIR')
+ swupdate_expand_bitbake_variables(d, s, workdir)
- swupdate_write_sha256(s)
+ swupdate_write_sha256(workdir)
encrypt = d.getVar('SWUPDATE_ENCRYPT_SWDESC')
if encrypt:
bb.note("Encryption of sw-description")
- shutil.copyfile(os.path.join(s, 'sw-description'), os.path.join(s, 'sw-description.plain'))
+ shutil.copyfile(os.path.join(workdir, 'sw-description'), os.path.join(workdir, 'sw-description.plain'))
key,iv = swupdate_extract_keys(d.getVar('SWUPDATE_AES_FILE'))
- iv = swupdate_get_IV(d, s, 'sw-description')
- swupdate_encrypt_file(os.path.join(s, 'sw-description.plain'), os.path.join(s, 'sw-description'), key, iv)
+ iv = swupdate_get_IV(d, workdir, 'sw-description')
+ swupdate_encrypt_file(os.path.join(workdir, 'sw-description.plain'), os.path.join(workdir, 'sw-description'), key, iv)
signing = d.getVar('SWUPDATE_SIGNING')
if signing == "1":
@@ -191,8 +192,8 @@ def prepare_sw_description(d):
signing = "RSA"
if signing:
- sw_desc_sig = os.path.join(s, 'sw-description.sig')
- sw_desc = os.path.join(s, 'sw-description.plain' if encrypt else 'sw-description')
+ sw_desc_sig = os.path.join(workdir, 'sw-description.sig')
+ sw_desc = os.path.join(workdir, 'sw-description.plain' if encrypt else 'sw-description')
if signing == "CUSTOM":
signcmd = []
@@ -233,6 +234,7 @@ def swupdate_add_src_uri(d, list_for_cpio):
import shutil
s = d.getVar('S')
+ workdir = d.getVar('WORKDIR')
exclude = (d.getVar("SWUPDATE_SRC_URI_EXCLUDE") or "").split()
fetch = bb.fetch2.Fetch([], d)
@@ -310,9 +312,14 @@ def swupdate_add_artifacts(d, list_for_cpio):
def swupdate_create_cpio(d, swudeploydir, list_for_cpio):
s = d.getVar('S')
- os.chdir(s)
+ workdir = d.getVar('WORKDIR')
+ os.chdir(workdir)
+ for file in list_for_cpio:
+ if not os.path.exists(file):
+ os.symlink(os.path.join(s, file), file)
+
updateimage = d.getVar('IMAGE_NAME') + '.swu'
- line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible > ' + os.path.join(swudeploydir, updateimage)
+ line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible -L > ' + os.path.join(swudeploydir, updateimage)
os.system(line)
os.chdir(swudeploydir)
updateimage_link = d.getVar('IMAGE_LINK_NAME')
diff --git a/classes-recipe/swupdate-image.bbclass b/classes-recipe/swupdate-image.bbclass
index 1cd3eeb..bf8c5cc 100644
--- a/classes-recipe/swupdate-image.bbclass
+++ b/classes-recipe/swupdate-image.bbclass
@@ -40,7 +40,7 @@ python do_swupdate_copy_swdescription() {
import shutil
- workdir = d.getVar('S')
+ workdir = d.getVar('WORKDIR')
filespath = d.getVar('FILESPATH')
sw_desc_path = bb.utils.which(filespath, "sw-description")
shutil.copyfile(sw_desc_path, os.path.join(workdir, "sw-description"))
--
2.34.1
Dan Walkes
Dec 1, 2024, 4:17:08 PM12/1/24
to swupdate
Sending this for review and comment.
Khem I think as discussed at https://groups.google.com/g/swupdate/c/8K-9H7C9o5E/m/a9fqhIOjAAAJ the correct fix here is more complicated.
These changes work for me on https://github.com/OE4T/tegra-demo-distro/tree/master/layers/meta-tegrademo/dynamic-layers/meta-swupdate but I could use some help testing the encryption/signing setup since I don't have a demo setup for this.
Also if you prefer attempting to rollback and split S and UNPACKDIR for a simpler version which still modifies S let me know and I can try that instead.
Dan Walkes
Jul 16, 2025, 5:14:24 PMJul 16
to swupdate
I've verified this is still an issue with latest master, and have updated the patch at https://github.com/Trellis-Logic/meta-swupdate/tree/fix-hash-issues and https://github.com/Trellis-Logic/meta-swupdate/commit/f829424cd38a1f549de050bffcc26434799e28ff with more details about how to reproduce and about the underlying issue causing it and also to rebase on the latest master.
The biggest new realization is that this only happens when `rm_work` is not enabled. If you use `rm_work` this effectively is a workaround for this issue, at least as far as I've tested. This likely explains why I appear to be the only one seeing this, as the underlying demo distro at https://github.com/OE4T/tegra-demo-distro does not use `rm_work` and this is how I've been testing it.
I think the patch I have is possibly not the simplest way to solve the problem and I'm willing to implement in a different way if anyone has suggestions.
If everyone else is content to just use rm_work as a workaround I'll just keep this in my local branch too, and perhaps I can change the defaults on tegra-demo-distro to use rm_work so we won't hit this either using upstream master.
If you'd like me to resend the current patch rebased on the latest master to the mailing list I'll do that too. The only change is the description, however.
Dan
Stefano Babic
Jul 17, 2025, 11:15:00 AMJul 17
to Dan Walkes, swupdate
Hi Dan,
On 7/16/25 23:14, Dan Walkes wrote:
> I've verified this is still an issue with latest master, and have
> updated the patch at https://github.com/Trellis-Logic/meta-swupdate/
> tree/fix-hash-issues and https://github.com/Trellis-Logic/meta-swupdate/
> commit/f829424cd38a1f549de050bffcc26434799e28ff with more details about
> how to reproduce and about the underlying issue causing it and also to
> rebase on the latest master.
Does it mean you have a new patch to be posted here for review or is the
posted patch enough ?
I will first ask something - the issue you report seems due to some
missing dependencies, and the task do_swupdate does not run again if you
change rootfs. I am quite missing the role of rm_work, because it
shouldn't play a role if dependency is correct.
You are mentioning that changes in rootfs does not reflect into the SWU.
Is just sw-description affected in your test ?
Surely, sw-description must be regenerated if one of the artifacts changed.
>
> The biggest new realization is that this only happens when `rm_work` is
> not enabled.
Well, I have not understood why this makes a difference. Can you explain
this ?
> If you use `rm_work` this effectively is a workaround for
> this issue, at least as far as I've tested. This likely explains why I
> appear to be the only one seeing this, as the underlying demo distro
> at https://github.com/OE4T/tegra-demo-distro does not use `rm_work` and
> this is how I've been testing it.
> > I think the patch I have is possibly not the simplest way to solve the
> problem and I'm willing to implement in a different way if anyone has
> suggestions.
>
> If everyone else is content to just use rm_work as a workaround I'll
> just keep this in my local branch too, and perhaps I can change the
> defaults on tegra-demo-distro to use rm_work so we won't hit this either
> using upstream master.
>
> If you'd like me to resend the current patch rebased on the latest
> master to the mailing list I'll do that too. The only change is the
> description, however.
Ok, the just wait. I review the original patch, and you can resend then
after comments.
Best regards,
Stefano Babic
>
> Dan
> On Sunday, December 1, 2024 at 2:17:08 PM UTC-7 Dan Walkes wrote:
>
> Sending this for review and comment.
>
> Khem I think as discussed at https://groups.google.com/g/swupdate/
> c/8K-9H7C9o5E/m/a9fqhIOjAAAJ <https://groups.google.com/g/swupdate/
> c/8K-9H7C9o5E/m/a9fqhIOjAAAJ> the correct fix here is more complicated.
> layers/meta-tegrademo/dynamic-layers/meta-swupdate> but I could use
> a9fqhIOjAAAJ <https://groups.google.com/g/swupdate/
> c/8K-9H7C9o5E/m/a9fqhIOjAAAJ>
> --
> You received this message because you are subscribed to the Google
> Groups "swupdate" group.
> To unsubscribe from this group and stop receiving emails from it, send
> an email to swupdate+u...@googlegroups.com
> <mailto:swupdate+u...@googlegroups.com>.
> To view this discussion visit https://groups.google.com/d/msgid/
> swupdate/1cf15b67-2723-4c0f-8e66-8d2d32af1ec9n%40googlegroups.com
> <https://groups.google.com/d/msgid/
> swupdate/1cf15b67-2723-4c0f-8e66-8d2d32af1ec9n%40googlegroups.com?
> utm_medium=email&utm_source=footer>.
On 7/16/25 23:14, Dan Walkes wrote:
> I've verified this is still an issue with latest master, and have
> updated the patch at https://github.com/Trellis-Logic/meta-swupdate/
> tree/fix-hash-issues and https://github.com/Trellis-Logic/meta-swupdate/
> commit/f829424cd38a1f549de050bffcc26434799e28ff with more details about
> how to reproduce and about the underlying issue causing it and also to
> rebase on the latest master.
posted patch enough ?
I will first ask something - the issue you report seems due to some
missing dependencies, and the task do_swupdate does not run again if you
change rootfs. I am quite missing the role of rm_work, because it
shouldn't play a role if dependency is correct.
You are mentioning that changes in rootfs does not reflect into the SWU.
Is just sw-description affected in your test ?
Surely, sw-description must be regenerated if one of the artifacts changed.
>
> The biggest new realization is that this only happens when `rm_work` is
> not enabled.
this ?
> If you use `rm_work` this effectively is a workaround for
> this issue, at least as far as I've tested. This likely explains why I
> appear to be the only one seeing this, as the underlying demo distro
> at https://github.com/OE4T/tegra-demo-distro does not use `rm_work` and
> this is how I've been testing it.
> > I think the patch I have is possibly not the simplest way to solve the
> problem and I'm willing to implement in a different way if anyone has
> suggestions.
>
> If everyone else is content to just use rm_work as a workaround I'll
> just keep this in my local branch too, and perhaps I can change the
> defaults on tegra-demo-distro to use rm_work so we won't hit this either
> using upstream master.
>
> If you'd like me to resend the current patch rebased on the latest
> master to the mailing list I'll do that too. The only change is the
> description, however.
after comments.
Best regards,
Stefano Babic
>
> Dan
> On Sunday, December 1, 2024 at 2:17:08 PM UTC-7 Dan Walkes wrote:
>
> Sending this for review and comment.
>
> Khem I think as discussed at https://groups.google.com/g/swupdate/
> c/8K-9H7C9o5E/m/a9fqhIOjAAAJ> the correct fix here is more complicated.
>
> These changes work for me on https://github.com/OE4T/tegra-demo-
> distro/tree/master/layers/meta-tegrademo/dynamic-layers/meta-
> swupdate <https://github.com/OE4T/tegra-demo-distro/tree/master/
> These changes work for me on https://github.com/OE4T/tegra-demo-
> distro/tree/master/layers/meta-tegrademo/dynamic-layers/meta-
> layers/meta-tegrademo/dynamic-layers/meta-swupdate> but I could use
> c/8K-9H7C9o5E/m/a9fqhIOjAAAJ>
> You received this message because you are subscribed to the Google
> Groups "swupdate" group.
> To unsubscribe from this group and stop receiving emails from it, send
> an email to swupdate+u...@googlegroups.com
> <mailto:swupdate+u...@googlegroups.com>.
> To view this discussion visit https://groups.google.com/d/msgid/
> swupdate/1cf15b67-2723-4c0f-8e66-8d2d32af1ec9n%40googlegroups.com
> <https://groups.google.com/d/msgid/
> swupdate/1cf15b67-2723-4c0f-8e66-8d2d32af1ec9n%40googlegroups.com?
> utm_medium=email&utm_source=footer>.
Stefano Babic
Jul 17, 2025, 11:31:04 AMJul 17
to Dan Walkes, swup...@googlegroups.com
Hi Dan,
On 12/1/24 22:13, Dan Walkes wrote:
> The fix discussed in [1] resolves build issues on styhead but
> introduces an issue which causes hash mismatches.
>
> The first build succeeds to generate a working .swu, however
> future builds fail with hash mismatch and a message like this:
>
> ```
> [ERROR] : SWUPDATE failed [0] ERROR cpio_utils.c : __swupdate_copy : 670 : HASH mismatch : 5bb39965bfcedd87fc02bd51fac6636b924aed606d0984dcebae6a3250b01af6 <--> ff70a8dbc50a9ce2bad724554f5f14b8c933ad87fc544d054142755e8ef07f1e
> [ERROR] : SWUPDATE failed [0] ERROR archive_handler.c : install_archive_image : 344 : Error copying extracted file
> ```
>
> This happens because the modifications made to files in the
> S directory only happen once. This means there's a single
> conversion of `$swupdate_get_sha256(` to the file sha, and future
> builds use the hash previously placed in the file in the S directory
> which is no longer correct when the rootfs file hash changes.
>
> It would probably be possible to make a simpler change here which
> splits the UNPACKDIR and S and continues to make runtime
> edits in S. However, it seems the intent of S and/or UNPACKDIR
> is to contain unmodified source as discussed in [1].
In principle it is ok to move from $S to $WORKDIR - until scarthgap, $S
was $WORKDIR.
swupdate_add_artifacts() already copies files from DEPLOY_DIR (listed in
SWUPDATE_IMAGES) into a work directory (it was $S). Is it not enough by
copying them into $WORKDIR then without links ?
"s" here is just the parameter of the function. What it counts, it is
how the function is called. Do we really need to change it ?
ok - so we have original sw-description in $S, generated sw-description
in $WORKDIR
So we are talking here about files in SRC_URI, right ? Images listed are
also searched in $DEPLOY_DIR_IMAGE. They are worked in
swupdate_add_src_uri(), and they are copied into WORKDIR by changing
"dst" in that function.
> +
> updateimage = d.getVar('IMAGE_NAME') + '.swu'
> - line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible > ' + os.path.join(swudeploydir, updateimage)
> + line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible -L > ' + os.path.join(swudeploydir, updateimage)
> os.system(line)
> os.chdir(swudeploydir)
> updateimage_link = d.getVar('IMAGE_LINK_NAME')
> diff --git a/classes-recipe/swupdate-image.bbclass b/classes-recipe/swupdate-image.bbclass
> index 1cd3eeb..bf8c5cc 100644
> --- a/classes-recipe/swupdate-image.bbclass
> +++ b/classes-recipe/swupdate-image.bbclass
> @@ -40,7 +40,7 @@ python do_swupdate_copy_swdescription() {
>
> import shutil
>
> - workdir = d.getVar('S')
> + workdir = d.getVar('WORKDIR')
> filespath = d.getVar('FILESPATH')
> sw_desc_path = bb.utils.which(filespath, "sw-description")
> shutil.copyfile(sw_desc_path, os.path.join(workdir, "sw-description"))
Best regards,
Stefano
On 12/1/24 22:13, Dan Walkes wrote:
> The fix discussed in [1] resolves build issues on styhead but
> introduces an issue which causes hash mismatches.
>
> The first build succeeds to generate a working .swu, however
> future builds fail with hash mismatch and a message like this:
>
> ```
> [ERROR] : SWUPDATE failed [0] ERROR cpio_utils.c : __swupdate_copy : 670 : HASH mismatch : 5bb39965bfcedd87fc02bd51fac6636b924aed606d0984dcebae6a3250b01af6 <--> ff70a8dbc50a9ce2bad724554f5f14b8c933ad87fc544d054142755e8ef07f1e
> [ERROR] : SWUPDATE failed [0] ERROR archive_handler.c : install_archive_image : 344 : Error copying extracted file
> ```
>
> This happens because the modifications made to files in the
> S directory only happen once. This means there's a single
> conversion of `$swupdate_get_sha256(` to the file sha, and future
> builds use the hash previously placed in the file in the S directory
> which is no longer correct when the rootfs file hash changes.
>
> It would probably be possible to make a simpler change here which
> splits the UNPACKDIR and S and continues to make runtime
> edits in S. However, it seems the intent of S and/or UNPACKDIR
> is to contain unmodified source as discussed in [1].
was $WORKDIR.
swupdate_add_artifacts() already copies files from DEPLOY_DIR (listed in
SWUPDATE_IMAGES) into a work directory (it was $S). Is it not enough by
copying them into $WORKDIR then without links ?
how the function is called. Do we really need to change it ?
in $WORKDIR
also searched in $DEPLOY_DIR_IMAGE. They are worked in
swupdate_add_src_uri(), and they are copied into WORKDIR by changing
"dst" in that function.
> +
> updateimage = d.getVar('IMAGE_NAME') + '.swu'
> - line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible > ' + os.path.join(swudeploydir, updateimage)
> + line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible -L > ' + os.path.join(swudeploydir, updateimage)
> os.system(line)
> os.chdir(swudeploydir)
> updateimage_link = d.getVar('IMAGE_LINK_NAME')
> diff --git a/classes-recipe/swupdate-image.bbclass b/classes-recipe/swupdate-image.bbclass
> index 1cd3eeb..bf8c5cc 100644
> --- a/classes-recipe/swupdate-image.bbclass
> +++ b/classes-recipe/swupdate-image.bbclass
> @@ -40,7 +40,7 @@ python do_swupdate_copy_swdescription() {
>
> import shutil
>
> - workdir = d.getVar('S')
> + workdir = d.getVar('WORKDIR')
> filespath = d.getVar('FILESPATH')
> sw_desc_path = bb.utils.which(filespath, "sw-description")
> shutil.copyfile(sw_desc_path, os.path.join(workdir, "sw-description"))
Stefano
Dan Walkes
Jul 17, 2025, 12:21:47 PMJul 17
to Stefano Babic, swupdate
Hi Stefano,
Thanks for the response.
On Thu, Jul 17, 2025 at 9:14 AM Stefano Babic
<stefan...@swupdate.org> wrote:
>
> Hi Dan,
>
> On 7/16/25 23:14, Dan Walkes wrote:
> > I've verified this is still an issue with latest master, and have
> > updated the patch at https://github.com/Trellis-Logic/meta-swupdate/
> > tree/fix-hash-issues and https://github.com/Trellis-Logic/meta-swupdate/
> > commit/f829424cd38a1f549de050bffcc26434799e28ff with more details about
> > how to reproduce and about the underlying issue causing it and also to
> > rebase on the latest master.
>
> Does it mean you have a new patch to be posted here for review or is the
> posted patch enough ?
The posted patch is enough, just rebased on the latest master. I just
added more detail to the description.
>
> I will first ask something - the issue you report seems due to some
> missing dependencies, and the task do_swupdate does not run again if you
> change rootfs. I am quite missing the role of rm_work, because it
> shouldn't play a role if dependency is correct.
The task do_swupdate does run, and the sw-description file is rebuilt
(timestamp changes). It's just rebuilt from the copy used in the
previous build, which has overwritten the `$swupdate_get_sha256(` with
the actual file sha. Therefore the logic at
https://github.com/sbabic/meta-swupdate/blob/cbd0ed50a61dd7cbad09740d5d0edd295e56f16a/classes-recipe/swupdate-common.bbclass#L72-L74
does not recognize the sha field and does not overwrite the sha from
previous build with the new sha.
If rm_work is used, then the sw-description file is removed from the S
directory after the build. The next build generates the
sw-description from the copy provided in the SRC_URI which includes
the `$swupdate_get_sha256(` placeholders, and everything works.
>
> You are mentioning that changes in rootfs does not reflect into the SWU.
> Is just sw-description affected in your test ?
It's just the sha field in the sw-description I've noticed this
problem with, but I think any variable expansion could have the same
issue, if the variable changes without re-initializing the
sw-description file from the SRC_URI and instead using a copy where
variable expansion has already been performed as a part of
>
> Surely, sw-description must be regenerated if one of the artifacts changed.
It's definitely regenerated in that the timestamp changes and the
logic in swupdate_expand_bitbake_variables and swupdate_write_sha256
is run. However both functions run on a sw-description where variable
placeholders have already been replaced in a previous run. The reason
is that the sw-description is not replaced in the S directory with the
copy in SRC_URI unless rm_work is used, at least in my
testing/configuration.
Maybe one other way to solve this would be for prepare_sw_description
to overwrite the sw-description in S with the copy from SRC_URI. It
looks like there's some logic today which specifically avoids this for
sw-description at
https://github.com/sbabic/meta-swupdate/blob/cbd0ed50a61dd7cbad09740d5d0edd295e56f16a/classes-recipe/swupdate-common.bbclass#L256
which went in at
https://github.com/sbabic/meta-swupdate/commit/cfb7ca5b3dc37f2889205bd79757f0c0b4410ef4
however.
>
> >
> > The biggest new realization is that this only happens when `rm_work` is
> > not enabled.
>
> Well, I have not understood why this makes a difference. Can you explain
> this ?
I think I have above but let me know if this is unclear. Also if
there's a public example you'd like me to demonstrate this on, let me
know. I think I could try this with raspberry pi on
https://github.com/sbabic/meta-swupdate-boards.
I expect you'll be able to reproduce this in any build setup based on
master if you:
1. Turn off rm_work on your builds, or for at least your swu image recipe.
2. Build the swu target at least once.
3. Make some modification to the rootfs.
4. Build the swu target again.
5. Run the swupdate on target.
At step 4 if you look at the $S directory in the build/tmp/work subdir
you'll see the timestamp of the sw-description file is updated but the
hash has not changed and does not match the hash of the updated rootfs
file, but instead contains the has for the rootfs built in step 2.
>
> > If you use `rm_work` this effectively is a workaround for
> > this issue, at least as far as I've tested. This likely explains why I
> > appear to be the only one seeing this, as the underlying demo distro
> > at https://github.com/OE4T/tegra-demo-distro does not use `rm_work` and
> > this is how I've been testing it.
> > > I think the patch I have is possibly not the simplest way to solve the
> > problem and I'm willing to implement in a different way if anyone has
> > suggestions.
> >
> > If everyone else is content to just use rm_work as a workaround I'll
> > just keep this in my local branch too, and perhaps I can change the
> > defaults on tegra-demo-distro to use rm_work so we won't hit this either
> > using upstream master.
> >
> > If you'd like me to resend the current patch rebased on the latest
> > master to the mailing list I'll do that too. The only change is the
> > description, however.
>
> Ok, the just wait. I review the original patch, and you can resend then
> after comments.
>
OK I'll respond to the review now, thank you.
--
Dan Walkes
Trellis-Logic
cell: 720-239-2553
Thanks for the response.
On Thu, Jul 17, 2025 at 9:14 AM Stefano Babic
<stefan...@swupdate.org> wrote:
>
> Hi Dan,
>
> On 7/16/25 23:14, Dan Walkes wrote:
> > I've verified this is still an issue with latest master, and have
> > updated the patch at https://github.com/Trellis-Logic/meta-swupdate/
> > tree/fix-hash-issues and https://github.com/Trellis-Logic/meta-swupdate/
> > commit/f829424cd38a1f549de050bffcc26434799e28ff with more details about
> > how to reproduce and about the underlying issue causing it and also to
> > rebase on the latest master.
>
> Does it mean you have a new patch to be posted here for review or is the
> posted patch enough ?
added more detail to the description.
>
> I will first ask something - the issue you report seems due to some
> missing dependencies, and the task do_swupdate does not run again if you
> change rootfs. I am quite missing the role of rm_work, because it
> shouldn't play a role if dependency is correct.
(timestamp changes). It's just rebuilt from the copy used in the
previous build, which has overwritten the `$swupdate_get_sha256(` with
the actual file sha. Therefore the logic at
https://github.com/sbabic/meta-swupdate/blob/cbd0ed50a61dd7cbad09740d5d0edd295e56f16a/classes-recipe/swupdate-common.bbclass#L72-L74
does not recognize the sha field and does not overwrite the sha from
previous build with the new sha.
If rm_work is used, then the sw-description file is removed from the S
directory after the build. The next build generates the
sw-description from the copy provided in the SRC_URI which includes
the `$swupdate_get_sha256(` placeholders, and everything works.
>
> You are mentioning that changes in rootfs does not reflect into the SWU.
> Is just sw-description affected in your test ?
problem with, but I think any variable expansion could have the same
issue, if the variable changes without re-initializing the
sw-description file from the SRC_URI and instead using a copy where
variable expansion has already been performed as a part of
>
> Surely, sw-description must be regenerated if one of the artifacts changed.
logic in swupdate_expand_bitbake_variables and swupdate_write_sha256
is run. However both functions run on a sw-description where variable
placeholders have already been replaced in a previous run. The reason
is that the sw-description is not replaced in the S directory with the
copy in SRC_URI unless rm_work is used, at least in my
testing/configuration.
Maybe one other way to solve this would be for prepare_sw_description
to overwrite the sw-description in S with the copy from SRC_URI. It
looks like there's some logic today which specifically avoids this for
sw-description at
https://github.com/sbabic/meta-swupdate/blob/cbd0ed50a61dd7cbad09740d5d0edd295e56f16a/classes-recipe/swupdate-common.bbclass#L256
which went in at
https://github.com/sbabic/meta-swupdate/commit/cfb7ca5b3dc37f2889205bd79757f0c0b4410ef4
however.
>
> >
> > The biggest new realization is that this only happens when `rm_work` is
> > not enabled.
>
> Well, I have not understood why this makes a difference. Can you explain
> this ?
there's a public example you'd like me to demonstrate this on, let me
know. I think I could try this with raspberry pi on
https://github.com/sbabic/meta-swupdate-boards.
I expect you'll be able to reproduce this in any build setup based on
master if you:
1. Turn off rm_work on your builds, or for at least your swu image recipe.
2. Build the swu target at least once.
3. Make some modification to the rootfs.
4. Build the swu target again.
5. Run the swupdate on target.
At step 4 if you look at the $S directory in the build/tmp/work subdir
you'll see the timestamp of the sw-description file is updated but the
hash has not changed and does not match the hash of the updated rootfs
file, but instead contains the has for the rootfs built in step 2.
>
> > If you use `rm_work` this effectively is a workaround for
> > this issue, at least as far as I've tested. This likely explains why I
> > appear to be the only one seeing this, as the underlying demo distro
> > at https://github.com/OE4T/tegra-demo-distro does not use `rm_work` and
> > this is how I've been testing it.
> > > I think the patch I have is possibly not the simplest way to solve the
> > problem and I'm willing to implement in a different way if anyone has
> > suggestions.
> >
> > If everyone else is content to just use rm_work as a workaround I'll
> > just keep this in my local branch too, and perhaps I can change the
> > defaults on tegra-demo-distro to use rm_work so we won't hit this either
> > using upstream master.
> >
> > If you'd like me to resend the current patch rebased on the latest
> > master to the mailing list I'll do that too. The only change is the
> > description, however.
>
> Ok, the just wait. I review the original patch, and you can resend then
> after comments.
>
Dan Walkes
Trellis-Logic
cell: 720-239-2553
Dan Walkes
Jul 17, 2025, 12:35:04 PMJul 17
to Stefano Babic, swup...@googlegroups.com
Hi Stefano,
Thanks for the review.
On Thu, Jul 17, 2025 at 9:31 AM Stefano Babic
<stefan...@swupdate.org> wrote:
>
> Hi Dan,
>
> On 12/1/24 22:13, Dan Walkes wrote:
> > The fix discussed in [1] resolves build issues on styhead but
> > introduces an issue which causes hash mismatches.
> >
> > The first build succeeds to generate a working .swu, however
> > future builds fail with hash mismatch and a message like this:
> >
> > ```
> > [ERROR] : SWUPDATE failed [0] ERROR cpio_utils.c : __swupdate_copy : 670 : HASH mismatch : 5bb39965bfcedd87fc02bd51fac6636b924aed606d0984dcebae6a3250b01af6 <--> ff70a8dbc50a9ce2bad724554f5f14b8c933ad87fc544d054142755e8ef07f1e
> > [ERROR] : SWUPDATE failed [0] ERROR archive_handler.c : install_archive_image : 344 : Error copying extracted file
> > ```
> >
> > This happens because the modifications made to files in the
> > S directory only happen once. This means there's a single
> > conversion of `$swupdate_get_sha256(` to the file sha, and future
> > builds use the hash previously placed in the file in the S directory
> > which is no longer correct when the rootfs file hash changes.
> >
> > It would probably be possible to make a simpler change here which
> > splits the UNPACKDIR and S and continues to make runtime
> > edits in S. However, it seems the intent of S and/or UNPACKDIR
> > is to contain unmodified source as discussed in [1].
>
> In principle it is ok to move from $S to $WORKDIR - until scarthgap, $S
> was $WORKDIR.
>
> swupdate_add_artifacts() already copies files from DEPLOY_DIR (listed in
> SWUPDATE_IMAGES) into a work directory (it was $S). Is it not enough by
> copying them into $WORKDIR then without links ?
I hadn't considered that approach but that might be easier, and I
think it will work. We shouldn't need symlinks in WORKDIR if we have
the content there, and we don't need content in both S and WORKDIR. I
think we'd need a similar change in swupdate_add_src_uri to copy to
WORKDIR or we'd need symlinks for those files as well.
The important thing is that we only write the sw-description file in
WORKDIR and not in UNPACKDIR/S, so we can keep a version of
sw-description without modification in UNPACKDIR/S. This is
accomplished with the current code by writing to WORKDIR but reading
from S in swupdate_expand_bitbake_variables.
Yes.
Symlinks will exist for all content including swupdate_add_artifacts
and swupdate_add_src_uri with this patch, since both still only add to
S as implemented. I could experiment with doing away with the symlinks
by copying content from these functions into WORKDIR instead if you
prefer.
> > +
> > updateimage = d.getVar('IMAGE_NAME') + '.swu'
> > - line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible > ' + os.path.join(swudeploydir, updateimage)
> > + line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible -L > ' + os.path.join(swudeploydir, updateimage)
> > os.system(line)
> > os.chdir(swudeploydir)
> > updateimage_link = d.getVar('IMAGE_LINK_NAME')
> > diff --git a/classes-recipe/swupdate-image.bbclass b/classes-recipe/swupdate-image.bbclass
> > index 1cd3eeb..bf8c5cc 100644
> > --- a/classes-recipe/swupdate-image.bbclass
> > +++ b/classes-recipe/swupdate-image.bbclass
> > @@ -40,7 +40,7 @@ python do_swupdate_copy_swdescription() {
> >
> > import shutil
> >
> > - workdir = d.getVar('S')
> > + workdir = d.getVar('WORKDIR')
> > filespath = d.getVar('FILESPATH')
> > sw_desc_path = bb.utils.which(filespath, "sw-description")
> > shutil.copyfile(sw_desc_path, os.path.join(workdir, "sw-description"))
>
> Best regards,
> Stefano
>
Thanks
Dan
Thanks for the review.
On Thu, Jul 17, 2025 at 9:31 AM Stefano Babic
<stefan...@swupdate.org> wrote:
>
> Hi Dan,
>
> On 12/1/24 22:13, Dan Walkes wrote:
> > The fix discussed in [1] resolves build issues on styhead but
> > introduces an issue which causes hash mismatches.
> >
> > The first build succeeds to generate a working .swu, however
> > future builds fail with hash mismatch and a message like this:
> >
> > ```
> > [ERROR] : SWUPDATE failed [0] ERROR cpio_utils.c : __swupdate_copy : 670 : HASH mismatch : 5bb39965bfcedd87fc02bd51fac6636b924aed606d0984dcebae6a3250b01af6 <--> ff70a8dbc50a9ce2bad724554f5f14b8c933ad87fc544d054142755e8ef07f1e
> > [ERROR] : SWUPDATE failed [0] ERROR archive_handler.c : install_archive_image : 344 : Error copying extracted file
> > ```
> >
> > This happens because the modifications made to files in the
> > S directory only happen once. This means there's a single
> > conversion of `$swupdate_get_sha256(` to the file sha, and future
> > builds use the hash previously placed in the file in the S directory
> > which is no longer correct when the rootfs file hash changes.
> >
> > It would probably be possible to make a simpler change here which
> > splits the UNPACKDIR and S and continues to make runtime
> > edits in S. However, it seems the intent of S and/or UNPACKDIR
> > is to contain unmodified source as discussed in [1].
>
> In principle it is ok to move from $S to $WORKDIR - until scarthgap, $S
> was $WORKDIR.
>
> swupdate_add_artifacts() already copies files from DEPLOY_DIR (listed in
> SWUPDATE_IMAGES) into a work directory (it was $S). Is it not enough by
> copying them into $WORKDIR then without links ?
think it will work. We shouldn't need symlinks in WORKDIR if we have
the content there, and we don't need content in both S and WORKDIR. I
think we'd need a similar change in swupdate_add_src_uri to copy to
WORKDIR or we'd need symlinks for those files as well.
WORKDIR and not in UNPACKDIR/S, so we can keep a version of
sw-description without modification in UNPACKDIR/S. This is
accomplished with the current code by writing to WORKDIR but reading
from S in swupdate_expand_bitbake_variables.
and swupdate_add_src_uri with this patch, since both still only add to
S as implemented. I could experiment with doing away with the symlinks
by copying content from these functions into WORKDIR instead if you
prefer.
> > +
> > updateimage = d.getVar('IMAGE_NAME') + '.swu'
> > - line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible > ' + os.path.join(swudeploydir, updateimage)
> > + line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible -L > ' + os.path.join(swudeploydir, updateimage)
> > os.system(line)
> > os.chdir(swudeploydir)
> > updateimage_link = d.getVar('IMAGE_LINK_NAME')
> > diff --git a/classes-recipe/swupdate-image.bbclass b/classes-recipe/swupdate-image.bbclass
> > index 1cd3eeb..bf8c5cc 100644
> > --- a/classes-recipe/swupdate-image.bbclass
> > +++ b/classes-recipe/swupdate-image.bbclass
> > @@ -40,7 +40,7 @@ python do_swupdate_copy_swdescription() {
> >
> > import shutil
> >
> > - workdir = d.getVar('S')
> > + workdir = d.getVar('WORKDIR')
> > filespath = d.getVar('FILESPATH')
> > sw_desc_path = bb.utils.which(filespath, "sw-description")
> > shutil.copyfile(sw_desc_path, os.path.join(workdir, "sw-description"))
>
> Best regards,
> Stefano
>
Dan
Stefano Babic
Jul 17, 2025, 4:11:57 PMJul 17
to Dan Walkes, Stefano Babic, swup...@googlegroups.com
Hi Dan,
I think so.
Right, full agree.
> so we can keep a version of
> sw-description without modification in UNPACKDIR/S. This is
> accomplished with the current code by writing to WORKDIR but reading
> from S in swupdate_expand_bitbake_variables.
Right
Yes, please do it.
> so we can keep a version of
> sw-description without modification in UNPACKDIR/S. This is
> accomplished with the current code by writing to WORKDIR but reading
> from S in swupdate_expand_bitbake_variables.
Dan Walkes
Jul 18, 2025, 4:58:40 PMJul 18
to swup...@googlegroups.com, Dan Walkes
Change from V1 at [1]:
* Resolves issues mentioned in code review there
The fix discussed in [2] resolves build issues on styhead but
introduces an issue which causes hash mismatches when building
without `rm_work`.
All builds succeed in generate a working .swu, however any build
which modifies the rootfs will fail to deploy.
On attempt to deploy, a hash mismatch will be noted like this:
```
[TRACE] : SWUPDATE running : [install_archive_image] : Installing file demo-image-base-jetson-orin-nano-devkit-nvme.rootfs.tar.gz on /tmp/datadst/Pfw8nm//, preserving attributes
[WARN ] : SWUPDATE running : [extract] : archive_read_next_header(): Pathname can't be converted from UTF-8 to current locale. for './etc/ssl/certs/NetLock_Arany_=Class_Gold=_Főtanúsítvány.pem': Invalid or incomplete multibyte or wide character
[WARN ] : SWUPDATE running : [extract] : archive_read_next_header(): Pathname can't be converted from UTF-8 to current locale. for './usr/share/ca-certificates/mozilla/NetLock_Arany_=Class_Gold=_Főtanúsítvány.crt': Invalid or incomplete multibyte or wide character
[ERROR] : SWUPDATE failed [0] ERROR cpio_utils.c : hash_compare : 465 : HASH mismatch : 0f7591f6ffa4aa8e1d919d774974f06d42f184ef5a552f9887201cf3649414da <--> 2497e5b3eaec665ec2ec7db4bb1025d2c92e38c9d8fc22d22aa5e71314f15e6f
[ERROR] : SWUPDATE failed [0] ERROR stream_interface.c : extract_files : 319 : Error streaming demo-image-base-jetson-orin-nano-devkit-nvme.rootfs.tar.gz
swupdate_image_write failed: Connection reset by peer
[ERROR] : SWUPDATE failed [1] Image invalid or corrupted. Not installing ...
[ERROR] : SWUPDATE failed [0] ERROR install_from_file.c : endupdate : 55 : SWUpdate *failed* !
```
This happens because the sha256 modifications made to files in the
S directory only happen once before variables are expanded
unless `rm_work` is used to remove the source dir on each build.
So anyone using `rm_work` for their builds would not see this issue.
The previous logic means there's a single expansion of
`$swupdate_get_sha256(` or similar variables, and future
1. Moves the modified sw-description file to WORKDIR in
the function swupdate_expand_bitbake_variables, keeping
a copy in S which is unmodified and without exapanded vars.
2. Modifies swupdate_add_artifacts and swupdate_add_src_uri
to target WORKDIR instead of S as the copy destination.
3. Modifies swupdate_create_cpio to use WORKDIR for generation
of cpio instead of S, since all up to date content and
artifacts are stored there.
4. Modifies swupdate_exec_functions to target workdir instead
of s, since all exec functions in swupdate-lib.common operate
on artifacts which are now in workdir and none require the
unmodified sw-description file which is the only content
remaining in S.
5. Updates all references to s and workdir in swupdate-common.bbclass
to reference the correct directories. The references to s in
swupdate-lib.bbclass were not modified, however in all
cases the s variable will actually be set to workdir due to
the modification to swupdate_exec_functions above.
1: https://groups.google.com/g/swupdate/c/ZrDn9hVCet8/m/4lZhxoFfAgAJ
2: https://groups.google.com/g/swupdate/c/8K-9H7C9o5E/m/a9fqhIOjAAAJ
To reproduce this problem before this fix:
1. Turn off rm-work on your builds.
1. Ensure `rm_work` is disabled.
2. Started with `bitbake -c cleanall $my_swupdate_image_recipe`
3. Bitbaked with `bitbake $my_swupdate_image_recipe`
4. Examine the recipe work/$machine*/$my_swupdate_image_recipe/ directory
4a. Verify sources dir contains only sw-description, without any parameter fields (sha256, etc) filed in
4b. Verify base dir (workdir) contains sw-description with all parameter fields filed in
4c. Verify sha256sum on the rootfs file in workdir matches value in sw-description in the workdir.
5. Make a minor change to the image by including another package, for instance in local.conf add `IMAGE_INSTALL:append = " curl"`
6. Bitbaked with `bitbake $my_swupdate_image_recipe`
7. Examine the recipe work/$machine*/$my_swupdate_image_recipe/ directory
7a. Verify timestamp on sources/sw-description file has not changed, still contains variable references.
7b. Verify timestamp on sw-description on base workdir has changed
7c. Verify sha256sum on the rootfs in workdir matches value in sw-description in the workdir.
8. Run swupdate with resulting .swu file on target
8a. Swupdate run succeeds.
classes-recipe/swupdate-common.bbclass | 45 +++++++++++++-------------
1 file changed, 23 insertions(+), 22 deletions(-)
diff --git a/classes-recipe/swupdate-common.bbclass b/classes-recipe/swupdate-common.bbclass
index ef016ca..80a5d5b 100644
--- a/classes-recipe/swupdate-common.bbclass
+++ b/classes-recipe/swupdate-common.bbclass
@@ -65,22 +65,22 @@ def swupdate_getdepends(d):
return depstr
-def swupdate_write_sha256(s):
+def swupdate_write_sha256(workdir):
write_lines.append(line)
- swupdate_exec_functions(d, s, write_lines)
+ swupdate_exec_functions(d, workdir, write_lines)
- with open(os.path.join(s, "sw-description"), 'w+') as f:
+ with open(os.path.join(workdir, "sw-description"), 'w+') as f:
for line in write_lines:
f.write(line)
@@ -172,17 +172,18 @@ def prepare_sw_description(d):
encrypted = (d.getVarFlag("SWUPDATE_IMAGES_ENCRYPTED", filename) or "")
- dst = os.path.join(s, "%s" % filename )
+ dst = os.path.join(workdir, "%s" % filename )
if encrypted == '1':
bb.note("Encryption requested for %s" %(filename))
if not key or not iv:
bb.fatal("Encryption required, but no key found")
- iv = swupdate_get_IV(d, s, filename)
+ iv = swupdate_get_IV(d, workdir, filename)
swupdate_encrypt_file(local, dst, key, iv)
else:
shutil.copyfile(local, dst)
@@ -290,7 +291,7 @@ def swupdate_add_artifacts(d, list_for_cpio):
images = (d.getVar('SWUPDATE_IMAGES') or "").split()
deploydir = d.getVar('DEPLOY_DIR_IMAGE')
imgdeploydir = d.getVar('SWUDEPLOYDIR')
- s = d.getVar('S')
fstypes = (d.getVarFlag("SWUPDATE_IMAGES_FSTYPES", image) or "").split()
encrypted = (d.getVarFlag("SWUPDATE_IMAGES_ENCRYPTED", image) or "")
@@ -305,19 +306,19 @@ def swupdate_add_artifacts(d, list_for_cpio):
for fstype in fstypes:
image_found = False
for imagebase in imagebases:
- image_found = add_image_to_swu(d, deploydir, imagebase + fstype, s, encrypted, list_for_cpio)
+ image_found = add_image_to_swu(d, deploydir, imagebase + fstype, workdir, encrypted, list_for_cpio)
if image_found:
break
if not image_found:
bb.fatal("swupdate cannot find image file: %s" % os.path.join(deploydir, imagebase + fstype))
else: # Allow also complete entries like "image.ext4.gz" in SWUPDATE_IMAGES
- if not add_image_to_swu(d, deploydir, image, s, encrypted, list_for_cpio):
+ if not add_image_to_swu(d, deploydir, image, workdir, encrypted, list_for_cpio):
bb.fatal("swupdate cannot find %s image file" % image)
def swupdate_create_cpio(d, swudeploydir, list_for_cpio):
- s = d.getVar('S')
--
2.34.1
* Resolves issues mentioned in code review there
The fix discussed in [2] resolves build issues on styhead but
introduces an issue which causes hash mismatches when building
without `rm_work`.
All builds succeed in generate a working .swu, however any build
which modifies the rootfs will fail to deploy.
On attempt to deploy, a hash mismatch will be noted like this:
```
[TRACE] : SWUPDATE running : [install_archive_image] : Installing file demo-image-base-jetson-orin-nano-devkit-nvme.rootfs.tar.gz on /tmp/datadst/Pfw8nm//, preserving attributes
[WARN ] : SWUPDATE running : [extract] : archive_read_next_header(): Pathname can't be converted from UTF-8 to current locale. for './etc/ssl/certs/NetLock_Arany_=Class_Gold=_Főtanúsítvány.pem': Invalid or incomplete multibyte or wide character
[WARN ] : SWUPDATE running : [extract] : archive_read_next_header(): Pathname can't be converted from UTF-8 to current locale. for './usr/share/ca-certificates/mozilla/NetLock_Arany_=Class_Gold=_Főtanúsítvány.crt': Invalid or incomplete multibyte or wide character
[ERROR] : SWUPDATE failed [0] ERROR cpio_utils.c : hash_compare : 465 : HASH mismatch : 0f7591f6ffa4aa8e1d919d774974f06d42f184ef5a552f9887201cf3649414da <--> 2497e5b3eaec665ec2ec7db4bb1025d2c92e38c9d8fc22d22aa5e71314f15e6f
[ERROR] : SWUPDATE failed [0] ERROR archive_handler.c : install_archive_image : 344 : Error copying extracted file
[TRACE] : SWUPDATE running : [install_single_image] : Installer for archive not successful !
[ERROR] : SWUPDATE failed [0] ERROR stream_interface.c : extract_files : 319 : Error streaming demo-image-base-jetson-orin-nano-devkit-nvme.rootfs.tar.gz
swupdate_image_write failed: Connection reset by peer
[ERROR] : SWUPDATE failed [1] Image invalid or corrupted. Not installing ...
[ERROR] : SWUPDATE failed [0] ERROR install_from_file.c : endupdate : 55 : SWUpdate *failed* !
```
This happens because the sha256 modifications made to files in the
S directory only happen once before variables are expanded
unless `rm_work` is used to remove the source dir on each build.
So anyone using `rm_work` for their builds would not see this issue.
The previous logic means there's a single expansion of
`$swupdate_get_sha256(` or similar variables, and future
builds use the hash previously placed in the file in the S directory
which is no longer correct when the rootfs file hash changes.
This change:
which is no longer correct when the rootfs file hash changes.
1. Moves the modified sw-description file to WORKDIR in
the function swupdate_expand_bitbake_variables, keeping
a copy in S which is unmodified and without exapanded vars.
2. Modifies swupdate_add_artifacts and swupdate_add_src_uri
to target WORKDIR instead of S as the copy destination.
3. Modifies swupdate_create_cpio to use WORKDIR for generation
of cpio instead of S, since all up to date content and
artifacts are stored there.
4. Modifies swupdate_exec_functions to target workdir instead
of s, since all exec functions in swupdate-lib.common operate
on artifacts which are now in workdir and none require the
unmodified sw-description file which is the only content
remaining in S.
5. Updates all references to s and workdir in swupdate-common.bbclass
to reference the correct directories. The references to s in
swupdate-lib.bbclass were not modified, however in all
cases the s variable will actually be set to workdir due to
the modification to swupdate_exec_functions above.
1: https://groups.google.com/g/swupdate/c/ZrDn9hVCet8/m/4lZhxoFfAgAJ
2: https://groups.google.com/g/swupdate/c/8K-9H7C9o5E/m/a9fqhIOjAAAJ
To reproduce this problem before this fix:
1. Turn off rm-work on your builds.
2. Build the swu target at least once.
3. Make some modification to the rootfs.
4. Build the swu target again.
5. Run the swupdate on target.
Testing (tested with my_swupdate_image_recipe="swupdate-image-tegra" and machine="jetson-orin-nano-devkit-nvme")
3. Make some modification to the rootfs.
4. Build the swu target again.
5. Run the swupdate on target.
1. Ensure `rm_work` is disabled.
2. Started with `bitbake -c cleanall $my_swupdate_image_recipe`
3. Bitbaked with `bitbake $my_swupdate_image_recipe`
4. Examine the recipe work/$machine*/$my_swupdate_image_recipe/ directory
4a. Verify sources dir contains only sw-description, without any parameter fields (sha256, etc) filed in
4b. Verify base dir (workdir) contains sw-description with all parameter fields filed in
4c. Verify sha256sum on the rootfs file in workdir matches value in sw-description in the workdir.
5. Make a minor change to the image by including another package, for instance in local.conf add `IMAGE_INSTALL:append = " curl"`
6. Bitbaked with `bitbake $my_swupdate_image_recipe`
7. Examine the recipe work/$machine*/$my_swupdate_image_recipe/ directory
7a. Verify timestamp on sources/sw-description file has not changed, still contains variable references.
7b. Verify timestamp on sw-description on base workdir has changed
7c. Verify sha256sum on the rootfs in workdir matches value in sw-description in the workdir.
8. Run swupdate with resulting .swu file on target
8a. Swupdate run succeeds.
classes-recipe/swupdate-common.bbclass | 45 +++++++++++++-------------
1 file changed, 23 insertions(+), 22 deletions(-)
diff --git a/classes-recipe/swupdate-common.bbclass b/classes-recipe/swupdate-common.bbclass
index ef016ca..80a5d5b 100644
--- a/classes-recipe/swupdate-common.bbclass
+++ b/classes-recipe/swupdate-common.bbclass
@@ -65,22 +65,22 @@ def swupdate_getdepends(d):
return depstr
-def swupdate_write_sha256(s):
+def swupdate_write_sha256(workdir):
import re
write_lines = []
- with open(os.path.join(s, "sw-description"), 'r') as f:
+ with open(os.path.join(workdir, "sw-description"), 'r') as f:
for line in f:
shastr = r"sha256.+=.+@(.+\")"
m = re.match(r"^(?P<before_placeholder>.+)(sha256|version).+[=:].*(?P<quote>[\'\"])@(?P<filename>.*)(?P=quote)", line)
if m:
filename = m.group('filename')
bb.warn("Syntax for sha256 changed, please use $swupdate_get_sha256(%s)" % filename)
- hash = swupdate_get_sha256(None, s, filename)
+ hash = swupdate_get_sha256(None, workdir, filename)
write_lines.append(line.replace("@%s" % (filename), hash))
else:
write_lines.append(line)
- with open(os.path.join(s, "sw-description"), 'w+') as f:
+ with open(os.path.join(workdir, "sw-description"), 'w+') as f:
for line in write_lines:
f.write(line)
@@ -96,7 +96,7 @@ def swupdate_exec_functions(d, s, write_lines):
write_lines = []
- with open(os.path.join(s, "sw-description"), 'r') as f:
+ with open(os.path.join(workdir, "sw-description"), 'r') as f:
for line in f:
shastr = r"sha256.+=.+@(.+\")"
m = re.match(r"^(?P<before_placeholder>.+)(sha256|version).+[=:].*(?P<quote>[\'\"])@(?P<filename>.*)(?P=quote)", line)
if m:
filename = m.group('filename')
bb.warn("Syntax for sha256 changed, please use $swupdate_get_sha256(%s)" % filename)
- hash = swupdate_get_sha256(None, s, filename)
+ hash = swupdate_get_sha256(None, workdir, filename)
write_lines.append(line.replace("@%s" % (filename), hash))
else:
write_lines.append(line)
- with open(os.path.join(s, "sw-description"), 'w+') as f:
+ with open(os.path.join(workdir, "sw-description"), 'w+') as f:
for line in write_lines:
f.write(line)
write_lines[index] = line
-def swupdate_expand_bitbake_variables(d, s):
+def swupdate_expand_bitbake_variables(d, s, workdir):
write_lines = []
with open(os.path.join(s, "sw-description"), 'r') as f:
@@ -128,9 +128,9 @@ def swupdate_expand_bitbake_variables(d, s):
-def swupdate_expand_bitbake_variables(d, s):
+def swupdate_expand_bitbake_variables(d, s, workdir):
write_lines = []
with open(os.path.join(s, "sw-description"), 'r') as f:
write_lines.append(line)
- swupdate_exec_functions(d, s, write_lines)
+ swupdate_exec_functions(d, workdir, write_lines)
- with open(os.path.join(s, "sw-description"), 'w+') as f:
+ with open(os.path.join(workdir, "sw-description"), 'w+') as f:
for line in write_lines:
f.write(line)
import subprocess
s = d.getVar('S')
- swupdate_expand_bitbake_variables(d, s)
+ workdir = d.getVar('WORKDIR')
+ swupdate_expand_bitbake_variables(d, s, workdir)
- swupdate_write_sha256(s)
+ swupdate_write_sha256(workdir)
encrypt = d.getVar('SWUPDATE_ENCRYPT_SWDESC')
if encrypt:
bb.note("Encryption of sw-description")
- shutil.copyfile(os.path.join(s, 'sw-description'), os.path.join(s, 'sw-description.plain'))
+ shutil.copyfile(os.path.join(workdir, 'sw-description'), os.path.join(workdir, 'sw-description.plain'))
key,iv = swupdate_extract_keys(d.getVar('SWUPDATE_AES_FILE'))
- iv = swupdate_get_IV(d, s, 'sw-description')
- swupdate_encrypt_file(os.path.join(s, 'sw-description.plain'), os.path.join(s, 'sw-description'), key, iv)
+ iv = swupdate_get_IV(d, workdir, 'sw-description')
+ swupdate_encrypt_file(os.path.join(workdir, 'sw-description.plain'), os.path.join(workdir, 'sw-description'), key, iv)
signing = d.getVar('SWUPDATE_SIGNING')
if signing == "1":
@@ -190,8 +191,8 @@ def prepare_sw_description(d):
s = d.getVar('S')
- swupdate_expand_bitbake_variables(d, s)
+ workdir = d.getVar('WORKDIR')
+ swupdate_expand_bitbake_variables(d, s, workdir)
- swupdate_write_sha256(s)
+ swupdate_write_sha256(workdir)
encrypt = d.getVar('SWUPDATE_ENCRYPT_SWDESC')
if encrypt:
bb.note("Encryption of sw-description")
- shutil.copyfile(os.path.join(s, 'sw-description'), os.path.join(s, 'sw-description.plain'))
+ shutil.copyfile(os.path.join(workdir, 'sw-description'), os.path.join(workdir, 'sw-description.plain'))
key,iv = swupdate_extract_keys(d.getVar('SWUPDATE_AES_FILE'))
- iv = swupdate_get_IV(d, s, 'sw-description')
- swupdate_encrypt_file(os.path.join(s, 'sw-description.plain'), os.path.join(s, 'sw-description'), key, iv)
+ iv = swupdate_get_IV(d, workdir, 'sw-description')
+ swupdate_encrypt_file(os.path.join(workdir, 'sw-description.plain'), os.path.join(workdir, 'sw-description'), key, iv)
signing = d.getVar('SWUPDATE_SIGNING')
if signing == "1":
signing = "RSA"
if signing:
- sw_desc_sig = os.path.join(s, 'sw-description.sig')
- sw_desc = os.path.join(s, 'sw-description.plain' if encrypt else 'sw-description')
+ sw_desc_sig = os.path.join(workdir, 'sw-description.sig')
+ sw_desc = os.path.join(workdir, 'sw-description.plain' if encrypt else 'sw-description')
if signing == "CUSTOM":
signcmd = []
@@ -239,7 +240,7 @@ def prepare_sw_description(d):
if signing:
- sw_desc_sig = os.path.join(s, 'sw-description.sig')
- sw_desc = os.path.join(s, 'sw-description.plain' if encrypt else 'sw-description')
+ sw_desc_sig = os.path.join(workdir, 'sw-description.sig')
+ sw_desc = os.path.join(workdir, 'sw-description.plain' if encrypt else 'sw-description')
if signing == "CUSTOM":
signcmd = []
def swupdate_add_src_uri(d, list_for_cpio):
import shutil
- s = d.getVar('S')
import shutil
+ workdir = d.getVar('WORKDIR')
exclude = (d.getVar("SWUPDATE_SRC_URI_EXCLUDE") or "").split()
fetch = bb.fetch2.Fetch([], d)
@@ -255,12 +256,12 @@ def swupdate_add_src_uri(d, list_for_cpio):
exclude = (d.getVar("SWUPDATE_SRC_URI_EXCLUDE") or "").split()
fetch = bb.fetch2.Fetch([], d)
key,iv = swupdate_extract_keys(d.getVar('SWUPDATE_AES_FILE'))
if (filename != 'sw-description') and (os.path.isfile(local)):
encrypted = (d.getVarFlag("SWUPDATE_IMAGES_ENCRYPTED", filename) or "")
- dst = os.path.join(s, "%s" % filename )
+ dst = os.path.join(workdir, "%s" % filename )
if encrypted == '1':
bb.note("Encryption requested for %s" %(filename))
if not key or not iv:
bb.fatal("Encryption required, but no key found")
- iv = swupdate_get_IV(d, s, filename)
+ iv = swupdate_get_IV(d, workdir, filename)
swupdate_encrypt_file(local, dst, key, iv)
else:
shutil.copyfile(local, dst)
@@ -290,7 +291,7 @@ def swupdate_add_artifacts(d, list_for_cpio):
images = (d.getVar('SWUPDATE_IMAGES') or "").split()
deploydir = d.getVar('DEPLOY_DIR_IMAGE')
imgdeploydir = d.getVar('SWUDEPLOYDIR')
- s = d.getVar('S')
+ workdir = d.getVar('WORKDIR')
for image in images:
fstypes = (d.getVarFlag("SWUPDATE_IMAGES_FSTYPES", image) or "").split()
encrypted = (d.getVarFlag("SWUPDATE_IMAGES_ENCRYPTED", image) or "")
@@ -305,19 +306,19 @@ def swupdate_add_artifacts(d, list_for_cpio):
for fstype in fstypes:
image_found = False
for imagebase in imagebases:
- image_found = add_image_to_swu(d, deploydir, imagebase + fstype, s, encrypted, list_for_cpio)
+ image_found = add_image_to_swu(d, deploydir, imagebase + fstype, workdir, encrypted, list_for_cpio)
if image_found:
break
if not image_found:
bb.fatal("swupdate cannot find image file: %s" % os.path.join(deploydir, imagebase + fstype))
else: # Allow also complete entries like "image.ext4.gz" in SWUPDATE_IMAGES
- if not add_image_to_swu(d, deploydir, image, s, encrypted, list_for_cpio):
+ if not add_image_to_swu(d, deploydir, image, workdir, encrypted, list_for_cpio):
bb.fatal("swupdate cannot find %s image file" % image)
def swupdate_create_cpio(d, swudeploydir, list_for_cpio):
- s = d.getVar('S')
- os.chdir(s)
+ workdir = d.getVar('WORKDIR')
+ os.chdir(workdir)
+ workdir = d.getVar('WORKDIR')
+ os.chdir(workdir)
updateimage = d.getVar('IMAGE_NAME') + '.swu'
line = 'for i in ' + ' '.join(list_for_cpio) + '; do echo $i;done | cpio -ov -H crc --reproducible > ' + os.path.join(swudeploydir, updateimage)
os.system(line)
--
2.34.1
Stefano Babic
Jul 22, 2025, 6:00:21 AMJul 22
to Dan Walkes, swup...@googlegroups.com
Hi Dan,
Applied to -master, thanks !
Best regards,
Stefano Babic
Best regards,
Stefano Babic
Reply all
Reply to author
Forward
0 new messages