[PATCH 0/9] Introduce delta update with Hawkbit

[Stefano Babic]

Delta update requires two files, the SWU with the list of the chunks and their hashes
and the file for the artifact in ZCK format. Both must be available in network.

It is currently not possible to put both on the Hawkbit Server, as SWUpdate expects that they are SWU.
SWUpdate expects that the URL for the ZCK is set into sw-description, but this is
hard to reach because the URL is generated by Hawkbit when a file is uploaded, and further
uploads can modify the URL.

This series introduce the concept of list of servers where to download artifacts.
The Hawkbit Connector will first scan all modules returned by the server, and for files
of a selected extension ("rule"), a specific action is taken. For Hawkbit,
the URL of the artifact is stored into0 a list that will be deleted after the update
has terminated.

When the delta handler is running, if no URL or "dynamic" as URL is set, the handler will
check the list and look for an entry set to the value of "zckfile" in the properties.
This URL is then used for the update.

Note that Hawkbit in the current and tested versions has a bug for multipart messages - this
is reported to Hawkbit as issue #2918. An additional patch that force the multipart parser
is sent separately, but it is not planned to be merged as the issue should be solved in Hawkbit.


Stefano Babic (9):
hawkbit: move out parsing of artifact
hawkbit: add rules for incoming artifacts
Add IPC to send a list of URL as sources
delta handler: retrieve URL from core if not present
delta: check if a config file is set before parsing
delta_downloader: activate ssl as default
delta_downloader: enable debug of channel
Factorize function to set authorization token
delta_downloader: load tokens from configuration

core/installer.c | 5 +
core/network_thread.c | 17 +++
core/swupdate.c | 1 +
corelib/server_utils.c | 19 ++++
doc/source/handlers.rst | 53 +++++++++
handlers/delta_downloader.c | 81 +++++++++----
handlers/delta_handler.c | 22 +++-
include/network_ipc.h | 7 ++
include/server_utils.h | 1 +
include/swupdate.h | 1 +
include/swupdate_image.h | 1 +
ipc/network_ipc-if.c | 17 +++
parser/parser.c | 1 +
suricatta/server_hawkbit.c | 221 ++++++++++++++++++++++++------------
suricatta/server_hawkbit.h | 11 ++
15 files changed, 360 insertions(+), 98 deletions(-)

--
2.43.0

[Stefano Babic]

Hawkbit can deliver multiple files for each update, while SWUpdate
relies on a single SWU. In case of delta update, a second file in the
ZCK format is necessary.

Add rules to be applied for each artifact to execute taks before an
update is started.

Signed-off-by: Stefano Babic <stefan...@swupdate.org>
---
suricatta/server_hawkbit.c | 64 +++++++++++++++++++++++++++++++++-----
suricatta/server_hawkbit.h | 2 ++
2 files changed, 58 insertions(+), 8 deletions(-)

diff --git a/suricatta/server_hawkbit.c b/suricatta/server_hawkbit.c
index dbd3e63c..c7b3f1fc 100644
--- a/suricatta/server_hawkbit.c
+++ b/suricatta/server_hawkbit.c
@@ -38,6 +38,39 @@
#define JSON_OBJECT_FREED 1
#define SERVER_NAME "hawkbit"

+/*
+ * Rules depending on artifact extension
+ * Standard SWU are always forwarded to SWUpdate core
+ * Other files can be managed according to the rule
+ * Previous version have always rejected not SWU files,
+ * so add a rule to reject if previous rules do not apply.
+ */
+static bool swu_rule(artifact_t __attribute__ ((__unused__)) *artifact) {
+ return true;
+}
+static bool zck_rule(artifact_t *artifact) {
+ artifact = artifact;
+ return true;
+}
+static bool reject_rule(artifact_t __attribute__ ((__unused__)) *artifact) {
+ return false;
+}
+
+struct artifact_rule {
+ const char *fileext;
+ rule fn;
+};
+
+/*
+ * Rule to be applied when an artifact is found
+ * Last rule in the array is applied when there is not match
+ */
+static struct artifact_rule rules[] = {
+ { "swu", swu_rule },
+ { "zck", zck_rule },
+ { "*" , reject_rule }
+};
+
static struct option long_options[] = {
{"tenant", required_argument, NULL, 't'},
{"id", required_argument, NULL, 'i'},
@@ -1156,16 +1189,31 @@ static server_op_res_t json_extract_artifact(struct array_list *json_data_artifa
artifacts[json_data_artifact_count].size = json_object_get_int(json_data_artifact_size);

/*
- * Check if the file is a .swu image
- * and skip if it not
+ * Check filename and apply rules
+ * default: skip if it is not a SWU
*/
- const char *s = artifacts[json_data_artifact_count].filename;
- int endfilename = strlen(s) - strlen(".swu");
- if (endfilename <= 0 ||
- strncmp(&s[endfilename], ".swu", 4)) {
- DEBUG("File '%s' is not a SWU image, skipping", s);
+ char **fnames;
+ fnames = string_split(artifacts[json_data_artifact_count].filename, '.');
+ int cnt = count_string_array((const char **)fnames);
+ TRACE("Number of strings CNT %d", cnt);
+ if (cnt > 0) {
+ int iter;
+ for (iter = 0; iter < ARRAY_SIZE(rules); iter++) {
+ if (!strcmp(rules[iter].fileext, fnames[cnt - 1])) {
+ TRACE("FOUND RULE %s", rules[iter].fileext);
+ rules[iter].fn(&artifacts[json_data_artifact_count]);
+ break;
+ }
+ }
+ /* Not found, run last rule */
+ if (iter == ARRAY_SIZE(rules))
+ rules[ARRAY_SIZE(rules) - 1].fn(&artifacts[json_data_artifact_count]);
+ } else {
+ /* No file without extension, rejected */
+ TRACE("File '%s' has no extension, skipping", fnames[cnt - 1]);
artifacts[json_data_artifact_count].skip = true;
}
+ free_string_array(fnames);
}

return SERVER_OK;
@@ -1216,7 +1264,7 @@ server_op_res_t server_process_update_artifact(int action_id,
}

/*
- * Elaborate aritifacts, extract URL for download
+ * Elaborate artifacts, extract URL for download
* and check if some of them need some extra work or should be skipped
*/
if (json_extract_artifact(json_data_artifact_array, json_data_artifact_max, artifacts) != SERVER_OK)
diff --git a/suricatta/server_hawkbit.h b/suricatta/server_hawkbit.h
index 69113b3f..4c17ebb9 100644
--- a/suricatta/server_hawkbit.h
+++ b/suricatta/server_hawkbit.h
@@ -90,3 +90,5 @@ typedef struct {
ssize_t size;
bool skip;
} artifact_t;
+
+typedef bool(*rule)(artifact_t *artifact);
--
2.43.0

[Stefano Babic]

Move away the function from server_hawkbit, so that this can be used by
other components (like delta_downloader).

Signed-off-by: Stefano Babic <stefan...@swupdate.org>
---

corelib/server_utils.c | 19 +++++++++++++++++++
include/server_utils.h | 1 +
suricatta/server_hawkbit.c | 23 ++---------------------
3 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/corelib/server_utils.c b/corelib/server_utils.c
index ff78aa2d..13923d50 100644
--- a/corelib/server_utils.c
+++ b/corelib/server_utils.c
@@ -119,3 +119,22 @@ struct json_object *server_tokenize_msg(char *buf, size_t size)

return json_root;
}
+
+/*
+ * Just called once to setup the tokens
+ */
+void channel_settoken(const char *type, const char *token, channel_data_t *chan)
+{
+ char *tokens_header = NULL;
+ if (!token)
+ return;
+
+ if (ENOMEM_ASPRINTF ==
+ asprintf(&tokens_header, "Authorization: %s %s", type, token)) {
+ ERROR("OOM when setting %s.", type);
+ return;
+ }
+ if (tokens_header != NULL && strlen(tokens_header))
+ SETSTRING(chan->auth_token, tokens_header);
+ free(tokens_header);
+}
diff --git a/include/server_utils.h b/include/server_utils.h
index 3b262e59..b3e28265 100644
--- a/include/server_utils.h
+++ b/include/server_utils.h
@@ -17,3 +17,4 @@ struct json_object;
int channel_settings(void *elem, void *data);
server_op_res_t map_channel_retcode(channel_op_res_t response);
struct json_object *server_tokenize_msg(char *buf, size_t size);
+void channel_settoken(const char *type, const char *token, channel_data_t *chan);
diff --git a/suricatta/server_hawkbit.c b/suricatta/server_hawkbit.c
index 84ec3600..a9d09ac5 100644
--- a/suricatta/server_hawkbit.c
+++ b/suricatta/server_hawkbit.c
@@ -182,25 +182,6 @@ static channel_data_t channel_data_defaults = {.debug = false,

static struct timeval server_time;

-/*
- * Just called once to setup the tokens
- */
-static inline void server_hawkbit_settoken(const char *type, const char *token)
-{
- char *tokens_header = NULL;
- if (!token)
- return;
-
- if (ENOMEM_ASPRINTF ==
- asprintf(&tokens_header, "Authorization: %s %s", type, token)) {
- ERROR("OOM when setting %s.", type);
- return;
- }
- if (tokens_header != NULL && strlen(tokens_header))
- SETSTRING(channel_data_defaults.auth_token, tokens_header);
- free(tokens_header);
-}
-
/*
* This is called when a general error is found before sending the stream
* to the installer. In this way, errors are collected in the same way as
@@ -2045,8 +2026,8 @@ static server_op_res_t server_start(const char *fname, int argc, char *argv[])
"but just one at a time is supported.\n");
exit(EXIT_FAILURE);
}
- server_hawkbit_settoken("TargetToken", server_hawkbit.targettoken);
- server_hawkbit_settoken("GatewayToken", server_hawkbit.gatewaytoken);
+ channel_settoken("TargetToken", server_hawkbit.targettoken, &channel_data_defaults);
+ channel_settoken("GatewayToken", server_hawkbit.gatewaytoken, &channel_data_defaults);

/*
* Allocate a channel to communicate with the server
--
2.43.0

[Stefano Babic]

Add a token configuration as done for Hawkbit, rename some vars to make
them more readable.

Signed-off-by: Stefano Babic <stefan...@swupdate.org>
---

handlers/delta_downloader.c | 80 ++++++++++++++++++++++++++-----------
1 file changed, 57 insertions(+), 23 deletions(-)

diff --git a/handlers/delta_downloader.c b/handlers/delta_downloader.c
index 994c80d8..fb8493a6 100644
--- a/handlers/delta_downloader.c
+++ b/handlers/delta_downloader.c
@@ -36,6 +36,16 @@
#include "delta_process.h"
#include "swupdate_settings.h"
#include "server_utils.h"
+#include "parselib.h"
+
+/*
+ * Structure to maintain transferate data
+ * of the downloader
+ */
+typedef struct {
+ char *targettoken;
+ char *gatewaytoken;
+} dwl_priv_t;

/*
* Structure used in curl callbacks
@@ -44,7 +54,7 @@ typedef struct {
unsigned int id; /* Request id */
int writefd; /* IPC file descriptor */
range_answer_t *answer;
-} dwl_data_t;
+} dwl_transfer_t;

extern channel_op_res_t channel_curl_init(void);

@@ -71,7 +81,7 @@ static size_t wrdata_callback(char *buffer, size_t size, size_t nmemb, void *dat
return 0;

channel_data_t *channel_data = (channel_data_t *)data;
- dwl_data_t *dwl = (dwl_data_t *)channel_data->user;
+ dwl_transfer_t *dwl = (dwl_transfer_t *)channel_data->user;
ssize_t nbytes = nmemb * size;
int ret;
if (!nmemb) {
@@ -112,7 +122,7 @@ static size_t wrdata_callback(char *buffer, size_t size, size_t nmemb, void *dat
static size_t delta_callback_headers(char *buffer, size_t size, size_t nitems, void *data)
{
channel_data_t *channel_data = (channel_data_t *)data;
- dwl_data_t *dwl = (dwl_data_t *)channel_data->user;
+ dwl_transfer_t *dwl = (dwl_transfer_t *)channel_data->user;
int ret;

range_answer_t *answer = dwl->answer;
@@ -132,10 +142,28 @@ static size_t delta_callback_headers(char *buffer, size_t size, size_t nitems, v
return nitems * size;
}

+/*
+ * Read setup from configuration file
+ */
+static int delta_downloader_settings(void *elem, void *data)
+{
+ char tmp[128];
+ dwl_priv_t *priv = (dwl_priv_t *)data;
+
+ GET_FIELD_STRING_RESET(LIBCFG_PARSER, elem, "targettoken", tmp);
+ if (strlen(tmp))
+ SETSTRING(priv->targettoken, tmp);
+ GET_FIELD_STRING_RESET(LIBCFG_PARSER, elem, "gatewaytoken", tmp);
+ if (strlen(tmp))
+ SETSTRING(priv->gatewaytoken, tmp);
+
+ return 0;
+}
+
/*
* Process that is spawned by the handler to download the missing chunks.
* Downloading should be done in a separate process to not break
- * privilige separation
+ * privilege separation
*/
int start_delta_downloader(const char __attribute__ ((__unused__)) *fname,
int __attribute__ ((__unused__)) argc,
@@ -143,12 +171,16 @@ int start_delta_downloader(const char __attribute__ ((__unused__)) *fname,
{
ssize_t ret;
range_request_t *req = NULL;
- channel_op_res_t transfer;
+ swupdate_cfg_handle handle;
+ channel_op_res_t result;
range_answer_t *answer;
struct dict httpheaders;
- dwl_data_t priv;
+ dwl_transfer_t dwltransfer;
+ dwl_priv_t dwldata;

TRACE("Starting Internal process for downloading chunks");
+ memset (&dwldata, 0, sizeof(dwldata));
+
if (channel_curl_init() != CHANNEL_OK) {
ERROR("Cannot initialize curl");
return SERVER_EINIT;
@@ -177,6 +209,16 @@ int start_delta_downloader(const char __attribute__ ((__unused__)) *fname,
exit (EXIT_FAILURE);
}

+ swupdate_cfg_init(&handle);
+ if (fname && swupdate_cfg_read_file(&handle, fname) == 0) {
+ read_module_settings(&handle, "delta", channel_settings, &channel_data);
+ read_module_settings(&handle, "delta", delta_downloader_settings, &dwldata);
+ }
+ swupdate_cfg_destroy(&handle);
+
+ channel_settoken("TargetToken", dwldata.targettoken, &channel_data);
+ channel_settoken("GatewayToken", dwldata.gatewaytoken, &channel_data);
+
for (;;) {
ret = read(sw_sockfd, req, sizeof(range_request_t));
if (ret < 0) {
@@ -188,9 +230,9 @@ int start_delta_downloader(const char __attribute__ ((__unused__)) *fname,
ERROR("Malformed data");
continue;
}
- priv.writefd = sw_sockfd;
- priv.id = req->id;
- priv.answer = answer;
+ dwltransfer.writefd = sw_sockfd;
+ dwltransfer.id = req->id;
+ dwltransfer.answer = answer;
channel_data.url = req->data;
channel_data.noipc = true;
channel_data.usessl = true;
@@ -199,30 +241,22 @@ int start_delta_downloader(const char __attribute__ ((__unused__)) *fname,
channel_data.headers = delta_callback_headers;
channel_data.dwlwrdata = wrdata_callback;
channel_data.range = &req->data[req->urllen + 1];
- channel_data.user = &priv;
+ channel_data.user = &dwltransfer;

if (loglevel >= DEBUGLEVEL) {
channel_data_defaults.debug = true;
- }
-
- swupdate_cfg_handle handle;
- swupdate_cfg_init(&handle);
-
- if (fname && swupdate_cfg_read_file(&handle, fname) == 0) {
- read_module_settings(&handle, "delta", channel_settings, &channel_data);
- }
-
- swupdate_cfg_destroy(&handle);
+ } else
+ channel_data_defaults.debug = false;

if (channel->open(channel, &channel_data) == CHANNEL_OK) {
- transfer = channel->get_file(channel, (void *)&channel_data);
+ result = channel->get_file(channel, (void *)&channel_data);
} else {
ERROR("Cannot open channel for communication");
- transfer = CHANNEL_EINIT;
+ result = CHANNEL_EINIT;
}

answer->id = req->id;
- answer->type = (transfer == CHANNEL_OK) ? RANGE_COMPLETED : RANGE_ERROR;
+ answer->type = (result == CHANNEL_OK) ? RANGE_COMPLETED : RANGE_ERROR;
answer->len = 0;
if (write(sw_sockfd, answer, sizeof(*answer)) != sizeof(*answer)) {
ERROR("Answer cannot be sent back, maybe deadlock !!");
--
2.43.0

[Stefano Babic]

Activate debug of libcurl as done in suricatta.

Signed-off-by: Stefano Babic <stefan...@swupdate.org>
---

handlers/delta_downloader.c | 4 ++++
1 file changed, 4 insertions(+)

diff --git a/handlers/delta_downloader.c b/handlers/delta_downloader.c
index e3e20dcb..994c80d8 100644
--- a/handlers/delta_downloader.c
+++ b/handlers/delta_downloader.c
@@ -201,6 +201,10 @@ int start_delta_downloader(const char __attribute__ ((__unused__)) *fname,

channel_data.range = &req->data[req->urllen + 1];

channel_data.user = &priv;

+ if (loglevel >= DEBUGLEVEL) {
+ channel_data_defaults.debug = true;
+ }
+
swupdate_cfg_handle handle;
swupdate_cfg_init(&handle);

--
2.43.0

[Stefano Babic]

Signed-off-by: Stefano Babic <stefan...@swupdate.org>
---

handlers/delta_downloader.c | 1 +
1 file changed, 1 insertion(+)

diff --git a/handlers/delta_downloader.c b/handlers/delta_downloader.c
index 890cd43e..e3e20dcb 100644
--- a/handlers/delta_downloader.c
+++ b/handlers/delta_downloader.c
@@ -193,6 +193,7 @@ int start_delta_downloader(const char __attribute__ ((__unused__)) *fname,
priv.answer = answer;

channel_data.url = req->data;
channel_data.noipc = true;

+ channel_data.usessl = true;
channel_data.method = CHANNEL_GET;
channel_data.content_type = "*";
channel_data.headers = delta_callback_headers;
--
2.43.0