[PATCH v2 0/6] kunit: Add dynamically-extending log

[Richard Fitzgerald]

Replace the original fixed-size log buffer with a dynamically-
extending log.

Patch 1 provides the basic implementation. The following patches
add test cases, support for logging long strings, and an optimization
to the string formatting that is now more thoroughly testable.

Richard Fitzgerald (6):
kunit: Replace fixed-size log with dynamically-extending buffer
kunit: kunit-test: Add test cases for extending log buffer
kunit: Handle logging of lines longer than the fragment buffer size
kunit: kunit-test: Add test cases for logging very long lines
kunit: kunit-test: Add test of logging only a newline
kunit: Don't waste first attempt to format string in
kunit_log_append()

include/kunit/test.h | 25 +++-
lib/kunit/debugfs.c | 65 +++++++--
lib/kunit/kunit-test.c | 321 ++++++++++++++++++++++++++++++++++++++++-
lib/kunit/test.c | 127 +++++++++++++---
4 files changed, 489 insertions(+), 49 deletions(-)

--
2.30.2

[Richard Fitzgerald]

Re-implement the log buffer as a list of buffer fragments that can
be extended as the size of the log info grows.

When using parameterization the test case can run many times and create
a large amount of log. It's not really practical to keep increasing the
size of the fixed buffer every time a test needs more space. And a big
fixed buffer wastes memory.

The original char *log pointer is replaced by a pointer to a list of
struct kunit_log_frag, each containing a fixed-size buffer.

kunit_log_append() now attempts to append to the last kunit_log_frag in
the list. If there isn't enough space it will append a new kunit_log_frag
to the list. This simple implementation does not attempt to completely
fill the buffer in every kunit_log_frag.

The 'log' member of kunit_suite, kunit_test_case and kunit_suite must be a
pointer because the API of kunit_log() requires that is the same type in
all three structs. As kunit.log is a pointer to the 'log' of the current
kunit_case, it must be a pointer in the other two structs.

The existing kunit-test.c log tests have been updated to build against the
new fragmented log implementation.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---
include/kunit/test.h | 25 +++++++++++-----
lib/kunit/debugfs.c | 65 ++++++++++++++++++++++++++++++++++--------
lib/kunit/kunit-test.c | 29 +++++++++++++------
lib/kunit/test.c | 63 ++++++++++++++++++++++++++++------------
4 files changed, 136 insertions(+), 46 deletions(-)

diff --git a/include/kunit/test.h b/include/kunit/test.h
index 011e0d6bb506..ef8e09aafe1b 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -33,8 +33,8 @@ DECLARE_STATIC_KEY_FALSE(kunit_running);

struct kunit;

-/* Size of log associated with test. */
-#define KUNIT_LOG_SIZE 2048
+/* Size of log buffer fragments. */
+#define KUNIT_LOG_FRAGMENT_SIZE (256 - sizeof(struct list_head))

/* Maximum size of parameter description string. */
#define KUNIT_PARAM_DESC_SIZE 128
@@ -85,6 +85,11 @@ struct kunit_attributes {
enum kunit_speed speed;
};

+struct kunit_log_frag {
+ struct list_head list;
+ char buf[KUNIT_LOG_FRAGMENT_SIZE];
+};
+
/**
* struct kunit_case - represents an individual test case.
*
@@ -132,7 +137,7 @@ struct kunit_case {
/* private: internal use only. */
enum kunit_status status;
char *module_name;
- char *log;
+ struct list_head *log;
};

static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
@@ -252,7 +257,7 @@ struct kunit_suite {
/* private: internal use only */
char status_comment[KUNIT_STATUS_COMMENT_SIZE];
struct dentry *debugfs;
- char *log;
+ struct list_head *log;
int suite_init_err;
};

@@ -272,7 +277,7 @@ struct kunit {

/* private: internal use only. */
const char *name; /* Read only after initialization! */
- char *log; /* Points at case log after initialization */
+ struct list_head *log; /* Points at case log after initialization */
struct kunit_try_catch try_catch;
/* param_value is the current parameter value for a test case. */
const void *param_value;
@@ -304,7 +309,7 @@ static inline void kunit_set_failure(struct kunit *test)

bool kunit_enabled(void);

-void kunit_init_test(struct kunit *test, const char *name, char *log);
+void kunit_init_test(struct kunit *test, const char *name, struct list_head *log);

int kunit_run_tests(struct kunit_suite *suite);

@@ -317,6 +322,12 @@ int __kunit_test_suites_init(struct kunit_suite * const * const suites, int num_

void __kunit_test_suites_exit(struct kunit_suite **suites, int num_suites);

+static inline void kunit_init_log_frag(struct kunit_log_frag *frag)
+{
+ INIT_LIST_HEAD(&frag->list);
+ frag->buf[0] = '\0';
+}
+
#if IS_BUILTIN(CONFIG_KUNIT)
int kunit_run_all_tests(void);
#else
@@ -451,7 +462,7 @@ static inline void *kunit_kcalloc(struct kunit *test, size_t n, size_t size, gfp

void kunit_cleanup(struct kunit *test);

-void __printf(2, 3) kunit_log_append(char *log, const char *fmt, ...);
+void __printf(2, 3) kunit_log_append(struct list_head *log, const char *fmt, ...);

/**
* kunit_mark_skipped() - Marks @test_or_suite as skipped
diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c
index 22c5c496a68f..a26b6d31bd2f 100644
--- a/lib/kunit/debugfs.c
+++ b/lib/kunit/debugfs.c
@@ -5,6 +5,7 @@
*/

#include <linux/debugfs.h>
+#include <linux/list.h>
#include <linux/module.h>

#include <kunit/test.h>
@@ -37,14 +38,15 @@ void kunit_debugfs_init(void)
debugfs_rootdir = debugfs_create_dir(KUNIT_DEBUGFS_ROOT, NULL);
}

-static void debugfs_print_result(struct seq_file *seq,
- struct kunit_suite *suite,
- struct kunit_case *test_case)
+static void debugfs_print_log(struct seq_file *seq, const struct list_head *log)
{
- if (!test_case || !test_case->log)
+ struct kunit_log_frag *frag;
+
+ if (!log)
return;

- seq_printf(seq, "%s", test_case->log);
+ list_for_each_entry(frag, log, list)
+ seq_puts(seq, frag->buf);
}

/*
@@ -69,10 +71,9 @@ static int debugfs_print_results(struct seq_file *seq, void *v)
seq_printf(seq, KUNIT_SUBTEST_INDENT "1..%zd\n", kunit_suite_num_test_cases(suite));

kunit_suite_for_each_test_case(suite, test_case)
- debugfs_print_result(seq, suite, test_case);
+ debugfs_print_log(seq, test_case->log);

- if (suite->log)
- seq_printf(seq, "%s", suite->log);
+ debugfs_print_log(seq, suite->log);

seq_printf(seq, "%s %d %s\n",
kunit_status_to_ok_not_ok(success), 1, suite->name);
@@ -100,14 +101,53 @@ static const struct file_operations debugfs_results_fops = {
.release = debugfs_release,
};

+static struct list_head *kunit_debugfs_alloc_log(void)
+{
+ struct list_head *log;
+ struct kunit_log_frag *frag;
+
+ log = kzalloc(sizeof(*log), GFP_KERNEL);
+ if (!log)
+ return NULL;
+
+ INIT_LIST_HEAD(log);
+
+ frag = kmalloc(sizeof(*frag), GFP_KERNEL);
+ if (!frag) {
+ kfree(log);
+ return NULL;
+ }
+
+ kunit_init_log_frag(frag);
+ list_add_tail(&frag->list, log);
+
+ return log;
+}
+
+static void kunit_debugfs_free_log(struct list_head *log)
+{
+ struct kunit_log_frag *frag, *n;
+
+ if (!log)
+ return;
+
+ list_for_each_entry_safe(frag, n, log, list) {
+ list_del(&frag->list);
+ kfree(frag);
+ }
+
+ kfree(log);
+}
+
void kunit_debugfs_create_suite(struct kunit_suite *suite)
{
struct kunit_case *test_case;

/* Allocate logs before creating debugfs representation. */
- suite->log = kzalloc(KUNIT_LOG_SIZE, GFP_KERNEL);
+ suite->log = kunit_debugfs_alloc_log();
+
kunit_suite_for_each_test_case(suite, test_case)
- test_case->log = kzalloc(KUNIT_LOG_SIZE, GFP_KERNEL);
+ test_case->log = kunit_debugfs_alloc_log();

suite->debugfs = debugfs_create_dir(suite->name, debugfs_rootdir);

@@ -121,7 +161,8 @@ void kunit_debugfs_destroy_suite(struct kunit_suite *suite)
struct kunit_case *test_case;

debugfs_remove_recursive(suite->debugfs);
- kfree(suite->log);
+ kunit_debugfs_free_log(suite->log);
+
kunit_suite_for_each_test_case(suite, test_case)
- kfree(test_case->log);
+ kunit_debugfs_free_log(test_case->log);
}
diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index 83d8e90ca7a2..54dc011c8980 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -533,9 +533,16 @@ static struct kunit_suite kunit_resource_test_suite = {
static void kunit_log_test(struct kunit *test)
{
struct kunit_suite suite;
+ struct kunit_log_frag *frag;

- suite.log = kunit_kzalloc(test, KUNIT_LOG_SIZE, GFP_KERNEL);
+ suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ INIT_LIST_HEAD(suite.log);
+ frag = kunit_kmalloc(test, sizeof(*frag), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
+ kunit_init_log_frag(frag);
+ KUNIT_EXPECT_EQ(test, frag->buf[0], '\0');
+ list_add_tail(&frag->list, suite.log);

kunit_log(KERN_INFO, test, "put this in log.");
kunit_log(KERN_INFO, test, "this too.");
@@ -543,14 +550,17 @@ static void kunit_log_test(struct kunit *test)
kunit_log(KERN_INFO, &suite, "along with this.");

#ifdef CONFIG_KUNIT_DEBUGFS
+ frag = list_first_entry(test->log, struct kunit_log_frag, list);
KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(test->log, "put this in log."));
+ strstr(frag->buf, "put this in log."));
KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(test->log, "this too."));
+ strstr(frag->buf, "this too."));
+
+ frag = list_first_entry(suite.log, struct kunit_log_frag, list);
KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(suite.log, "add to suite log."));
+ strstr(frag->buf, "add to suite log."));
KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(suite.log, "along with this."));
+ strstr(frag->buf, "along with this."));
#else
KUNIT_EXPECT_NULL(test, test->log);
#endif
@@ -558,11 +568,14 @@ static void kunit_log_test(struct kunit *test)

static void kunit_log_newline_test(struct kunit *test)
{
+ struct kunit_log_frag *frag;
+
kunit_info(test, "Add newline\n");
if (test->log) {
- KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(test->log, "Add newline\n"),
- "Missing log line, full log:\n%s", test->log);
- KUNIT_EXPECT_NULL(test, strstr(test->log, "Add newline\n\n"));
+ frag = list_first_entry(test->log, struct kunit_log_frag, list);
+ KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(frag->buf, "Add newline\n"),
+ "Missing log line, full log:\n%s", frag->buf);
+ KUNIT_EXPECT_NULL(test, strstr(frag->buf, "Add newline\n\n"));
} else {
kunit_skip(test, "only useful when debugfs is enabled");
}
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index cb9797fa6303..bdb361741214 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -11,6 +11,7 @@
#include <kunit/test-bug.h>
#include <kunit/attributes.h>
#include <linux/kernel.h>
+#include <linux/list.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/panic.h>
@@ -114,46 +115,66 @@ static void kunit_print_test_stats(struct kunit *test,
* already present.
* @log: The log to add the newline to.
*/
-static void kunit_log_newline(char *log)
+static void kunit_log_newline(struct kunit_log_frag *frag)
{
int log_len, len_left;

- log_len = strlen(log);
- len_left = KUNIT_LOG_SIZE - log_len - 1;
+ log_len = strlen(frag->buf);
+ len_left = sizeof(frag->buf) - log_len - 1;

- if (log_len > 0 && log[log_len - 1] != '\n')
- strncat(log, "\n", len_left);
+ if (log_len > 0 && frag->buf[log_len - 1] != '\n')
+ strncat(frag->buf, "\n", len_left);
}

-/*
- * Append formatted message to log, size of which is limited to
- * KUNIT_LOG_SIZE bytes (including null terminating byte).
- */
-void kunit_log_append(char *log, const char *fmt, ...)
+static struct kunit_log_frag *kunit_log_extend(struct list_head *log)
+{
+ struct kunit_log_frag *frag;
+
+ frag = kmalloc(sizeof(*frag), GFP_KERNEL);
+ if (!frag)
+ return NULL;
+
+ kunit_init_log_frag(frag);
+ list_add_tail(&frag->list, log);
+
+ return frag;
+}
+
+/* Append formatted message to log, extending the log buffer if necessary. */
+void kunit_log_append(struct list_head *log, const char *fmt, ...)
{
va_list args;
+ struct kunit_log_frag *frag;
int len, log_len, len_left;

if (!log)
return;

- log_len = strlen(log);
- len_left = KUNIT_LOG_SIZE - log_len - 1;
- if (len_left <= 0)
- return;
+ frag = list_last_entry(log, struct kunit_log_frag, list);
+ log_len = strlen(frag->buf);
+ len_left = sizeof(frag->buf) - log_len - 1;

/* Evaluate length of line to add to log */
va_start(args, fmt);
len = vsnprintf(NULL, 0, fmt, args) + 1;
va_end(args);

+ if (len > len_left) {
+ frag = kunit_log_extend(log);
+ if (!frag)
+ return;
+
+ len_left = sizeof(frag->buf) - 1;
+ log_len = 0;
+ }
+
/* Print formatted line to the log */
va_start(args, fmt);
- vsnprintf(log + log_len, min(len, len_left), fmt, args);
+ vsnprintf(frag->buf + log_len, min(len, len_left), fmt, args);
va_end(args);

/* Add newline to end of log if not already present. */
- kunit_log_newline(log);
+ kunit_log_newline(frag);
}
EXPORT_SYMBOL_GPL(kunit_log_append);

@@ -359,14 +380,18 @@ void __kunit_do_failed_assertion(struct kunit *test,
}
EXPORT_SYMBOL_GPL(__kunit_do_failed_assertion);

-void kunit_init_test(struct kunit *test, const char *name, char *log)
+void kunit_init_test(struct kunit *test, const char *name, struct list_head *log)
{
spin_lock_init(&test->lock);
INIT_LIST_HEAD(&test->resources);
test->name = name;
test->log = log;
- if (test->log)
- test->log[0] = '\0';
+ if (test->log) {
+ struct kunit_log_frag *frag = list_first_entry(test->log,
+ struct kunit_log_frag,
+ list);
+ frag->buf[0] = '\0';
+ }
test->status = KUNIT_SUCCESS;
test->status_comment[0] = '\0';
}
--
2.30.2

[Richard Fitzgerald]

Add handling to kunit_log_append() for log messages that are longer than
a single buffer fragment.

The initial implementation of fragmented buffers did not change the logic
of the original kunit_log_append(). A consequence was that it still had
the original assumption that a log line will fit into one buffer.

This patch checks for log messages that are larger than one fragment
buffer. In that case, kvasprintf() is used to format it into a temporary
buffer and that content is then split across as many fragments as
necessary.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/test.c | 65 +++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 61 insertions(+), 4 deletions(-)

diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index bdb361741214..b00f077314e3 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -140,25 +140,82 @@ static struct kunit_log_frag *kunit_log_extend(struct list_head *log)
return frag;
}

+static void kunit_log_append_string(struct list_head *log, const char *src)
+{
+ struct kunit_log_frag *frag, *new_frag;
+ int log_len, bytes_left;
+ ssize_t written;
+ char *p;
+

+ frag = list_last_entry(log, struct kunit_log_frag, list);
+ log_len = strlen(frag->buf);

+ bytes_left = sizeof(frag->buf) - log_len;
+
+ written = strscpy(frag->buf + log_len, src, bytes_left);
+ if (written != -E2BIG)
+ goto newline;
+
+ src += bytes_left - 1;
+ do {
+ new_frag = kunit_log_extend(log);
+ if (!new_frag)
+ goto newline;
+
+ frag = new_frag;
+ written = strscpy(frag->buf, src, sizeof(frag->buf));
+ src += sizeof(frag->buf) - 1;
+ } while (written == -E2BIG);
+
+newline:
+ if (written == -E2BIG)
+ written = strlen(frag->buf);
+
+ p = &frag->buf[written - 1];
+ if (*p != '\n') {
+ if (strlcat(frag->buf, "\n", sizeof(frag->buf)) >= sizeof(frag->buf)) {

+ frag = kunit_log_extend(log);
+ if (!frag) {

+ *p = '\n';
+ return;
+ }
+
+ frag->buf[0] = '\n';
+ frag->buf[1] = '\0';
+ }
+ }
+}
+

/* Append formatted message to log, extending the log buffer if necessary. */

void kunit_log_append(struct list_head *log, const char *fmt, ...)
{
va_list args;

struct kunit_log_frag *frag;
int len, log_len, len_left;

+ char *tmp = NULL;

if (!log)
return;

- frag = list_last_entry(log, struct kunit_log_frag, list);
- log_len = strlen(frag->buf);
- len_left = sizeof(frag->buf) - log_len - 1;
-

/* Evaluate length of line to add to log */
va_start(args, fmt);
len = vsnprintf(NULL, 0, fmt, args) + 1;
va_end(args);

+ if (len > sizeof(frag->buf) - 1) {
+ va_start(args, fmt);
+ tmp = kvasprintf(GFP_KERNEL, fmt, args);
+ va_end(args);
+
+ kunit_log_append_string(log, tmp);
+ kfree(tmp);
+
+ return;
+ }
+

+ frag = list_last_entry(log, struct kunit_log_frag, list);
+ log_len = strlen(frag->buf);
+ len_left = sizeof(frag->buf) - log_len - 1;

+
if (len > len_left) {
frag = kunit_log_extend(log);
if (!frag)
--
2.30.2

[Richard Fitzgerald]

It's wasteful to call vsnprintf() only to figure out the length of the
string. The string might fit in the available buffer space but we have to
vsnprintf() again to do that.

Instead, attempt to format the string to the available buffer at the same
time as finding the string length. Only if the string didn't fit the
available space is it necessary to extend the log and format the string
again to a new fragment buffer.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/test.c | 33 +++++++++++++++++----------------
1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index b00f077314e3..e7f0a46d184a 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -196,11 +196,21 @@ void kunit_log_append(struct list_head *log, const char *fmt, ...)
if (!log)
return;

- /* Evaluate length of line to add to log */

+ frag = list_last_entry(log, struct kunit_log_frag, list);
+ log_len = strlen(frag->buf);
+ len_left = sizeof(frag->buf) - log_len - 1;
+

+ /* Attempt to print formatted line to current fragment */
va_start(args, fmt);
- len = vsnprintf(NULL, 0, fmt, args) + 1;
+ len = vsnprintf(frag->buf + log_len, len_left, fmt, args) + 1;
va_end(args);

+ if (len <= len_left)
+ goto out_newline;
+
+ /* Abandon the truncated result of vsnprintf */
+ frag->buf[log_len] = '\0';

+
if (len > sizeof(frag->buf) - 1) {

va_start(args, fmt);

tmp = kvasprintf(GFP_KERNEL, fmt, args);

@@ -212,24 +222,15 @@ void kunit_log_append(struct list_head *log, const char *fmt, ...)

return;
}

- frag = list_last_entry(log, struct kunit_log_frag, list);
- log_len = strlen(frag->buf);
- len_left = sizeof(frag->buf) - log_len - 1;
-

- if (len > len_left) {
- frag = kunit_log_extend(log);
- if (!frag)
- return;
-
- len_left = sizeof(frag->buf) - 1;
- log_len = 0;
- }

+ frag = kunit_log_extend(log);
+ if (!frag)

+ return;

/* Print formatted line to the log */
va_start(args, fmt);

- vsnprintf(frag->buf + log_len, min(len, len_left), fmt, args);
+ vsnprintf(frag->buf, sizeof(frag->buf) - 1, fmt, args);
va_end(args);
-
+out_newline:

/* Add newline to end of log if not already present. */

kunit_log_newline(frag);
}
--
2.30.2

[Richard Fitzgerald]

Add a test that logging a string containing only a newline appends
one newline to the log.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/kunit-test.c | 6 ++++++
1 file changed, 6 insertions(+)

diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index 621acdb5385e..ce80fb42128c 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -613,6 +613,12 @@ static void kunit_log_newline_test(struct kunit *test)
kunit_init_log_frag(frag);
list_add_tail(&frag->list, suite.log);

+ /* Log only a newline */
+ kunit_log_append(suite.log, "\n");
+ KUNIT_EXPECT_TRUE(test, list_is_singular(suite.log));
+ KUNIT_EXPECT_STREQ(test, frag->buf, "\n");
+ frag->buf[0] = '\0';
+
/* String that exactly fills fragment leaving no room for \n */
memset(frag->buf, 0, sizeof(frag->buf));
memset(frag->buf, 'x', sizeof(frag->buf) - 9);
--
2.30.2

[Richard Fitzgerald]

Add kunit_log_long_line_test() to test that logging a line longer than
the buffer fragment size doesn't truncate the line.

Add extra tests to kunit_log_newline_test() for lines longer than the
buffer fragment size.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/kunit-test.c | 84 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 83 insertions(+), 1 deletion(-)

diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index 48967d12e053..621acdb5385e 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -596,7 +596,7 @@ static void kunit_log_newline_test(struct kunit *test)
{
struct kunit_suite suite;
struct kunit_log_frag *frag;
- char *p;
+ char *p, *line;

kunit_info(test, "Add newline\n");
if (test->log) {

@@ -621,6 +621,33 @@ static void kunit_log_newline_test(struct kunit *test)
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
KUNIT_EXPECT_NOT_NULL_MSG(test, strstr(p, "x12345678\n"),
"Newline not appended when fragment is full. Log is:\n'%s'", p);
+ kunit_kfree(test, p);
+
+ /* String that is much longer than a fragment */
+ line = kunit_kzalloc(test, sizeof(frag->buf) * 6, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, line);
+ memset(line, 'x', (sizeof(frag->buf) * 6) - 1);
+ kunit_log_append(suite.log, "%s", line);
+ p = get_concatenated_log(test, suite.log, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
+ KUNIT_EXPECT_EQ(test, p[strlen(p) - 1], '\n');
+ KUNIT_EXPECT_NULL(test, strstr(p, "\n\n"));
+ kunit_kfree(test, p);
+
+ kunit_log_append(suite.log, "%s\n", line);
+ p = get_concatenated_log(test, suite.log, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
+ KUNIT_EXPECT_EQ(test, p[strlen(p) - 1], '\n');
+ KUNIT_EXPECT_NULL(test, strstr(p, "\n\n"));
+ kunit_kfree(test, p);
+
+ line[strlen(line) - 1] = '\n';
+ kunit_log_append(suite.log, "%s", line);
+ p = get_concatenated_log(test, suite.log, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
+ KUNIT_EXPECT_EQ(test, p[strlen(p) - 1], '\n');
+ KUNIT_EXPECT_NULL(test, strstr(p, "\n\n"));
+ kunit_kfree(test, p);

} else {
kunit_skip(test, "only useful when debugfs is enabled");
}

@@ -782,12 +809,67 @@ static void kunit_log_frag_sized_line_test(struct kunit *test)
#endif
}

+static void kunit_log_long_line_test(struct kunit *test)
+{
+#ifdef CONFIG_KUNIT_DEBUGFS
+ struct kunit_suite suite;
+ struct kunit_log_frag *frag;
+ struct rnd_state rnd;
+ char *line, *p, *pn;
+ size_t line_buf_size, len;
+ int num_frags, i;
+

+ suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);

+ INIT_LIST_HEAD(suite.log);
+ frag = kunit_kmalloc(test, sizeof(*frag), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
+ kunit_init_log_frag(frag);
+ KUNIT_EXPECT_EQ(test, frag->buf[0], '\0');
+ list_add_tail(&frag->list, suite.log);

+
+ /* Create a very long string to be logged */
+ line_buf_size = sizeof(frag->buf) * 6;
+ line = kunit_kmalloc(test, line_buf_size, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, line);
+ line[0] = '\0';
+
+ prandom_seed_state(&rnd, 3141592653589793238ULL);
+ len = 0;
+ do {
+ static const char fill[] =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+ i = prandom_u32_state(&rnd) % (sizeof(fill) - 1);
+ len = strlcat(line, &fill[i], line_buf_size);
+ } while (len < line_buf_size);
+
+ kunit_log_append(suite.log, "%s\n", line);
+
+ p = get_concatenated_log(test, suite.log, &num_frags);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
+ KUNIT_EXPECT_GT(test, num_frags, 1);
+
+ kunit_info(test, "num_frags:%d total len:%zu\n", num_frags, strlen(p));
+
+ /* Don't compare the trailing '\n' */
+ pn = strrchr(p, '\n');
+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test, pn);
+ *pn = '\0';
+ KUNIT_EXPECT_EQ(test, strlen(p), strlen(line));
+ KUNIT_EXPECT_STREQ(test, p, line);
+#else
+ kunit_skip(test, "only useful when debugfs is enabled");
+#endif
+}
+
static struct kunit_case kunit_log_test_cases[] = {
KUNIT_CASE(kunit_log_test),
KUNIT_CASE(kunit_log_newline_test),
KUNIT_CASE(kunit_log_extend_test_1),
KUNIT_CASE(kunit_log_extend_test_2),
KUNIT_CASE(kunit_log_frag_sized_line_test),
+ KUNIT_CASE(kunit_log_long_line_test),
{}
};

--
2.30.2

[David Gow]

On Tue, 8 Aug 2023 at 20:35, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>
> Re-implement the log buffer as a list of buffer fragments that can
> be extended as the size of the log info grows.
>
> When using parameterization the test case can run many times and create
> a large amount of log. It's not really practical to keep increasing the
> size of the fixed buffer every time a test needs more space. And a big
> fixed buffer wastes memory.
>
> The original char *log pointer is replaced by a pointer to a list of
> struct kunit_log_frag, each containing a fixed-size buffer.
>
> kunit_log_append() now attempts to append to the last kunit_log_frag in
> the list. If there isn't enough space it will append a new kunit_log_frag
> to the list. This simple implementation does not attempt to completely
> fill the buffer in every kunit_log_frag.
>
> The 'log' member of kunit_suite, kunit_test_case and kunit_suite must be a
> pointer because the API of kunit_log() requires that is the same type in
> all three structs. As kunit.log is a pointer to the 'log' of the current
> kunit_case, it must be a pointer in the other two structs.
>
> The existing kunit-test.c log tests have been updated to build against the
> new fragmented log implementation.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Looks good to me.

A few small notes inline below, mostly around the possibility of
either embedding the list_head in the kunit_case struct directly
(rather than using a pointer), or of pointing directly to the first
fragment, rather than a separately-allocated struct list_head. Neither
are showstoppers, though (and if it increases complexity at all, it's
possibly premature optimization).

Otherwise, some test nitpicks and the fact that this will need a
trivial rebase due to the module filtering stuff landing in
kselftest/kunit.

Reviewed-by: David Gow <davi...@google.com>

The other patches in the series pass the initial sniff test: I'll try
to get a more thorough review done in the next day or two.

Cheers,
-- David

I wonder if this has to be a pointer? Would it make more sense to
embed the struct list_head (or possibly a whole struct
kunit_log_fragment if we weren't worried about kernel image size, see
below) here, to avoid an extra allocation and a bunch of extra
indirect memory accesses.

Even if we still want to pass a pointer to a struct list_head around,
we could just take the address of this one, rather than allocating it
separately. We'd have to copy the whole struct list_head around,
rather than just the pointer, and it'd increase the size of struct
kunit_case and similar, but struct list_head is just two pointers, so
it shouldn't be drastic enough to matter.

Not a problem either way: I doubt this would be a performance or
memory bottleneck, so if it's simpler to keep it as a pointer I don't
mind, but if it's an easy enough change, it may be worth it.

> };
>
> static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
> @@ -252,7 +257,7 @@ struct kunit_suite {
> /* private: internal use only */
> char status_comment[KUNIT_STATUS_COMMENT_SIZE];
> struct dentry *debugfs;
> - char *log;
> + struct list_head *log;

As above, should this be a pointer?

> int suite_init_err;
> };
>
> @@ -272,7 +277,7 @@ struct kunit {
>
> /* private: internal use only. */
> const char *name; /* Read only after initialization! */
> - char *log; /* Points at case log after initialization */
> + struct list_head *log; /* Points at case log after initialization */

I could imagine this either being a pointer to &(case.log), or a copy
of the list_head which is then copied back into the case structure if
we went with a less pointer-y implementation.

> struct kunit_try_catch try_catch;
> /* param_value is the current parameter value for a test case. */
> const void *param_value;
> @@ -304,7 +309,7 @@ static inline void kunit_set_failure(struct kunit *test)
>
> bool kunit_enabled(void);
>
> -void kunit_init_test(struct kunit *test, const char *name, char *log);
> +void kunit_init_test(struct kunit *test, const char *name, struct list_head *log);
>
> int kunit_run_tests(struct kunit_suite *suite);
>
> @@ -317,6 +322,12 @@ int __kunit_test_suites_init(struct kunit_suite * const * const suites, int num_
>
> void __kunit_test_suites_exit(struct kunit_suite **suites, int num_suites);
>
> +static inline void kunit_init_log_frag(struct kunit_log_frag *frag)
> +{
> + INIT_LIST_HEAD(&frag->list);
> + frag->buf[0] = '\0';
> +}
> +

There's now a (trivial) conflict between this and the latest
kselftest/kunit branch (with the module filtering patches). If you're
doing a v3, could you rebase?

If we're always allocating at least one fragment, would it make sense
to embed a while kunit_log_frag in the test struct, rather than just a
list_head (so the first fragment doesn't need allocating separately)?

Of course, that could bloat the kunit_case / kunit_suite structs too
much (and therefore the .kunit_test_suites section). But maybe even a
pointer to a kunit_log_frag would work.

Probably not worth the extra complexity, but it's a thought...

I'm not super thrilled that this only operates on the first fragment.
Could we at least note that this is not the "full log" in the
assertion message here, and maybe also assert that the log hasn't
grown to a second fragment?

I was going to wonder whether or not we should cache the length of the
current fragment somewhere, but thinking about it, it's probably not
worth it given we're only measuring a single fragment, and it's capped
at 256 bytes.

[Richard Fitzgerald]

I did start out trying to use the first fragment as the list head.
Trouble with this is that the functions in list.h expect to have a
dummy list_head node that is only the head, but not an actual list
member. It's possible to workaround this but the shenanigans involved is
likely to trip someone up later so reverted to doing the list the way
the API intended.

For the pointers, I did consider embedding the list_head instead of
using a pointer. But then the struct kunit can't refer to the
kunit_case list, it can only take it over. There can only be one list
head because the ->prev and ->next pointers of the first and last
members in the list can only point to one head.

After playing around with it I decided that it wasn't worth trying to
avoid the pointers. At least... it wasn't worth spending a lot of time
trying to avoid them for an initial implementation.

Maybe some magic with typeof() in the kunit_log() would let us use
different types for the members of kunit_suite, kunit_case, kunit?
Then the list_head can be directly embedded in the first two but a
pointer in kunit?

> Otherwise, some test nitpicks and the fact that this will need a
> trivial rebase due to the module filtering stuff landing in
> kselftest/kunit.
>
> Reviewed-by: David Gow <davi...@google.com>
>

...

>> static void kunit_log_newline_test(struct kunit *test)
>> {
>> + struct kunit_log_frag *frag;
>> +
>> kunit_info(test, "Add newline\n");
>> if (test->log) {
>> - KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(test->log, "Add newline\n"),
>> - "Missing log line, full log:\n%s", test->log);
>> - KUNIT_EXPECT_NULL(test, strstr(test->log, "Add newline\n\n"));
>> + frag = list_first_entry(test->log, struct kunit_log_frag, list);
>> + KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(frag->buf, "Add newline\n"),
>> + "Missing log line, full log:\n%s", frag->buf);
>
> I'm not super thrilled that this only operates on the first fragment.
> Could we at least note that this is not the "full log" in the
> assertion message here, and maybe also assert that the log hasn't
> grown to a second fragment?
>

The only aim in this first patch is to make sure that kunit-test.c still
builds. I've added extra newline test cases in later patches.

...

>
> I was going to wonder whether or not we should cache the length of the
> current fragment somewhere, but thinking about it, it's probably not
> worth it given we're only measuring a single fragment, and it's capped
> at 256 bytes.
>

Yes, I had the same thought but decided to leave it as something that
can be done later. But as you say it's doubtful whether it's worth the
extra storage space when the buffer fragments are small. On x86_64
simply adding a length member could add 8 bytes per fragment (because of
rounding). If the size of the fragment buffer is capped at 256 we could
use single byte for the length and hope the compiler doesn't insert
padding between a char and a char[] array.

Take a look at what happens when you log a message to the kernel log and
by comparison this kunit logging is super-lightweight.

(I did look at whether we could re-use the existing kernel log
implementation but decided it was too heavily hardcoded to how the
kernel log works.)

[Richard Fitzgerald]

Replace the original fixed-size log buffer with a dynamically-
extending log.

Patch 1 provides the basic implementation. The following patches
add test cases, support for logging long strings, and an optimization
to the string formatting that is now more thoroughly testable.

Changes since v2:
- Fixed uninitialized string bug in get_concatenated_log().
- Moved get_concatenated_log() into first patch so that
kunit_log_newline_test() dumps the entire log on error.
- Moved kunit_log_frag_sized_line_test() to the correct point in
the chain, after the change that it depends on. Also log another
line after the long line to test that the log extends correctly.
- Added kunit_log_init_frag_test() to test kunit_init_log_frag()
instead of testing it as part of every other test.

Richard Fitzgerald (7):

kunit: Replace fixed-size log with dynamically-extending buffer
kunit: kunit-test: Add test cases for extending log buffer
kunit: Handle logging of lines longer than the fragment buffer size

kunit: kunit-test: Test logging a line that exactly fills a fragment

kunit: kunit-test: Add test cases for logging very long lines
kunit: kunit-test: Add test of logging only a newline
kunit: Don't waste first attempt to format string in
kunit_log_append()

include/kunit/test.h | 25 ++-
lib/kunit/debugfs.c | 65 ++++++--
lib/kunit/kunit-test.c | 339 ++++++++++++++++++++++++++++++++++++++++-
lib/kunit/test.c | 127 ++++++++++++---
4 files changed, 507 insertions(+), 49 deletions(-)

--
2.30.2

[Richard Fitzgerald]

Re-implement the log buffer as a list of buffer fragments that can
be extended as the size of the log info grows.

When using parameterization the test case can run many times and create
a large amount of log. It's not really practical to keep increasing the
size of the fixed buffer every time a test needs more space. And a big
fixed buffer wastes memory.

The original char *log pointer is replaced by a pointer to a list of
struct kunit_log_frag, each containing a fixed-size buffer.

kunit_log_append() now attempts to append to the last kunit_log_frag in
the list. If there isn't enough space it will append a new kunit_log_frag
to the list. This simple implementation does not attempt to completely
fill the buffer in every kunit_log_frag.

The 'log' member of kunit_suite, kunit_test_case and kunit_suite must be a
pointer because the API of kunit_log() requires that is the same type in
all three structs. As kunit.log is a pointer to the 'log' of the current
kunit_case, it must be a pointer in the other two structs.

The existing kunit-test.c log tests have been updated to build against the

new fragmented log implementation. If the test fails the new function
get_concatenated_log() constructs a single contiguous string from the
log fragments so that the whole log can be emitted in the failure
message.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

include/kunit/test.h | 25 +++++++++++-----
lib/kunit/debugfs.c | 65 ++++++++++++++++++++++++++++++++++--------

lib/kunit/kunit-test.c | 48 +++++++++++++++++++++++++------
lib/kunit/test.c | 63 ++++++++++++++++++++++++++++------------
4 files changed, 155 insertions(+), 46 deletions(-)

diff --git a/include/kunit/test.h b/include/kunit/test.h
index d33114097d0d..cb5082efc91c 100644

--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -33,8 +33,8 @@ DECLARE_STATIC_KEY_FALSE(kunit_running);

struct kunit;

-/* Size of log associated with test. */
-#define KUNIT_LOG_SIZE 2048
+/* Size of log buffer fragments. */
+#define KUNIT_LOG_FRAGMENT_SIZE (256 - sizeof(struct list_head))

/* Maximum size of parameter description string. */
#define KUNIT_PARAM_DESC_SIZE 128

@@ -85,6 +85,17 @@ struct kunit_attributes {

enum kunit_speed speed;
};

+struct kunit_log_frag {
+ struct list_head list;
+ char buf[KUNIT_LOG_FRAGMENT_SIZE];
+};
+

+static inline void kunit_init_log_frag(struct kunit_log_frag *frag)
+{
+ INIT_LIST_HEAD(&frag->list);
+ frag->buf[0] = '\0';
+}
+

/**
* struct kunit_case - represents an individual test case.
*

@@ -132,7 +143,7 @@ struct kunit_case {

/* private: internal use only. */
enum kunit_status status;
char *module_name;
- char *log;
+ struct list_head *log;

};

static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)

@@ -252,7 +263,7 @@ struct kunit_suite {

/* private: internal use only */
char status_comment[KUNIT_STATUS_COMMENT_SIZE];
struct dentry *debugfs;
- char *log;
+ struct list_head *log;

int suite_init_err;
};

@@ -278,7 +289,7 @@ struct kunit {

/* private: internal use only. */
const char *name; /* Read only after initialization! */
- char *log; /* Points at case log after initialization */
+ struct list_head *log; /* Points at case log after initialization */

struct kunit_try_catch try_catch;
/* param_value is the current parameter value for a test case. */
const void *param_value;

@@ -314,7 +325,7 @@ const char *kunit_filter_glob(void);
char *kunit_filter(void);
char *kunit_filter_action(void);

-void kunit_init_test(struct kunit *test, const char *name, char *log);
+void kunit_init_test(struct kunit *test, const char *name, struct list_head *log);

int kunit_run_tests(struct kunit_suite *suite);

@@ -472,7 +483,7 @@ static inline void *kunit_kcalloc(struct kunit *test, size_t n, size_t size, gfp

void kunit_cleanup(struct kunit *test);

-void __printf(2, 3) kunit_log_append(char *log, const char *fmt, ...);
+void __printf(2, 3) kunit_log_append(struct list_head *log, const char *fmt, ...);

/**
* kunit_mark_skipped() - Marks @test_or_suite as skipped
diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c
index 22c5c496a68f..a26b6d31bd2f 100644
--- a/lib/kunit/debugfs.c
+++ b/lib/kunit/debugfs.c
@@ -5,6 +5,7 @@
*/

#include <linux/debugfs.h>
+#include <linux/list.h>
#include <linux/module.h>

#include <kunit/test.h>
@@ -37,14 +38,15 @@ void kunit_debugfs_init(void)
debugfs_rootdir = debugfs_create_dir(KUNIT_DEBUGFS_ROOT, NULL);
}

-static void debugfs_print_result(struct seq_file *seq,
- struct kunit_suite *suite,
- struct kunit_case *test_case)
+static void debugfs_print_log(struct seq_file *seq, const struct list_head *log)
{
- if (!test_case || !test_case->log)

+ struct kunit_log_frag *frag;
+

+ if (!log)
return;

- seq_printf(seq, "%s", test_case->log);
+ list_for_each_entry(frag, log, list)
+ seq_puts(seq, frag->buf);
}

/*
@@ -69,10 +71,9 @@ static int debugfs_print_results(struct seq_file *seq, void *v)
seq_printf(seq, KUNIT_SUBTEST_INDENT "1..%zd\n", kunit_suite_num_test_cases(suite));

kunit_suite_for_each_test_case(suite, test_case)
- debugfs_print_result(seq, suite, test_case);
+ debugfs_print_log(seq, test_case->log);

- if (suite->log)
- seq_printf(seq, "%s", suite->log);
+ debugfs_print_log(seq, suite->log);

seq_printf(seq, "%s %d %s\n",
kunit_status_to_ok_not_ok(success), 1, suite->name);
@@ -100,14 +101,53 @@ static const struct file_operations debugfs_results_fops = {
.release = debugfs_release,
};

+static struct list_head *kunit_debugfs_alloc_log(void)
+{
+ struct list_head *log;

+ struct kunit_log_frag *frag;
+

+ log = kzalloc(sizeof(*log), GFP_KERNEL);
+ if (!log)
+ return NULL;
+
+ INIT_LIST_HEAD(log);
+
+ frag = kmalloc(sizeof(*frag), GFP_KERNEL);

index 83d8e90ca7a2..a199f83bac67 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -530,12 +530,37 @@ static struct kunit_suite kunit_resource_test_suite = {
.test_cases = kunit_resource_test_cases,
};

+static char *get_concatenated_log(struct kunit *test, const struct list_head *log)
+{
+ struct kunit_log_frag *frag;
+ size_t len = 0;
+ char *p;
+
+ list_for_each_entry(frag, log, list)
+ len += strlen(frag->buf);
+
+ len++; /* for terminating '\0' */
+ p = kunit_kzalloc(test, len, GFP_KERNEL);
+
+ list_for_each_entry(frag, log, list)
+ strlcat(p, frag->buf, len);
+
+ return p;
+}
+

static void kunit_log_test(struct kunit *test)
{
struct kunit_suite suite;
+ struct kunit_log_frag *frag;

- suite.log = kunit_kzalloc(test, KUNIT_LOG_SIZE, GFP_KERNEL);
+ suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ INIT_LIST_HEAD(suite.log);
+ frag = kunit_kmalloc(test, sizeof(*frag), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
+ kunit_init_log_frag(frag);
+ KUNIT_EXPECT_EQ(test, frag->buf[0], '\0');
+ list_add_tail(&frag->list, suite.log);

kunit_log(KERN_INFO, test, "put this in log.");
kunit_log(KERN_INFO, test, "this too.");

@@ -543,14 +568,17 @@ static void kunit_log_test(struct kunit *test)

kunit_log(KERN_INFO, &suite, "along with this.");

#ifdef CONFIG_KUNIT_DEBUGFS

+ frag = list_first_entry(test->log, struct kunit_log_frag, list);

KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(test->log, "put this in log."));
+ strstr(frag->buf, "put this in log."));
KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(test->log, "this too."));
+ strstr(frag->buf, "this too."));
+
+ frag = list_first_entry(suite.log, struct kunit_log_frag, list);
KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(suite.log, "add to suite log."));
+ strstr(frag->buf, "add to suite log."));
KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(suite.log, "along with this."));
+ strstr(frag->buf, "along with this."));
#else
KUNIT_EXPECT_NULL(test, test->log);
#endif

@@ -558,11 +586,15 @@ static void kunit_log_test(struct kunit *test)

static void kunit_log_newline_test(struct kunit *test)
{
+ struct kunit_log_frag *frag;
+
kunit_info(test, "Add newline\n");
if (test->log) {
- KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(test->log, "Add newline\n"),
- "Missing log line, full log:\n%s", test->log);
- KUNIT_EXPECT_NULL(test, strstr(test->log, "Add newline\n\n"));
+ frag = list_first_entry(test->log, struct kunit_log_frag, list);
+ KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(frag->buf, "Add newline\n"),
+ "Missing log line, full log:\n%s",

+ get_concatenated_log(test, test->log));

+ KUNIT_EXPECT_NULL(test, strstr(frag->buf, "Add newline\n\n"));
} else {
kunit_skip(test, "only useful when debugfs is enabled");
}
diff --git a/lib/kunit/test.c b/lib/kunit/test.c

index 49698a168437..dfe51bc2b387 100644

+ struct kunit_log_frag *frag;
+

+ frag = kmalloc(sizeof(*frag), GFP_KERNEL);
+ if (!frag)
+ return NULL;
+
+ kunit_init_log_frag(frag);
+ list_add_tail(&frag->list, log);
+
+ return frag;
+}
+
+/* Append formatted message to log, extending the log buffer if necessary. */
+void kunit_log_append(struct list_head *log, const char *fmt, ...)
{
va_list args;
+ struct kunit_log_frag *frag;
int len, log_len, len_left;

if (!log)
return;

- log_len = strlen(log);
- len_left = KUNIT_LOG_SIZE - log_len - 1;
- if (len_left <= 0)
- return;
+ frag = list_last_entry(log, struct kunit_log_frag, list);
+ log_len = strlen(frag->buf);

[Richard Fitzgerald]

Add test cases for the dynamically-extending log buffer.

kunit_log_init_frag_test() tests that kunit_init_log_frag() correctly
initializes new struct kunit_log_frag.

kunit_log_extend_test_1() logs a series of numbered lines then tests
that the resulting log contains all the lines.

kunit_log_extend_test_2() logs a large number of lines of varying length
to create many fragments, then tests that all lines are present.

kunit_log_newline_test() has a new test to append a line that is exactly
the length of the available space in the current fragment and check that
the resulting log has a trailing '\n'.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/kunit-test.c | 182 +++++++++++++++++++++++++++++++++++++++--
1 file changed, 174 insertions(+), 8 deletions(-)

diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index a199f83bac67..c0ee33a8031e 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -7,6 +7,7 @@
*/
#include <kunit/test.h>
#include <kunit/test-bug.h>
+#include <linux/prandom.h>

#include "try-catch-impl.h"

@@ -530,10 +531,12 @@ static struct kunit_suite kunit_resource_test_suite = {
.test_cases = kunit_resource_test_cases,
};

-static char *get_concatenated_log(struct kunit *test, const struct list_head *log)
+static char *get_concatenated_log(struct kunit *test, const struct list_head *log,
+ int *num_frags)
{
struct kunit_log_frag *frag;
size_t len = 0;
+ int frag_count = 0;
char *p;

list_for_each_entry(frag, log, list)
@@ -542,24 +545,42 @@ static char *get_concatenated_log(struct kunit *test, const struct list_head *lo

len++; /* for terminating '\0' */

p = kunit_kzalloc(test, len, GFP_KERNEL);

- list_for_each_entry(frag, log, list)
+ list_for_each_entry(frag, log, list) {
strlcat(p, frag->buf, len);
+ ++frag_count;
+ }
+
+ if (num_frags)
+ *num_frags = frag_count;

return p;
}

-static void kunit_log_test(struct kunit *test)
+static void kunit_log_init_frag_test(struct kunit *test)
{
- struct kunit_suite suite;
struct kunit_log_frag *frag;

- suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
- KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
- INIT_LIST_HEAD(suite.log);

frag = kunit_kmalloc(test, sizeof(*frag), GFP_KERNEL);

KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
+ memset(frag, 0x5a, sizeof(*frag));
+
kunit_init_log_frag(frag);

KUNIT_EXPECT_EQ(test, frag->buf[0], '\0');

+ KUNIT_EXPECT_TRUE(test, list_is_first(&frag->list, &frag->list));
+ KUNIT_EXPECT_TRUE(test, list_is_last(&frag->list, &frag->list));
+}
+
+static void kunit_log_test(struct kunit *test)
+{
+ struct kunit_suite suite;
+ struct kunit_log_frag *frag;
+

+ suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ INIT_LIST_HEAD(suite.log);
+ frag = kunit_kzalloc(test, sizeof(*frag), GFP_KERNEL);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
+ kunit_init_log_frag(frag);

list_add_tail(&frag->list, suite.log);

kunit_log(KERN_INFO, test, "put this in log.");

@@ -586,23 +607,168 @@ static void kunit_log_test(struct kunit *test)

static void kunit_log_newline_test(struct kunit *test)
{

+ struct kunit_suite suite;
struct kunit_log_frag *frag;
+ char *p;

kunit_info(test, "Add newline\n");
if (test->log) {

frag = list_first_entry(test->log, struct kunit_log_frag, list);

KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(frag->buf, "Add newline\n"),

"Missing log line, full log:\n%s",

- get_concatenated_log(test, test->log));
+ get_concatenated_log(test, test->log, NULL));

KUNIT_EXPECT_NULL(test, strstr(frag->buf, "Add newline\n\n"));

+

+ suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ INIT_LIST_HEAD(suite.log);
+ frag = kunit_kzalloc(test, sizeof(*frag), GFP_KERNEL);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
+ kunit_init_log_frag(frag);

+ list_add_tail(&frag->list, suite.log);
+

+ /* String that exactly fills fragment leaving no room for \n */

+ memset(frag->buf, 0, sizeof(frag->buf));
+ memset(frag->buf, 'x', sizeof(frag->buf) - 9);
+ kunit_log_append(suite.log, "12345678");

+ p = get_concatenated_log(test, suite.log, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);

+ KUNIT_EXPECT_NOT_NULL_MSG(test, strstr(p, "x12345678\n"),
+ "Newline not appended when fragment is full. Log is:\n'%s'", p);

} else {
kunit_skip(test, "only useful when debugfs is enabled");
}
}

+static void kunit_log_extend_test_1(struct kunit *test)
+{
+#ifdef CONFIG_KUNIT_DEBUGFS
+ struct kunit_suite suite;
+ struct kunit_log_frag *frag;
+ char line[60];
+ char *p, *pn;
+ size_t len, n;
+ int num_lines, num_frags, i;
+

+ suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ INIT_LIST_HEAD(suite.log);
+ frag = kunit_kzalloc(test, sizeof(*frag), GFP_KERNEL);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
+ kunit_init_log_frag(frag);

+ list_add_tail(&frag->list, suite.log);
+

+ i = 0;

+ len = 0;
+ do {

+ n = snprintf(line, sizeof(line),
+ "The quick brown fox jumps over the lazy penguin %d\n", i);
+ KUNIT_ASSERT_LT(test, n, sizeof(line));
+ kunit_log_append(suite.log, line);
+ ++i;
+ len += n;
+ } while (len < (sizeof(frag->buf) * 30));
+ num_lines = i;

+
+ p = get_concatenated_log(test, suite.log, &num_frags);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
+ KUNIT_EXPECT_GT(test, num_frags, 1);
+

+ kunit_info(test, "num lines:%d num_frags:%d total len:%zu\n",
+ num_lines, num_frags, strlen(p));
+
+ i = 0;
+ while ((pn = strchr(p, '\n')) != NULL) {
+ *pn = '\0';
+ snprintf(line, sizeof(line),
+ "The quick brown fox jumps over the lazy penguin %d", i);
+ KUNIT_EXPECT_STREQ(test, p, line);
+ p = pn + 1;
+ ++i;
+ }
+ KUNIT_EXPECT_EQ(test, i, num_lines);
+#else
+ kunit_skip(test, "only useful when debugfs is enabled");
+#endif
+}
+
+static void kunit_log_extend_test_2(struct kunit *test)
+{
+#ifdef CONFIG_KUNIT_DEBUGFS
+ struct kunit_suite suite;
+ struct kunit_log_frag *frag;
+ struct rnd_state rnd;
+ char line[101];
+ char *p, *pn;
+ size_t len;
+ int num_lines, num_frags, n, i;
+

+ suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ INIT_LIST_HEAD(suite.log);
+ frag = kunit_kzalloc(test, sizeof(*frag), GFP_KERNEL);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
+ kunit_init_log_frag(frag);

+ list_add_tail(&frag->list, suite.log);
+

+ /* Build log line of varying content */

+ line[0] = '\0';

+ i = 0;
+ do {
+ char tmp[9];
+
+ snprintf(tmp, sizeof(tmp), "%x", i++);
+ len = strlcat(line, tmp, sizeof(line));
+ } while (len < sizeof(line) - 1);
+
+ /*
+ * Log lines of different lengths until we have created
+ * many fragments.
+ * The "randomness" must be repeatable.
+ */
+ prandom_seed_state(&rnd, 3141592653589793238ULL);
+ i = 0;
+ len = 0;
+ num_lines = 0;
+ do {
+ kunit_log_append(suite.log, "%s\n", &line[i]);
+ len += sizeof(line) - i;
+ num_lines++;
+ i = prandom_u32_state(&rnd) % (sizeof(line) - 1);
+ } while (len < (sizeof(frag->buf) * 30));
+
+ /* There must be more than one buffer fragment now */
+ KUNIT_EXPECT_FALSE(test, list_is_singular(suite.log));

+
+ p = get_concatenated_log(test, suite.log, &num_frags);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
+ KUNIT_EXPECT_GT(test, num_frags, 1);
+

+ kunit_info(test, "num lines:%d num_frags:%d total len:%zu\n",
+ num_lines, num_frags, strlen(p));
+
+ prandom_seed_state(&rnd, 3141592653589793238ULL);
+ i = 0;
+ n = 0;
+ while ((pn = strchr(p, '\n')) != NULL) {
+ *pn = '\0';
+ KUNIT_EXPECT_STREQ(test, p, &line[i]);
+ p = pn + 1;
+ n++;
+ i = prandom_u32_state(&rnd) % (sizeof(line) - 1);
+ }
+ KUNIT_EXPECT_EQ_MSG(test, n, num_lines, "Not enough lines.");

+#else
+ kunit_skip(test, "only useful when debugfs is enabled");
+#endif
+}
+
static struct kunit_case kunit_log_test_cases[] = {

+ KUNIT_CASE(kunit_log_init_frag_test),
KUNIT_CASE(kunit_log_test),
KUNIT_CASE(kunit_log_newline_test),
+ KUNIT_CASE(kunit_log_extend_test_1),
+ KUNIT_CASE(kunit_log_extend_test_2),
{}
};

--
2.30.2

[Richard Fitzgerald]

If a log string is the exact length of a log fragment buffer
kunit_log_append() should now exactly fill that fragment without
extending the log.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/kunit-test.c | 37 +++++++++++++++++++++++++++++++++++++
1 file changed, 37 insertions(+)

diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index c0ee33a8031e..9ac81828d018 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -763,12 +763,49 @@ static void kunit_log_extend_test_2(struct kunit *test)
#endif
}

+static void kunit_log_frag_sized_line_test(struct kunit *test)

+{
+#ifdef CONFIG_KUNIT_DEBUGFS
+ struct kunit_suite suite;

+ struct kunit_log_frag *frag, *src;

+
+ suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ INIT_LIST_HEAD(suite.log);
+ frag = kunit_kzalloc(test, sizeof(*frag), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
+ kunit_init_log_frag(frag);
+ list_add_tail(&frag->list, suite.log);
+

+ src = kunit_kzalloc(test, sizeof(*src), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, src);
+ memset(src->buf, 'x', sizeof(src->buf) - 2);
+ KUNIT_ASSERT_EQ(test, strlen(src->buf), sizeof(src->buf) - 2);
+
+ /* Log a string that exactly fills the fragment */
+ kunit_log_append(suite.log, "%s\n", src->buf);
+ KUNIT_EXPECT_TRUE(test, list_is_singular(suite.log));
+ KUNIT_EXPECT_EQ(test, strlen(frag->buf), sizeof(frag->buf) - 1);
+ strlcat(src->buf, "\n", sizeof(src->buf));
+ KUNIT_EXPECT_STREQ(test, frag->buf, src->buf);
+
+ /* Logging another string should extend the log */
+ kunit_log_append(suite.log, "Next\n");
+ KUNIT_EXPECT_EQ(test, list_count_nodes(suite.log), 2);
+ frag = list_last_entry(suite.log, struct kunit_log_frag, list);
+ KUNIT_EXPECT_STREQ(test, frag->buf, "Next\n");

+#else
+ kunit_skip(test, "only useful when debugfs is enabled");
+#endif
+}
+
static struct kunit_case kunit_log_test_cases[] = {

KUNIT_CASE(kunit_log_init_frag_test),
KUNIT_CASE(kunit_log_test),
KUNIT_CASE(kunit_log_newline_test),
KUNIT_CASE(kunit_log_extend_test_1),
KUNIT_CASE(kunit_log_extend_test_2),
+ KUNIT_CASE(kunit_log_frag_sized_line_test),
{}
};

--
2.30.2

[Richard Fitzgerald]

Add handling to kunit_log_append() for log messages that are longer than
a single buffer fragment.

The initial implementation of fragmented buffers did not change the logic
of the original kunit_log_append(). A consequence was that it still had
the original assumption that a log line will fit into one buffer.

This patch checks for log messages that are larger than one fragment
buffer. In that case, kvasprintf() is used to format it into a temporary
buffer and that content is then split across as many fragments as
necessary.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/test.c | 65 +++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 61 insertions(+), 4 deletions(-)

diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index dfe51bc2b387..28d0048d6171 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c

@@ -140,25 +140,82 @@ static struct kunit_log_frag *kunit_log_extend(struct list_head *log)
return frag;
}

+static void kunit_log_append_string(struct list_head *log, const char *src)

+{

+ struct kunit_log_frag *frag, *new_frag;
+ int log_len, bytes_left;
+ ssize_t written;

+ char *p;
+

+ frag = list_last_entry(log, struct kunit_log_frag, list);
+ log_len = strlen(frag->buf);

+ bytes_left = sizeof(frag->buf) - log_len;
+
+ written = strscpy(frag->buf + log_len, src, bytes_left);
+ if (written != -E2BIG)
+ goto newline;
+
+ src += bytes_left - 1;
+ do {
+ new_frag = kunit_log_extend(log);
+ if (!new_frag)
+ goto newline;
+
+ frag = new_frag;
+ written = strscpy(frag->buf, src, sizeof(frag->buf));
+ src += sizeof(frag->buf) - 1;
+ } while (written == -E2BIG);
+
+newline:
+ if (written == -E2BIG)
+ written = strlen(frag->buf);
+
+ p = &frag->buf[written - 1];
+ if (*p != '\n') {
+ if (strlcat(frag->buf, "\n", sizeof(frag->buf)) >= sizeof(frag->buf)) {

+ frag = kunit_log_extend(log);
+ if (!frag) {

+ *p = '\n';
+ return;
+ }
+
+ frag->buf[0] = '\n';
+ frag->buf[1] = '\0';
+ }
+ }
+}
+

/* Append formatted message to log, extending the log buffer if necessary. */

void kunit_log_append(struct list_head *log, const char *fmt, ...)
{
va_list args;

struct kunit_log_frag *frag;
int len, log_len, len_left;

+ char *tmp = NULL;

if (!log)
return;

- frag = list_last_entry(log, struct kunit_log_frag, list);
- log_len = strlen(frag->buf);

- len_left = sizeof(frag->buf) - log_len - 1;
-

/* Evaluate length of line to add to log */
va_start(args, fmt);
len = vsnprintf(NULL, 0, fmt, args) + 1;
va_end(args);

+ if (len > sizeof(frag->buf) - 1) {
+ va_start(args, fmt);
+ tmp = kvasprintf(GFP_KERNEL, fmt, args);
+ va_end(args);
+
+ kunit_log_append_string(log, tmp);
+ kfree(tmp);
+
+ return;
+ }
+

+ frag = list_last_entry(log, struct kunit_log_frag, list);
+ log_len = strlen(frag->buf);
+ len_left = sizeof(frag->buf) - log_len - 1;

[Richard Fitzgerald]

Add kunit_log_long_line_test() to test that logging a line longer than
the buffer fragment size doesn't truncate the line.

Add extra tests to kunit_log_newline_test() for lines longer than the
buffer fragment size.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/kunit-test.c | 84 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 83 insertions(+), 1 deletion(-)

diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index 9ac81828d018..c079550c3afd 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -609,7 +609,7 @@ static void kunit_log_newline_test(struct kunit *test)
{

struct kunit_suite suite;
struct kunit_log_frag *frag;

- char *p;
+ char *p, *line;

kunit_info(test, "Add newline\n");
if (test->log) {

@@ -635,6 +635,33 @@ static void kunit_log_newline_test(struct kunit *test)
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);

KUNIT_EXPECT_NOT_NULL_MSG(test, strstr(p, "x12345678\n"),

"Newline not appended when fragment is full. Log is:\n'%s'", p);

+ kunit_kfree(test, p);
+

+ /* String that is much longer than a fragment */
+ line = kunit_kzalloc(test, sizeof(frag->buf) * 6, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, line);
+ memset(line, 'x', (sizeof(frag->buf) * 6) - 1);
+ kunit_log_append(suite.log, "%s", line);

+ p = get_concatenated_log(test, suite.log, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);

+ KUNIT_EXPECT_EQ(test, p[strlen(p) - 1], '\n');
+ KUNIT_EXPECT_NULL(test, strstr(p, "\n\n"));
+ kunit_kfree(test, p);
+

+ kunit_log_append(suite.log, "%s\n", line);

+ p = get_concatenated_log(test, suite.log, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);

+ KUNIT_EXPECT_EQ(test, p[strlen(p) - 1], '\n');
+ KUNIT_EXPECT_NULL(test, strstr(p, "\n\n"));
+ kunit_kfree(test, p);
+
+ line[strlen(line) - 1] = '\n';
+ kunit_log_append(suite.log, "%s", line);

+ p = get_concatenated_log(test, suite.log, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);

+ KUNIT_EXPECT_EQ(test, p[strlen(p) - 1], '\n');
+ KUNIT_EXPECT_NULL(test, strstr(p, "\n\n"));
+ kunit_kfree(test, p);
} else {

kunit_skip(test, "only useful when debugfs is enabled");
}

@@ -799,6 +826,60 @@ static void kunit_log_frag_sized_line_test(struct kunit *test)
#endif
}

+static void kunit_log_long_line_test(struct kunit *test)

+{
+#ifdef CONFIG_KUNIT_DEBUGFS
+ struct kunit_suite suite;

+ struct kunit_log_frag *frag;
+ struct rnd_state rnd;

+ char *line, *p, *pn;
+ size_t line_buf_size, len;
+ int num_frags, i;

+
+ suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ INIT_LIST_HEAD(suite.log);

+ frag = kunit_kmalloc(test, sizeof(*frag), GFP_KERNEL);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
+ kunit_init_log_frag(frag);

+ KUNIT_EXPECT_EQ(test, frag->buf[0], '\0');

+ list_add_tail(&frag->list, suite.log);
+

+ /* Create a very long string to be logged */
+ line_buf_size = sizeof(frag->buf) * 6;
+ line = kunit_kmalloc(test, line_buf_size, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, line);

+ line[0] = '\0';
+

+ prandom_seed_state(&rnd, 3141592653589793238ULL);

+ len = 0;
+ do {

+ static const char fill[] =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+ i = prandom_u32_state(&rnd) % (sizeof(fill) - 1);
+ len = strlcat(line, &fill[i], line_buf_size);
+ } while (len < line_buf_size);
+

+ kunit_log_append(suite.log, "%s\n", line);

+
+ p = get_concatenated_log(test, suite.log, &num_frags);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
+ KUNIT_EXPECT_GT(test, num_frags, 1);
+

+ kunit_info(test, "num_frags:%d total len:%zu\n", num_frags, strlen(p));
+
+ /* Don't compare the trailing '\n' */
+ pn = strrchr(p, '\n');
+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test, pn);

+ *pn = '\0';

+ KUNIT_EXPECT_EQ(test, strlen(p), strlen(line));
+ KUNIT_EXPECT_STREQ(test, p, line);

+#else
+ kunit_skip(test, "only useful when debugfs is enabled");
+#endif
+}
+
static struct kunit_case kunit_log_test_cases[] = {
KUNIT_CASE(kunit_log_init_frag_test),
KUNIT_CASE(kunit_log_test),

@@ -806,6 +887,7 @@ static struct kunit_case kunit_log_test_cases[] = {

[Richard Fitzgerald]

Add a test that logging a string containing only a newline appends
one newline to the log.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/kunit-test.c | 6 ++++++
1 file changed, 6 insertions(+)

diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index c079550c3afd..7e710c73c7e5 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -627,6 +627,12 @@ static void kunit_log_newline_test(struct kunit *test)

kunit_init_log_frag(frag);
list_add_tail(&frag->list, suite.log);

+ /* Log only a newline */
+ kunit_log_append(suite.log, "\n");
+ KUNIT_EXPECT_TRUE(test, list_is_singular(suite.log));
+ KUNIT_EXPECT_STREQ(test, frag->buf, "\n");

+ frag->buf[0] = '\0';

+
/* String that exactly fills fragment leaving no room for \n */

memset(frag->buf, 0, sizeof(frag->buf));

memset(frag->buf, 'x', sizeof(frag->buf) - 9);

--
2.30.2

[Richard Fitzgerald]

It's wasteful to call vsnprintf() only to figure out the length of the
string. The string might fit in the available buffer space but we have to
vsnprintf() again to do that.

Instead, attempt to format the string to the available buffer at the same
time as finding the string length. Only if the string didn't fit the
available space is it necessary to extend the log and format the string
again to a new fragment buffer.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/test.c | 33 +++++++++++++++++----------------
1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 28d0048d6171..230ec5e9824f 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -196,11 +196,21 @@ void kunit_log_append(struct list_head *log, const char *fmt, ...)
if (!log)
return;

- /* Evaluate length of line to add to log */

+ frag = list_last_entry(log, struct kunit_log_frag, list);
+ log_len = strlen(frag->buf);
+ len_left = sizeof(frag->buf) - log_len - 1;
+

+ /* Attempt to print formatted line to current fragment */
va_start(args, fmt);
- len = vsnprintf(NULL, 0, fmt, args) + 1;

+ len = vsnprintf(frag->buf + log_len, len_left, fmt, args) + 1;
va_end(args);

+ if (len <= len_left)
+ goto out_newline;
+
+ /* Abandon the truncated result of vsnprintf */
+ frag->buf[log_len] = '\0';

+
if (len > sizeof(frag->buf) - 1) {

va_start(args, fmt);

tmp = kvasprintf(GFP_KERNEL, fmt, args);

@@ -212,24 +222,15 @@ void kunit_log_append(struct list_head *log, const char *fmt, ...)

return;
}

- frag = list_last_entry(log, struct kunit_log_frag, list);
- log_len = strlen(frag->buf);
- len_left = sizeof(frag->buf) - log_len - 1;
-

- if (len > len_left) {
- frag = kunit_log_extend(log);
- if (!frag)
- return;
-
- len_left = sizeof(frag->buf) - 1;
- log_len = 0;
- }

+ frag = kunit_log_extend(log);
+ if (!frag)

+ return;

/* Print formatted line to the log */
va_start(args, fmt);

- vsnprintf(frag->buf + log_len, min(len, len_left), fmt, args);
+ vsnprintf(frag->buf, sizeof(frag->buf) - 1, fmt, args);
va_end(args);
-
+out_newline:

/* Add newline to end of log if not already present. */

kunit_log_newline(frag);
}
--
2.30.2

[Rae Moar]

On Wed, Aug 9, 2023 at 11:54 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>
> Re-implement the log buffer as a list of buffer fragments that can
> be extended as the size of the log info grows.
>
> When using parameterization the test case can run many times and create
> a large amount of log. It's not really practical to keep increasing the
> size of the fixed buffer every time a test needs more space. And a big
> fixed buffer wastes memory.
>
> The original char *log pointer is replaced by a pointer to a list of
> struct kunit_log_frag, each containing a fixed-size buffer.
>
> kunit_log_append() now attempts to append to the last kunit_log_frag in
> the list. If there isn't enough space it will append a new kunit_log_frag
> to the list. This simple implementation does not attempt to completely
> fill the buffer in every kunit_log_frag.
>
> The 'log' member of kunit_suite, kunit_test_case and kunit_suite must be a
> pointer because the API of kunit_log() requires that is the same type in
> all three structs. As kunit.log is a pointer to the 'log' of the current
> kunit_case, it must be a pointer in the other two structs.
>
> The existing kunit-test.c log tests have been updated to build against the
> new fragmented log implementation. If the test fails the new function
> get_concatenated_log() constructs a single contiguous string from the
> log fragments so that the whole log can be emitted in the failure
> message.

Hello!

All the tests now pass for me and this patch now looks good to me. I
have tested it and it seems to be working well.

I just have a few nits below. But I am overall happy with this patch.

Reviewed-by: Rae Moar <rm...@google.com>

-Rae

To respond to some of the comments from the previous version, I am
fine with the list_head being a pointer considering there can only be
one list_head in this struct definition.

I wonder if we could change the name of p to be a bit more
descriptive. Maybe concat_log?

> +
> + list_for_each_entry(frag, log, list)
> + len += strlen(frag->buf);
> +
> + len++; /* for terminating '\0' */
> + p = kunit_kzalloc(test, len, GFP_KERNEL);
> +
> + list_for_each_entry(frag, log, list)
> + strlcat(p, frag->buf, len);
> +
> + return p;
> +}
> +
> static void kunit_log_test(struct kunit *test)
> {
> struct kunit_suite suite;
> + struct kunit_log_frag *frag;
>
> - suite.log = kunit_kzalloc(test, KUNIT_LOG_SIZE, GFP_KERNEL);
> + suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
> KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
> + INIT_LIST_HEAD(suite.log);

This section of the test is pretty dense. I would love to see at least
one comment in this section. Maybe it could be here and say something
like:
"/* Allocate, initialize, and then add the first fragment of log */"

This test passes when CONFIG_KUNIT_DEBUGFS=n while most of the other
tests are skipped. Should this test be skipped instead?

I'm assuming since the assert/expect statements at the beginning are
run even if CONFIG_KUNIT_DEBUGFS=n, this test should not be skipped
but I just wanted to check.

> #endif
> @@ -558,11 +586,15 @@ static void kunit_log_test(struct kunit *test)
>
> static void kunit_log_newline_test(struct kunit *test)
> {
> + struct kunit_log_frag *frag;
> +
> kunit_info(test, "Add newline\n");
> if (test->log) {

This is a small nit but I would prefer that the if statements to
decide whether CONFIG_KUNIT_DEBUGFS is enabled were uniform. So I
would prefer that we choose between if (test->log) and #ifdef
CONFIG_KUNIT_DEBUGFS. I think we originally chose to use if
(test->log) here to avoid the compile-time #ifdef.

I would love to see just a short comment describing kunit_log_extend
to prevent confusion with kunit_log_append.

[Rae Moar]

On Wed, Aug 9, 2023 at 11:54 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Add test cases for the dynamically-extending log buffer.
>
> kunit_log_init_frag_test() tests that kunit_init_log_frag() correctly
> initializes new struct kunit_log_frag.
>
> kunit_log_extend_test_1() logs a series of numbered lines then tests
> that the resulting log contains all the lines.
>
> kunit_log_extend_test_2() logs a large number of lines of varying length
> to create many fragments, then tests that all lines are present.
>
> kunit_log_newline_test() has a new test to append a line that is exactly
> the length of the available space in the current fragment and check that
> the resulting log has a trailing '\n'.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

Hello!

These tests now pass for me. Thanks!

I do have a few comments below mostly regarding comments and a few
clarifying questions.

-Rae

Why is the fragment getting filled here with memset? Should this be
tested? Feel free to let me know, I'm just uncertain.

> kunit_init_log_frag(frag);
> KUNIT_EXPECT_EQ(test, frag->buf[0], '\0');
> + KUNIT_EXPECT_TRUE(test, list_is_first(&frag->list, &frag->list));
> + KUNIT_EXPECT_TRUE(test, list_is_last(&frag->list, &frag->list));
> +}
> +
> +static void kunit_log_test(struct kunit *test)
> +{
> + struct kunit_suite suite;
> + struct kunit_log_frag *frag;
> +
> + suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
> + INIT_LIST_HEAD(suite.log);
> + frag = kunit_kzalloc(test, sizeof(*frag), GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
> + kunit_init_log_frag(frag);
> list_add_tail(&frag->list, suite.log);
>
> kunit_log(KERN_INFO, test, "put this in log.");
> @@ -586,23 +607,168 @@ static void kunit_log_test(struct kunit *test)
>
> static void kunit_log_newline_test(struct kunit *test)
> {
> + struct kunit_suite suite;
> struct kunit_log_frag *frag;
> + char *p;

Similar to last email, could we change p to be a more descriptive name
such as concat_log?

>
> kunit_info(test, "Add newline\n");
> if (test->log) {
> frag = list_first_entry(test->log, struct kunit_log_frag, list);
> KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(frag->buf, "Add newline\n"),
> "Missing log line, full log:\n%s",
> - get_concatenated_log(test, test->log));
> + get_concatenated_log(test, test->log, NULL));
> KUNIT_EXPECT_NULL(test, strstr(frag->buf, "Add newline\n\n"));
> +

Should this section of kunit_log_newline_test be separated into a new
test? This test seems a bit long and seems to have two distinct
sections?

> + suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
> + INIT_LIST_HEAD(suite.log);

I would love to see a comment here to explain and break up this
section similar to the comment from the previous email.

> + frag = kunit_kzalloc(test, sizeof(*frag), GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
> + kunit_init_log_frag(frag);
> + list_add_tail(&frag->list, suite.log);
> +
> + /* String that exactly fills fragment leaving no room for \n */
> + memset(frag->buf, 0, sizeof(frag->buf));
> + memset(frag->buf, 'x', sizeof(frag->buf) - 9);
> + kunit_log_append(suite.log, "12345678");
> + p = get_concatenated_log(test, suite.log, NULL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
> + KUNIT_EXPECT_NOT_NULL_MSG(test, strstr(p, "x12345678\n"),
> + "Newline not appended when fragment is full. Log is:\n'%s'", p);
> } else {
> kunit_skip(test, "only useful when debugfs is enabled");
> }
> }
>
> +static void kunit_log_extend_test_1(struct kunit *test)

In general, I would really like to see more comments in the next two
tests describing the test behavior. I would prefer a comment for each
of the while/do-while loops below. I just found the behavior to be
slightly confusing to understand without comments (although I do
appreciate the comments that are in kunit_log_extend_test_2).

Also, I really appreciate how detailed these tests are.

Another potential idea is to rename these two tests to be
kunit_log_extend_test() and kunit_log_rand_extend_test() instead to be
more descriptive?

> +{
> +#ifdef CONFIG_KUNIT_DEBUGFS
> + struct kunit_suite suite;
> + struct kunit_log_frag *frag;
> + char line[60];
> + char *p, *pn;

Similar to before, could we change p and pn to be slightly more
descriptive names? Maybe concat_log and newline_ptr or newline_log or
newline_char?

> + size_t len, n;
> + int num_lines, num_frags, i;
> +
> + suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
> + INIT_LIST_HEAD(suite.log);
> + frag = kunit_kzalloc(test, sizeof(*frag), GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
> + kunit_init_log_frag(frag);
> + list_add_tail(&frag->list, suite.log);
> +
> + i = 0;
> + len = 0;
> + do {
> + n = snprintf(line, sizeof(line),
> + "The quick brown fox jumps over the lazy penguin %d\n", i);
> + KUNIT_ASSERT_LT(test, n, sizeof(line));
> + kunit_log_append(suite.log, line);
> + ++i;
> + len += n;
> + } while (len < (sizeof(frag->buf) * 30));

Are we trying to restrict the num_frags to less than 30? And then we
could check that with a KUNIT_EXPECT? Currently, the num_frags are
just above 30. That is ok too. I just was wondering if this was
intentional? (Same as kunit_log_extend_test_2)

Similar to above, could p and pn be renamed to be more descriptive?

> + size_t len;
> + int num_lines, num_frags, n, i;
> +
> + suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
> + INIT_LIST_HEAD(suite.log);
> + frag = kunit_kzalloc(test, sizeof(*frag), GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
> + kunit_init_log_frag(frag);
> + list_add_tail(&frag->list, suite.log);
> +
> + /* Build log line of varying content */
> + line[0] = '\0';
> + i = 0;
> + do {
> + char tmp[9];
> +
> + snprintf(tmp, sizeof(tmp), "%x", i++);
> + len = strlcat(line, tmp, sizeof(line));
> + } while (len < sizeof(line) - 1);

Could there be an expectation statement here to check the line has
been properly filled. Maybe checking the length?

Is it possible for this to be too many lines instead? Should this
comment instead be "Unexpected number of lines". Also could we have a
similar message for the test above for this expectation regarding the
number of lines.

[Rae Moar]

On Wed, Aug 9, 2023 at 11:54 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> If a log string is the exact length of a log fragment buffer
> kunit_log_append() should now exactly fill that fragment without
> extending the log.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

Hello!

This test looks good to me. I have tested it and it seems to be working well.

I appreciate all of the assert and expect statements. I do have one
comment below.

Although, I would be happy to set this as reviewed by me after that
comment is responded to.

Thanks!
-Rae

Should this be an EXPECT instead? It doesn't seem like the test needs
to fail immediately if this fails. Let me know what you think.

[Richard Fitzgerald]

...

>> +static char *get_concatenated_log(struct kunit *test, const struct list_head *log)
>> +{
>> + struct kunit_log_frag *frag;
>> + size_t len = 0;
>> + char *p;
>
> I wonder if we could change the name of p to be a bit more
> descriptive. Maybe concat_log?

I'll do that.

>> +
>> + list_for_each_entry(frag, log, list)
>> + len += strlen(frag->buf);
>> +
>> + len++; /* for terminating '\0' */
>> + p = kunit_kzalloc(test, len, GFP_KERNEL);
>> +
>> + list_for_each_entry(frag, log, list)
>> + strlcat(p, frag->buf, len);
>> +
>> + return p;
>> +}
>> +
>> static void kunit_log_test(struct kunit *test)
>> {
>> struct kunit_suite suite;
>> + struct kunit_log_frag *frag;
>>
>> - suite.log = kunit_kzalloc(test, KUNIT_LOG_SIZE, GFP_KERNEL);
>> + suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
>> KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
>> + INIT_LIST_HEAD(suite.log);
>
> This section of the test is pretty dense. I would love to see at least
> one comment in this section. Maybe it could be here and say something
> like:
> "/* Allocate, initialize, and then add the first fragment of log */"

Good. You like comments. So do I.
But many people hate them so when I'm starting on a new subsystem I go
with however the existing code looks. And the bits of kunit I've looked
at are light on comments.

...

That the existing code, which I didn't change.
I think the best thing here would be to move the fragment testing into a
new test case.

>> #endif
>> @@ -558,11 +586,15 @@ static void kunit_log_test(struct kunit *test)
>>
>> static void kunit_log_newline_test(struct kunit *test)
>> {
>> + struct kunit_log_frag *frag;
>> +
>> kunit_info(test, "Add newline\n");
>> if (test->log) {
>
> This is a small nit but I would prefer that the if statements to
> decide whether CONFIG_KUNIT_DEBUGFS is enabled were uniform. So I
> would prefer that we choose between if (test->log) and #ifdef
> CONFIG_KUNIT_DEBUGFS. I think we originally chose to use if
> (test->log) here to avoid the compile-time #ifdef.

Actually the existing code did it both ways. The newline test used
if (test->log) - but this makes sense because it actually does testing
on test->log.

The original kunit_log_test() used #ifdef CONFIG_KUNIT_DEBUGFS. I based
my new functions on kunit_log_test(). But I can change them. Would you
prefer

if (!test->log)
skip

or
if (!IS_ENABLED(CONFIG_KUNIT_DEBUGFS))
skip

[Richard Fitzgerald]

On 9/8/23 22:10, Rae Moar wrote:

> On Wed, Aug 9, 2023 at 11:54 AM Richard Fitzgerald
> <r...@opensource.cirrus.com> wrote:
>>
>> Add test cases for the dynamically-extending log buffer.
>>
>> kunit_log_init_frag_test() tests that kunit_init_log_frag() correctly
>> initializes new struct kunit_log_frag.
>>
>> kunit_log_extend_test_1() logs a series of numbered lines then tests
>> that the resulting log contains all the lines.
>>
>> kunit_log_extend_test_2() logs a large number of lines of varying length
>> to create many fragments, then tests that all lines are present.
>>
>> kunit_log_newline_test() has a new test to append a line that is exactly
>> the length of the available space in the current fragment and check that
>> the resulting log has a trailing '\n'.
>>
>> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
>
> Hello!
>
> These tests now pass for me. Thanks!
>
> I do have a few comments below mostly regarding comments and a few
> clarifying questions.
>
> -Rae

...

>> +static void kunit_log_init_frag_test(struct kunit *test)
>> {
>> - struct kunit_suite suite;
>> struct kunit_log_frag *frag;
>>
>> - suite.log = kunit_kzalloc(test, sizeof(*suite.log), GFP_KERNEL);
>> - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
>> - INIT_LIST_HEAD(suite.log);
>> frag = kunit_kmalloc(test, sizeof(*frag), GFP_KERNEL);
>> KUNIT_ASSERT_NOT_ERR_OR_NULL(test, frag);
>> + memset(frag, 0x5a, sizeof(*frag));
>> +
>
> Why is the fragment getting filled here with memset? Should this be
> tested? Feel free to let me know, I'm just uncertain.

I'll add a comment in V4. It's to prove that kunit_init_log_frag()
really did change something. kzalloc() is no good for this because we
want to see that kunit_log_frag() zeroed buf[0].

...

>> kunit_info(test, "Add newline\n");
>> if (test->log) {
>> frag = list_first_entry(test->log, struct kunit_log_frag, list);
>> KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(frag->buf, "Add newline\n"),
>> "Missing log line, full log:\n%s",
>> - get_concatenated_log(test, test->log));
>> + get_concatenated_log(test, test->log, NULL));
>> KUNIT_EXPECT_NULL(test, strstr(frag->buf, "Add newline\n\n"));
>> +
>
> Should this section of kunit_log_newline_test be separated into a new
> test? This test seems a bit long and seems to have two distinct
> sections?

Yes, it makes sense to add a separate test case for when newlines cause
the log to extend.

...

> Another potential idea is to rename these two tests to be
> kunit_log_extend_test() and kunit_log_rand_extend_test() instead to be
> more descriptive?

TBH I had trouble thinking of a short description. But I'll spend some
time thinking about naming.

...

>> + do {
>> + n = snprintf(line, sizeof(line),
>> + "The quick brown fox jumps over the lazy penguin %d\n", i);
>> + KUNIT_ASSERT_LT(test, n, sizeof(line));
>> + kunit_log_append(suite.log, line);
>> + ++i;
>> + len += n;
>> + } while (len < (sizeof(frag->buf) * 30));
>
> Are we trying to restrict the num_frags to less than 30? And then we
> could check that with a KUNIT_EXPECT? Currently, the num_frags are
> just above 30. That is ok too. I just was wondering if this was
> intentional? (Same as kunit_log_extend_test_2)

I'll comment on this in V4.
It's just trying to create "a lot" of data without assuming exactly
how kunit_log_append() breaks up the lines across fragments. I don't
want to have to keep changing this code if the fragmenting algorithm
changes slightly. So the idea is to generate "about 30" buffers worth.
I don't mind if it's a bit more, or a bit less. It's done this way,
instead of just counting how many fragments were created, to prevent
getting into an infinite loop if for some reason kunit_log_append()
fails to add fragments.

...

>> + /* Build log line of varying content */
>> + line[0] = '\0';
>> + i = 0;
>> + do {
>> + char tmp[9];
>> +
>> + snprintf(tmp, sizeof(tmp), "%x", i++);
>> + len = strlcat(line, tmp, sizeof(line));
>> + } while (len < sizeof(line) - 1);
>
> Could there be an expectation statement here to check the line has
> been properly filled. Maybe checking the length?

Yes

>> + prandom_seed_state(&rnd, 3141592653589793238ULL);
>> + i = 0;
>> + n = 0;
>> + while ((pn = strchr(p, '\n')) != NULL) {
>> + *pn = '\0';
>> + KUNIT_EXPECT_STREQ(test, p, &line[i]);
>> + p = pn + 1;
>> + n++;
>> + i = prandom_u32_state(&rnd) % (sizeof(line) - 1);
>> + }
>> + KUNIT_EXPECT_EQ_MSG(test, n, num_lines, "Not enough lines.");
>
> Is it possible for this to be too many lines instead? Should this
> comment instead be "Unexpected number of lines". Also could we have a
> similar message for the test above for this expectation regarding the
> number of lines.

Fair point. It's only found that the number of lines is wrong, it
could be less or more.

[Richard Fitzgerald]

I think ASSERT is appropriate here. This isn't testing anything
(unless you don't trust memset). It's ensuring that the test data
I generate is what I expect otherwise the following testing is
invalid.

This is redundant because the first 3 lines must produce the expected
string, but I put it in to prove to myself that I can do math and
decided to leave it in.

[David Gow]

On Wed, 9 Aug 2023 at 23:54, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>
> Add handling to kunit_log_append() for log messages that are longer than
> a single buffer fragment.
>
> The initial implementation of fragmented buffers did not change the logic
> of the original kunit_log_append(). A consequence was that it still had
> the original assumption that a log line will fit into one buffer.
>
> This patch checks for log messages that are larger than one fragment
> buffer. In that case, kvasprintf() is used to format it into a temporary
> buffer and that content is then split across as many fragments as
> necessary.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

I think this looks good (and is a long-overdue addition to the logging
functionality).

One thought I have (and I'm kicking myself for not thinking of it
earlier) is that this is starting to get very similar to the "string
stream" functionality in lib/kunit/string-stream.{h,c}. Now, I
actually think this implementation is much more efficient (using
larger fragments, whereas the string stream uses variable-sized ones).
Particularly since there are a lot of cases where string streams are
created, converted to a string, and then logged, there's almost
certainly a bunch of redundant work being done here.

My gut feeling is that we should stick with this as-is, and maybe try
to either work out some better integration between string streams and
logging (to avoid that extra string allocation) or find some way of
combining them.

Regardless, this seems good as-is to me. There are some minor comments
below, but nothing disastrous. I'll let Rae have a look over the
various strscpy and strlcat calls, though, as while I did check them
carefully (and KASAN hasn't complained), it's always best to have as
many eyes on C string code as possible. :-)

But in my eyes, this is
Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

I think that this can only be true if kunit_log_extend() fails. If the
do/while loop ends naturally, then written != -E2BIG, as is the case
with the strscpy goto above.

Do you think it's cleaner to move the 'written = strlen(frag->buf)
into the if (!new_frag) statement within the loop?

> + written = strlen(frag->buf);
> +
> + p = &frag->buf[written - 1];
> + if (*p != '\n') {
> + if (strlcat(frag->buf, "\n", sizeof(frag->buf)) >= sizeof(frag->buf)) {
> + frag = kunit_log_extend(log);
> + if (!frag) {

A comment here could be useful. Something like "If we can't extend the
log to add a newline, truncate the last message".

Should we handle the case where kvasprintf() fails here and drop the
log message?

[Richard Fitzgerald]

I completely failed to notice string_stream. I could re-implement this
to use string_stream. I wonder whether appending newlines gets
a bit inefficient with the current string_stream implementation.
Could add newline support to string_stream and alloc one extra byte for
each fragment just in case we need to add a newline.

The string_stream implementation would waste a lot a memory if you log
many short lines. My current code wastes memory if you log lots of lines
that don't fit in available space in the current fragment - though it's
simple to shuffle the formatted string backwards to fill up the previous
fragment (I just haven't done that yet).

[Rae Moar]

Hello!

I have tested and taken a look at this and it looks overall good to
me. I think all of the string copying and concatenating is right.

Other than David's comments below, especially whether we should do
this with string-stream, I am pretty happy to accept this as is.

Thanks!
Rae

[Rae Moar]

On Wed, Aug 9, 2023 at 10:54 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>
> Add kunit_log_long_line_test() to test that logging a line longer than
> the buffer fragment size doesn't truncate the line.
>
> Add extra tests to kunit_log_newline_test() for lines longer than the
> buffer fragment size.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

Hello!

This test looks good to me. I have included just a few comments below.

Reviewed-by: Rae Moar <rm...@google.com>

Thanks!
-Rae

> ---
> lib/kunit/kunit-test.c | 84 +++++++++++++++++++++++++++++++++++++++++-
> 1 file changed, 83 insertions(+), 1 deletion(-)
>
> diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
> index 9ac81828d018..c079550c3afd 100644
> --- a/lib/kunit/kunit-test.c
> +++ b/lib/kunit/kunit-test.c
> @@ -609,7 +609,7 @@ static void kunit_log_newline_test(struct kunit *test)
> {
> struct kunit_suite suite;
> struct kunit_log_frag *frag;
> - char *p;
> + char *p, *line;
>
> kunit_info(test, "Add newline\n");
> if (test->log) {
> @@ -635,6 +635,33 @@ static void kunit_log_newline_test(struct kunit *test)
> KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
> KUNIT_EXPECT_NOT_NULL_MSG(test, strstr(p, "x12345678\n"),
> "Newline not appended when fragment is full. Log is:\n'%s'", p);
> + kunit_kfree(test, p);
> +

I really like the thoroughness of this test. However, I do wonder if
this newline test could be broken into at least 2 parts as the test is
quite long with all these additions. I spoke on this in a previous
patch and just wanted to touch on it here as well.

> + /* String that is much longer than a fragment */
> + line = kunit_kzalloc(test, sizeof(frag->buf) * 6, GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, line);
> + memset(line, 'x', (sizeof(frag->buf) * 6) - 1);
> + kunit_log_append(suite.log, "%s", line);
> + p = get_concatenated_log(test, suite.log, NULL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p);
> + KUNIT_EXPECT_EQ(test, p[strlen(p) - 1], '\n');
> + KUNIT_EXPECT_NULL(test, strstr(p, "\n\n"));
> + kunit_kfree(test, p);
> +

I would also consider adding comments between these three cases to
describe their differences and maybe what the desired behavior would
be.

I was a little worried about including a randomized string but since
it does not need to be reproduced here it should be fine. I also
haven't seen any issues with the tests with the randomized strings
being nondeterministic.

[Rae Moar]

On Wed, Aug 9, 2023 at 10:54 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Add a test that logging a string containing only a newline appends
> one newline to the log.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

This is a really great idea to include a test for the simple addition
of only a newline character.

This patch looks good to me. The only comment is that the newline test
is quite large and could be broken up as I said on the previous patch.

Reviewed-by: Rae Moar <rm...@google.com>

Thanks!
-Rae

[Rae Moar]

On Wed, Aug 9, 2023 at 10:54 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> It's wasteful to call vsnprintf() only to figure out the length of the
> string. The string might fit in the available buffer space but we have to
> vsnprintf() again to do that.
>
> Instead, attempt to format the string to the available buffer at the same
> time as finding the string length. Only if the string didn't fit the
> available space is it necessary to extend the log and format the string
> again to a new fragment buffer.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

Hello!

I find this version slightly more confusing but I realize that the
extra vsnprintf() call is unnecessary. So I am ok with this version
except for one question below.

I am curious what David Gow thinks and it would also be good to have
another set of eyes here.

My testing of this showed no problems.

Thanks!
-Rae

Are we meant to be using the remainder of the last fragment to its
capacity or just start again with a new fragment and leave some of the
last fragment empty? I worry we abandoned using the last fragment or
is that the intention? Let me know if I understand this correctly.

[David Gow]

Yeah: I think your implementation here is overall better than the
string_stream one. string_stream might handle concurrency a bit
better, which would be nice to have as people start wanting to try
multithreaded tests.

I think the ideal solution is:
- Update string_stream to basically use this implementation.
- Update the logging code to then use this via the string_stream API
(probably with some tweaks to handle newlines)
- Optimize the string_stream append implementation to not create a
temporary string, as string streams are written to logs often. (If you
were prepared to allow string stream fragments to have variable
lengths, and do some ownership shenanigans, this could even become
O(1), though I suspect it's not worth the added complexity.)

That being said, I don't think we'd need to land all of that at once.
Even if we ported to the suboptimal string_stream API now (which would
still gain us the extensible log and some concurrency support), and
optimized string_stream later if it turned out to be tricky, that'd be
fine. (I wouldn't want to hold this up if changing string_stream was
regressing the other code which uses it, for example.)

How does that sound?

-- David

[David Gow]

On Wed, 9 Aug 2023 at 23:54, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> It's wasteful to call vsnprintf() only to figure out the length of the
> string. The string might fit in the available buffer space but we have to
> vsnprintf() again to do that.
>
> Instead, attempt to format the string to the available buffer at the same
> time as finding the string length. Only if the string didn't fit the
> available space is it necessary to extend the log and format the string
> again to a new fragment buffer.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

This looks good.

The only case I'm not totally convinced about is the last one, where
the message doesn't fit in the current log fragment, but is not a
whole fragment itself. I'm not sure if the added efficiency of not
doing a second vsprintf() is worth the memory fragmentation of always
starting a new fragment: the worst case where a log message is just
over half the length of frag->buf would result in every message being
in its own fragment (which would not necessarily have a consistent
size), and memory use would be ~doubled.

But I could accept it either way: until there's a real-world example
which strongly benefits from either implementation, let's go with
whatever we have working.

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

[Richard Fitzgerald]

This patch chain changes the logging implementation to use string_stream
so that the log will grow dynamically.

The first 8 patches add test code for string_stream, and make some
changes to string_stream needed to be able to use it for the log.

The final patch adds a performance report of string_stream.

CHANGES SINCE V3:

Completely rewritten to use string_stream instead of implementing a
separate extending-buffer implementation for logging.

I have used the performance test from the final patch on my original
fixed-size-fragment implementation from V3 to get a comparison of the
two implementations (run on i3-8145UE CPU @ 2.20GHz):

string_stream V3 fixed-size-buffer
Time elapsed: 7748 us 3251 us
Total string length: 573890 573890
Bytes requested: 823994 728336
Actual bytes allocated: 1061440 728352

I don't think the difference is enough to be worth complicating the
string_stream implementation with my fixed-fragment implementation from
V3 of this patch chain.

Richard Fitzgerald (10):
kunit: string-stream: Improve testing of string_stream
kunit: string-stream: Don't create a fragment for empty strings
kunit: string-stream: Add cases for adding empty strings to a
string_stream
kunit: string-stream: Add option to make all lines end with newline
kunit: string-stream: Add cases for string_stream newline appending
kunit: string-stream: Pass struct kunit to string_stream_get_string()
kunit: string-stream: Decouple string_stream from kunit
kunit: string-stream: Add test for freeing resource-managed
string_stream
kunit: Use string_stream for test log
kunit: string-stream: Test performance of string_stream

include/kunit/test.h | 14 +-
lib/kunit/Makefile | 5 +-
lib/kunit/debugfs.c | 36 ++-
lib/kunit/kunit-test.c | 52 +---
lib/kunit/log-test.c | 72 ++++++
lib/kunit/string-stream-test.c | 447 +++++++++++++++++++++++++++++++--
lib/kunit/string-stream.c | 129 +++++++---
lib/kunit/string-stream.h | 22 +-
lib/kunit/test.c | 48 +---
9 files changed, 656 insertions(+), 169 deletions(-)
create mode 100644 lib/kunit/log-test.c

--
2.30.2

[Richard Fitzgerald]

Adds string_stream_append_empty_string_test() to test that adding an
empty string to a string_stream doesn't create a new empty fragment.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream-test.c | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 1d46d5f06d2a..efe13e3322b5 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -206,11 +206,32 @@ static void string_stream_append_test(struct kunit *test)
KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream_1), stream_2_content);
}

+/* Adding an empty string should not create a fragment. */
+static void string_stream_append_empty_string_test(struct kunit *test)
+{
+ struct string_stream *stream;
+
+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+ /* Formatted empty string */
+ string_stream_add(stream, "%s", "");
+ KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
+ KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));
+
+ /* Adding an empty string to a non-empty stream */
+ string_stream_add(stream, "Add this line");
+ string_stream_add(stream, "%s", "");
+ KUNIT_EXPECT_EQ(test, list_count_nodes(&stream->fragments), 1);
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream), "Add this line");
+}
+
static struct kunit_case string_stream_test_cases[] = {
KUNIT_CASE(string_stream_init_test),
KUNIT_CASE(string_stream_line_add_test),
KUNIT_CASE(string_stream_variable_length_line_test),
KUNIT_CASE(string_stream_append_test),
+ KUNIT_CASE(string_stream_append_empty_string_test),
{}
};

--
2.30.2

[Richard Fitzgerald]

If the result of the formatted string is an empty string just return
instead of creating an empty fragment.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream.c | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index cc32743c1171..ed24d86af9f5 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -50,11 +50,17 @@ int string_stream_vadd(struct string_stream *stream,
/* Make a copy because `vsnprintf` could change it */
va_copy(args_for_counting, args);

- /* Need space for null byte. */
- len = vsnprintf(NULL, 0, fmt, args_for_counting) + 1;
+ /* Evaluate length of formatted string */
+ len = vsnprintf(NULL, 0, fmt, args_for_counting);

va_end(args_for_counting);

+ if (len == 0)
+ return 0;
+
+ /* Need space for null byte. */
+ len++;
+
frag_container = alloc_string_stream_fragment(stream->test,
len,
stream->gfp);
--
2.30.2

[Richard Fitzgerald]

Replace the minimal tests with more-thorough testing.

string_stream_init_test() tests that struct string_stream is
initialized correctly.

string_stream_line_add_test() adds a series of numbered lines and
checks that the resulting string contains all the lines.

string_stream_variable_length_line_test() adds a large number of

lines of varying length to create many fragments, then tests that all
lines are present.

string_stream_append_test() tests various cases of using
string_stream_append() to append the content of one stream to another.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream-test.c | 200 ++++++++++++++++++++++++++++++---
1 file changed, 184 insertions(+), 16 deletions(-)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 110f3a993250..1d46d5f06d2a 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -11,38 +11,206 @@

#include "string-stream.h"

-static void string_stream_test_empty_on_creation(struct kunit *test)
+/* string_stream object is initialized correctly. */
+static void string_stream_init_test(struct kunit *test)
{
- struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL);

+ struct string_stream *stream;
+
+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+

+ KUNIT_EXPECT_EQ(test, stream->length, 0);
+ KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));
+ KUNIT_EXPECT_PTR_EQ(test, stream->test, test);
+ KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL);

KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
}

-static void string_stream_test_not_empty_after_add(struct kunit *test)
+/*
+ * Add a series of lines to a string_stream. Check that all lines
+ * appear in the correct order and no characters are dropped.
+ */
+static void string_stream_line_add_test(struct kunit *test)
{
- struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL);
+ struct string_stream *stream;
+ char line[60];
+ char *concat_string, *pos, *string_end;
+ size_t len, total_len;
+ int num_lines, i;

- string_stream_add(stream, "Foo");

+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

- KUNIT_EXPECT_FALSE(test, string_stream_is_empty(stream));
+ /* Add series of sequence numbered lines */
+ total_len = 0;
+ for (i = 0; i < 100; ++i) {
+ len = snprintf(line, sizeof(line),

+ "The quick brown fox jumps over the lazy penguin %d\n", i);
+

+ /* Sanity-check that our test string isn't truncated */
+ KUNIT_ASSERT_LT(test, len, sizeof(line));
+
+ string_stream_add(stream, line);
+ total_len += len;
+ }

+ num_lines = i;
+

+ concat_string = string_stream_get_string(stream);
+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test, concat_string);
+ KUNIT_EXPECT_EQ(test, strlen(concat_string), total_len);
+
+ /*
+ * Split the concatenated string at the newlines and check that
+ * all the original added strings are present.
+ */
+ pos = concat_string;
+ for (i = 0; i < num_lines; ++i) {
+ string_end = strchr(pos, '\n');
+ KUNIT_EXPECT_NOT_NULL(test, string_end);
+
+ /* Convert to NULL-terminated string */
+ *string_end = '\0';
+
+ snprintf(line, sizeof(line),
+ "The quick brown fox jumps over the lazy penguin %d", i);
+ KUNIT_EXPECT_STREQ(test, pos, line);
+
+ pos = string_end + 1;
+ }
+
+ /* There shouldn't be any more data after this */
+ KUNIT_EXPECT_EQ(test, strlen(pos), 0);
}

-static void string_stream_test_get_string(struct kunit *test)
+/* Add a series of lines of variable length to a string_stream. */
+static void string_stream_variable_length_line_test(struct kunit *test)
{
- struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL);
- char *output;
+ static const char line[] =
+ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ " 0123456789!$%^&*()_-+={}[]:;@'~#<>,.?/|";
+ struct string_stream *stream;
+ struct rnd_state rnd;
+ char *concat_string, *pos, *string_end;
+ size_t offset, total_len;
+ int num_lines, i;

- string_stream_add(stream, "Foo");
- string_stream_add(stream, " %s", "bar");

+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

- output = string_stream_get_string(stream);
- KUNIT_ASSERT_STREQ(test, output, "Foo bar");
+ /*
+ * Log many lines of varying lengths until we have created

+ * many fragments.
+ * The "randomness" must be repeatable.
+ */
+ prandom_seed_state(&rnd, 3141592653589793238ULL);

+ total_len = 0;
+ for (i = 0; i < 100; ++i) {
+ offset = prandom_u32_state(&rnd) % (sizeof(line) - 1);
+ string_stream_add(stream, "%s\n", &line[offset]);
+ total_len += sizeof(line) - offset;
+ }

+ num_lines = i;
+

+ concat_string = string_stream_get_string(stream);
+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test, concat_string);
+ KUNIT_EXPECT_EQ(test, strlen(concat_string), total_len);
+
+ /*
+ * Split the concatenated string at the newlines and check that
+ * all the original added strings are present.

+ */
+ prandom_seed_state(&rnd, 3141592653589793238ULL);

+ pos = concat_string;
+ for (i = 0; i < num_lines; ++i) {
+ string_end = strchr(pos, '\n');
+ KUNIT_EXPECT_NOT_NULL(test, string_end);
+
+ /* Convert to NULL-terminated string */
+ *string_end = '\0';
+
+ offset = prandom_u32_state(&rnd) % (sizeof(line) - 1);
+ KUNIT_EXPECT_STREQ(test, pos, &line[offset]);
+
+ pos = string_end + 1;
+ }
+
+ /* There shouldn't be any more data after this */
+ KUNIT_EXPECT_EQ(test, strlen(pos), 0);
+}
+
+/* Appending the content of one string stream to another. */
+static void string_stream_append_test(struct kunit *test)
+{
+ static const char * const strings_1[] = {
+ "one", "two", "three", "four", "five", "six",
+ "seven", "eight", "nine", "ten",
+ };
+ static const char * const strings_2[] = {
+ "ONE", "TWO", "THREE", "FOUR", "FIVE", "SIZE",
+ "SEVEN", "EIGHT", "NINE", "TEN",
+ };
+ struct string_stream *stream_1, *stream_2;
+ const char *original_content, *stream_2_content;
+ char *combined_content;
+ size_t combined_length;
+ int i;
+
+ stream_1 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);
+
+ stream_2 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2);
+
+ /* Append content of empty stream to empty stream */
+ string_stream_append(stream_1, stream_2);
+ KUNIT_EXPECT_EQ(test, strlen(string_stream_get_string(stream_1)), 0);
+
+ /* Add some data to stream_1 */
+ for (i = 0; i < ARRAY_SIZE(strings_1); ++i)
+ string_stream_add(stream_1, "%s\n", strings_1[i]);
+
+ original_content = string_stream_get_string(stream_1);
+
+ /* Append content of empty stream to non-empty stream */
+ string_stream_append(stream_1, stream_2);
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream_1), original_content);
+
+ /* Add some data to stream_2 */
+ for (i = 0; i < ARRAY_SIZE(strings_2); ++i)
+ string_stream_add(stream_2, "%s\n", strings_2[i]);
+
+ /* Append content of non-empty stream to non-empty stream */
+ string_stream_append(stream_1, stream_2);
+
+ /*
+ * End result should be the original content of stream_1 plus
+ * the content of stream_2.
+ */
+ stream_2_content = string_stream_get_string(stream_2);
+ combined_length = strlen(original_content) + strlen(stream_2_content);
+ combined_length++; /* for terminating \0 */
+ combined_content = kunit_kmalloc(test, combined_length, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, combined_content);
+ snprintf(combined_content, combined_length, "%s%s", original_content, stream_2_content);
+
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream_1), combined_content);
+
+ /* Append content of non-empty stream to empty stream */
+ string_stream_destroy(stream_1);
+
+ stream_1 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);
+
+ string_stream_append(stream_1, stream_2);
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream_1), stream_2_content);

}

static struct kunit_case string_stream_test_cases[] = {

- KUNIT_CASE(string_stream_test_empty_on_creation),
- KUNIT_CASE(string_stream_test_not_empty_after_add),
- KUNIT_CASE(string_stream_test_get_string),
+ KUNIT_CASE(string_stream_init_test),
+ KUNIT_CASE(string_stream_line_add_test),
+ KUNIT_CASE(string_stream_variable_length_line_test),
+ KUNIT_CASE(string_stream_append_test),
{}
};

--
2.30.2

[Richard Fitzgerald]

Add an optional feature to string_stream that will append a newline to
any added string that does not already end with a newline. The purpose
of this is so that string_stream can be used to collect log lines.

This is enabled/disabled by calling string_stream_set_append_newlines().

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream.c | 28 +++++++++++++++++++++-------
lib/kunit/string-stream.h | 7 +++++++
2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index ed24d86af9f5..1dcf6513b692 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -44,32 +44,46 @@ int string_stream_vadd(struct string_stream *stream,
va_list args)
{
struct string_stream_fragment *frag_container;
- int len;
+ int buf_len, result_len;
va_list args_for_counting;

/* Make a copy because `vsnprintf` could change it */
va_copy(args_for_counting, args);

/* Evaluate length of formatted string */

- len = vsnprintf(NULL, 0, fmt, args_for_counting);

+ buf_len = vsnprintf(NULL, 0, fmt, args_for_counting);

va_end(args_for_counting);

- if (len == 0)
+ if (buf_len == 0)
return 0;

+ /* Reserve one extra for possible appended newline. */
+ if (stream->append_newlines)
+ buf_len++;

+
/* Need space for null byte. */

- len++;
+ buf_len++;

frag_container = alloc_string_stream_fragment(stream->test,
- len,
+ buf_len,
stream->gfp);
if (IS_ERR(frag_container))
return PTR_ERR(frag_container);

- len = vsnprintf(frag_container->fragment, len, fmt, args);
+ if (stream->append_newlines) {
+ /* Don't include reserved newline byte in writeable length. */
+ result_len = vsnprintf(frag_container->fragment, buf_len - 1, fmt, args);
+
+ /* Append newline if necessary. */
+ if (frag_container->fragment[result_len - 1] != '\n')
+ result_len = strlcat(frag_container->fragment, "\n", buf_len);
+ } else {
+ result_len = vsnprintf(frag_container->fragment, buf_len, fmt, args);
+ }
+
spin_lock(&stream->lock);
- stream->length += len;
+ stream->length += result_len;
list_add_tail(&frag_container->node, &stream->fragments);
spin_unlock(&stream->lock);

diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h
index b669f9a75a94..048930bf97f0 100644
--- a/lib/kunit/string-stream.h
+++ b/lib/kunit/string-stream.h
@@ -25,6 +25,7 @@ struct string_stream {
spinlock_t lock;
struct kunit *test;
gfp_t gfp;
+ bool append_newlines;
};

struct kunit;
@@ -47,4 +48,10 @@ bool string_stream_is_empty(struct string_stream *stream);

void string_stream_destroy(struct string_stream *stream);

+static inline void string_stream_set_append_newlines(struct string_stream *stream,
+ bool append_newlines)
+{
+ stream->append_newlines = append_newlines;
+}
+
#endif /* _KUNIT_STRING_STREAM_H */
--
2.30.2

[Richard Fitzgerald]

Pass a struct kunit* and gfp_t to string_stream_get_string(). Allocate
the returned buffer using these instead of using the stream->test and
stream->gfp.

This is preparation for removing the dependence of string_stream on
struct kunit, so that string_stream can be used for the debugfs log.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream-test.c | 26 +++++++++++++++-----------
lib/kunit/string-stream.c | 8 ++++----
lib/kunit/string-stream.h | 3 ++-
lib/kunit/test.c | 2 +-
4 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 46c2ac162fe8..8a30bb7d5fb7 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -57,7 +57,7 @@ static void string_stream_line_add_test(struct kunit *test)
}
num_lines = i;

- concat_string = string_stream_get_string(stream);
+ concat_string = string_stream_get_string(test, stream, GFP_KERNEL);
KUNIT_EXPECT_NOT_ERR_OR_NULL(test, concat_string);
KUNIT_EXPECT_EQ(test, strlen(concat_string), total_len);

@@ -113,7 +113,7 @@ static void string_stream_variable_length_line_test(struct kunit *test)
}
num_lines = i;

- concat_string = string_stream_get_string(stream);
+ concat_string = string_stream_get_string(test, stream, GFP_KERNEL);
KUNIT_EXPECT_NOT_ERR_OR_NULL(test, concat_string);
KUNIT_EXPECT_EQ(test, strlen(concat_string), total_len);

@@ -165,17 +165,18 @@ static void string_stream_append_test(struct kunit *test)

/* Append content of empty stream to empty stream */

string_stream_append(stream_1, stream_2);
- KUNIT_EXPECT_EQ(test, strlen(string_stream_get_string(stream_1)), 0);
+ KUNIT_EXPECT_EQ(test, strlen(string_stream_get_string(test, stream_1, GFP_KERNEL)), 0);

/* Add some data to stream_1 */

for (i = 0; i < ARRAY_SIZE(strings_1); ++i)

string_stream_add(stream_1, "%s\n", strings_1[i]);

- original_content = string_stream_get_string(stream_1);
+ original_content = string_stream_get_string(test, stream_1, GFP_KERNEL);

/* Append content of empty stream to non-empty stream */

string_stream_append(stream_1, stream_2);
- KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream_1), original_content);
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(test, stream_1, GFP_KERNEL),
+ original_content);

/* Add some data to stream_2 */

for (i = 0; i < ARRAY_SIZE(strings_2); ++i)

@@ -188,14 +189,15 @@ static void string_stream_append_test(struct kunit *test)

* End result should be the original content of stream_1 plus

* the content of stream_2.

*/
- stream_2_content = string_stream_get_string(stream_2);
+ stream_2_content = string_stream_get_string(test, stream_2, GFP_KERNEL);

combined_length = strlen(original_content) + strlen(stream_2_content);

combined_length++; /* for terminating \0 */

combined_content = kunit_kmalloc(test, combined_length, GFP_KERNEL);

KUNIT_ASSERT_NOT_ERR_OR_NULL(test, combined_content);

snprintf(combined_content, combined_length, "%s%s", original_content, stream_2_content);

- KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream_1), combined_content);
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(test, stream_1, GFP_KERNEL),
+ combined_content);

/* Append content of non-empty stream to empty stream */

string_stream_destroy(stream_1);
@@ -204,7 +206,8 @@ static void string_stream_append_test(struct kunit *test)
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);

string_stream_append(stream_1, stream_2);
- KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream_1), stream_2_content);
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(test, stream_1, GFP_KERNEL),
+ stream_2_content);

}

/* Adding an empty string should not create a fragment. */

@@ -224,7 +227,8 @@ static void string_stream_append_empty_string_test(struct kunit *test)

string_stream_add(stream, "Add this line");

string_stream_add(stream, "%s", "");
KUNIT_EXPECT_EQ(test, list_count_nodes(&stream->fragments), 1);
- KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream), "Add this line");
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(test, stream, GFP_KERNEL),
+ "Add this line");
}

/* Adding strings without automatic newline appending */
@@ -244,7 +248,7 @@ static void string_stream_no_auto_newline_test(struct kunit *test)
string_stream_add(stream, "Two\n");
string_stream_add(stream, "%s\n", "Three");
string_stream_add(stream, "Four");
- KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream),
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(test, stream, GFP_KERNEL),
"OneTwo\nThree\nFour");
}

@@ -271,7 +275,7 @@ static void string_stream_auto_newline_test(struct kunit *test)
string_stream_add(stream, "Five\n%s", "Six");
string_stream_add(stream, "Seven\n\n");
string_stream_add(stream, "Eight");
- KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream),
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(test, stream, GFP_KERNEL),
"One\nTwo\nThree\nFour\nFive\nSix\nSeven\n\nEight\n");
}

diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index 1dcf6513b692..eb673d3ea3bd 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -117,13 +117,14 @@ static void string_stream_clear(struct string_stream *stream)
spin_unlock(&stream->lock);
}

-char *string_stream_get_string(struct string_stream *stream)
+char *string_stream_get_string(struct kunit *test, struct string_stream *stream,
+ gfp_t gfp)
{
struct string_stream_fragment *frag_container;
size_t buf_len = stream->length + 1; /* +1 for null byte. */
char *buf;

- buf = kunit_kzalloc(stream->test, buf_len, stream->gfp);
+ buf = kunit_kzalloc(test, buf_len, gfp);
if (!buf)
return NULL;

@@ -140,8 +141,7 @@ int string_stream_append(struct string_stream *stream,
{
const char *other_content;

- other_content = string_stream_get_string(other);
-
+ other_content = string_stream_get_string(other->test, other, other->gfp);
if (!other_content)
return -ENOMEM;

diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h
index 048930bf97f0..6b4a747881c5 100644
--- a/lib/kunit/string-stream.h
+++ b/lib/kunit/string-stream.h
@@ -39,7 +39,8 @@ int __printf(2, 0) string_stream_vadd(struct string_stream *stream,
const char *fmt,
va_list args);

-char *string_stream_get_string(struct string_stream *stream);
+char *string_stream_get_string(struct kunit *test, struct string_stream *stream,
+ gfp_t gfp);

int string_stream_append(struct string_stream *stream,
struct string_stream *other);
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 49698a168437..520e15f49d0d 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -286,7 +286,7 @@ static void kunit_print_string_stream(struct kunit *test,
if (string_stream_is_empty(stream))
return;

- buf = string_stream_get_string(stream);
+ buf = string_stream_get_string(test, stream, GFP_KERNEL);
if (!buf) {
kunit_err(test,
"Could not allocate buffer, dumping stream:\n");
--
2.30.2

[Richard Fitzgerald]

string_stream_resource_free_test() allocates a resource-managed
string_stream and tests that raw_free_string_stream() is called when
the test resources are cleaned up.

string_stream_init_test() is extended to test allocating a
string_stream that is not resource-managed.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream-test.c | 117 ++++++++++++++++++++++++++++++++-
lib/kunit/string-stream.c | 3 +
2 files changed, 119 insertions(+), 1 deletion(-)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 437aa4b3179d..05bfade2bd8a 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -6,16 +6,27 @@
* Author: Brendan Higgins <brendan...@google.com>
*/

+#include <kunit/static_stub.h>
#include <kunit/test.h>
#include <linux/slab.h>

#include "string-stream.h"

+struct string_stream_test_priv {
+ struct string_stream *raw_stream;
+
+ /* For testing resource-managed free */
+ struct string_stream *freed_stream;
+ bool stream_free_again;
+};
+

/* string_stream object is initialized correctly. */

static void string_stream_init_test(struct kunit *test)
{
+ struct string_stream_test_priv *priv = test->priv;
struct string_stream *stream;

+ /* Resource-managed initialization */
stream = alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

@@ -26,6 +37,86 @@ static void string_stream_init_test(struct kunit *test)
KUNIT_EXPECT_FALSE(test, stream->append_newlines);

KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
+
+ free_string_stream(test, stream);
+
+ /*
+ * Raw initialization. This allocation is not resource-managed
+ * so store it in priv->raw_stream to be cleaned up by the
+ * exit function.
+ */
+ priv->raw_stream = raw_alloc_string_stream(GFP_KERNEL);
+ stream = priv->raw_stream;

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+ KUNIT_EXPECT_EQ(test, stream->length, 0);
+ KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));

+ KUNIT_EXPECT_PTR_EQ(test, stream->test, NULL);

+ KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL);

+ KUNIT_EXPECT_FALSE(test, stream->append_newlines);
+
+ KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
+}
+
+static void string_stream_raw_free_string_stream_stub(struct string_stream *stream)
+{
+ struct kunit *fake_test = kunit_get_current_test();
+ struct string_stream_test_priv *priv = fake_test->priv;
+
+ if (priv->freed_stream)
+ priv->stream_free_again = true;
+
+ priv->freed_stream = stream;
+
+ /*
+ * Avoid calling deactivate_static_stub() or changing
+ * current->kunit_test during cleanup. Leave the stream to
+ * be freed during the test exit.
+ */
+}
+
+/* string_stream object is freed when test is cleaned up. */
+static void string_stream_resource_free_test(struct kunit *test)
+{
+ struct string_stream_test_priv *priv = test->priv;
+ struct kunit *fake_test;

+ struct string_stream *stream;
+

+ fake_test = kunit_kzalloc(test, sizeof(*fake_test), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fake_test);
+
+ kunit_init_test(fake_test, "string_stream_fake_test", NULL);
+ fake_test->priv = priv;
+
+ /*
+ * Activate stub before creating string_stream so the
+ * string_stream will be cleaned up first.
+ */
+ priv->freed_stream = NULL;
+ priv->stream_free_again = false;
+ kunit_activate_static_stub(fake_test,
+ raw_free_string_stream,
+ string_stream_raw_free_string_stream_stub);
+
+ stream = alloc_string_stream(fake_test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+ /*
+ * Ensure the stream is freed when this test terminates.
+ */
+ priv->raw_stream = stream;
+
+ /* Set current->kunit_test to fake_test so the static stub will be called. */
+ current->kunit_test = fake_test;
+
+ /* Cleanup test - the stub function should be called */
+ kunit_cleanup(fake_test);
+
+ /* Set current->kunit_test back to current test. */
+ current->kunit_test = test;
+
+ KUNIT_EXPECT_PTR_EQ(test, priv->freed_stream, stream);
+ KUNIT_EXPECT_FALSE(test, priv->stream_free_again);
}

/*
@@ -279,8 +370,30 @@ static void string_stream_auto_newline_test(struct kunit *test)

"One\nTwo\nThree\nFour\nFive\nSix\nSeven\n\nEight\n");
}

+static int string_stream_test_init(struct kunit *test)
+{
+ struct string_stream_test_priv *priv;
+
+ priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ test->priv = priv;
+

+ return 0;
+}
+

+static void string_stream_test_exit(struct kunit *test)
+{
+ struct string_stream_test_priv *priv = test->priv;
+
+ if (priv && priv->raw_stream)
+ raw_free_string_stream(priv->raw_stream);
+}
+

static struct kunit_case string_stream_test_cases[] = {

KUNIT_CASE(string_stream_init_test),
+ KUNIT_CASE(string_stream_resource_free_test),
KUNIT_CASE(string_stream_line_add_test),
KUNIT_CASE(string_stream_variable_length_line_test),
KUNIT_CASE(string_stream_append_test),
@@ -292,6 +405,8 @@ static struct kunit_case string_stream_test_cases[] = {

static struct kunit_suite string_stream_test_suite = {
.name = "string-stream-test",
- .test_cases = string_stream_test_cases
+ .test_cases = string_stream_test_cases,
+ .init = string_stream_test_init,
+ .exit = string_stream_test_exit,
};
kunit_test_suites(&string_stream_test_suite);
diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index 06104a729b45..1b55ac1be2e5 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -6,6 +6,7 @@
* Author: Brendan Higgins <brendan...@google.com>
*/

+#include <kunit/static_stub.h>
#include <kunit/test.h>
#include <linux/list.h>
#include <linux/slab.h>
@@ -167,6 +168,8 @@ bool string_stream_is_empty(struct string_stream *stream)

void raw_free_string_stream(struct string_stream *stream)
{
+ KUNIT_STATIC_STUB_REDIRECT(raw_free_string_stream, stream);
+
string_stream_clear(stream);
kfree(stream);
}
--
2.30.2

[Richard Fitzgerald]

Re-work string_stream so that it is not tied to a struct kunit. This is
to allow using it for the log of struct kunit_suite.

Instead of resource-managing individual allocations the whole string_stream
object can be resource-managed as a single object:

alloc_string_stream() API is unchanged and takes a pointer to a
struct kunit but it now registers the returned string_stream object to
be resource-managed.

raw_alloc_string_stream() is a new function that allocates a
bare string_stream without any association to a struct kunit.

free_string_stream() is a new function that frees a resource-managed
string_stream allocated by alloc_string_stream().

raw_free_string_stream() is a new function that frees a non-managed
string_stream allocated by raw_alloc_string_stream().

The confusing function string_stream_destroy() has been removed. This only
called string_stream_clear() but didn't free the struct string_stream.
Instead string_stream_clear() has been exported, and the new functions use
the more conventional naming of "free" as the opposite of "alloc".

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream-test.c | 2 +-
lib/kunit/string-stream.c | 92 +++++++++++++++++++++++-----------
lib/kunit/string-stream.h | 12 ++++-
lib/kunit/test.c | 2 +-
4 files changed, 77 insertions(+), 31 deletions(-)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 8a30bb7d5fb7..437aa4b3179d 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -200,7 +200,7 @@ static void string_stream_append_test(struct kunit *test)

combined_content);

/* Append content of non-empty stream to empty stream */

- string_stream_destroy(stream_1);
+ string_stream_clear(stream_1);

stream_1 = alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);
diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index eb673d3ea3bd..06104a729b45 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -13,30 +13,28 @@
#include "string-stream.h"


-static struct string_stream_fragment *alloc_string_stream_fragment(
- struct kunit *test, int len, gfp_t gfp)
+static struct string_stream_fragment *alloc_string_stream_fragment(int len, gfp_t gfp)
{
struct string_stream_fragment *frag;

- frag = kunit_kzalloc(test, sizeof(*frag), gfp);
+ frag = kzalloc(sizeof(*frag), gfp);
if (!frag)
return ERR_PTR(-ENOMEM);

- frag->fragment = kunit_kmalloc(test, len, gfp);
+ frag->fragment = kmalloc(len, gfp);
if (!frag->fragment) {
- kunit_kfree(test, frag);
+ kfree(frag);
return ERR_PTR(-ENOMEM);
}

return frag;
}

-static void string_stream_fragment_destroy(struct kunit *test,
- struct string_stream_fragment *frag)
+static void string_stream_fragment_destroy(struct string_stream_fragment *frag)
{
list_del(&frag->node);
- kunit_kfree(test, frag->fragment);
- kunit_kfree(test, frag);
+ kfree(frag->fragment);
+ kfree(frag);
}

int string_stream_vadd(struct string_stream *stream,
@@ -65,9 +63,7 @@ int string_stream_vadd(struct string_stream *stream,

/* Need space for null byte. */

buf_len++;

- frag_container = alloc_string_stream_fragment(stream->test,
- buf_len,
- stream->gfp);
+ frag_container = alloc_string_stream_fragment(buf_len, stream->gfp);
if (IS_ERR(frag_container))
return PTR_ERR(frag_container);

@@ -102,7 +98,7 @@ int string_stream_add(struct string_stream *stream, const char *fmt, ...)
return result;
}

-static void string_stream_clear(struct string_stream *stream)
+void string_stream_clear(struct string_stream *stream)
{
struct string_stream_fragment *frag_container, *frag_container_safe;

@@ -111,16 +107,28 @@ static void string_stream_clear(struct string_stream *stream)
frag_container_safe,
&stream->fragments,
node) {
- string_stream_fragment_destroy(stream->test, frag_container);
+ string_stream_fragment_destroy(frag_container);
}
stream->length = 0;
spin_unlock(&stream->lock);
}

+static void _string_stream_concatenate_to_buf(struct string_stream *stream,
+ char *buf, size_t buf_len)
+{
+ struct string_stream_fragment *frag_container;
+
+ buf[0] = '\0';
+
+ spin_lock(&stream->lock);
+ list_for_each_entry(frag_container, &stream->fragments, node)
+ strlcat(buf, frag_container->fragment, buf_len);
+ spin_unlock(&stream->lock);
+}
+

char *string_stream_get_string(struct kunit *test, struct string_stream *stream,

gfp_t gfp)
{
- struct string_stream_fragment *frag_container;

size_t buf_len = stream->length + 1; /* +1 for null byte. */
char *buf;

@@ -128,10 +136,7 @@ char *string_stream_get_string(struct kunit *test, struct string_stream *stream,
if (!buf)
return NULL;

- spin_lock(&stream->lock);
- list_for_each_entry(frag_container, &stream->fragments, node)
- strlcat(buf, frag_container->fragment, buf_len);
- spin_unlock(&stream->lock);
+ _string_stream_concatenate_to_buf(stream, buf, buf_len);

return buf;
}
@@ -139,13 +144,20 @@ char *string_stream_get_string(struct kunit *test, struct string_stream *stream,

int string_stream_append(struct string_stream *stream,
struct string_stream *other)

{
- const char *other_content;
+ size_t other_buf_len = other->length + 1; /* +1 for null byte. */
+ char *other_buf;
+ int ret;

- other_content = string_stream_get_string(other->test, other, other->gfp);
- if (!other_content)
+ other_buf = kmalloc(other_buf_len, GFP_KERNEL);
+ if (!other_buf)
return -ENOMEM;

- return string_stream_add(stream, other_content);
+ _string_stream_concatenate_to_buf(other, other_buf, other_buf_len);
+
+ ret = string_stream_add(stream, other_buf);
+ kfree(other_buf);
+
+ return ret;
}

bool string_stream_is_empty(struct string_stream *stream)
@@ -153,23 +165,47 @@ bool string_stream_is_empty(struct string_stream *stream)
return list_empty(&stream->fragments);
}

-struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp)
+void raw_free_string_stream(struct string_stream *stream)
+{
+ string_stream_clear(stream);
+ kfree(stream);
+}
+
+struct string_stream *raw_alloc_string_stream(gfp_t gfp)
{
struct string_stream *stream;

- stream = kunit_kzalloc(test, sizeof(*stream), gfp);
+ stream = kzalloc(sizeof(*stream), gfp);
if (!stream)
return ERR_PTR(-ENOMEM);

stream->gfp = gfp;
- stream->test = test;
INIT_LIST_HEAD(&stream->fragments);
spin_lock_init(&stream->lock);

return stream;
}

-void string_stream_destroy(struct string_stream *stream)
+struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp)
{
- string_stream_clear(stream);

+ struct string_stream *stream;
+

+ stream = raw_alloc_string_stream(gfp);
+ if (IS_ERR(stream))
+ return stream;
+
+ stream->test = test;
+
+ if (kunit_add_action_or_reset(test, (kunit_action_t *)raw_free_string_stream, stream) != 0)
+ return ERR_PTR(-ENOMEM);
+
+ return stream;
+}
+
+void free_string_stream(struct kunit *test, struct string_stream *stream)
+{
+ if (!stream)
+ return;
+
+ kunit_release_action(test, (kunit_action_t *)raw_free_string_stream, (void *)stream);
}
diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h
index 6b4a747881c5..fbeda486382a 100644
--- a/lib/kunit/string-stream.h
+++ b/lib/kunit/string-stream.h
@@ -23,14 +23,24 @@ struct string_stream {
struct list_head fragments;
/* length and fragments are protected by this lock */
spinlock_t lock;
+
+ /*
+ * Pointer to kunit this stream is associated with, or NULL if
+ * not associated with a kunit.
+ */
struct kunit *test;
+
gfp_t gfp;
bool append_newlines;
};

struct kunit;

+struct string_stream *raw_alloc_string_stream(gfp_t gfp);
+void raw_free_string_stream(struct string_stream *stream);
+
struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp);
+void free_string_stream(struct kunit *test, struct string_stream *stream);

int __printf(2, 3) string_stream_add(struct string_stream *stream,
const char *fmt, ...);
@@ -47,7 +57,7 @@ int string_stream_append(struct string_stream *stream,

bool string_stream_is_empty(struct string_stream *stream);

-void string_stream_destroy(struct string_stream *stream);
+void string_stream_clear(struct string_stream *stream);

static inline void string_stream_set_append_newlines(struct string_stream *stream,

bool append_newlines)
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 520e15f49d0d..4b69d12dfc96 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -322,7 +322,7 @@ static void kunit_fail(struct kunit *test, const struct kunit_loc *loc,

kunit_print_string_stream(test, stream);

- string_stream_destroy(stream);
+ free_string_stream(test, stream);
}

void __noreturn __kunit_abort(struct kunit *test)
--
2.30.2

[Richard Fitzgerald]

Replace the fixed-size log buffer with a string_stream so that the
log can grow as lines are added.

The existing kunit log tests have been updated for using a
string_stream as the log. As they now depend on string_stream
functions they cannot build when kunit-test is a module. They have
been moved to a separate source file to be built only if kunit-test
is built-in.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

include/kunit/test.h | 14 ++++----
lib/kunit/Makefile | 5 +--
lib/kunit/debugfs.c | 36 +++++++++++++--------
lib/kunit/kunit-test.c | 52 +-----------------------------
lib/kunit/log-test.c | 72 ++++++++++++++++++++++++++++++++++++++++++
lib/kunit/test.c | 44 +++-----------------------
6 files changed, 110 insertions(+), 113 deletions(-)
create mode 100644 lib/kunit/log-test.c

diff --git a/include/kunit/test.h b/include/kunit/test.h
index d33114097d0d..b915a0fe93c0 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -32,9 +32,7 @@
DECLARE_STATIC_KEY_FALSE(kunit_running);

struct kunit;
-

-/* Size of log associated with test. */
-#define KUNIT_LOG_SIZE 2048

+struct string_stream;

/* Maximum size of parameter description string. */
#define KUNIT_PARAM_DESC_SIZE 128

@@ -132,7 +130,7 @@ struct kunit_case {

/* private: internal use only. */
enum kunit_status status;
char *module_name;
- char *log;

+ struct string_stream *log;

};

static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)

@@ -252,7 +250,7 @@ struct kunit_suite {

/* private: internal use only */
char status_comment[KUNIT_STATUS_COMMENT_SIZE];
struct dentry *debugfs;
- char *log;

+ struct string_stream *log;
int suite_init_err;
};

@@ -278,7 +276,7 @@ struct kunit {

/* private: internal use only. */
const char *name; /* Read only after initialization! */
- char *log; /* Points at case log after initialization */

+ struct string_stream *log; /* Points at case log after initialization */

struct kunit_try_catch try_catch;
/* param_value is the current parameter value for a test case. */
const void *param_value;

@@ -314,7 +312,7 @@ const char *kunit_filter_glob(void);

char *kunit_filter(void);
char *kunit_filter_action(void);

-void kunit_init_test(struct kunit *test, const char *name, char *log);

+void kunit_init_test(struct kunit *test, const char *name, struct string_stream *log);

int kunit_run_tests(struct kunit_suite *suite);

@@ -472,7 +470,7 @@ static inline void *kunit_kcalloc(struct kunit *test, size_t n, size_t size, gfp

void kunit_cleanup(struct kunit *test);

-void __printf(2, 3) kunit_log_append(char *log, const char *fmt, ...);
+void __printf(2, 3) kunit_log_append(struct string_stream *log, const char *fmt, ...);

/**
* kunit_mark_skipped() - Marks @test_or_suite as skipped

diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile
index 46f75f23dfe4..65735c2e1d14 100644
--- a/lib/kunit/Makefile
+++ b/lib/kunit/Makefile
@@ -18,9 +18,10 @@ obj-y += hooks.o

obj-$(CONFIG_KUNIT_TEST) += kunit-test.o

-# string-stream-test compiles built-in only.
+# string-stream-test and log-test compiles built-in only.
ifeq ($(CONFIG_KUNIT_TEST),y)
-obj-$(CONFIG_KUNIT_TEST) += string-stream-test.o
+obj-$(CONFIG_KUNIT_TEST) += string-stream-test.o \
+ log-test.o
endif

obj-$(CONFIG_KUNIT_EXAMPLE_TEST) += kunit-example-test.o
diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c
index 22c5c496a68f..ab53a7e5c898 100644
--- a/lib/kunit/debugfs.c
+++ b/lib/kunit/debugfs.c
@@ -37,14 +37,21 @@ void kunit_debugfs_init(void)

debugfs_rootdir = debugfs_create_dir(KUNIT_DEBUGFS_ROOT, NULL);
}

-static void debugfs_print_result(struct seq_file *seq,
- struct kunit_suite *suite,
- struct kunit_case *test_case)

+static void debugfs_print_result(struct seq_file *seq, struct string_stream *log)

{
- if (!test_case || !test_case->log)

+ struct string_stream_fragment *frag_container;
+

+ if (!log)
return;

- seq_printf(seq, "%s", test_case->log);

+ /*
+ * Walk the fragments so we don't need to allocate a temporary
+ * buffer to hold the entire string.
+ */
+ spin_lock(&log->lock);
+ list_for_each_entry(frag_container, &log->fragments, node)
+ seq_printf(seq, "%s", frag_container->fragment);
+ spin_unlock(&log->lock);
}

/*
@@ -69,10 +76,9 @@ static int debugfs_print_results(struct seq_file *seq, void *v)

seq_printf(seq, KUNIT_SUBTEST_INDENT "1..%zd\n", kunit_suite_num_test_cases(suite));

kunit_suite_for_each_test_case(suite, test_case)
- debugfs_print_result(seq, suite, test_case);

+ debugfs_print_result(seq, test_case->log);

- if (suite->log)
- seq_printf(seq, "%s", suite->log);

+ debugfs_print_result(seq, suite->log);

seq_printf(seq, "%s %d %s\n",
kunit_status_to_ok_not_ok(success), 1, suite->name);

@@ -105,9 +111,13 @@ void kunit_debugfs_create_suite(struct kunit_suite *suite)

struct kunit_case *test_case;

/* Allocate logs before creating debugfs representation. */
- suite->log = kzalloc(KUNIT_LOG_SIZE, GFP_KERNEL);

- kunit_suite_for_each_test_case(suite, test_case)

- test_case->log = kzalloc(KUNIT_LOG_SIZE, GFP_KERNEL);

+ suite->log = raw_alloc_string_stream(GFP_KERNEL);
+ string_stream_set_append_newlines(suite->log, true);
+
+ kunit_suite_for_each_test_case(suite, test_case) {
+ test_case->log = raw_alloc_string_stream(GFP_KERNEL);
+ string_stream_set_append_newlines(test_case->log, true);
+ }

suite->debugfs = debugfs_create_dir(suite->name, debugfs_rootdir);

@@ -121,7 +131,7 @@ void kunit_debugfs_destroy_suite(struct kunit_suite *suite)

struct kunit_case *test_case;

debugfs_remove_recursive(suite->debugfs);
- kfree(suite->log);

+ raw_free_string_stream(suite->log);

kunit_suite_for_each_test_case(suite, test_case)
- kfree(test_case->log);

+ raw_free_string_stream(test_case->log);
}
diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index 83d8e90ca7a2..ecc39d5f7411 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -530,55 +530,6 @@ static struct kunit_suite kunit_resource_test_suite = {
.test_cases = kunit_resource_test_cases,
};

-static void kunit_log_test(struct kunit *test)
-{
- struct kunit_suite suite;
-
- suite.log = kunit_kzalloc(test, KUNIT_LOG_SIZE, GFP_KERNEL);
- KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
-
- kunit_log(KERN_INFO, test, "put this in log.");
- kunit_log(KERN_INFO, test, "this too.");
- kunit_log(KERN_INFO, &suite, "add to suite log.");
- kunit_log(KERN_INFO, &suite, "along with this.");
-
-#ifdef CONFIG_KUNIT_DEBUGFS
- KUNIT_EXPECT_NOT_ERR_OR_NULL(test,

- strstr(test->log, "put this in log."));

- KUNIT_EXPECT_NOT_ERR_OR_NULL(test,

- strstr(test->log, "this too."));

- KUNIT_EXPECT_NOT_ERR_OR_NULL(test,

- strstr(suite.log, "add to suite log."));

- KUNIT_EXPECT_NOT_ERR_OR_NULL(test,

- strstr(suite.log, "along with this."));

-#else
- KUNIT_EXPECT_NULL(test, test->log);
-#endif
-}
-
-static void kunit_log_newline_test(struct kunit *test)
-{
- kunit_info(test, "Add newline\n");
- if (test->log) {

- KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(test->log, "Add newline\n"),
- "Missing log line, full log:\n%s", test->log);

- KUNIT_EXPECT_NULL(test, strstr(test->log, "Add newline\n\n"));
- } else {
- kunit_skip(test, "only useful when debugfs is enabled");
- }
-}
-
-static struct kunit_case kunit_log_test_cases[] = {
- KUNIT_CASE(kunit_log_test),
- KUNIT_CASE(kunit_log_newline_test),
- {}
-};
-
-static struct kunit_suite kunit_log_test_suite = {
- .name = "kunit-log-test",
- .test_cases = kunit_log_test_cases,
-};
-
static void kunit_status_set_failure_test(struct kunit *test)
{
struct kunit fake;
@@ -658,7 +609,6 @@ static struct kunit_suite kunit_current_test_suite = {
};

kunit_test_suites(&kunit_try_catch_test_suite, &kunit_resource_test_suite,
- &kunit_log_test_suite, &kunit_status_test_suite,
- &kunit_current_test_suite);
+ &kunit_status_test_suite, &kunit_current_test_suite);

MODULE_LICENSE("GPL v2");
diff --git a/lib/kunit/log-test.c b/lib/kunit/log-test.c
new file mode 100644
index 000000000000..a93d87112fea
--- /dev/null
+++ b/lib/kunit/log-test.c
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit test for logging.
+ *
+ * Based on code:
+ * Copyright (C) 2019, Google LLC.
+ * Author: Brendan Higgins <brendan...@google.com>
+ */
+#include <kunit/test.h>
+
+#include "string-stream.h"
+
+static void kunit_log_test(struct kunit *test)
+{
+ struct kunit_suite suite;
+ char *full_log;
+
+ suite.log = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ string_stream_set_append_newlines(suite.log, true);
+
+ kunit_log(KERN_INFO, test, "put this in log.");
+ kunit_log(KERN_INFO, test, "this too.");
+ kunit_log(KERN_INFO, &suite, "add to suite log.");
+ kunit_log(KERN_INFO, &suite, "along with this.");
+
+#ifdef CONFIG_KUNIT_DEBUGFS
+ KUNIT_EXPECT_TRUE(test, test->log->append_newlines);
+
+ full_log = string_stream_get_string(test, test->log, GFP_KERNEL);
+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
+ strstr(full_log, "put this in log."));
+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
+ strstr(full_log, "this too."));
+
+ full_log = string_stream_get_string(test, suite.log, GFP_KERNEL);
+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
+ strstr(full_log, "add to suite log."));
+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
+ strstr(full_log, "along with this."));
+#else
+ KUNIT_EXPECT_NULL(test, test->log);
+#endif
+}
+
+static void kunit_log_newline_test(struct kunit *test)
+{
+ char *full_log;
+
+ kunit_info(test, "Add newline\n");
+ if (test->log) {
+ full_log = string_stream_get_string(test, test->log, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(full_log, "Add newline\n"),
+ "Missing log line, full log:\n%s", full_log);
+ KUNIT_EXPECT_NULL(test, strstr(full_log, "Add newline\n\n"));
+ } else {

+ kunit_skip(test, "only useful when debugfs is enabled");
+ }

+}
+
+static struct kunit_case kunit_log_test_cases[] = {
+ KUNIT_CASE(kunit_log_test),
+ KUNIT_CASE(kunit_log_newline_test),
+ {}
+};
+
+static struct kunit_suite kunit_log_test_suite = {
+ .name = "kunit-log-test",
+ .test_cases = kunit_log_test_cases,
+};
+
+kunit_test_suites(&kunit_log_test_suite);
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 4b69d12dfc96..14e889e80129 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -109,51 +109,17 @@ static void kunit_print_test_stats(struct kunit *test,
stats.total);
}

-/**
- * kunit_log_newline() - Add newline to the end of log if one is not
- * already present.
- * @log: The log to add the newline to.
- */
-static void kunit_log_newline(char *log)
-{
- int log_len, len_left;
-
- log_len = strlen(log);
- len_left = KUNIT_LOG_SIZE - log_len - 1;
-

- if (log_len > 0 && log[log_len - 1] != '\n')
- strncat(log, "\n", len_left);

-}
-

-/*
- * Append formatted message to log, size of which is limited to
- * KUNIT_LOG_SIZE bytes (including null terminating byte).
- */

-void kunit_log_append(char *log, const char *fmt, ...)
+/* Append formatted message to log. */
+void kunit_log_append(struct string_stream *log, const char *fmt, ...)
{
va_list args;
- int len, log_len, len_left;

if (!log)
return;

- log_len = strlen(log);
- len_left = KUNIT_LOG_SIZE - log_len - 1;
- if (len_left <= 0)
- return;

-

- /* Evaluate length of line to add to log */

va_start(args, fmt);
- len = vsnprintf(NULL, 0, fmt, args) + 1;
+ string_stream_vadd(log, fmt, args);
va_end(args);
-
- /* Print formatted line to the log */
- va_start(args, fmt);
- vsnprintf(log + log_len, min(len, len_left), fmt, args);
- va_end(args);
-
- /* Add newline to end of log if not already present. */
- kunit_log_newline(log);
}
EXPORT_SYMBOL_GPL(kunit_log_append);

@@ -359,14 +325,14 @@ void __kunit_do_failed_assertion(struct kunit *test,

}
EXPORT_SYMBOL_GPL(__kunit_do_failed_assertion);

-void kunit_init_test(struct kunit *test, const char *name, char *log)

+void kunit_init_test(struct kunit *test, const char *name, struct string_stream *log)

{
spin_lock_init(&test->lock);
INIT_LIST_HEAD(&test->resources);
test->name = name;
test->log = log;

if (test->log)
- test->log[0] = '\0';

+ string_stream_clear(log);

[Richard Fitzgerald]

Add a test of the speed and memory use of string_stream.

string_stream_performance_test() doesn't actually "test" anything (it
cannot fail unless the system has run out of allocatable memory) but it
measures the speed and memory consumption of the string_stream and reports
the result.

This allows changes in the string_stream implementation to be compared.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream-test.c | 54 ++++++++++++++++++++++++++++++++++
1 file changed, 54 insertions(+)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 05bfade2bd8a..b55cc14f43fb 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -8,7 +8,9 @@

#include <kunit/static_stub.h>
#include <kunit/test.h>
+#include <linux/ktime.h>
#include <linux/slab.h>
+#include <linux/timekeeping.h>

#include "string-stream.h"

@@ -370,6 +372,57 @@ static void string_stream_auto_newline_test(struct kunit *test)

"One\nTwo\nThree\nFour\nFive\nSix\nSeven\n\nEight\n");
}

+/*
+ * This doesn't actually "test" anything. It reports time taken
+ * and memory used for logging a large number of lines.
+ */
+static void string_stream_performance_test(struct kunit *test)
+{
+ struct string_stream_fragment *frag_container;
+ struct string_stream *stream;
+ char test_line[101];
+ ktime_t start_time, end_time;
+ size_t len, bytes_requested, actual_bytes_used, total_string_length;
+ int offset, i;
+
+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+ memset(test_line, 'x', sizeof(test_line) - 1);
+ test_line[sizeof(test_line) - 1] = '\0';
+
+ start_time = ktime_get();
+ for (i = 0; i < 10000; i++) {
+ offset = i % (sizeof(test_line) - 1);
+ string_stream_add(stream, "%s: %d\n", &test_line[offset], i);
+ }
+ end_time = ktime_get();
+
+ /*
+ * Calculate memory used. This doesn't include invisible
+ * overhead due to kernel allocator fragment size rounding.
+ */
+ bytes_requested = sizeof(*stream);
+ actual_bytes_used = ksize(stream);
+ total_string_length = 0;
+

+ list_for_each_entry(frag_container, &stream->fragments, node) {

+ bytes_requested += sizeof(*frag_container);
+ actual_bytes_used += ksize(frag_container);
+
+ len = strlen(frag_container->fragment);
+ total_string_length += len;
+ bytes_requested += len + 1; /* +1 for '\0' */
+ actual_bytes_used += ksize(frag_container->fragment);
+ }
+
+ kunit_info(test, "Time elapsed: %lld us\n",
+ ktime_us_delta(end_time, start_time));
+ kunit_info(test, "Total string length: %zu\n", total_string_length);
+ kunit_info(test, "Bytes requested: %zu\n", bytes_requested);
+ kunit_info(test, "Actual bytes allocated: %zu\n", actual_bytes_used);
+}
+
static int string_stream_test_init(struct kunit *test)
{
struct string_stream_test_priv *priv;
@@ -400,6 +453,7 @@ static struct kunit_case string_stream_test_cases[] = {
KUNIT_CASE(string_stream_append_empty_string_test),
KUNIT_CASE(string_stream_no_auto_newline_test),
KUNIT_CASE(string_stream_auto_newline_test),
+ KUNIT_CASE(string_stream_performance_test),
{}
};

--
2.30.2

[Richard Fitzgerald]

Add test cases for testing the string_stream feature that appends a
newline to strings that do not already end with a newline.

string_stream_no_auto_newline_test() tests with this feature disabled.
Newlines should not be added or dropped.

string_stream_auto_newline_test() tests with this feature enabled.
Newlines should be added to lines that do not end with a newline.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream-test.c | 51 ++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index efe13e3322b5..46c2ac162fe8 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -23,6 +23,7 @@ static void string_stream_init_test(struct kunit *test)
KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));
KUNIT_EXPECT_PTR_EQ(test, stream->test, test);

KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL);
+ KUNIT_EXPECT_FALSE(test, stream->append_newlines);

KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
}
@@ -226,12 +227,62 @@ static void string_stream_append_empty_string_test(struct kunit *test)

KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream), "Add this line");
}

+/* Adding strings without automatic newline appending */
+static void string_stream_no_auto_newline_test(struct kunit *test)
+{
+ struct string_stream *stream;

+
+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+

+ /*
+ * Add some strings with and without newlines. All formatted
+ * newlines should be preserved. No extra newlines should be
+ * added.
+ */
+ string_stream_add(stream, "One");
+ string_stream_add(stream, "Two\n");
+ string_stream_add(stream, "%s\n", "Three");
+ string_stream_add(stream, "Four");
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream),
+ "OneTwo\nThree\nFour");
+}
+
+/* Adding strings with automatic newline appending */
+static void string_stream_auto_newline_test(struct kunit *test)
+{
+ struct string_stream *stream;

+
+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+

+ string_stream_set_append_newlines(stream, true);
+ KUNIT_EXPECT_TRUE(test, stream->append_newlines);
+
+ /*
+ * Add some strings with and without newlines. Newlines should
+ * be appended to lines that do not end with \n, but newlines
+ * resulting from the formatting should not be changed.
+ */
+ string_stream_add(stream, "One");
+ string_stream_add(stream, "Two\n");
+ string_stream_add(stream, "%s\n", "Three");
+ string_stream_add(stream, "%s", "Four\n");
+ string_stream_add(stream, "Five\n%s", "Six");
+ string_stream_add(stream, "Seven\n\n");
+ string_stream_add(stream, "Eight");
+ KUNIT_EXPECT_STREQ(test, string_stream_get_string(stream),
+ "One\nTwo\nThree\nFour\nFive\nSix\nSeven\n\nEight\n");
+}
+

static struct kunit_case string_stream_test_cases[] = {

KUNIT_CASE(string_stream_init_test),
KUNIT_CASE(string_stream_line_add_test),
KUNIT_CASE(string_stream_variable_length_line_test),
KUNIT_CASE(string_stream_append_test),
KUNIT_CASE(string_stream_append_empty_string_test),
+ KUNIT_CASE(string_stream_no_auto_newline_test),
+ KUNIT_CASE(string_stream_auto_newline_test),
{}
};

--
2.30.2

[kernel test robot]

Hi Richard,

kernel test robot noticed the following build warnings:

[auto build test WARNING on shuah-kselftest/kunit]
[also build test WARNING on shuah-kselftest/kunit-fixes linus/master v6.5-rc6 next-20230809]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url: https://github.com/intel-lab-lkp/linux/commits/Richard-Fitzgerald/kunit-string-stream-Improve-testing-of-string_stream/20230814-212947
base: https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit
patch link: https://lore.kernel.org/r/20230814132309.32641-8-rf%40opensource.cirrus.com
patch subject: [PATCH v4 07/10] kunit: string-stream: Decouple string_stream from kunit
config: hexagon-randconfig-r014-20230815 (https://download.01.org/0day-ci/archive/20230815/202308150347...@intel.com/config)
compiler: clang version 17.0.0 (https://github.com/llvm/llvm-project.git 4a5ac14ee968ff0ad5d2cc1ffa0299048db4c88a)
reproduce: (https://download.01.org/0day-ci/archive/20230815/202308150347...@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <l...@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202308150347...@intel.com/

All warnings (new ones prefixed by >>):

>> lib/kunit/string-stream.c:199:38: warning: cast from 'void (*)(struct string_stream *)' to 'kunit_action_t *' (aka 'void (*)(void *)') converts to incompatible function type [-Wcast-function-type-strict]
199 | if (kunit_add_action_or_reset(test, (kunit_action_t *)raw_free_string_stream, stream) != 0)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/compiler.h:55:47: note: expanded from macro 'if'
55 | #define if(cond, ...) if ( __trace_if_var( !!(cond , ## __VA_ARGS__) ) )
| ^~~~
include/linux/compiler.h:57:52: note: expanded from macro '__trace_if_var'
57 | #define __trace_if_var(cond) (__builtin_constant_p(cond) ? (cond) : __trace_if_value(cond))
| ^~~~
>> lib/kunit/string-stream.c:199:38: warning: cast from 'void (*)(struct string_stream *)' to 'kunit_action_t *' (aka 'void (*)(void *)') converts to incompatible function type [-Wcast-function-type-strict]
199 | if (kunit_add_action_or_reset(test, (kunit_action_t *)raw_free_string_stream, stream) != 0)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/compiler.h:55:47: note: expanded from macro 'if'
55 | #define if(cond, ...) if ( __trace_if_var( !!(cond , ## __VA_ARGS__) ) )
| ^~~~
include/linux/compiler.h:57:61: note: expanded from macro '__trace_if_var'
57 | #define __trace_if_var(cond) (__builtin_constant_p(cond) ? (cond) : __trace_if_value(cond))
| ^~~~
>> lib/kunit/string-stream.c:199:38: warning: cast from 'void (*)(struct string_stream *)' to 'kunit_action_t *' (aka 'void (*)(void *)') converts to incompatible function type [-Wcast-function-type-strict]
199 | if (kunit_add_action_or_reset(test, (kunit_action_t *)raw_free_string_stream, stream) != 0)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/compiler.h:55:47: note: expanded from macro 'if'
55 | #define if(cond, ...) if ( __trace_if_var( !!(cond , ## __VA_ARGS__) ) )
| ^~~~
include/linux/compiler.h:57:86: note: expanded from macro '__trace_if_var'
57 | #define __trace_if_var(cond) (__builtin_constant_p(cond) ? (cond) : __trace_if_value(cond))
| ^~~~
include/linux/compiler.h:68:3: note: expanded from macro '__trace_if_value'
68 | (cond) ? \
| ^~~~
lib/kunit/string-stream.c:210:29: warning: cast from 'void (*)(struct string_stream *)' to 'kunit_action_t *' (aka 'void (*)(void *)') converts to incompatible function type [-Wcast-function-type-strict]
210 | kunit_release_action(test, (kunit_action_t *)raw_free_string_stream, (void *)stream);
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4 warnings generated.


vim +199 lib/kunit/string-stream.c

188
189 struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp)
190 {
191 struct string_stream *stream;
192
193 stream = raw_alloc_string_stream(gfp);
194 if (IS_ERR(stream))
195 return stream;
196
197 stream->test = test;
198
> 199 if (kunit_add_action_or_reset(test, (kunit_action_t *)raw_free_string_stream, stream) != 0)
200 return ERR_PTR(-ENOMEM);
201
202 return stream;
203 }
204

--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

[kernel test robot]

Hi Richard,

kernel test robot noticed the following build warnings:

[auto build test WARNING on shuah-kselftest/kunit]
[also build test WARNING on shuah-kselftest/kunit-fixes linus/master v6.5-rc6 next-20230809]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url: https://github.com/intel-lab-lkp/linux/commits/Richard-Fitzgerald/kunit-string-stream-Improve-testing-of-string_stream/20230814-212947
base: https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit

patch link: https://lore.kernel.org/r/20230814132309.32641-2-rf%40opensource.cirrus.com
patch subject: [PATCH v4 01/10] kunit: string-stream: Improve testing of string_stream
config: arc-randconfig-r073-20230815 (https://download.01.org/0day-ci/archive/20230815/202308151555...@intel.com/config)
compiler: arc-elf-gcc (GCC) 12.3.0
reproduce: (https://download.01.org/0day-ci/archive/20230815/202308151555...@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <l...@intel.com>

| Closes: https://lore.kernel.org/oe-kbuild-all/202308151555...@intel.com/

sparse warnings: (new ones prefixed by >>)
>> lib/kunit/string-stream-test.c:25:9: sparse: sparse: incorrect type in initializer (different base types) @@ expected long long left_value @@ got restricted gfp_t const __left @@
lib/kunit/string-stream-test.c:25:9: sparse: expected long long left_value
lib/kunit/string-stream-test.c:25:9: sparse: got restricted gfp_t const __left
>> lib/kunit/string-stream-test.c:25:9: sparse: sparse: incorrect type in initializer (different base types) @@ expected long long right_value @@ got restricted gfp_t const __right @@
lib/kunit/string-stream-test.c:25:9: sparse: expected long long right_value
lib/kunit/string-stream-test.c:25:9: sparse: got restricted gfp_t const __right

vim +25 lib/kunit/string-stream-test.c

13
14 /* string_stream object is initialized correctly. */
15 static void string_stream_init_test(struct kunit *test)
16 {
17 struct string_stream *stream;
18
19 stream = alloc_string_stream(test, GFP_KERNEL);
20 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
21
22 KUNIT_EXPECT_EQ(test, stream->length, 0);
23 KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));
24 KUNIT_EXPECT_PTR_EQ(test, stream->test, test);
> 25 KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL);
26
27 KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
28 }
29

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>
> Replace the minimal tests with more-thorough testing.
>
> string_stream_init_test() tests that struct string_stream is
> initialized correctly.
>
> string_stream_line_add_test() adds a series of numbered lines and
> checks that the resulting string contains all the lines.
>
> string_stream_variable_length_line_test() adds a large number of
> lines of varying length to create many fragments, then tests that all
> lines are present.
>
> string_stream_append_test() tests various cases of using
> string_stream_append() to append the content of one stream to another.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Thanks. These tests are much better than the original string stream ones.

It looks good to me. I left a few notes below (mostly to myself), but
nothing that would require a new version by itself.

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

> lib/kunit/string-stream-test.c | 200 ++++++++++++++++++++++++++++++---
> 1 file changed, 184 insertions(+), 16 deletions(-)
>
> diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
> index 110f3a993250..1d46d5f06d2a 100644
> --- a/lib/kunit/string-stream-test.c
> +++ b/lib/kunit/string-stream-test.c
> @@ -11,38 +11,206 @@
>
> #include "string-stream.h"
>
> -static void string_stream_test_empty_on_creation(struct kunit *test)
> +/* string_stream object is initialized correctly. */
> +static void string_stream_init_test(struct kunit *test)
> {
> - struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL);
> + struct string_stream *stream;
> +
> + stream = alloc_string_stream(test, GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
> +
> + KUNIT_EXPECT_EQ(test, stream->length, 0);
> + KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));
> + KUNIT_EXPECT_PTR_EQ(test, stream->test, test);
> + KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL);

As the kernel test robot notes, this may trigger a sparse warning, as
KUnit stores integer types as 'long long' internally in assertions.
Ignore it for now, we'll see if we can fix it in KUnit.

>
> KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
> }
>
> -static void string_stream_test_not_empty_after_add(struct kunit *test)
> +/*
> + * Add a series of lines to a string_stream. Check that all lines
> + * appear in the correct order and no characters are dropped.
> + */
> +static void string_stream_line_add_test(struct kunit *test)
> {
> - struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL);
> + struct string_stream *stream;
> + char line[60];
> + char *concat_string, *pos, *string_end;
> + size_t len, total_len;
> + int num_lines, i;
>
> - string_stream_add(stream, "Foo");
> + stream = alloc_string_stream(test, GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
>
> - KUNIT_EXPECT_FALSE(test, string_stream_is_empty(stream));
> + /* Add series of sequence numbered lines */
> + total_len = 0;
> + for (i = 0; i < 100; ++i) {
> + len = snprintf(line, sizeof(line),
> + "The quick brown fox jumps over the lazy penguin %d\n", i);

Assuming the 'd' in '%d' counts, this still has every letter of the
alphabet in it. :-)

This being deterministic is good. There are a few tests which are
doing similar things with randomness, and I think we'll want to have a
shared framework for it at some point (to enable a 'fuzzing' mode
which is _not_ deterministic), but this is good for now.

This is fine, but I wonder if it'd be more resilient to potential
changes to the string-string implementation if these arrays didn't
have the same shape (in terms of length and length of substrings).

e.g., if we made the 2nd one "NINE", "EIGHT", "SEVEN"..., so it
doesn't have the same number of strings (due to missing "TEN") and the
lengths of them are not equivalent (strlen("one") != strlen("NINE")).

I doubt it'd make much difference, but maybe it'd catch some nasty
use-after-free or similar errors, and having the strings be different
makes it more obvious that there's not something being tested which
relies on them being the same.

(Don't bother resending the series just for this, though. But if we
have to do a new version anyway...)

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> If the result of the formatted string is an empty string just return
> instead of creating an empty fragment.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Nice catch!

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Adds string_stream_append_empty_string_test() to test that adding an
> empty string to a string_stream doesn't create a new empty fragment.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Looks good. One minor note below (not worth resending the series to
fix by itself, though).

If you wanted to combine this with the previous patch, that'd be fine
too. (I don't mind either way.)

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

While this is fine, I do wonder whether the more future-proof thing to
do would be to check that the number of fragments hasn't changed after
adding the empty string, rather than that it's definitely 1.

(In practice, even with a fancier allocation strategy, I can't imagine
us splitting "Add this line" into multiple fragments, but I think it's
slightly clearer that what we're testing is that the empty string
doesn't increase it.)

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Add an optional feature to string_stream that will append a newline to
> any added string that does not already end with a newline. The purpose
> of this is so that string_stream can be used to collect log lines.
>
> This is enabled/disabled by calling string_stream_set_append_newlines().
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Looks good. I don't mind the extra 'wasted' byte if a message already
ends in a newline.

Reviewed-by: David Gow <davi...@google.com>

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Add test cases for testing the string_stream feature that appends a
> newline to strings that do not already end with a newline.
>
> string_stream_no_auto_newline_test() tests with this feature disabled.
> Newlines should not be added or dropped.
>
> string_stream_auto_newline_test() tests with this feature enabled.
> Newlines should be added to lines that do not end with a newline.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Looks good to me. I wouldn't mind if you added the extra test strings
to the non-newline case, but can do without if you feel it's
excessive.

Reviewed-by: David Gow <davi...@google.com>


-- David

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Pass a struct kunit* and gfp_t to string_stream_get_string(). Allocate
> the returned buffer using these instead of using the stream->test and
> stream->gfp.
>
> This is preparation for removing the dependence of string_stream on
> struct kunit, so that string_stream can be used for the debugfs log.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Makes sense to me.

I think that, if we weren't going to remove the struct kunit
dependency, we'd want to either keep a version of
string_stream_get_string() which uses it, or, e.g., fall back to it if
the struct passed in is NULL.

The other option is to have a version which doesn't manage the string
at all, so just takes a gfp and requires the caller to free it (or
register an action to do so). If we did that, we could get rid of the
struct kunit pointer totally (though it may be better to keep it and
have both versions).

So -- to switch to some stream-of-consciousness thoughts about this --
basically there are three possible variants of
string_stream_get_string():
- Current version: uses the stream's struct kunit. Useless if this is
NULL, but very convenient otherwise.
- This patch: manage the string using the struct kunit passed as an
argument. Still can't be used directly outside a test, but adds enough
flexibility to get _append to work.
- Totally unmanaged: the generated string is allocated separately, and
must be freed (directly or in a deferred action) by the caller. Works
well outside tests, but less convenient.

Personally, I feel that the design of string_stream is heading towards
the third option. By the end of this series, everything uses
_string_stream_concatenate_to_buf() anyway. There's only one call to
string_stream_get_string() outside of the logging / string_stream
tests, and all that does is log the results (and it has a fallback to
log each fragment separately if the allocation fails).

Then again, if this is only really being used in tests, then we can
probably just stick with string_stream_get_string() as-is, remove the
string_stream->test member totally, and if we need it, we can make
_string_stream_concatenate_to_buf() public, or add an unmanaged
version anyway.

So, after all that, I think this is probably good as-is. _Maybe_ we
could rename string_stream_get_string() to something like
string_stream_get_managed_string(), now that it's the only function
which is "managed" as a KUnit resource, but we can equally put that
off until we need to add an unmanaged version.

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Re-work string_stream so that it is not tied to a struct kunit. This is
> to allow using it for the log of struct kunit_suite.
>
> Instead of resource-managing individual allocations the whole string_stream
> object can be resource-managed as a single object:
>
> alloc_string_stream() API is unchanged and takes a pointer to a
> struct kunit but it now registers the returned string_stream object to
> be resource-managed.
>
> raw_alloc_string_stream() is a new function that allocates a
> bare string_stream without any association to a struct kunit.
>
> free_string_stream() is a new function that frees a resource-managed
> string_stream allocated by alloc_string_stream().
>
> raw_free_string_stream() is a new function that frees a non-managed
> string_stream allocated by raw_alloc_string_stream().
>
> The confusing function string_stream_destroy() has been removed. This only
> called string_stream_clear() but didn't free the struct string_stream.
> Instead string_stream_clear() has been exported, and the new functions use
> the more conventional naming of "free" as the opposite of "alloc".
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

I'm in favour of this. Should we go further and get rid of the struct
kunit member from string_stream totally?

Also, note that the kunit_action_t casting is causing warnings on some
clang configs (and technically isn't valid C). Personally, I still
like it, but expect more emails from the kernel test robot and others.

Reviewed-by: David Gow <davi...@google.com>

As the kernel test robot notes, this sort of function pointer casting
is not technically valid C, and some compiler setups are starting to
warn on it.

Personally, I'm still okay with it, but expect a bunch of robot email
complaining about it if it lands. If more compilers / configs start
warning, though, I'll try to fix all callers of the KUnit action
functions which are affected.

> +
> + return stream;
> +}
> +
> +void free_string_stream(struct kunit *test, struct string_stream *stream)
> +{
> + if (!stream)
> + return;
> +
> + kunit_release_action(test, (kunit_action_t *)raw_free_string_stream, (void *)stream);

(As above.)

> }
> diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h
> index 6b4a747881c5..fbeda486382a 100644
> --- a/lib/kunit/string-stream.h
> +++ b/lib/kunit/string-stream.h
> @@ -23,14 +23,24 @@ struct string_stream {
> struct list_head fragments;
> /* length and fragments are protected by this lock */
> spinlock_t lock;
> +
> + /*
> + * Pointer to kunit this stream is associated with, or NULL if
> + * not associated with a kunit.
> + */
> struct kunit *test;
> +

Can we just get rid of this totally? I don't think anything's actually
using it now. (Or have I missed something?)

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> string_stream_resource_free_test() allocates a resource-managed
> string_stream and tests that raw_free_string_stream() is called when
> the test resources are cleaned up.
>
> string_stream_init_test() is extended to test allocating a
> string_stream that is not resource-managed.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Glad to see this tested. It's nice to see the static_stub being useful
here, too.

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

Again, this will probably annoy sparse, so expect an email from ktr.
We'll look into how to fix this in either KUnit or sparse separately.

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Replace the fixed-size log buffer with a string_stream so that the
> log can grow as lines are added.
>
> The existing kunit log tests have been updated for using a
> string_stream as the log. As they now depend on string_stream
> functions they cannot build when kunit-test is a module. They have
> been moved to a separate source file to be built only if kunit-test
> is built-in.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Yay! This is much nicer.

A part of me wonders if it makes sense to do something to allow the
tests to continue to work as a module (either exporting the needed
string stream functions, or moving the tests into the main KUnit
module, and just exporting the suite definition to a stub module).
Probably not worth it, though...

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Add a test of the speed and memory use of string_stream.
>
> string_stream_performance_test() doesn't actually "test" anything (it
> cannot fail unless the system has run out of allocatable memory) but it
> measures the speed and memory consumption of the string_stream and reports
> the result.
>
> This allows changes in the string_stream implementation to be compared.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Thanks, this is very useful.

My results are (UML):
# string_stream_performance_test: Time elapsed: 5021 us
# string_stream_performance_test: Total string length: 573890
# string_stream_performance_test: Bytes requested: 823930
# string_stream_performance_test: Actual bytes allocated: 1048312

KUnit isn't really ideal for performance testing, but this works well
enough and fits in well alongside the other string stream tests.

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

[David Gow]

On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> This patch chain changes the logging implementation to use string_stream
> so that the log will grow dynamically.
>
> The first 8 patches add test code for string_stream, and make some
> changes to string_stream needed to be able to use it for the log.
>
> The final patch adds a performance report of string_stream.
>
> CHANGES SINCE V3:
>
> Completely rewritten to use string_stream instead of implementing a
> separate extending-buffer implementation for logging.
>
> I have used the performance test from the final patch on my original
> fixed-size-fragment implementation from V3 to get a comparison of the
> two implementations (run on i3-8145UE CPU @ 2.20GHz):
>
> string_stream V3 fixed-size-buffer
> Time elapsed: 7748 us 3251 us
> Total string length: 573890 573890
> Bytes requested: 823994 728336
> Actual bytes allocated: 1061440 728352
>
> I don't think the difference is enough to be worth complicating the
> string_stream implementation with my fixed-fragment implementation from
> V3 of this patch chain.
>

Thanks very much! I think this is good to go: I've added a few small
comments on various patches, but none of them are serious enough to
make me feel we _need_ a new version.
I'll leave it for a day or two in case there are any other comments or
any changes you want to make, otherwise this can be merged.

I agree the performance difference isn't worth the effort. If we
change our minds and want to change the implementation over later,
that shouldn't be a problem either. So let's stick with it as is.

Cheers,
-- David

[Richard Fitzgerald]

On 15/8/23 10:16, David Gow wrote:
> On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
> <r...@opensource.cirrus.com> wrote:
>>
>> Pass a struct kunit* and gfp_t to string_stream_get_string(). Allocate
>> the returned buffer using these instead of using the stream->test and
>> stream->gfp.
>>
>> This is preparation for removing the dependence of string_stream on
>> struct kunit, so that string_stream can be used for the debugfs log.
>>
>> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
>> ---
>
> Makes sense to me.
>
> I think that, if we weren't going to remove the struct kunit
> dependency, we'd want to either keep a version of
> string_stream_get_string() which uses it, or, e.g., fall back to it if
> the struct passed in is NULL.
>

That was my first thought. But I thought that was open to subtle
accidental bugs in calling code - it might return you a managed
allocation, or it might return you an unmanaged allocation that you
must free.

As there weren't many callers of string_stream_get_string() I decided
to go with changing the API to pass in test and gfp like other managed
allocations. This makes it more generalized, since the returned buffer
is not part of the stream itself, it's a temporary buffer owned by the
caller. It also makes it clearer that what you are getting back is
likely to be a managed allocation.

If you'd prefer to leave string_stream_get_string() as it was, or make
it return an unmanged buffer, I can send a new version.

> The other option is to have a version which doesn't manage the string
> at all, so just takes a gfp and requires the caller to free it (or

I didn't add a companion function to get a raw unmanaged string buffer
because there's nothing that needs it. It could be added later if
it's needed.

> register an action to do so). If we did that, we could get rid of the
> struct kunit pointer totally (though it may be better to keep it and
> have both versions).
>

Another option is to make string_stream_get_string() return a raw
buffer and add a kunit_string_stream_geT_string() that returns a
managed buffer. This follows some consistency with the normal mallocs
where kunit_xxxx() is the managed version.

> So -- to switch to some stream-of-consciousness thoughts about this --
> basically there are three possible variants of
> string_stream_get_string():
> - Current version: uses the stream's struct kunit. Useless if this is
> NULL, but very convenient otherwise.
> - This patch: manage the string using the struct kunit passed as an
> argument. Still can't be used directly outside a test, but adds enough
> flexibility to get _append to work.
> - Totally unmanaged: the generated string is allocated separately, and
> must be freed (directly or in a deferred action) by the caller. Works
> well outside tests, but less convenient.
>
> Personally, I feel that the design of string_stream is heading towards
> the third option. By the end of this series, everything uses
> _string_stream_concatenate_to_buf() anyway. There's only one call to
> string_stream_get_string() outside of the logging / string_stream
> tests, and all that does is log the results (and it has a fallback to
> log each fragment separately if the allocation fails).
>
> Then again, if this is only really being used in tests, then we can
> probably just stick with string_stream_get_string() as-is, remove the
> string_stream->test member totally, and if we need it, we can make
> _string_stream_concatenate_to_buf() public, or add an unmanaged
> version anyway.
>

I didn't remove ->test because there's some existing code in assert.c
that uses it, and I didn't want to break that.

[Richard Fitzgerald]

On 15/8/23 10:16, David Gow wrote:

> On Mon, 14 Aug 2023 at 21:23, Richard Fitzgerald
> <r...@opensource.cirrus.com> wrote:
>>
>> Re-work string_stream so that it is not tied to a struct kunit. This is
>> to allow using it for the log of struct kunit_suite.
>>
>> Instead of resource-managing individual allocations the whole string_stream
>> object can be resource-managed as a single object:
>>
>> alloc_string_stream() API is unchanged and takes a pointer to a
>> struct kunit but it now registers the returned string_stream object to
>> be resource-managed.
>>
>> raw_alloc_string_stream() is a new function that allocates a
>> bare string_stream without any association to a struct kunit.
>>
>> free_string_stream() is a new function that frees a resource-managed
>> string_stream allocated by alloc_string_stream().
>>
>> raw_free_string_stream() is a new function that frees a non-managed
>> string_stream allocated by raw_alloc_string_stream().
>>
>> The confusing function string_stream_destroy() has been removed. This only
>> called string_stream_clear() but didn't free the struct string_stream.
>> Instead string_stream_clear() has been exported, and the new functions use
>> the more conventional naming of "free" as the opposite of "alloc".
>>
>> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
>> ---
>
> I'm in favour of this. Should we go further and get rid of the struct
> kunit member from string_stream totally?
>

I can do that. I was worried about some hairy-looking code in assert.c
that used stream->test. But I've just looked at it again and it's
really quite simple, and doesn't even need ->test. is_literal()
allocates a temporary managed buffer, but it frees it before returning
so it doesn't need to be managed.

> Also, note that the kunit_action_t casting is causing warnings on some
> clang configs (and technically isn't valid C). Personally, I still
> like it, but expect more emails from the kernel test robot and others.
>

I can send a new version to fix that.

[Rae Moar]

On Mon, Aug 14, 2023 at 9:23 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>
> Add an optional feature to string_stream that will append a newline to
> any added string that does not already end with a newline. The purpose
> of this is so that string_stream can be used to collect log lines.
>
> This is enabled/disabled by calling string_stream_set_append_newlines().
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

Hello!

I just wanted to look over handling the newline and this looks good to
me. I agree with David that I don't mind one extra allocated byte.

Reviewed-by: Rae Moar <rm...@google.com>

Thanks!
-Rae

[Rae Moar]

On Mon, Aug 14, 2023 at 9:23 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Add test cases for testing the string_stream feature that appends a
> newline to strings that do not already end with a newline.
>
> string_stream_no_auto_newline_test() tests with this feature disabled.
> Newlines should not be added or dropped.
>
> string_stream_auto_newline_test() tests with this feature enabled.
> Newlines should be added to lines that do not end with a newline.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

Hello!

These two cases seem very clean to me. I appreciate the organization
and commenting on the tests.

Reviewed-by: Rae Moar <rm...@google.com>

Thanks!
-Rae

[Rae Moar]

On Mon, Aug 14, 2023 at 9:23 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> This patch chain changes the logging implementation to use string_stream
> so that the log will grow dynamically.
>
> The first 8 patches add test code for string_stream, and make some
> changes to string_stream needed to be able to use it for the log.
>
> The final patch adds a performance report of string_stream.
>
> CHANGES SINCE V3:
>
> Completely rewritten to use string_stream instead of implementing a
> separate extending-buffer implementation for logging.
>
> I have used the performance test from the final patch on my original
> fixed-size-fragment implementation from V3 to get a comparison of the
> two implementations (run on i3-8145UE CPU @ 2.20GHz):
>
> string_stream V3 fixed-size-buffer
> Time elapsed: 7748 us 3251 us
> Total string length: 573890 573890
> Bytes requested: 823994 728336
> Actual bytes allocated: 1061440 728352
>
> I don't think the difference is enough to be worth complicating the
> string_stream implementation with my fixed-fragment implementation from
> V3 of this patch chain.

Hello!

I just wanted to note that I have tested this series and it looks good
to me. I specifically looked over the newline handling and that looks
really good.

I understand you will likely be doing a new version of this. But other
than the things noted in comments by David, this seems to be working
really well.

Tested-by: Rae Moar <rm...@google.com>

Thanks for all the work you did in moving this framework to string-stream!
-Rae

[David Gow]

Yeah, let's get rid of that. Having a stream->kunit exist but be NULL
half the time is asking for issues down the line.

> > Also, note that the kunit_action_t casting is causing warnings on some
> > clang configs (and technically isn't valid C). Personally, I still
> > like it, but expect more emails from the kernel test robot and others.
> >
>
> I can send a new version to fix that.
>

That's probably best. If you want to keep it as-is, I'll fight for it,
but it's probably better to err on the side of not introducing the
warnings.

Thanks,
-- David

[David Gow]

Fair enough.

> > register an action to do so). If we did that, we could get rid of the
> > struct kunit pointer totally (though it may be better to keep it and
> > have both versions).
> >
>
> Another option is to make string_stream_get_string() return a raw
> buffer and add a kunit_string_stream_geT_string() that returns a
> managed buffer. This follows some consistency with the normal mallocs
> where kunit_xxxx() is the managed version.
>

Ooh... I like this best. Let's go with that.

> > So -- to switch to some stream-of-consciousness thoughts about this --
> > basically there are three possible variants of
> > string_stream_get_string():
> > - Current version: uses the stream's struct kunit. Useless if this is
> > NULL, but very convenient otherwise.
> > - This patch: manage the string using the struct kunit passed as an
> > argument. Still can't be used directly outside a test, but adds enough
> > flexibility to get _append to work.
> > - Totally unmanaged: the generated string is allocated separately, and
> > must be freed (directly or in a deferred action) by the caller. Works
> > well outside tests, but less convenient.
> >
> > Personally, I feel that the design of string_stream is heading towards
> > the third option. By the end of this series, everything uses
> > _string_stream_concatenate_to_buf() anyway. There's only one call to
> > string_stream_get_string() outside of the logging / string_stream
> > tests, and all that does is log the results (and it has a fallback to
> > log each fragment separately if the allocation fails).
> >
> > Then again, if this is only really being used in tests, then we can
> > probably just stick with string_stream_get_string() as-is, remove the
> > string_stream->test member totally, and if we need it, we can make
> > _string_stream_concatenate_to_buf() public, or add an unmanaged
> > version anyway.
> >
>
> I didn't remove ->test because there's some existing code in assert.c
> that uses it, and I didn't want to break that.
>

Since it seems the assert stuff isn't too difficult to make unmanaged,
can we go ahead and remove ->test?

If it turns out to be too nasty, let me know and we'll keep it (it's
not worth making major excavations), but I'd prefer to get rid of it
if we can.

Thanks,
-- David

[Richard Fitzgerald]

I was busy last week with other things and have only got back to looking
at this.

I'm trying to decide what to do with string_stream_get_string() and I'd
value an opinion.

The only use for a test-managed get_string() is the string_stream test.
So I've thought of 4 options:

1) Add a kunit_string_stream_get_string() anyway. But only the test code
uses it.

2) Change the tests to have a local function to do the same thing, and
add an explicit test case for the (unmanaged)
string_stream_get_string() that ensures it's freed.

3) Change the tests to have a local function to get the string, which
calls string_stream_get_string() then copies it to a test-managed buffer
and frees the unmanaged buffer.

4) Implement a kunit_kfree_on_exit() function that takes an already-
allocated buffer and kfree()s it when the test exists, so that we can
do:

// on success kunit_kfree_on_exit() returns the buffer it was given
str = kunit_kfree_on_exit(test, string_stream_get_string(stream));
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, str);

I'm leaning towards (2) but open to going with any of the other options.

[David Gow]

On Tue, 22 Aug 2023 at 00:10, Richard Fitzgerald

I'd be happy with any of those options (though 3 is my least favourite).

There is already a kfree_at_end() function in the executor test. It's
not quite as nice as your proposed kunit_kfree_on_exit() function (I
like this idea of passing the pointer through), but it proves the
concept. I think your version of this would be a useful thing to have
as an actual part of the KUnit API, rather than just a per-test hack:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/lib/kunit/executor_test.c#n131

Cheers,
-- David

[Richard Fitzgerald]

Add an optional feature to string_stream that will append a newline to
any added string that does not already end with a newline. The purpose
of this is so that string_stream can be used to collect log lines.

This is enabled/disabled by calling string_stream_set_append_newlines().

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream.c | 28 +++++++++++++++++++++-------
lib/kunit/string-stream.h | 7 +++++++
2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index ed24d86af9f5..1dcf6513b692 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c

diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h
index b669f9a75a94..048930bf97f0 100644
--- a/lib/kunit/string-stream.h
+++ b/lib/kunit/string-stream.h

[Richard Fitzgerald]

Add test cases for testing the string_stream feature that appends a
newline to strings that do not already end with a newline.

string_stream_no_auto_newline_test() tests with this feature disabled.
Newlines should not be added or dropped.

string_stream_auto_newline_test() tests with this feature enabled.
Newlines should be added to lines that do not end with a newline.

string_stream_append_auto_newline_test() tests appending the
content of one stream to another stream when the target stream
has newline appending enabled.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

Changes since V4:
- string_stream_append_auto_newline_test() doesn't clear the destination
stream_1 between the newline and no-newline case. This is just a
simplification of the code.

- string_stream_no_auto_newline_test() uses the same set of test strings
as string_stream_auto_newline_test().
---
lib/kunit/string-stream-test.c | 93 ++++++++++++++++++++++++++++++++++
1 file changed, 93 insertions(+)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 2b761ba01835..2a9936db1b9f 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -32,6 +32,7 @@ static void string_stream_init_test(struct kunit *test)

KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));
KUNIT_EXPECT_PTR_EQ(test, stream->test, test);
KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL);
+ KUNIT_EXPECT_FALSE(test, stream->append_newlines);

KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
}

@@ -214,6 +215,45 @@ static void string_stream_append_test(struct kunit *test)
KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), stream_2_content);
}

+/* Appending the content of one string stream to one with auto-newlining. */
+static void string_stream_append_auto_newline_test(struct kunit *test)
+{
+ struct string_stream *stream_1, *stream_2;
+
+ /* Stream 1 has newline appending enabled */
+ stream_1 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);
+ string_stream_set_append_newlines(stream_1, true);
+ KUNIT_EXPECT_TRUE(test, stream_1->append_newlines);
+
+ /* Stream 2 does not append newlines */
+ stream_2 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2);
+
+ /* Appending a stream with a newline should not add another newline */
+ string_stream_add(stream_1, "Original string\n");
+ string_stream_add(stream_2, "Appended content\n");
+ string_stream_add(stream_2, "More stuff\n");
+ string_stream_append(stream_1, stream_2);
+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1),
+ "Original string\nAppended content\nMore stuff\n");
+
+ string_stream_destroy(stream_2);
+ stream_2 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2);
+
+ /*
+ * Appending a stream without newline should add a final newline.
+ * The appended string_stream is treated as a single string so newlines
+ * should not be inserted between fragments.
+ */
+ string_stream_add(stream_2, "Another");
+ string_stream_add(stream_2, "And again");
+ string_stream_append(stream_1, stream_2);
+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1),
+ "Original string\nAppended content\nMore stuff\nAnotherAnd again\n");
+}
+

/* Adding an empty string should not create a fragment. */

static void string_stream_append_empty_string_test(struct kunit *test)

{
@@ -237,12 +277,65 @@ static void string_stream_append_empty_string_test(struct kunit *test)
KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream), "Add this line");

}

+/* Adding strings without automatic newline appending */
+static void string_stream_no_auto_newline_test(struct kunit *test)
+{
+ struct string_stream *stream;
+
+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+ /*

+ * Add some strings with and without newlines. All formatted newlines
+ * should be preserved. It should not add any extra newlines.

+ */
+ string_stream_add(stream, "One");
+ string_stream_add(stream, "Two\n");

+ string_stream_add(stream, "%s\n", "Three");
+ string_stream_add(stream, "%s", "Four\n");
+ string_stream_add(stream, "Five\n%s", "Six");
+ string_stream_add(stream, "Seven\n\n");
+ string_stream_add(stream, "Eight");

+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream),
+ "OneTwo\nThree\nFour\nFive\nSixSeven\n\nEight");

+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream),

+ "One\nTwo\nThree\nFour\nFive\nSix\nSeven\n\nEight\n");
+}
+
static struct kunit_case string_stream_test_cases[] = {
KUNIT_CASE(string_stream_init_test),
KUNIT_CASE(string_stream_line_add_test),
KUNIT_CASE(string_stream_variable_length_line_test),
KUNIT_CASE(string_stream_append_test),

+ KUNIT_CASE(string_stream_append_auto_newline_test),

[Richard Fitzgerald]

Replace the fixed-size log buffer with a string_stream so that the
log can grow as lines are added.

string_stream_clear() has been made public for the log truncation
done in kunit_init_test().

The existing kunit log tests have been updated for using a

string_stream as the log. No new test have been added because there
are already tests for the underlying string_stream.

As the log tests now depend on string_stream functions they cannot
build when kunit-test is a module. They have been surrounded by
a #if to replace them with skipping version when the test is
build as a module. Though this isn't pretty, it avoids moving
code to another file while that code is also being changed.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---
Changes since V4:

- Don't move the log tests to another file. Deal with only including them
when the test is built-in by wrapping them in a #if. This is to simplify
code review, because it avoids having a block of code which moves from
one file to another but at the same time the code has changed.
- Use kunit_add_action() to automatically free the string returned by
string_stream_get_string().
---
include/kunit/test.h | 14 +++++-------
lib/kunit/debugfs.c | 36 +++++++++++++++++++-----------
lib/kunit/kunit-test.c | 46 ++++++++++++++++++++++++++++++++-------
lib/kunit/string-stream.c | 2 +-
lib/kunit/string-stream.h | 2 ++
lib/kunit/test.c | 44 +++++--------------------------------
6 files changed, 75 insertions(+), 69 deletions(-)

diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c
index 22c5c496a68f..270d185737e6 100644

--- a/lib/kunit/debugfs.c
+++ b/lib/kunit/debugfs.c
@@ -37,14 +37,21 @@ void kunit_debugfs_init(void)
debugfs_rootdir = debugfs_create_dir(KUNIT_DEBUGFS_ROOT, NULL);
}

-static void debugfs_print_result(struct seq_file *seq,
- struct kunit_suite *suite,
- struct kunit_case *test_case)
+static void debugfs_print_result(struct seq_file *seq, struct string_stream *log)
{
- if (!test_case || !test_case->log)

+ struct string_stream_fragment *frag_container;
+

+ suite->log = alloc_string_stream(GFP_KERNEL);

+ string_stream_set_append_newlines(suite->log, true);
+
+ kunit_suite_for_each_test_case(suite, test_case) {

+ test_case->log = alloc_string_stream(GFP_KERNEL);

+ string_stream_set_append_newlines(test_case->log, true);
+ }

suite->debugfs = debugfs_create_dir(suite->name, debugfs_rootdir);

@@ -121,7 +131,7 @@ void kunit_debugfs_destroy_suite(struct kunit_suite *suite)
struct kunit_case *test_case;

debugfs_remove_recursive(suite->debugfs);
- kfree(suite->log);

+ string_stream_destroy(suite->log);

kunit_suite_for_each_test_case(suite, test_case)
- kfree(test_case->log);

+ string_stream_destroy(test_case->log);
}
diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index 83d8e90ca7a2..616e287aa4bf 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -8,6 +8,7 @@
#include <kunit/test.h>
#include <kunit/test-bug.h>

+#include "string-stream.h"
#include "try-catch-impl.h"

struct kunit_try_catch_test_context {
@@ -530,12 +531,19 @@ static struct kunit_suite kunit_resource_test_suite = {
.test_cases = kunit_resource_test_cases,
};

+/*
+ * Log tests call string_stream functions, which aren't exported. So only
+ * build this code if this test is built-in.
+ */
+#if IS_BUILTIN(CONFIG_KUNIT_TEST)
static void kunit_log_test(struct kunit *test)
{

struct kunit_suite suite;
+ char *full_log;

- suite.log = kunit_kzalloc(test, KUNIT_LOG_SIZE, GFP_KERNEL);

+ suite.log = kunit_alloc_string_stream(test, GFP_KERNEL);

KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ string_stream_set_append_newlines(suite.log, true);

kunit_log(KERN_INFO, test, "put this in log.");

kunit_log(KERN_INFO, test, "this too.");

@@ -543,14 +551,21 @@ static void kunit_log_test(struct kunit *test)

kunit_log(KERN_INFO, &suite, "along with this.");

#ifdef CONFIG_KUNIT_DEBUGFS
+ KUNIT_EXPECT_TRUE(test, test->log->append_newlines);
+

+ full_log = string_stream_get_string(test->log);
+ kunit_add_action(test, (kunit_action_t *)kfree, full_log);

KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(test->log, "put this in log."));

+ strstr(full_log, "put this in log."));

KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(test->log, "this too."));

+ strstr(full_log, "this too."));
+

+ full_log = string_stream_get_string(suite.log);
+ kunit_add_action(test, (kunit_action_t *)kfree, full_log);

KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(suite.log, "add to suite log."));

+ strstr(full_log, "add to suite log."));

KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(suite.log, "along with this."));

+ strstr(full_log, "along with this."));

#else
KUNIT_EXPECT_NULL(test, test->log);
#endif
@@ -558,15 +573,30 @@ static void kunit_log_test(struct kunit *test)

static void kunit_log_newline_test(struct kunit *test)
{

+ char *full_log;

+
kunit_info(test, "Add newline\n");

if (test->log) {
- KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(test->log, "Add newline\n"),
- "Missing log line, full log:\n%s", test->log);
- KUNIT_EXPECT_NULL(test, strstr(test->log, "Add newline\n\n"));

+ full_log = string_stream_get_string(test->log);
+ kunit_add_action(test, (kunit_action_t *)kfree, full_log);

+ KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(full_log, "Add newline\n"),
+ "Missing log line, full log:\n%s", full_log);
+ KUNIT_EXPECT_NULL(test, strstr(full_log, "Add newline\n\n"));

} else {

kunit_skip(test, "only useful when debugfs is enabled");
}
}

+#else

+static void kunit_log_test(struct kunit *test)
+{

+ kunit_skip(test, "Log tests only run when built-in");

+}
+
+static void kunit_log_newline_test(struct kunit *test)
+{

+ kunit_skip(test, "Log tests only run when built-in");
+}
+#endif /* IS_BUILTIN(CONFIG_KUNIT_TEST) */

static struct kunit_case kunit_log_test_cases[] = {
KUNIT_CASE(kunit_log_test),
diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index d2ded5207e9e..a6f3616c2048 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -99,7 +99,7 @@ int string_stream_add(struct string_stream *stream, const char *fmt, ...)

return result;
}

-static void string_stream_clear(struct string_stream *stream)
+void string_stream_clear(struct string_stream *stream)
{
struct string_stream_fragment *frag_container, *frag_container_safe;

diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h
index c55925a9b67f..7be2450c7079 100644
--- a/lib/kunit/string-stream.h
+++ b/lib/kunit/string-stream.h
@@ -42,6 +42,8 @@ int __printf(2, 0) string_stream_vadd(struct string_stream *stream,

const char *fmt,
va_list args);

+void string_stream_clear(struct string_stream *stream);
+
char *string_stream_get_string(struct string_stream *stream);

int string_stream_append(struct string_stream *stream,
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 2ad45a4ac06a..b153808ff1ec 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c

[Richard Fitzgerald]

Replace the minimal tests with more-thorough testing.

string_stream_init_test() tests that struct string_stream is
initialized correctly.

string_stream_line_add_test() adds a series of numbered lines and
checks that the resulting string contains all the lines.

string_stream_variable_length_line_test() adds a large number of
lines of varying length to create many fragments, then tests that all
lines are present.

string_stream_append_test() tests various cases of using

string_stream_append() to append the content of one stream to another.

Adds string_stream_append_empty_string_test() to test that adding an
empty string to a string_stream doesn't create a new empty fragment.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---
Changes since V4:

- Test cases for appending empty strings have been squashed into this
patch.
- Call to string_stream_get_string() is wrapped in a local function
get_concatenated_string(). This is to avoid a lot of code churn later
when string_stream_get_string() is changed to return an unmanaged buffer.
---
lib/kunit/string-stream-test.c | 232 ++++++++++++++++++++++++++++++---
1 file changed, 216 insertions(+), 16 deletions(-)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 110f3a993250..2b761ba01835 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -11,38 +11,238 @@

#include "string-stream.h"

-static void string_stream_test_empty_on_creation(struct kunit *test)

+static char *get_concatenated_string(struct kunit *test, struct string_stream *stream)

{
- struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL);

+ char *str = string_stream_get_string(stream);
+
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, str);
+
+ return str;
+}
+

+/* string_stream object is initialized correctly. */

+static void string_stream_init_test(struct kunit *test)

+{
+ struct string_stream *stream;
+
+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+

+ KUNIT_EXPECT_EQ(test, stream->length, 0);
+ KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));

+ KUNIT_EXPECT_PTR_EQ(test, stream->test, test);

+ KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL);

KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
}

-static void string_stream_test_not_empty_after_add(struct kunit *test)
+/*
+ * Add a series of lines to a string_stream. Check that all lines
+ * appear in the correct order and no characters are dropped.
+ */
+static void string_stream_line_add_test(struct kunit *test)
{
- struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL);

+ struct string_stream *stream;

+ char line[60];
+ char *concat_string, *pos, *string_end;
+ size_t len, total_len;
+ int num_lines, i;

- string_stream_add(stream, "Foo");

+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

- KUNIT_EXPECT_FALSE(test, string_stream_is_empty(stream));
+ /* Add series of sequence numbered lines */
+ total_len = 0;
+ for (i = 0; i < 100; ++i) {
+ len = snprintf(line, sizeof(line),
+ "The quick brown fox jumps over the lazy penguin %d\n", i);

+
+ /* Sanity-check that our test string isn't truncated */
+ KUNIT_ASSERT_LT(test, len, sizeof(line));
+
+ string_stream_add(stream, line);
+ total_len += len;
+ }
+ num_lines = i;
+

+ concat_string = get_concatenated_string(test, stream);

+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test, concat_string);
+ KUNIT_EXPECT_EQ(test, strlen(concat_string), total_len);

+
+ /*

+ struct string_stream *stream;

+ struct rnd_state rnd;
+ char *concat_string, *pos, *string_end;
+ size_t offset, total_len;
+ int num_lines, i;

- string_stream_add(stream, "Foo");
- string_stream_add(stream, " %s", "bar");

+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

- output = string_stream_get_string(stream);
- KUNIT_ASSERT_STREQ(test, output, "Foo bar");
+ /*
+ * Log many lines of varying lengths until we have created
+ * many fragments.
+ * The "randomness" must be repeatable.
+ */
+ prandom_seed_state(&rnd, 3141592653589793238ULL);

+ total_len = 0;

+ for (i = 0; i < 100; ++i) {
+ offset = prandom_u32_state(&rnd) % (sizeof(line) - 1);
+ string_stream_add(stream, "%s\n", &line[offset]);
+ total_len += sizeof(line) - offset;
+ }
+ num_lines = i;
+

+ concat_string = get_concatenated_string(test, stream);

+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test, concat_string);
+ KUNIT_EXPECT_EQ(test, strlen(concat_string), total_len);

+
+ /*

+ static const char * const strings_2[] = {
+ "Apple", "Pear", "Orange", "Banana", "Grape", "Apricot",

+ };
+ struct string_stream *stream_1, *stream_2;

+ const char *original_content, *stream_2_content;
+ char *combined_content;
+ size_t combined_length;
+ int i;
+

+ stream_1 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);
+

+ stream_2 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2);
+

+ /* Append content of empty stream to empty stream */
+ string_stream_append(stream_1, stream_2);

+ KUNIT_EXPECT_EQ(test, strlen(get_concatenated_string(test, stream_1)), 0);
+
+ /* Add some data to stream_1 */
+ for (i = 0; i < ARRAY_SIZE(strings_1); ++i)
+ string_stream_add(stream_1, "%s\n", strings_1[i]);
+
+ original_content = get_concatenated_string(test, stream_1);
+
+ /* Append content of empty stream to non-empty stream */
+ string_stream_append(stream_1, stream_2);
+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), original_content);

+
+ /* Add some data to stream_2 */
+ for (i = 0; i < ARRAY_SIZE(strings_2); ++i)
+ string_stream_add(stream_2, "%s\n", strings_2[i]);
+

+ /* Append content of non-empty stream to non-empty stream */
+ string_stream_append(stream_1, stream_2);
+
+ /*

+ * End result should be the original content of stream_1 plus
+ * the content of stream_2.
+ */

+ stream_2_content = get_concatenated_string(test, stream_2);
+ combined_length = strlen(original_content) + strlen(stream_2_content);
+ combined_length++; /* for terminating \0 */
+ combined_content = kunit_kmalloc(test, combined_length, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, combined_content);
+ snprintf(combined_content, combined_length, "%s%s", original_content, stream_2_content);
+
+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), combined_content);
+
+ /* Append content of non-empty stream to empty stream */
+ string_stream_destroy(stream_1);
+

+ stream_1 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);
+

+ string_stream_append(stream_1, stream_2);
+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), stream_2_content);
+}
+
+/* Adding an empty string should not create a fragment. */
+static void string_stream_append_empty_string_test(struct kunit *test)

+{
+ struct string_stream *stream;

+ int original_frag_count;

+
+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+

+ /* Formatted empty string */
+ string_stream_add(stream, "%s", "");
+ KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
+ KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));

+

+ /* Adding an empty string to a non-empty stream */

+ string_stream_add(stream, "Add this line");
+ original_frag_count = list_count_nodes(&stream->fragments);
+

+ string_stream_add(stream, "%s", "");

+ KUNIT_EXPECT_EQ(test, list_count_nodes(&stream->fragments), original_frag_count);
+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream), "Add this line");

}

static struct kunit_case string_stream_test_cases[] = {

- KUNIT_CASE(string_stream_test_empty_on_creation),
- KUNIT_CASE(string_stream_test_not_empty_after_add),
- KUNIT_CASE(string_stream_test_get_string),
+ KUNIT_CASE(string_stream_init_test),
+ KUNIT_CASE(string_stream_line_add_test),
+ KUNIT_CASE(string_stream_variable_length_line_test),

+ KUNIT_CASE(string_stream_append_test),
+ KUNIT_CASE(string_stream_append_empty_string_test),
{}
};

--
2.30.2

[Richard Fitzgerald]

Re-work string_stream so that it is not tied to a struct kunit. This is
to allow using it for the log of struct kunit_suite.

Instead of resource-managing individual allocations the whole string_stream

can be resource-managed, if required.

alloc_string_stream() now allocates a string stream that is
not resource-managed.

string_stream_destroy() now works on an unmanaged string_stream
allocated by alloc_string_stream() and frees the entire
string_stream (previously it only freed the fragments).

For resource-managed allocations use kunit_alloc_string_stream()
and kunit_free_string_stream().

In addition to this, string_stream_get_string() now returns an
unmanaged buffer that the caller must kfree().

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---
Changes since V4:

- Adding the kunit_[alloc|free]_string_stream() functions has been split
out into the previous patch to reduce the amount of code churn in
this patch.
- string_stream_destroy() has been kept and re-used instead of replacing
it with a new function.
- string_stream_get_string() now returns an unmanaged buffer. This avoids
a large code change to string_stream_append().
- Added wrapper function for resource free to prevent the type warning of
passing string_stream_destroy() directly to kunit_add_action_or_reset().
---
lib/kunit/string-stream-test.c | 2 +-
lib/kunit/string-stream.c | 59 ++++++++++++++++++++++------------
lib/kunit/string-stream.h | 4 ++-
lib/kunit/test.c | 2 +-
4 files changed, 44 insertions(+), 23 deletions(-)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 89549c237069..45a2d221f1b5 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -16,6 +16,7 @@ static char *get_concatenated_string(struct kunit *test, struct string_stream *s
char *str = string_stream_get_string(stream);

KUNIT_ASSERT_NOT_ERR_OR_NULL(test, str);
+ kunit_add_action(test, (kunit_action_t *)kfree, (void *)str);

return str;
}
@@ -30,7 +31,6 @@ static void string_stream_init_test(struct kunit *test)

KUNIT_EXPECT_EQ(test, stream->length, 0);
KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));
- KUNIT_EXPECT_PTR_EQ(test, stream->test, test);
KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL);
KUNIT_EXPECT_FALSE(test, stream->append_newlines);

diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index 12ecf15e1f6b..c39f1cba3bcd 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c

@@ -65,9 +63,7 @@ int string_stream_vadd(struct string_stream *stream,

/* Need space for null byte. */

buf_len++;

- frag_container = alloc_string_stream_fragment(stream->test,
- buf_len,
- stream->gfp);

+ frag_container = alloc_string_stream_fragment(buf_len, stream->gfp);
if (IS_ERR(frag_container))
return PTR_ERR(frag_container);

@@ -111,7 +107,7 @@ static void string_stream_clear(struct string_stream *stream)

frag_container_safe,
&stream->fragments,
node) {
- string_stream_fragment_destroy(stream->test, frag_container);
+ string_stream_fragment_destroy(frag_container);
}
stream->length = 0;
spin_unlock(&stream->lock);

@@ -123,7 +119,7 @@ char *string_stream_get_string(struct string_stream *stream)

size_t buf_len = stream->length + 1; /* +1 for null byte. */
char *buf;

- buf = kunit_kzalloc(stream->test, buf_len, stream->gfp);

+ buf = kzalloc(buf_len, stream->gfp);
if (!buf)
return NULL;

@@ -139,13 +135,17 @@ int string_stream_append(struct string_stream *stream,
struct string_stream *other)
{
const char *other_content;
+ int ret;

other_content = string_stream_get_string(other);

if (!other_content)

return -ENOMEM;

- return string_stream_add(stream, other_content);

+ ret = string_stream_add(stream, other_content);
+ kfree(other_content);

+
+ return ret;
}

bool string_stream_is_empty(struct string_stream *stream)

@@ -153,16 +153,15 @@ bool string_stream_is_empty(struct string_stream *stream)
return list_empty(&stream->fragments);
}

-static struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp)
+struct string_stream *alloc_string_stream(gfp_t gfp)

{
struct string_stream *stream;

- stream = kunit_kzalloc(test, sizeof(*stream), gfp);
+ stream = kzalloc(sizeof(*stream), gfp);
if (!stream)
return ERR_PTR(-ENOMEM);

stream->gfp = gfp;
- stream->test = test;
INIT_LIST_HEAD(&stream->fragments);
spin_lock_init(&stream->lock);

@@ -171,15 +170,35 @@ static struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp)

void string_stream_destroy(struct string_stream *stream)

{
+ if (!stream)
+ return;
+

string_stream_clear(stream);
+ kfree(stream);
+}
+

+static void resource_free_string_stream(void *p)
+{
+ struct string_stream *stream = p;
+
+ string_stream_destroy(stream);
}

struct string_stream *kunit_alloc_string_stream(struct kunit *test, gfp_t gfp)
{
- return alloc_string_stream(test, gfp);

+ struct string_stream *stream;
+

+ stream = alloc_string_stream(gfp);

+ if (IS_ERR(stream))
+ return stream;
+

+ if (kunit_add_action_or_reset(test, resource_free_string_stream, stream) != 0)
+ return ERR_PTR(-ENOMEM);
+
+ return stream;
}

void kunit_free_string_stream(struct kunit *test, struct string_stream *stream)
{
- string_stream_destroy(stream);
+ kunit_release_action(test, resource_free_string_stream, (void *)stream);
}
diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h
index 3e70ee9d66e9..c55925a9b67f 100644
--- a/lib/kunit/string-stream.h
+++ b/lib/kunit/string-stream.h
@@ -23,7 +23,6 @@ struct string_stream {

struct list_head fragments;
/* length and fragments are protected by this lock */
spinlock_t lock;

- struct kunit *test;
gfp_t gfp;
bool append_newlines;
};
@@ -33,6 +32,9 @@ struct kunit;
struct string_stream *kunit_alloc_string_stream(struct kunit *test, gfp_t gfp);
void kunit_free_string_stream(struct kunit *test, struct string_stream *stream);

+struct string_stream *alloc_string_stream(gfp_t gfp);
+void free_string_stream(struct string_stream *stream);
+
int __printf(2, 3) string_stream_add(struct string_stream *stream,
const char *fmt, ...);

diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 93d9225d61e3..2ad45a4ac06a 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -296,7 +296,7 @@ static void kunit_print_string_stream(struct kunit *test,
kunit_err(test, "\n");
} else {
kunit_err(test, "%s", buf);
- kunit_kfree(test, buf);
+ kfree(buf);
}
}

--
2.30.2

[Richard Fitzgerald]

Add a test of the speed and memory use of string_stream.

string_stream_performance_test() doesn't actually "test" anything (it
cannot fail unless the system has run out of allocatable memory) but it
measures the speed and memory consumption of the string_stream and reports
the result.

This allows changes in the string_stream implementation to be compared.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream-test.c | 54 ++++++++++++++++++++++++++++++++++
1 file changed, 54 insertions(+)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 6897c57e0db7..7d81d139b8f8 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c

@@ -8,7 +8,9 @@

#include <kunit/static_stub.h>
#include <kunit/test.h>
+#include <linux/ktime.h>
#include <linux/slab.h>
+#include <linux/timekeeping.h>

#include "string-stream.h"

@@ -437,6 +439,57 @@ static void string_stream_auto_newline_test(struct kunit *test)

"One\nTwo\nThree\nFour\nFive\nSix\nSeven\n\nEight\n");
}

+/*
+ * This doesn't actually "test" anything. It reports time taken
+ * and memory used for logging a large number of lines.
+ */

+static void string_stream_performance_test(struct kunit *test)
+{
+ struct string_stream_fragment *frag_container;
+ struct string_stream *stream;

+ char test_line[101];
+ ktime_t start_time, end_time;
+ size_t len, bytes_requested, actual_bytes_used, total_string_length;
+ int offset, i;
+

+ stream = kunit_alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+

+ memset(test_line, 'x', sizeof(test_line) - 1);
+ test_line[sizeof(test_line) - 1] = '\0';
+
+ start_time = ktime_get();
+ for (i = 0; i < 10000; i++) {
+ offset = i % (sizeof(test_line) - 1);
+ string_stream_add(stream, "%s: %d\n", &test_line[offset], i);
+ }
+ end_time = ktime_get();

+
+ /*

@@ -462,6 +515,7 @@ static struct kunit_case string_stream_test_cases[] = {

[Richard Fitzgerald]

If the result of the formatted string is an empty string just return
instead of creating an empty fragment.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

lib/kunit/string-stream.c | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index cc32743c1171..ed24d86af9f5 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -50,11 +50,17 @@ int string_stream_vadd(struct string_stream *stream,

/* Make a copy because `vsnprintf` could change it */
va_copy(args_for_counting, args);

[Richard Fitzgerald]

This patch chain changes the logging implementation to use string_stream
so that the log will grow dynamically.

The first 8 patches add test code for string_stream, and make some
changes to string_stream needed to be able to use it for the log.

The final patch adds a performance report of string_stream.

CHANGES SINCE V4:
- Re-ordered the first 3 patches from V4 to squash the first two sets
of string_stream tests into a single patch.
- Changed is_literal() so it doesn't need a struct kunit.
- Split out the new resource-managed alloc and free functions into
a pre-patch to reduce the amount of code churn when the string_stream
is decoupled from kunit.
- Wrapped the call to string_stream_geT_string() in string-stream-test
in a local function to reduce the amount of code churn when the
string_stream is decoupled from kunit.
- Some minor changes to test implementations.
- string_stream is now completely separated from kunit and the 'test'
member of struct string_stream has been eliminated.

Richard Fitzgerald (10):

kunit: string-stream: Don't create a fragment for empty strings
kunit: string-stream: Improve testing of string_stream

kunit: string-stream: Add option to make all lines end with newline
kunit: string-stream: Add cases for string_stream newline appending

kunit: Don't use a managed alloc in is_literal()
kunit: string-stream: Add kunit_alloc_string_stream()

kunit: string-stream: Decouple string_stream from kunit

kunit: string-stream: Add tests for freeing resource-managed

string_stream
kunit: Use string_stream for test log
kunit: string-stream: Test performance of string_stream

include/kunit/test.h | 14 +-

lib/kunit/assert.c | 14 +-
lib/kunit/debugfs.c | 36 ++-
lib/kunit/kunit-test.c | 46 ++-
lib/kunit/string-stream-test.c | 508 +++++++++++++++++++++++++++++++--
lib/kunit/string-stream.c | 100 +++++--
lib/kunit/string-stream.h | 16 +-
lib/kunit/test.c | 50 +---
8 files changed, 662 insertions(+), 122 deletions(-)

--
2.30.2

[Rae Moar]

On Thu, Aug 24, 2023 at 10:31 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>
> Replace the minimal tests with more-thorough testing.
>
> string_stream_init_test() tests that struct string_stream is
> initialized correctly.
>
> string_stream_line_add_test() adds a series of numbered lines and
> checks that the resulting string contains all the lines.
>
> string_stream_variable_length_line_test() adds a large number of
> lines of varying length to create many fragments, then tests that all
> lines are present.
>
> string_stream_append_test() tests various cases of using
> string_stream_append() to append the content of one stream to another.
>
> Adds string_stream_append_empty_string_test() to test that adding an
> empty string to a string_stream doesn't create a new empty fragment.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

Hello!

These tests all look good to me. I like all of the details and
comments. Great to see these additions to the string-stream-test!

Reviewed-by: Rae Moar <rm...@google.com>

Thanks!
-Rae

As mentioned in the last version, if this causes a warning we will
look into it on the KUnit side.

I would maybe consider changing the name original_content to
stream_1_content but definitely not worth it as this version looks
very good.

[Rae Moar]

On Thu, Aug 24, 2023 at 10:32 AM 'Richard Fitzgerald' via KUnit
Development <kuni...@googlegroups.com> wrote:
>
> If the result of the formatted string is an empty string just return
> instead of creating an empty fragment.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

This looks good to me!

Reviewed-by: Rae Moar <rm...@google.com>

Thanks!

-Rae

> --
> You received this message because you are subscribed to the Google Groups "KUnit Development" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to kunit-dev+...@googlegroups.com.
> To view this discussion on the web visit https://groups.google.com/d/msgid/kunit-dev/20230824143129.1957914-2-rf%40opensource.cirrus.com.

[Rae Moar]

On Thu, Aug 24, 2023 at 10:32 AM Richard Fitzgerald

<r...@opensource.cirrus.com> wrote:
>
> Add an optional feature to string_stream that will append a newline to
> any added string that does not already end with a newline. The purpose
> of this is so that string_stream can be used to collect log lines.
>
> This is enabled/disabled by calling string_stream_set_append_newlines().
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

Hello,

This again looks good to me!

Reviewed-by: Rae Moar <rm...@google.com>

Thanks!
-Rae

[Rae Moar]

On Thu, Aug 24, 2023 at 10:32 AM Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Add test cases for testing the string_stream feature that appends a
> newline to strings that do not already end with a newline.
>
> string_stream_no_auto_newline_test() tests with this feature disabled.
> Newlines should not be added or dropped.
>
> string_stream_auto_newline_test() tests with this feature enabled.
> Newlines should be added to lines that do not end with a newline.
>
> string_stream_append_auto_newline_test() tests appending the
> content of one stream to another stream when the target stream
> has newline appending enabled.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

Hi!

This looks good to me! I like the changes.

Reviewed-by: Rae Moar <rm...@google.com>

Thanks!
-Rae

[kernel test robot]

Hi Richard,

kernel test robot noticed the following build warnings:

[auto build test WARNING on shuah-kselftest/kunit]
[also build test WARNING on next-20230824]
[cannot apply to shuah-kselftest/kunit-fixes linus/master v6.5-rc7]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url: https://github.com/intel-lab-lkp/linux/commits/Richard-Fitzgerald/kunit-string-stream-Don-t-create-a-fragment-for-empty-strings/20230824-223722
base: https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit
patch link: https://lore.kernel.org/r/20230824143129.1957914-10-rf%40opensource.cirrus.com
patch subject: [PATCH v5 09/10] kunit: Use string_stream for test log
config: hexagon-randconfig-002-20230825 (https://download.01.org/0day-ci/archive/20230825/202308251443...@intel.com/config)
compiler: clang version 17.0.0 (https://github.com/llvm/llvm-project.git 4a5ac14ee968ff0ad5d2cc1ffa0299048db4c88a)
reproduce: (https://download.01.org/0day-ci/archive/20230825/202308251443...@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <l...@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202308251443...@intel.com/

All warnings (new ones prefixed by >>):

>> lib/kunit/kunit-test.c:542:8: warning: unused variable 'full_log' [-Wunused-variable]
542 | char *full_log;
| ^
lib/kunit/kunit-test.c:581:26: warning: cast from 'void (*)(const void *)' to 'kunit_action_t *' (aka 'void (*)(void *)') converts to incompatible function type [-Wcast-function-type-strict]
581 | kunit_add_action(test, (kunit_action_t *)kfree, full_log);
| ^~~~~~~~~~~~~~~~~~~~~~~
2 warnings generated.


vim +/full_log +542 lib/kunit/kunit-test.c

533
534 /*
535 * Log tests call string_stream functions, which aren't exported. So only
536 * build this code if this test is built-in.
537 */
538 #if IS_BUILTIN(CONFIG_KUNIT_TEST)
539 static void kunit_log_test(struct kunit *test)
540 {
541 struct kunit_suite suite;
> 542 char *full_log;
543
544 suite.log = kunit_alloc_string_stream(test, GFP_KERNEL);
545 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
546 string_stream_set_append_newlines(suite.log, true);
547
548 kunit_log(KERN_INFO, test, "put this in log.");
549 kunit_log(KERN_INFO, test, "this too.");
550 kunit_log(KERN_INFO, &suite, "add to suite log.");
551 kunit_log(KERN_INFO, &suite, "along with this.");
552

--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

[kernel test robot]

Hi Richard,

kernel test robot noticed the following build warnings:

[auto build test WARNING on shuah-kselftest/kunit]

[also build test WARNING on shuah-kselftest/kunit-fixes linus/master v6.5-rc7 next-20230824]

[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url: https://github.com/intel-lab-lkp/linux/commits/Richard-Fitzgerald/kunit-string-stream-Don-t-create-a-fragment-for-empty-strings/20230824-223722
base: https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit

patch link: https://lore.kernel.org/r/20230824143129.1957914-8-rf%40opensource.cirrus.com
patch subject: [PATCH v5 07/10] kunit: string-stream: Decouple string_stream from kunit
config: hexagon-randconfig-001-20230825 (https://download.01.org/0day-ci/archive/20230825/202308251401...@intel.com/config)

compiler: clang version 17.0.0 (https://github.com/llvm/llvm-project.git 4a5ac14ee968ff0ad5d2cc1ffa0299048db4c88a)

reproduce: (https://download.01.org/0day-ci/archive/20230825/202308251401...@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <l...@intel.com>

| Closes: https://lore.kernel.org/oe-kbuild-all/202308251401...@intel.com/

All warnings (new ones prefixed by >>):

>> lib/kunit/string-stream-test.c:19:25: warning: cast from 'void (*)(const void *)' to 'kunit_action_t *' (aka 'void (*)(void *)') converts to incompatible function type [-Wcast-function-type-strict]
19 | kunit_add_action(test, (kunit_action_t *)kfree, (void *)str);
| ^~~~~~~~~~~~~~~~~~~~~~~
1 warning generated.


vim +19 lib/kunit/string-stream-test.c

13
14 static char *get_concatenated_string(struct kunit *test, struct string_stream *stream)
15 {
16 char *str = string_stream_get_string(stream);
17
18 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, str);
> 19 kunit_add_action(test, (kunit_action_t *)kfree, (void *)str);
20
21 return str;
22 }
23

[David Gow]

On Thu, 24 Aug 2023 at 22:32, 'Richard Fitzgerald' via KUnit
Development <kuni...@googlegroups.com> wrote:
>
> If the result of the formatted string is an empty string just return
> instead of creating an empty fragment.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

[David Gow]

On Thu, 24 Aug 2023 at 22:33, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>
> Replace the minimal tests with more-thorough testing.
>
> string_stream_init_test() tests that struct string_stream is
> initialized correctly.
>
> string_stream_line_add_test() adds a series of numbered lines and
> checks that the resulting string contains all the lines.
>
> string_stream_variable_length_line_test() adds a large number of
> lines of varying length to create many fragments, then tests that all
> lines are present.
>
> string_stream_append_test() tests various cases of using
> string_stream_append() to append the content of one stream to another.
>
> Adds string_stream_append_empty_string_test() to test that adding an
> empty string to a string_stream doesn't create a new empty fragment.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---
> Changes since V4:
> - Test cases for appending empty strings have been squashed into this
> patch.
> - Call to string_stream_get_string() is wrapped in a local function
> get_concatenated_string(). This is to avoid a lot of code churn later
> when string_stream_get_string() is changed to return an unmanaged buffer.
> ---

This looks good to me. I'm not 100% sold on the
'get_concatenated_string()' function long-term (despite its obvious
benefits during the refactoring), but that's just personal taste. This
version is fine regardless.

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

[David Gow]

On Thu, 24 Aug 2023 at 22:32, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>
> Add an optional feature to string_stream that will append a newline to
> any added string that does not already end with a newline. The purpose
> of this is so that string_stream can be used to collect log lines.
>
> This is enabled/disabled by calling string_stream_set_append_newlines().
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

This is the same as v4, patch 4, and still looks good.

(In the future, feel free to leave the Reviewed-by: tag from the
previous version, so long as there are no significant changes.)

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

[David Gow]

On Thu, 24 Aug 2023 at 22:32, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Add test cases for testing the string_stream feature that appends a
> newline to strings that do not already end with a newline.
>
> string_stream_no_auto_newline_test() tests with this feature disabled.
> Newlines should not be added or dropped.
>
> string_stream_auto_newline_test() tests with this feature enabled.
> Newlines should be added to lines that do not end with a newline.
>
> string_stream_append_auto_newline_test() tests appending the
> content of one stream to another stream when the target stream
> has newline appending enabled.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---
> Changes since V4:
> - string_stream_append_auto_newline_test() doesn't clear the destination
> stream_1 between the newline and no-newline case. This is just a
> simplification of the code.
>
> - string_stream_no_auto_newline_test() uses the same set of test strings
> as string_stream_auto_newline_test().
> ---

Much better, thanks!

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

[David Gow]

On Thu, 24 Aug 2023 at 22:32, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

The changes all make sense to me, and work here. Thanks!

The only slight issue is there's one missing spot which still casts
the kunit_action_t function pointer directly, in the test. Up to you
if you want to change that, too (though it may need a helper of its
own.)

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

> lib/kunit/string-stream-test.c | 2 +-
> lib/kunit/string-stream.c | 59 ++++++++++++++++++++++------------
> lib/kunit/string-stream.h | 4 ++-
> lib/kunit/test.c | 2 +-
> 4 files changed, 44 insertions(+), 23 deletions(-)
>
> diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
> index 89549c237069..45a2d221f1b5 100644
> --- a/lib/kunit/string-stream-test.c
> +++ b/lib/kunit/string-stream-test.c
> @@ -16,6 +16,7 @@ static char *get_concatenated_string(struct kunit *test, struct string_stream *s
> char *str = string_stream_get_string(stream);
>
> KUNIT_ASSERT_NOT_ERR_OR_NULL(test, str);
> + kunit_add_action(test, (kunit_action_t *)kfree, (void *)str);

This still directly casts kfree to kunit_action_t, so triggers a
warning. I'm okay with it personally (and at some point we'll probably
implement a public kunit_free_at_end() function to do things like
this, which we already have in some other tests).

[David Gow]

On Thu, 24 Aug 2023 at 22:33, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Add a test of the speed and memory use of string_stream.
>
> string_stream_performance_test() doesn't actually "test" anything (it
> cannot fail unless the system has run out of allocatable memory) but it
> measures the speed and memory consumption of the string_stream and reports
> the result.
>
> This allows changes in the string_stream implementation to be compared.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---

Still looks good to me, so still:

Reviewed-by: David Gow <davi...@google.com>

My results are:
# string_stream_performance_test: Time elapsed: 4457 us
# string_stream_performance_test: Total string length: 573890
# string_stream_performance_test: Bytes requested: 823922
# string_stream_performance_test: Actual bytes allocated: 1048280

Cheers,
-- David

[David Gow]

On Thu, 24 Aug 2023 at 22:33, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>

> Replace the fixed-size log buffer with a string_stream so that the
> log can grow as lines are added.
>
> string_stream_clear() has been made public for the log truncation
> done in kunit_init_test().
>
> The existing kunit log tests have been updated for using a
> string_stream as the log. No new test have been added because there
> are already tests for the underlying string_stream.
>
> As the log tests now depend on string_stream functions they cannot
> build when kunit-test is a module. They have been surrounded by
> a #if to replace them with skipping version when the test is
> build as a module. Though this isn't pretty, it avoids moving
> code to another file while that code is also being changed.
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---
> Changes since V4:
> - Don't move the log tests to another file. Deal with only including them
> when the test is built-in by wrapping them in a #if. This is to simplify
> code review, because it avoids having a block of code which moves from
> one file to another but at the same time the code has changed.
> - Use kunit_add_action() to automatically free the string returned by
> string_stream_get_string().
> ---

Looks pretty good to me, and works fine here.

The kunit_add_action() cast does trigger the clang warning (but again,
it's not something which bothers me much personally). But since you've
cleaned it up elsewhere, it may be worth adding a wrapper here, at
least until we have a kunit_free_at_end() function or similar.

Otherwise,

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David

[David Gow]

On Thu, 24 Aug 2023 at 22:32, 'Richard Fitzgerald' via KUnit
Development <kuni...@googlegroups.com> wrote:
>

Thanks a lot for sticking with this. I think we're in pretty good
shape now. There are a few minor comments, only one of which really
concerns me. That's the freeing of string streams in the
resource-managed string stream tests. I can't quite see how the actual
stream is freed after being "fake freed" by the stub. Is there
something I'm missing?

Otherwise, this seems good enough to go. I fear we're probably past
the point where it can make it into 6.6 (pull requests are already
being sent out, and I'd really rather have the final version of this
soak in linux-next for a while before sending it to Linus. But we'll
make it the first thing to go into 6.7, I think.

Cheers,
-- David

> include/kunit/test.h | 14 +-
> lib/kunit/assert.c | 14 +-
> lib/kunit/debugfs.c | 36 ++-
> lib/kunit/kunit-test.c | 46 ++-
> lib/kunit/string-stream-test.c | 508 +++++++++++++++++++++++++++++++--
> lib/kunit/string-stream.c | 100 +++++--
> lib/kunit/string-stream.h | 16 +-
> lib/kunit/test.c | 50 +---
> 8 files changed, 662 insertions(+), 122 deletions(-)
>
> --
> 2.30.2
>

> --
> You received this message because you are subscribed to the Google Groups "KUnit Development" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to kunit-dev+...@googlegroups.com.

> To view this discussion on the web visit https://groups.google.com/d/msgid/kunit-dev/20230824143129.1957914-1-rf%40opensource.cirrus.com.

[kernel test robot]

Hi Richard,

kernel test robot noticed the following build warnings:

[auto build test WARNING on shuah-kselftest/kunit]

[also build test WARNING on next-20230824]
[cannot apply to shuah-kselftest/kunit-fixes linus/master v6.5-rc7]

[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url: https://github.com/intel-lab-lkp/linux/commits/Richard-Fitzgerald/kunit-string-stream-Don-t-create-a-fragment-for-empty-strings/20230824-223722
base: https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit

patch link: https://lore.kernel.org/r/20230824143129.1957914-10-rf%40opensource.cirrus.com
patch subject: [PATCH v5 09/10] kunit: Use string_stream for test log
config: hexagon-randconfig-001-20230825 (https://download.01.org/0day-ci/archive/20230825/202308251509...@intel.com/config)

compiler: clang version 17.0.0 (https://github.com/llvm/llvm-project.git 4a5ac14ee968ff0ad5d2cc1ffa0299048db4c88a)

reproduce: (https://download.01.org/0day-ci/archive/20230825/202308251509...@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <l...@intel.com>

| Closes: https://lore.kernel.org/oe-kbuild-all/202308251509...@intel.com/

All warnings (new ones prefixed by >>):

>> lib/kunit/kunit-test.c:557:25: warning: cast from 'void (*)(const void *)' to 'kunit_action_t *' (aka 'void (*)(void *)') converts to incompatible function type [-Wcast-function-type-strict]
557 | kunit_add_action(test, (kunit_action_t *)kfree, full_log);
| ^~~~~~~~~~~~~~~~~~~~~~~
lib/kunit/kunit-test.c:564:25: warning: cast from 'void (*)(const void *)' to 'kunit_action_t *' (aka 'void (*)(void *)') converts to incompatible function type [-Wcast-function-type-strict]
564 | kunit_add_action(test, (kunit_action_t *)kfree, full_log);

| ^~~~~~~~~~~~~~~~~~~~~~~
lib/kunit/kunit-test.c:581:26: warning: cast from 'void (*)(const void *)' to 'kunit_action_t *' (aka 'void (*)(void *)') converts to incompatible function type [-Wcast-function-type-strict]
581 | kunit_add_action(test, (kunit_action_t *)kfree, full_log);
| ^~~~~~~~~~~~~~~~~~~~~~~

3 warnings generated.


vim +557 lib/kunit/kunit-test.c

533
534 /*
535 * Log tests call string_stream functions, which aren't exported. So only
536 * build this code if this test is built-in.
537 */
538 #if IS_BUILTIN(CONFIG_KUNIT_TEST)
539 static void kunit_log_test(struct kunit *test)
540 {
541 struct kunit_suite suite;
542 char *full_log;
543
544 suite.log = kunit_alloc_string_stream(test, GFP_KERNEL);
545 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
546 string_stream_set_append_newlines(suite.log, true);
547
548 kunit_log(KERN_INFO, test, "put this in log.");
549 kunit_log(KERN_INFO, test, "this too.");
550 kunit_log(KERN_INFO, &suite, "add to suite log.");

551 kunit_log(KERN_INFO, &suite, "along with this.");
552
553 #ifdef CONFIG_KUNIT_DEBUGFS
554 KUNIT_EXPECT_TRUE(test, test->log->append_newlines);
555
556 full_log = string_stream_get_string(test->log);
> 557 kunit_add_action(test, (kunit_action_t *)kfree, full_log);
558 KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
559 strstr(full_log, "put this in log."));
560 KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
561 strstr(full_log, "this too."));
562
563 full_log = string_stream_get_string(suite.log);
564 kunit_add_action(test, (kunit_action_t *)kfree, full_log);
565 KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
566 strstr(full_log, "add to suite log."));
567 KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
568 strstr(full_log, "along with this."));
569 #else
570 KUNIT_EXPECT_NULL(test, test->log);
571 #endif
572 }
573

[Richard Fitzgerald]

Yes, we can remove it later. I just wanted to avoid having one enormous
patch that changes everything all over the place.

[Richard Fitzgerald]

On 24/08/2023 23:42, Rae Moar wrote:
> On Thu, Aug 24, 2023 at 10:31 AM Richard Fitzgerald
> <r...@opensource.cirrus.com> wrote:
>>
>> Replace the minimal tests with more-thorough testing.
>>

<SNIP>

>> + KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL);
>
> As mentioned in the last version, if this causes a warning we will
> look into it on the KUnit side.
>

It does. I left it because you said you'd do a fix.
But maybe it's better to change it to

KUNIT_EXPECT_TRUE(test, stream_gfp == GFP_KERNEL);

to avoid the warning for now.

[Richard Fitzgerald]

This patch chain changes the logging implementation to use string_stream
so that the log will grow dynamically.

The first 8 patches add test code for string_stream, and make some
changes to string_stream needed to be able to use it for the log.

The final patch adds a performance report of string_stream.

CHANGES SINCE V5:
Patch 2:
- Avoid cast warning when using KUNIT_EXPECT_EQ() on a gfp_t. Instead pass
the result of the comparison to KUNIT_EXPECT_TRUE(). While it would be
nice to use KUNIT_EXPECT_EQ(), it's probably better to avoid introducing
build or sparse warnings.

- In string_stream_append_test() rename original_content to
stream1_content_before_append.

Patch 7:
- Make string_stream_clear() public (in v5 this was done in patch #8).
- In string-stream-test.c add a wrapper for kfree() to prevent a cast
warning when calling kunit_add_action().

Patch 8:
- Fix memory leak when calling the redirected string_stream_destroy_stub().

Patch 9:
- In kunit-test.c: add wrapper function around kfree() to prevent cast
warning when calling kunit_add_action().
- Fix unused variable warning in kunit_log_test() when built as a module.

Richard Fitzgerald (10):
kunit: string-stream: Don't create a fragment for empty strings
kunit: string-stream: Improve testing of string_stream
kunit: string-stream: Add option to make all lines end with newline

kunit: string-stream-test: Add cases for string_stream newline

appending
kunit: Don't use a managed alloc in is_literal()
kunit: string-stream: Add kunit_alloc_string_stream()
kunit: string-stream: Decouple string_stream from kunit
kunit: string-stream: Add tests for freeing resource-managed
string_stream

kunit: Use string_stream for test log

kunit: string-stream: Test performance of string_stream

include/kunit/test.h | 14 +-
lib/kunit/assert.c | 14 +-
lib/kunit/debugfs.c | 36 ++-

lib/kunit/kunit-test.c | 56 +++-
lib/kunit/string-stream-test.c | 525 +++++++++++++++++++++++++++++++--

lib/kunit/string-stream.c | 100 +++++--
lib/kunit/string-stream.h | 16 +-
lib/kunit/test.c | 50 +---

8 files changed, 688 insertions(+), 123 deletions(-)

--
2.30.2

[Richard Fitzgerald]

Re-work string_stream so that it is not tied to a struct kunit. This is
to allow using it for the log of struct kunit_suite.

Instead of resource-managing individual allocations the whole string_stream
can be resource-managed, if required.

alloc_string_stream() now allocates a string stream that is
not resource-managed.

string_stream_destroy() now works on an unmanaged string_stream
allocated by alloc_string_stream() and frees the entire
string_stream (previously it only freed the fragments).

string_stream_clear() has been made public for callers that
want to free the fragments without destroying the string_stream.

For resource-managed allocations use kunit_alloc_string_stream()
and kunit_free_string_stream().

In addition to this, string_stream_get_string() now returns an
unmanaged buffer that the caller must kfree().

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>

Reviewed-by: David Gow <davi...@google.com>
---
Changes since v5:

- Make string_stream_clear() public (in v5 this was done in patch #8).
- In string-stream-test.c add a wrapper for kfree() to prevent a cast
warning when calling kunit_add_action().

---
lib/kunit/string-stream-test.c | 8 ++++-
lib/kunit/string-stream.c | 61 ++++++++++++++++++++++------------
lib/kunit/string-stream.h | 6 +++-
lib/kunit/test.c | 2 +-
4 files changed, 53 insertions(+), 24 deletions(-)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 46b18e940b73..58ba1ef5207f 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -11,11 +11,18 @@

#include "string-stream.h"

+/* This avoids a cast warning if kfree() is passed direct to kunit_add_action(). */
+static void kfree_wrapper(void *p)
+{
+ kfree(p);
+}
+
static char *get_concatenated_string(struct kunit *test, struct string_stream *stream)
{

char *str = string_stream_get_string(stream);

KUNIT_ASSERT_NOT_ERR_OR_NULL(test, str);

+ kunit_add_action(test, kfree_wrapper, (void *)str);

return str;
}
@@ -30,7 +37,6 @@ static void string_stream_init_test(struct kunit *test)

KUNIT_EXPECT_EQ(test, stream->length, 0);
KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));
- KUNIT_EXPECT_PTR_EQ(test, stream->test, test);

KUNIT_EXPECT_TRUE(test, (stream->gfp == GFP_KERNEL));

KUNIT_EXPECT_FALSE(test, stream->append_newlines);
KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));

diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index 12ecf15e1f6b..64abceb7b716 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c

@@ -102,7 +98,7 @@ int string_stream_add(struct string_stream *stream, const char *fmt, ...)

return result;
}

-static void string_stream_clear(struct string_stream *stream)
+void string_stream_clear(struct string_stream *stream)
{
struct string_stream_fragment *frag_container, *frag_container_safe;

+ struct string_stream *stream;
+

+ stream = alloc_string_stream(gfp);
+ if (IS_ERR(stream))
+ return stream;
+
+ if (kunit_add_action_or_reset(test, resource_free_string_stream, stream) != 0)
+ return ERR_PTR(-ENOMEM);
+
+ return stream;
}

void kunit_free_string_stream(struct kunit *test, struct string_stream *stream)
{
- string_stream_destroy(stream);
+ kunit_release_action(test, resource_free_string_stream, (void *)stream);
}

diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h
index 3e70ee9d66e9..7be2450c7079 100644
--- a/lib/kunit/string-stream.h
+++ b/lib/kunit/string-stream.h

@@ -23,7 +23,6 @@ struct string_stream {
struct list_head fragments;
/* length and fragments are protected by this lock */
spinlock_t lock;
- struct kunit *test;
gfp_t gfp;
bool append_newlines;
};
@@ -33,6 +32,9 @@ struct kunit;
struct string_stream *kunit_alloc_string_stream(struct kunit *test, gfp_t gfp);
void kunit_free_string_stream(struct kunit *test, struct string_stream *stream);

+struct string_stream *alloc_string_stream(gfp_t gfp);
+void free_string_stream(struct string_stream *stream);
+

int __printf(2, 3) string_stream_add(struct string_stream *stream,
const char *fmt, ...);

@@ -40,6 +42,8 @@ int __printf(2, 0) string_stream_vadd(struct string_stream *stream,

const char *fmt,
va_list args);

+void string_stream_clear(struct string_stream *stream);
+
char *string_stream_get_string(struct string_stream *stream);

int string_stream_append(struct string_stream *stream,
diff --git a/lib/kunit/test.c b/lib/kunit/test.c

index 93d9225d61e3..2ad45a4ac06a 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c

[Richard Fitzgerald]

Replace the fixed-size log buffer with a string_stream so that the
log can grow as lines are added.

The existing kunit log tests have been updated for using a
string_stream as the log. No new test have been added because there
are already tests for the underlying string_stream.

As the log tests now depend on string_stream functions they cannot
build when kunit-test is a module. They have been surrounded by
a #if to replace them with skipping version when the test is
build as a module. Though this isn't pretty, it avoids moving
code to another file while that code is also being changed.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
Reviewed-by: David Gow <davi...@google.com>
---

Changes since V5:

- In kunit-test.c: add wrapper function around kfree() to prevent cast
warning when calling kunit_add_action().
- Fix unused variable warning in kunit_log_test() when built as a module.

---
include/kunit/test.h | 14 +++++------
lib/kunit/debugfs.c | 36 +++++++++++++++++----------
lib/kunit/kunit-test.c | 56 +++++++++++++++++++++++++++++++++++-------
lib/kunit/test.c | 44 ++++-----------------------------
4 files changed, 81 insertions(+), 69 deletions(-)

index 83d8e90ca7a2..99d2a3a528e1 100644

--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -8,6 +8,7 @@
#include <kunit/test.h>
#include <kunit/test-bug.h>

+#include "string-stream.h"
#include "try-catch-impl.h"

struct kunit_try_catch_test_context {

@@ -530,12 +531,27 @@ static struct kunit_suite kunit_resource_test_suite = {
.test_cases = kunit_resource_test_cases,
};

+/*
+ * Log tests call string_stream functions, which aren't exported. So only
+ * build this code if this test is built-in.
+ */
+#if IS_BUILTIN(CONFIG_KUNIT_TEST)
+

+/* This avoids a cast warning if kfree() is passed direct to kunit_add_action(). */
+static void kfree_wrapper(void *p)
+{
+ kfree(p);
+}
+

static void kunit_log_test(struct kunit *test)
{
struct kunit_suite suite;

-

- suite.log = kunit_kzalloc(test, KUNIT_LOG_SIZE, GFP_KERNEL);

+#ifdef CONFIG_KUNIT_DEBUGFS
+ char *full_log;
+#endif

+ suite.log = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, suite.log);
+ string_stream_set_append_newlines(suite.log, true);

kunit_log(KERN_INFO, test, "put this in log.");

kunit_log(KERN_INFO, test, "this too.");

@@ -543,14 +559,21 @@ static void kunit_log_test(struct kunit *test)

kunit_log(KERN_INFO, &suite, "along with this.");

#ifdef CONFIG_KUNIT_DEBUGFS
+ KUNIT_EXPECT_TRUE(test, test->log->append_newlines);
+
+ full_log = string_stream_get_string(test->log);
+ kunit_add_action(test, (kunit_action_t *)kfree, full_log);
KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(test->log, "put this in log."));
+ strstr(full_log, "put this in log."));
KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(test->log, "this too."));
+ strstr(full_log, "this too."));
+
+ full_log = string_stream_get_string(suite.log);

+ kunit_add_action(test, kfree_wrapper, full_log);

KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(suite.log, "add to suite log."));
+ strstr(full_log, "add to suite log."));
KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
- strstr(suite.log, "along with this."));
+ strstr(full_log, "along with this."));
#else
KUNIT_EXPECT_NULL(test, test->log);
#endif

@@ -558,15 +581,30 @@ static void kunit_log_test(struct kunit *test)

static void kunit_log_newline_test(struct kunit *test)
{
+ char *full_log;
+
kunit_info(test, "Add newline\n");
if (test->log) {
- KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(test->log, "Add newline\n"),
- "Missing log line, full log:\n%s", test->log);
- KUNIT_EXPECT_NULL(test, strstr(test->log, "Add newline\n\n"));
+ full_log = string_stream_get_string(test->log);

+ kunit_add_action(test, kfree_wrapper, full_log);

+ KUNIT_ASSERT_NOT_NULL_MSG(test, strstr(full_log, "Add newline\n"),
+ "Missing log line, full log:\n%s", full_log);
+ KUNIT_EXPECT_NULL(test, strstr(full_log, "Add newline\n\n"));
} else {
kunit_skip(test, "only useful when debugfs is enabled");
}
}
+#else
+static void kunit_log_test(struct kunit *test)
+{
+ kunit_skip(test, "Log tests only run when built-in");
+}
+
+static void kunit_log_newline_test(struct kunit *test)
+{
+ kunit_skip(test, "Log tests only run when built-in");
+}
+#endif /* IS_BUILTIN(CONFIG_KUNIT_TEST) */

static struct kunit_case kunit_log_test_cases[] = {
KUNIT_CASE(kunit_log_test),

diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 2ad45a4ac06a..b153808ff1ec 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c

[Richard Fitzgerald]

Replace the minimal tests with more-thorough testing.

string_stream_init_test() tests that struct string_stream is
initialized correctly.

string_stream_line_add_test() adds a series of numbered lines and
checks that the resulting string contains all the lines.

string_stream_variable_length_line_test() adds a large number of
lines of varying length to create many fragments, then tests that all
lines are present.

string_stream_append_test() tests various cases of using
string_stream_append() to append the content of one stream to another.

Adds string_stream_append_empty_string_test() to test that adding an
empty string to a string_stream doesn't create a new empty fragment.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
Reviewed-by: Rae Moar <rm...@google.com>

Reviewed-by: David Gow <davi...@google.com>
---

Changes since v5:

- Avoid cast warning when using KUNIT_EXPECT_EQ() on a gfp_t. Instead pass
the result of the comparison to KUNIT_EXPECT_TRUE(). While it would be
nice to use KUNIT_EXPECT_EQ(), it's probably better to avoid introducing
build or sparse warnings.

- In string_stream_append_test() rename original_content to
stream1_content_before_append.

---
lib/kunit/string-stream-test.c | 233 ++++++++++++++++++++++++++++++---
1 file changed, 217 insertions(+), 16 deletions(-)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 110f3a993250..7e17307ca78c 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -11,38 +11,239 @@

#include "string-stream.h"

-static void string_stream_test_empty_on_creation(struct kunit *test)
+static char *get_concatenated_string(struct kunit *test, struct string_stream *stream)
{
- struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL);
+ char *str = string_stream_get_string(stream);

+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, str);
+
+ return str;
+}
+
+/* string_stream object is initialized correctly. */
+static void string_stream_init_test(struct kunit *test)

+{

+ struct string_stream *stream;
+

+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+

+ KUNIT_EXPECT_EQ(test, stream->length, 0);
+ KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));
+ KUNIT_EXPECT_PTR_EQ(test, stream->test, test);

+ KUNIT_EXPECT_TRUE(test, (stream->gfp == GFP_KERNEL));

KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
}

-static void string_stream_test_not_empty_after_add(struct kunit *test)
+/*
+ * Add a series of lines to a string_stream. Check that all lines
+ * appear in the correct order and no characters are dropped.
+ */
+static void string_stream_line_add_test(struct kunit *test)
{
- struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL);

+ struct string_stream *stream;

+ char line[60];
+ char *concat_string, *pos, *string_end;
+ size_t len, total_len;
+ int num_lines, i;

- string_stream_add(stream, "Foo");

+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

- KUNIT_EXPECT_FALSE(test, string_stream_is_empty(stream));
+ /* Add series of sequence numbered lines */
+ total_len = 0;
+ for (i = 0; i < 100; ++i) {
+ len = snprintf(line, sizeof(line),
+ "The quick brown fox jumps over the lazy penguin %d\n", i);
+
+ /* Sanity-check that our test string isn't truncated */
+ KUNIT_ASSERT_LT(test, len, sizeof(line));
+
+ string_stream_add(stream, line);
+ total_len += len;
+ }
+ num_lines = i;
+
+ concat_string = get_concatenated_string(test, stream);
+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test, concat_string);
+ KUNIT_EXPECT_EQ(test, strlen(concat_string), total_len);

+
+ /*

+ struct string_stream *stream;

+ struct rnd_state rnd;
+ char *concat_string, *pos, *string_end;
+ size_t offset, total_len;
+ int num_lines, i;

- string_stream_add(stream, "Foo");
- string_stream_add(stream, " %s", "bar");

+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

- output = string_stream_get_string(stream);
- KUNIT_ASSERT_STREQ(test, output, "Foo bar");
+ /*
+ * Log many lines of varying lengths until we have created
+ * many fragments.
+ * The "randomness" must be repeatable.
+ */
+ prandom_seed_state(&rnd, 3141592653589793238ULL);
+ total_len = 0;
+ for (i = 0; i < 100; ++i) {
+ offset = prandom_u32_state(&rnd) % (sizeof(line) - 1);
+ string_stream_add(stream, "%s\n", &line[offset]);
+ total_len += sizeof(line) - offset;
+ }
+ num_lines = i;
+
+ concat_string = get_concatenated_string(test, stream);
+ KUNIT_EXPECT_NOT_ERR_OR_NULL(test, concat_string);
+ KUNIT_EXPECT_EQ(test, strlen(concat_string), total_len);

+
+ /*

+ };
+ struct string_stream *stream_1, *stream_2;

+ const char *stream1_content_before_append, *stream_2_content;

+ char *combined_content;
+ size_t combined_length;
+ int i;
+

+ stream_1 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);
+

+ stream_2 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2);
+

+ /* Append content of empty stream to empty stream */
+ string_stream_append(stream_1, stream_2);
+ KUNIT_EXPECT_EQ(test, strlen(get_concatenated_string(test, stream_1)), 0);
+
+ /* Add some data to stream_1 */
+ for (i = 0; i < ARRAY_SIZE(strings_1); ++i)
+ string_stream_add(stream_1, "%s\n", strings_1[i]);
+

+ stream1_content_before_append = get_concatenated_string(test, stream_1);

+
+ /* Append content of empty stream to non-empty stream */

+ string_stream_append(stream_1, stream_2);
+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1),

+ stream1_content_before_append);

+
+ /* Add some data to stream_2 */
+ for (i = 0; i < ARRAY_SIZE(strings_2); ++i)
+ string_stream_add(stream_2, "%s\n", strings_2[i]);
+
+ /* Append content of non-empty stream to non-empty stream */
+ string_stream_append(stream_1, stream_2);

+
+ /*

+ * End result should be the original content of stream_1 plus
+ * the content of stream_2.
+ */
+ stream_2_content = get_concatenated_string(test, stream_2);

+ combined_length = strlen(stream1_content_before_append) + strlen(stream_2_content);

+ combined_length++; /* for terminating \0 */
+ combined_content = kunit_kmalloc(test, combined_length, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, combined_content);
+ snprintf(combined_content, combined_length, "%s%s",

+ stream1_content_before_append, stream_2_content);

+
+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), combined_content);
+
+ /* Append content of non-empty stream to empty stream */
+ string_stream_destroy(stream_1);
+

+ stream_1 = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);
+

+ string_stream_append(stream_1, stream_2);

+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), stream_2_content);
+}
+
+/* Adding an empty string should not create a fragment. */
+static void string_stream_append_empty_string_test(struct kunit *test)

+{
+ struct string_stream *stream;
+ int original_frag_count;
+
+ stream = alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+

+ /* Formatted empty string */
+ string_stream_add(stream, "%s", "");
+ KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
+ KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));
+
+ /* Adding an empty string to a non-empty stream */
+ string_stream_add(stream, "Add this line");
+ original_frag_count = list_count_nodes(&stream->fragments);
+
+ string_stream_add(stream, "%s", "");
+ KUNIT_EXPECT_EQ(test, list_count_nodes(&stream->fragments), original_frag_count);

+ KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream), "Add this line");

}

static struct kunit_case string_stream_test_cases[] = {

[Richard Fitzgerald]

If the result of the formatted string is an empty string just return

instead of creating an empty fragment.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
Reviewed-by: Rae Moar <rm...@google.com>
Reviewed-by: David Gow <davi...@google.com>
---

lib/kunit/string-stream.c | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index cc32743c1171..ed24d86af9f5 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -50,11 +50,17 @@ int string_stream_vadd(struct string_stream *stream,

/* Make a copy because `vsnprintf` could change it */
va_copy(args_for_counting, args);

[Richard Fitzgerald]

Add an optional feature to string_stream that will append a newline to
any added string that does not already end with a newline. The purpose
of this is so that string_stream can be used to collect log lines.

This is enabled/disabled by calling string_stream_set_append_newlines().

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
Reviewed-by: Rae Moar <rm...@google.com>
Reviewed-by: David Gow <davi...@google.com>
---

lib/kunit/string-stream.c | 28 +++++++++++++++++++++-------
lib/kunit/string-stream.h | 7 +++++++

2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index ed24d86af9f5..1dcf6513b692 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c

@@ -44,32 +44,46 @@ int string_stream_vadd(struct string_stream *stream,
va_list args)
{
struct string_stream_fragment *frag_container;
- int len;
+ int buf_len, result_len;
va_list args_for_counting;

/* Make a copy because `vsnprintf` could change it */
va_copy(args_for_counting, args);

/* Evaluate length of formatted string */

- len = vsnprintf(NULL, 0, fmt, args_for_counting);

+ buf_len = vsnprintf(NULL, 0, fmt, args_for_counting);

va_end(args_for_counting);

- if (len == 0)
+ if (buf_len == 0)
return 0;

+ /* Reserve one extra for possible appended newline. */
+ if (stream->append_newlines)
+ buf_len++;

+
/* Need space for null byte. */

- len++;
+ buf_len++;

frag_container = alloc_string_stream_fragment(stream->test,
- len,
+ buf_len,

stream->gfp);
if (IS_ERR(frag_container))
return PTR_ERR(frag_container);

- len = vsnprintf(frag_container->fragment, len, fmt, args);
+ if (stream->append_newlines) {
+ /* Don't include reserved newline byte in writeable length. */
+ result_len = vsnprintf(frag_container->fragment, buf_len - 1, fmt, args);
+
+ /* Append newline if necessary. */
+ if (frag_container->fragment[result_len - 1] != '\n')
+ result_len = strlcat(frag_container->fragment, "\n", buf_len);
+ } else {
+ result_len = vsnprintf(frag_container->fragment, buf_len, fmt, args);
+ }
+
spin_lock(&stream->lock);
- stream->length += len;
+ stream->length += result_len;
list_add_tail(&frag_container->node, &stream->fragments);
spin_unlock(&stream->lock);

diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h
index b669f9a75a94..048930bf97f0 100644
--- a/lib/kunit/string-stream.h
+++ b/lib/kunit/string-stream.h

@@ -25,6 +25,7 @@ struct string_stream {
spinlock_t lock;

struct kunit *test;
gfp_t gfp;

[Richard Fitzgerald]

There is no need to use a test-managed alloc in is_literal().
The function frees the temporary buffer before returning.

This removes the only use of the test and gfp members of
struct string_stream outside of the string_stream implementation.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
Reviewed-by: David Gow <davi...@google.com>
---
lib/kunit/assert.c | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/lib/kunit/assert.c b/lib/kunit/assert.c
index 05a09652f5a1..dd1d633d0fe2 100644
--- a/lib/kunit/assert.c
+++ b/lib/kunit/assert.c
@@ -89,8 +89,7 @@ void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert,
EXPORT_SYMBOL_GPL(kunit_ptr_not_err_assert_format);

/* Checks if `text` is a literal representing `value`, e.g. "5" and 5 */
-static bool is_literal(struct kunit *test, const char *text, long long value,
- gfp_t gfp)
+static bool is_literal(const char *text, long long value)
{
char *buffer;
int len;
@@ -100,14 +99,15 @@ static bool is_literal(struct kunit *test, const char *text, long long value,
if (strlen(text) != len)
return false;

- buffer = kunit_kmalloc(test, len+1, gfp);
+ buffer = kmalloc(len+1, GFP_KERNEL);
if (!buffer)
return false;

snprintf(buffer, len+1, "%lld", value);
ret = strncmp(buffer, text, len) == 0;

- kunit_kfree(test, buffer);
+ kfree(buffer);
+
return ret;
}

@@ -125,14 +125,12 @@ void kunit_binary_assert_format(const struct kunit_assert *assert,
binary_assert->text->left_text,
binary_assert->text->operation,
binary_assert->text->right_text);
- if (!is_literal(stream->test, binary_assert->text->left_text,
- binary_assert->left_value, stream->gfp))
+ if (!is_literal(binary_assert->text->left_text, binary_assert->left_value))
string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %lld (0x%llx)\n",
binary_assert->text->left_text,
binary_assert->left_value,
binary_assert->left_value);
- if (!is_literal(stream->test, binary_assert->text->right_text,
- binary_assert->right_value, stream->gfp))
+ if (!is_literal(binary_assert->text->right_text, binary_assert->right_value))
string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %lld (0x%llx)",
binary_assert->text->right_text,
binary_assert->right_value,
--
2.30.2

[Richard Fitzgerald]

Add function kunit_alloc_string_stream() to do a resource-managed
allocation of a string stream, and corresponding
kunit_free_string_stream() to free the resource-managed stream.

This is preparing for decoupling the string_stream
implementation from struct kunit, to reduce the amount of code
churn when that happens. Currently:
- kunit_alloc_string_stream() only calls alloc_string_stream().
- kunit_free_string_stream() takes a struct kunit* which
isn't used yet.

Callers of the old alloc_string_stream() and
string_stream_destroy() are all requesting a managed allocation
so have been changed to use the new functions.

alloc_string_stream() has been temporarily made static because
its current behavior has been replaced with
kunit_alloc_string_stream().

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
Reviewed-by: David Gow <davi...@google.com>
---

lib/kunit/string-stream-test.c | 28 ++++++++++++++--------------
lib/kunit/string-stream.c | 12 +++++++++++-
lib/kunit/string-stream.h | 3 ++-
lib/kunit/test.c | 4 ++--
4 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index f117c4b7389b..46b18e940b73 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -25,7 +25,7 @@ static void string_stream_init_test(struct kunit *test)
{
struct string_stream *stream;

- stream = alloc_string_stream(test, GFP_KERNEL);
+ stream = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

KUNIT_EXPECT_EQ(test, stream->length, 0);
@@ -48,7 +48,7 @@ static void string_stream_line_add_test(struct kunit *test)
size_t len, total_len;
int num_lines, i;

- stream = alloc_string_stream(test, GFP_KERNEL);
+ stream = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

/* Add series of sequence numbered lines */

@@ -104,7 +104,7 @@ static void string_stream_variable_length_line_test(struct kunit *test)
size_t offset, total_len;
int num_lines, i;

- stream = alloc_string_stream(test, GFP_KERNEL);
+ stream = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

/*
@@ -164,10 +164,10 @@ static void string_stream_append_test(struct kunit *test)
size_t combined_length;
int i;

- stream_1 = alloc_string_stream(test, GFP_KERNEL);
+ stream_1 = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);

- stream_2 = alloc_string_stream(test, GFP_KERNEL);
+ stream_2 = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2);

/* Append content of empty stream to empty stream */

@@ -207,9 +207,9 @@ static void string_stream_append_test(struct kunit *test)
KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1), combined_content);

/* Append content of non-empty stream to empty stream */

- string_stream_destroy(stream_1);
+ kunit_free_string_stream(test, stream_1);

- stream_1 = alloc_string_stream(test, GFP_KERNEL);
+ stream_1 = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);

string_stream_append(stream_1, stream_2);
@@ -222,13 +222,13 @@ static void string_stream_append_auto_newline_test(struct kunit *test)
struct string_stream *stream_1, *stream_2;

/* Stream 1 has newline appending enabled */

- stream_1 = alloc_string_stream(test, GFP_KERNEL);
+ stream_1 = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_1);
string_stream_set_append_newlines(stream_1, true);
KUNIT_EXPECT_TRUE(test, stream_1->append_newlines);

/* Stream 2 does not append newlines */

- stream_2 = alloc_string_stream(test, GFP_KERNEL);
+ stream_2 = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2);

/* Appending a stream with a newline should not add another newline */

@@ -239,8 +239,8 @@ static void string_stream_append_auto_newline_test(struct kunit *test)
KUNIT_EXPECT_STREQ(test, get_concatenated_string(test, stream_1),

"Original string\nAppended content\nMore stuff\n");

- string_stream_destroy(stream_2);
- stream_2 = alloc_string_stream(test, GFP_KERNEL);
+ kunit_free_string_stream(test, stream_2);
+ stream_2 = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream_2);

/*
@@ -261,7 +261,7 @@ static void string_stream_append_empty_string_test(struct kunit *test)
struct string_stream *stream;
int original_frag_count;

- stream = alloc_string_stream(test, GFP_KERNEL);
+ stream = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

/* Formatted empty string */

@@ -283,7 +283,7 @@ static void string_stream_no_auto_newline_test(struct kunit *test)
{
struct string_stream *stream;

- stream = alloc_string_stream(test, GFP_KERNEL);
+ stream = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

/*
@@ -306,7 +306,7 @@ static void string_stream_auto_newline_test(struct kunit *test)
{
struct string_stream *stream;

- stream = alloc_string_stream(test, GFP_KERNEL);
+ stream = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

string_stream_set_append_newlines(stream, true);
diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index 1dcf6513b692..12ecf15e1f6b 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -153,7 +153,7 @@ bool string_stream_is_empty(struct string_stream *stream)
return list_empty(&stream->fragments);
}

-struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp)
+static struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp)
{
struct string_stream *stream;

@@ -173,3 +173,13 @@ void string_stream_destroy(struct string_stream *stream)
{
string_stream_clear(stream);
}
+
+struct string_stream *kunit_alloc_string_stream(struct kunit *test, gfp_t gfp)
+{
+ return alloc_string_stream(test, gfp);
+}
+
+void kunit_free_string_stream(struct kunit *test, struct string_stream *stream)
+{
+ string_stream_destroy(stream);
+}
diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h
index 048930bf97f0..3e70ee9d66e9 100644
--- a/lib/kunit/string-stream.h
+++ b/lib/kunit/string-stream.h
@@ -30,7 +30,8 @@ struct string_stream {

struct kunit;

-struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp);
+struct string_stream *kunit_alloc_string_stream(struct kunit *test, gfp_t gfp);
+void kunit_free_string_stream(struct kunit *test, struct string_stream *stream);

int __printf(2, 3) string_stream_add(struct string_stream *stream,
const char *fmt, ...);

diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 49698a168437..93d9225d61e3 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -308,7 +308,7 @@ static void kunit_fail(struct kunit *test, const struct kunit_loc *loc,

kunit_set_failure(test);

- stream = alloc_string_stream(test, GFP_KERNEL);
+ stream = kunit_alloc_string_stream(test, GFP_KERNEL);
if (IS_ERR(stream)) {
WARN(true,
"Could not allocate stream to print failed assertion in %s:%d\n",
@@ -322,7 +322,7 @@ static void kunit_fail(struct kunit *test, const struct kunit_loc *loc,

kunit_print_string_stream(test, stream);

- string_stream_destroy(stream);
+ kunit_free_string_stream(test, stream);
}

void __noreturn __kunit_abort(struct kunit *test)
--
2.30.2

[Richard Fitzgerald]

string_stream_managed_free_test() allocates a resource-managed
string_stream and tests that kunit_free_string_stream() calls
string_stream_destroy().

string_stream_resource_free_test() allocates a resource-managed
string_stream and tests that string_stream_destroy() is called
when the test resources are cleaned up.

The old string_stream_init_test() has been split into two tests,
one for kunit_alloc_string_stream() and the other for
alloc_string_stream().

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
---

Changes since v5:

- Fix memory leak when calling the redirected string_stream_destroy_stub().

---
lib/kunit/string-stream-test.c | 147 +++++++++++++++++++++++++++++++--
lib/kunit/string-stream.c | 3 +
2 files changed, 145 insertions(+), 5 deletions(-)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index 58ba1ef5207f..b759974d9cec 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c
@@ -6,17 +6,33 @@
* Author: Brendan Higgins <brendan...@google.com>
*/

+#include <kunit/static_stub.h>
#include <kunit/test.h>
#include <linux/slab.h>

#include "string-stream.h"

-/* This avoids a cast warning if kfree() is passed direct to kunit_add_action(). */
+struct string_stream_test_priv {
+ /* For testing resource-managed free. */
+ struct string_stream *expected_free_stream;
+ bool stream_was_freed;
+ bool stream_free_again;
+};
+
+/* Avoids a cast warning if kfree() is passed direct to kunit_add_action(). */
static void kfree_wrapper(void *p)
{
kfree(p);
}

+/* Avoids a cast warning if string_stream_destroy() is passed direct to kunit_add_action(). */
+static void cleanup_raw_stream(void *p)
+{

+ struct string_stream *stream = p;
+
+ string_stream_destroy(stream);

+}
+
static char *get_concatenated_string(struct kunit *test, struct string_stream *stream)
{
char *str = string_stream_get_string(stream);
@@ -27,11 +43,12 @@ static char *get_concatenated_string(struct kunit *test, struct string_stream *s
return str;
}

-/* string_stream object is initialized correctly. */
-static void string_stream_init_test(struct kunit *test)
+/* Managed string_stream object is initialized correctly. */
+static void string_stream_managed_init_test(struct kunit *test)
{
struct string_stream *stream;

+ /* Resource-managed initialization. */

stream = kunit_alloc_string_stream(test, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);

@@ -42,6 +59,109 @@ static void string_stream_init_test(struct kunit *test)
KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
}

+/* Unmanaged string_stream object is initialized correctly. */
+static void string_stream_unmanaged_init_test(struct kunit *test)

+{
+ struct string_stream *stream;
+

+ stream = alloc_string_stream(GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+ kunit_add_action(test, cleanup_raw_stream, stream);

+
+ KUNIT_EXPECT_EQ(test, stream->length, 0);
+ KUNIT_EXPECT_TRUE(test, list_empty(&stream->fragments));

+ KUNIT_EXPECT_EQ(test, stream->gfp, GFP_KERNEL);
+ KUNIT_EXPECT_FALSE(test, stream->append_newlines);
+
+ KUNIT_EXPECT_TRUE(test, string_stream_is_empty(stream));
+}
+
+static void string_stream_destroy_stub(struct string_stream *stream)
+{
+ struct kunit *fake_test = kunit_get_current_test();
+ struct string_stream_test_priv *priv = fake_test->priv;
+
+ /* The kunit could own string_streams other than the one we are testing. */
+ if (stream == priv->expected_free_stream) {
+ if (priv->stream_was_freed)
+ priv->stream_free_again = true;
+ else
+ priv->stream_was_freed = true;
+ }
+
+ /*
+ * Calling string_stream_destroy() will only call this function again
+ * because the redirection stub is still active.
+ * Avoid calling deactivate_static_stub() or changing current->kunit_test
+ * during cleanup.
+ */

+ string_stream_clear(stream);
+ kfree(stream);
+}
+

+/* kunit_free_string_stream() calls string_stream_desrtoy() */
+static void string_stream_managed_free_test(struct kunit *test)
+{
+ struct string_stream_test_priv *priv = test->priv;
+
+ priv->expected_free_stream = NULL;
+ priv->stream_was_freed = false;
+ priv->stream_free_again = false;
+
+ kunit_activate_static_stub(test,
+ string_stream_destroy,
+ string_stream_destroy_stub);
+
+ priv->expected_free_stream = kunit_alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv->expected_free_stream);
+
+ /* This should call the stub function. */
+ kunit_free_string_stream(test, priv->expected_free_stream);
+
+ KUNIT_EXPECT_TRUE(test, priv->stream_was_freed);
+ KUNIT_EXPECT_FALSE(test, priv->stream_free_again);
+}
+
+/* string_stream object is freed when test is cleaned up. */
+static void string_stream_resource_free_test(struct kunit *test)
+{
+ struct string_stream_test_priv *priv = test->priv;
+ struct kunit *fake_test;
+
+ fake_test = kunit_kzalloc(test, sizeof(*fake_test), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fake_test);
+
+ kunit_init_test(fake_test, "string_stream_fake_test", NULL);
+ fake_test->priv = priv;
+
+ /*
+ * Activate stub before creating string_stream so the
+ * string_stream will be cleaned up first.
+ */
+ priv->expected_free_stream = NULL;
+ priv->stream_was_freed = false;
+ priv->stream_free_again = false;
+
+ kunit_activate_static_stub(fake_test,
+ string_stream_destroy,
+ string_stream_destroy_stub);
+
+ priv->expected_free_stream = kunit_alloc_string_stream(fake_test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv->expected_free_stream);
+
+ /* Set current->kunit_test to fake_test so the static stub will be called. */
+ current->kunit_test = fake_test;
+
+ /* Cleanup test - the stub function should be called */
+ kunit_cleanup(fake_test);
+
+ /* Set current->kunit_test back to current test. */
+ current->kunit_test = test;
+
+ KUNIT_EXPECT_TRUE(test, priv->stream_was_freed);
+ KUNIT_EXPECT_FALSE(test, priv->stream_free_again);
+}
+
/*

* Add a series of lines to a string_stream. Check that all lines

* appear in the correct order and no characters are dropped.

@@ -334,8 +454,24 @@ static void string_stream_auto_newline_test(struct kunit *test)

"One\nTwo\nThree\nFour\nFive\nSix\nSeven\n\nEight\n");
}

+static int string_stream_test_init(struct kunit *test)
+{
+ struct string_stream_test_priv *priv;
+
+ priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ test->priv = priv;
+

+ return 0;
+}
+

static struct kunit_case string_stream_test_cases[] = {

- KUNIT_CASE(string_stream_init_test),
+ KUNIT_CASE(string_stream_managed_init_test),
+ KUNIT_CASE(string_stream_unmanaged_init_test),
+ KUNIT_CASE(string_stream_managed_free_test),
+ KUNIT_CASE(string_stream_resource_free_test),
KUNIT_CASE(string_stream_line_add_test),
KUNIT_CASE(string_stream_variable_length_line_test),
KUNIT_CASE(string_stream_append_test),
@@ -348,6 +484,7 @@ static struct kunit_case string_stream_test_cases[] = {

static struct kunit_suite string_stream_test_suite = {
.name = "string-stream-test",
- .test_cases = string_stream_test_cases
+ .test_cases = string_stream_test_cases,
+ .init = string_stream_test_init,
};
kunit_test_suites(&string_stream_test_suite);
diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index 64abceb7b716..a6f3616c2048 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -6,6 +6,7 @@
* Author: Brendan Higgins <brendan...@google.com>
*/

+#include <kunit/static_stub.h>
#include <kunit/test.h>
#include <linux/list.h>
#include <linux/slab.h>
@@ -170,6 +171,8 @@ struct string_stream *alloc_string_stream(gfp_t gfp)

void string_stream_destroy(struct string_stream *stream)
{
+ KUNIT_STATIC_STUB_REDIRECT(string_stream_destroy, stream);
+
if (!stream)
return;

--
2.30.2

[Richard Fitzgerald]

Add a test of the speed and memory use of string_stream.

string_stream_performance_test() doesn't actually "test" anything (it
cannot fail unless the system has run out of allocatable memory) but it
measures the speed and memory consumption of the string_stream and reports
the result.

This allows changes in the string_stream implementation to be compared.

Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
Reviewed-by: David Gow <davi...@google.com>
---

lib/kunit/string-stream-test.c | 54 ++++++++++++++++++++++++++++++++++
1 file changed, 54 insertions(+)

diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c
index b759974d9cec..06822766f29a 100644
--- a/lib/kunit/string-stream-test.c
+++ b/lib/kunit/string-stream-test.c

@@ -8,7 +8,9 @@

#include <kunit/static_stub.h>
#include <kunit/test.h>
+#include <linux/ktime.h>
#include <linux/slab.h>
+#include <linux/timekeeping.h>

#include "string-stream.h"

@@ -454,6 +456,57 @@ static void string_stream_auto_newline_test(struct kunit *test)

"One\nTwo\nThree\nFour\nFive\nSix\nSeven\n\nEight\n");
}

+/*
+ * This doesn't actually "test" anything. It reports time taken
+ * and memory used for logging a large number of lines.
+ */

+static void string_stream_performance_test(struct kunit *test)
+{
+ struct string_stream_fragment *frag_container;
+ struct string_stream *stream;

+ char test_line[101];
+ ktime_t start_time, end_time;
+ size_t len, bytes_requested, actual_bytes_used, total_string_length;
+ int offset, i;
+
+ stream = kunit_alloc_string_stream(test, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+ memset(test_line, 'x', sizeof(test_line) - 1);
+ test_line[sizeof(test_line) - 1] = '\0';
+
+ start_time = ktime_get();
+ for (i = 0; i < 10000; i++) {
+ offset = i % (sizeof(test_line) - 1);
+ string_stream_add(stream, "%s: %d\n", &test_line[offset], i);
+ }
+ end_time = ktime_get();

+
+ /*

@@ -479,6 +532,7 @@ static struct kunit_case string_stream_test_cases[] = {

[David Gow]

On Mon, 28 Aug 2023 at 18:41, Richard Fitzgerald
<r...@opensource.cirrus.com> wrote:
>
> string_stream_managed_free_test() allocates a resource-managed
> string_stream and tests that kunit_free_string_stream() calls
> string_stream_destroy().
>
> string_stream_resource_free_test() allocates a resource-managed
> string_stream and tests that string_stream_destroy() is called
> when the test resources are cleaned up.
>
> The old string_stream_init_test() has been split into two tests,
> one for kunit_alloc_string_stream() and the other for
> alloc_string_stream().
>
> Signed-off-by: Richard Fitzgerald <r...@opensource.cirrus.com>
> ---
> Changes since v5:
> - Fix memory leak when calling the redirected string_stream_destroy_stub().
> ---

Much better, thanks.

Reviewed-by: David Gow <davi...@google.com>

Cheers,
-- David