[COMMIT seastar master] Merge 'Minor improvements to the json subsystem' from Stephan Dollberg

[Commit Bot]

From: Pavel Emelyanov <xe...@scylladb.com>
Committer: Pavel Emelyanov <xe...@scylladb.com>
Branch: master

Merge 'Minor improvements to the json subsystem' from Stephan Dollberg

Some improvements to the json subsystem:

- Fix a lifetime issue in stream_range_as_array
- Make formatter::write(const jsonable&) recursively call `write` and not `to_json`. This allows for recursive streamable encoding and hence avoids some potentially very large string allocations
- Adds some tests for the above

Note that because whitespace bahaviour is inconsistent across the different ways of encoding things (`write`, `to_json`, vector encoding, map encoding, ...) some of the tests need adapting as there is now less whitespace.

Closes scylladb/seastar#2287

* https://github.com/scylladb/seastar:
json: Add a test for jsonable objects
json: Make formatter::write(vector/map/umap) copy their arguments
json: Make formatter call write for jsonable

---
diff --git a/include/seastar/json/formatter.hh b/include/seastar/json/formatter.hh
--- a/include/seastar/json/formatter.hh
+++ b/include/seastar/json/formatter.hh
@@ -302,18 +302,24 @@ public:
* @return the given vector in a json format
*/
template<typename... Args>
- static future<> write(output_stream<char>& s, const std::vector<Args...>& vec) {
- return write(s, state::array, vec.begin(), vec.end());
+ static future<> write(output_stream<char>& s, std::vector<Args...> vec) {
+ return do_with(std::move(vec), [&s] (const auto& vec) {
+ return write(s, state::array, vec.begin(), vec.end());
+ });
}

template<typename... Args>
- static future<> write(output_stream<char>& s, const std::map<Args...>& map) {
- return write(s, state::map, map.begin(), map.end());
+ static future<> write(output_stream<char>& s, std::map<Args...> map) {
+ return do_with(std::move(map), [&s] (const auto& map) {
+ return write(s, state::map, map.begin(), map.end());
+ });
}

template<typename... Args>
- static future<> write(output_stream<char>& s, const std::unordered_map<Args...>& map) {
- return write(s, state::map, map.begin(), map.end());
+ static future<> write(output_stream<char>& s, std::unordered_map<Args...> map) {
+ return do_with(std::move(map), [&s] (const auto& map) {
+ return write(s, state::map, map.begin(), map.end());
+ });
}

/**
@@ -330,9 +336,12 @@ public:
* @param obj the date_time to format
* @return the given json object in a json format
*/
- static future<> write(output_stream<char>& s, const jsonable& obj) {
- return s.write(to_json(obj));
- }
+ template <std::derived_from<jsonable> Jsonable>
+ static future<> write(output_stream<char>& s, Jsonable obj) {
+ return do_with(std::move(obj), [&s] (const auto& obj) {
+ return obj.write(s);
+ });
+ }

/**
* return a json formatted unsigned long
diff --git a/tests/unit/httpd_test.cc b/tests/unit/httpd_test.cc
--- a/tests/unit/httpd_test.cc
+++ b/tests/unit/httpd_test.cc
@@ -735,18 +735,18 @@ std::string get_value(int size) {
/*
* A helper object that map to a big json string
* in the format of:
- * {"valu": "aaa....aa", "valu": "aaa....aa", "valu": "aaa....aa"...}
+ * {"valu":"aaa....aa","valu":"aaa....aa","valu":"aaa....aa"...}
*
* The object can have an arbitrary size in multiplication of 10000 bytes
* */
struct extra_big_object : public json::json_base {
json::json_element<sstring>* value;
extra_big_object(size_t size) {
value = new json::json_element<sstring>;
- // size = brackets + (name + ": " + get_value) * n + ", " * (n-1)
- // size = 2 + (name + 6 + get_value) * n - 2
+ // size = brackets + ("name" + ":" + get_value()) * n + "," * (n-1)
+ // size = 2 + (valu + 4 + get_value) * n - 1
value->_name = "valu";
- *value = get_value(9990);
+ *value = get_value(9992);
for (size_t i = 0; i < size/10000; i++) {
_elements.emplace_back(value);
}
@@ -769,7 +769,9 @@ struct extra_big_object : public json::json_base {
SEASTAR_TEST_CASE(json_stream) {
std::vector<extra_big_object> vec;
size_t num_objects = 1000;
- size_t total_size = num_objects * 1000001 + 1;
+ // each object is 1000001 bytes plus a comma for all but the last and plus
+ // two for the [] backets
+ size_t total_size = num_objects * 1000002 + 1;
for (size_t i = 0; i < num_objects; i++) {
vec.emplace_back(1000000);
}
diff --git a/tests/unit/json_formatter_test.cc b/tests/unit/json_formatter_test.cc
--- a/tests/unit/json_formatter_test.cc
+++ b/tests/unit/json_formatter_test.cc
@@ -24,8 +24,10 @@
#include <seastar/core/do_with.hh>
#include <seastar/testing/test_case.hh>
#include <seastar/core/sstring.hh>
-#include <seastar/core/do_with.hh>
+#include <seastar/core/vector-data-sink.hh>
#include <seastar/json/formatter.hh>
+#include <seastar/json/json_elements.hh>
+#include <seastar/testing/thread_test_case.hh>

using namespace seastar;
using namespace json;
@@ -59,3 +61,57 @@ SEASTAR_TEST_CASE(test_collections) {

return make_ready_future();
}
+
+struct object_json : public json_base {
+ json_element<sstring> subject;
+ json_list<long> values;
+
+ void register_params() {
+ add(&subject, "subject");
+ add(&values, "values");
+ }
+
+ object_json() { register_params(); }
+
+ object_json(const object_json &e) {
+ register_params();
+ subject = e.subject;
+ values = e.values;
+ }
+};
+
+SEASTAR_TEST_CASE(test_jsonable) {
+ object_json obj;
+ obj.subject = "foo";
+ obj.values.push(1);
+ obj.values.push(2);
+ obj.values.push(3);
+
+ BOOST_CHECK_EQUAL("{\"subject\": \"foo\", \"values\": [1,2,3]}", formatter::to_json(obj));
+ return make_ready_future();
+}
+
+SEASTAR_THREAD_TEST_CASE(test_stream_range_as_array) {
+ auto vec = std::vector<net::packet>{};
+ auto out = output_stream<char>(data_sink(std::make_unique<vector_data_sink>(vec)), 8);
+
+ auto mapper = stream_range_as_array(std::vector<int>{1,2,3}, [] (auto i) {
+ object_json obj;
+ obj.subject = std::to_string(i);
+ obj.values.push(i);
+ return obj;
+ });
+
+ mapper(std::move(out)).get();
+
+ auto packets = net::packet{};
+ for (auto &p : vec) {
+ packets.append(std::move(p));
+ }
+ packets.linearize();
+ auto buf = packets.release();
+
+ sstring result(buf.front().get(), buf.front().size());
+ sstring expected = "[{\"subject\":\"1\",\"values\":[1]}, {\"subject\":\"2\",\"values\":[2]}, {\"subject\":\"3\",\"values\":[3]}]";
+ BOOST_CHECK_EQUAL(expected, result);
+}