[PATCH 1/2] build: Include dpdk as a single object in libseastar.a

[Kefu Chai]

this change brings 4bd84425084e25bc957c5693b7a299739b7f039f back.
which was previously reverted by
76c87384dc8866aae603a4b598ff0adc8c9bac5a.

76c87384dc8866aae603a4b598ff0adc8c9bac5a intended to bring the DPDK
dependencies as an interface library instead of *including* the DPDK
libraries in libseastar.a as a relocatable output file. after
76c87384dc8866aae603a4b598ff0adc8c9bac5a, in addition to linking against
libseastar.a, we have to link against DPDK libraries when linking a
Seastar application when DPDK is enabled. this renders the
redistribution of the Seastar library more complicated. because we
always build our own dpdk, without including dpdk into libseastar.a,
we have to redistribute the DPDK libraries along with libseastar.

also, the rationales explained in the commit message of
4bd84425084e25bc957c5693b7a299739b7f039f still hold.

Signed-off-by: Kefu Chai <tcha...@gmail.com>
---
CMakeLists.txt | 12 ++++++++++++
cmake/Finddpdk.cmake | 12 ++++++++++++
2 files changed, 24 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9b1193b9..4a9c5702 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -428,9 +428,14 @@ seastar_generate_ragel (
IN_FILE ${CMAKE_CURRENT_SOURCE_DIR}/src/http/response_parser.rl
OUT_FILE ${Seastar_GEN_BINARY_DIR}/include/seastar/http/response_parser.hh)

+if (Seastar_DPDK)
+ set (seastar_dpdk_obj seastar-dpdk.o)
+endif ()
+
add_library (seastar STATIC
${http_chunk_parsers_file}
${http_request_parser_file}
+ ${seastar_dpdk_obj}
include/seastar/core/abort_source.hh
include/seastar/core/alien.hh
include/seastar/core/align.hh
@@ -882,6 +887,13 @@ if (Seastar_DPDK)
target_compile_definitions (seastar
PUBLIC SEASTAR_HAVE_DPDK)

+ # No pmd driver code will be pulled in without "--whole-archive". To
+ # avoid exposing that to seastar users, combine dpdk into a single
+ # .o file.
+ add_custom_command (
+ OUTPUT seastar-dpdk.o
+ COMMAND ${CMAKE_C_COMPILER} -r -o seastar-dpdk.o ${dpdk_LIBRARIES}
+ )
# This just provides the include path to cmake
target_include_directories (seastar
PUBLIC $<TARGET_PROPERTY:dpdk::dpdk,INTERFACE_INCLUDE_DIRECTORIES>)
diff --git a/cmake/Finddpdk.cmake b/cmake/Finddpdk.cmake
index c8a381ea..e8257df0 100644
--- a/cmake/Finddpdk.cmake
+++ b/cmake/Finddpdk.cmake
@@ -38,9 +38,21 @@ if (dpdk_PC_FOUND AND dpdk_PC_STATIC_LIBRARIES)
INTERFACE_INCLUDE_DIRECTORIES "${dpdk_PC_STATIC_INCLUDE_DIRS}"
INTERFACE_LINK_OPTIONS "${dpdk_PC_STATIC_LDFLAGS}"
INTERFACE_LINK_DIRECTORIES "${dpdk_PC_STATIC_LIBRARY_DIRS}")
+ list(FIND dpdk_PC_STATIC_LDFLAGS "-Wl,--whole-archive" begin)
+ list(FIND dpdk_PC_STATIC_LDFLAGS "-Wl,--no-whole-archive" end)
+ if (begin EQUAL -1 OR end EQUAL -1)
+ message(FATAL_ERROR "failed to parse dpdk LDFLAGS: ${dpdk_PC_STATIC_LDFLAGS}")
+ endif ()
+ math(EXPR length "${end} - ${begin}")
+ list(SUBLIST dpdk_PC_STATIC_LDFLAGS ${begin} ${length} archives)
+ list(FILTER dpdk_PC_STATIC_LDFLAGS INCLUDE REGEX "-L.*")
+ set(dpdk_LIBRARIES
+ "${dpdk_PC_STATIC_LDFLAGS}"
+ ${archives})
return ()
endif ()
elseif (dpdk_PC_FOUND)
+ set(dpdk_LIBRARIES "${dpdk_PC_LDFLAGS}")
find_package_handle_standard_args (dpdk
REQUIRED_VARS
dpdk_PC_CFLAGS
--
2.33.0

[Kefu Chai]

to address the FTBFS with clang-13, like

build/debug/_cooking/stow/dpdk/include/generic/rte_spinlock.h: In function ‘void rte_spinlock_recursive_lock(rte_spinlock_recursive_t*)’:
build/debug/_cooking/stow/dpdk/include/generic/rte_spinlock.h:221:14: error: ‘++’ expression of ‘volatile’-qualified type is deprecated [-Werror=volatile]
221 | slr->count++;
| ~~~~~^~~~~

Signed-off-by: Kefu Chai <tcha...@gmail.com>
---

CMakeLists.txt | 9 +++++++++
1 file changed, 9 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4a9c5702..cd419648 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -803,6 +803,15 @@ if (MaybeUninitialized_FOUND)
-Wno-maybe-uninitialized)
endif ()

+if (Seastar_DPDK)
+ seastar_supports_flag ("-Wno-error=volatile" ErrorVolatile_FOUND)
+ if (ErrorVolatile_FOUND)
+ target_compile_options (seastar
+ PUBLIC
+ -Wno-error=volatile
+ endif ()
+endif ()
+
if (Seastar_SSTRING)
target_compile_options (seastar PUBLIC -DSEASTAR_SSTRING)
endif ()
--
2.33.0

[Commit Bot]

From: Kefu Chai <tcha...@gmail.com>
Committer: Avi Kivity <a...@scylladb.com>
Branch: master

build: Include dpdk as a single object in libseastar.a

this change brings 4bd84425084e25bc957c5693b7a299739b7f039f back.
which was previously reverted by
76c87384dc8866aae603a4b598ff0adc8c9bac5a.

76c87384dc8866aae603a4b598ff0adc8c9bac5a intended to bring the DPDK
dependencies as an interface library instead of *including* the DPDK
libraries in libseastar.a as a relocatable output file. after
76c87384dc8866aae603a4b598ff0adc8c9bac5a, in addition to linking against
libseastar.a, we have to link against DPDK libraries when linking a
Seastar application when DPDK is enabled. this renders the
redistribution of the Seastar library more complicated. because we
always build our own dpdk, without including dpdk into libseastar.a,
we have to redistribute the DPDK libraries along with libseastar.

also, the rationales explained in the commit message of
4bd84425084e25bc957c5693b7a299739b7f039f still hold.

Signed-off-by: Kefu Chai <tcha...@gmail.com>

Message-Id: <20211018172240.4...@gmail.com>

---
diff --git a/CMakeLists.txt b/CMakeLists.txt

[Commit Bot]

From: Kefu Chai <tcha...@gmail.com>
Committer: Avi Kivity <a...@scylladb.com>
Branch: master

build: add -Wno-error=volatile to CXX_FLAGS

to address the FTBFS with clang-13, like

build/debug/_cooking/stow/dpdk/include/generic/rte_spinlock.h: In function ‘void rte_spinlock_recursive_lock(rte_spinlock_recursive_t*)’:
build/debug/_cooking/stow/dpdk/include/generic/rte_spinlock.h:221:14: error: ‘++’ expression of ‘volatile’-qualified type is deprecated [-Werror=volatile]
221 | slr->count++;
| ~~~~~^~~~~

Signed-off-by: Kefu Chai <tcha...@gmail.com>

Message-Id: <20211018172240.4...@gmail.com>

---
diff --git a/CMakeLists.txt b/CMakeLists.txt

[Commit Bot]

From: Kefu Chai <tcha...@gmail.com>
Committer: Avi Kivity <a...@scylladb.com>
Branch: master

build: add -Wno-error=volatile to CXX_FLAGS

to address the FTBFS with clang-13, like

build/debug/_cooking/stow/dpdk/include/generic/rte_spinlock.h: In function ‘void rte_spinlock_recursive_lock(rte_spinlock_recursive_t*)’:
build/debug/_cooking/stow/dpdk/include/generic/rte_spinlock.h:221:14: error: ‘++’ expression of ‘volatile’-qualified type is deprecated [-Werror=volatile]
221 | slr->count++;
| ~~~~~^~~~~

Signed-off-by: Kefu Chai <tcha...@gmail.com>

[avi: close parentheses in cmake goo]

Message-Id: <20211018172240.4...@gmail.com>

---
diff --git a/CMakeLists.txt b/CMakeLists.txt

--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -803,6 +803,15 @@ if (MaybeUninitialized_FOUND)
-Wno-maybe-uninitialized)
endif ()

+if (Seastar_DPDK)
+ seastar_supports_flag ("-Wno-error=volatile" ErrorVolatile_FOUND)
+ if (ErrorVolatile_FOUND)
+ target_compile_options (seastar
+ PUBLIC

+ -Wno-error=volatile)

[Avi Kivity]

Even with this closing parentheses added, I get errors with clang 12:


error: unknown warning option '-Werror=volatile'
[-Werror,-Wunknown-warning-option]


So it looks like the detection code isn't working well.

[Avi Kivity]

On 10/18/21 20:22, Kefu Chai wrote:
> this change brings 4bd84425084e25bc957c5693b7a299739b7f039f back.
> which was previously reverted by
> 76c87384dc8866aae603a4b598ff0adc8c9bac5a.
>
> 76c87384dc8866aae603a4b598ff0adc8c9bac5a intended to bring the DPDK
> dependencies as an interface library instead of *including* the DPDK
> libraries in libseastar.a as a relocatable output file. after
> 76c87384dc8866aae603a4b598ff0adc8c9bac5a, in addition to linking against
> libseastar.a, we have to link against DPDK libraries when linking a
> Seastar application when DPDK is enabled. this renders the
> redistribution of the Seastar library more complicated. because we
> always build our own dpdk, without including dpdk into libseastar.a,
> we have to redistribute the DPDK libraries along with libseastar.
>
> also, the rationales explained in the commit message of
> 4bd84425084e25bc957c5693b7a299739b7f039f still hold.

I get these nice errors:


FAILED: seastar-dpdk.o
cd /home/avi/scylla-maint/build/release/seastar && clang -r -o
seastar-dpdk.o
-L/home/avi/scylla-maint/build/release/seastar/_cooking/stow/dpdk/lib64
-L/usr/usr/lib64 -L/usr/usr/lib64 -L/usr/usr/lib64 -Wl,--whole-archive
-L/home/avi/scylla-maint/build/release/seastar/_cooking/stow/dpdk/lib64
-l:librte_common_cpt.a -l:librte_common_dpaax.a -l:librte_common_iavf.a
-l:librte_common_octeontx.a -l:librte_common_octeontx2.a
-l:librte_bus_auxiliary.a -l:librte_bus_dpaa.a -l:librte_bus_fslmc.a
-l:librte_bus_ifpga.a -l:librte_bus_pci.a -l:librte_bus_vdev.a
-l:librte_bus_vmbus.a -l:librte_common_cnxk.a -l:librte_common_mlx5.a
-l:librte_common_qat.a -l:librte_common_sfc_efx.a
-l:librte_mempool_bucket.a -l:librte_mempool_cnxk.a
-l:librte_mempool_dpaa.a -l:librte_mempool_dpaa2.a
-l:librte_mempool_octeontx.a -l:librte_mempool_octeontx2.a
-l:librte_mempool_ring.a -l:librte_mempool_stack.a
-l:librte_net_af_packet.a -l:librte_net_ark.a -l:librte_net_atlantic.a
-l:librte_net_avp.a -l:librte_net_axgbe.a -l:librte_net_bnx2x.a
-l:librte_net_bnxt.a -l:librte_net_bond.a -l:librte_net_cnxk.a
-l:librte_net_cxgbe.a -l:librte_net_dpaa.a -l:librte_net_dpaa2.a
-l:librte_net_e1000.a -l:librte_net_ena.a -l:librte_net_enetc.a
-l:librte_net_enic.a -l:librte_net_failsafe.a -l:librte_net_fm10k.a
-l:librte_net_hinic.a -l:librte_net_hns3.a -l:librte_net_i40e.a
-l:librte_net_iavf.a -l:librte_net_ice.a -l:librte_net_igc.a
-l:librte_net_ionic.a -l:librte_net_ixgbe.a -l:librte_net_kni.a
-l:librte_net_liquidio.a -l:librte_net_memif.a -l:librte_net_mlx4.a
-l:librte_net_mlx5.a -l:librte_net_netvsc.a -l:librte_net_nfp.a
-l:librte_net_ngbe.a -l:librte_net_null.a -l:librte_net_octeontx.a
-l:librte_net_octeontx2.a -l:librte_net_octeontx_ep.a
-l:librte_net_pfe.a -l:librte_net_qede.a -l:librte_net_ring.a
-l:librte_net_sfc.a -l:librte_net_softnic.a -l:librte_net_tap.a
-l:librte_net_thunderx.a -l:librte_net_txgbe.a
-l:librte_net_vdev_netvsc.a -l:librte_net_vhost.a -l:librte_net_virtio.a
-l:librte_net_vmxnet3.a -l:librte_raw_cnxk_bphy.a
-l:librte_raw_dpaa2_cmdif.a -l:librte_raw_dpaa2_qdma.a
-l:librte_raw_ioat.a -l:librte_raw_ntb.a -l:librte_raw_octeontx2_dma.a
-l:librte_raw_octeontx2_ep.a -l:librte_raw_skeleton.a
-l:librte_crypto_bcmfs.a -l:librte_crypto_caam_jr.a
-l:librte_crypto_ccp.a -l:librte_crypto_cnxk.a
-l:librte_crypto_dpaa_sec.a -l:librte_crypto_dpaa2_sec.a
-l:librte_crypto_mlx5.a -l:librte_crypto_nitrox.a
-l:librte_crypto_null.a -l:librte_crypto_octeontx.a
-l:librte_crypto_octeontx2.a -l:librte_crypto_openssl.a
-l:librte_crypto_scheduler.a -l:librte_crypto_virtio.a
-l:librte_compress_mlx5.a -l:librte_compress_octeontx.a
-l:librte_compress_zlib.a -l:librte_regex_mlx5.a
-l:librte_regex_octeontx2.a -l:librte_vdpa_ifc.a -l:librte_vdpa_mlx5.a
-l:librte_event_cnxk.a -l:librte_event_dlb2.a -l:librte_event_dpaa.a
-l:librte_event_dpaa2.a -l:librte_event_dsw.a
-l:librte_event_octeontx2.a -l:librte_event_opdl.a
-l:librte_event_skeleton.a -l:librte_event_sw.a
-l:librte_event_octeontx.a -l:librte_baseband_acc100.a
-l:librte_baseband_fpga_5gnr_fec.a -l:librte_baseband_fpga_lte_fec.a
-l:librte_baseband_null.a -l:librte_baseband_turbo_sw.a -l:librte_node.a
-l:librte_graph.a -l:librte_bpf.a -l:librte_flow_classify.a
-l:librte_pipeline.a -l:librte_table.a -l:librte_port.a -l:librte_fib.a
-l:librte_ipsec.a -l:librte_vhost.a -l:librte_stack.a
-l:librte_security.a -l:librte_sched.a -l:librte_reorder.a
-l:librte_rib.a -l:librte_regexdev.a -l:librte_rawdev.a
-l:librte_pdump.a -l:librte_member.a -l:librte_lpm.a
-l:librte_latencystats.a -l:librte_kni.a -l:librte_jobstats.a
-l:librte_ip_frag.a -l:librte_gso.a -l:librte_gro.a -l:librte_eventdev.a
-l:librte_efd.a -l:librte_distributor.a -l:librte_cryptodev.a
-l:librte_compressdev.a -l:librte_cfgfile.a -l:librte_bitratestats.a
-l:librte_bbdev.a -l:librte_acl.a -l:librte_timer.a -l:librte_hash.a
-l:librte_metrics.a -l:librte_cmdline.a -l:librte_pci.a
-l:librte_ethdev.a -l:librte_meter.a -l:librte_net.a -l:librte_mbuf.a
-l:librte_mempool.a -l:librte_rcu.a -l:librte_ring.a -l:librte_eal.a
-l:librte_telemetry.a -l:librte_kvargs.a
/usr/bin/ld: cannot find -lgcc_s
/usr/bin/ld: cannot find -lc
/usr/bin/ld: /usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_muldi3.o): in
function `__multi3':
(.text+0x0): multiple definition of `__multi3';
/usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_muldi3.o):(.text+0x0):
first defined here
/usr/bin/ld: /usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_negdi2.o): in
function `__negti2':
(.text+0x0): multiple definition of `__negti2';
/usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_negdi2.o):(.text+0x0):
first defined here
/usr/bin/ld: /usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_lshrdi3.o):
in function `__lshrti3':
(.text+0x0): multiple definition of `__lshrti3';
/usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_lshrdi3.o):(.text+0x0):
first defined here
/usr/bin/ld: /usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_ashldi3.o):
in function `__ashlti3':
(.text+0x0): multiple definition of `__ashlti3';
/usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_ashldi3.o):(.text+0x0):
first defined here
/usr/bin/ld: /usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_ashrdi3.o):
in function `__ashrti3':
(.text+0x0): multiple definition of `__ashrti3';
/usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_ashrdi3.o):(.text+0x0):
first defined here
/usr/bin/ld: /usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_cmpdi2.o): in
function `__cmpti2':
(.text+0x0): multiple definition of `__cmpti2';
/usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_cmpdi2.o):(.text+0x0):
first defined here
/usr/bin/ld: /usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_ucmpdi2.o):
in function `__ucmpti2':
(.text+0x0): multiple definition of `__ucmpti2';
/usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_ucmpdi2.o):(.text+0x0):
first defined here
/usr/bin/ld:
/usr/lib/gcc/x86_64-redhat-linux/11/libgcc.a(_clear_cache.o): in
function `__clear_cache':

[Nicolas Le Scouarnec]

Hi,

The command succeeds when compiling everything with gcc (DPDK and Seastar). However, I see that the command line passed to clang misses "-Wl,--no-archive" at the end (when comparing to what I had previously). If I add them, the number of "defined" twice symbols is greatly reduced. There remains this warning, which explains why -lgcc_s is drawn. Wouldn't it be possible to pass the compiler from seastar into DPDK ? I.e., compiling DPDK with clang if clang is used for DPDK.

lescouarnecn@explo-server [ubuntu]~/u/b/release ❯❯ clang-10 -r -o seastar-dpdk.o -L/home/lescouarnecn/upstream_seastar/build/release/_cooking/stow/dpdk/lib/x86_64-linux-gnu -Wl,--whole-archive -l:librte_common_cpt.a ... -l:librte_kvargs.a -Wl,--no-whole-archive empty.cc
/usr/bin/ld: cannot find -lgcc_s
/usr/bin/ld: /home/lescouarnecn/upstream_seastar/build/release/_cooking/stow/dpdk/lib/x86_64-linux-gnu/librte_eal.a(eal_common_eal_common_trace_utils.c.o): in function `trace_mkdir':
eal_common_trace_utils.c:(.text+0xa20): warning: Using 'getpwuid' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
/usr/bin/ld: cannot find -lgcc_s
clang: error: linker command failed with exit code 1 (use -v to see invocation)

Best regards,
Nicolas

> -----Original Message-----
> From: seast...@googlegroups.com <seast...@googlegroups.com>
> On Behalf Of Avi Kivity
> Sent: Monday, 18 October 2021 19:43
> To: Kefu Chai <tcha...@gmail.com>; seast...@googlegroups.com
> Subject: Re: [seastar-dev] [PATCH 1/2] build: Include dpdk as a single object
> in libseastar.a
>

> > + set (seastar_dpdk_obj seastar-dpdk.o) endif ()

> > +
> > add_library (seastar STATIC
> > ${http_chunk_parsers_file}
> > ${http_request_parser_file}
> > + ${seastar_dpdk_obj}
> > include/seastar/core/abort_source.hh
> > include/seastar/core/alien.hh
> > include/seastar/core/align.hh
> > @@ -882,6 +887,13 @@ if (Seastar_DPDK)
> > target_compile_definitions (seastar
> > PUBLIC SEASTAR_HAVE_DPDK)
> >
> > + # No pmd driver code will be pulled in without "--whole-archive".

> > + To # avoid exposing that to seastar users, combine dpdk into a
> > + single # .o file.

> > + add_custom_command (
> > + OUTPUT seastar-dpdk.o
> > + COMMAND ${CMAKE_C_COMPILER} -r -o seastar-dpdk.o

> > + ${dpdk_LIBRARIES}

> --
> You received this message because you are subscribed to the Google Groups
> "seastar-dev" group.
> To unsubscribe from this group and stop receiving emails from it, send an
> email to seastar-dev...@googlegroups.com.
> To view this discussion on the web visit
> https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgrou
> ps.google.com%2Fd%2Fmsgid%2Fseastar-dev%2Ff1b604f6-c387-0262-09ed-
> 761f46e9f469%2540scylladb.com&amp;data=04%7C01%7Cnicolas.lescouarne
> c%40broadpeak.tv%7C7c52d1001e064c224cf008d9925ecbc4%7C0ebe44eac9c
> 9438da0407e699f358ed4%7C0%7C0%7C637701758120510418%7CUnknown%
> 7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haW
> wiLCJXVCI6Mn0%3D%7C1000&amp;sdata=UP%2Fg81J2tird0mtwCK%2FPH%2
> FeYX9OkRE%2Bvc2WkpVFEYic%3D&amp;reserved=0.

[Nicolas Le Scouarnec]

Hi Kefu,

As a followup to my previous message, adding -Wl,--no-whole-archive -nodefaultlibs seems to help for building seastar-dpdk.o. However, I could not build the complete seastar, as I got some errors with clang-10 on dpdk_rte.cc . Also, I removed the target_link_libraries as I believe they aren't necessary now that everything is in seastar-dpdk.o

In case it can help you progress toward a solution:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4a9c5702..c6439e1d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -740,11 +740,11 @@ target_link_libraries (seastar
rt::rt
yaml-cpp::yaml-cpp
Threads::Threads)
-if (Seastar_DPDK)
- target_link_libraries (seastar
- PUBLIC
- dpdk::dpdk)
-endif()
+#if (Seastar_DPDK)
+# target_link_libraries (seastar
+# PUBLIC
+# dpdk::dpdk)
+#endif()

set (Seastar_SANITIZE_MODES "Debug" "Sanitize")
if ((Seastar_SANITIZE STREQUAL "ON") OR
@@ -892,7 +892,7 @@ if (Seastar_DPDK)
# .o file.
add_custom_command (
OUTPUT seastar-dpdk.o
- COMMAND ${CMAKE_C_COMPILER} -r -o seastar-dpdk.o ${dpdk_LIBRARIES}
+ COMMAND ${CMAKE_C_COMPILER} -r -o seastar-dpdk.o ${dpdk_LIBRARIES} -Wl,-no-whole-archive -nodefaultlibs

)
# This just provides the include path to cmake

> -----Original Message-----
> From: seast...@googlegroups.com <seast...@googlegroups.com>
> On Behalf Of Nicolas Le Scouarnec
> Sent: Monday, 18 October 2021 21:40
> To: Avi Kivity <a...@scylladb.com>; Kefu Chai <tcha...@gmail.com>; seastar-
> d...@googlegroups.com
> Subject: RE: [seastar-dev] [PATCH 1/2] build: Include dpdk as a single object
> in libseastar.a
>

> dev%2FDB8PR10MB34814F89ED29CF9B27D58C4C90BC9%2540DB8PR10MB34
> 81.EURPRD10.PROD.OUTLOOK.COM&amp;data=04%7C01%7Cnicolas.lescoua
> rnec%40broadpeak.tv%7C7a6b33bc0cbe4eb8fd4808d9926f210a%7C0ebe44e
> ac9c9438da0407e699f358ed4%7C0%7C0%7C637701828259982991%7CUnkno
> wn%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1
> haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=Rvm1CVVIrBTXhKcQdOFIA4P
> %2FIfJu7bdl%2BO2jGSVEcpU%3D&amp;reserved=0.

[Kefu Chai]

v2

* add missing closing "-Wl,--no-whole-archive" to the ldflags
* add -Werror to fail the build if the an unknown warning option is
specified

Kefu Chai (2):

build: add -Wno-error=volatile to CXX_FLAGS

build: add the closing "-Wl,--no-whole-archive" to the ldflags

CMakeLists.txt | 9 +++++++++
cmake/Finddpdk.cmake | 2 +-
2 files changed, 10 insertions(+), 1 deletion(-)

--
2.33.0

[Kefu Chai]

to address the FTBFS with clang-13, like

build/debug/_cooking/stow/dpdk/include/generic/rte_spinlock.h: In function ‘void rte_spinlock_recursive_lock(rte_spinlock_recursive_t*)’:
build/debug/_cooking/stow/dpdk/include/generic/rte_spinlock.h:221:14: error: ‘++’ expression of ‘volatile’-qualified type is deprecated [-Werror=volatile]
221 | slr->count++;
| ~~~~~^~~~~

Signed-off-by: Kefu Chai <tcha...@gmail.com>
---

CMakeLists.txt | 9 +++++++++
1 file changed, 9 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4a9c5702..3296952a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt

@@ -803,6 +803,15 @@ if (MaybeUninitialized_FOUND)
-Wno-maybe-uninitialized)
endif ()

+if (Seastar_DPDK)

+ seastar_supports_flag ("-Wno-error=volatile -Werror" ErrorVolatile_FOUND)

+ if (ErrorVolatile_FOUND)
+ target_compile_options (seastar
+ PUBLIC
+ -Wno-error=volatile)

+ endif ()
+endif ()
+
if (Seastar_SSTRING)
target_compile_options (seastar PUBLIC -DSEASTAR_SSTRING)
endif ()

--
2.33.0

[Kefu Chai]

this should address the link failure

Signed-off-by: Kefu Chai <tcha...@gmail.com>
---

cmake/Finddpdk.cmake | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/Finddpdk.cmake b/cmake/Finddpdk.cmake
index e8257df0..fb3eba9a 100644
--- a/cmake/Finddpdk.cmake
+++ b/cmake/Finddpdk.cmake
@@ -43,7 +43,7 @@ if (dpdk_PC_FOUND AND dpdk_PC_STATIC_LIBRARIES)

if (begin EQUAL -1 OR end EQUAL -1)

message(FATAL_ERROR "failed to parse dpdk LDFLAGS: ${dpdk_PC_STATIC_LDFLAGS}")

endif ()
- math(EXPR length "${end} - ${begin}")
+ math(EXPR length "${end} - ${begin} + 1")

list(SUBLIST dpdk_PC_STATIC_LDFLAGS ${begin} ${length} archives)

list(FILTER dpdk_PC_STATIC_LDFLAGS INCLUDE REGEX "-L.*")

set(dpdk_LIBRARIES
--
2.33.0

[tcha...@gmail.com]

On Tuesday, October 19, 2021 at 5:08:35 AM UTC+8 nicolas.l...@broadpeak.tv wrote:

Hi Kefu,

As a followup to my previous message, adding -Wl,--no-whole-archive -nodefaultlibs seems to help for building seastar-dpdk.o. However, I could not build the complete seastar, as I got some errors with clang-10 on dpdk_rte.cc . Also, I removed the target_link_libraries as I believe they aren't necessary now that everything is in seastar-dpdk.o

much appreciated, Nicolas! i just added the missing "-Wl,--no-whole-archive" to the ldflags. and the tests build just fine. let's see how things go on the reviewers' (hey Avi!) side.

[Commit Bot]

From: Kefu Chai <tcha...@gmail.com>
Committer: Avi Kivity <a...@scylladb.com>
Branch: master

build: add -Wno-error=volatile to CXX_FLAGS

to address the FTBFS with clang-13, like

build/debug/_cooking/stow/dpdk/include/generic/rte_spinlock.h: In function ‘void rte_spinlock_recursive_lock(rte_spinlock_recursive_t*)’:
build/debug/_cooking/stow/dpdk/include/generic/rte_spinlock.h:221:14: error: ‘++’ expression of ‘volatile’-qualified type is deprecated [-Werror=volatile]
221 | slr->count++;
| ~~~~~^~~~~

Signed-off-by: Kefu Chai <tcha...@gmail.com>

Message-Id: <20211019001334.1...@gmail.com>

---
diff --git a/CMakeLists.txt b/CMakeLists.txt

[Commit Bot]

From: Kefu Chai <tcha...@gmail.com>
Committer: Avi Kivity <a...@scylladb.com>
Branch: master

build: add the closing "-Wl,--no-whole-archive" to the ldflags

this should address the link failure

Signed-off-by: Kefu Chai <tcha...@gmail.com>

Message-Id: <20211019001334.1...@gmail.com>

---
diff --git a/cmake/Finddpdk.cmake b/cmake/Finddpdk.cmake

[Avi Kivity]

On 19/10/2021 03.16, tcha...@gmail.com wrote:

On Tuesday, October 19, 2021 at 5:08:35 AM UTC+8 nicolas.l...@broadpeak.tv wrote:

Hi Kefu,

As a followup to my previous message, adding -Wl,--no-whole-archive -nodefaultlibs seems to help for building seastar-dpdk.o. However, I could not build the complete seastar, as I got some errors with clang-10 on dpdk_rte.cc . Also, I removed the target_link_libraries as I believe they aren't necessary now that everything is in seastar-dpdk.o

much appreciated, Nicolas! i just added the missing "-Wl,--no-whole-archive" to the ldflags. and the tests build just fine. let's see how things go on the reviewers' (hey Avi!) side.

I thought it worked, but a clean build (removing .cooking_memory) gets me this:

FAILED: seastar-dpdk.o
cd /home/avi/scylla-maint/build/release/seastar && clang -r -o seastar-dpdk.o -L/home/avi/scylla-maint/build/release/seastar/_cooking/stow/dpdk/lib64 -L/usr/usr/lib64 -L/usr/usr/lib64 -L/usr/usr/lib64 -Wl,--whole-archive -L/home/avi/scylla-maint/build/release/seastar/_cooking/stow/dpdk/lib64 -l:librte_common_cpt.a -l:librte_common_dpaax.a -l:librte_common_iavf.a -l:librte_common_octeontx.a -l:librte_common_octeontx2.a -l:librte_bus_auxiliary.a -l:librte_bus_dpaa.a -l:librte_bus_fslmc.a -l:librte_bus_ifpga.a -l:librte_bus_pci.a -l:librte_bus_vdev.a -l:librte_bus_vmbus.a -l:librte_common_cnxk.a -l:librte_common_mlx5.a -l:librte_common_qat.a -l:librte_common_sfc_efx.a -l:librte_mempool_bucket.a -l:librte_mempool_cnxk.a -l:librte_mempool_dpaa.a -l:librte_mempool_dpaa2.a -l:librte_mempool_octeontx.a -l:librte_mempool_octeontx2.a -l:librte_mempool_ring.a -l:librte_mempool_stack.a -l:librte_net_af_packet.a -l:librte_net_ark.a -l:librte_net_atlantic.a -l:librte_net_avp.a -l:librte_net_axgbe.a -l:librte_net_bnx2x.a -l:librte_net_bnxt.a -l:librte_net_bond.a -l:librte_net_cnxk.a -l:librte_net_cxgbe.a -l:librte_net_dpaa.a -l:librte_net_dpaa2.a -l:librte_net_e1000.a -l:librte_net_ena.a -l:librte_net_enetc.a -l:librte_net_enic.a -l:librte_net_failsafe.a -l:librte_net_fm10k.a -l:librte_net_hinic.a -l:librte_net_hns3.a -l:librte_net_i40e.a -l:librte_net_iavf.a -l:librte_net_ice.a -l:librte_net_igc.a -l:librte_net_ionic.a -l:librte_net_ixgbe.a -l:librte_net_kni.a -l:librte_net_liquidio.a -l:librte_net_memif.a -l:librte_net_mlx4.a -l:librte_net_mlx5.a -l:librte_net_netvsc.a -l:librte_net_nfp.a -l:librte_net_ngbe.a -l:librte_net_null.a -l:librte_net_octeontx.a -l:librte_net_octeontx2.a -l:librte_net_octeontx_ep.a -l:librte_net_pfe.a -l:librte_net_qede.a -l:librte_net_ring.a -l:librte_net_sfc.a -l:librte_net_softnic.a -l:librte_net_tap.a -l:librte_net_thunderx.a -l:librte_net_txgbe.a -l:librte_net_vdev_netvsc.a -l:librte_net_vhost.a -l:librte_net_virtio.a -l:librte_net_vmxnet3.a -l:librte_raw_cnxk_bphy.a -l:librte_raw_dpaa2_cmdif.a -l:librte_raw_dpaa2_qdma.a -l:librte_raw_ioat.a -l:librte_raw_ntb.a -l:librte_raw_octeontx2_dma.a -l:librte_raw_octeontx2_ep.a -l:librte_raw_skeleton.a -l:librte_crypto_bcmfs.a -l:librte_crypto_caam_jr.a -l:librte_crypto_cnxk.a -l:librte_crypto_dpaa_sec.a -l:librte_crypto_dpaa2_sec.a -l:librte_crypto_mlx5.a -l:librte_crypto_nitrox.a -l:librte_crypto_null.a -l:librte_crypto_octeontx.a -l:librte_crypto_octeontx2.a -l:librte_crypto_scheduler.a -l:librte_crypto_virtio.a -l:librte_compress_mlx5.a -l:librte_compress_octeontx.a -l:librte_compress_zlib.a -l:librte_regex_mlx5.a -l:librte_regex_octeontx2.a -l:librte_vdpa_ifc.a -l:librte_vdpa_mlx5.a -l:librte_event_cnxk.a -l:librte_event_dlb2.a -l:librte_event_dpaa.a -l:librte_event_dpaa2.a -l:librte_event_dsw.a -l:librte_event_octeontx2.a -l:librte_event_opdl.a -l:librte_event_skeleton.a -l:librte_event_sw.a -l:librte_event_octeontx.a -l:librte_baseband_acc100.a -l:librte_baseband_fpga_5gnr_fec.a -l:librte_baseband_fpga_lte_fec.a -l:librte_baseband_null.a -l:librte_baseband_turbo_sw.a -l:librte_node.a -l:librte_graph.a -l:librte_bpf.a -l:librte_flow_classify.a -l:librte_pipeline.a -l:librte_table.a -l:librte_port.a -l:librte_fib.a -l:librte_ipsec.a -l:librte_vhost.a -l:librte_stack.a -l:librte_security.a -l:librte_sched.a -l:librte_reorder.a -l:librte_rib.a -l:librte_regexdev.a -l:librte_rawdev.a -l:librte_pdump.a -l:librte_member.a -l:librte_lpm.a -l:librte_latencystats.a -l:librte_kni.a -l:librte_jobstats.a -l:librte_ip_frag.a -l:librte_gso.a -l:librte_gro.a -l:librte_eventdev.a -l:librte_efd.a -l:librte_distributor.a -l:librte_cryptodev.a -l:librte_compressdev.a -l:librte_cfgfile.a -l:librte_bitratestats.a -l:librte_bbdev.a -l:librte_acl.a -l:librte_timer.a -l:librte_hash.a -l:librte_metrics.a -l:librte_cmdline.a -l:librte_pci.a -l:librte_ethdev.a -l:librte_meter.a -l:librte_net.a -l:librte_mbuf.a -l:librte_mempool.a -l:librte_rcu.a -l:librte_ring.a -l:librte_eal.a -l:librte_telemetry.a -l:librte_kvargs.a -Wl,--no-whole-archive

/usr/bin/ld: cannot find -lgcc_s
/usr/bin/ld: cannot find -lc

/usr/bin/ld: cannot find -lgcc_s
clang-12: error: linker command failed with exit code 1 (use -v to see invocation)
ninja: build stopped: subcommand failed.

To view this discussion on the web visit https://groups.google.com/d/msgid/seastar-dev/ea6f4438-b95e-4008-bbf9-578cc3b6dc4an%40googlegroups.com.

[Avi Kivity]

On 19/10/2021 17.29, Avi Kivity wrote:

On 19/10/2021 03.16, tcha...@gmail.com wrote:

On Tuesday, October 19, 2021 at 5:08:35 AM UTC+8 nicolas.l...@broadpeak.tv wrote:

Hi Kefu,

As a followup to my previous message, adding -Wl,--no-whole-archive -nodefaultlibs seems to help for building seastar-dpdk.o. However, I could not build the complete seastar, as I got some errors with clang-10 on dpdk_rte.cc . Also, I removed the target_link_libraries as I believe they aren't necessary now that everything is in seastar-dpdk.o

much appreciated, Nicolas! i just added the missing "-Wl,--no-whole-archive" to the ldflags. and the tests build just fine. let's see how things go on the reviewers' (hey Avi!) side.

I thought it worked, but a clean build (removing .cooking_memory) gets me this:

FAILED: seastar-dpdk.o
cd /home/avi/scylla-maint/build/release/seastar && clang -r -o seastar-dpdk.o -L/home/avi/scylla-maint/build/release/seastar/_cooking/stow/dpdk/lib64 -L/usr/usr/lib64 -L/usr/usr/lib64 -L/usr/usr/lib64 -Wl,--whole-archive -L/home

Replacing clang with gcc in this command makes it work. So there's some difference between gcc and clang in partial links.

[Avi Kivity]

On 19/10/2021 17.52, Avi Kivity wrote:

On 19/10/2021 17.29, Avi Kivity wrote:

On 19/10/2021 03.16, tcha...@gmail.com wrote:

On Tuesday, October 19, 2021 at 5:08:35 AM UTC+8 nicolas.l...@broadpeak.tv wrote:

Hi Kefu,

As a followup to my previous message, adding -Wl,--no-whole-archive -nodefaultlibs seems to help for building seastar-dpdk.o. However, I could not build the complete seastar, as I got some errors with clang-10 on dpdk_rte.cc . Also, I removed the target_link_libraries as I believe they aren't necessary now that everything is in seastar-dpdk.o

much appreciated, Nicolas! i just added the missing "-Wl,--no-whole-archive" to the ldflags. and the tests build just fine. let's see how things go on the reviewers' (hey Avi!) side.

I thought it worked, but a clean build (removing .cooking_memory) gets me this:

FAILED: seastar-dpdk.o
cd /home/avi/scylla-maint/build/release/seastar && clang -r -o seastar-dpdk.o -L/home/avi/scylla-maint/build/release/seastar/_cooking/stow/dpdk/lib64 -L/usr/usr/lib64 -L/usr/usr/lib64 -L/usr/usr/lib64 -Wl,--whole-archive -L/home

Replacing clang with gcc in this command makes it work. So there's some difference between gcc and clang in partial links.

Adding -nostdlib to the command line makes it work with both clang and gcc!

[kefu chai]

Le mar. 19 oct. 2021 à 22:29, Avi Kivity <a...@scylladb.com> a écrit :

On 19/10/2021 03.16, tcha...@gmail.com wrote:

On Tuesday, October 19, 2021 at 5:08:35 AM UTC+8 nicolas.l...@broadpeak.tv wrote:

Hi Kefu,

As a followup to my previous message, adding -Wl,--no-whole-archive -nodefaultlibs seems to help for building seastar-dpdk.o. However, I could not build the complete seastar, as I got some errors with clang-10 on dpdk_rte.cc . Also, I removed the target_link_libraries as I believe they aren't necessary now that everything is in seastar-dpdk.o

much appreciated, Nicolas! i just added the missing "-Wl,--no-whole-archive" to the ldflags. and the tests build just fine. let's see how things go on the reviewers' (hey Avi!) side.

I thought it worked, but a clean build (removing .cooking_memory) gets me this:

Sorry, Avi. I need to reproduce in my building host using probably fc-32 and clang-12. Was using clang-13 and libstdc++ from gcc-11 in my own test bed. Guess the interesting part is that the linker is looking for gcc. Maybe this echos Nicolas comment that we need to build dodo with specified c compiler. But I double checked that clang was used for building dpdk if it is used for building Seastar.

As am traveling today, will try to fix this in this week.

--

Regards
Kefu Chai

[Nicolas Le Scouarnec]

Hi,

 

I also tested and indeed -nostdlib seems to do the trick. In addition, on my machine I have a few “dependencies” installed such as libbsd, libpcap, libopenssl, Mellanox drivers, … which ends up missing in the final link (this will not occur on “clean” build machines as DPDK’s meson will automatically disable the corresponding drivers/libraries).  This is the part of the pkg-config that is not “fetched” by the static linking command and that should be put in CMake, but also in the “Libs” section of seatar.pc.in (pkg-config).

 

PKG_CONFIG_PATH=build/release/_cooking/installed/lib/x86_64-linux-gnu/pkgconfig pkg-config --static --libs libdpdk

...

-latomic -lmlx5 -lpthread -libverbs -lmlx5 -libverbs -lpthread -lnl-route-3 -lnl-3 -lcrypto -ldl -pthread -lz -lpcap -pthread -lm -ldl -lnuma -lpcap -lbsd

 

I see two “evolutions”

 

1) Adding CMake stuff to handle -lbsd/-lpcap/-lmlx5/-libverbs (directly getting them from DPDK pkg-config – although in our project we hardcoded them temporarily)

dpdk_deps=-lmlx5 -lpthread -libverbs -lmlx5 -libverbs -lpthread -lnl-route-3 -lnl-3 -lz -lbsd -lpcap -lnuma

…

Libs: ${seastar_libs} ${boost_program_options_libs} ${boost_thread_libs} ${c_ares_libs} ${cryptopp_libs} ${fmt_libs} ${gnutls_libs} ${hwloc_libs} ${rt_libs} ${dpdk_deps}

 

2) Disabling more drivers/libs that are not used/exposed through seastar  (obviously disabling mlx5 as it’s a NIC driver wouldn’t make sense). For this, we’re using the following in cooking.recipes, that disable compressdev/cryptodev/regex/… to reduce dependencies.

-Ddisable_drivers="net/softnic,net/bonding,common/dpaax,common/octeontx,common/octeontx2,common/sfc_efx,common/qat,common/cpt,bus/dpaa,net/sfc,bus/ifpga,bus/fslmc,compress/*,crypto/*,baseband/*,regex/*,vdpa/*,event/*"

-Ddisable_libs="kni,jobstats,lpm,acl,power,ip_frag,distributor,reorder,port,table,pipeline,flow_classify,bpf,efd,member, compressdev, cryptodev"

 

Best regards,

To view this discussion on the web visit https://groups.google.com/d/msgid/seastar-dev/d220d9ab-ccd1-e100-7255-f39682f82289%40scylladb.com.

[Avi Kivity]

I'm seeing this too now. I'll revert the dpdk changes for now, and we can reapply after both Nicolas and me see that it works for our uses. There's too much shrapnel flying.

[kefu chai]

Le mar. 19 oct. 2021 à 23:46, Avi Kivity <a...@scylladb.com> a écrit :

I'm seeing this too now. I'll revert the dpdk changes for now, and we can reapply after both Nicolas and me see that it works for our uses. There's too much shrapnel flying.

Will try to fix the issues. And resubmit the patches.

Sorry for the inconvenience!

--

Regards
Kefu Chai