STLVmFixChecksumHw for trex stl vxlan ipv6 for mellanox nic

70 views

Skip to first unread message

dheer...@gmail.com

unread,

Mar 12, 2026, 2:25:57 AMMar 12

to TRex Traffic Generator

Hi ,

I am using mellanox nic [ConnectX-6 Lx] . If I pump vxlan ipv6 traffic using trex stl , I am getting "bad udp cksum" on the receiver DUT.

I narrowed it down and observed that this is due to STLVmFixChecksumHw incorrectly calculating the checksum.

I cannot make inner header checksum as 0, as it is required for the internal test.

If I send some sample packets streams without using field engine, then there is no checksum issue. Even this issue is not seen while using XXV710 nic.

Is there any fix or workaround without affecting the performance ?

Below is the code snippet :

base_pkt = Ether(src= source_mac, dst = dest_mac)/ IP(src=source_ip, dst=dest_ip)/UDP(sport=1054,dport=4789,chksum=0x0000)/VXLAN(vni=1000)/Ether(src="02:00:00:02:15:01", dst= "02:00:00:02:14:01")/IPv6(dst="2001:db8:abcd:11:f816:3eff:0002:1401", src="2001:db8:abcd:11:f816:3eff:0002:1501")/UDP(sport=60001, dport=5005)

if (1):
vm1 = STLScVmRaw( [

STLVmFlowVar(name="dst_port",
min_value=5005,
max_value=30000,
size=2),

STLVmWrFlowVar(fv_name="dst_port",
pkt_offset= "UDP:1.dport"), # fix udp len

STLVmFixChecksumHw(l3_offset="IPv6",
l4_offset="UDP:1",
l4_type=CTRexVmInsFixHwCs.L4_TYPE_UDP),

Regards,

Dheeraj

dheeraj kumar

unread,

Jul 6, 2026, 7:42:37 AMJul 6

to TRex Traffic Generator

Hi TRex Team,

I wanted to highlight that the issue mentioned in my previous email seems to be originating from the TRex side and not from the Mellanox NIC. I have got a proposed fix using the AI tool on TRex version 3.06, and it is working successfully.

To provide a brief recap:
**Problem:** TRex's STLVmFixChecksumHw produces an incorrect inner UDP checksum for VXLAN-encapsulated IPv6 packets on Mellanox ConnectX-6 Lx NICs, due to the mlx5 PMD requiring explicit tunnel metadata in the mbuf, which TRex doesn't set.
**Proposed Fix:** Detect the Mellanox NIC at port initialization time (`g_mlx_nic` flag), and then at stream load time and runtime, auto-detect VXLAN tunnel and set `RTE_MBUF_F_TX_TUNNEL_VXLAN` along with the outer IP offload flags and IPv6 pseudo-header checksum. This adjustment ensures that the NIC computes the correct inner L4 checksum in hardware, while Intel NICs can bypass this path entirely.

Could you please take this proposed fix as reference and provide the fix in the latest TRex STL code.

git diff -c | cat

diff --git a/src/drivers/trex_driver_base.cpp b/src/drivers/trex_driver_base.cpp
index 775dfb3b2..11b227b12 100755
--- a/src/drivers/trex_driver_base.cpp
+++ b/src/drivers/trex_driver_base.cpp
@@ -62,7 +62,9 @@ port_cfg_t::port_cfg_t() {
tx_offloads.common_best_effort =
RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
- RTE_ETH_TX_OFFLOAD_TCP_CKSUM;
+ RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
+ RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
+ RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO;

tx_offloads.common_required =
RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
diff --git a/src/main_dpdk.cpp b/src/main_dpdk.cpp
index 2a98f9231..ad4e3dadf 100644
--- a/src/main_dpdk.cpp
+++ b/src/main_dpdk.cpp
@@ -5932,6 +5932,11 @@ COLD_FUNC void CPhyEthIF::_conf_queues(uint16_t tx_qs,
// disable non-supported best-effort offloads
tx_offloads &= dev_info->tx_offload_capa;

+ /* Global variable for Mellanox NICs */
+ extern bool g_mlx_nic;
+ if (CTRexExtendedDriverDb::Ins()->get_driver_name().find("mlx") != std::string::npos) {
+ g_mlx_nic = true;
+ }

tx_offloads |= cfg.tx_offloads.common_required;

diff --git a/src/stx/stl/trex_stl_stream_vm.cpp b/src/stx/stl/trex_stl_stream_vm.cpp
index a44a251a9..7931f786e 100644
--- a/src/stx/stl/trex_stl_stream_vm.cpp
+++ b/src/stx/stl/trex_stl_stream_vm.cpp
@@ -33,6 +33,8 @@ limitations under the License.
#include "trex_stl_stream.h"
#include "trex_stl_stream_vm.h"

+/* Global flag: true if NIC is Mellanox (mlx4/mlx5) */
+bool g_mlx_nic = false;

/**
* provides some tools for the fast rand function
@@ -783,6 +785,9 @@ void StreamVm::build_program(){
StreamDPOpHwCsFix ipv_fix;
ipv_fix.m_l2_len = lpFix->m_l2_len;
ipv_fix.m_l3_len = l3_len;
+ ipv_fix.m_outer_l2_len = 0;
+ ipv_fix.m_outer_l3_len = 0;
+
ipv_fix.m_op = StreamDPVmInstructions::ditFIX_HW_CS;

if (packet_is_ipv4) {
@@ -800,21 +805,58 @@ void StreamVm::build_program(){
l4_header_size = 0;
}
}
- }else{
+ } else {
/* Ipv6*/
- /* in this case we need to scan the Ipv6 headers */
- /* TBD replace with parser of IPv6 function */
- if ( lpFix->m_l4_type==StreamVmInstructionFixHwChecksum::L4_TYPE_TCP ){
- ipv_fix.m_ol_flags = (RTE_MBUF_F_TX_IPV6 | RTE_MBUF_F_TX_TCP_CKSUM);
- l4_header_size = TCP_HEADER_LEN;
- }else{
- if ( lpFix->m_l4_type==StreamVmInstructionFixHwChecksum::L4_TYPE_UDP ){
- ipv_fix.m_ol_flags = (RTE_MBUF_F_TX_IPV6 | RTE_MBUF_F_TX_UDP_CKSUM);
+ /* Check if inner IPv6 is inside a VXLAN tunnel for Mellanox NIC */
+ extern bool g_mlx_nic;
+ bool is_ipv6_tunnel = false;
+ if (g_mlx_nic && lpFix->m_l2_len > ETH_HDR_LEN) {
+ /* Find outer L3: skip Ethernet + optional VLAN tags */
+ uint16_t outer_l2_len = ETH_HDR_LEN;
+ uint16_t ethertype = PKT_NTOHS(*((uint16_t*)(m_pkt + ETH_HDR_LEN - 2 /* ETHER_TYPE_LEN */)));
+ while (ethertype == 0x8100 || ethertype == 0x88A8 || ethertype == 0x9100) {
+ outer_l2_len += 4; /* VLAN_HDR_LEN */
+ ethertype = PKT_NTOHS(*((uint16_t*)(m_pkt + outer_l2_len - 2 /* ETHER_TYPE_LEN */)));
+ }
+ if (ethertype == 0x0800) {
+ IPHeader *outer = (IPHeader *)(m_pkt + outer_l2_len);
+ if (outer->getVersion() == 4 &&
+ outer->getNextProtocol() == IPHeader::Protocol::UDP) {
+ is_ipv6_tunnel = true;
+ ipv_fix.m_outer_l2_len = outer_l2_len;
+ ipv_fix.m_outer_l3_len = outer->getSize();
+ ipv_fix.m_l2_len = lpFix->m_l2_len - outer_l2_len - outer->getSize();
+ }
+ }
+ }
+
+ if (g_mlx_nic && is_ipv6_tunnel) {
+ /* VXLAN tunnel with inner IPv6 (Mellanox only) */
+ if ( lpFix->m_l4_type==StreamVmInstructionFixHwChecksum::L4_TYPE_TCP ){
+ ipv_fix.m_ol_flags = (RTE_MBUF_F_TX_IPV6 | RTE_MBUF_F_TX_TCP_CKSUM |
+ RTE_MBUF_F_TX_TUNNEL_VXLAN |
+ RTE_MBUF_F_TX_OUTER_IP_CKSUM | RTE_MBUF_F_TX_OUTER_IPV4);
+ l4_header_size = TCP_HEADER_LEN;
+ } else {
+ ipv_fix.m_ol_flags = (RTE_MBUF_F_TX_IPV6 | RTE_MBUF_F_TX_UDP_CKSUM |
+ RTE_MBUF_F_TX_TUNNEL_VXLAN |
+ RTE_MBUF_F_TX_OUTER_IP_CKSUM | RTE_MBUF_F_TX_OUTER_IPV4);
l4_header_size = UDP_HEADER_LEN;
- }else{
- std::stringstream ss;
- ss << "instruction id '" << ins_id << "' fix hw command offsets should be TCP or UDP for IPv6 ";
- err(ss.str());
+ }
+ } else {
+ /* Non-tunnel IPv6 (original) — covers all NICs and plain IPv6 on mlx */
+ if ( lpFix->m_l4_type==StreamVmInstructionFixHwChecksum::L4_TYPE_TCP ){
+ ipv_fix.m_ol_flags = (RTE_MBUF_F_TX_IPV6 | RTE_MBUF_F_TX_TCP_CKSUM);
+ l4_header_size = TCP_HEADER_LEN;
+ } else {
+ if ( lpFix->m_l4_type==StreamVmInstructionFixHwChecksum::L4_TYPE_UDP ){
+ ipv_fix.m_ol_flags = (RTE_MBUF_F_TX_IPV6 | RTE_MBUF_F_TX_UDP_CKSUM);
+ l4_header_size = UDP_HEADER_LEN;
+ } else {
+ std::stringstream ss;
+ ss << "instruction id '" << ins_id << "' fix hw command offsets should be TCP or UDP for IPv6 ";
+ err(ss.str());
+ }
}
}
}
diff --git a/src/stx/stl/trex_stl_stream_vm.h b/src/stx/stl/trex_stl_stream_vm.h
index 4d0f2307b..a4c998a5e 100644
--- a/src/stx/stl/trex_stl_stream_vm.h
+++ b/src/stx/stl/trex_stl_stream_vm.h
@@ -801,46 +801,73 @@ public:

} __attribute__((packed));

+extern bool g_mlx_nic;
+
/* fix checksum using hardware */
struct StreamDPOpHwCsFix {
uint8_t m_op;
uint16_t m_l2_len;
uint16_t m_l3_len;
- uint64_t m_ol_flags;
+ uint64_t m_ol_flags;
+ uint16_t m_outer_l2_len; /* 0 = non-tunnel, >0 = tunnel mode (IPv6 inner only) */
+ uint16_t m_outer_l3_len;

public:
void dump(FILE *fd,std::string opt);
void HOT_FUNC run(uint8_t * pkt_base,rte_mbuf_t * m){
- IPHeader * ipv4 = (IPHeader *)(pkt_base+m_l2_len);
union {
TCPHeader * tcp;
UDPHeader * udp;
} u;

- u.tcp = (TCPHeader*)(pkt_base+m_l2_len+m_l3_len);
- /* set the mbuf info */
- m->l2_len = m_l2_len;
- m->l3_len = m_l3_len;
- m->ol_flags |= m_ol_flags;
- if (m_ol_flags & RTE_MBUF_F_TX_IPV4 ){ /* splitting to 4 instructions didn't improve performance .. */
- ipv4->ClearCheckSum();
- if ((m_ol_flags & RTE_MBUF_F_TX_L4_MASK) == RTE_MBUF_F_TX_TCP_CKSUM ){
- u.tcp->setChecksumRaw(rte_ipv4_phdr_cksum((struct rte_ipv4_hdr *)ipv4,m_ol_flags));
- }else{
- u.udp->setChecksumRaw(rte_ipv4_phdr_cksum((struct rte_ipv4_hdr *)ipv4,m_ol_flags));
- }
-
- }else{
- if ((m_ol_flags & RTE_MBUF_F_TX_L4_MASK) == RTE_MBUF_F_TX_TCP_CKSUM ){
- u.tcp->setChecksumRaw(rte_ipv6_phdr_cksum((struct rte_ipv6_hdr *)ipv4,m_ol_flags));
- }else{
- u.udp->setChecksumRaw(rte_ipv6_phdr_cksum((struct rte_ipv6_hdr *)ipv4,m_ol_flags));
- }
- }
- }
-} __attribute__((packed));
-
-
+ if (m_outer_l2_len > 0 && g_mlx_nic) {
+ /* === TUNNEL MODE (VXLAN with IPv6 inner, Mellanox only) ===
+ * Use flat l2_len model with tunnel flags in ol_flags.
+ * l2_len = total offset to inner IPv6 header
+ * l3_len = inner IPv6 header length
+ * Tunnel flags in ol_flags enable mlx5 tunnel-aware TX.
+ */
+ uint16_t inner_l3_offset = m_outer_l2_len + m_outer_l3_len + m_l2_len;
+ uint8_t *inner_l3 = pkt_base + inner_l3_offset;
+ u.tcp = (TCPHeader *)(inner_l3 + m_l3_len);
+
+ m->l2_len = inner_l3_offset;
+ m->l3_len = m_l3_len;
+ m->l4_len = UDP_HEADER_LEN;
+ m->ol_flags |= m_ol_flags;
+
+ /* set inner IPv6 pseudo-header checksum for NIC to complete */
+ if ((m_ol_flags & RTE_MBUF_F_TX_L4_MASK) == RTE_MBUF_F_TX_TCP_CKSUM) {
+ u.tcp->setChecksumRaw(rte_ipv6_phdr_cksum((struct rte_ipv6_hdr *)inner_l3, m->ol_flags));
+ } else {
+ u.udp->setChecksumRaw(rte_ipv6_phdr_cksum((struct rte_ipv6_hdr *)inner_l3, m->ol_flags));
+ }
+ } else {
+ /* === NON-TUNNEL MODE (original logic for Intel NICs and non-tunnel packets) === */
+ IPHeader *ipv4 = (IPHeader *)(pkt_base + m_l2_len);
+ u.tcp = (TCPHeader *)(pkt_base + m_l2_len + m_l3_len);
+
+ m->l2_len = m_l2_len;
+ m->l3_len = m_l3_len;
+ m->ol_flags |= m_ol_flags;
+
+ if (m_ol_flags & RTE_MBUF_F_TX_IPV4) {
+ ipv4->ClearCheckSum();
+ if ((m_ol_flags & RTE_MBUF_F_TX_L4_MASK) == RTE_MBUF_F_TX_TCP_CKSUM) {
+ u.tcp->setChecksumRaw(rte_ipv4_phdr_cksum((struct rte_ipv4_hdr *)ipv4, m_ol_flags));
+ } else {
+ u.udp->setChecksumRaw(rte_ipv4_phdr_cksum((struct rte_ipv4_hdr *)ipv4, m_ol_flags));
+ }
+ } else {
+ if ((m_ol_flags & RTE_MBUF_F_TX_L4_MASK) == RTE_MBUF_F_TX_TCP_CKSUM) {
+ u.tcp->setChecksumRaw(rte_ipv6_phdr_cksum((struct rte_ipv6_hdr *)ipv4, m_ol_flags));
+ } else {
+ u.udp->setChecksumRaw(rte_ipv6_phdr_cksum((struct rte_ipv6_hdr *)ipv4, m_ol_flags));
+ }
+ }
+ }
+ }
+ } __attribute__((packed));

/* flow varible of Client command */
struct StreamDPFlowClient {

Regards ,

Dheeraj

--
You received this message because you are subscribed to the Google Groups "TRex Traffic Generator" group.
To unsubscribe from this group and stop receiving emails from it, send an email to trex-tgn+u...@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/trex-tgn/c5b277bb-9ba6-4f9a-bbbf-658b965c96dcn%40googlegroups.com.

Reply all

Reply to author

Forward

0 new messages