C++ Example is much slower than Python example?

[Eben Kadile]

Hello,

working from the examples given in the inivation documentation, I created some simple examples to view live events from a pair of dvxplorer cameras. I will paste the python code, which is working fine, below (hopefully the formatting imported from vscode makes it easier to read).

```python

from datetime import timedelta

import cv2 as cv

import numpy as np

import dv_processing as dv

# Open cameras

capture_left = dv.io.camera.open("DXM00089")

capture_right = dv.io.camera.open("DXM00090")

# Make sure it supports event stream output, throw an error otherwise

if not (capture_left.isEventStreamAvailable() and capture_right.isEventStreamAvailable()):

raise RuntimeError("Input camera does not provide an event stream.")

# Initialize an accumulator with some resolution

visualizer_left = dv.visualization.EventVisualizer(capture_left.getEventResolution())

# Apply color scheme configuration, these values can be modified to taste

visualizer_left.setBackgroundColor(dv.visualization.colors.black())

visualizer_left.setPositiveColor(dv.visualization.colors.blue())

visualizer_left.setNegativeColor(dv.visualization.colors.green())

# Initialize an accumulator with some resolution

visualizer_right = dv.visualization.EventVisualizer(capture_right.getEventResolution())

# Apply color scheme configuration, these values can be modified to taste

visualizer_right.setBackgroundColor(dv.visualization.colors.black())

visualizer_right.setPositiveColor(dv.visualization.colors.iniBlue())

visualizer_right.setNegativeColor(dv.visualization.colors.green())

# Initialize a preview window

cv.namedWindow("Left", cv.WINDOW_NORMAL)

cv.namedWindow("Right", cv.WINDOW_NORMAL)

# Initialize a slicer

slicer_left = dv.EventStreamSlicer()

slicer_right = dv.EventStreamSlicer()

# background noise filter

high_pass_left = dv.noise.BackgroundActivityNoiseFilter((640, 480), backgroundActivityDuration=timedelta(milliseconds=0.01))

high_pass_right = dv.noise.BackgroundActivityNoiseFilter((640, 480), backgroundActivityDuration=timedelta(milliseconds=0.01))

# refractory period filter

low_pass_left = dv.noise.LowPassFilter((640, 480), 500)

low_pass_right = dv.noise.LowPassFilter((640, 480), 500)

# hot pixel filter

mask_left = 255*np.ones((640, 480), dtype=np.uint8)

mask_right = 255*np.ones((640, 480), dtype=np.uint8)

hot_pixels_left_x = np.load("hot_pixels_mimir_jr/hot_pixels_left_x.npy")

hot_pixels_left_y = np.load("hot_pixels_mimir_jr/hot_pixels_left_y.npy")

hot_pixels_right_x = np.load("hot_pixels_mimir_jr/hot_pixels_right_x.npy")

hot_pixels_right_y = np.load("hot_pixels_mimir_jr/hot_pixels_right_y.npy")

mask_left[hot_pixels_left_x, hot_pixels_left_y] = 0

mask_right[hot_pixels_right_x, hot_pixels_right_y] = 0

print(np.sum((255 - mask_left)/255))

print(np.sum((255 - mask_right)/255))

cold_pass_left = dv.EventMaskFilter(mask_left.T)

cold_pass_right = dv.EventMaskFilter(mask_right.T)

# Declare the callback method for slicer

def slicing_callback_left(events: dv.EventStore):

# Generate a preview frame

frame = visualizer_left.generateImage(events)

# Show the accumulated image

cv.imshow("Left", frame)

cv.waitKey(2)

def slicing_callback_right(events: dv.EventStore):

# Generate a preview frame

frame = visualizer_right.generateImage(events)

# Show the accumulated image

cv.imshow("Right", frame)

cv.waitKey(2)

# Register callback to be performed every 33 milliseconds

slicer_left.doEveryTimeInterval(timedelta(milliseconds=30), slicing_callback_left)

slicer_right.doEveryTimeInterval(timedelta(milliseconds=30), slicing_callback_right)

# Run the event processing while the camera is connected

while capture_left.isRunning() and capture_right.isRunning():

# Receive events

events_left = capture_left.getNextEventBatch()

events_right = capture_right.getNextEventBatch()

# Check if anything was received

if events_left is not None:

# If so, pass the events into the slicer to handle them

high_pass_left.accept(events_left)

events_left = high_pass_left.generateEvents()

low_pass_left.accept(events_left)

events_left = low_pass_left.generateEvents()

cold_pass_left.accept(events_left)

events_left = cold_pass_left.generateEvents()

slicer_left.accept(events_left)

if events_right is not None:

high_pass_right.accept(events_right)

events_right = high_pass_right.generateEvents()

low_pass_right.accept(events_right)

events_right = low_pass_right.generateEvents()

cold_pass_right.accept(events_right)

events_right = cold_pass_right.generateEvents()

slicer_right.accept(events_right)

```

Now, I wanted to translate that example into C++, because it should be better for performance to work with C++ in general. I am not so good with C++, but the snippet below, has a visualization which runs a whole 5 seconds behind what is going on in the scene. Besides lacking the mask filter, the only other difference with the C++ filter is the background activity filter having a longer time scale, but I checked and this doesn't change anything. At the very bottom there is a commented section where I naively tried to speed it up with std::async, but this did nothing to help the extreme latency.

```cpp

#include <dv-processing/io/camera/discovery.hpp>

#include <dv-processing/visualization/event_visualizer.hpp>

#include <dv-processing/data/generate.hpp>

#include <dv-processing/noise/background_activity_noise_filter.hpp>

#include <dv-processing/noise/frequency_filters.hpp>

#include <future>

#include <opencv2/highgui.hpp>

int main()

{

using namespace std::chrono_literals;

using namespace std;

// Open any camera

auto capture_left = dv::io::camera::open("DXM00089");

auto capture_right = dv::io::camera::open("DXM00090");

// Make sure it supports event stream output, throw an error otherwise

if (!capture_left->isEventStreamAvailable() || !capture_right->isEventStreamAvailable())

{

throw dv::exceptions::RuntimeError("Input camera does not provide an event stream.");

}

// Initialize an accumulator with some resolution

auto resolution = *capture_left->getEventResolution();

dv::visualization::EventVisualizer visualizer_left(resolution);

dv::visualization::EventVisualizer visualizer_right(resolution);

// Apply color scheme configuration, these values can be modified to taste

visualizer_left.setBackgroundColor(dv::visualization::colors::black);

visualizer_left.setPositiveColor(dv::visualization::colors::blue);

visualizer_left.setNegativeColor(dv::visualization::colors::green);

visualizer_right.setBackgroundColor(dv::visualization::colors::black);

visualizer_right.setPositiveColor(dv::visualization::colors::blue);

visualizer_right.setNegativeColor(dv::visualization::colors::green);

// Initialize a preview window

cv::namedWindow("Left", cv::WINDOW_NORMAL);

cv::namedWindow("Right", cv::WINDOW_NORMAL);

// Initialize a slicer

dv::EventStreamSlicer slicer_left;

dv::EventStreamSlicer slicer_right;

// Register a callback every 33 milliseconds

slicer_left.doEveryTimeInterval(30ms, [&visualizer_left](const dv::EventStore &events)

{

// Generate a preview frame

cv::Mat image = visualizer_left.generateImage(events);

// Show the accumulated image

cv::imshow("Left", image);

cv::waitKey(2); });

slicer_right.doEveryTimeInterval(30ms, [&visualizer_right](const dv::EventStore &events)

{

// Generate a preview frame

cv::Mat image = visualizer_right.generateImage(events);

// Show the accumulated image

cv::imshow("Right", image);

cv::waitKey(2); });

// Initialize a background activity noise filter with 0.01-millisecond activity period

dv::noise::BackgroundActivityNoiseFilter high_pass_left(resolution, 1ms);

dv::noise::BackgroundActivityNoiseFilter high_pass_right(resolution, 1ms);

dv::noise::LowPassFilter low_pass_left(resolution, 500.0f);

dv::noise::LowPassFilter low_pass_right(resolution, 500.0f);

// Run the event processing while the camera is connected

//*

// Sequential version which I thought should just work

while (capture_left->isRunning() && capture_right->isRunning())

{

if (const auto events_left = capture_left->getNextEventBatch())

{

high_pass_left.accept(*events_left);

const dv::EventStore filtered1 = high_pass_left.generateEvents();

low_pass_left.accept(filtered1);

const dv::EventStore filtered2 = low_pass_left.generateEvents();

slicer_left.accept(filtered2);

}

//*

if (const auto events_right = capture_right->getNextEventBatch())

{

high_pass_right.accept(*events_right);

const dv::EventStore filtered1 = high_pass_right.generateEvents();

low_pass_right.accept(filtered1);

const dv::EventStore filtered2 = low_pass_right.generateEvents();

slicer_right.accept(filtered2);

}

}

//*/

/*

while (capture_left->isRunning() && capture_right->isRunning()) {

const auto events_left = capture_left->getNextEventBatch();

const auto events_right = capture_right->getNextEventBatch();

auto left_future = std::async(std::launch::async, [&]() -> std::optional<dv::EventStore> {

if (events_left) {

high_pass_left.accept(*events_left);

const dv::EventStore filtered1 = high_pass_left.generateEvents();

low_pass_left.accept(filtered1);

const dv::EventStore filtered2 = low_pass_left.generateEvents();

return filtered2;

}

return std::nullopt;

});

auto right_future = std::async(std::launch::async, [&]() -> std::optional<dv::EventStore> {

if (events_right) {

high_pass_right.accept(*events_right);

const dv::EventStore filtered1 = high_pass_right.generateEvents();

low_pass_right.accept(filtered1);

const dv::EventStore filtered2 = low_pass_right.generateEvents();

return filtered2;

}

return std::nullopt;

});

if (auto filtered = left_future.get()) {

slicer_left.accept(*filtered);

}

if (auto filtered = right_future.get()) {

slicer_right.accept(*filtered);

}

}

return 0;

*/

}

```

To be clear, I'm not changing machines or running on inside a container or anything. I can run the C++ example, verify that it is slow, kill it run, the python example, see that it is fast, kill it and immediately run the C++ version to see that it slow again. I am quite sure I did something obviously wrong.

[Luca Longinotti]

Hello! The code looks okay. How are you compiling the C++ code? By default no optimizations are enabled by the compiler, usually.

Make sure to add -O3 to the compiler flags to optimize performance (or, if you're using CMake, to switch to a RELEASE build).

-- 

Luca Longinotti (llongi)

Senior Software Engineer
iniVation AG - https://inivation.com/
A SynSense Group company

[Eben Kadile]

Hi Luca, wow, what a simple fix! 😅 Do you have any idea what the compiler was *not* optimizing before I switched to a cmake release build? I actually only have opencv experience from the python side, so I have no idea. Anyway, thanks for the tip!

[Luca Longinotti]

Generally working with events is fairly loop-intensive, you go over an event-packet and work with individual events a lot, and these kinds of loop-optimizations (unrolling, vectoring, ...) help performance a lot. Probably also some function call and math optimizations.

import dv_processingas dv

# Open cameras

capture_left = dv.io.camera.open("DXM00089")

capture_right = dv.io.camera.open("DXM00090")

# Make sure it supports event stream output, throw an error otherwise

if not (capture_left.isEventStreamAvailable()and capture_right.isEventStreamAvailable()):

mask_right[hot_pixels_right_x,hot_pixels_right_y]= 0

print(np.sum((255 - mask_left)/255))

print(np.sum((255 - mask_right)/255))

cold_pass_left = dv.EventMaskFilter(mask_left.T)

cold_pass_right = dv.EventMaskFilter(mask_right.T)

# Declare the callback method for slicer

def slicing_callback_left(events:dv.EventStore):

# Generate a preview frame

frame = visualizer_left.generateImage(events)

# Show the accumulated image

cv.imshow("Left",frame)

cv.waitKey(2)

def slicing_callback_right(events:dv.EventStore):

# Generate a preview frame

frame = visualizer_right.generateImage(events)

# Show the accumulated image

cv.imshow("Right",frame)

cv.waitKey(2)

# Register callback to be performed every 33 milliseconds

slicer_left.doEveryTimeInterval(timedelta(milliseconds=30),slicing_callback_left)

slicer_right.doEveryTimeInterval(timedelta(milliseconds=30),slicing_callback_right)

# Run the event processing while the camera is connected

while capture_left.isRunning()and capture_right.isRunning():

# Receive events

events_left = capture_left.getNextEventBatch()

events_right = capture_right.getNextEventBatch()

# Check if anything was received

if events_left is not None:

# If so, pass the events into the slicer to handle them

high_pass_left.accept(events_left)

events_left = high_pass_left.generateEvents()

low_pass_left.accept(events_left)

events_left = low_pass_left.generateEvents()

cold_pass_left.accept(events_left)

events_left = cold_pass_left.generateEvents()

slicer_left.accept(events_left)

if events_right is not None:

high_pass_right.accept(events_right)

events_right = high_pass_right.generateEvents()

low_pass_right.accept(events_right)

events_right = low_pass_right.generateEvents()

cold_pass_right.accept(events_right)

events_right = cold_pass_right.generateEvents()

slicer_right.accept(events_right)

```

Now, I wanted to translate that example into C++, because it should be better for performance to work with C++ in general. I am not so good with C++, but the snippet below, has a visualization which runs a whole5 seconds behind what is going on in the scene. Besides lacking the mask filter, the only other difference with the C++ filter is the background activity filter having a longer time scale, but I checked and this doesn't change anything. At the very bottom there is a commented section where I naively tried to speed it up with std::async, but this did nothing to help the extreme latency.

```cpp

cv::namedWindow("Right",cv::WINDOW_NORMAL);

// Initialize a slicer

dv::EventStreamSlicer slicer_left;

dv::EventStreamSlicer slicer_right;

// Register a callback every 33 milliseconds

slicer_left.doEveryTimeInterval(30ms, [&visualizer_left](const dv::EventStore &events)

{

// Generate a preview frame

cv::Mat image = visualizer_left.generateImage(events);

// Show the accumulated image

cv::imshow("Left",image);

cv::waitKey(2); });

slicer_right.doEveryTimeInterval(30ms, [&visualizer_right](const dv::EventStore &events)

{

// Generate a preview frame

cv::Mat image = visualizer_right.generateImage(events);

// Show the accumulated image

cv::imshow("Right",image);

cv::waitKey(2); });

// Initialize a background activity noise filter with 0.01-millisecond activity period

dv::noise::BackgroundActivityNoiseFilter high_pass_left(resolution,1ms);

dv::noise::BackgroundActivityNoiseFilter high_pass_right(resolution,1ms);

dv::noise::LowPassFilter low_pass_left(resolution,500.0f);

dv::noise::LowPassFilter low_pass_right(resolution,500.0f);

// Run the event processing while the camera is connected

//*

// Sequential version which I thought should just work

while (capture_left->isRunning()&& capture_right->isRunning())