cmake project with nanopb in-tree, not using system-wide install or manual venv

[Andrew Kohlsmith]

I'm trying to use nanopb with a raspberry pi pico (RP2040) project. I've used nanopb previously and know how to use protoc to generate the .c and .h for my protobufs, but it's always bugged me how the project now has untracked system dependencies. 

One of cmake's benefits is an abstracted build system and nanopb certainly builds well with cmake, but I cannot figure out how to add nanopb as a git submodule and have my cmake project build nanopb and then use that build to generate the .c and .h files. It seems like no matter what, it builds nanopb but then appears to try using the system protoc to generate my source/headers instead of the freshly-build-in-project version of nanopb.

First I tried to do it simply, in the toplevel CMakeLists.txt:

list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/nanopb/extra)

find_package(Nanopb MODULE REQUIRED)

set(PROTO_FILES
    "${CMAKE_SOURCE_DIR}/myprotos/foo.proto"
    # Add more proto files here
)

nanopb_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES})

This builds nanopb just fine, but then when it tries to generate the header/source files it bombs out with 

[ 12%] Generating nanopb/generator/nanopb_generator.py, nanopb/generator/proto/nanopb.proto
[ 13%] Generating nanopb/generator/proto/nanopb_pb2.py
Traceback (most recent call last):
  File "/Users/andrew/testproj/nanopb/generator/protoc", line 44, in <module>
    status = invoke_protoc(['protoc'] + args)
  File "/Users/andrew/testproj/nanopb/generator/proto/_utils.py", line 76, in invoke_protoc
    return subprocess.call(argv)
  File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/subprocess.py", line 349, in call
    with Popen(*popenargs, **kwargs) as p:
  File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/subprocess.py", line 951, in __init__
    self._execute_child(args, executable, preexec_fn, close_fds,
  File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/subprocess.py", line 1821, in _execute_child
    raise child_exception_type(errno_num, err_msg, err_filename)
FileNotFoundError: [Errno 2] No such file or directory: 'protoc'
make[2]: *** [nanopb/generator/proto/nanopb_pb2.py] Error 1
make[1]: *** [CMakeFiles/testproj.dir/all] Error 2
make: *** [all] Error 2

Now I've tried various incarnations of this, and after some googling even tried having cmake create a venv in the project directory and install nanopb in there, but it bombs out the same when it tries to generate the .c and .h protobuf files.

I see issues #911, #481 and others, but they seem to be stale. 

Does anyone have a working example of how to get nanopb to build within a project without a) having nanopb installed in the host system or b) requiring manual installation of a venv?

Thanks,

Andrew

[Petteri Aimonen]

Hi,

System-wide installation of nanopb is not necessary, but the generator
uses the Google's python-protobuf library and protoc.
These are often available from e.g. system package manager
already, so forcibly including another copy as nanopb
submodule wouldn't make sense.

Currently nanopb's build rules do not attempt to install
python-protobuf. Easiest way to do that is to use Python
venv and install grpcio-tools in it.

It could be nice if the cmake rules could optionally
do this automatically. The cmake files are community
maintained as I don't have enough time to handle all
of the build systems.

--
Petteri

[Andrew Kohlsmith]

Thank you for your response, it's helping me understand a bit more.

I installed the protobuf homebrew package, but that changes the error to 

[ 12%] Generating nanopb/generator/nanopb_generator.py, nanopb/generator/proto/nanopb.proto
[ 13%] Generating nanopb/generator/proto/nanopb_pb2.py

[ 13%] Running C++ protocol buffer compiler using nanopb plugin on /Users/andrew/testproj/meshtastic-pb/nanopb.proto

         **********************************************************************
         *** Could not import the Google protobuf Python libraries          ***
         ***                                                                ***
         *** Easiest solution is often to install the dependencies via pip: ***
         ***    pip install protobuf grpcio-tools                           ***
         **********************************************************************

Traceback (most recent call last):

  File "/Users/andrew/testproj/build/nanopb/generator/protoc-gen-nanopb", line 7, in <module>
    from nanopb_generator import *
  File "/Users/andrew/testproj/build/nanopb/generator/nanopb_generator.py", line 34, in <module>
    import google.protobuf.text_format as text_format
ModuleNotFoundError: No module named 'google'
--nanopb_out: protoc-gen-nanopb: Plugin failed with status code 1.
make[2]: *** [nanopb.pb.c] Error 1
make[1]: *** [CMakeFiles/picomgmt.dir/all] Error 2

make: *** [all] Error 2

So ok, let's create a venv and use that -- I'm trying to not have to require specific out-of-project (ie system) venv. This is where I am very much out of my depth with both python and cmake:

# initialize nanopb

list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/nanopb/extra)
find_package(Nanopb MODULE REQUIRED)

# use the system python3 to create a venv
find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
set(SYSTEM_PYTHON_EXE_PATH ${Python3_EXECUTABLE})
execute_process(
COMMAND "${Python3_EXECUTABLE}" -m venv "${CMAKE_SOURCE_DIR}/venv" --upgrade-deps
COMMAND_ERROR_IS_FATAL ANY
)

# Update the environment with VIRTUAL_ENV variable (mimic the activate script)
set(VENV_PATH "${CMAKE_SOURCE_DIR}/venv")
set(ENV{VIRTUAL_ENV} "${VENV_PATH}")

# now look for the venv
set(Python3_FIND_VIRTUALENV ONLY)
unset(Python3_EXECUTABLE)
find_package(Python3 COMPONENTS Interpreter Development REQUIRED)

# I'm sure this isn't right, nanopb should be doing this
execute_process(
COMMAND "${Python3_EXECUTABLE}" -m pip install -r "${CMAKE_SOURCE_DIR}/nanopb/requirements.txt"
COMMAND_ERROR_IS_FATAL ANY
)

# install grpcio-tools (and google, as the error I'm getting says it can't find that package)
execute_process(
COMMAND "${Python3_EXECUTABLE}" -m pip install grpcio-tools google
COMMAND_ERROR_IS_FATAL ANY
)

set(PROTO_FILES
    "${CMAKE_SOURCE_DIR}/myproto/myproto.proto"

    # Add more proto files here
)

nanopb_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES})

message("NANOPB_FOUND=" ${NANOPB_FOUND})
message("PROTOBUF_PROTOC_EXECUTABLE" ${PROTOBUF_PROTOC_EXECUTABLE})
message("NANOPB_GENERATOR_SOURCE_DIR" ${NANOPB_GENERATOR_SOURCE_DIR})

This does successfully create the venv and appears to install the grpcio-tools and google packages into it, but the exact same error persists; it's like the venv isn't being used by nanopb_generate_cpp().

Thanks,

Andrew

[Andrew Kohlsmith]

Well, I sort of made some progress.

The issue is that nanopb is calling protoc as you said, but it does not update the path that protoc uses, so protoc tries to use the system python interpreter/environment, which does not have the python packages.

I worked around this by setting CUSTOM_COMMAND_PREFIX before calling nanopb_generate_cpp(). I saw this variable used for working around the exact same thing with MSVC based builds and fortunately it appears to work for other builds as well:

set(CUSTOM_COMMAND_PREFIX PATH=${NANOPB_VENV_DIR}/bin;${PATH})

This "correctly" generates the myproto.pb.c and myprot.pb.h files. I have "correctly" in quotes because they fail to build:

[ 12%] Building C object CMakeFiles/proto.dir/myproto.pb.c.o
/opt/homebrew/bin/arm-none-eabi-gcc  -I/Users/andrew/testproj/build -I/Users/andrew/testproj/nanopb -mcpu=cortex-m0plus -mthumb -g -O3 -DNDEBUG -std=gnu11 -MD -MT CMakeFiles/proto.dir/myproto.pb.c.o -MF CMakeFiles/proto.dir/myproto.pb.c.o.d -o CMakeFiles/proto.dir/myproto.pb.c.o -c /Users/andrew/testproj/build/myproto.pb.c
In file included from /Users/andrew/testproj/build/myproto.pb.c:4:
/Users/andrew/testproj/build/myproto.pb.h:141:5: error: unknown type name 'google_protobuf_FieldDescriptorProto_Type'
  141 |     google_protobuf_FieldDescriptorProto_Type type_override;
      |     ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
make[2]: *** [CMakeFiles/proto.dir/myproto.pb.c.o] Error 1

Now "google_protobuf_FieldDescriptorProto_Type" is from google's protobuf implementation, not nanopb's. I cannot figure out why that is being generated, because I believe I'm using the nanopb cmake correctly:

list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/nanopb/extra)

set(PROTO_FILES
"${CMAKE_SOURCE_DIR}/myproto/myproto.proto"
# Add more proto files here
)

find_package(Nanopb REQUIRED)
set(CUSTOM_COMMAND_PREFIX PATH=${NANOPB_VENV_DIR}/bin;${PATH})
nanopb_generate_cpp(TARGET proto ${PROTO_FILES})

target_link_libraries(${PROJECT_NAME} PUBLIC ... proto)

This is becoming very frustrating. It should not be this hard to use nanopb in a cmake based project. I have also tried the variant of nanopb_generate_cpp() which creates a list of source/header files but the result is the same -- the generator seems to be using the google reference library, not nanopb.

I have verified that nanopb_generate_cpp() seems to be doing the right thing by passing along VERBOSE=1 to the make command:

[ 12%] Running C++ protocol buffer compiler using nanopb plugin on /Users/andrew/testproj/myproto/myproto.proto
PATH=/Users/andrew/testproj/build/venv/bin /Users/andrew/testproj/nanopb/generator/protoc -I/Users/andrew/testproj/myproto -I/Users/andrew/testproj/build/nanopb/generator -I/Users/andrew/testproj/build/nanopb/generator/proto -I/Users/andrew/testproj/build --plugin=protoc-gen-nanopb=/Users/andrew/testproj/build/nanopb/generator/protoc-gen-nanopb --nanopb_opt= --nanopb_out=/Users/andrew/testproj/build /Users/andrew/testproj/myproto/myproto.proto
cd /Users/andrew/testproj/build && /opt/homebrew/bin/cmake -E cmake_depends "Unix Makefiles" /Users/andrew/testproj /Users/andrew/testproj /Users/andrew/testproj/build /Users/andrew/testproj/build /Users/andrew/testproj/build/CMakeFiles/proto.dir/DependInfo.cmake "--color="
/Library/Developer/CommandLineTools/usr/bin/make  -f CMakeFiles/proto.dir/build.make CMakeFiles/proto.dir/build

Also, the nanopb library (composed of pb_decode.c, pb_encode.c and pb_common.c) is being built without error.

-A.

[Andrew Kohlsmith]

(minor fix, the correct way to set the environment path for a command run with add_custom_command() is apparently not how I had it. My CUSTOM_COMMAND_PREFIX now looks like this:

set(CUSTOM_COMMAND_PREFIX ${CMAKE_COMMAND} -E env PATH=${NANOPB_VENV_DIR}/bin:$ENV{PATH})

which is correctly prepending the venv directory to the path that add_custom_command() was using by default. Still no change with the error I'm getting, though.

-A.

[Petteri Aimonen]

Hi,

One approach could be changing the CMake rules to instead of
calling nanopb as a protoc plugin, running nanopb_generator.py
directly. It will then call protoc by itself.

Nanopb generator can work in both modes since 0.4 IIRC, but the CMake
rules probably date further back. Giving .proto files directly to
nanopb_generator.py was added exactly to avoid this kind of path problems.

--
Petteri

> --
> You received this message because you are subscribed to the Google Groups "nanopb" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to nanopb+un...@googlegroups.com.
> To view this discussion visit https://groups.google.com/d/msgid/nanopb/3821299c-98a2-4d0f-abdb-878c792dbe33n%40googlegroups.com.

> --
> You received this message because you are subscribed to the Google
> Groups "nanopb" group.
> To unsubscribe from this group and stop receiving emails from it, send
> an email to [1]nanopb+un...@googlegroups.com.
> To view this discussion visit
> [2]https://groups.google.com/d/msgid/nanopb/3821299c-98a2-4d0f-abdb-878
> c792dbe33n%40googlegroups.com.
>
> References
>
> 1. mailto:nanopb+un...@googlegroups.com
> 2. https://groups.google.com/d/msgid/nanopb/3821299c-98a2-4d0f...@googlegroups.com?utm_medium=email&utm_source=footer

[Andrew Kohlsmith]

On Thursday, January 8, 2026 at 10:53:14 PM UTC-8 Petteri Aimonen wrote:

One approach could be changing the CMake rules to instead of
calling nanopb as a protoc plugin, running nanopb_generator.py
directly. It will then call protoc by itself.

Nanopb generator can work in both modes since 0.4 IIRC, but the CMake
rules probably date further back. Giving .proto files directly to
nanopb_generator.py was added exactly to avoid this kind of path problems.

 

I have absolutely no idea how to do that and am not super familiar with cmake but out of sheer stubbornness I think I succeeded.

My cmake project's directory structure is as follows:

/CMakeLists.txt

/src

/nanopb

/meshtastic-pb

src has my source code, nanopb is a submodule for https://github.com/nanopb/nanopb.git, and meshtastic-pb is another submodule for https://github.com/meshtastic/protobufs.git. 

My CMakeLists.txt has the following, which creates a venv, installs the required packages and then uses it. It also overrides the PATH environment variable passed to the host's protoc so that uses the correct Python environment for the nanopb plugin. That was probably the trickiest part. To my knowledge this setup has no host dependencies other than protoc, which I'm comfortable with, although I'd like to eliminate that as well at some point.  I am sure this is inelegant and someone much better with CMake than myself could probably do a much better job, but this is sufficient for my objectives which was to have no manual build steps and work entirely within CMake. The specific minimum Python version (3.6) is unimportant, it's just what I ended up using and didn't want to mess with it.

1. Create a venv, "activate" it and install the required packages:

# Create venv
# Only create venv if it doesn't exist
find_package(Python3 3.6 REQUIRED COMPONENTS Interpreter)
set(NANOPB_VENV_DIR "${CMAKE_BINARY_DIR}/venv")

if(NOT EXISTS "${NANOPB_VENV_DIR}")
message(STATUS "Creating Python virtual environment at ${NANOPB_VENV_DIR}")

execute_process(
COMMAND ${Python3_EXECUTABLE} -m venv ${NANOPB_VENV_DIR}
RESULT_VARIABLE VENV_RESULT
OUTPUT_VARIABLE VENV_OUTPUT
ERROR_VARIABLE VENV_ERROR
)

if(VENV_RESULT EQUAL 0)
set(ENV{VIRTUAL_ENV} "${NANOPB_VENV_DIR}") # mimic activation
set(Python3_FIND_VIRTUALENV ONLY) # require venv Python
unset(Python3_EXECUTABLE) # force re‑search

# find python (now it'll be in the venv)
find_package(Python3 3.6 REQUIRED COMPONENTS Interpreter)
set(_pip "${Python3_EXECUTABLE}")
message(STATUS "Using venv Python at ${Python3_EXECUTABLE}")

# install/update python modules
execute_process(
COMMAND "${_pip}" -m pip install --upgrade pip setuptools wheel protobuf grpcio-tools nanopb
RESULT_VARIABLE _pip_ret
)

if(_pip_ret)
message(FATAL_ERROR "pip install failed in venv (code ${_pip_ret})")
endif()
else()
message(FATAL_ERROR "Failed to create venv:\n${VENV_ERROR}")
endif()
endif()

2. Use the venv, set up the list of .proto files you want (bonus points because it's got subdirectories so thank you for RELPATH):

# try to find the venv
set(ENV{VIRTUAL_ENV} "${NANOPB_VENV_DIR}") # mimic activation
set(Python3_FIND_VIRTUALENV ONLY) # require venv Python
unset(Python3_EXECUTABLE) # force re‑search

# find python (it should be in the venv)
find_package(Python3 3.6 REQUIRED COMPONENTS Interpreter)

# choose the protobuf .proto files we need
set(NANOPB_IMPORT_DIRS ${CMAKE_SOURCE_DIR}/meshtastic-pb)
set(PROTO_FILES
"meshtastic-pb/meshtastic/mesh.proto"
"meshtastic-pb/meshtastic/channel.proto"
"meshtastic-pb/meshtastic/config.proto"
"meshtastic-pb/meshtastic/device_ui.proto"
"meshtastic-pb/meshtastic/module_config.proto"
"meshtastic-pb/meshtastic/portnums.proto"
"meshtastic-pb/meshtastic/telemetry.proto"
"meshtastic-pb/meshtastic/xmodem.proto"

# Add more proto files here
)

# now call nanopb_generate_cpp() to generate the .pb.c and .pb.h files
# this will call the host's protoc, and unfortunately it won't use the correct python path (the one in the venv)
# so we have to mess around a little. The FindNanopb.cmake has a way to prefix the protoc command (intended to be
# used for MSVC) but we can (ab)use it to append the venv's bin/ to the path so the host's protoc will use the
# python environment we created above

list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/nanopb/extra)

find_package(Nanopb REQUIRED)

set(CUSTOM_COMMAND_PREFIX ${CMAKE_COMMAND} -E env PATH=${NANOPB_VENV_DIR}/bin:$ENV{PATH})

nanopb_generate_cpp(TARGET proto RELPATH meshtastic-pb ${PROTO_FILES})

3. Add the nanopb/generated protobuf library (in this case it's called "proto") to the target_link_libraries() list:

target_link_libraries(${PROJECT_NAME} PUBLIC ... proto)

Using this, I am now able to successfully build and use the meshtastic protobufs, generated with nanopb.

-A.