Converting Google's MediaPipe to Web Assembly using Emscripten from within Bazel
74 views
Skip to first unread message
Pinku Deb Nath
Jul 8, 2020, 6:02:01 AM7/8/20
to emscripten-discuss
Has anyone tried to convert MediaPipe project to Web Assembly using Bazel? I am stuck with two errors based on what dependency I include in the BUILD file. The following is one of the errors, which I get if I just omit MediaPipe/graph/calculators from the dependency:
Execution platform: @local_execution_config_platform//:platformmediapipe/framework/deps/threadpool_pthread_impl.cc:63:64: error: use of undeclared identifier 'SYS_gettid' internal::CreateThreadName(thread->name_prefix_, syscall(SYS_gettid));Here is the error I get if I include all the dependencies:
/home/prantoran/.cache/bazel/_bazel_prantoran/53d0985b61cdf3fef140f3aded9a4395/external/cpuinfo/BUILD.bazel:96:11: Configurable attribute "srcs" doesn't match this configuration (would a default condition help?).Conditions checked: @cpuinfo//:linux_x86_64 @cpuinfo//:linux_arm @cpuinfo//:linux_armhf @cpuinfo//:linux_aarch64 @cpuinfo//:macos_x86_64 @cpuinfo//:windows_x86_64 @cpuinfo//:android_armv7 @cpuinfo//:android_arm64 @cpuinfo//:android_x86 @cpuinfo//:android_x86_64 @cpuinfo//:ios_x86_64 @cpuinfo//:ios_x86 @cpuinfo//:ios_armv7 @cpuinfo//:ios_arm64 @cpuinfo//:ios_arm64e @cpuinfo//:watchos_x86_64 @cpuinfo//:watchos_x86 @cpuinfo//:watchos_armv7k @cpuinfo//:watchos_arm64_32 @cpuinfo//:tvos_x86_64 @cpuinfo//:tvos_arm64ERROR: Analysis of target '//javascript:khaliwasm' failed; build aborted: Analysis failed
I tried to it by getting the .bazelrc and WORKSPACE from this blog: hackernoon.com/c-to-webassembly-using-bazel-and-emscripten-4him3ymc. I can produce Web Assembly code for simple cc code with no dependencies.
However, I am stuck with 2 types of errors based on what MediaPipe dependency I include in the BUILD file.
Here is my .bazelrc:
# The bazelrc file for MediaPipe OSS.
# Tensorflow needs remote repo
common --experimental_repo_remote_exec
# Basic build settings
build --jobs 128
build --define='absl=1'
build --enable_platform_specific_config
# Linux
build:linux --cxxopt=-std=c++14
build:linux --host_cxxopt=-std=c++14
build:linux --copt=-w
# emscripten
##### WASM #####
# Use our custom-configured c++ toolchain.
build:wasm --crosstool_top=//javascript/toolchain:emscripten
# Use --cpu as a differentiator.
build:wasm --cpu=wasm
# Use the default C++ toolchain to build the tools used during the build.
build:wasm --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
# These compile flags are active no matter which build mode we are in
# (dbg vs opt). For flags specific to build mode, see cc_toolchain_config.bzl.
#build:wasm --cxxopt="-flto" --copt=-flto
#build:wasm --host_cxxopt="-fno-rtti" # disable generation of info of every class for runtime access
build:wasm --host_cxxopt="-fno-exceptions"
#build:wasm --host_cxxopt="-fomit-frame-pointer"
build:wasm --cxxopt=-std=c++14
build:wasm --host_cxxopt=-std=c++14
#build:wasm --copt=-w
# Disable sandbox environment because emsdk caches files by writing to
# home directory.
build:wasm --spawn_strategy=localI turned off the optimization flags --cxxopt="-flto", --host_cxxopt="-fomit-frame-pointer", --host_cxxopt="-fno-rtti" on purpose. In fact, --host_cxxopt="-fno-rtti" prevented dynamic type inferencing.
Here is the WORKSPACE:
workspace(name = "practice")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
skylib_version = "0.9.0"
http_archive(
name = "bazel_skylib",
type = "tar.gz",
url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version),
sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
)
load("@bazel_skylib//lib:versions.bzl", "versions")
versions.check(minimum_bazel_version = "2.0.0")
# ABSL cpp library lts_2020_02_25
http_archive(
name = "com_google_absl",
urls = [
"https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz",
],
# Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
patches = [
"@//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff"
],
patch_args = [
"-p1",
],
strip_prefix = "abseil-cpp-20200225",
sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353"
)
http_archive(
name = "rules_cc",
strip_prefix = "rules_cc-master",
urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
)
# GoogleTest/GoogleMock framework. Used by most unit-tests.
http_archive(
name = "com_google_googletest",
urls = ["https://github.com/google/googletest/archive/master.zip"],
strip_prefix = "googletest-master",
)
# Google Benchmark library.
http_archive(
name = "com_google_benchmark",
urls = ["https://github.com/google/benchmark/archive/master.zip"],
strip_prefix = "benchmark-master",
build_file = "@//third_party:benchmark.BUILD",
)
# gflags needed by glog
http_archive(
name = "com_github_gflags_gflags",
strip_prefix = "gflags-2.2.2",
sha256 = "19713a36c9f32b33df59d1c79b4958434cb005b5b47dc5400a7a4b078111d9b5",
url = "https://github.com/gflags/gflags/archive/v2.2.2.zip",
)
# glog v0.3.5
# TODO: Migrate MediaPipe to use com_github_glog_glog on all platforms.
http_archive(
name = "com_github_glog_glog_v_0_3_5",
url = "https://github.com/google/glog/archive/v0.3.5.zip",
sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
strip_prefix = "glog-0.3.5",
build_file = "@//third_party:glog.BUILD",
patches = [
"@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
],
patch_args = [
"-p1",
],
)
# 2020-02-16
http_archive(
name = "com_github_glog_glog",
strip_prefix = "glog-3ba8976592274bc1f907c402ce22558011d6fc5e",
sha256 = "feca3c7e29a693cab7887409756d89d342d4a992d54d7c5599bebeae8f7b50be",
urls = [
"https://github.com/google/glog/archive/3ba8976592274bc1f907c402ce22558011d6fc5e.zip",
],
)
# easyexif
http_archive(
name = "easyexif",
url = "https://github.com/mayanklahiri/easyexif/archive/master.zip",
strip_prefix = "easyexif-master",
build_file = "@//third_party:easyexif.BUILD",
)
# libyuv
http_archive(
name = "libyuv",
urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"],
build_file = "@//third_party:libyuv.BUILD",
)
# Note: protobuf-javalite is no longer released as a separate download, it's included in the main Java download.
# ...but the Java download is currently broken, so we use the "source" download.
http_archive(
name = "com_google_protobuf_javalite",
sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
strip_prefix = "protobuf-3.11.4",
urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
)
http_archive(
name = "com_google_protobuf",
sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
strip_prefix = "protobuf-3.11.4",
urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
patches = [
"@//third_party:com_google_protobuf_fixes.diff"
],
patch_args = [
"-p1",
],
)
http_archive(
name = "com_google_audio_tools",
strip_prefix = "multichannel-audio-tools-master",
urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
)
http_archive(
name = "ceres_solver",
url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip",
patches = [
"@//third_party:ceres_solver_compatibility_fixes.diff"
],
patch_args = [
"-p1",
],
strip_prefix = "ceres-solver-1.14.0",
sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
)
new_local_repository(
name = "linux_opencv",
build_file = "@//third_party:opencv_linux.BUILD",
path = "/usr/local",
)
new_local_repository(
name = "linux_ffmpeg",
build_file = "@//third_party:ffmpeg_linux.BUILD",
path = "/usr"
)
# Maven dependencies.
RULES_JVM_EXTERNAL_TAG = "3.2"
RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af"
http_archive(
name = "rules_jvm_external",
strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
sha256 = RULES_JVM_EXTERNAL_SHA,
url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
)
load("@rules_jvm_external//:defs.bzl", "maven_install")
# Needed by TensorFlow
http_archive(
name = "io_bazel_rules_closure",
sha256 = "e0a111000aeed2051f29fcc7a3f83be3ad8c6c93c186e64beb1ad313f0c7f9f9",
strip_prefix = "rules_closure-cf1e44edb908e9616030cc83d085989b8e6cd6df",
urls = [
"http://mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",
"https://github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz", # 2019-04-04
],
)
#Tensorflow repo should always go after the other external dependencies.
# 2020-05-11
_TENSORFLOW_GIT_COMMIT = "7c09d15f9fcc14343343c247ebf5b8e0afe3e4aa"
_TENSORFLOW_SHA256= "673d00cbd2676ae43df1993e0d28c10b5ffbe96d9e2ab29f88a77b43c0211299"
http_archive(
name = "org_tensorflow",
urls = [
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
],
patches = [
"@//third_party:org_tensorflow_compatibility_fixes.diff",
],
patch_args = [
"-p1",
],
strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
sha256 = _TENSORFLOW_SHA256,
)
load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
tf_workspace(tf_repo_name = "org_tensorflow")
load("//javascript/toolchain:cc_toolchain_config.bzl", "emsdk_configure")
emsdk_configure(name = "emsdk")In particular, I added the lines in the end:
load("//javascript/toolchain:cc_toolchain_config.bzl", "emsdk_configure")
emsdk_configure(name = "emsdk")
The contents of //javascript/toolchain:cc_toolchain_config.bzl:
load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES")
load(
"@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl",
"feature",
"flag_group",
"flag_set",
"tool_path",
"with_feature_set",
)
def _impl(ctx):
tool_paths = [
tool_path(
name = "gcc",
path = "emcc.sh",
),
tool_path(
name = "ld",
path = "emcc.sh",
),
tool_path(
name = "ar",
path = "emar.sh",
),
tool_path(
name = "cpp",
path = "false.sh",
),
tool_path(
name = "gcov",
path = "false.sh",
),
tool_path(
name = "nm",
path = "NOT_USED",
),
tool_path(
name = "objdump",
path = "false.sh",
),
tool_path(
name = "strip",
path = "NOT_USED",
),
]
preprocessor_compile_actions = [
ACTION_NAMES.c_compile,
ACTION_NAMES.cpp_compile,
ACTION_NAMES.linkstamp_compile,
ACTION_NAMES.preprocess_assemble,
ACTION_NAMES.cpp_header_parsing,
ACTION_NAMES.cpp_module_compile,
ACTION_NAMES.clif_match,
]
all_link_actions = [
ACTION_NAMES.cpp_link_executable,
ACTION_NAMES.cpp_link_dynamic_library,
ACTION_NAMES.cpp_link_nodeps_dynamic_library,
]
all_compile_actions = [
ACTION_NAMES.c_compile,
ACTION_NAMES.cpp_compile,
ACTION_NAMES.linkstamp_compile,
ACTION_NAMES.assemble,
ACTION_NAMES.preprocess_assemble,
ACTION_NAMES.cpp_header_parsing,
ACTION_NAMES.cpp_module_compile,
ACTION_NAMES.cpp_module_codegen,
ACTION_NAMES.clif_match,
ACTION_NAMES.lto_backend,
]
toolchain_include_directories_feature = feature(
name = "toolchain_include_directories",
enabled = True,
flag_sets = [
flag_set(
actions = all_compile_actions,
flag_groups = [
flag_group(
flags = [
"-isystem",
"external/emsdk/emsdk/upstream/emscripten/system/include/libcxx",
"-isystem",
"external/emsdk/emsdk/upstream/emscripten/system/lib/libcxxabi/include",
"-isystem",
"external/emsdk/emsdk/upstream/emscripten/system/include/compat",
"-isystem",
"external/emsdk/emsdk/upstream/emscripten/system/include",
"-isystem",
"external/emsdk/emsdk/upstream/emscripten/system/include/libc",
"-isystem",
"external/emsdk/emsdk/upstream/emscripten/system/lib/libc/musl/arch/emscripten",
"-isystem",
"external/emsdk/emsdk/upstream/emscripten/system/local/include",
],
),
],
),
],
)
crosstool_default_flag_sets = [
# Optimized (opt)
flag_set(
actions = preprocessor_compile_actions,
flag_groups = [flag_group(flags = ["-DNDEBUG"])],
with_features = [with_feature_set(features = ["opt"])],
),
# In advanced C++ apps, -O3 can break asmjs.
flag_set(
actions = all_compile_actions + all_link_actions,
flag_groups = [flag_group(flags = ["-g0", "-O3"])],
with_features = [with_feature_set(features = ["opt"])],
),
# Fastbuild (fastbuild)
flag_set(
actions = all_compile_actions + all_link_actions,
flag_groups = [flag_group(flags = ["-O2"])],
with_features = [with_feature_set(features = ["fastbuild"])],
),
# Debug (dbg)
flag_set(
actions = all_compile_actions + all_link_actions,
flag_groups = [flag_group(flags = ["-g2", "-O0"])],
with_features = [with_feature_set(features = ["dbg"])],
),
]
features = [
toolchain_include_directories_feature,
# These 3 features will be automatically enabled by blaze in the
# corresponding build mode.
feature(
name = "opt",
provides = ["variant:crosstool_build_mode"],
),
feature(
name = "dbg",
provides = ["variant:crosstool_build_mode"],
),
feature(
name = "fastbuild",
provides = ["variant:crosstool_build_mode"],
),
feature(
name = "crosstool_default_flags",
enabled = True,
flag_sets = crosstool_default_flag_sets,
),
]
return cc_common.create_cc_toolchain_config_info(
ctx = ctx,
toolchain_identifier = "wasm-toolchain",
host_system_name = "i686-unknown-linux-gnu",
target_system_name = "wasm-unknown-emscripten",
target_cpu = "wasm",
target_libc = "unknown",
compiler = "emscripten",
abi_version = "unknown",
abi_libc_version = "unknown",
tool_paths = tool_paths,
features = features,
)
cc_toolchain_config = rule(
implementation = _impl,
attrs = {},
provides = [CcToolchainConfigInfo],
)
def _emsdk_impl(ctx):
if "EMSDK" not in ctx.os.environ or ctx.os.environ["EMSDK"].strip() == "":
fail("The environment variable EMSDK is not found. " +
"Did you run source ./emsdk_env.sh ?")
path = ctx.os.environ["EMSDK"]
ctx.symlink(path, "emsdk")
ctx.file("BUILD", """
filegroup(
name = "all",
srcs = glob(["emsdk/**"]),
visibility = ["//visibility:public"],
)
""")
emsdk_configure = repository_rule(
implementation = _emsdk_impl,
local = True,
)
And the dummy cc file:
#include <emscripten/bind.h>
#include <iostream>
#include <sys/syscall.h>
int main() {
std::cout << "empty inside\n";
}
And the BUILD file used for bazel build:
package(default_visibility = ["//visibility:public"])
DEFAULT_EMSCRIPTEN_LINKOPTS = [
# "-flto", # Specify lto (has to be set on for compiler as well)
"-O0",
"--bind", # Compiles the source code using the Embind bindings to connect C/C++ and JavaScript
"--closure 1", # Run the closure compiler
"-s MALLOC=emmalloc", # Switch to using the much smaller implementation
"-s ALLOW_MEMORY_GROWTH=1", # Our example doesn't need memory growth
"-s USE_PTHREADS=1", # Disable pthreads
"-s ASSERTIONS=0", # Turn off assertions
"-s EXPORT_ES6=1", # Export as es6 module, used for rollup
"-s MODULARIZE=1", # Allows us to manually invoke the initializatio of wasm
"-s EXPORT_NAME=createModule", # Not used, but good to specify
"-s USE_ES6_IMPORT_META=0", # Disable loading from import meta since we use rollup
"-s SINGLE_FILE=1" # Pack all webassembly into base64
]
WASM_LINKOPTS = [
"-s WASM=1", # Specify wasm output
]
linux_or_darwin_copts = [
# Disable warnings that exists in glog.
"-Wno-sign-compare",
"-Wno-unused-function",
"-Wno-unused-local-typedefs",
"-Wno-unused-variable",
# # Allows src/base/mutex.h to include pthread.h.
"-DHAVE_PTHREAD",
# # Allows src/logging.cc to determine the host name.
"-DHAVE_SYS_UTSNAME_H",
# For src/utilities.cc.
"-DHAVE_SYS_SYSCALL_H",
"-DHAVE_SYS_TIME_H",
# # Enable dumping stacktrace upon sigaction.
# "-DHAVE_SIGACTION",
# # For logging.cc.
"-DHAVE_PREAD",
"-DHAVE___ATTRIBUTE__",
]
cc_binary(
name = "khaliwasm",
srcs = ["bindings/khali.cpp"],
linkopts = DEFAULT_EMSCRIPTEN_LINKOPTS + WASM_LINKOPTS + linux_or_darwin_copts,
deps = [
"//desktop:main_gpu",
# "//mediapipe/graphs/hand_tracking:multi_hand_mobile_calculators",
],
)
The desktop:main_gpu contains the BUILD contents of the Two Hands GPU example.
The command I used for bazel build:
bazel build -c opt //javascript:khaliwasm --config=wasm --sandbox_debug --verbose_failuresReply all
Reply to author
Forward
0 new messages