Cross compile IREE for aarch64 linux

[cycheng....@gmail.com]

Hey,

I am trying to cross compile IREE for aarch64 linux, but I still can't make it : (

My build step:

1. Setup clang cross build environment:

wget https://developer.arm.com/-/media/Files/downloads/gnu-a/9.2-2019.12/binrel/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu.tar.xz

tar -xf gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu.tar.xz

2. Test clang (version 9.0.0-2~ubuntu18.04.2) cross build

clang++ --target=aarch64-linux-gnu \

  --gcc-toolchain=gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu \

  --sysroot=gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu/aarch64-none-linux-gnu/libc \

  test.cpp

=> "Hello world" (on aarch64 device)

test.cpp

#include <iostream>

int main(int argc, char* argv[]) { std::cout << "hello world\n"; return 0; }

3. Prepare clang-aarch64-toolchain.cmake

SET(CMAKE_SYSTEM_NAME Linux)

SET(CMAKE_SYSTEM_VERSION 1)

SET(CMAKE_C_COMPILER   clang)

SET(CMAKE_CXX_COMPILER clang++)

SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)

SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)

SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

set(CLANG_COMPILER_FLAGS)

list(APPEND --target=aarch64-linux-gnu

       --gcc-toolchain=/home/saber/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu

       --sysroot=/home/saber/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu/aarch64-none-linux-gnu/libc)

set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CLANG_COMPILER_FLAGS}")                  

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CLANG_COMPILER_FLAGS}")

#add_compile_options(--target=aarch64-linux-gnu)

#add_compile_options(-fuse-ld=gold)

#add_compile_options(-mcpu=cortex-a72)

#add_compile_options(--gcc-toolchain=/home/saber/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu)

#add_compile_options(--sysroot=/home/saber/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu/aarch64-none-linux-gnu/libc)

4. Configure IREE

cmake -G Ninja ../iree \
-DCMAKE_TOOLCHAIN_FILE=/clang-aarch64-toolchain.cmake \
-DIREE_BUILD_COMPILER=OFF -DIREE_BUILD_SAMPLES=OFF \
-DIREE_HOST_C_COMPILER=/home/saber/img-mlir-build/x86/bin/clang \

-DIREE_HOST_CXX_COMPILER=/home/saber/img-mlir-build/x86/bin/clang++

But I checked the build.ninja, it stil built for x86, not for aarch64, I also tried some different things, but I got cmake configuration error.

Please guide me how to cross compile.

By the way, can I also turn on '-DIREE_BUILD_COMPILER' in cross compilation?

Many thanks for help =)

[Lei Zhang]

Hey Chuang-Yu Cheng,

Thanks for the interest in IREE! Keep in mind that the cross compilation functionality is recently added and we haven't tested it yet outside Android; so don't be too surprised when you see issues. :) 

With that said, I think in the above CMake toolchain file you are actually not setting CMAKE_C_FLAGS and CMAKE_CXX_FLAGS as expected? I'd expect the toolchain file to look something like this:

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR aarch64)

set(CMAKE_C_COMPILER   clang)
set(CMAKE_CXX_COMPILER clang++)

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

set(EXTRA_CLNAG_ARGS
    --target=aarch64-linux-gnu
    --gcc-toolchain=/path/to/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu
    --sysroot=/path/to/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu/aarch64-none-linux-gnu/libc
)
list(JOIN EXTRA_CLNAG_ARGS " " EXTRA_CLNAG_ARG_STRING)
set(CMAKE_C_FLAGS ${EXTRA_CLNAG_ARG_STRING})
set(CMAKE_CXX_FLAGS ${EXTRA_CLNAG_ARG_STRINGS})

Thanks,

Lei

If you also want the compilers to run on aarch64 then you will need to turn that on. But we haven't spent time to look into making it work yet; so probably you'll see some compilation issues.

Many thanks for help =)

--
You received this message because you are subscribed to the Google Groups "iree-discuss" group.
To unsubscribe from this group and stop receiving emails from it, send an email to iree-discuss...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/iree-discuss/77a49161-29d1-409d-a3ce-6722c6969349o%40googlegroups.com.

[Lei Zhang]

Also it seems you are mixing Clang and GCC when cross compiling by using Clang as a driver to look for GCC toolchains? I'm not sure about the flow and environment requirements on your side; but would it make sense to use GCC totally when you want GCC toolchain? For example, for Ubuntu you probably can install g++-aarch64-linux-gnu (https://packages.ubuntu.com/bionic/g++-aarch64-linux-gnu) and I think the following CMake toolchain file should work?

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR aarch64)

set(CMAKE_C_COMPILER   "aarch64-linux-gnu-gcc-10")
set(CMAKE_CXX_COMPILER "aarch64-linux-gnu-g++-10")

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

set(CMAKE_C_FLAGS "-march=armv8-a")
set(CMAKE_CXX_FLAGS "-march=armv8-a")

Thanks,

Lei

[Chuang-Yu Cheng]

(I forgot CC iree-discuss channel)

Hi Lei,

Thanks for your help!! I made some mistakes in the toolchain script!

Now I change the toolchain script to:

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR aarch64)

# clang version 12.0.0

set(CMAKE_C_COMPILER   "/path/to/clang")
set(CMAKE_CXX_COMPILER "/path/to/clang++")

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

set(EXTRA_CLNAG_ARGS
    --target=aarch64-linux-gnu -march=armv8-a
    -I /usr/aarch64-linux-gnu/include/c++/7/aarch64-linux-gnu
    -I /usr/aarch64-linux-gnu/include

)
list(JOIN EXTRA_CLNAG_ARGS " " EXTRA_CLNAG_ARG_STRING)
set(CMAKE_C_FLAGS ${EXTRA_CLNAG_ARG_STRING})
set(CMAKE_CXX_FLAGS ${EXTRA_CLNAG_ARG_STRINGS})

with the cmake command:
cmake -G Ninja /home/cycheng/mlir-project/iree \

  -DCMAKE_TOOLCHAIN_FILE=/to/clang-aarch64-toolchain.cmake \

  -DIREE_BUILD_COMPILER=OFF -DIREE_BUILD_SAMPLES=OFF \

  -DIREE_HOST_C_COMPILER=/to/clang \

  -DIREE_HOST_CXX_COMPILER=/to/clang++
cmake --build .

So now, clang-12 can correctly find the correct cross compile libraries

(I install it by:
sudo apt-get install gcc-aarch64-linux-gnu
sudo apt-get install libstdc++-7-dev-arm64-cross 
sudo apt-get install g++-aarch64-linux-gnu)

When I built, I got the linking error:

third_party/llvm-project/llvm/lib/libLLVMSupport.a: error adding symbols: File in wrong format
clang-12: error: linker command failed with exit code 1 (use -v to see invocation)
[18/1803] Configuring IREE for HOST build...

It looks like linker try to link x86 objects with aarch64 libraries.

This happens in the 'HOST' build stage.

I also checked my output directory, for example 'RegisterInfoEmitter.cpp.o'

(1) ./host/third_party/llvm-project/llvm/utils/TableGen/CMakeFiles/llvm-tblgen.dir/RegisterInfoEmitter.cpp.o
(2) ./third_party/llvm-project/llvm/utils/TableGen/CMakeFiles/llvm-tblgen.dir/RegisterInfoEmitter.cpp.o

I found the two files are both x86, but I expect the second one should be aarch64. 

===============

The reason I use gcc toolchain is because I read the two documents, it looks like if I want to use clang to cross build, then clang will still need gcc toolchain's help (headers, libraries). I found I don't have to specify it if I use clang-12.

See:

https://llvm.org/docs/HowToCrossCompileLLVM.html

However, if you’re using Clang, the driver might not be up-to-date with your specific Linux distribution, version or GCC layout, so you’ll need to fudge.

In addition to the ones above, you’ll also need:

• '-target arm-linux-gnueabihf' or whatever is the triple of your cross GCC.
• '--sysroot=/usr/arm-linux-gnueabihf', '--sysroot=/opt/gcc/arm-linux-gnueabihf' or whatever is the location of your GCC’s sysroot (where /lib, /bin etc are).
• Appropriate use of -I and -L, ...

https://s3.amazonaws.com/connect.linaro.org/bkk19/presentations/bkk19-210.pdf

See: p17 (Using a Clang installation for cross compiling). and p.18 (Finding the location of a GCC cross compiler)

And if I use gcc, I will get different build errors, for example:

FAILED: iree/hal/CMakeFiles/iree_hal_device_manager.dir/device_manager.cc.o
/usr/bin/aarch64-linux-gnu-g++ ... 

/home/cycheng/mlir-project/iree/iree/hal/buffer.h:292:24: error: enclosing class of constexpr non-static member function ‘iree::hal::Allocator* iree::hal::Buffer::allocator() const’ is not a literal type
constexpr Allocator* allocator() const {
^

===============

Could you give some directions so I could try it later?

Thanks for your time and help : )

CY

[Lei Zhang]

What version of GCC are you using? We just got GCC working quite recently (https://github.com/google/iree/pull/2569) so that may only work with specific versions. Sorry about the inconvenience!  

I tried the following command locally

cmake -GNinja /path/to/iree/source -DCMAKE_TOOLCHAIN_FILE=/path/to/the/previous/suggested/toolchain -DIREE_BUILD_COMPILER=OFF -DIREE_BUILD_SAMPLES=OFF -DIREE_HOST_C_COMPILER=`which clang` -DIREE_HOST_CXX_COMPILER=`which clang++`

and it works for me for compilation. (I haven't tested for real on an AArch64 Linux though.) I'm using GCC 10 cross toolchain and Clang 9.

(Note that you might be wondering about the mixing of GCC and Clang in the above; but that's fine given the GCC is used for cross compilation and Clang is used for compiling host binaries. They are for different CMake invocations. Again sorry about the weirdness here; requiring specifying host compilers is really a limitation we had back when we cannot support GCC; now GCC is supported so probably we can remove it.)

[Stella Laurenzo]

FYI - The one bot where we are building with gcc is gcc v9.1, I believe.

To view this discussion on the web visit https://groups.google.com/d/msgid/iree-discuss/CAGhUxBDjqWXn6tj2XDnM4rTd7kuRd5CLEC5UyfRkdySDhbrNOQ%40mail.gmail.com.

--

- Stella

[Chuang-Yu Cheng]

Cool! Thanks for the info! I will need to pull the latest IREE codebase.

My aarch64-linux-gnu-gcc is (Ubuntu/Linaro 7.5.0-3ubuntu1~18.04) 7.5.0, I will test it tomorrow. (Sorry I broke my machine just few minutes ago : P)

I am  also trying qemu-aarch64-static + chroot + ubuntu-base-18.04-base-arm64. Will let you know if I succeed!

wget http://cdimage.ubuntu.com/ubuntu-base/releases/18.04/release/ubuntu-base-18.04-base-arm64.tar.gz
mkdir ubuntu-base-18.04-base-arm64
cd ubuntu-base-18.04-base-arm64
export ROOTFS=/home/saber/sysroot/ubuntu-base-18.04-base-arm64
sudo tar -xf ../ubuntu-base-18.04-base-arm64.tar.gz -C ./
sudo cp /usr/bin/qemu-arm-static $ROOTFS/usr/bin/
sudo cp /usr/bin/qemu-aarch64-static $ROOTFS/usr/bin/
sudo cp /etc/resolv.conf $ROOTFS/etc
for m in `echo 'sys dev proc'`; do sudo mount /$m $ROOTFS/$m -o bind; done
sudo mount --bind /dev/pts $ROOTFS/dev/pts
sudo mount --bind /proc/sys/fs/binfmt_misc $ROOTFS/proc/sys/fs/binfmt_misc

sudo su
echo ':aarch64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7:\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/bin/qemu-aarch64:' > /proc/sys/fs/binfmt_misc/register

exit

sudo chroot .

[Stella Laurenzo]

Feel free to capture the compiler errors you are seeing with that GCC version and send them over for us to look at. We're not opposed to fixing things for older compiler versions within reason, but we take that on lazily, since it is otherwise a never-ending task.

--

- Stella

[Chuang-Yu Cheng]

Hi Stella,

Thanks, the policy makes sense to me!

Hi Lei,

I have updated the iree codebase and gcc/g++ toolchains by:

# rebase iree to ced7477f4c5

cd iree

git pull --rebase
git submodule update --init

# update gcc/g++ (and aarch versions)

sudo add-apt-repository ppa:ubuntu-toolchain-r/test
sudo vi /etc/apt/sources.list
    deb http://de.archive.ubuntu.com/ubuntu groovy main

sudo apt-get update
sudo apt install gcc-10 g++-10
sudo apt install gcc-10-aarch64-linux-gnu g++-10-aarch64-linux-gnu

# my system: ubuntu 18.04 LTS x86_64

I use your cross toolchain script with your cmake commands (and several different combinations), I got this error, but I think it almost works:

[1860/2057] Generating check_dylib-llvm-aot_dylib_success.mlir_module.module

FAILED: iree/modules/check/test/check_dylib-llvm-aot_dylib_success.mlir_module.module 

cd /home/saber/img-iree-build/gcc-aarch64-2/iree/modules/check/test && /home/saber/img-iree-build/gcc-aarch64-2/host/bin/iree-translate -iree-mlir-to-vm-bytecode-module --iree-hal-target-backends=dylib-llvm-aot /home/cycheng/mlir-project/iree/iree/modules/check/test/success.mlir -o check_dylib-llvm-aot_dylib_success.mlir_module.module

/home/cycheng/mlir-project/iree/iree/modules/check/test/success.mlir:58:13: error: Can't link executable and generate target dylib using lld::elf::link

  %result = "mhlo.add"(%c5, %c5) : (tensor<i32>, tensor<i32>) -> tensor<i32>

            ^

Do I need to set up anything for llvm AOT??

https://github.com/google/iree/blob/27f4f6114aa7a8e3289ccea66bc7569dd5ba5e83/iree/compiler/Dialect/HAL/Target/LLVM/LLVMAOTTarget.cpp#L111

Thanks 

[Stella Laurenzo]

We are just discussing that on the builds discord channel.

We are currently using this workaround:

# TODO: Revert once https://github.com/google/iree/issues/2645 resolved. export IREE_LLVMAOT_LINKER_PATH="$(which ld)"  

--

- Stella

[Lei Zhang]

+1 to what Stella said. This is a known issue tracked by https://github.com/google/iree/issues/2645. LLVM AOT support is still in progress at the moment. But given you are interested in GPU and PowerVR, you can largely ignore it.

(BTW, if you also want the LLVM AOT to work for cross compilation, you'll need the environment variable but pointing to a linker for cross compilation. See https://google.github.io/iree/get-started/getting-started-android-cmake#dylib-llvm-aot-backend for steps.)

Thanks,

Lei

[Chuang-Yu Cheng]

Thank you all!!

Now I can successfully cross build, and run iree-run-module on aarch64 device =)

(By the way, to fix the libm.so.6: version `GLIBC_2.29' not found the problem, I directly copy the /usr/aarch64-linux-gnu/lib/libm.so.6 to my device, and setup the LD_LIBRARY_PATH)

Thanks Lei for the LLVM AOT info!

[Lei Zhang]

Woohoo! Glad to hear it! :)

Thanks,

Lei

[Chuang-Yu Cheng]

I will write a document for it later. I am doing some experiments: try to cross build compiler with some work around and use gold linker.

[Chuang-Yu Cheng]

=== Build IREE for aarch64.by by cross toolchain ===

* IREE version: ced7477f4c5  (Jul 28, 2020)

# Use gcc-10/g++-10

sudo add-apt-repository ppa:ubuntu-toolchain-r/test
sudo vi /etc/apt/sources.list

    # add this line

  deb http://de.archive.ubuntu.com/ubuntu groovy main

sudo apt-get update
sudo apt install gcc-10 g++-10
sudo apt install gcc-10-aarch64-linux-gnu g++-10-aarch64-linux-gnu

vi aarch64-toolchain.cmake

  # Add the following lines

  set(CMAKE_SYSTEM_NAME Linux)
  set(CMAKE_SYSTEM_PROCESSOR aarch64)

  set(CMAKE_C_COMPILER   "aarch64-linux-gnu-gcc-10")
  set(CMAKE_CXX_COMPILER "aarch64-linux-gnu-g++-10")

  set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
  set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
  set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

  set(CMAKE_C_FLAGS "-march=armv8-a")
  set(CMAKE_CXX_FLAGS "-march=armv8-a")

cmake -G Ninja /path/to/iree \

  -DCMAKE_TOOLCHAIN_FILE=aarch64-toolchain.cmake \
  -B gcc-aarch64 \
  -DIREE_BUILD_COMPILER=OFF -DIREE_BUILD_SAMPLES=OFF \
  -DIREE_HOST_C_COMPILER=/usr/bin/gcc-10 \
  -DIREE_HOST_CXX_COMPILER=/usr/bin/g++-10

cmake --build gcc-aarch64

=== Build IREE for aarch64.by using qemu + chroot in Ubuntu 18.04 ===

* IREE version: ced7477f4c5  (Jul 28, 2020)

* It may take 6 ~ 7 hours to build everything, including the experimental directory, on the x86 with 8 hw threads + 16gb ram.

# install qemu user

sudo apt-get install -y qemu-user-static

# prepare ubuntu 18.04 rootfs

export ROOTFS=$PWD

wget http://cdimage.ubuntu.com/ubuntu-base/releases/18.04/release/ubuntu-base-18.04-base-arm64.tar.gz

mkdir ubuntu-base-18.04-base-arm64
cd ubuntu-base-18.04-base-arm64

sudo tar -xf ../ubuntu-base-18.04-base-arm64.tar.gz -C ./

sudo cp /usr/bin/qemu-aarch64-static $ROOTFS/usr/bin/
sudo cp /etc/resolv.conf $ROOTFS/etc

# create shared folders between host pc and rootfs

sudo mkdir -p $ROOTFS/path/to/iree

sudo mkdir -p $ROOTFS/path/to/iree/build

sudo mount --bind /path/to/iree $ROOTFS/path/to/iree

sudo mount --bind /path/to/iree/build $ROOTFS/path/to/iree/build

sudo mount --bind /proc proc
sudo mount --bind /sys sys
sudo mount --bind /dev dev

# chroot.
sudo chroot $ROOTFS

apt-get update
apt-get install build-essential wget git python3 cmake ninja-build

# install cmake 3.15
wget https://github.com/Kitware/CMake/releases/download/v3.15.2/cmake-3.15.2.tar.gz
tar -zxf cmake-3.15.2.tar.gz

cd cmake-3.15.2
./bootstrap
make -j8 && sudo make install

# download clang10

wget https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang+llvm-10.0.0-aarch64-linux-gnu.tar.xz

tar -xf clang+llvm-10.0.0-aarch64-linux-gnu.tar.xz

# [optional] prepare vulkan

mkdir vulkan-sdk

# copy vulkan include and lib to vulkan-sdk

# build iree:

export IREE_LLVMAOT_LINKER_PATH="$(which ld)"
export VULKAN_SDK=/path/to/vulkan-sdk
cmake -G Ninja /path/to/iree \
  -B aarch64 \
  -DIREE_BUILD_EXPERIMENTAL=ON \
  -DIREE_ENABLE_LLD=ON \
  -DCMAKE_C_COMPILER=./clang+llvm-10.0.0-aarch64-linux-gnu/bin/clang \
  -DCMAKE_CXX_COMPILER=./clang+llvm-10.0.0-aarch64-linux-gnu/bin/clang++

cmake --build aarch64

# Currently we need patch two things for ced7477f4c5

(1)

ld.lld: error: undefined symbol: LLVMInitializeAArch64Target
>>> referenced by llvmjit_driver_module.cc
>>>               llvmjit_driver_module.cc.o:(iree::hal::llvmjit::CreateLLVMJITDriver()) in archive iree/hal/llvmjit/libiree_hal_llvmjit_llvmjit_driver_module.a

patch: iree/hal/llvmjit/CMakeLists.txt 

iree_cc_library(
     ::llvmjit_driver
     LLVMSupport
     LLVMX86CodeGen
+    LLVMAArch64CodeGen

(2)

iree/experimental/ModelBuilder/test/TestMNISTJIT.cpp:186:7: error: use of undeclared identifier 'makeInitializedUnrank
edDescriptor'; did you mean 'makeInitializedStridedMemRefDescriptor'?
      makeInitializedUnrankedDescriptor<float, 2>({B, W3}, outputLinearInit);

patch: experimental/ModelBuilder/test/CMakeLists.txt

For now I just skip the 'test-mnist-jit' build.

[Lei Zhang]

Awesome, thanks Chuang-Yu!

Would you mind opening a pull request to put these instructions in a md file at https://github.com/google/iree/tree/main/docs/get_started? It can be useful for others looking to cross compile towards general aarch64 linux too. You can refer to existing md files for style etc. :)

BTW, feel free to create issues on GitHub for the issues you've hit. We might not be able to get to it immediately; but at least keep track of.

Thanks,

Lei

[Chuang-Yu Cheng]

Hi Lei,

Sure =)

Okay! I will create the issues for tracking!

Cheers!!

CY