[PATCH 1/2] make: add new uniform kernel.elf artifact intended for direct-kernel boot mode

[Waldemar Kozaczuk]

Loader.elf, the main build artifact, besides kernel code also contains small
bootfs filesystem. The purpose of bootfs is to provide mkfs.so, zpool.so and
other binaries intended to help create ZFS filesystem on he main drive which
is then populated by upload_manifest.py tool during build process. Once usr.img
is created, the bootfs filesystem, which is still part of the original loader.elf
and therefore usr.img, is no longer necessary to boot OSv image as the ZFS file system
is already created. On average bootfs adds 800K to loader-stripped.elf
which causes unnecessary memory waste and slightly slows down kernel loading.
It is worth pointing out the loader.elf created for ROFS filesystem has empty bootfs.

This patch changes main makefile to produce new artifact - kernel.elf -
that in effect is loader.elf with empty bootfs. The kernel.elf is intended
for direct-kernel boot with firecracker and qemu '-kernel' mode and can be used
to boot with either ZFS or ROFS disk.

This patch alse changes firecracker.elf and run.py with '-k' option to use kernel.elf
instead of loader.elf to boot OSv from.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---
Makefile | 17 ++++++++++++++++-
empty_bootfs.S | 4 ++++
scripts/firecracker.py | 4 ++--
scripts/run.py | 4 +++-
4 files changed, 25 insertions(+), 4 deletions(-)
create mode 100644 empty_bootfs.S

diff --git a/Makefile b/Makefile
index 9401e809..db3c68cf 100644
--- a/Makefile
+++ b/Makefile
@@ -125,7 +125,7 @@ endif
quiet = $(if $V, $1, @echo " $2"; $1)
very-quiet = $(if $V, $1, @$1)

-all: $(out)/loader.img links
+all: $(out)/loader.img links $(out)/kernel.elf
ifeq ($(arch),x64)
all: $(out)/vmlinuz.bin
endif
@@ -1868,6 +1868,19 @@ $(out)/loader.elf: $(stage1_targets) arch/$(arch)/loader.ld $(out)/bootfs.o $(lo
@scripts/libosv.py $(out)/osv.syms $(out)/libosv.ld `scripts/osv-version.sh` | $(CC) -c -o $(out)/osv.o -x assembler -
$(call quiet, $(CC) $(out)/osv.o -nostdlib -shared -o $(out)/libosv.so -T $(out)/libosv.ld, LIBOSV.SO)

+$(out)/kernel.elf: $(stage1_targets) arch/$(arch)/loader.ld $(out)/empty_bootfs.o $(loader_options_dep)
+ $(call quiet, $(LD) -o $@ --defsym=OSV_KERNEL_BASE=$(kernel_base) \
+ --defsym=OSV_KERNEL_VM_BASE=$(kernel_vm_base) --defsym=OSV_KERNEL_VM_SHIFT=$(kernel_vm_shift) \
+ -Bdynamic --export-dynamic --eh-frame-hdr --enable-new-dtags -L$(out)/arch/$(arch) \
+ $(^:%.ld=-T %.ld) \
+ --whole-archive \
+ $(libstdc++.a) $(libgcc_eh.a) \
+ $(boost-libs) \
+ --no-whole-archive $(libgcc.a), \
+ LINK kernel.elf)
+ $(call quiet, $(STRIP) $(out)/kernel.elf -o $(out)/kernel-stripped.elf, STRIP kernel.elf -> kernel-stripped.elf )
+ $(call very-quiet, cp $(out)/kernel-stripped.elf $(out)/kernel.elf)
+
$(out)/bsd/%.o: COMMON += -DSMP -D'__FBSDID(__str__)=extern int __bogus__'

environ_sources = $(addprefix libc/, $(environ_libc))
@@ -1907,6 +1920,8 @@ $(out)/bootfs.bin: scripts/mkbootfs.py $(bootfs_manifest) $(bootfs_manifest_dep)
$(out)/bootfs.o: $(out)/bootfs.bin
$(out)/bootfs.o: ASFLAGS += -I$(out)

+$(out)/empty_bootfs.o: ASFLAGS += -I$(out)
+
$(out)/tools/mkfs/mkfs.so: $(out)/tools/mkfs/mkfs.o $(out)/libzfs.so
$(makedir)
$(call quiet, $(CC) $(CFLAGS) -o $@ $(out)/tools/mkfs/mkfs.o -L$(out) -lzfs, LINK mkfs.so)
diff --git a/empty_bootfs.S b/empty_bootfs.S
new file mode 100644
index 00000000..c72b9d94
--- /dev/null
+++ b/empty_bootfs.S
@@ -0,0 +1,4 @@
+.pushsection .data
+.global bootfs_start
+bootfs_start:
+.popsection
diff --git a/scripts/firecracker.py b/scripts/firecracker.py
index 8656ae79..3cf47c58 100755
--- a/scripts/firecracker.py
+++ b/scripts/firecracker.py
@@ -266,7 +266,7 @@ def main(options):
# Prepare arguments we are going to pass when creating VM instance
kernel_path = options.kernel
if not kernel_path:
- kernel_path = os.path.join(dirname, '../build/release/loader-stripped.elf')
+ kernel_path = os.path.join(dirname, '../build/release/kernel.elf')

qemu_disk_path = options.image
if not qemu_disk_path:
@@ -364,7 +364,7 @@ if __name__ == "__main__":
parser.add_argument("-i", "--image", action="store", default=None, metavar="CMD",
help="path to disk image file. defaults to ../build/release/usr.img")
parser.add_argument("-k", "--kernel", action="store", default=None, metavar="CMD",
- help="path to kernel loader file. defaults to ../build/release/loader-stripped.elf")
+ help="path to kernel loader file. defaults to ../build/release/kernel.elf")
parser.add_argument("-n", "--networking", action="store_true",
help="needs root to setup tap networking first time")
parser.add_argument("-b", "--bridge", action="store", default=None,
diff --git a/scripts/run.py b/scripts/run.py
index 26177d57..76cecb56 100755
--- a/scripts/run.py
+++ b/scripts/run.py
@@ -548,6 +548,8 @@ if __name__ == "__main__":
help="Path to the optional cloud-init image that should be attached to the instance")
parser.add_argument("-k", "--kernel", action="store_true",
help="Run OSv in QEMU kernel mode as PVH.")
+ parser.add_argument("--kernel-path", action="store",
+ help="path to kernel.elf. defaults to build/$mode/kernel.elf")
parser.add_argument("--virtio", action="store", choices=["legacy","transitional","modern"], default="transitional",
help="specify virtio version: legacy, transitional or modern")
parser.add_argument("--arch", action="store", choices=["x86_64","aarch64"], default="x86_64",
@@ -558,7 +560,7 @@ if __name__ == "__main__":
if cmdargs.arch == 'aarch64':
cmdargs.kernel_file = os.path.join(osv_base, "build/%s/loader.img" % cmdargs.opt_path)
else:
- cmdargs.kernel_file = os.path.join(osv_base, "build/%s/loader-stripped.elf" % cmdargs.opt_path)
+ cmdargs.kernel_file = os.path.abspath(cmdargs.kernel_path or os.path.join(osv_base, "build/%s/kernel.elf" % cmdargs.opt_path))
if not os.path.exists(cmdargs.image_file):
raise Exception('Image file %s does not exist.' % cmdargs.image_file)
if cmdargs.cloud_init_image:
--
2.20.1

[Waldemar Kozaczuk]

This patch enhances the build shell script by adding new '--create-disk' option
that allows building kernel-less disk images. Kernel-less disk images are
intended to be used with direct kernel boot. Please note it can only be used
with QEMU 4.0 or later as it requires PVH boot support.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---

scripts/build | 55 +++++++++++++++++++++++++++-----------
scripts/upload_manifest.py | 10 ++++++-
2 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/scripts/build b/scripts/build
index dbf1a9d7..5265ad67 100755
--- a/scripts/build
+++ b/scripts/build
@@ -34,6 +34,7 @@ usage() {
(can be used to customize specific app/module build process)
-j<N> Set number of parallel jobs for make
--append-manifest Append build/<mode>/append.manifest to usr.manifest
+ --create-disk Instead of usr.img create kernel-less disk.img

Examples:
./scripts/build -j4 fs=rofs image=native-example # Create image with native-example app
@@ -75,7 +76,7 @@ do
case $i in
--help|-h)
usage ;;
- image=*|modules=*|fs=*|usrskel=*|check|--append-manifest) ;;
+ image=*|modules=*|fs=*|usrskel=*|check|--append-manifest|--create-disk) ;;
clean)
stage1_args=clean ;;
*) # yuck... Is there a prettier way to append to array?
@@ -147,6 +148,8 @@ do
vars[image]=tests;;
--append-manifest)
vars[append_manifest]="true";;
+ --create-disk)
+ vars[create_disk]="true";;
esac
done

@@ -261,26 +264,46 @@ if [ "$export" != "none" ]; then
exit 0
fi

-case $fs_type in
-zfs)
- cp loader.img bare.raw
- zfs_size=$(($fs_size - $kernel_end))
- "$SRC"/scripts/imgedit.py setpartition "-f raw bare.raw" 2 $kernel_end $zfs_size
+if [[ ${vars[create_disk]} == "true" ]]; then
+ partition_offset=512
+ bare=boot.bin
+ raw_disk=disk
+ qcow2_disk=disk
+ upload_kernel_mode="-k"
+else
+ partition_offset=$kernel_end
+ bare=loader.img
+ raw_disk=bare
+ qcow2_disk=usr
+fi
+
+create_zfs_disk() {
+ cp $bare $raw_disk.raw
+ "$SRC"/scripts/imgedit.py setpartition "-f raw ${raw_disk}.raw" 2 $partition_offset $partition_size
+ qemu-img convert -f raw -O qcow2 $raw_disk.raw $qcow2_disk.img
+ qemu-img resize $qcow2_disk.img ${image_size}b >/dev/null 2>&1
+ "$SRC"/scripts/upload_manifest.py -o $qcow2_disk.img -m usr.manifest -D libgcc_s_dir="$libgcc_s_dir" $upload_kernel_mode
+}

- qemu-img convert -f raw -O qcow2 bare.raw usr.img
- qemu-img resize usr.img ${fs_size}b >/dev/null 2>&1
+create_rofs_disk() {
+ cp $bare $raw_disk.raw
+ "$SRC"/scripts/imgedit.py setpartition "-f raw ${raw_disk}.raw" 2 $partition_offset $partition_size
+ qemu-img resize ${raw_disk}.raw ${image_size}b >/dev/null 2>&1
+ dd if=rofs.img of=${raw_disk}.raw obs=$partition_offset seek=1 >/dev/null 2>&1
+ qemu-img convert -f raw -O qcow2 $raw_disk.raw $qcow2_disk.img
+}

- "$SRC"/scripts/upload_manifest.py -o usr.img -m usr.manifest -D libgcc_s_dir="$libgcc_s_dir" ;;
+case $fs_type in
+zfs)
+ partition_size=$((fs_size - partition_offset))
+ image_size=$fs_size
+ create_zfs_disk ;;
rofs)
rm -rf rofs.img
"$SRC"/scripts/gen-rofs-img.py -o rofs.img -m usr.manifest -D libgcc_s_dir="$libgcc_s_dir"
- rofs_size=`stat --printf %s rofs.img`
- img_size=$((kernel_end + rofs_size))
- cp loader.img bare.raw
- "$SRC"/scripts/imgedit.py setpartition "-f raw bare.raw" 2 $kernel_end $rofs_size
- qemu-img resize bare.raw ${img_size}b >/dev/null 2>&1
- dd if=rofs.img of=bare.raw obs=${kernel_end} seek=1 >/dev/null 2>&1
- qemu-img convert -f raw -O qcow2 bare.raw usr.img ;;
+ partition_size=`stat --printf %s rofs.img`
+ image_size=$((partition_offset + partition_size))
+ create_rofs_disk ;;
ramfs)
qemu-img convert -f raw -O qcow2 loader.img usr.img ;;
esac
diff --git a/scripts/upload_manifest.py b/scripts/upload_manifest.py
index f036cbb9..97c59e5b 100755
--- a/scripts/upload_manifest.py
+++ b/scripts/upload_manifest.py
@@ -134,6 +134,10 @@ def main():
metavar='VAR=DATA',
action='callback',
callback=add_var),
+ make_option('-k',
+ dest='kernel',
+ action='store_true',
+ help='run OSv in direct kernel mode')
])

(options, args) = opt.parse_args()
@@ -147,7 +151,11 @@ def main():

image_path = os.path.abspath(options.output)
upload_port = find_free_port()
- osv = subprocess.Popen('cd ../..; scripts/run.py --vnc none -m 512 -c1 -i "%s" --block-device-cache unsafe -s -e "--norandom --nomount --noinit /tools/mkfs.so; /tools/cpiod.so --prefix /zfs/zfs/; /zfs.so set compression=off osv" --forward tcp:127.0.0.1:%s-:10000' % (image_path,upload_port), shell=True, stdout=subprocess.PIPE)
+ if options.kernel:
+ kernel_mode_flag = '-k --kernel-path build/release/loader-stripped.elf'
+ else:
+ kernel_mode_flag = ''
+ osv = subprocess.Popen('cd ../..; scripts/run.py %s --vnc none -m 512 -c1 -i "%s" --block-device-cache unsafe -s -e "--norandom --nomount --noinit /tools/mkfs.so; /tools/cpiod.so --prefix /zfs/zfs/; /zfs.so set compression=off osv" --forward tcp:127.0.0.1:%s-:10000' % (kernel_mode_flag,image_path,upload_port), shell=True, stdout=subprocess.PIPE)

upload(osv, manifest, depends, upload_port)

--
2.20.1

[Commit Bot]

From: Waldemar Kozaczuk <jwkoz...@gmail.com>
Committer: Waldemar Kozaczuk <jwkoz...@gmail.com>
Branch: master

make: add new uniform kernel.elf artifact intended for direct-kernel boot mode

Loader.elf, the main build artifact, besides kernel code also contains small
bootfs filesystem. The purpose of bootfs is to provide mkfs.so, zpool.so and
other binaries intended to help create ZFS filesystem on he main drive which
is then populated by upload_manifest.py tool during build process. Once usr.img
is created, the bootfs filesystem, which is still part of the original loader.elf
and therefore usr.img, is no longer necessary to boot OSv image as the ZFS file system
is already created. On average bootfs adds 800K to loader-stripped.elf
which causes unnecessary memory waste and slightly slows down kernel loading.
It is worth pointing out the loader.elf created for ROFS filesystem has empty bootfs.

This patch changes main makefile to produce new artifact - kernel.elf -
that in effect is loader.elf with empty bootfs. The kernel.elf is intended
for direct-kernel boot with firecracker and qemu '-kernel' mode and can be used
to boot with either ZFS or ROFS disk.

This patch alse changes firecracker.elf and run.py with '-k' option to use kernel.elf
instead of loader.elf to boot OSv from.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>

Message-Id: <20200411040454.1...@gmail.com>

---
diff --git a/Makefile b/Makefile

--- a/empty_bootfs.S

+++ b/empty_bootfs.S
@@ -0,0 +1,4 @@
+.pushsection .data
+.global bootfs_start
+bootfs_start:
+.popsection
diff --git a/scripts/firecracker.py b/scripts/firecracker.py

--- a/scripts/firecracker.py
+++ b/scripts/firecracker.py
@@ -266,7 +266,7 @@ def main(options):
# Prepare arguments we are going to pass when creating VM instance
kernel_path = options.kernel
if not kernel_path:
- kernel_path = os.path.join(dirname, '../build/release/loader-stripped.elf')
+ kernel_path = os.path.join(dirname, '../build/release/kernel.elf')

qemu_disk_path = options.image
if not qemu_disk_path:

@@ -364,7 +364,7 @@ def main(options):

parser.add_argument("-i", "--image", action="store", default=None, metavar="CMD",
help="path to disk image file. defaults to ../build/release/usr.img")
parser.add_argument("-k", "--kernel", action="store", default=None, metavar="CMD",
- help="path to kernel loader file. defaults to ../build/release/loader-stripped.elf")
+ help="path to kernel loader file. defaults to ../build/release/kernel.elf")
parser.add_argument("-n", "--networking", action="store_true",
help="needs root to setup tap networking first time")
parser.add_argument("-b", "--bridge", action="store", default=None,
diff --git a/scripts/run.py b/scripts/run.py

--- a/scripts/run.py
+++ b/scripts/run.py
@@ -552,6 +552,8 @@ def main(options):

help="Path to the optional cloud-init image that should be attached to the instance")
parser.add_argument("-k", "--kernel", action="store_true",
help="Run OSv in QEMU kernel mode as PVH.")
+ parser.add_argument("--kernel-path", action="store",
+ help="path to kernel.elf. defaults to build/$mode/kernel.elf")
parser.add_argument("--virtio", action="store", choices=["legacy","transitional","modern"], default="transitional",
help="specify virtio version: legacy, transitional or modern")
parser.add_argument("--arch", action="store", choices=["x86_64","aarch64"], default="x86_64",

@@ -562,7 +564,7 @@ def main(options):

[Commit Bot]

From: Waldemar Kozaczuk <jwkoz...@gmail.com>
Committer: Waldemar Kozaczuk <jwkoz...@gmail.com>
Branch: master

build: allow building kernel-less ZFS and ROFS disk images

This patch enhances the build shell script by adding new '--create-disk' option
that allows building kernel-less disk images. Kernel-less disk images are
intended to be used with direct kernel boot. Please note it can only be used
with QEMU 4.0 or later as it requires PVH boot support.

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>

Message-Id: <20200411040454.1...@gmail.com>

---
diff --git a/scripts/build b/scripts/build