[PATCH] Upgrade cli, lua and httpserver-api modules to use OpenSSL 1.1 and Lua 5.3

Waldemar Kozaczuk

unread,

Aug 1, 2019, 5:44:48 PM8/1/19

to osv...@googlegroups.com, Waldemar Kozaczuk

As the issue #1022 explains, building the default OSv image that
implicitly adds cli module that in turn implicitly pulls httpserver-api
and cli modules, requires some bizarre workarounds on modern Linux distribution.
This is caused by the fact that the lua module is based on Lua 5.2 and older version
of LuaSec (OpenSSL library for Lua) which requires old version of OpenSSL 1.0.

This patch updates following modules:
1) cli:
- makes necessary changes to work with Lua 5.3
- drops explicit dependency on the modules ncurses and libedit in lieu of
new terminfo and the script manifest_from_host.sh that pulls all necessary
dependencies from host
- drops obsolete rpmbuild

2) httpserver-api
- drops explicit dependency on the module libyaml in lieu of using
manifest_from_host.sh that pulls all necessary dependencies from host

3) lua
- revamps this module completely by upgrading to Lua 5.3
- extensively employs manifest_from_host.sh to pull Lua libraries
from host (see changes to setup.py) and minimize building from source
- updates Lua modules (like LuaSec) to the newest versions

4) openssl
- instead of using hardcoded usr.manifest, new Makefile employs manifest_from_host.sh
to find relevant main OpenSSL 1.1 library and its required dependencies from host

The patch also deletes the obsolete module libedit in lieu of using relevant files from host.

Finally it adds new terminfo module.

This patch has been tested with following Linux distribution using Docker:
- Ubuntu 18.10 (OpenSSL 1.0 -> OpenSSL 1.1 upgrade)
- Ubuntu 19.04
- Fedora 27 (OpenSSL 1.0 -> OpenSSL 1.1 upgrade)
- Ubuntu 29

Fixes #1022

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---
modules/cli/.gitignore | 3 +-
modules/cli/Makefile | 33 ++----
modules/cli/cli.c | 63 ++++++-----
modules/cli/lib/osv_api.lua | 4 +-
modules/cli/module.py | 6 +-
modules/cli/rpmbuild/.gitignore | 6 --
modules/cli/rpmbuild/Makefile | 45 --------
modules/cli/rpmbuild/SPECS/osv-cli.spec | 54 ----------
modules/httpserver-api/Makefile | 1 +
modules/httpserver-api/module.py | 1 -
modules/libedit/.gitignore | 2 -
modules/libedit/Makefile | 41 --------
modules/libedit/module.py | 7 --
modules/lua/.gitignore | 8 +-
modules/lua/Makefile | 133 ++++++------------------
modules/lua/check-openssl-version | 16 ---
modules/lua/module.py | 6 +-
modules/lua/src/Makefile | 119 ---------------------
modules/openssl/.gitignore | 1 +
modules/openssl/Makefile | 23 ++++
modules/openssl/usr.manifest | 37 -------
modules/terminfo/Makefile | 16 +++
modules/terminfo/module.py | 4 +
scripts/setup.py | 16 +--
24 files changed, 143 insertions(+), 502 deletions(-)
delete mode 100644 modules/cli/rpmbuild/.gitignore
delete mode 100644 modules/cli/rpmbuild/Makefile
delete mode 100644 modules/cli/rpmbuild/SPECS/osv-cli.spec
delete mode 100644 modules/libedit/.gitignore
delete mode 100644 modules/libedit/Makefile
delete mode 100644 modules/libedit/module.py
delete mode 100755 modules/lua/check-openssl-version
delete mode 100644 modules/lua/src/Makefile
create mode 100644 modules/openssl/.gitignore
create mode 100644 modules/openssl/Makefile
delete mode 100644 modules/openssl/usr.manifest
create mode 100644 modules/terminfo/Makefile
create mode 100644 modules/terminfo/module.py

diff --git a/modules/cli/.gitignore b/modules/cli/.gitignore
index 773de15c..7d357ae8 100644
--- a/modules/cli/.gitignore
+++ b/modules/cli/.gitignore
@@ -1,2 +1,3 @@
-cli
cli.so
+cli-stripped.so
+usr.manifest
diff --git a/modules/cli/Makefile b/modules/cli/Makefile
index e2abe632..41e788bf 100644
--- a/modules/cli/Makefile
+++ b/modules/cli/Makefile
@@ -1,34 +1,23 @@
+LUA_LIB_NAME = $(shell ldconfig -p | grep -Po "liblua*.5\.3.so" | grep -o "lua.*5.3" | head -1)
+LUA_INCLUDES = $(shell $(CXX) -E -xc++ - -v </dev/null 2>&1 | awk '/^End/ {exit} /^ \/.*include/ {print $$0}' | grep -vP 'c\+\+|gnu' | xargs -I '{}' find {} -name lualib.h | xargs dirname | awk '// { printf("-I%s\n", $$0)}')
+
CC=gcc
CFLAGS=-O2 -g -Wall -std=gnu99
-INCLUDES=-I../lua/src \
- -I$(lastword $(wildcard ../ncurses/build/*/include)) \
- -I$(lastword $(wildcard ../libedit/build/*/src))
-LFLAGS=-L../lua/src \
- -L$(lastword $(wildcard ../ncurses/build/*/lib)) \
- -L$(lastword $(wildcard ../libedit/build/*/src/.libs))
-LIBS=-ledit -ltinfo -llua
+LIBS=-ledit -ltinfo -l$(LUA_LIB_NAME)
SRCS=cli.c
MAIN=cli.so

-# For compiling executable running locally
-LOC_INCLUDES=-I../lua/src
-LOC_LFLAGS=-L../lua/src
-LOC_LDFLAGS=-Wl,-rpath,$(abspath ../lua/src)
-LOC_MAIN=cli
+INCLUDES = $(LUA_INCLUDES)
+
+SRC = $(shell readlink -f ../..)

module: $(MAIN)
+ $(SRC)/scripts/manifest_from_host.sh cli.so > usr.manifest

$(MAIN): $(SRCS)
- $(CC) -DOSV_CLI $(CFLAGS) $(INCLUDES) -fPIC -shared -o $@ $(SRCS) $(LFLAGS) $(LIBS)
-
-$(LOC_MAIN): $(SRCS)
- $(CC) $(CFLAGS) $(LOC_INCLUDES) $^ -o $@ $(LOC_LDFLAGS) $(LOC_LFLAGS) $(LIBS)
+ $(CC) $(CFLAGS) $(INCLUDES) $^ -fPIC -pie -o $@ $(LIBS)

clean:
- rm -f $(MAIN) $(LOC_MAIN)
- make -f rpmbuild/Makefile clean
-
-rpm: $(LOC_MAIN)
- make -C rpmbuild
+ rm -f $(MAIN)

-.PHONY: module clean rpm
+.PHONY: module clean
diff --git a/modules/cli/cli.c b/modules/cli/cli.c
index 01c6a1e8..5b39d68c 100644
--- a/modules/cli/cli.c
+++ b/modules/cli/cli.c
@@ -5,6 +5,7 @@
#include <signal.h>
#include <getopt.h>
#include <signal.h>
+#include <gnu/libc-version.h>

#include <lua.h>
#include <lauxlib.h>
@@ -16,17 +17,15 @@
#include <sys/poll.h>
#include <termios.h>

-#ifdef OSV_CLI
-#define CLI_LUA "/cli/cli.lua"
-#define CLI_LUA_PATH "/usr/share/lua/5.2/?.lua;/cli/lib/?.lua;/cli/lua/share/?.lua"
-#define CLI_LUA_CPATH "/usr/lib/lua/5.2/?.so;/cli/lua/lib/?.so"
-#define CLI_COMMANDS_PATH "/cli/commands"
-#else
-#define CLI_LUA "cli.lua"
-#define CLI_LUA_PATH "lib/?.lua;../lua/out/share/lua/5.2/?.lua;./lib/share/lua/5.2/?.lua"
-#define CLI_LUA_CPATH "../lua/out/lib/lua/5.2/?.so;./lib/lib/lua/5.2/?.so"
-#define CLI_COMMANDS_PATH "./commands"
-#endif
+#define OSV_CLI_LUA "/cli/cli.lua"
+#define OSV_CLI_LUA_PATH "/usr/share/lua/5.3/?.lua;/usr/share/lua/5.3/?/init.lua;/cli/lib/?.lua;/cli/lua/share/?.lua"
+#define OSV_CLI_LUA_CPATH "/usr/lib/lua/5.3/?.so;/cli/lua/lib/?.so"
+#define OSV_CLI_COMMANDS_PATH "/cli/commands"
+
+#define HOST_CLI_LUA "cli.lua"
+#define HOST_CLI_LUA_PATH "lib/?.lua;./lib/share/lua/5.3/?.lua;../lua/install/lua_modules/share/lua/5.3/?.lua;../lua/install/lua_modules/share/lua/5.3/?/init.lua"
+#define HOST_CLI_LUA_CPATH "./lib/lib/lua/5.3/?.so;../lua/install/lua_modules/lib/lua/5.3/?.so"
+#define HOST_CLI_COMMANDS_PATH "./commands"

#define PROMPT_MAXLEN 128
static char sprompt[PROMPT_MAXLEN];
@@ -66,22 +65,26 @@ static struct {
/* Misc */
void print_usage();

+static int is_osv() {
+ return strcmp("OSv", gnu_get_libc_release()) == 0;
+}
+
int main (int argc, char* argv[]) {
-#ifdef OSV_CLI
- putenv("TERM=vt100-qemu");
+ if (is_osv()) {
+ putenv("TERM=vt100-qemu");

- cli_console_size_dirty();
-#else
- struct winsize sz;
- ioctl(0, TIOCGWINSZ, &sz);
+ cli_console_size_dirty();
+ } else {
+ struct winsize sz;
+ ioctl(0, TIOCGWINSZ, &sz);

- if (sz.ws_col > 0 && sz.ws_row > 0) {
- con_width = sz.ws_col;
- con_height = sz.ws_row;
+ if (sz.ws_col > 0 && sz.ws_row > 0) {
+ con_width = sz.ws_col;
+ con_height = sz.ws_row;

- signal(SIGWINCH, cli_sigwinch_handler);
+ signal(SIGWINCH, cli_sigwinch_handler);
+ }
}
-#endif

int i;

@@ -137,6 +140,9 @@ int main (int argc, char* argv[]) {

/* Lua state */
L = cli_luaL_newstate();
+ if (L == NULL) {
+ exit(2);
+ }

if (test_command != NULL) {
if (L == NULL) {
@@ -292,10 +298,15 @@ lua_State *cli_luaL_newstate() {
lua_State *L = luaL_newstate();
luaL_openlibs(L);

- cli_lua_settable(L, "package", "path", CLI_LUA_PATH);
- cli_lua_settable(L, "package", "cpath", CLI_LUA_CPATH);
+ if (is_osv()) {
+ cli_lua_settable(L, "package", "path", OSV_CLI_LUA_PATH);
+ cli_lua_settable(L, "package", "cpath", OSV_CLI_LUA_CPATH);
+ } else {
+ cli_lua_settable(L, "package", "path", HOST_CLI_LUA_PATH);
+ cli_lua_settable(L, "package", "cpath", HOST_CLI_LUA_CPATH);
+ }

- int error = luaL_loadfile(L, CLI_LUA) || lua_pcall(L, 0, 0, 0);
+ int error = luaL_loadfile(L, is_osv() ? OSV_CLI_LUA : HOST_CLI_LUA) || lua_pcall(L, 0, 0, 0);
if (error) {
fprintf(stderr, "Failed to load shell: %s\n", lua_tostring(L, -1));
lua_pop(L, 1);
@@ -304,7 +315,7 @@ lua_State *cli_luaL_newstate() {
return NULL;
}

- cli_lua_settable(L, "context", "commands_path", CLI_COMMANDS_PATH);
+ cli_lua_settable(L, "context", "commands_path", is_osv() ? OSV_CLI_COMMANDS_PATH : HOST_CLI_COMMANDS_PATH);
for (int i=0; i<CTXC; i++) {
if (ctxv[i]) {
cli_lua_settable(L, "context", ctx[i].name, ctxv[i]);
diff --git a/modules/cli/lib/osv_api.lua b/modules/cli/lib/osv_api.lua
index 7155298d..affc3056 100644
--- a/modules/cli/lib/osv_api.lua
+++ b/modules/cli/lib/osv_api.lua
@@ -91,12 +91,12 @@ local function create_ssl_socket()

local params = {
mode = "client",
- protocol = "tlsv1",
+ protocol = "any",
key = context.ssl_key,
certificate = context.ssl_cert,
cafile = context.ssl_cacert,
verify = context.ssl_verify,
- options = "all"
+ options = {"all", "no_sslv3"}
}

local st = getmetatable(conn.sock).__index.settimeout
diff --git a/modules/cli/module.py b/modules/cli/module.py
index e38aa2db..a0ace526 100644
--- a/modules/cli/module.py
+++ b/modules/cli/module.py
@@ -3,16 +3,14 @@ from osv.modules.filemap import FileMap
from osv.modules import api

require('lua')
-require('ncurses')
-require('libedit')
+require('terminfo')
require_running('httpserver')

usr_files = FileMap()
usr_files.add('${OSV_BASE}/modules/cli').to('/cli') \
- .include('cli.so') \
.include('cli.lua') \
.include('lib/**') \
.include('commands/**')

-full = api.run('/cli/cli.so')
+full = api.run('/cli.so')
default = full
diff --git a/modules/cli/rpmbuild/.gitignore b/modules/cli/rpmbuild/.gitignore
deleted file mode 100644
index f44d1b0b..00000000
--- a/modules/cli/rpmbuild/.gitignore
+++ /dev/null
@@ -1,6 +0,0 @@
-BUILD/*
-BUILDROOT/*
-osv-cli-1.0/*
-RPMS/*
-SOURCES/*
-SRPMS/*
diff --git a/modules/cli/rpmbuild/Makefile b/modules/cli/rpmbuild/Makefile
deleted file mode 100644
index be9915d6..00000000
--- a/modules/cli/rpmbuild/Makefile
+++ /dev/null
@@ -1,45 +0,0 @@
-ROOT_DIR := $(shell pwd)
-CLI_ROOT=$(ROOT_DIR)/..
-LUA_DIR=$(CLI_ROOT)/../lua
-VERSION=1.0
-
-MAIN=rpm
-CLI_DIR=osv-cli-$(VERSION)/usr/lib64/osv-cli
-LOC_MAIN=cli
-
-module: clean all
-
-all: $(MAIN)
-
-init:
- mkdir -p BUILD
- mkdir -p BUILDROOT
- mkdir -p $(CLI_DIR)/lib
- mkdir -p RPMS
- mkdir -p SOURCES
- mkdir -p SRPMS
- echo $(ROOT_DIR)
-
-$(MAIN): init copy-files
- tar -zcvf osv-cli-1.0.tar.gz osv-cli-1.0/
- mv osv-cli-1.0.tar.gz SOURCES/
- rpmbuild --define "_topdir `pwd`" -ba SPECS/osv-cli.spec
-
-copy-files:
- cp $(CLI_ROOT)/cli $(CLI_DIR)
- cp $(CLI_ROOT)/cli.lua $(CLI_DIR)
- cp $(CLI_ROOT)/lib/*.lua $(CLI_DIR)/lib
- cp -r $(CLI_ROOT)/commands $(CLI_DIR)
- cp -r $(LUA_DIR)/out/share $(CLI_DIR)/lib
- cp -r $(LUA_DIR)/out/lib $(CLI_DIR)/lib
- cp $(LUA_DIR)/src/liblua.so $(CLI_DIR)/lib
-
-clean:
- rm -rf BUILD
- rm -rf BUILDROOT
- rm -rf $(CLI_DIR)
- rm -rf RPMS
- rm -rf SOURCES
- rm -rf SRPMS
-
-.PHONY: module clean
diff --git a/modules/cli/rpmbuild/SPECS/osv-cli.spec b/modules/cli/rpmbuild/SPECS/osv-cli.spec
deleted file mode 100644
index 5aff3bcb..00000000
--- a/modules/cli/rpmbuild/SPECS/osv-cli.spec
+++ /dev/null
@@ -1,54 +0,0 @@
-%define __spec_install_post %{nil}
-%define debug_package %{nil}
-%define __os_install_post %{_dbpath}/brp-compress
-
-Name: osv-cli
-Version: 1.0
-Release: 1%{?dist}
-Summary: A standalon CLI for the OSv
-
-Group: Development/Tools
-License: FreeBSD
-URL: http://cloudius-systems.com/
-Source0: %{name}-%{version}.tar.gz
-
-BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
-Requires: libedit
-Requires: openssl-libs
-Requires: ncurses
-%description
-
-
-%prep
-%setup -q
-
-
-%build
-
-
-%install
-rm -rf %{buildroot}
-mkdir -p %{buildroot}
-
-# in builddir
-cp -a * %{buildroot}
-
-
-%post
-echo "(cd %{_libdir}/osv-cli && exec" './cli $@)' > %{_bindir}/cli
-chmod a+x %{_bindir}/cli
-
-%clean
-rm -rf %{buildroot}
-
-
-%files
-%defattr(-,root,root,-)
-%{_libdir}/*
-%doc
-
-%postun
-rm %{_bindir}/cli
-
-
-%changelog
diff --git a/modules/httpserver-api/Makefile b/modules/httpserver-api/Makefile
index 0f581e6e..30fdc1a1 100644
--- a/modules/httpserver-api/Makefile
+++ b/modules/httpserver-api/Makefile
@@ -46,6 +46,7 @@ module: all

all: lib$(TARGET).so api_api api_app api_env api_file api_fs api_hardware api_network api_os api_trace
$(call quiet, cat _usr_*.manifest | sort | uniq > usr.manifest, CREATE_MANIFEST)
+ $(call very-quiet, $(SRC)/scripts/manifest_from_host.sh lib$(TARGET).so >> usr.manifest)

add_api_to_manifest = \
echo "/usr/mgmt/plugins/libhttpserver-$(1).so: `pwd`/libhttpserver-$(1).so" > _usr_$(1).manifest
diff --git a/modules/httpserver-api/module.py b/modules/httpserver-api/module.py
index 25bc4948..e1da7447 100644
--- a/modules/httpserver-api/module.py
+++ b/modules/httpserver-api/module.py
@@ -13,7 +13,6 @@ usr_files.add(os.path.join(_module, 'api-doc')).to('/usr/mgmt/api')

api.require('openssl')
api.require('libtools')
-api.require('libyaml')

# only require next 3 modules if java (jre) is included in the list of modules
api.require_if_other_module_present('josvsym','java')
diff --git a/modules/libedit/.gitignore b/modules/libedit/.gitignore
deleted file mode 100644
index 5fc3ba00..00000000
--- a/modules/libedit/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-build
-download
diff --git a/modules/libedit/Makefile b/modules/libedit/Makefile
deleted file mode 100644
index 04979144..00000000
--- a/modules/libedit/Makefile
+++ /dev/null
@@ -1,41 +0,0 @@
-# A small Makefile "download and build" for libedit
-
-# Directories for downloading and building
-DOWNLOAD=download
-BUILD=build
-
-# libedit
-LE_VERSION=20140620-3.1
-LE_FOLDER=libedit-$(LE_VERSION)
-LE_DOWNLOAD=http://thrysoee.dk/editline/libedit-$(LE_VERSION).tar.gz
-LE_ARCHIVE=download/$(LE_FOLDER).tar.gz
-LE_BUILD=$(BUILD)/$(LE_FOLDER)
-
-CFLAGS:=-fPIC
-LDFLAGS:=-L$(abspath $(lastword $(wildcard ../ncurses/build/*/lib)))
-
-MAIN=$(LE_BUILD)/src/.libs/libedit.so.0
-
-module: $(MAIN)
-
-$(MAIN): $(LE_BUILD)/Makefile
- cd $(LE_BUILD) && make
-
-$(LE_BUILD)/Makefile: $(LE_BUILD)/configure
- cd $(LE_BUILD) && CFLAGS=$(CFLAGS) LDFLAGS=$(LDFLAGS) ./configure
-
-$(LE_BUILD)/configure: $(LE_ARCHIVE) | $(BUILD)
- cd $(BUILD) && tar xzf ../$(LE_ARCHIVE)
- touch $(LE_BUILD)/configure
-
-$(LE_ARCHIVE): | $(DOWNLOAD)
- cd $(DOWNLOAD) && \
- curl --remote-name --remote-time $(LE_DOWNLOAD)
-
-$(DOWNLOAD) $(BUILD):
- @mkdir -p $@
-
-clean:
- rm -rf $(BUILD) $(DOWNLOAD)
-
-.PHONY: module clean
diff --git a/modules/libedit/module.py b/modules/libedit/module.py
deleted file mode 100644
index 53582563..00000000
--- a/modules/libedit/module.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from osv.modules.filemap import FileMap
-
-VERSION = '20140620-3.1'
-
-usr_files = FileMap()
-usr_files.add('${OSV_BASE}/modules/libedit/build/libedit-%s/src/.libs' % VERSION).to('/usr/lib') \
- .include('lib*.so.?')
diff --git a/modules/lua/.gitignore b/modules/lua/.gitignore
index 0e3f9ff5..0089f1a3 100644
--- a/modules/lua/.gitignore
+++ b/modules/lua/.gitignore
@@ -1,5 +1,3 @@
-build
-download
-out
-src/*
-!src/Makefile
+install
+upstream
+usr.manifest
diff --git a/modules/lua/Makefile b/modules/lua/Makefile
index 620a78f8..91325455 100644
--- a/modules/lua/Makefile
+++ b/modules/lua/Makefile
@@ -1,138 +1,65 @@
-# Some directories
-OUT=out
-BUILD=build
-DOWNLOAD=download
-
-# Official version
-LUA_V=5.2
-
-# Archive details
-LUA_VERSION=5.2.3
-LUA_FOLDER=lua-$(LUA_VERSION)
-LUA_DOWNLOAD=http://www.lua.org/ftp/$(LUA_FOLDER).tar.gz
-LUA_ARCHIVE=download/$(LUA_FOLDER).tar.gz
-
-# Lua sources for dependencies
-LUA_SRCS=lapi.c lapi.h lauxlib.c lauxlib.h lbaselib.c lbitlib.c lcode.c \
- lcode.h lcorolib.c lctype.c lctype.h ldblib.c ldebug.c ldebug.h ldo.c ldo.h \
- ldump.c lfunc.c lfunc.h lgc.c lgc.h linit.c liolib.c llex.c llex.h \
- llimits.h lmathlib.c lmem.c lmem.h loadlib.c lobject.c lobject.h lopcodes.c \
- lopcodes.h loslib.c lparser.c lparser.h lstate.c lstate.h lstring.c \
- lstring.h lstrlib.c ltable.c ltable.h ltablib.c ltm.c ltm.h lua.c luac.c \
- luaconf.h lua.h lualib.h lundump.c lundump.h lvm.c lvm.h lzio.c lzio.h
-
-SRCS=$(addprefix src/, $(LUA_SRCS))
-
-# Out sub-directories
-BDIR=$(OUT)/bin
-LDIR=$(OUT)/lib/lua/$(LUA_V)
-CDIR=$(OUT)/share/lua/$(LUA_V)
-
-# Lua itself
-MAIN=src/liblua.so
-
-# Local executable used for LuaRocks
-LUA_BIN=$(BDIR)/lua
-LUA_ROCKS_BIN=$(BDIR)/luarocks
+SRC = $(shell readlink -f ../..)
+LUA_ROCKS=upstream/luarocks-3.1.1-linux-x86_64/luarocks
+MODULES_DIR=install/lua_modules
+LDIR=install/lua_modules/lib/lua/5.3
+CDIR=install/lua_modules/share/lua/5.3

# List of Lua modules, each module has its own target
LUA_MODULES=LuaSocket LuaJSON Lua_stdlib LuaFileSystem LuaPath LuaSec

-module: $(MAIN) $(LUA_MODULES)
-
-$(MAIN): $(SRCS)
- cd src && $(MAKE)
-
-$(LUA_BIN): $(MAIN) | $(BDIR)
- cp src/lua $(LUA_BIN)
-
-$(LUA_ARCHIVE): | $(DOWNLOAD)
- cd $(DOWNLOAD) && \
- curl --remote-name --remote-time $(LUA_DOWNLOAD)
-
-$(SRCS): $(LUA_ARCHIVE)
- tar -Oxzf $< $(LUA_FOLDER)/$@ > $@
-
-$(DOWNLOAD) $(BUILD) $(LDIR) $(CDIR) $(BDIR):
- @mkdir -p $@
-
-# == LuaRocks ==
-LuaRocks_V=2.1.2
-LuaRocks_F=luarocks-$(LuaRocks_V)
-LuaRocks_A=$(LuaRocks_F).tar.gz
-LuaRocks_D=http://luarocks.org/releases/$(LuaRocks_A)
-
-$(LUA_ROCKS_BIN): $(LUA_BIN) $(BUILD)/$(LuaRocks_F)/src/luarocks
- cd $(BUILD)/$(LuaRocks_F) && \
- make install
-
-$(BUILD)/$(LuaRocks_F)/src/luarocks: $(BUILD)/$(LuaRocks_F)/Makefile
- cd $(BUILD)/$(LuaRocks_F) && \
- make build
-
-$(BUILD)/$(LuaRocks_F)/Makefile: $(BUILD)/$(LuaRocks_F)/configure
- $(eval PREFIX=$(shell pwd))
- cd $(BUILD)/$(LuaRocks_F) && \
- ./configure --prefix=$(PREFIX)/$(OUT) --force-config \
- --with-lua-include=$(PREFIX)/src --lua-version=$(LUA_V) \
- --with-lua=$(PREFIX)/$(OUT)
- touch $@
+LUA_LIBRARY := $(shell ldconfig -p | grep -Po "liblua*.5\.3.so.0" | head -1)
+ifndef LUA_LIBRARY
+ LUA_LIBRARY := $(shell ldconfig -p | grep -Po "liblua*.5\.3.so" | head -1)
+endif

-$(BUILD)/$(LuaRocks_F)/configure: $(DOWNLOAD)/$(LuaRocks_A) | $(BUILD)
- tar --directory $(BUILD) -xzf $<
- touch $@
+module: $(LUA_MODULES)
+ mkdir -p $(MODULES_DIR)
+ $(SRC)/scripts/manifest_from_host.sh -l $(LUA_LIBRARY) > usr.manifest

-$(DOWNLOAD)/$(LuaRocks_A): | $(DOWNLOAD)
- curl --remote-time --location $(LuaRocks_D) > $@
+$(LUA_ROCKS):
+ mkdir -p upstream
+ cd upstream && wget -c https://luarocks.github.io/luarocks/releases/luarocks-3.1.1-linux-x86_64.zip
+ cd upstream && unzip luarocks-3.1.1-linux-x86_64.zip

# == LuaSocket ==
LuaSocket: $(LDIR)/socket/core.so

-$(LDIR)/socket/core.so: $(LUA_ROCKS_BIN)
- out/bin/luarocks install luasocket 3.0-rc1
+$(LDIR)/socket/core.so: $(LUA_ROCKS)
+ $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) luasocket 3.0rc1-2

# == LuaJSON ==
LuaJSON: $(CDIR)/json.lua

-$(CDIR)/json.lua: $(LUA_ROCKS_BIN)
- out/bin/luarocks install luajson 1.3.3
+$(CDIR)/json.lua: $(LUA_ROCKS)
+ $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) luajson 1.3.4-1

# == Lua_stdlib ==
Lua_stdlib: $(CDIR)/std.lua

-$(CDIR)/std.lua: $(LUA_ROCKS_BIN)
- out/bin/luarocks install stdlib 40
+$(CDIR)/std.lua: $(LUA_ROCKS)
+ $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) stdlib 41.2.2-1

# == LuaFileSystem ==
LuaFileSystem: $(LDIR)/lfs.so

-$(LDIR)/lfs.so: $(LUA_ROCKS_BIN)
- out/bin/luarocks install LuaFileSystem 1.6.2
+$(LDIR)/lfs.so: $(LUA_ROCKS)
+ $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) LuaFileSystem 1.7.0-2

# == LuaPath ==
LuaPath: $(CDIR)/path.lua

-$(CDIR)/path.lua: $(LUA_ROCKS_BIN)
- out/bin/luarocks install lua-path 0.2.1
+$(CDIR)/path.lua: $(LUA_ROCKS)
+ $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) lua-path 0.3.1-1

# == LuaSec ==
LuaSec: $(CDIR)/ssl.lua

-$(CDIR)/ssl.lua: $(LUA_ROCKS_BIN)
- ./check-openssl-version
-
-# Workaround because LuaRocks ignores /lib64
-ifneq ("$(wildcard /usr/lib64/libssl.so*)", "")
- out/bin/luarocks install LuaSec 0.5 OPENSSL_LIBDIR=/usr/lib64
-else
- out/bin/luarocks install LuaSec 0.5
-endif
+$(CDIR)/ssl.lua: $(LUA_ROCKS)
+ $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) LuaSec 0.8-1

-##
clean:
- rm -f $(SRCS)
- rm -rf $(DOWNLOAD) $(BUILD) $(OUT)
- cd src && $(MAKE) clean
+ rm -rf install
+ rm -rf upstream

.PHONY: module clean $(LUA_MODULES)
.NOTPARALLEL: $(LUA_MODULES)
diff --git a/modules/lua/check-openssl-version b/modules/lua/check-openssl-version
deleted file mode 100755
index 90ae479a..00000000
--- a/modules/lua/check-openssl-version
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-
-# This is here to help cope with the transition period that the distro
-# use for openssl-1.0 to openssl-1.1
-if [ -f /usr/include/openssl/opensslv.h ]; then
- has_new_openssl=$(grep "OpenSSL 1.1" /usr/include/openssl/opensslv.h)
-fi
-if [ "$has_new_openssl"x != "x" ]; then
- echo "OpenSSL 1.1 header detected."
- echo "On Fedora >= 26 please use the compat headers by doing:"
- echo ""
- echo " dnf install --allowerasing compat-openssl10-devel -y"
- echo ""
- exit 1
-fi
-exit 0
diff --git a/modules/lua/module.py b/modules/lua/module.py
index b15ebbc2..721454a6 100644
--- a/modules/lua/module.py
+++ b/modules/lua/module.py
@@ -1,10 +1,6 @@
from osv.modules.filemap import FileMap

usr_files = FileMap()
-usr_files.add('${OSV_BASE}/modules/lua/src/liblua.so').to('/usr/lib/liblua.so')
-
-usr_files.add('${OSV_BASE}/modules/lua/out').to('/usr') \
- .exclude('bin/*') \
- .exclude('etc/*') \
+usr_files.add('${OSV_BASE}/modules/lua/install/lua_modules/').to('/usr') \
.exclude('lib/luarocks/**') \
.exclude('share/lua/*/luarocks/**')
diff --git a/modules/lua/src/Makefile b/modules/lua/src/Makefile
deleted file mode 100644
index c6ed82bb..00000000
--- a/modules/lua/src/Makefile
+++ /dev/null
@@ -1,119 +0,0 @@
-# == MODIFIED VERSION OF ORIGINAL LUA MAKEFILE ==
-
-CC= gcc
-CFLAGS=-O2 -Wall -fPIC -DLUA_COMPAT_ALL -DLUA_USE_DLOPEN -DLUA_USE_STRTODHEX \
- -DLUA_USE_AFORMAT -DLUA_USE_LONGLONG -DLUA_USE_POSIX
-LDFLAGS=
-LIBS= -lm -ldl
-
-AR= ar rcu
-RANLIB= ranlib
-RM= rm -f
-
-LUA_A= liblua.a
-CORE_O= lapi.o lcode.o lctype.o ldebug.o ldo.o ldump.o lfunc.o lgc.o llex.o \
- lmem.o lobject.o lopcodes.o lparser.o lstate.o lstring.o ltable.o \
- ltm.o lundump.o lvm.o lzio.o
-LIB_O= lauxlib.o lbaselib.o lbitlib.o lcorolib.o ldblib.o liolib.o \
- lmathlib.o loslib.o lstrlib.o ltablib.o loadlib.o linit.o
-BASE_O= $(CORE_O) $(LIB_O)
-
-LUA_T= lua
-LUA_O= lua.o
-
-LUA_SO= liblua.so
-
-ALL_O= $(BASE_O) $(LUA_O)
-ALL_T= $(LUA_SO) $(LUA_A) $(LUA_T)
-
-all: $(ALL_T)
-
-o: $(ALL_O)
-
-$(LUA_A): $(BASE_O)
- $(AR) $@ $(BASE_O)
- $(RANLIB) $@
-
-$(LUA_T): $(LUA_O) $(LUA_A)
- $(CC) -o $@ $(LDFLAGS) $(LUA_O) $(LUA_A) $(LIBS)
-
-$(LUA_SO): $(ALL_O)
- $(CC) $(CFLAGS) -shared -o $@ $(LDFLAGS) $? $(LIBS)
-
-clean:
- $(RM) $(ALL_T) $(ALL_O)
-
-depend:
- @$(CC) $(CFLAGS) -MM l*.c
-
-echo:
- @echo "CC= $(CC)"
- @echo "CFLAGS= $(CFLAGS)"
- @echo "LDFLAGS= $(SYSLDFLAGS)"
- @echo "LIBS= $(LIBS)"
- @echo "RM= $(RM)"
-
-# list targets that do not create files (but not all makes understand .PHONY)
-.PHONY: all o clean depend echo
-
-# DO NOT DELETE
-
-lapi.o: lapi.c lua.h luaconf.h lapi.h llimits.h lstate.h lobject.h ltm.h \
- lzio.h lmem.h ldebug.h ldo.h lfunc.h lgc.h lstring.h ltable.h lundump.h \
- lvm.h
-lauxlib.o: lauxlib.c lua.h luaconf.h lauxlib.h
-lbaselib.o: lbaselib.c lua.h luaconf.h lauxlib.h lualib.h
-lbitlib.o: lbitlib.c lua.h luaconf.h lauxlib.h lualib.h
-lcode.o: lcode.c lua.h luaconf.h lcode.h llex.h lobject.h llimits.h \
- lzio.h lmem.h lopcodes.h lparser.h ldebug.h lstate.h ltm.h ldo.h lgc.h \
- lstring.h ltable.h lvm.h
-lcorolib.o: lcorolib.c lua.h luaconf.h lauxlib.h lualib.h
-lctype.o: lctype.c lctype.h lua.h luaconf.h llimits.h
-ldblib.o: ldblib.c lua.h luaconf.h lauxlib.h lualib.h
-ldebug.o: ldebug.c lua.h luaconf.h lapi.h llimits.h lstate.h lobject.h \
- ltm.h lzio.h lmem.h lcode.h llex.h lopcodes.h lparser.h ldebug.h ldo.h \
- lfunc.h lstring.h lgc.h ltable.h lvm.h
-ldo.o: ldo.c lua.h luaconf.h lapi.h llimits.h lstate.h lobject.h ltm.h \
- lzio.h lmem.h ldebug.h ldo.h lfunc.h lgc.h lopcodes.h lparser.h \
- lstring.h ltable.h lundump.h lvm.h
-ldump.o: ldump.c lua.h luaconf.h lobject.h llimits.h lstate.h ltm.h \
- lzio.h lmem.h lundump.h
-lfunc.o: lfunc.c lua.h luaconf.h lfunc.h lobject.h llimits.h lgc.h \
- lstate.h ltm.h lzio.h lmem.h
-lgc.o: lgc.c lua.h luaconf.h ldebug.h lstate.h lobject.h llimits.h ltm.h \
- lzio.h lmem.h ldo.h lfunc.h lgc.h lstring.h ltable.h
-linit.o: linit.c lua.h luaconf.h lualib.h lauxlib.h
-liolib.o: liolib.c lua.h luaconf.h lauxlib.h lualib.h
-llex.o: llex.c lua.h luaconf.h lctype.h llimits.h ldo.h lobject.h \
- lstate.h ltm.h lzio.h lmem.h llex.h lparser.h lstring.h lgc.h ltable.h
-lmathlib.o: lmathlib.c lua.h luaconf.h lauxlib.h lualib.h
-lmem.o: lmem.c lua.h luaconf.h ldebug.h lstate.h lobject.h llimits.h \
- ltm.h lzio.h lmem.h ldo.h lgc.h
-loadlib.o: loadlib.c lua.h luaconf.h lauxlib.h lualib.h
-lobject.o: lobject.c lua.h luaconf.h lctype.h llimits.h ldebug.h lstate.h \
- lobject.h ltm.h lzio.h lmem.h ldo.h lstring.h lgc.h lvm.h
-lopcodes.o: lopcodes.c lopcodes.h llimits.h lua.h luaconf.h
-loslib.o: loslib.c lua.h luaconf.h lauxlib.h lualib.h
-lparser.o: lparser.c lua.h luaconf.h lcode.h llex.h lobject.h llimits.h \
- lzio.h lmem.h lopcodes.h lparser.h ldebug.h lstate.h ltm.h ldo.h lfunc.h \
- lstring.h lgc.h ltable.h
-lstate.o: lstate.c lua.h luaconf.h lapi.h llimits.h lstate.h lobject.h \
- ltm.h lzio.h lmem.h ldebug.h ldo.h lfunc.h lgc.h llex.h lstring.h \
- ltable.h
-lstring.o: lstring.c lua.h luaconf.h lmem.h llimits.h lobject.h lstate.h \
- ltm.h lzio.h lstring.h lgc.h
-lstrlib.o: lstrlib.c lua.h luaconf.h lauxlib.h lualib.h
-ltable.o: ltable.c lua.h luaconf.h ldebug.h lstate.h lobject.h llimits.h \
- ltm.h lzio.h lmem.h ldo.h lgc.h lstring.h ltable.h lvm.h
-ltablib.o: ltablib.c lua.h luaconf.h lauxlib.h lualib.h
-ltm.o: ltm.c lua.h luaconf.h lobject.h llimits.h lstate.h ltm.h lzio.h \
- lmem.h lstring.h lgc.h ltable.h
-lua.o: lua.c lua.h luaconf.h lauxlib.h lualib.h
-luac.o: luac.c lua.h luaconf.h lauxlib.h lobject.h llimits.h lstate.h \
- ltm.h lzio.h lmem.h lundump.h ldebug.h lopcodes.h
-lundump.o: lundump.c lua.h luaconf.h ldebug.h lstate.h lobject.h \
- llimits.h ltm.h lzio.h lmem.h ldo.h lfunc.h lstring.h lgc.h lundump.h
-lvm.o: lvm.c lua.h luaconf.h ldebug.h lstate.h lobject.h llimits.h ltm.h \
- lzio.h lmem.h ldo.h lfunc.h lgc.h lopcodes.h lstring.h ltable.h lvm.h
-lzio.o: lzio.c lua.h luaconf.h llimits.h lmem.h lstate.h lobject.h ltm.h \
- lzio.h
diff --git a/modules/openssl/.gitignore b/modules/openssl/.gitignore
new file mode 100644
index 00000000..f9235a6b
--- /dev/null
+++ b/modules/openssl/.gitignore
@@ -0,0 +1 @@
+usr.manifest
diff --git a/modules/openssl/Makefile b/modules/openssl/Makefile
new file mode 100644
index 00000000..fd1617bb
--- /dev/null
+++ b/modules/openssl/Makefile
@@ -0,0 +1,23 @@
+#
+# Copyright (C) 2019 Waldemar Kozaczuk, Ltd.
+#
+# This work is open source software, licensed under the terms of the
+# BSD license as described in the LICENSE file in the top-level directory.
+#
+SRC = $(shell readlink -f ../..)
+
+module:
+ $(SRC)/scripts/manifest_from_host.sh -l libssl.so.1.1 > usr.manifest
+ # From krb5-libs
+ #/&/etc/krb5.conf: %(miscbase)s/&
+ $(SRC)/scripts/manifest_from_host.sh -l libgssapi_krb5.so.2 >> usr.manifest
+ # From libselinux
+ $(SRC)/scripts/manifest_from_host.sh -l libselinux.so.1 >> usr.manifest
+ # From xz-libs
+ $(SRC)/scripts/manifest_from_host.sh -l liblzma.so.5 >> usr.manifest
+
+.PHONY: module
+
+clean:
+ rm -rf usr.manifest
+.PHONY: clean
diff --git a/modules/openssl/usr.manifest b/modules/openssl/usr.manifest
deleted file mode 100644
index e9d13ad3..00000000
--- a/modules/openssl/usr.manifest
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Copyright (C) 2014 Cloudius Systems, Ltd.
-#
-# This work is open source software, licensed under the terms of the
-# BSD license as described in the LICENSE file in the top-level directory.
-#
-
-# From openssl-libs
-/usr/lib/openssl/**: %(miscbase)s/usr/lib64/openssl/**
-#/usr/lib/&/libssl.so.1.0.1k: %(miscbase)s/usr/lib64/&
-/usr/lib/&/libssl.so.10: %(miscbase)s/usr/lib64/&
-#/usr/lib/&/libcrypto.so.1.0.1k: %(miscbase)s/usr/lib64/&
-/usr/lib/&/libcrypto.so.10: %(miscbase)s/usr/lib64/&
-
-# From libcom_err
-/usr/lib/&/libcom_err.so.2: %(miscbase)s/usr/lib64/&
-#/usr/lib/&/libcom_err.so.2.1: %(miscbase)s/usr/lib64/&
-
-# From krb5-libs
-/&/etc/krb5.conf: %(miscbase)s/&
-/usr/lib/&/libgssapi_krb5.so.2: %(miscbase)s/usr/lib64/&
-/usr/lib/&/libk5crypto.so.3: %(miscbase)s/usr/lib64/&
-/usr/lib/&/libkrb5.so.3: %(miscbase)s/usr/lib64/&
-#/usr/lib/&/libkrb5.so.3.3: %(miscbase)s/usr/lib64/&
-/usr/lib/&/libkrb5support.so.0: %(miscbase)s/usr/lib64/&
-
-# From libselinux
-/usr/lib/&/libselinux.so.1: %(miscbase)s/usr/lib64/&
-
-# From keyutils-libs
-/usr/lib/&/libkeyutils.so.1: %(miscbase)s/usr/lib64/&
-
-# From pcre
-/usr/lib/&/libpcre.so.1: %(miscbase)s/usr/lib64/&
-
-# From xz-libs
-/usr/lib/&/liblzma.so.5: %(miscbase)s/usr/lib64/&
diff --git a/modules/terminfo/Makefile b/modules/terminfo/Makefile
new file mode 100644
index 00000000..ad4fbb81
--- /dev/null
+++ b/modules/terminfo/Makefile
@@ -0,0 +1,16 @@
+OUT=out
+TI_VT100=$(OUT)/terminfo/v/vt100-qemu
+
+module: $(TI_VT100)
+
+$(TI_VT100): $(MAIN)
+ @mkdir -p $(OUT)/terminfo
+ infocmp vt100 | sed 's/cud1=$^J\|\\n$,/cud1=\\E[B,/; s/ind=$^J\|\\n$,/ind=\\E[S,/; s/vt100/vt100-qemu/' \
+ > $(OUT)/terminfo.tmp
+ tic -o $(OUT)/terminfo $(OUT)/terminfo.tmp
+ rm -f $(OUT)/terminfo.tmp
+
+clean:
+ rm -rf $(OUT)
+
+.PHONY: module clean
diff --git a/modules/terminfo/module.py b/modules/terminfo/module.py
new file mode 100644
index 00000000..a422d75c
--- /dev/null
+++ b/modules/terminfo/module.py
@@ -0,0 +1,4 @@
+from osv.modules.filemap import FileMap
+
+usr_files = FileMap()
+usr_files.add('${OSV_BASE}/modules/terminfo/out/terminfo').to('/usr/share/terminfo')
diff --git a/scripts/setup.py b/scripts/setup.py
index de76800c..211b40de 100755
--- a/scripts/setup.py
+++ b/scripts/setup.py
@@ -46,7 +46,7 @@ class Fedora(object):
'ncurses',
'ncurses-devel',
'openssl',
- 'openssl-libs',
+ 'openssl-libs.x86_64',
'p11-kit',
'patch',
'python-dpkt',
@@ -56,41 +56,42 @@ class Fedora(object):
'unzip',
'wget',
'yaml-cpp-devel',
+ 'pax-utils',
]
ec2_packages = standard_ec2_packages
test_packages = ['openssl-devel']
ec2_post_install = standard_ec2_post_install

class Fedora_25(object):
- packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel']
+ packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel.x86_64', 'lua-5.3.4', 'lua-devel-5.3.4']
ec2_packages = []
test_packages = []
ec2_post_install = None
version = '25'

class Fedora_26(object):
- packages = ['java-1.8.0-openjdk', 'python2-requests', 'compat-openssl10-devel']
+ packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel.x86_64', 'lua-5.3.4', 'lua-devel-5.3.4']
ec2_packages = []
test_packages = []
ec2_post_install = None
version = '26'

class Fedora_27(object):
- packages = ['java-1.8.0-openjdk', 'python2-requests', 'compat-openssl10-devel']
+ packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel.x86_64', 'lua-5.3.4', 'lua-devel-5.3.4']
ec2_packages = []
test_packages = []
ec2_post_install = None
version = '27'

class Fedora_28(object):
- packages = ['java-1.8.0-openjdk', 'python2-requests', 'compat-openssl10-devel']
+ packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel.x86_64', 'lua-5.3.4', 'lua-devel-5.3.4']
ec2_packages = []
test_packages = []
ec2_post_install = None
version = '28'

class Fedora_29(object):
- packages = ['java-1.8.0-openjdk', 'python2-requests', 'compat-openssl10-devel']
+ packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel.x86_64', 'lua-5.3.5', 'lua-devel-5.3.5']
ec2_packages = []
test_packages = []
ec2_post_install = None
@@ -220,6 +221,9 @@ class Ubuntu(object):
'tcpdump',
'unzip',
'wget',
+ 'pax-utils',
+ 'lua5.3',
+ 'liblua5.3',
]

ec2_packages = standard_ec2_packages
--
2.20.1

Waldek Kozaczuk

unread,

Aug 1, 2019, 5:49:09 PM8/1/19

to OSv Development

I am not 100% sure the "setup.py" part of this patch will work correctly on the build machine that uses Fedora 27. I did test the upgrade scenario on Fedora 27 using Docker-based installation so in theory it should all work. But who knows.

Waldemar Kozaczuk

unread,

Aug 4, 2019, 11:33:46 AM8/4/19

to osv...@googlegroups.com, Waldemar Kozaczuk

As the issue #1022 explains, building the default OSv image that
implicitly adds cli module that in turn implicitly pulls httpserver-api
and cli modules, requires some bizarre workarounds on modern Linux distribution.
This is caused by the fact that the lua module is based on Lua 5.2 and older version
of LuaSec (OpenSSL library for Lua) which requires old version of OpenSSL 1.0.

This patch updates following modules:
1) cli:
- makes necessary changes to work with Lua 5.3
- drops explicit dependency on the modules ncurses and libedit in lieu of
new terminfo and the script manifest_from_host.sh that pulls all necessary
dependencies from host

- creates common cli executable that can be used on both host and OSv side

2) httpserver-api
- drops explicit dependency on the module libyaml in lieu of using
manifest_from_host.sh that pulls all necessary dependencies from host

3) lua
- revamps this module completely by upgrading to Lua 5.3
- extensively employs manifest_from_host.sh to pull Lua libraries
from host (see changes to setup.py) and minimize building from source
- updates Lua modules (like LuaSec) to the newest versions

4) openssl
- instead of using hardcoded usr.manifest, new Makefile employs manifest_from_host.sh
to find relevant main OpenSSL 1.1 library and its required dependencies from host

The patch also deletes the obsolete module libedit in lieu of using relevant files from host.

Finally it adds new terminfo module.

This patch has been tested with following Linux distribution using Docker:
- Ubuntu 18.10 (OpenSSL 1.0 -> OpenSSL 1.1 upgrade)
- Ubuntu 19.04
- Fedora 27 (OpenSSL 1.0 -> OpenSSL 1.1 upgrade)
- Ubuntu 29

Fixes #1022

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---

modules/httpserver-api/Makefile | 1 +
modules/httpserver-api/module.py | 1 -
modules/libedit/.gitignore | 2 -
modules/libedit/Makefile | 41 -------

modules/libedit/module.py | 7 --
modules/lua/.gitignore | 8 +-

modules/lua/Makefile | 133 +++++----------------

modules/lua/check-openssl-version | 16 ---
modules/lua/module.py | 6 +-
modules/lua/src/Makefile | 119 ------------------

modules/openssl/.gitignore | 1 +
modules/openssl/Makefile | 23 ++++
modules/openssl/usr.manifest | 37 ------

modules/terminfo/Makefile | 16 +++
modules/terminfo/module.py | 4 +

modules/terminfo/out/terminfo/v/vt100-am | 1 +
modules/terminfo/out/terminfo/v/vt100-qemu | Bin 0 -> 1184 bytes
scripts/setup.py | 16 ++-
24 files changed, 151 insertions(+), 401 deletions(-)

delete mode 100644 modules/libedit/.gitignore
delete mode 100644 modules/libedit/Makefile
delete mode 100644 modules/libedit/module.py
delete mode 100755 modules/lua/check-openssl-version
delete mode 100644 modules/lua/src/Makefile
create mode 100644 modules/openssl/.gitignore
create mode 100644 modules/openssl/Makefile
delete mode 100644 modules/openssl/usr.manifest
create mode 100644 modules/terminfo/Makefile
create mode 100644 modules/terminfo/module.py

create mode 120000 modules/terminfo/out/terminfo/v/vt100-am
create mode 100644 modules/terminfo/out/terminfo/v/vt100-qemu

diff --git a/modules/cli/.gitignore b/modules/cli/.gitignore
index 773de15c..64545030 100644
--- a/modules/cli/.gitignore
+++ b/modules/cli/.gitignore
@@ -1,2 +1,2 @@
cli
-cli.so

+usr.manifest
diff --git a/modules/cli/Makefile b/modules/cli/Makefile

index e2abe632..857aae93 100644
--- a/modules/cli/Makefile
+++ b/modules/cli/Makefile
@@ -1,34 +1,27 @@

+LUA_LIB_NAME = $(shell ldconfig -p | grep -Po "liblua*.5\.3.so" | grep -o "lua.*5.3" | head -1)
+LUA_INCLUDES = $(shell $(CXX) -E -xc++ - -v </dev/null 2>&1 | awk '/^End/ {exit} /^ \/.*include/ {print $$0}' | grep -vP 'c\+\+|gnu' | xargs -I '{}' find {} -name lualib.h | xargs dirname | awk '// { printf("-I%s\n", $$0)}')
+
CC=gcc
CFLAGS=-O2 -g -Wall -std=gnu99
-INCLUDES=-I../lua/src \
- -I$(lastword $(wildcard ../ncurses/build/*/include)) \
- -I$(lastword $(wildcard ../libedit/build/*/src))
-LFLAGS=-L../lua/src \
- -L$(lastword $(wildcard ../ncurses/build/*/lib)) \
- -L$(lastword $(wildcard ../libedit/build/*/src/.libs))
-LIBS=-ledit -ltinfo -llua
+LIBS=-ledit -ltinfo -l$(LUA_LIB_NAME)
SRCS=cli.c

-MAIN=cli.so
+MAIN=cli
+
+INCLUDES = $(LUA_INCLUDES)

-# For compiling executable running locally
-LOC_INCLUDES=-I../lua/src
-LOC_LFLAGS=-L../lua/src
-LOC_LDFLAGS=-Wl,-rpath,$(abspath ../lua/src)
-LOC_MAIN=cli

+SRC = $(shell readlink -f ../..)

module: $(MAIN)

+ $(SRC)/scripts/manifest_from_host.sh cli > usr.manifest

$(MAIN): $(SRCS)
- $(CC) -DOSV_CLI $(CFLAGS) $(INCLUDES) -fPIC -shared -o $@ $(SRCS) $(LFLAGS) $(LIBS)

+ $(CC) $(CFLAGS) $(INCLUDES) $^ -fPIC -pie -o $@ $(LIBS)

-$(LOC_MAIN): $(SRCS)
- $(CC) $(CFLAGS) $(LOC_INCLUDES) $^ -o $@ $(LOC_LDFLAGS) $(LOC_LFLAGS) $(LIBS)

+rpm: $(MAIN)
+ make -C rpmbuild

clean:
- rm -f $(MAIN) $(LOC_MAIN)

+ rm -f $(MAIN)
make -f rpmbuild/Makefile clean

-rpm: $(LOC_MAIN)
- make -C rpmbuild

-

-.PHONY: module clean rpm
+.PHONY: module clean
diff --git a/modules/cli/cli.c b/modules/cli/cli.c

index 01c6a1e8..68dfbba0 100644

--- a/modules/cli/cli.c
+++ b/modules/cli/cli.c
@@ -5,6 +5,7 @@
#include <signal.h>
#include <getopt.h>
#include <signal.h>
+#include <gnu/libc-version.h>

#include <lua.h>
#include <lauxlib.h>
@@ -16,17 +17,15 @@
#include <sys/poll.h>
#include <termios.h>

-#ifdef OSV_CLI
-#define CLI_LUA "/cli/cli.lua"
-#define CLI_LUA_PATH "/usr/share/lua/5.2/?.lua;/cli/lib/?.lua;/cli/lua/share/?.lua"
-#define CLI_LUA_CPATH "/usr/lib/lua/5.2/?.so;/cli/lua/lib/?.so"
-#define CLI_COMMANDS_PATH "/cli/commands"
-#else
-#define CLI_LUA "cli.lua"
-#define CLI_LUA_PATH "lib/?.lua;../lua/out/share/lua/5.2/?.lua;./lib/share/lua/5.2/?.lua"
-#define CLI_LUA_CPATH "../lua/out/lib/lua/5.2/?.so;./lib/lib/lua/5.2/?.so"
-#define CLI_COMMANDS_PATH "./commands"
-#endif

+#define OSV_CLI_LUA "/cli-app/cli.lua"
+#define OSV_CLI_LUA_PATH "/usr/share/lua/5.3/?.lua;/usr/share/lua/5.3/?/init.lua;/cli-app/lib/?.lua;/cli-app/lua/share/?.lua"
+#define OSV_CLI_LUA_CPATH "/usr/lib/lua/5.3/?.so;/cli-app/lua/lib/?.so"
+#define OSV_CLI_COMMANDS_PATH "/cli-app/commands"

index e38aa2db..aec10615 100644

--- a/modules/cli/module.py
+++ b/modules/cli/module.py
@@ -3,16 +3,14 @@ from osv.modules.filemap import FileMap
from osv.modules import api

require('lua')
-require('ncurses')
-require('libedit')
+require('terminfo')
require_running('httpserver')

usr_files = FileMap()

-usr_files.add('${OSV_BASE}/modules/cli').to('/cli') \
- .include('cli.so') \
+usr_files.add('${OSV_BASE}/modules/cli').to('/cli-app') \

.include('cli.lua') \
.include('lib/**') \
.include('commands/**')

-full = api.run('/cli/cli.so')

+full = api.run('/cli')
default = full
diff --git a/modules/cli/rpmbuild/Makefile b/modules/cli/rpmbuild/Makefile
index be9915d6..c91d30f6 100644
--- a/modules/cli/rpmbuild/Makefile
+++ b/modules/cli/rpmbuild/Makefile
@@ -5,7 +5,7 @@ VERSION=1.0

MAIN=rpm
CLI_DIR=osv-cli-$(VERSION)/usr/lib64/osv-cli
-LOC_MAIN=cli
+LUA_LIB_PATH = $(shell ldconfig -p | grep -Po "/.*liblua*.5\.3.so" | head -1)

module: clean all

@@ -30,9 +30,9 @@ copy-files:
cp $(CLI_ROOT)/cli.lua $(CLI_DIR)
cp $(CLI_ROOT)/lib/*.lua $(CLI_DIR)/lib

cp -r $(CLI_ROOT)/commands $(CLI_DIR)
- cp -r $(LUA_DIR)/out/share $(CLI_DIR)/lib
- cp -r $(LUA_DIR)/out/lib $(CLI_DIR)/lib
- cp $(LUA_DIR)/src/liblua.so $(CLI_DIR)/lib

+ cp -r $(LUA_DIR)/install/lua_modules/share $(CLI_DIR)/lib
+ cp -r $(LUA_DIR)/install/lua_modules/lib $(CLI_DIR)/lib
+ cp $(LUA_LIB_PATH) $(CLI_DIR)/lib

clean:
rm -rf BUILD

diff --git a/modules/terminfo/out/terminfo/v/vt100-am b/modules/terminfo/out/terminfo/v/vt100-am
new file mode 120000
index 00000000..7de1ca04
--- /dev/null
+++ b/modules/terminfo/out/terminfo/v/vt100-am
@@ -0,0 +1 @@
+vt100-qemu
\ No newline at end of file
diff --git a/modules/terminfo/out/terminfo/v/vt100-qemu b/modules/terminfo/out/terminfo/v/vt100-qemu
new file mode 100644
index 0000000000000000000000000000000000000000..d228a60e1d29ebd2bee4c3263d27c32c2bb7427b
GIT binary patch
literal 1184
zcmdT?JBt%h6h3zpcNI*kWp$?sk}I1+&E(;%Ox%!Ni=Zxv4+M$mhD@HDNha^dX1B4m
z6<bS7u~<++K|w)5!NS7A!otGB($e_dJCofF`Ukw^JLfy!dCZ+NSzsE?(+qjo38mZC
zg27Xbyww>wG;58_e9k8W-(v4f*6e0$xxDG?md$)UK*Z#aoPz*|5lho79iy>42jGbm
zI!JT0Kqu+6)OET{X|U@ux&eDzGN-3NC2H)8$KxU0rF&5B(*sw3NROr}oX+w1@jms5
z%XjD*y`We0hThQ!`b3}U3w?ES-(3IQ^*`t*{R00DDC`hB!X}fay~ow(nGY=BMj@uN
znB)m*Z^GVUcUX>9*bewpsftO@*-Q4Cy=CvE@^M#JlvzdOJ<<xccjo?~?PdQzc3hdE
zqoihdncEsS4Q_|H*@O@-!X(wqMSm<BTn4}7kA;Qb6{cE}C==Ng(S`q>npyS7LPAMM
z?*WTiuB1(f{uhZjiWlXU*DaqEIUci9cuHyjq=7zg1-J@a1KL0b=mJ%s2V4MFfdt0J
zwW=!a71v7Yd9n){5CXzL1iIXVcnwL==@7I#M9c`5VIsmrVPfhs#)XZD5v_vg#!^Ym
zV24Yj>L|gb*4*s`d0b4j&+-v(8yYu690H9v6sfRG=}sI$D#{H@l%3;k56^qK;cO*I
zhw(xUpatM*DI2KK`g%57C=`p8%Em^uYFV{fy>8o$Mzh&!wcDLex7+LW`-8!7xVbqR
bZEbCD<BD;;BrW!}xS|Gvn6w5#bH4c>xsE~Q

literal 0
HcmV?d00001

Nadav Har'El

unread,

Aug 6, 2019, 10:13:15 AM8/6/19

to Waldemar Kozaczuk, Osv Dev

Hi, thanks!

I have some comments and questions inline below:

On Sun, Aug 4, 2019 at 6:33 PM Waldemar Kozaczuk <jwkoz...@gmail.com> wrote:

As the issue #1022 explains, building the default OSv image that
implicitly adds cli module that in turn implicitly pulls httpserver-api
and cli modules, requires some bizarre workarounds on modern Linux distribution.
This is caused by the fact that the lua module is based on Lua 5.2 and older version
of LuaSec (OpenSSL library for Lua) which requires old version of OpenSSL 1.0.

This patch updates following modules:
1) cli:
- makes necessary changes to work with Lua 5.3
- drops explicit dependency on the modules ncurses and libedit in lieu of
new terminfo and the script manifest_from_host.sh that pulls all necessary
dependencies from host

Was this change necessary? Does the new Lua not work with the older ncurses or libedit

we had?

In general I see below several places where you're changing over from building our own libraries and

using them to using the ones from the host, and I wonder if these changes are necessary or

related to this patch, or just something you wanted to do for other reasons?

I agree that on new packages, it's easier to just take stuff from the host. But why stop using

things we already know how to build?

For openssl itself, it's different - you didn't switch from compiling it to taking it from the host -

you switched (if I understand correctly...) from taking it from external/ to taking it from the host.

- creates common cli executable that can be used on both host and OSv side

Nice.

Would be nice to have some more elaborate error checking for missing lualib.h - I for example didn't have it installed (I guess I need to install a lua-devel package?).

But why do you need this LUA_INCLUDES at all? If Lua is already installed on the host system, you don't need any special -I at all, no?

+
CC=gcc
CFLAGS=-O2 -g -Wall -std=gnu99
-INCLUDES=-I../lua/src \
- -I$(lastword $(wildcard ../ncurses/build/*/include)) \
- -I$(lastword $(wildcard ../libedit/build/*/src))
-LFLAGS=-L../lua/src \
- -L$(lastword $(wildcard ../ncurses/build/*/lib)) \
- -L$(lastword $(wildcard ../libedit/build/*/src/.libs))
-LIBS=-ledit -ltinfo -llua
+LIBS=-ledit -ltinfo -l$(LUA_LIB_NAME)
SRCS=cli.c
-MAIN=cli.so
+MAIN=cli
+
+INCLUDES = $(LUA_INCLUDES)

-# For compiling executable running locally
-LOC_INCLUDES=-I../lua/src
-LOC_LFLAGS=-L../lua/src
-LOC_LDFLAGS=-Wl,-rpath,$(abspath ../lua/src)
-LOC_MAIN=cli
+SRC = $(shell readlink -f ../..)

module: $(MAIN)
+ $(SRC)/scripts/manifest_from_host.sh cli > usr.manifest

Maybe use $(MAIN) instead of "cli" here would make it clearer that it's the same thing.

$(MAIN): $(SRCS)
- $(CC) -DOSV_CLI $(CFLAGS) $(INCLUDES) -fPIC -shared -o $@ $(SRCS) $(LFLAGS) $(LIBS)
+ $(CC) $(CFLAGS) $(INCLUDES) $^ -fPIC -pie -o $@ $(LIBS)

-$(LOC_MAIN): $(SRCS)
- $(CC) $(CFLAGS) $(LOC_INCLUDES) $^ -o $@ $(LOC_LDFLAGS) $(LOC_LFLAGS) $(LIBS)
+rpm: $(MAIN)
+ make -C rpmbuild

clean:
- rm -f $(MAIN) $(LOC_MAIN)
+ rm -f $(MAIN)
make -f rpmbuild/Makefile clean

-rpm: $(LOC_MAIN)
- make -C rpmbuild
-
-.PHONY: module clean rpm
+.PHONY: module clean

Why did you change this? "rpm" target seems to be just as PHONY as it was?

What I'm worried about this is that it will stop working once Lua 5.4, or whatever, is released. Why do we need to specifically look for this version?

I think you committed this file by mistake?

What is this change? Why do you need to specify "x86_64" explictly here?

'p11-kit',
'patch',
'python-dpkt',
@@ -56,41 +56,42 @@ class Fedora(object):
'unzip',
'wget',
'yaml-cpp-devel',
+ 'pax-utils',

Hmm, why pax?

]

ec2_packages = standard_ec2_packages
test_packages = ['openssl-devel']
ec2_post_install = standard_ec2_post_install

class Fedora_25(object):
- packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel']
+ packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel.x86_64', 'lua-5.3.4', 'lua-devel-5.3.4']

Again, why the "x86_64"?

And why do you need to specify the specific version of Lua, and specifically for Fedora 25, and can't just install "lua" and "lua-devel" for all Fedoras?

--
You received this message because you are subscribed to the Google Groups "OSv Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+u...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/osv-dev/20190804153324.14890-1-jwkozaczuk%40gmail.com.

Waldek Kozaczuk

unread,

Aug 6, 2019, 11:26:43 AM8/6/19

to OSv Development

Thanks for the review!

On Tuesday, August 6, 2019 at 10:13:15 AM UTC-4, Nadav Har'El wrote:

Hi, thanks!

I have some comments and questions inline below:

On Sun, Aug 4, 2019 at 6:33 PM Waldemar Kozaczuk <jwkoz...@gmail.com> wrote:
As the issue #1022 explains, building the default OSv image that
implicitly adds cli module that in turn implicitly pulls httpserver-api
and cli modules, requires some bizarre workarounds on modern Linux distribution.
This is caused by the fact that the lua module is based on Lua 5.2 and older version
of LuaSec (OpenSSL library for Lua) which requires old version of OpenSSL 1.0.

This patch updates following modules:
1) cli:
- makes necessary changes to work with Lua 5.3
- drops explicit dependency on the modules ncurses and libedit in lieu of
new terminfo and the script manifest_from_host.sh that pulls all necessary
dependencies from host

Was this change necessary? Does the new Lua not work with the older ncurses or libedit
we had?

In general I see below several places where you're changing over from building our own libraries and
using them to using the ones from the host, and I wonder if these changes are necessary or
related to this patch, or just something you wanted to do for other reasons?

I agree that on new packages, it's easier to just take stuff from the host. But why stop using
things we already know how to build?

For openssl itself, it's different - you didn't switch from compiling it to taking it from the host -
you switched (if I understand correctly...) from taking it from external/ to taking it from the host.

My goal for the upcoming OSv release has been to make it as close to binary compatible with Linux as possible so that we can take more and more ELF files from the host.

So honestly I did not even care to test if new lua, cli would work with the older ncurses and libedit. The fewer makefiles we have to maintain the better I think. Maybe I could have tried to use old libedit and ncurses but who knows if it was compatible with new lua and cli. Because we take most stuff from host whoever prepared that Linux distribution made sure and tested things play together well (we hope ;-)).

Our old version of lua module used to use 5.2 version and build from source which would take a while. My patch only requires building the lua libraries like luasec using luarocks which is way faster. Given I had to upgrade to 5.3 (because new luasec that works with openssl 1.1 depended on 5.3) I decided to simply pull it from source which is why I added it to setup.py. Also I think we need a specific (or known) version of Lua as there have been some changes to Lua itself or luasec at least that I had to change to make it work with openssl. Yeah it is all somewhat entangled.

New version of setup.py install lua-5.3* and lua-devel-5.3.* so you do not have to do it manually.

Also the cli.c does include lua headers.

Sure.

$(MAIN): $(SRCS)
- $(CC) -DOSV_CLI $(CFLAGS) $(INCLUDES) -fPIC -shared -o $@ $(SRCS) $(LFLAGS) $(LIBS)
+ $(CC) $(CFLAGS) $(INCLUDES) $^ -fPIC -pie -o $@ $(LIBS)

-$(LOC_MAIN): $(SRCS)
- $(CC) $(CFLAGS) $(LOC_INCLUDES) $^ -o $@ $(LOC_LDFLAGS) $(LOC_LFLAGS) $(LIBS)
+rpm: $(MAIN)
+ make -C rpmbuild

clean:
- rm -f $(MAIN) $(LOC_MAIN)
+ rm -f $(MAIN)
make -f rpmbuild/Makefile clean

-rpm: $(LOC_MAIN)
- make -C rpmbuild
-
-.PHONY: module clean rpm
+.PHONY: module clean

Why did you change this? "rpm" target seems to be just as PHONY as it was?

I may have done by accident.

It should work not matter what version of lua you have installed but we are controlling which version we install in setup.py anyway. I think in my case (my Ubuntu version) I still have lua 5.2 and 5.3 at the same time and I this patch handles my case properly as well and takes the 5.3 version.

Again for 5.4 we would have to change other things as well like luasec, etc so ..

You mean the "vt100-am" or terminfo in general. My intention was to add terminfo but maybe some files should not be added to git as they are generated.

This provides lddtree utility that manifest_from_host script heavily uses.

]
ec2_packages = standard_ec2_packages
test_packages = ['openssl-devel']
ec2_post_install = standard_ec2_post_install

class Fedora_25(object):
- packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel']
+ packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel.x86_64', 'lua-5.3.4', 'lua-devel-5.3.4']

Again, why the "x86_64"?
And why do you need to specify the specific version of Lua, and specifically for Fedora 25, and can't just install "lua" and "lua-devel" for all Fedoras?

We need the specific version of lua to make cli work properly. Also how do we know which version to use if there are 2 version installed like I had in my case? I doubt we could make it work for any version. At least it would be hard to test. Has Lua 5.4 been officially released? I am not too fmailar with Lua. Is it in general backward compatible?

As for "x86_64" in one test case (I do not remember which one - Fedora 27 or 29) yum would fail complaining about some i686 version of same package conflict. Somewhat similar to what happened on the jenkins build machine I pointed in my other email. Not sure what is the right solution. Drop "x86_64" and hope it will work?

To unsubscribe from this group and stop receiving emails from it, send an email to osv...@googlegroups.com.

Waldek Kozaczuk

unread,

Aug 11, 2019, 9:21:52 AM8/11/19

to OSv Development

Hi,

I wonder what do you think we should with this patch? Which elements shall I address before we can commit it?

Have you been successful applying it and building lua on your fedora distribution?

I wanted this to be the last patch before I cut 0.54 release. In reality it addresses an annoyance but that annoyance becomes even bigger when cutting a release.

Sent from my iPhone

You received this message because you are subscribed to a topic in the Google Groups "OSv Development" group.
To unsubscribe from this topic, visit https://groups.google.com/d/topic/osv-dev/vaNoiVMCJ6E/unsubscribe.
To unsubscribe from this group and all its topics, send an email to osv-dev+u...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/osv-dev/7fab6f90-1ab9-4bb0-9e59-f9af1966de0e%40googlegroups.com.

Nadav Har'El

unread,

Aug 11, 2019, 5:46:23 PM8/11/19

to Waldek Kozaczuk, OSv Development

I wasn't sure exactly what is the scope, or goal, of this patch. I thought that you're working not only on https://github.com/cloudius-systems/osv/issues/1022, but now I understand you're also trying, in paralel, to work on https://github.com/cloudius-systems/osv/issues/743 (don't take openssl from external/) and also on reducing recompilation and instead take stuff which hopefully is installed on the build machine.

So let's be more explicit about these other goals, and perhaps split the patches up - can you have one patch to take openssl from the build machine instead of external/? Another patch to take ncurses (or whatever) from the build machine instead of compiling it? And so on. Maybe it will be easier for me to understand what you changed and what.

Our old version of lua module used to use 5.2 version and build from source which would take a while. My patch only requires building the lua libraries like luasec using luarocks which is way faster. Given I had to upgrade to 5.3 (because new luasec that works with openssl 1.1 depended on 5.3) I decided to simply pull it from source which is why I added it to setup.py. Also I think we need a specific (or known) version of Lua as there have been some changes to Lua itself or luasec at least that I had to change to make it work with openssl. Yeah it is all somewhat entangled.

Isn't the only thing which you need to do is to take all of these (Lua, luasec and openssl) from the same era? You can't take a 3 year old Lua with today's openssl, or the other way around. But it doesn't mean you need to care if Lua is 5.3.4 or 5.3.5 and other version specifics like I saw in scripts below. I'm afraid that it might become annoying to fix these scripts every time a new minor version of Lua comes along.

My Fedora has just one version of Lua install. Interesting Ubuntu has both.

Anyway, I have no idea when 5.4 will be released, but I don't understand why the same build code

will not work for it - if it just wasn't hardcoded with the version number "5.3".

This file is called "out/*" so I assumed it shouldn't have been committed? I don't know.

The next file is even worse, looks like a binary file you obviously generated by some code (a tic command?)

and shouldn't be in the source repository.

I don't understand why. If the user has an hypothetical future Lua 5.4 installed, and the same version is used to build

the cli and the other Lua stuff, wouldn't it all just work?

Of course, it's possible that Lua sucks at backward compatibility. Maybe Lua 5.4 wouldn't be able to run code written 5.3. But it's even more likely that the opposite is true.

All this specification of specific Lua versions added a lot of complexity to setup.py as you needed to find specific version numbers in all historic versions of Fedora.

Also how do we know which version to use if there are 2 version installed like I had in my case? I doubt we could make it work for any version.

How does this "yum install" command help if you already have 2 versions installed? I don't understand how this is relevant.

At least it would be hard to test. Has Lua 5.4 been officially released? I am not too fmailar with Lua. Is it in general backward compatible?

I have no idea :-)

As for "x86_64" in one test case (I do not remember which one - Fedora 27 or 29) yum would fail complaining about some i686 version of same package conflict.

Would be good to check why... But it's very strange to specify this x86_64, which is relevant to *all* the packages, just for this one package.

Somewhat similar to what happened on the jenkins build machine I pointed in my other email. Not sure what is the right solution. Drop "x86_64" and hope it will work?

I don't know. If there's a good reason, or even just a Fedora bug, that causes us to need this x86_64, then it's not that terrible, but I just don't understand what this reason is.

To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+u...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/osv-dev/7fab6f90-1ab9-4bb0-9e59-f9af1966de0e%40googlegroups.com.

Waldek Kozaczuk

unread,

Aug 12, 2019, 4:10:00 PM8/12/19

to OSv Development

For sure I want to make this patch as manageable to review as possible. So let us find a middle ground because it is also hard for me to prepare smaller patch or even break it down into many smaller ones that would also be independent.

On one hand, you are right that I am trying to accomplish too many things at once with one patch, but then it is also hard to do one main thing - upgrade OpenSSL - without inadvertently touching/changing other modules.

So first, I do have to stop using OpenSSL from external and start using it from the host in order to accomplish the main goal (1022). And I hope we agree on that.

Secondly, I do have to upgrade lua module from 5.2.x to 5.3.x because it comes with luasec that has to be upgraded to a newer version of Lua to work with OpenSSL 1.1. And I hope we agree on that. The tricky part of it this exercise is the fact that the lua module has its own custom-made makefile (https://github.com/cloudius-systems/osv/blob/master/modules/lua/Makefile#L15-L22) that explicitly extracts very specific header and source files from the source tarball and links them. The problem is that there are new files in Lua 5.3 vs Lua 5.2. More specifically this:

# Official version

-LUA_V=5.2

+LUA_V=5.3

# Archive details

-LUA_VERSION=5.2.3

+LUA_VERSION=5.3.5

LUA_FOLDER=lua-$(LUA_VERSION)

LUA_DOWNLOAD=http://www.lua.org/ftp/$(LUA_FOLDER).tar.gz

LUA_ARCHIVE=download/$(LUA_FOLDER).tar.gz

is not enough. I had to add new header lprefix.h, then it would complain aboit some missing object.

gcc -o lua lua.o liblua.a -lm -ldl

/usr/bin/ld: liblua.a(linit.o):(.data.rel.ro+0x88): undefined reference to `luaopen_utf8'

collect2: error: ld returned 1 exit status

make[1]: *** [Makefile:38: lua] Error 1

make[1]: Leaving directory '/home/wkozaczuk/projects/osv/modules/lua/src'

make: *** [Makefile:44: src/liblua.so] Error 2

which means I have to learn how to build lua which I do not have time to and long term we do not want to maintain this part of the Makefile. Shouldn't we take it from host instead?

My patch modifies setup.py to install lua53 and llibua53 for that reason and simplifies lua Makefile to mainly delegate to luarocks to build the lua modules like luasec we use and skip building lua from the source. I think this makes it more manageable in the future. All in all, I think my patch makes building lua module less version-specific than it is now, don't you agree?

Lastly, I do have to update cli as well.

Regarding the "lua 5.3.5/5.3.4" thing, which is Fedora specific, this might be solved to specifying package names in yum using a more generic pattern like lua-5.3.* (in Ubuntu I can say lua53). There are a couple of quirks in the cli Makefile where I have to make sure I pick a single version of lua if the host happens to have many version of lua installed.

So how do I make this patch more manageable? I can probably eliminate the libedit, ncurses part.

I do not see a way to break this patch into smaller ones that modifies openssl, lua and cli separately. What do you think?

...

Waldek Kozaczuk

unread,

Aug 12, 2019, 4:18:03 PM8/12/19

to OSv Development

Regarding the "x86_64 yum" thing on Fedora I have found this article which seems to explain how to fix it - https://access.redhat.com/solutions/158883

...

Waldek Kozaczuk

unread,

Aug 12, 2019, 4:28:14 PM8/12/19

to OSv Development

Also, the changes to setup.py can be simplified by using a wildcard like so:

'lua-5.3.*', 'lua-devel-5.3.*'

...

Waldek Kozaczuk

unread,

Aug 12, 2019, 4:35:43 PM8/12/19

to OSv Development

Lastly, regarding the lua module, there is a second custom-made Makefile that would need to be adjusted for 5.3 - https://github.com/cloudius-systems/osv/blob/master/modules/lua/src/Makefile#L61-L119

Waldemar Kozaczuk

unread,

Aug 12, 2019, 11:33:03 PM8/12/19

to osv...@googlegroups.com, Waldemar Kozaczuk

As the issue #1022 explains, building the default OSv image that
implicitly adds cli module that in turn implicitly pulls httpserver-api
and cli modules, requires some bizarre workarounds on modern Linux distribution.
This is caused by the fact that the lua module is based on Lua 5.2 and older version
of LuaSec (OpenSSL library for Lua) which requires old version of OpenSSL 1.0.

This patch updates following modules:
1) cli:
- makes necessary changes to work with Lua 5.3
- drops explicit dependency on the modules ncurses and libedit in lieu of
new terminfo and the script manifest_from_host.sh that pulls all necessary
dependencies from host

- creates common cli executable that can be used on both host and OSv side

2) httpserver-api
- drops explicit dependency on the module libyaml in lieu of using
manifest_from_host.sh that pulls all necessary dependencies from host

3) lua
- revamps this module completely by upgrading to Lua 5.3

- removes custom-made makefiles to build Lua from the source

- extensively employs manifest_from_host.sh to pull Lua libraries
from host (see changes to setup.py) and minimize building from source
- updates Lua modules (like LuaSec) to the newest versions

4) openssl

- instead of using hardcoded usr.manifest that pulls openssl 1.0 libraries from external,

new Makefile employs manifest_from_host.sh to find relevant main OpenSSL 1.1 library
and its required dependencies from host

Finally it adds new terminfo module and changes setup.py to uninstall
compat-openssl10-devel if present and install openssl-devel and lua 5.3 packages.

This patch has been tested with following Linux distribution using Docker:
- Ubuntu 18.10 (OpenSSL 1.0 -> OpenSSL 1.1 upgrade)
- Ubuntu 19.04
- Fedora 27 (OpenSSL 1.0 -> OpenSSL 1.1 upgrade)
- Ubuntu 29

Fixes #1022

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---
modules/cli/.gitignore | 2 +-

modules/cli/Makefile | 33 +++-----
modules/cli/cli.c | 65 +++++++++------

delete mode 100755 modules/lua/check-openssl-version
delete mode 100644 modules/lua/src/Makefile
create mode 100644 modules/openssl/.gitignore
create mode 100644 modules/openssl/Makefile
delete mode 100644 modules/openssl/usr.manifest

create mode 100644 modules/terminfo/.gitignore

create mode 100644 modules/terminfo/Makefile
create mode 100644 modules/terminfo/module.py

diff --git a/modules/cli/.gitignore b/modules/cli/.gitignore
index 773de15c..64545030 100644
--- a/modules/cli/.gitignore
+++ b/modules/cli/.gitignore
@@ -1,2 +1,2 @@
cli
-cli.so
+usr.manifest
diff --git a/modules/cli/Makefile b/modules/cli/Makefile

index e2abe632..8a3b037e 100644

--- a/modules/cli/Makefile
+++ b/modules/cli/Makefile
@@ -1,34 +1,27 @@

+LUA_LIB = $(shell pkg-config --libs lua53 2>/dev/null || pkg-config --libs lua || echo 'ERROR: Could not find lua, please run ./scripts/setup.py')
+LUA_INCLUDES = $(shell pkg-config --cflags lua53 2>/dev/null || pkg-config --cflags lua || echo 'ERROR: Could not find lua, please run ./scripts/setup.py')

+
CC=gcc
CFLAGS=-O2 -g -Wall -std=gnu99
-INCLUDES=-I../lua/src \
- -I$(lastword $(wildcard ../ncurses/build/*/include)) \
- -I$(lastword $(wildcard ../libedit/build/*/src))
-LFLAGS=-L../lua/src \
- -L$(lastword $(wildcard ../ncurses/build/*/lib)) \
- -L$(lastword $(wildcard ../libedit/build/*/src/.libs))
-LIBS=-ledit -ltinfo -llua

+LIBS=-ledit -ltinfo $(LUA_LIB)

SRCS=cli.c
-MAIN=cli.so
+MAIN=cli
+
+INCLUDES = $(LUA_INCLUDES)

-# For compiling executable running locally
-LOC_INCLUDES=-I../lua/src
-LOC_LFLAGS=-L../lua/src
-LOC_LDFLAGS=-Wl,-rpath,$(abspath ../lua/src)
-LOC_MAIN=cli
+SRC = $(shell readlink -f ../..)

module: $(MAIN)

+ $(SRC)/scripts/manifest_from_host.sh $(MAIN) > usr.manifest

$(MAIN): $(SRCS)
- $(CC) -DOSV_CLI $(CFLAGS) $(INCLUDES) -fPIC -shared -o $@ $(SRCS) $(LFLAGS) $(LIBS)
+ $(CC) $(CFLAGS) $(INCLUDES) $^ -fPIC -pie -o $@ $(LIBS)

-$(LOC_MAIN): $(SRCS)
- $(CC) $(CFLAGS) $(LOC_INCLUDES) $^ -o $@ $(LOC_LDFLAGS) $(LOC_LFLAGS) $(LIBS)
+rpm: $(MAIN)
+ make -C rpmbuild

clean:
- rm -f $(MAIN) $(LOC_MAIN)
+ rm -f $(MAIN)
make -f rpmbuild/Makefile clean

-rpm: $(LOC_MAIN)
- make -C rpmbuild
-

.PHONY: module clean rpm
diff --git a/modules/cli/cli.c b/modules/cli/cli.c
index 01c6a1e8..99c4046c 100644

@@ -443,6 +454,6 @@ void print_usage(char* program) {
"-T, --test=[COMMAND] run tests for a specifiec command\n"
"-h, --help print this help and exit\n\n"
"For more help on the CLI, see "
-"https://github.com/cloudius-systems/osv/wiki/Command-line-interface\n",
+"https://github.com/cloudius-systems/osv/wiki/Command-Line-Interface-(CLI)\n",
program);

index 00000000..b7727a37
--- /dev/null
+++ b/modules/openssl/Makefile
@@ -0,0 +1,22 @@

+#
+# Copyright (C) 2019 Waldemar Kozaczuk, Ltd.
+#
+# This work is open source software, licensed under the terms of the
+# BSD license as described in the LICENSE file in the top-level directory.
+#
+SRC = $(shell readlink -f ../..)
+
+module:
+ $(SRC)/scripts/manifest_from_host.sh -l libssl.so.1.1 > usr.manifest
+ # From krb5-libs

+ $(SRC)/scripts/manifest_from_host.sh -l libgssapi_krb5.so.2 >> usr.manifest

diff --git a/modules/terminfo/.gitignore b/modules/terminfo/.gitignore
new file mode 100644
index 00000000..1fcb1529
--- /dev/null
+++ b/modules/terminfo/.gitignore
@@ -0,0 +1 @@
+out

diff --git a/modules/terminfo/Makefile b/modules/terminfo/Makefile
new file mode 100644
index 00000000..ad4fbb81
--- /dev/null
+++ b/modules/terminfo/Makefile
@@ -0,0 +1,16 @@
+OUT=out
+TI_VT100=$(OUT)/terminfo/v/vt100-qemu
+
+module: $(TI_VT100)
+
+$(TI_VT100): $(MAIN)
+ @mkdir -p $(OUT)/terminfo
+ infocmp vt100 | sed 's/cud1=$^J\|\\n$,/cud1=\\E[B,/; s/ind=$^J\|\\n$,/ind=\\E[S,/; s/vt100/vt100-qemu/' \
+ > $(OUT)/terminfo.tmp
+ tic -o $(OUT)/terminfo $(OUT)/terminfo.tmp

+ rm -f $(OUT)/terminfo.tmp
+
+clean:
+ rm -rf $(OUT)

+
+.PHONY: module clean
diff --git a/modules/terminfo/module.py b/modules/terminfo/module.py

new file mode 100644

index 00000000..a422d75c
--- /dev/null
+++ b/modules/terminfo/module.py
@@ -0,0 +1,4 @@
+from osv.modules.filemap import FileMap
+
+usr_files = FileMap()
+usr_files.add('${OSV_BASE}/modules/terminfo/out/terminfo').to('/usr/share/terminfo')

diff --git a/scripts/setup.py b/scripts/setup.py
index 88bf6015..9be80d69 100755
--- a/scripts/setup.py
+++ b/scripts/setup.py
@@ -22,6 +22,7 @@ standard_ec2_post_install = ['pip install awscli &&'

class Fedora(object):
name = 'Fedora'
+ pre_install = '(yum list installed compat-openssl10-devel 2>/dev/null && yum -y remove compat-openssl10-devel) || echo "package compat-openssl10-devel not found -> no need to remove it"'
install = 'yum -y install --allowerasing'
packages = [
'ant',
@@ -63,35 +64,35 @@ class Fedora(object):

ec2_post_install = standard_ec2_post_install

class Fedora_25(object):
- packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel']