[PATCH] lua: change build process to download artifacts from lua binaries

[Waldemar Kozaczuk]

As the issue #1166 explains, building lua module does not work on Fedora
33 and up. In short, lua module depends on a specific version 5.3 of lua
interpreter, library and header files which may not be available on
given version of Fedora and which may be the case with other Linux
distributions.

The issue #1166 describes at least three alternative solutions to the
problem, but this patch solves it by changing the makefile to
download the lua interpreter (lua executable), library and header files
from a well maintained repository - LuaBinaries at http://luabinaries.sourceforge.net/
- logically in a similar way we download luarocks. The LuaBinaries has
been in place since 2005 so there is good chance we can keep relying
on it in foreseeable future.

At the moment the makefile downloads fairly recent version 5.3.6 of lua binaries
which are compatible with the versions of lua modules (like socket, etc)
and luarocks. In future we may upgrade all elements needed to build the
module as we see fit.

As the result of this patch, lua module should in theory be build-able on any
Linux distribution and version. in reality with newer versions of gcc
one can imagine that lua modules themselves will stop compiling at which
point we will need to upgrade those and possibly lua and luarocks itself.
Also please note that lua module no longer depends on version of lua
installed on host if any.

Fixes #1166

Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com>
---
modules/cli/Makefile | 15 +++++------
modules/lua/Makefile | 63 +++++++++++++++++++++++++++++++++-----------
2 files changed, 53 insertions(+), 25 deletions(-)

diff --git a/modules/cli/Makefile b/modules/cli/Makefile
index 8a3b037e..ab648879 100644
--- a/modules/cli/Makefile
+++ b/modules/cli/Makefile
@@ -1,21 +1,18 @@
-LUA_LIB = $(shell pkg-config --libs lua53 2>/dev/null || pkg-config --libs lua || echo 'ERROR: Could not find lua, please run ./scripts/setup.py')
-LUA_INCLUDES = $(shell pkg-config --cflags lua53 2>/dev/null || pkg-config --cflags lua || echo 'ERROR: Could not find lua, please run ./scripts/setup.py')
+SRC = $(shell readlink -f ../..)
+
+LUA_DIR = $(SRC)/modules/lua/upstream/lua5.3

CC=gcc
-CFLAGS=-O2 -g -Wall -std=gnu99
-LIBS=-ledit -ltinfo $(LUA_LIB)
+CFLAGS=-O2 -g -Wall -std=gnu99 -I $(LUA_DIR)/include
+LIBS=-ledit -ltinfo -ldl -L$(LUA_DIR) -llua53
SRCS=cli.c
MAIN=cli

-INCLUDES = $(LUA_INCLUDES)
-
-SRC = $(shell readlink -f ../..)
-
module: $(MAIN)
$(SRC)/scripts/manifest_from_host.sh $(MAIN) > usr.manifest

$(MAIN): $(SRCS)
- $(CC) $(CFLAGS) $(INCLUDES) $^ -fPIC -pie -o $@ $(LIBS)
+ $(CC) $(CFLAGS) $^ -fPIC -pie -o $@ $(LIBS)

rpm: $(MAIN)
make -C rpmbuild
diff --git a/modules/lua/Makefile b/modules/lua/Makefile
index e48791af..a2412894 100644
--- a/modules/lua/Makefile
+++ b/modules/lua/Makefile
@@ -1,62 +1,93 @@
SRC = $(shell readlink -f ../..)
+
+# This makefile orchestrates building some key lua modules used by the OSv cli
+# module. Please note that both lua binaries, header files and luarocks are
+# downloaded from internet and lua artifacts if installed on the host are not used.
+# This should make maintenance of lua module much less painful as regardless
+# of the Linux distribution and version it will use lua 5.3 and luarocks 3.1.1
+# until we specifically upgrade them by modifying this makefile.
+
+LUA=lua5.3
+LUA_DIR=upstream/$(LUA)
LUA_ROCKS=upstream/luarocks-3.1.1-linux-x86_64/luarocks
+
MODULES_DIR=install/lua_modules
+LUA_ROCKS_INSTALL_MODULE := $(LUA_ROCKS) --lua-dir=$(LUA_DIR) install --no-doc --tree $(MODULES_DIR)
+
LDIR=install/lua_modules/lib/lua/5.3
CDIR=install/lua_modules/share/lua/5.3

+# Set LUAROCKS_CONFIG to make luarocks use lua binaries downloaded in upstream/lua5.3
+export LUAROCKS_CONFIG=$(SRC)/modules/lua/upstream/config.lua
+
# List of Lua modules, each module has its own target
LUA_MODULES=LuaSocket LuaJSON Lua_stdlib LuaFileSystem LuaPath LuaSec

-LUA_LIBRARY := $(shell ldconfig -p | grep -Po "liblua*.5\.3.so.0" | head -1)
-ifndef LUA_LIBRARY
- LUA_LIBRARY := $(shell ldconfig -p | grep -Po "liblua*.5\.3.so" | head -1)
-endif
-
module: $(LUA_MODULES)
mkdir -p $(MODULES_DIR)
- $(SRC)/scripts/manifest_from_host.sh -l $(LUA_LIBRARY) > usr.manifest
-
-$(LUA_ROCKS):
+ echo "/usr/lib/liblua53.so: $(SRC)/modules/lua/$(LUA_DIR)/liblua53.so" > usr.manifest
+
+# Download lua interpreter from lua binaries
+$(LUA_DIR)/lua53:
+ mkdir -p $(LUA_DIR)
+ cd upstream && wget -c "https://sourceforge.net/projects/luabinaries/files/5.3.6/Tools%20Executables/lua-5.3.6_Linux54_64_bin.tar.gz"
+ cd $(LUA_DIR) && tar xf ../lua-5.3.6_Linux54_64_bin.tar.gz
+
+# Download lua shared library and header files from lua binaries
+$(LUA_DIR)/liblua53.so:
+ mkdir -p $(LUA_DIR)
+ cd upstream && wget -c "https://sourceforge.net/projects/luabinaries/files/5.3.6/Linux%20Libraries/lua-5.3.6_Linux54_64_lib.tar.gz"
+ cd $(LUA_DIR) && tar xf ../lua-5.3.6_Linux54_64_lib.tar.gz
+
+# In order for luarocks to use the downloaded version of lua in upstream, we need to create a config file below
+upstream/config.lua:
+ echo "variables = {" > upstream/config.lua
+ echo " LUA_INCDIR = \"$(SRC)/modules/lua/$(LUA_DIR)/include\"," >> upstream/config.lua
+ echo " LUA_LIBDIR = \"$(SRC)/modules/lua/$(LUA_DIR)\"," >> upstream/config.lua
+ echo "}" >> upstream/config.lua
+
+$(LUA_ROCKS): $(LUA_DIR)/lua53 $(LUA_DIR)/liblua53.so upstream/config.lua
mkdir -p upstream
- cd upstream && wget -c https://luarocks.github.io/luarocks/releases/luarocks-3.1.1-linux-x86_64.zip
+ cd upstream && wget -c "https://luarocks.github.io/luarocks/releases/luarocks-3.1.1-linux-x86_64.zip"
cd upstream && unzip luarocks-3.1.1-linux-x86_64.zip
+ touch $(LUA_ROCKS) #To prevent re-running the rule in case $(LUA_ROCKS) is older than $(LUA_DIR)/liblua53.so and/or $(LUA_DIR)/lua53

# == LuaSocket ==
LuaSocket: $(LDIR)/socket/core.so

$(LDIR)/socket/core.so: $(LUA_ROCKS)
- $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) luasocket 3.0rc1-2
+ $(LUA_ROCKS_INSTALL_MODULE) luasocket 3.0rc1-2

# == LuaJSON ==
LuaJSON: $(CDIR)/json.lua

$(CDIR)/json.lua: $(LUA_ROCKS)
- $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) lpeg 1.0.2-1
- $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) luajson 1.3.4-1
+ $(LUA_ROCKS_INSTALL_MODULE) lpeg 1.0.2-1
+ $(LUA_ROCKS_INSTALL_MODULE) luajson 1.3.4-1

# == Lua_stdlib ==
Lua_stdlib: $(CDIR)/std.lua

$(CDIR)/std.lua: $(LUA_ROCKS)
- $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) stdlib 41.2.2-1
+ $(LUA_ROCKS_INSTALL_MODULE) stdlib 41.2.2-1

# == LuaFileSystem ==
LuaFileSystem: $(LDIR)/lfs.so

$(LDIR)/lfs.so: $(LUA_ROCKS)
- $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) LuaFileSystem 1.7.0-2
+ $(LUA_ROCKS_INSTALL_MODULE) LuaFileSystem 1.7.0-2

# == LuaPath ==
LuaPath: $(CDIR)/path.lua

$(CDIR)/path.lua: $(LUA_ROCKS)
- $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) lua-path 0.3.1-1
+ $(LUA_ROCKS_INSTALL_MODULE) lua-path 0.3.1-1

# == LuaSec ==
LuaSec: $(CDIR)/ssl.lua

$(CDIR)/ssl.lua: $(LUA_ROCKS)
- $(LUA_ROCKS) install --no-doc --tree $(MODULES_DIR) LuaSec 0.8-1
+ $(LUA_ROCKS_INSTALL_MODULE) LuaSec 0.8-1

clean:
rm -rf install
--
2.31.1

[Waldek Kozaczuk]

I have tested it on Fedora 33 and Ubuntu 21.10. It would be nice to see if it works on newer versions of Fedora - 34,35 and possibly some older ones.

[Commit Bot]

From: Waldemar Kozaczuk <jwkoz...@gmail.com>
Committer: Waldemar Kozaczuk <jwkoz...@gmail.com>
Branch: master

lua: change build process to download artifacts from lua binaries

diff --git a/modules/cli/Makefile b/modules/cli/Makefile