[PATCH] nginx: make it work on read-only fs

[Fotis Xenakis]

By default, nginx writes to various files, mainly for logging and
caching. In order for it to work with read-only file systems, this patch
provides an alternative nginx configuration file redirecting all these
write paths to /tmp (which is backed by ramfs and thus is read-write).

On the way, nginx is also updated to 1.19.7 (current latest).

Signed-off-by: Fotis Xenakis <fo...@windowslive.com>
---
nginx/Makefile | 6 +-
nginx/patches/nginx-ro.conf | 147 ++++++++++++++++++++++++++++++++++++
2 files changed, 151 insertions(+), 2 deletions(-)
create mode 100644 nginx/patches/nginx-ro.conf

diff --git a/nginx/Makefile b/nginx/Makefile
index 6a44e00..99c5cb8 100644
--- a/nginx/Makefile
+++ b/nginx/Makefile
@@ -1,10 +1,12 @@
-VERSION=1.17.4
+VERSION=1.19.7
SOURCE=http://nginx.org/download/nginx-${VERSION}.tar.gz
CONFIGURE_MODULES=--prefix=/nginx/ --with-debug --without-http_rewrite_module --with-threads --with-http_stub_status_module

.PHONY: module clean

SRC=upstream/nginx
+# Use nginx-ro.conf when on a read-only root file system (currenty rofs and virtio-fs)
+NGINX_CONF=$(if $(filter $(fs_type),rofs virtiofs),nginx-ro.conf,nginx.conf)

module: usr.manifest

@@ -14,7 +16,7 @@ usr.manifest: $(SRC)/nginx.so
echo '/nginx/html/**: $${MODULE_DIR}/upstream/nginx/html/**' >> usr.manifest
echo '/nginx/logs/**: $${MODULE_DIR}/upstream/nginx/logs/**' >> usr.manifest
echo '/nginx/conf/**: $${MODULE_DIR}/upstream/nginx/conf/**' >> usr.manifest
- echo '/nginx/conf/nginx.conf: $${MODULE_DIR}/patches/nginx.conf' >> usr.manifest
+ echo '/nginx/conf/nginx.conf: $${MODULE_DIR}/patches/$(NGINX_CONF)' >> usr.manifest

clean:
rm -fr upstream
diff --git a/nginx/patches/nginx-ro.conf b/nginx/patches/nginx-ro.conf
new file mode 100644
index 0000000..58b1c4f
--- /dev/null
+++ b/nginx/patches/nginx-ro.conf
@@ -0,0 +1,147 @@
+
+#user nobody;
+worker_processes 1;
+
+# Set error_log to stderr so that log messages are displayed on
+# OSv console that started "scripts/run.py -nvd".
+# Although this is less ideal when compared to redirecting error
+# and access logs to syslog for example, it is a workable first
+# solution that is comparable to redirection used while starting
+# Nginx in a container
+# (reference: http://serverfault.com/questions/657863/nginx-how-to-use-docker-log-collector-when-nginx-is-running-under-supervisord).
+error_log stderr error;
+
+# Write pid to ramfs
+pid /tmp/nginx_pid;
+
+# Run in foreground, primarily because fork() is stubbed in OSv.
+# This setting is consistent with official Nginx Dockerfile configuration:
+# https://github.com/nginxinc/docker-nginx/blob/41aa13f7d2c24407e483c40fb1e8b33e73462ff1/mainline/jessie/Dockerfile#L27
+daemon off;
+
+events {
+ worker_connections 1024;
+}
+
+
+http {
+ include mime.types;
+ default_type application/octet-stream;
+
+ #log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+ # '$status $body_bytes_sent "$http_referer" '
+ # '"$http_user_agent" "$http_x_forwarded_for"';
+
+ access_log off;
+
+ client_body_buffer_size 10m;
+ client_max_body_size 10m;
+ client_body_in_single_buffer on;
+ client_body_temp_path /tmp/nginx_client_body_temp;
+
+ proxy_temp_path /tmp/nginx_proxy_temp;
+ fastcgi_temp_path /tmp/nginx_fastcgi_temp;
+ uwsgi_temp_path /tmp/nginx_uwsgi_temp;
+ scgi_temp_path /tmp/nginx_uwsgi_temp;
+
+ sendfile on;
+ tcp_nopush on;
+
+ # Default keepalive param values
+ #keepalive_requests 100;
+ # Long keepalive to avoid/reduce preblems with TCP port resue
+ # See https://github.com/cloudius-systems/osv/issues/889
+ keepalive_requests 1000000000;
+
+ #gzip on;
+
+ server {
+ listen 80;
+ server_name localhost;
+
+ # server_name 192.168.122.1;
+
+ #charset koi8-r;
+
+ #access_log logs/host.access.log main;
+
+ location / {
+ root html;
+ index index.html index.htm;
+ #aio threads;
+ }
+
+ location /basic_status {
+ stub_status;
+ }
+
+ #error_page 404 /404.html;
+
+ # redirect server error pages to the static page /50x.html
+ #
+ error_page 500 502 503 504 /50x.html;
+ location = /50x.html {
+ root html;
+ }
+
+ # proxy the PHP scripts to Apache listening on 127.0.0.1:80
+ #
+ #location ~ \.php$ {
+ # proxy_pass http://127.0.0.1;
+ #}
+
+ # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
+ #
+ #location ~ \.php$ {
+ # root html;
+ # fastcgi_pass 127.0.0.1:9000;
+ # fastcgi_index index.php;
+ # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
+ # include fastcgi_params;
+ #}
+
+ # deny access to .htaccess files, if Apache's document root
+ # concurs with nginx's one
+ #
+ #location ~ /\.ht {
+ # deny all;
+ #}
+ }
+
+
+ # another virtual host using mix of IP-, name-, and port-based configuration
+ #
+ #server {
+ # listen 8000;
+ # listen somename:8080;
+ # server_name somename alias another.alias;
+
+ # location / {
+ # root html;
+ # index index.html index.htm;
+ # }
+ #}
+
+
+ # HTTPS server
+ #
+ #server {
+ # listen 443 ssl;
+ # server_name localhost;
+
+ # ssl_certificate cert.pem;
+ # ssl_certificate_key cert.key;
+
+ # ssl_session_cache shared:SSL:1m;
+ # ssl_session_timeout 5m;
+
+ # ssl_ciphers HIGH:!aNULL:!MD5;
+ # ssl_prefer_server_ciphers on;
+
+ # location / {
+ # root html;
+ # index index.html index.htm;
+ # }
+ #}
+
+}
--
2.30.1

[Waldek Kozaczuk]

The patch looks good but when I build and run it I still get a warning about read-only FS:

./scripts/build fs=virtiofs export=all image=nginx -j$(nproc)

sudo PATH=build/downloaded_packages/qemu/build/tools/virtiofsd:$PATH ./scripts/run.py --virtio-fs-tag=myfs --virtio-fs-dir=$(pwd)/build/export --qemu-path build/downloaded_packages/qemu/build/qemu-system-x86_64
OSv v0.55.0-209-g2275ed48
eth0: 192.168.122.15
Booted up in 170.08 ms
Cmdline: /nginx.so -c /nginx/conf/nginx.conf
nginx: [alert] could not open error log file: open() "/nginx//logs/error.log" failed (30: Read-only file system)
WARNING: af_local::ioctl(FIOASYNC) stubbed
WARNING: fcntl(F_SETOWN) stubbed
initgroups() stubbed
2021/03/13 16:18:38 [emerg] 2#56: initgroups(nobody, 0) failed
2021/03/13 16:18:38 [alert] 2#56: epoll_wait() timed out (110: Unknown error)

Waldek

--
You received this message because you are subscribed to the Google Groups "OSv Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+u...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/osv-dev/VI1PR03MB3773463CF1CDCEC2935FF22AA6959%40VI1PR03MB3773.eurprd03.prod.outlook.com.

[Fotis Xenakis]

This is an nginx idiosyncrasy: it accesses the default error log location even when it has been changed or turned off: [1]

Failure during this access was not blocking: it was reported, but everything worked just fine. This is why it was viable to ignore it.

Fortunately, very recently they addressed this: [2] (I had originally investigated the issue before this fix, kudos for bringing it up!)

Let me iterate on the patch to try and make use of the new command line option when on a read-only fs.

[1] https://stackoverflow.com/a/39124579

[2] https://trac.nginx.org/nginx/ticket/147

---

From: Waldek Kozaczuk <jwkoz...@gmail.com>
Sent: Saturday, March 13, 2021 18:21
To: Fotis Xenakis <fo...@windowslive.com>
Cc: OSv Development <osv...@googlegroups.com>
Subject: Re: [osv-dev] [PATCH] nginx: make it work on read-only fs

 

[Fotis Xenakis]

By default, nginx writes to various files, mainly for logging and
caching. In order for it to work with read-only file systems, this patch
provides an alternative nginx configuration file redirecting all these
write paths to /tmp (which is backed by ramfs and thus is read-write).

On the way, nginx is also updated to 1.19.7 (current latest).

Signed-off-by: Fotis Xenakis <fo...@windowslive.com>
---
nginx/Makefile | 6 +-

nginx/module.py | 12 ++-
nginx/patches/nginx-ro.conf | 147 ++++++++++++++++++++++++++++++++++++
3 files changed, 162 insertions(+), 3 deletions(-)

create mode 100644 nginx/patches/nginx-ro.conf

diff --git a/nginx/Makefile b/nginx/Makefile

index 6a44e00..d4a0456 100644

--- a/nginx/Makefile
+++ b/nginx/Makefile
@@ -1,10 +1,12 @@
-VERSION=1.17.4

+VERSION=1.19.8

SOURCE=http://nginx.org/download/nginx-${VERSION}.tar.gz
CONFIGURE_MODULES=--prefix=/nginx/ --with-debug --without-http_rewrite_module --with-threads --with-http_stub_status_module

.PHONY: module clean

SRC=upstream/nginx
+# Use nginx-ro.conf when on a read-only root file system (currenty rofs and virtio-fs)
+NGINX_CONF=$(if $(filter $(fs_type),rofs virtiofs),nginx-ro.conf,nginx.conf)

module: usr.manifest

@@ -14,7 +16,7 @@ usr.manifest: $(SRC)/nginx.so
echo '/nginx/html/**: $${MODULE_DIR}/upstream/nginx/html/**' >> usr.manifest
echo '/nginx/logs/**: $${MODULE_DIR}/upstream/nginx/logs/**' >> usr.manifest
echo '/nginx/conf/**: $${MODULE_DIR}/upstream/nginx/conf/**' >> usr.manifest
- echo '/nginx/conf/nginx.conf: $${MODULE_DIR}/patches/nginx.conf' >> usr.manifest
+ echo '/nginx/conf/nginx.conf: $${MODULE_DIR}/patches/$(NGINX_CONF)' >> usr.manifest

clean:
rm -fr upstream

diff --git a/nginx/module.py b/nginx/module.py
index 6a7df37..1035dcd 100644
--- a/nginx/module.py
+++ b/nginx/module.py
@@ -1,4 +1,14 @@
+import os
+
from osv.modules import api

+
+ROFS = ['rofs', 'virtiofs'] # List of read-only filesystems
+
+
api.require('libz')
-default = api.run('/nginx.so -c /nginx/conf/nginx.conf')
+
+# NOTE: This is necessary to avoid the warning about the default error log
+# location. See https://trac.nginx.org/nginx/ticket/147.
+error_log = ' -e stderr' if os.environ.get('fs_type') in ROFS else ''
+default = api.run('/nginx.so -c /nginx/conf/nginx.conf' + error_log)

2.30.2

[Commit Bot]

From: Fotis Xenakis <fo...@windowslive.com>
Committer: Waldemar Kozaczuk <jwkoz...@gmail.com>
Branch: master

nginx: make it work on read-only fs

By default, nginx writes to various files, mainly for logging and
caching. In order for it to work with read-only file systems, this patch
provides an alternative nginx configuration file redirecting all these
write paths to /tmp (which is backed by ramfs and thus is read-write).

On the way, nginx is also updated to 1.19.7 (current latest).

Signed-off-by: Fotis Xenakis <fo...@windowslive.com>

Message-Id: <DB7PR03MB3771A3538F...@DB7PR03MB3771.eurprd03.prod.outlook.com>

---
diff --git a/nginx/Makefile b/nginx/Makefile

--- a/nginx/Makefile
+++ b/nginx/Makefile
@@ -1,10 +1,12 @@
-VERSION=1.17.4
+VERSION=1.19.8
SOURCE=http://nginx.org/download/nginx-${VERSION}.tar.gz
CONFIGURE_MODULES=--prefix=/nginx/ --with-debug --without-http_rewrite_module --with-threads --with-http_stub_status_module

.PHONY: module clean

SRC=upstream/nginx
+# Use nginx-ro.conf when on a read-only root file system (currenty rofs and virtio-fs)
+NGINX_CONF=$(if $(filter $(fs_type),rofs virtiofs),nginx-ro.conf,nginx.conf)

module: usr.manifest

@@ -14,7 +16,7 @@ usr.manifest: $(SRC)/nginx.so
echo '/nginx/html/**: $${MODULE_DIR}/upstream/nginx/html/**' >> usr.manifest
echo '/nginx/logs/**: $${MODULE_DIR}/upstream/nginx/logs/**' >> usr.manifest
echo '/nginx/conf/**: $${MODULE_DIR}/upstream/nginx/conf/**' >> usr.manifest
- echo '/nginx/conf/nginx.conf: $${MODULE_DIR}/patches/nginx.conf' >> usr.manifest
+ echo '/nginx/conf/nginx.conf: $${MODULE_DIR}/patches/$(NGINX_CONF)' >> usr.manifest

clean:
rm -fr upstream
diff --git a/nginx/module.py b/nginx/module.py

--- a/nginx/module.py
+++ b/nginx/module.py
@@ -1,4 +1,14 @@
+import os
+
from osv.modules import api

+
+ROFS = ['rofs', 'virtiofs'] # List of read-only filesystems
+
+
api.require('libz')
-default = api.run('/nginx.so -c /nginx/conf/nginx.conf')
+
+# NOTE: This is necessary to avoid the warning about the default error log
+# location. See https://trac.nginx.org/nginx/ticket/147.
+error_log = ' -e stderr' if os.environ.get('fs_type') in ROFS else ''
+default = api.run('/nginx.so -c /nginx/conf/nginx.conf' + error_log)
diff --git a/nginx/patches/nginx-ro.conf b/nginx/patches/nginx-ro.conf

--- a/nginx/patches/nginx-ro.conf