Nginx reverse proxy connection reset by peer when putting files to XNAT

[Tom Close]

Hi,

I am getting this strange error when trying to upload particular files (98% files can be uploaded okay but some won't upload no matter how many times you try). It looks like it is an issue with my Nginx reverse proxy setup as when I check the nginx logs I see that the connection was reset by XNAT. 

2017/09/26 13:35:44 [error] 2946#2946: *3842076 readv() failed (104: Connection reset by peer) while reading upstream, client: 118.138.254.185, server: mbi-xnat.erc.monash.edu.au, request: "PUT /data/archive/projects/MRH032F/subjects/MBI_XNAT_S02678/experiments/MBI_XNAT_E04877/scans/test_FRDA_2_betted_T1/resources/NIFTI_GZ/files/test_FRDA_2_betted_T1.nii.gz HTTP/1.1", upstream: "http://127.0.0.1:8080/data/archive/projects/MRH032F/subjects/MBI_XNAT_S02678/experiments/MBI_XNAT_E04877/scans/test_FRDA_2_betted_T1/resources/NIFTI_GZ/files/test_FRDA_2_betted_T1.nii.gz", host: "mbi-xnat.erc.monash.edu.au"

However, when I check the tomcat and XNAT access logs I don't find entries that match the nginx timestamps. So in my head I am envisaging that nginx attempts to connect to tomcat but this fails during the initialisation phase (perhaps a time out of some description). But why this would happen only with some files I have no idea. Could this be indicative with a problem with my XNAT instance (e.g. insufficient resources) or does anyone know enough about Nginx to advise me on appropriate values to try for the timeout values?

My nginx config (as generated by nginx -T) is as follows, with the part I have customised in blue:

user www-data;

worker_processes auto;

pid /run/nginx.pid;

events {

           worker_connections 768;

           # multi_accept on;

}

http {

           ##

           # Basic Settings

           ##

           sendfile on;

           tcp_nopush on;

           tcp_nodelay on;

           keepalive_timeout 65;

           types_hash_max_size 2048;

           # server_tokens off;

           # server_names_hash_bucket_size 64;

           # server_name_in_redirect off;

           include /etc/nginx/mime.types;

           default_type application/octet-stream;

           ##

           # SSL Settings

           ##

           ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE

           ssl_prefer_server_ciphers on;

           ##

           # Logging Settings

           ##

           access_log /var/log/nginx/access.log;

           error_log /var/log/nginx/error.log;

           ##

           # Gzip Settings

           ##

           gzip on;

           gzip_disable "msie6";

           # gzip_vary on;

           # gzip_proxied any;

           # gzip_comp_level 6;

           # gzip_buffers 16 8k;

           # gzip_http_version 1.1;

           # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;

           ##

           # Virtual Host Configs

           ##

           include /etc/nginx/conf.d/*.conf;

           include /etc/nginx/sites-enabled/*;

}

#mail {

#          # See sample authentication script at:

#          # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript

#

#          # auth_http localhost/auth.php;

#          # pop3_capabilities "TOP" "USER";

#          # imap_capabilities "IMAP4rev1" "UIDPLUS";

#

#          server {

#              listen     localhost:110;

#              protocol   pop3;

#              proxy      on;

#          }

#

#          server {

#              listen     localhost:143;

#              protocol   imap;

#              proxy      on;

#          }

#}

# configuration file /etc/nginx/mime.types:

types {

    text/html                             html htm shtml;

    text/css                              css;

    text/xml                              xml;

    image/gif                             gif;

    image/jpeg                            jpeg jpg;

    application/javascript                js;

    application/atom+xml                  atom;

    application/rss+xml                   rss;

    text/mathml                           mml;

    text/plain                            txt;

    text/vnd.sun.j2me.app-descriptor      jad;

    text/vnd.wap.wml                      wml;

    text/x-component                      htc;

    image/png                             png;

    image/tiff                            tif tiff;

    image/vnd.wap.wbmp                    wbmp;

    image/x-icon                          ico;

    image/x-jng                           jng;

    image/x-ms-bmp                        bmp;

    image/svg+xml                         svg svgz;

    image/webp                            webp;

    application/font-woff                 woff;

    application/java-archive              jar war ear;

    application/json                      json;

    application/mac-binhex40              hqx;

    application/msword                    doc;

    application/pdf                       pdf;

    application/postscript                ps eps ai;

    application/rtf                       rtf;

    application/vnd.apple.mpegurl         m3u8;

    application/vnd.ms-excel              xls;

    application/vnd.ms-fontobject         eot;

    application/vnd.ms-powerpoint         ppt;

    application/vnd.wap.wmlc              wmlc;

    application/vnd.google-earth.kml+xml  kml;

    application/vnd.google-earth.kmz      kmz;

    application/x-7z-compressed           7z;

    application/x-cocoa                   cco;

    application/x-java-archive-diff       jardiff;

    application/x-java-jnlp-file          jnlp;

    application/x-makeself                run;

    application/x-perl                    pl pm;

    application/x-pilot                   prc pdb;

    application/x-rar-compressed          rar;

    application/x-redhat-package-manager  rpm;

    application/x-sea                     sea;

    application/x-shockwave-flash         swf;

    application/x-stuffit                 sit;

    application/x-tcl                     tcl tk;

    application/x-x509-ca-cert            der pem crt;

    application/x-xpinstall               xpi;

    application/xhtml+xml                 xhtml;

    application/xspf+xml                  xspf;

    application/zip                       zip;

    application/octet-stream              bin exe dll;

    application/octet-stream              deb;

    application/octet-stream              dmg;

    application/octet-stream              iso img;

    application/octet-stream              msi msp msm;

    application/vnd.openxmlformats-officedocument.wordprocessingml.document    docx;

    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet          xlsx;

    application/vnd.openxmlformats-officedocument.presentationml.presentation  pptx;

    audio/midi                            mid midi kar;

    audio/mpeg                            mp3;

    audio/ogg                             ogg;

    audio/x-m4a                           m4a;

    audio/x-realaudio                     ra;

    video/3gpp                            3gpp 3gp;

    video/mp2t                            ts;

    video/mp4                             mp4;

    video/mpeg                            mpeg mpg;

    video/quicktime                       mov;

    video/webm                            webm;

    video/x-flv                           flv;

    video/x-m4v                           m4v;

    video/x-mng                           mng;

    video/x-ms-asf                        asx asf;

    video/x-ms-wmv                        wmv;

    video/x-msvideo                       avi;

}

# configuration file /etc/nginx/sites-enabled/xnat.ssl:

# Redirect http requests to https

server {

    listen 80 default_server;

    listen [::]:80 default_server;

    server_name mbi-xnat.erc.monash.edu.au;

    return 301 https://$server_name$request_uri;

}

# Listen for https requests and pass them on to the tomcat server, which listens on port 8080

server {

    listen mbi-xnat.erc.monash.edu.au:443;

    server_name mbi-xnat.erc.monash.edu.au;

    ssl    on;

    ssl_certificate    /etc/pki/tls/certs/my.crt;

    ssl_certificate_key    /etc/pki/tls/private/my.key;

    location / {

root /var/lib/tomcat/webapps/ROOT;

proxy_pass                          http://localhost:8080;

proxy_redirect                      http://localhost:8080 $scheme://localhost;

proxy_set_header Host               $host;

proxy_set_header X-Real-IP          $remote_addr;

proxy_set_header X-Forwarded-Host   $host;

proxy_set_header X-Forwarded-Server $host;

proxy_set_header X-Forwarded-For    $proxy_add_x_forwarded_for;

proxy_set_header X-Forwarded-Proto  $scheme;

proxy_connect_timeout               150;

proxy_send_timeout                  100;

proxy_read_timeout                  500;

proxy_buffers                       4 32k;

client_max_body_size                0;

client_body_buffer_size             128k;

    }

    access_log /var/log/nginx/xnat.ssl.access.log;

    error_log /var/log/nginx/xnat.ssl.error.log;

}

Any leads would be much appreciated as I am a little out of my depth with Nginx.

Cheers,

Tom 

[Herrick, Rick]

I suspect the issue is on the Tomcat side, because it’s not even responding. If you search for this error, you’ll see that most often the nginx error message will actually reference “while reading response header from upstream”. In that case, nginx established the connection with Tomcat but something went wrong. nginx returns that error code when “an error occurred while establishing a connection with the server, passing a request to it, or reading the response header.” But you know it’s not the response header so things are sideways before then.

 

Try this: in the server.xml for Tomcat, find the <Connector> element. See if you have a value set for maxHttpHeadersize. There’s not one set by default. The standard <Connector> element looks like this:

 

<Connector port="8080" protocol="HTTP/1.1"

           connectionTimeout="20000"

           URIEncoding="UTF-8"

           redirectPort="8443" />

 

Try setting that to something higher than the default 8,192, maybe times 4 or 8. You can also mess around with the connection timeout settings. I’m specifically thinking maybe raising connectionTimeout and keepAliveTimeout. See if those make any difference.

 

-- 

Rick Herrick

Sr. Programmer/Analyst

Neuroinformatics Research Group

Washington University School of Medicine

Phone: +1 (314) 273-1645

--
You received this message because you are subscribed to the Google Groups "xnat_discussion" group.
To unsubscribe from this group and stop receiving emails from it, send an email to xnat_discussi...@googlegroups.com.
To post to this group, send email to xnat_di...@googlegroups.com.
Visit this group at https://groups.google.com/group/xnat_discussion.
For more options, visit https://groups.google.com/d/optout.

 

---

The materials in this message are private and may contain Protected Healthcare Information or other information of a sensitive nature. If you are not the intended recipient, be advised that any unauthorized use, disclosure, copying or the taking of any action in reliance on the contents of this information is strictly prohibited. If you have received this email in error, please immediately notify the sender via telephone or return mail.

[Tom Close]

Hi Rick,

I upped those values you suggested to 

maxHttpHeadersize="65536"

connectionTimeout="40000"

and was able to upload one of the files that was giving us problems, so it looks like those settings were the problem (my money is on the header size but since this is my production box I am not going to play around with it). Will see if it pops up again.

Thanks for your help!

[Herrick, Rick]

Cool, glad that’s working for you!