Nginx not scaling more than 750 Connections when used as Reverse Proxy - nginx

I have a mobile application in which we are using Nginx as a Reverse proxy which routes request to Nginx. The Nginx request in app server is passed to Node.js for processing.
We are getting 504 Gateway timeout error when we are hitting more than 750 Users.we are seeing below error in Nginx Logs.
upstream timed out (110: Connection timed out) while connecting to upstream, client: LoadGenerator_IP, server: WebserverDNS, request: "GET /api/sample/profile HTTP/1.1", upstream: "https://app_server:443/api/sample/profile", host: "webserver_IP"
I tried to hit App server Nginx directly.We could able to hit more than 1000 users.But if we are using reverse proxy we are getting that error.
I tried alot of Linux system and Nginx settings.But did not overcome this issue.
NGINX.CONF
user nginx;
worker_processes auto;
worker_rlimit_nofile 100000;
error_log /var/log/nginx/error.log error;
pid /var/run/nginx.pid;
events {
worker_connections 4096;
use epoll;
multi_accept on;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"'
'uct="$upstream_connect_time"'
'uht="$upstream_header_time"'
'urt="$upstream_response_time"'
'rt="$request_time "';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
#tcp_nodelay on;
# to boost I/O on HDD we can disable access logs
access_log off;
keepalive_timeout 120;
keepalive_requests 10000;
# allow the server to close connection on non responding client, this will free up memory
reset_timedout_connection on;
# request timed out -- default 60
client_body_timeout 10;
# if client stop responding, free up memory -- default 60
send_timeout 2;
#gzip on;
# gzip_static on;
#gzip_min_length 10240;
#gzip_comp_level 1;
#gzip_vary on;
#gzip_disable msie6;
#gzip_proxied expired no-cache no-store private auth;
#gzip_types
# text/html is always compressed by HttpGzipModule
# text/css
# text/javascript
# text/xml
# text/plain
# text/x-component
# application/javascript
# application/x-javascript
# application/json
# application/xml
# application/rss+xml
# application/atom+xml
# font/truetype
# font/opentype
# application/vnd.ms-fontobject
# image/svg+xml;
include /etc/nginx/conf.d/Default.conf;
}
Default.conf
upstream app_server {
server app_server_ip:443;
}
server {
listen 80;
listen 443 ssl backlog=32768;
server_name someIP;
server_tokens off;
location ~ {
proxy_pass https://app_server_ip;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
#proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 75;
proxy_read_timeout 300;
proxy_send_timeout 300;
proxy_http_version 1.1;
proxy_set_header Connection "";
}
ssl_certificate /etc/ssl/certs/uatweb.crt;
ssl_certificate_key /etc/ssl/certs/uatweb.key;
ssl_protocols TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers On;
ssl_ciphers ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS;
#charset koi8-r;
#access_log /var/log/nginx/host.access.log main;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
}
server {
listen 8114 ssl;
server_name someServerName;
ssl_certificate /etc/ssl/certs/uatweb.crt;
ssl_certificate_key /etc/ssl/certs/uatweb.key;
ssl_protocols TLSv1.1 TLSv1.2;
location ~ {
proxy_pass https://someIP;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
}
Please let me know if any changes are required in Config Files.

Related

Nginx https reverse proxy is too slow

I have Implemented the Nginx cache with https reverse proxy in centos, My response time taking more than 1.5 seconds for each request. My nginx server configuration was 4 core, 8gb ram.
My configuration looks like below (nginx.config)
`user nginx;
worker_processes auto;
worker_rlimit_nofile 100000;
error_log /var/log/nginx/error.log;
pid /var/run/nginx.pid;
# Load dynamic modules. See /usr/share/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 80000;
use epoll;
multi_accept on;
}
http {
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 10m;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
log_format rt_cache '$remote_addr - $upstream_cache_status [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"';
# Below pattern will print
# Time stamp | Client IP | client Dev apps Name| Request | Status Returned| Time taken in ms| size Returned in bytes| Referer | hit or miss | User agent
log_format bf_log_format '[$time_local]|'
'$remote_addr|'
'$http_x_developer_username|$http_x_forwarded_for|'
'"$request"|'
'$status|$upstream_response_time|$body_bytes_sent|'
'"$http_referer"|'
'"$upstream_cache_status"|'
'"$http_user_agent"';
log_format json_log_format escape=json '{'
'"time": "$time_iso8601",'
'"trace_id": "$request_id",'
'"http": {'
'"body_bytes_sent": "$body_bytes_sent",'
'"x_developer_username": "$http_x_developer_username",'
'"remote_addr": "$remote_addr",'
'"method": "$request_method",'
'"request": "$request_uri",'
'"schema": "$scheme",'
'"request_time": "$request_time",'
'"host": "$host",'
'"uri": "$uri",'
'"user_agent": "$http_user_agent",'
'"status": "$status"'
'},'
'"proxy": {'
'"host": "$proxy_host"'
'},'
'"upstream": {'
'"response_time": "$upstream_response_time sec",'
'"cache_status": "$upstream_cache_status"'
'}'
'}';
# access_log /var/log/nginx/access.log main;
# access_log /var/log/nginx/access.log json_log_format;
access_log off;
sendfile on;
sendfile_max_chunk 512k;
# directio 4m;
# directio_alignment 512;
tcp_nopush on;
tcp_nodelay on;
reset_timedout_connection on;
keepalive_requests 100000;
types_hash_max_size 2048;
# reduce the data that needs to be sent over network -- for testing environment
gzip on;
# gzip_static on;
gzip_min_length 10240;
gzip_comp_level 1;
gzip_vary on;
gzip_disable msie6;
gzip_proxied expired no-cache no-store private auth;
gzip_types
text/css
text/javascript
text/xml
text/plain
text/x-component
application/javascript
application/x-javascript
application/json
application/xml
application/rss+xml
application/atom+xml
font/truetype
font/opentype
application/vnd.ms-fontobject
image/svg+xml;
include /etc/nginx/mime.types;
default_type application/octet-stream;
# Load modular configuration files from the /etc/nginx/conf.d directory.
# See http://nginx.org/en/docs/ngx_core_module.html#include
# for more information.
proxy_cache_path /opt/nginx/cache levels=1:2 keys_zone=api-cache:3000m max_size=100g inactive=43200m use_temp_path=off;
proxy_temp_path /opt/nginx/cache/other;
include /etc/nginx/conf.d/ssl.conf;
}`
My ssl.confg looks like below
server {
server_name _;
root /usr/share/nginx/html;
listen 443 ssl http2 default_server;
listen [::]:443 ssl;
ssl_certificate "/etc/private/ssl/cert.pem";
ssl_certificate_key "/etc/private/ssl/key.pem";
# ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
keepalive_timeout 100;
# Load configuration files for the default server block.
include /etc/nginx/default.d/*.conf;
location / {
}
error_page 404 /404.html;
location = /40x.html {
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
}
location /health {
default_type application/json;
return 200 '{"status":"UP"}';
}
location /nginx-status {
stub_status;
}
location /trellotest {
proxy_cache_bypass $http_no_cache_purge $arg_nocache;
proxy_cache_methods GET POST;
add_header Cache-Control "public";
proxy_cache api-cache;
proxy_cache_valid 200 40320m;
add_header X-Cache $upstream_cache_status;
add_header X-Time $request_time;
proxy_ignore_headers X-Accel-Expires Expires Cache-Control;
proxy_pass https://mytrelloapp;
}
}
If possible, Anyone could you please advise me if we have anyway to improve the above configurations?

Nginx is failing to load css, js and png of tomcat server

I'm trying to deploy the tomcat & Nginx server on a single AWS EC2 instance. I have 3 instances & on each instance, I wanted to deploy Nginx & Tomcat server. Below is my configuration file
/etc/nginx/nginx.conf
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
events {
worker_connections 768;
# multi_accept on;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
}"
/etc/nginx/conf.d/application.conf
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name localhost;
root /var/lib/tomcat9/webapps/ROOT;
index deploy.html;
location /admin {
try_files $uri $uri/ /deploy.html;
}
location /admin/admin-portal {
alias /opt/tomcat/webapps/ROOT/;
rewrite /admin-portal/(.*) /$1 break;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-Server $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://localhost:8080/;
}
location ~ \.css {
add_header Content-Type text/css;
}
location ~ \.js {
add_header Content-Type application/x-javascript;
}
My goal is, when I hit http://IP/ or HTTP://IP/admin then it should redirect to deploy.html and when I hit HTTP://IP/admin/admi-portal it should open tomcat server
NOTE: I got success in both conditions except when I hit HTTP://IP/admin/admi-portal then it is opening only HTML page and CSS/png/js files getting 404:not found error
/opt/tomcat/webapps/ROOT/ this is the file path for all tomcat static file CSS/js/png etc
Can anyone help me with this?
Try hitting the compete url of your EC2 instance
<instanceip>:8080/admin/admin-portal/
also,
you can add "/" in the end:-
location /admin/admin-portal/
then try hitting the url with
<instance-ip>:8080/admin/admin-portal
Now you don't need to add "/" at the end

Nginx not saving cached 404s to disk

Here's my nginx config (using nginx 1.16.1):
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 100000;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format upstream_time '$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"'
'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
##
# Logging Settings
##
error_log /var/log/nginx/error.log warn;
access_log /var/log/nginx/access.log upstream_time;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_min_length 256;
gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/vnd.ms-fontobject application/x-font-ttf font/opentype image/svg+xml image/x-icon;
#Nginx cache
proxy_cache_path /nginx_cache/product levels=1:2 keys_zone=product_cache:100m max_size=20g inactive=2d use_temp_path=off;
#Serve HTML, JS, CSS & Go requests
server {
client_max_body_size 102M;
listen 443 ssl http2;
server_name example.com;
root /html;
index /;
error_page 404 /404.html;
error_page 500 /500.html;
error_page 502 =503 /maintenance.html;
location = /404.html {
add_header x-nginx-cache-status $upstream_cache_status always;
}
location ~^/([a-zA-Z0-9/]+)$ {
set $product_id $1;
rewrite ^ /product?id=$product_id break;
proxy_cache product_cache;
proxy_http_version 1.1;
proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
proxy_cache_lock on;
proxy_cache_key product-$product_id;
add_header x-nginx-cache-key product-$product_id always;
add_header x-nginx-cache-status $upstream_cache_status always;
proxy_cache_valid 200 404 1d;
proxy_cache_bypass $nocache;
proxy_ignore_headers Cache-Control; #force cache
proxy_ignore_headers Set-Cookie;
proxy_intercept_errors on;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header Host $host;
proxy_set_header REQUEST_URI $request_uri;
proxy_pass http://go:2053;
proxy_buffer_size 128k;
proxy_buffers 4 256k;
proxy_busy_buffers_size 256k;
error_log /var/log/nginx/error.log;
access_log /var/log/nginx/access.log;
}
}
}
With this configuration, I can see that 404s are actually a cache HIT after the first request, however it's not saved in the nginx_cache folder as set.
Other requests which are 200 responses are cached appropriately and saved to disk as expected.
I've tried also adding the same caching config to the location = /404.html block, but that did not have any effect on whether the file was saved to disk.
I am guessing this has to do with overriding the error page by using proxy_intercept_errors and error_page, so nginx is no longer caching it using the parameters I set.
Is there a way to achieve this?

Odoo 13 Web Assets Failure

I have an Odoo instance on a VPS behind a nginx reverse proxy. When I access the Odoo instance directly at the VPS IP:8069 or domain:8069 then the page is displayed correctly and there are no asset errors. If I try to access through the proxy, using ip or domain name (default port 80), some javascript resources don't load and/or images are truncated.
nginx.conf
user nobody;
worker_processes 1;
error_log logs/error.log;
worker_rlimit_nofile 8192;
events {
worker_connections 4096; ## Default: 1024
}
http{
log_format main '$remote_addr - $remote_user [$time_local] $status '
'"$request" $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log logs/access.log main;
server {
listen 80;
server_name www.mysite.com;
rewrite ^(.*) https://$host$1 permanent;
}
server {
listen 443 ssl http2;
server_name www.mysite.com;
proxy_read_timeout 720s;
proxy_connect_timeout 720s;
proxy_send_timeout 720s;
# Add Headers for odoo proxy mode
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Real-IP $remote_addr;
#SSL parameters
ssl on;
ssl_certificate /etc/mycert/location/fullcert.pem;
ssl_certificate_key /etc/mycert/location/pkey.pem;
ssl_session_timeout 30m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
ssl_prefer_server_ciphers on;
ssl_session_tickets off;
#log
access_log /var/log/nginx/odoo.access.log;
error_log /var/log/nginx/odoo.error.log;
location ~* / {
proxy_redirect off;
proxy_pass http://odoo;
}
location /longpolling {
proxy_pass http://odoo-chat;
}
# gzip
gzip_types text/css text/less text/plain text/xml application/xml application/json application/javascript;
gzip on;
}
upstream odoo {
server 127.0.0.1:8069;
}
upstream odoo-chat {
server 127.0.0.1:8072;
}
}
I've tried Odoo 12 and 13 with Python 3.6 based on some Python errors that were mentioned in a forum, no change. I have proxy = true in my odoo.conf file.
2019/10/21 11:56:46 [crit] 2803#0: *1 open() "/var/lib/nginx/tmp/proxy/1/00/0000000001" failed (13: Permission denied) while reading upstream, client: 99.99.254.254, server: www.example.com, request: "GET /web/image/366?access_token=ff2c51a8-716b-448f-98b7-85b97349d39a HTTP/1.1", upstream: "http://127.0.0.1:8069/web/image/366?access_token=ff2c51a8-716b-448f-98b7-85b97349d39a", host: "example.com", referrer: "https://example.com/"
I experienced this problem just now. This is because nginx is dynamically creating temporary files in /var/lib/nginx/tmp/proxy and chown-ing will not solve the issue. What I learned was to set proxy_temp_path:
create a directory wherein user nobody has permissions: e.g. mkdir /home/nobody/tmp
Set proxy_temp_path in nginx.config: proxy_temp_path /home/nobody/tmp 1 2;
An excerpt of config is shown below:
location ~* / {
proxy_temp_path /home/nobody/tmp 1 2;
proxy_redirect off;
proxy_pass http://odoo;
}

Nginx : multiple download freeze my server

We have a problem with Nginx. We have a converter server it's convert MP4 video to MP3 file and 300 user online, so when they start download their MP3 files at the same time, server time response become so huge like if it is freezed even if %vCPU doesn't exceeds 10% when he start the conversion using mpeg library.
My server Configuration :
16 vCPU.
RAM:30G
Data transfert :5TB.
Nginx Configuration (nginx.conf)
user www-data;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
worker_rlimit_nofile 20240;
events {
worker_connections 4000;
multi_accept on;
use epoll;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log off;
limit_req_zone $binary_remote_addr zone=one:10m rate=1r/s;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
server_tokens off;
client_header_timeout 10m;
client_body_timeout 10m;
send_timeout 10m;
client_max_body_size 700m;
connection_pool_size 256;
client_body_buffer_size 1024k;
client_header_buffer_size 8k;
keepalive_timeout 30;
keepalive_requests 100000;
reset_timedout_connection on;
open_file_cache max=200000 inactive=20s;
open_file_cache_valid 30s;
open_file_cache_min_uses 2;
open_file_cache_errors on;
gzip on;
gzip_min_length 10240;
gzip_proxied expired no-cache no-store private auth;
gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml;
gzip_disable "MSIE [1-6]\.";
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
#include /etc/nginx/conf.d/*.conf;
}
Web site nginx configuration:
server {
listen 80 default_server;
listen [::]:80 default_server ipv6only=on;
root /var/www/html;
index index.php index.html index.htm;
client_max_body_size 700m;
connection_pool_size 256;
client_body_buffer_size 1024k;
client_header_buffer_size 8k;
limit_rate 125k;
limit_req zone=one burst=5;
# Make site accessible from http://localhost/
server_name localhost;
location / {
try_files $uri $uri/ =404;
}
error_page 404 /404.html;
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
location ~ \.php$ {
include snippets/fastcgi-php.conf;
fastcgi_pass unix:/run/php/php7.0-fpm.sock;
}
}
Do you know where the problem may come ?

Resources