Nginx reverse proxy on AWS Beanstalk and Cacheing - nginx

I have a reverse proxy set up in AWS Beanstalk. The purpose is for nginx to fetch a value from the upstream Location header and store the result with the cache key of the original request URI so that future clients don't need to follow the redirect too.
Disk space gets up to 68G here: /var/nginx/cache and 30G here: /var/nginx/temp-cache. So, my proxy server's disk space fills up pretty fast.
Anyone know how I can reduce or limit the size of my cache? Or if there is a more efficient better way of doing this so my disk doesn't fill up so fast? Thanks.
worker_processes 1;
user nginx;
pid /var/run/nginx.pid;
events {
worker_connections 65535;
}
worker_rlimit_nofile 30000;
http {
proxy_cache_path /var/nginx/cache keys_zone=rev-origin:100m levels=1:2 inactive=7d max_size=80g;
proxy_temp_path /var/nginx/temp-cache;
server {
listen 80 default_server;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
charset utf-8;
client_max_body_size 500M;
gzip on;
location / {
proxy_pass https://123456abc.execute-api.us-east-1.amazonaws.com/AB/;
proxy_ssl_server_name on;
proxy_ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
proxy_cache rev-origin;
proxy_cache_key $uri;
proxy_cache_valid 200 206 1d;
proxy_intercept_errors on;
recursive_error_pages on;
error_page 301 302 307 = #handle_redirects;
}
location #handle_redirects {
resolver 8.8.8.8;
set $original_uri $uri;
set $orig_loc $upstream_http_location;
# nginx goes to fetch the value from the upstream Location header
proxy_pass $orig_loc;
proxy_cache rev-origin;
proxy_intercept_errors on;
error_page 301 302 307 = #handle_redirect;
# But we store the result with the cache key of the original request URI
# so that future clients don't need to follow the redirect too
proxy_cache_key $original_uri;
proxy_cache_valid 200 206 3000h;
}
}
}

I found a solution by testing a few modules over the past few weeks. What seems to work and reduce the size of the cache is to use these modules:
proxy_max_temp_file_size 1024m;
proxy_buffer_size 4k;
proxy_buffers 4 32k;
proxy_busy_buffers_size 64k;
proxy_temp_file_write_size 64k;

Related

HTTP forwarding Plaintext warning (nginx)

So I was updating my nginx configuration to meet some security checks, to get grade A+ from Qualys SSL Labs.
I do get A+, but one of the warnings I don't understand. I get this:
Though, I do have http redirect to https and it seems its working fine. Does anyone know what would be the cause of this warning?
I found this question: HTTP forwarding PLAINTEXT warning but it talks about apache.
I also found this: https://github.com/ssllabs/ssllabs-scan/issues/154 so it was mentioned that it could be just a bug, but now its not clear (that issue is old).
My nginx configuration:
http {
upstream odoo-upstream {
server odoo:8069 weight=1 fail_timeout=0;
}
upstream odoo-im-upstream {
server odoo:8072 weight=1 fail_timeout=0;
}
# Basic Settings
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
# Enable SSL session caching for improved performance
# http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache
ssl_session_cache shared:ssl_session_cache:5m;
ssl_session_timeout 24h; # time which sessions can be re-used.
# Because the proper rotation of session ticket encryption key is
# not yet implemented in Nginx, we should turn this off for now.
ssl_session_tickets off;
# Default size is 16k, reducing it can slightly improve performance.
ssl_buffer_size 8k;
# Gzip Settings
gzip on;
# http redirects to https
server {
listen 80 default_server;
server_name _;
return 301 https://$host$request_uri;
}
charset utf-8;
server {
# server port and name
listen 443 ssl http2;
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
add_header X-Frame-Options sameorigin;
add_header X-Content-Type-Options nosniff;
add_header X-Xss-Protection "1; mode=block";
# Specifies the maximum accepted body size of a client request,
# as indicated by the request header Content-Length.
client_max_body_size 200m;
# add ssl specific settings
keepalive_timeout 60;
ssl_certificate /etc/ssl/nginx/domain.bundle.crt;
ssl_certificate_key /etc/ssl/nginx/domain.key;
ssl_stapling on;
ssl_stapling_verify on;
resolver 8.8.4.4 8.8.8.8;
ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:ECDHE-RSA-AES128-GCM-SHA256:AES256+EECDH:DHE-RSA-AES128-GCM-SHA256:AES256+EDH:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
ssl_protocols TLSv1.2 TLSv1.3;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/ssl/certs/dhparam.pem;
# increase proxy buffer to handle some Odoo web requests
proxy_buffers 16 64k;
proxy_buffer_size 128k;
#general proxy settings
# force timeouts if the backend dies
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
proxy_read_timeout 600s;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503;
# set headers
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
# Let the Odoo web service know that we’re using HTTPS, otherwise
# it will generate URL using http:// and not https://
proxy_set_header X-Forwarded-Proto $scheme;
# by default, do not forward anything
proxy_redirect off;
proxy_buffering off;
location / {
proxy_pass http://odoo-upstream;
}
location /longpolling {
proxy_pass http://odoo-im-upstream;
}
# cache some static data in memory for 60mins.
# under heavy load this should relieve stress on the Odoo web interface a bit.
location /web/static/ {
proxy_cache_valid 200 60m;
proxy_buffering on;
expires 864000;
proxy_pass http://odoo-upstream;
}
include /etc/nginx/custom_error_page.conf;
}
include /etc/nginx/conf.d/*.conf;
}
events {
worker_connections 1024;
}

How to fix Error 400 Hook should contain payload for Jenkins running behind Nginx server

Jenkins is running behind Nginx server on CentOS virtual machine. I am able to
access Jenkins via web interface in a web browser. Since I want to trigger the automatic builds when the code is pushed to the GitHub repository I have defined a Github repository web hook.
Then I edited the NGINX config file
/etc/nginx/nginx.conf
by adding the location with:
location /github-webhook {
proxy_pass http://localhost:8080/github-webhook;
proxy_method POST;
proxy_connect_timeout 150;
proxy_send_timeout 100;
proxy_read_timeout 100;
proxy_buffers 4 32k;
client_max_body_size 8m;
client_body_buffer_size 128k;
}
But when Github sends a POST request Jenkins sends back 400 Hook should contain payload response. Is there anything I could do to solve this issue?
Below is the complete Nginx config file (the domain name has been changed to xyz.com):
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
# Load modular configuration files from the /etc/nginx/conf.d directory.
# See http://nginx.org/en/docs/ngx_core_module.html#include
# for more information.
include /etc/nginx/conf.d/*.conf;
upstream jenkins{
server 127.0.0.1:8080;
keepalive 16;
}
server {
listen 80 default_server;
listen [::]:80 default_server;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name xyz.com;
ssl_certificate /etc/letsencrypt/live/xyz.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/xyz.com/privkey.pem;
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
ssl_session_tickets off;
# intermediate configuration
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
ssl_prefer_server_ciphers off;
# HSTS (ngx_http_headers_module is required) (63072000 seconds)
add_header Strict-Transport-Security "max-age=63072000" always;
# OCSP stapling
ssl_stapling on;
ssl_stapling_verify on;
# replace with the IP address of your resolver
resolver 127.0.0.1;
ignore_invalid_headers off;
location /github-webhook {
proxy_pass http://localhost:8080/github-webhook;
proxy_method POST;
proxy_connect_timeout 150;
proxy_send_timeout 100;
proxy_read_timeout 100;
proxy_buffers 4 32k;
client_max_body_size 8m;
client_body_buffer_size 128k;
}
location / {
proxy_pass http://jenkins;
# we want to connect to Jenkins via HTTP 1.1 with keep-alive connections
proxy_http_version 1.1;
# has to be copied from server block,
# since we are defining per-location headers, and in
# this case server headers are ignored
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# no Connection header means keep-alive
proxy_set_header Connection "";
# Jenkins will use this header to tell if the connection
# was made via http or https
proxy_set_header X-Forwarded-Proto $scheme;
# increase body size (default is 1mb)
client_max_body_size 10m;
# increase buffer size, not sure how this impacts Jenkins, but it is recommended
# by official guide
client_body_buffer_size 128k;
# block below is for HTTP CLI commands in Jenkins
# increase timeouts for long-running CLI commands (default is 60s)
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
# disable buffering
proxy_buffering off;
proxy_request_buffering off;
}
}
}
And here is the GitHub webhook settings:
In Jenkins projects configuration Github was configured as:
The problem was solved by setting Jenkins URL field with http://localhost:8080/ instead of being xyz.com:8080/. You can can access this field by going to Jenkins > Manage Jenkins > Configure System

Why does NGINX load balancer passive health check not detect when upstream server is offline?

I have an upstream block in an Nginx config file. This block lists multiple backend servers across which to load balance requests to.
...
upstream backend {
server backend1.com;
server backend2.com;
server backend3.com;
}
...
Each of the above 3 backend servers is running a Node application.
If I stop the application process on backend1 - Nginx recognises this, via passive health check, traffic is only directed to backend2 and backend3, as expected.
However, if I power down the server on which backend1 is hosted, Nginx does not recognise that it is offline and continues to attempt to send traffic/requests to it. Nginx still tries to direct traffic to the offline server, resulting in an error: 504.
Can someone shed some light on why this (scenario 2 above) may happen and if there is some further configuration needed that I am missing?
Update:
I'm beginning to wonder if the behaviour I'm seeing is because the above upstream block is located with an HTTP {} Nginx context. If backend1 was indeed powered down, this would be a connection error and so (maybe off the mark here, but just thinking aloud) should this be a TCP health check?
Update 2:
nginx.conf
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
events {
worker_connections 768;
# multi_accept on;
}
http {
upstream backends {
server xx.xx.xx.37:3000 fail_timeout=2s;
server xx.xx.xx.52:3000 fail_timeout=2s;
server xx.xx.xx.69:3000 fail_timeout=2s;
}
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_certificate …
ssl_certificate_key …
ssl_ciphers …;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
default
server {
listen 80;
listen [::]:80;
return 301 https://$host$request_uri;
#server_name ...;
}
server {
listen 443 ssl;
listen [::]:443 ssl;
# SSL configuration
...
# Add index.php to the list if you are using PHP
index index.html index.htm;
server_name _;
location / {
# First attempt to serve request as file, then
# as directory, then fall back to displaying a 404.
try_files $uri $uri/ /index.html;
#try_files $uri $uri/ =404;
}
location /api {
rewrite /api/(.*) /$1 break;
proxy_pass http://backends;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
}
# Requests for socket.io are passed on to Node on port 3000
location /socket.io/ {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_pass http://backends;
}
}
The reason for you to get a 504 is when nginx does HTTP health check it tries to connect to the location(ex: / for 200 status code) which you configured. Since the backend1 is powered down and the port is not listening and the socket is closed.
It will take some time to get timeout exception and hence the 504: gateway timeout.
It's a different case when you stop the application process.The port will not be listening and it will get connection refused which is identified pretty quick and marks the instance as unavailable.
To overcome this you can set fail_timeout=2s to mark the server as unavailable default is 10 seconds.
https://nginx.org/en/docs/http/ngx_http_upstream_module.html?&_ga=2.174685482.969425228.1595841929-1716500038.1594281802#fail_timeout

Redirect NiFi Azure AD OAuth Request usinig NGINX

I have been plugging away at this for a good number of days now and I have been unable to find a solution. We have deployed NiFi onto an HDInsight cluster Edge Node, which comes with NGINX (free) pre-installed. We have installed NiFi onto this node and had the basic, unsecured NiFi UI being served by NGINX. So far, so good.
The requirement is that NiFi be secured using Azure AD and it is this that is giving me issues. The problem is that no matter what I try, I am unable to configure NGINX such that it redirects the OAuth request from NiFi to https://login.microsoftonline.com. The closest I have got is to redirect to the required domain, and include the query string, but missing the location part of the URL (AD tenant ID), which means authentication fails.
So my question is how can I configure NGINX to correctly redirect the OAuth request? I am either able to get:
https://login.microsoftonline.com/?client_id=<ID>&response_type=code&scope=openid+email&state=4bibmlfesmgbmi9o1p8blibd4q&redirect_uri=https%3A%2F%2F<NODE HOSTNAME>%3A443%2Fnifi-api%2Faccess%2Foidc%2Fcallback
or
https://<NODE HOSTNAME>/<TENANT ID>/oauth2/v2.0/authorize?client_id=<ID>&response_type=code&scope=openid+email&state=4bibmlfesmgbmi9o1p8blibd4q&redirect_uri=https%3A%2F%2Fems-poc-tri-hdi-nfi.apps.azurehdinsight.net%3A443%2Fnifi-api%2Faccess%2Foidc%2Fcallback
I am unable to get:
https://login.microsoftonline.com/<TENANT ID>/oauth2/v2.0/authorize?client_id=<ID>&response_type=code&scope=openid+email&state=4bibmlfesmgbmi9o1p8blibd4q&redirect_uri=https%3A%2F%2Fems-poc-tri-hdi-nfi.apps.azurehdinsight.net%3A443%2Fnifi-api%2Faccess%2Foidc%2Fcallback
I have tried all sorts, but I have finally had to admit defeat. The current NGINX config is given below:
user www-data;
worker_processes 4;
pid /var/run/nginx.pid;
events {
worker_connections 768;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_names_hash_bucket_size 1024;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log notice;
rewrite_log on;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
#upstream nifi {
# server 10.1.0.10:9443;
# keepalive 512;
#}
server
{
listen 443 default_server;
#server_name localhost;
server_name ems-poc-tri-hdi-nfi.apps.azurehdinsight.net;
proxy_set_header X-ProxyHost ems-poc-tri-hdi-nfi.apps.azurehdinsight.net;
proxy_set_header X-ProxyScheme https;
proxy_set_header X-ProxyPort 443;
location = / {
return 301 https://10.1.0.10:9443/nifi;
}
location /nifi {
proxy_pass https://10.1.0.10:9443/nifi;
}
location /nifi-api {
proxy_pass https://10.1.0.10:9443/nifi-api;
}
location ~ "\/<AAD TENANT ID>" {
access_log /var/log/nginx/special.access.log;
proxy_set_header Host login.microsoftonline.com;
add_header X-uri "$uri";
add_header X-host "$host";
add_header X-request_uri "$request_uri";
add_header X-args "$args";
if ($request_uri ~ "/[^?]+\?[^?]+callback$") {
#return 307 https://login.microsoftonline.com$uri?$args&redirect=true;
#rewrite ^(?<location>/[^?]+) $location redirect;
#rewrite ^(?<location>/[^?]+) https://login.microsoftonline.com$location redirect;
rewrite ^ $scheme://login.microsoftonline.com$uri?$args&redirect=true? last;
}
if ($request_uri ~ "/[^?]+\?[^?]+true$") {
# proxy_pass https://login.microsoftonline.com;
rewrite ^ https://login.microsoftonline.com last;
#return 307 https://login.microsoftonline.com$uri?$args;
}
}
}
}

Nginx is not able to handle a large number of requests

I have an scenario on where a nginx is in front of an Artifactory server.
Recently, while trying to pull a big number of docker images in a for loop, all at the same time (first test was with 200 images, second test with 120 images), access to Artifactory gets blocked, as nginx is busy processing all the requests and users will not be able to reach it.
My nginx server is running with 4 cpu cores and 8192 of ram.
I have tried to improve the handling of files in the server, by adding the bellow:
sendfile on;
sendfile_max_chunk 512k;
tcp_nopush on;
This made it a bit better (but of course, pull's of images with 1gb+ take much more time, due to the chunk size) - still, access to the UI would cause a lot of timeouts.
Is there something else that i can do to improve the nginx performance, whenever a bigger load is pushed thru it?
I think that my last option is to increase the size of the machine (more cpu's) aswell as the number of processes on nginx (8 to 16).
The full nginx.conf file follows bellow:
user www-data;
worker_processes 8;
pid /var/run/nginx.pid;
events {
worker_connections 19000;
}
http {
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
include /etc/nginx/mime.types;
default_type application/octet-stream;
gzip on;
gzip_disable "msie6";
sendfile on;
sendfile_max_chunk 512k;
tcp_nopush on;
set_real_ip_from 138.190.190.168;
real_ip_header X-Forwarded-For;
log_format custome '$remote_addr - $realip_remote_addr - $remote_user [$time_local] $request_time'
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"';
server {
listen 80 default;
listen [::]:80 default;
server_name _;
return 301 https://$server_name$request_uri;
}
###########################################################
## this configuration was generated by JFrog Artifactory ##
###########################################################
## add ssl entries when https has been set in config
ssl_certificate /etc/ssl/certs/{{ hostname }}.cer;
ssl_certificate_key /etc/ssl/private/{{ hostname }}.key;
ssl_session_cache shared:SSL:1m;
ssl_prefer_server_ciphers on;
## server configuration
server {
listen 443 ssl;
server_name ~(?<repo>.+)\.{{ hostname }} {{ hostname }} _;
if ($http_x_forwarded_proto = '') {
set $http_x_forwarded_proto $scheme;
}
## Application specific logs
access_log /var/log/nginx/{{ hostname }}-access.log custome;
error_log /var/log/nginx/{{ hostname }}-error.log warn;
rewrite ^/$ /webapp/ redirect;
rewrite ^//?(/webapp)?$ /webapp/ redirect;
rewrite ^/(v1|v2)/(.*) /api/docker/$repo/$1/$2;
chunked_transfer_encoding on;
client_max_body_size 0;
location / {
proxy_read_timeout 900;
proxy_max_temp_file_size 10240m;
proxy_pass_header Server;
proxy_cookie_path ~*^/.* /;
proxy_pass http://{{ appserver }}:8081/artifactory/;
proxy_set_header X-Artifactory-Override-Base-Url $http_x_forwarded_proto://$host:$server_port;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
}
Thanks for the tips.
Cheers,
Ricardo

Resources