I have an NGINX server set up to serve static content. Moreover, this NGINX proxies requests coming from bots to a pre-rendering service. Suddenly, and for no apparent reason at all, NGINX stops redirecting these requests to the service. Instead, NGINX keeps on responding with 504 timeouts. I do not understand why it happens abruptly (instead of regularly), and the reasons behind it. I analyzed the request logs, and could not find a pattern.
I have NGINX v1.10.3 running on Ubuntu 16.04 machine.
Below is the configuration files for NGINX:
nginx.conf:
events {
worker_connections 768;
# multi_accept on;
}
http {
##
# Basic Settings
##
client_max_body_size 20M;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
proxy_connect_timeout 10s;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
send_timeout 10s;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
real_ip_header X-Forwarded-For;
set_real_ip_from 10.0.0.0/8;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
log_format main '"$time_local" client=$remote_addr '
'method=$request_method request="$request" '
'request_length=$request_length '
'status=$status bytes_sent=$bytes_sent '
'body_bytes_sent=$body_bytes_sent '
'referer=$http_referer '
'user_agent="$http_user_agent" '
'upstream_addr=$upstream_addr '
'upstream_status=$upstream_status '
'request_time=$request_time '
'upstream_response_time=$upstream_response_time '
'upstream_connect_time=$upstream_connect_time '
'upstream_header_time=$upstream_header_time';
access_log /var/log/nginx/access.log main;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
virtual server:
server {
listen 443 ssl http2;
server_name serverName.com;
ssl on;
ssl_certificate <crtLocation>;
ssl_certificate_key <keyLocation>;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
add_header 'Access-Control-Allow-Origin' "$http_origin" always;
add_header 'Access-Control-Allow-Credentials' 'true' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always;
client_body_timeout 7s;
client_header_timeout 7s;
location / {
set $prerender 0;
if ($http_user_agent ~* "Google|Googlebot|Googlebot\-News|Googlebot\-Image|Googlebot\-Video|Mediapartners\-Google|AdsBot\-Google|Bingbot|MSNBot|MSNBot\-Media|AdIdxBot|BingPreview|Slurp|DuckDuckBot|Baiduspider|Baiduspider|Baiduspider|Yandexbot|facebookexternalhit|FacebookExternalHit|Facebot|Twitterbot|LinkedInBot|rogerbot|DotBot|MJ12bot|AhrefsBot|ia\_archiver|embedly|quora\ link\ preview|showyoubot|outbrain|pinterest|slackbot|vkShare|W3C_Validator"){
set $prerender 1;
}
if ($args ~ "_escaped_fragment_") {
set $prerender 1;
}
if ($http_user_agent ~ "Prerender") {
set $prerender 0;
}
if ($uri ~* "\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)"){
set $prerender 0;
}
if ($prerender = 1) {
rewrite .* /$scheme://$host$request_uri? break;
proxy_pass <proxy server>;
}
try_files $uri $uri/ /index.php;
}
location ~ .php/ { ## Forward paths like /js/index.php/x.js to relevant handler
rewrite ^(.*.php)/ $1 last;
}
location ~ \.php$ {
include fastcgi.conf;
fastcgi_split_path_info ^(.+\.php)(/.+)$;
try_files $uri =404;
fastcgi_pass unix:/var/run/php/php7.0-fpm.sock;
fastcgi_index index.php;
}
}
Related
Here's my nginx config (using nginx 1.16.1):
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 100000;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format upstream_time '$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"'
'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
##
# Logging Settings
##
error_log /var/log/nginx/error.log warn;
access_log /var/log/nginx/access.log upstream_time;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_min_length 256;
gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/vnd.ms-fontobject application/x-font-ttf font/opentype image/svg+xml image/x-icon;
#Nginx cache
proxy_cache_path /nginx_cache/product levels=1:2 keys_zone=product_cache:100m max_size=20g inactive=2d use_temp_path=off;
#Serve HTML, JS, CSS & Go requests
server {
client_max_body_size 102M;
listen 443 ssl http2;
server_name example.com;
root /html;
index /;
error_page 404 /404.html;
error_page 500 /500.html;
error_page 502 =503 /maintenance.html;
location = /404.html {
add_header x-nginx-cache-status $upstream_cache_status always;
}
location ~^/([a-zA-Z0-9/]+)$ {
set $product_id $1;
rewrite ^ /product?id=$product_id break;
proxy_cache product_cache;
proxy_http_version 1.1;
proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
proxy_cache_lock on;
proxy_cache_key product-$product_id;
add_header x-nginx-cache-key product-$product_id always;
add_header x-nginx-cache-status $upstream_cache_status always;
proxy_cache_valid 200 404 1d;
proxy_cache_bypass $nocache;
proxy_ignore_headers Cache-Control; #force cache
proxy_ignore_headers Set-Cookie;
proxy_intercept_errors on;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header Host $host;
proxy_set_header REQUEST_URI $request_uri;
proxy_pass http://go:2053;
proxy_buffer_size 128k;
proxy_buffers 4 256k;
proxy_busy_buffers_size 256k;
error_log /var/log/nginx/error.log;
access_log /var/log/nginx/access.log;
}
}
}
With this configuration, I can see that 404s are actually a cache HIT after the first request, however it's not saved in the nginx_cache folder as set.
Other requests which are 200 responses are cached appropriately and saved to disk as expected.
I've tried also adding the same caching config to the location = /404.html block, but that did not have any effect on whether the file was saved to disk.
I am guessing this has to do with overriding the error page by using proxy_intercept_errors and error_page, so nginx is no longer caching it using the parameters I set.
Is there a way to achieve this?
I dont know what i am doing wrong.
First I tried to add Vhosts in NGINX making new file in sites-available then linken to sites-enabled. This doerst work.
Then I tried to make the vhosts directly in the nginx.conf file. There is the same Problem. It works perfectly for one host but then if I add a another one and try to /etc/init.d/nginx start it always says:
See "systemctl status nginx.service" and "journalctl -xe" for details.
failed!
How can I fix that?
Here ist my nginx.conf
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
events {
worker_connections 768;
# multi_accept on;
}
http {
server_names_hash_bucket_size 64;
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name_;
location / {
root /var/www/html/;
index index.html index.htm index.nginx-debian.html;
autoindex on;
}
location ~ \.php$ {
root /var/www/html/;
include snippets/fastcgi-php.conf;
fastcgi_pass unix:/var/run/php/php7.0-fpm.sock;
}
}
server {
listen 80;
server_name website.de;
location / {
root /var/www/dev1/;
index index.html index.htm;
autoindex on;
}
location ~ \.php$ {
root /var/www/dev1/;
include snippets/fastcgi-php.conf;
fastcgi_pass unix:/var/run/php/php7.0-fpm.sock;
}
}
}
I cannot access to directories inside server's document root.
It throw 403 Forbidden error!
here is /etc/nginx/nginx.conf file:
user www-data;
worker_processes auto;
pid /run/nginx.pid;
events {
worker_connections 768;
# multi_accept on;
}
http {
#################
# Custom config
################
#disable_symlinks off;
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_tokens off;
server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
/etc/nginx/sites-available/default file:
server {
listen 80 default_server;
listen [::]:80 default_server;
root /var/www/html;
index index.html index.htm index.nginx-debian.html;
server_name _;
location / {
try_files $uri $uri/ =404;
}
location ~ \.php$ {
include snippets/fastcgi-php.conf;
fastcgi_pass unix:/run/php/php7.0-fpm.sock;
}
location ~ /\.ht {
deny all;
}
}
server root directory map:
/var/www/html/index.php
<?php include 'folder/file.php';
/var/www/html/folder/file.php
<?php echo 'file included';
if I request http://localhost/index.php it respond me 'file included' but if I request directly for http://localhost/folder/file.php it says me 403 Forbidden!
Basically, my NGINX setup is working fine for 2 of my sites but adding a third redirects to the second one.
server {
listen 80;
root /var/www/html/link.com/public/;
index index.php index.html index.htm index.nginx-debian.html;
server_name www.link.com link.com;
location / {
# URLs to attempt, including pretty ones.
try_files $uri $uri/ /index.php?$query_string;
}
location ~ \.php$ {
include snippets/fastcgi-php.conf;
fastcgi_pass unix:/run/php/php7.0-fpm.sock;
fastcgi_split_path_info ^(.+\.php)(/.+)$;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
include fastcgi_params;
}
location ~ /\.ht {
deny all;
}
}
user www-data;
worker_processes auto;
pid /run/nginx.pid;
events {
worker_connections 768;
# multi_accept on;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
My other 3 sites have the same config but editted accordingly. I also have a default section.
All 4 sites have a symbolic link in sites-enabled. I also havent editted the nginx.conf I dont think.
What could be the issue here?
Just consolidating the links in the comments above and adding a few more for reference :
https://www.nginx.com/resources/wiki/start/topics/examples/full/
https://www.nginx.com/resources/wiki/start/topics/examples/server_blocks/
https://nginx.org/en/docs/example.html
+
multiple websites on nginx & sites-available
Below two are in turn referenced in one of the answers in the above SO post :
http://nginx.org/en/docs/http/request_processing.html
http://nginx.org/en/docs/http/server_names.html
+
https://www.digitalocean.com/community/tutorials/how-to-set-up-nginx-server-blocks-virtual-hosts-on-ubuntu-16-04
While its not quite the standard SO answer, until someone else with better understanding comes along, you can refer these.
when I open 127.0.0.1, the url would jump to 127.0.0.1/k.com/k.com/k/.com/......
my hosts
127.0.0.1 localhost
127.0.0.1 k.com
my nginx config
#user _www;
worker_processes 1;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
access_log off;
gzip off;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.1;
gzip_comp_level 2;
gzip_types text/plain application/json application/x-javascript text/css application/xml text/javascript;
gzip_vary on;
gzip_disable "MSIE [1-6]\.";
include mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
server {
listen 80;
server_name k.com;
client_max_body_size 20M;
root F:/xxxx/www;
underscores_in_headers on;
index index.html;
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Credentials' 'true';
add_header 'Access-Control-Allow-Methods' 'GET,POST,PUT,DELETE';
add_header 'Access-Control-Allow-Headers' 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,username,password,remember,token';
location / {
try_files $uri $uri/ /index.html;
}
location = /index.html {
add_header Cache-Control no-cache;
add_header Cache-Control private;
}
}
server {
listen 80;
server_name localhost;
root F:/xxxx/www/htdocs; #
index index.html index.php;
autoindex on;
location / {
try_files $uri $uri/ /index.html;
}
}
}
how to configure 127.0.0.1 , direct to another directory or empty page, either or direct to localhost.