I am running a few services from my VM at home, and I'm having some issue in connection with bad bots and setting up a https redirect for my subdomains. I would highly appreciate any help in fixing these issues.
The bad_bot issue is that if I enable it in the Nginx file, it won't let me open the webpage from any browser (throws a 403 error). The code is below:
map $http_user_agent $bad_bot {
default 1;
"~*\bUptimeRobot/2.0\b" 0;
}
The other issue is that if I visit any of my subdomains by typing out the link in a browser, it redirects me to Port 80 instead of Port 443 by default. I would like to redirect to Port 443 for all cases. My default file contents are below:
include /etc/nginx/blockuseragents.rules;
include /etc/nginx/bad_bots.rules;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
#server {
# listen 80 default_server;
# listen [::]:80 default_server;
# server_name *.example.in;
# return 301 https://$server_name$request_uri;
#}
#Main Server Configuration Part
server {
#BlockedAgent
if ($blockedagent) {
return 403;
}
#Bad Bots Filtering
#if ($bad_bot) {
# return 403;
#}
#Block Request Method
#if ($request_method !~ ^(GET|HEAD|POST)$) {
# return 444;
#}
listen [::]:443 ssl http2;
listen 443 ssl http2;
server_name example.in;
include /etc/nginx/conf.d/*.conf;
#location / {
#root /usr/share/nginx/html;
#index index.html index.htm index.nginx-debian.html;
#try_files $uri /index.html;
#}
#SSL Configuration
include /etc/nginx/ssl.conf;
#Tautulli
location /tautulli {
proxy_pass http://192.168.0.12:8181;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $server_name;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Ssl on;
}
#Transmission Torrent Client
location /transmission {
proxy_pass http://192.168.0.12:9091;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
#Radarr Movies
location /radarr {
proxy_pass http://192.168.0.12:7878;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
#Sonarr TV Shows
location /sonarr {
proxy_pass http://192.168.0.12:8989;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
#Ombi
location /ombi/ {
proxy_pass http://192.168.0.12:5000;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $server_name;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Ssl on;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 90;
proxy_redirect http://192.168.0.12:5000 https://$host;
}
if ($http_referer ~* /ombi/) {
rewrite ^/dist/([0-9\d*]).js /ombi/dist/$1.js last;
}
#Sabnzbd
location /sabnzbd {
proxy_pass http://192.168.0.12:8080;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
#Jackett
location /jackett {
proxy_pass http://192.168.0.12:9117;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
#Home Assistant Block
#Add entry in Cloudflare DNS ("CNAME home example.DynamicDNSProvider.com") to enable
server {
##BlockedAgent
#if ($blockedagent) {
# return 403;
#}
listen [::]:443 ssl http2;
listen 443 ssl http2;
server_name home.example.in;
#return 301 https://$host$request_uri;
include /etc/nginx/conf.d/*.conf;
#SSL Configuration
include /etc/nginx/ssl.conf;
#Home Assistant
location / {
proxy_pass http://192.168.0.12:8123/;
proxy_set_header Host $host;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
#Home Assistant Google Assistant Block
location /api/google_assistant {
proxy_pass http://192.168.0.12:8123;
proxy_set_header Host $host;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
#Home Assistant API and Websocket
location /api/websocket {
proxy_pass http://192.168.0.12:8123/api/websocket;
proxy_set_header Host $host;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
#Home Assistant Notifications Fix
location /api/notify.html5/callback {
if ($http_authorization = "") { return 403; }
allow all;
proxy_pass http://192.168.0.12:8123;
proxy_set_header Host $host;
proxy_redirect http:// https://;
}
}
#pfSense Block
#Add entry in Cloudflare DNS ("CNAME pfsense example.DynamicDNSProvider.com") to enable
server {
#BlockedAgent
if ($blockedagent) {
return 403;
}
listen [::]:443 ssl http2;
listen 443 ssl http2;
server_name pfsense.example.in;
#return 301 https://$host$request_uri;
include /etc/nginx/conf.d/*.conf;
#SSL Configuration
include /etc/nginx/ssl.conf;
location / {
auth_basic "Restricted";
auth_basic_user_file /etc/nginx/.htpasswd;
proxy_pass https://192.168.0.1:443;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_buffering off;
}
}
#UniFi Controller Block
#Add entry in Cloudflare DNS ("CNAME unifi example.DynamicDNSProvider.com") to enable
server {
#BlockedAgent
if ($blockedagent) {
return 403;
}
listen [::]:443 ssl http2;
listen 443 ssl http2;
server_name unifi.example.in;
#return 301 https://$host$request_uri;
include /etc/nginx/conf.d/*.conf;
#SSL Configuration
include /etc/nginx/ssl.conf;
location / {
#auth_basic "Restricted";
#auth_basic_user_file /etc/nginx/.htpasswd;
proxy_pass https://localhost:8443;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_buffering off;
}
}
#FreeNAS Block
#Add entry in Cloudflare DNS ("CNAME newton example.DynamicDNSProvider.com") to enable
server {
#BlockedAgent
if ($blockedagent) {
return 403;
}
listen [::]:443 ssl http2;
listen 443 ssl http2;
server_name newton.example.in;
#return 301 https://$host$request_uri;
include /etc/nginx/conf.d/*.conf;
#SSL Configuration
include /etc/nginx/ssl.conf;
location / {
auth_basic "Restricted";
auth_basic_user_file /etc/nginx/.htpasswd;
proxy_pass https://192.168.0.10:443;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_buffering off;
}
}
#IPMI Block
#Add entry in Cloudflare DNS ("CNAME ipmi example.DynamicDNSProvider.com") to enable
server {
#BlockedAgent
if ($blockedagent) {
return 403;
}
listen [::]:443 ssl http2;
listen 443 ssl http2;
server_name ipmi.example.in;
#return 301 https://$server_name$request_uri;
include /etc/nginx/conf.d/*.conf;
#SSL Configuration
include /etc/nginx/ssl.conf;
location / {
auth_basic "Restricted";
auth_basic_user_file /etc/nginx/.htpasswd;
proxy_pass https://192.168.0.8:443;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_buffering off;
}
}
Your map directive is back to front. You also need to move the ~* outside the quotes of your regex.
map $http_user_agent $bad_bot {
default 1; #This sets $bad_bot to 1 is nothing else matches
"~*\bUptimeRobot/2.0\b" 0; #This sets $bad_bot to 0 if the regex matches
}
So at this point, if you fixed your regex then UptimeRobot would be $bad_bot 0 and everyone else would be $bad_bot 1
It's not looking good for most people when they get to this part of your config:
if ($bad_bot) {
return 403;
}
Related
I have been trying to solve this issue for quite awhile now. Bots are hitting my sites hard with INVALID HOST HEADERS and Nginx forwards these requests to Gunicorn/Django. I need to stop them at Nginx. I have tried every solution I can find on SO, and elsewhere, but none seem to work for my setup.
Nginx.conf:
upstream backend_server {
server backend:8000;
}
upstream backend_asgi {
server backend_asgi:8001;
}
server {
listen 80;
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
location ~* ^/(api|admin|static|v2) {
return 301 https://$host$request_uri;
}
location / {
return 301 https://$host$request_uri;
}
}
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name example.site *.example.site;
ssl_certificate /etc/letsencrypt/live/example.site/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/example.site/privkey.pem;
include /etc/letsencrypt/options-ssl-nginx.conf;
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
location /ws/ {
proxy_pass http://backend_asgi;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
}
location ~ ^/v2(?:/(.*))?$ {
root /usr/share/nginx/html;
index index.html;
try_files $uri $uri/ /v2/index.html =404;
}
location /backend_static/ {
alias /backend/assets/;
}
location /media/ {
alias /backend/media/;
}
location ~* ^/(api|admin) {
proxy_pass http://backend_server$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $https;
proxy_connect_timeout 360s;
proxy_read_timeout 360s;
}
location / {
proxy_pass http://backend_server$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $https;
proxy_connect_timeout 360s;
proxy_read_timeout 360s;
# Set upload size for videos to be 500MB
client_max_body_size 500M;
}
}
What can i add to my Nginx configuration to stop invalid host headers, given that I have a wildcard subdomain and bots are also using HOST HEADERS w/ subdomains?
Need to configure nginx to have multiple services in same domain.
my nginx.conf file as below
server {
listen 80;
listen [::]:80;
server_name www.example.com example.com;
access_log /var/log/nginx/host.access.log main;
location ^~ /.well-known/acme-challenge/ {
alias /var/www/acme-challenge/;
}
return 301 https://$host$request_uri;
}
server {
listen 443;
listen [::]:443;
server_name www.example.com example.com;
ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem;
ssl on;
access_log /var/log/nginx/host.access.log main;
#
# # Enable server-side protection against BEAST attacks
# ssl_protocols TLSv1.2;
#
ssl_trusted_certificate /etc/letsencrypt/live/example.com/fullchain.pem;
location ^~ /.well-known/acme-challenge/ {
alias /var/www/acme-challenge/;
}
location /cms/ {
proxy_pass http://cms-container:1337;
rewrite ^/sdcms/?(.*)$ /$1 break;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
proxy_redirect off;
}
location /chat {
proxy_pass http://chat-container:3000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
proxy_redirect off;
}
location /auth {
proxy_pass http://auth-container:8080;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
proxy_redirect off;
}
location /service {
proxy_pass http://service-container:3000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
proxy_redirect off;
}
location / {
proxy_pass http://web:8060/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
proxy_redirect off;
}
}
Here, the baseurl (/) works fine. We are able to see other services pages(/auth, /cms, /chat, /services) on the initial load only. When we try to access the pages, it throws an error "page not found" from baseurl application.
I have nginx which I am receiving http request then I will redirect them to https,
server {
listen 80;
server_name example.com;
add_header X-Frame-Options "SAMEORIGIN";
return 301 https://example.com$request_uri;
}
server {
listen 443 ssl http2;
server_name example.com
.
.
.
location / {
proxy_http_version 1.1;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://$backend_server;
}
I have 2 custom header which named :
x_sub_ch1
x_sub_ch2
I want to have them with proxy_pass , I have tried below but does not work, would you please let me know I can add these 2 custom header to proxy_pass:
server {
listen 80;
server_name example.com;
add_header X-Frame-Options "SAMEORIGIN";
return 301 https://example.com$request_uri;
}
server {
listen 443 ssl http2;
server_name example.com
.
.
.
location / {
proxy_http_version 1.1;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# proxy_set_header X-Sub_Ch1 $http_x_sub_ch1; # this does not work
# proxy_set_header X-Sub_Ch1 $http_x_sub_ch2; # this does not work
proxy_set_header X-Sub_Ch1 $http_x-sub-ch1; # this does not work
proxy_set_header X-Sub_Ch1 $http_x-sub-ch2; # this does not work
proxy_pass http://$backend_server;
}
I have a host with two containers:
nginx
check_mk
the check_mk interface is accessible by http://172.17.0.2:5000/cmk
I have proxy_pass rule set up in nginx:
server {
listen 80;
server_name cmk.domain.com;
location / {
proxy_pass http://172.17.0.2:5000;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
}
}
When I hit the nginx on port 80 with cmk.domain.com/cmk it works.
What I want is that when hitting the server_name cmk.domain.com, the /cmk would be added automatically.
I tried doing proxy_pass http://172.17.0.2:5000/cmk; but then I get a page not found error.
What am I missing here?
Try this
server {
listen 80;
server_name cmk.domain.com;
location /cmk {
proxy_pass http://172.17.0.2:5000;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
}
location / {
return 301 http://cmk.domain.com/cmk$request_uri;
}
}
I have an Nginx config similar to:
server {
listen 80;
listen 443;
server_name api.mysite.dev;
location / {
proxy_set_header Host "api.mysite.dev";
proxy_set_header X-Real-IP $remote_addr;
proxy_pass $scheme://127.0.0.1:8001;
}
}
server {
listen 80;
listen 443;
server_name mysite.dev www.mysite.dev;
# Forward all /api/ requests ti api.mysite.dev
# sadly need to keep this around for backwards compatibility
location /api/ {
proxy_set_header Host "api.mysite.dev";
proxy_set_header X-Real-IP $remote_addr;
proxy_pass $scheme://127.0.0.1:8001/;
}
# The proxy_pass here ends up putting the port (8002) in the response URL.
location / {
proxy_set_header Host "www.mysite.dev";
proxy_set_header X-Real-IP $remote_addr;
proxy_pass $scheme://127.0.0.1:8002;
}
}
So, as said in the comment, when I request www.mysite.dev, my browser is forwarded to www.mysite.dev:8002.
Any ideas what I'm doing wrong here?
Thanks in advance!
You have to set the following options:
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host;
This is a sample that works:
server {
listen 80;
listen 443;
server_name api.mysite.dev;
location /api/ {
proxy_pass http://127.0.0.1:8001/;
proxy_redirect off;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header Host "api.mysite.dev";
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
}
}