How to make streaming MP4 from reverse proxy? - nginx

I try to make little youtube. I have two servers.
Backend server Ubuntu16. Server has application + mp4 files.
Django + uWSGI +
NGINX
user www-data;
worker_processes auto;
pid /etc/nginx/logs/nginx.pid;
events {
worker_connections 1024;
multi_accept on;
}
http {
upstream django {
server 127.0.0.1:8000;
}
server {
listen 80;
server_name XXX.XXX.XXX.191;
charset utf-8;
gzip on;
gzip_static on;
gzip_comp_level 5;
gzip_min_length 1024;
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
if_modified_since off;
add_header Accept-Ranges bytes;
location /static/ {
root /home/tube;
}
location / {
uwsgi_pass django;
include uwsgi_params;
}
}
}
When I load page with video from browser using http://www.backend/, I see:
1.mp4 - answer 206 - chunk 2mb. After I click to play, the video starts to play chunk by chunk with answer 206. Seems to be correct. This is backend server (it has public access )
Frontend server Ubuntu16. NGINX as reverse proxy.
user www-data;
worker_processes auto;
pid /etc/nginx/logs/nginx.pid;
events {worker_connections 1024;}
http {
include mime.types;
default_type application/octet-stream;
server {
listen 80;
server_name XXX.XXX.XXX.92;
proxy_http_version 1.1;
proxy_pass_request_headers on;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Range $http_range;
proxy_set_header If-Range $http_if_range;
proxy_no_cache $http_range $http_if_range;
location / {
proxy_pass http://XXX.XXX.XXX.191/;
}
}
}
When I load the same page with video from browser using http://www.frontend/
I see:
1.mp4 - answer 200 - preload starts until full lenght of video (~150mb), instead 2mb chunk. After preload finished, I can click play and video starts streaming, loading mp4 again (answer 200)
I want that frontend server streaming videos correctly. First chunk 2-4mb after I load html, the rest chunks after i click play.

Related

How to cache NextJS 10.0 images using NGINX

We would like to launch a NextJS 10 app using NGINX so we use a configuration similar to:
location /_next/static/ {
alias /home/ec2-user/my-app/.next/static/;
expires 1y;
access_log on;
}
It works great, it caches for a year our statics but as we use NextJS images I'm failing to add an expires tag on on-the-fly resized images.
If I do:
location /_next/image/ {
alias /home/ec2-user/my-app/.next/image;
expires 1y;
access_log on;
}
It just returns a 404 on images.
Here is my server part NGINX config :
server {
listen 80;
server_name *.my-website.com;
# root /usr/share/nginx/html;
# root /home/ec2-user/my-app;
charset utf-8;
client_max_body_size 20M;
client_body_buffer_size 20M;
proxy_connect_timeout 600;
proxy_send_timeout 600;
proxy_read_timeout 600;
send_timeout 600;
underscores_in_headers on;
add_header X-Frame-Options SAMEORIGIN always;
add_header X-Content-Type-Options nosniff always;
add_header X-XSS-Protection "1; mode=block" always;
add_header Referrer-Policy "same-origin" always;
location = /robots.txt {
proxy_pass https://api.my-website.com/robots.txt;
}
location /_next/static/ {
alias /home/ec2-user/my-app/.next/static/;
expires 1y;
access_log on;
}
location / {
# reverse proxy for merchant next server
proxy_pass http://localhost:3000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass_request_headers on;
proxy_cache_bypass $http_upgrade;
proxy_buffering off;
}
}
Here is an example how you can rely of upstream Content-Type header to set up the Expires and Cache-Control headers:
map $upstream_http_content_type $expire {
~^image/ 1y; # 'image/*' content type
default off;
}
server {
...
location / {
# reverse proxy for merchant next server
proxy_pass http://localhost:3000;
...
expires $expire;
}
}
The same way you can tune cache control headers for any other content type of proxied response. The $upstream_http_<name> nginx variable is described here.
Update
To add cache control headers only by specific URIs you can use two chained map blocks:
map $uri $expire_by_uri {
~^/_next/image/ 1y;
default off;
}
map $upstream_http_content_type $expire {
~^image/ $expire_by_uri;
default off;
}
And if you don't expect anything but the images from /_next/image/... URIs, you can just use the
map $uri $expire {
~^/_next/image/ 1y;
default off;
}

Nginx Static content caching proxy_cache_bypass proxy_no_cache

I have a problem with using nginx as a load balancer. I could configure it to work as a load balancer but I don't how to make it cache static contents from the proxied servers in the backend such as html,css,js, etc... This means I want nginx to
weather to cache or not based on the content of the response from the backend servers if it changed to bypass cache and send requests to the backend and if not to serve from cache. I tried and seached a lot in the internet to make it using many directives such as proxy_cache_bypass and proxy_no_cache but I couldn't. Is there any means to do this if anyone has experience in such topic. these are the configurations:
upstream backend {
server www.webserver1.com:443 max_fails=3 fail_timeout=15s;
server www.webserver2.com:443 max_fails=3 fail_timeout=15s;
}
server {
listen 443 ssl;
rewrite_log on;
error_log /var/log/nginx/lb.error.log;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Proxy-Cache $upstream_cache_status;
ssl_certificate /etc/nginx/client.crt;
ssl_certificate_key /etc/nginx/client.key;
ssl on;
location / {
proxy_cache backcache;
#proxy_cache_methods GET HEAD POST;
#proxy_cache_bypass $cookie_nocache $arg_nocache;
#proxy_no_cache $cookie_nocache $arg_nocache;
proxy_cache_min_uses 1;
#proxy_cache_revalidate on;
#proxy_cache_valid 200 4m;
proxy_cache_lock on;
proxy_cache_background_update on;
add_header X-Proxy-Cache $upstream_cache_status;
proxy_pass https://backend;
}
}
server {
listen 80 ;
if ($http_x_forwarded_proto != 'https') {
rewrite ^(.*) https://$host$1 redirect;
}
}
these are the contents of a config. file under /etc/nginx/conf.d/ which is included in the main config. file which is /etc/nginx/nginx.conf and also those 2 lines are in the main config. file :
proxy_cache_path /var/lib/nginx/cache keys_zone=backcache:20m max_size=100m;
proxy_cache_key "$scheme$request_method$host$request_uri$is_args$args$cookie_user";
Your backend servers could be the root cause of that problem, if those servers were improperly configured. For example sending Cache-Control headers on requests to static files.
According to that docs by default, NGINX respects the Cache-Control headers from origin servers. It does not cache responses with Cache-Control set to Private, No-Cache, or No-Store or with Set-Cookie in the response header.
You can permanently change this behavior by adding those directives:
proxy_ignore_headers Cache-Control;
proxy_cache_valid any 30m;
So the config will look like:
location / {
proxy_cache backcache;
proxy_cache_revalidate on;
proxy_cache_min_uses 3;
proxy_cache_valid 200 302 10m;;
proxy_cache_lock on;
proxy_cache_background_update on;
proxy_ignore_headers Cache-Control;
proxy_cache_valid any 30m;
add_header X-Proxy-Cache $upstream_cache_status;
proxy_pass https://backend;
}
Hope it will help you to figure out.

NGINX configuration for gunicorn and prerender.io

I am currently serving my website using Nginx and Gunicorn.
In particular, Nginx is serving static files and Gunicorn is serving rest-api.
This is my current Nginx configuration:
worker_processes 2;
user nobody nogroup;
# 'user nobody nobody;' for systems with 'nobody' as a group instead
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024; # increase if you have lots of clients
accept_mutex on; # set to 'on' if nginx worker_processes > 1
# 'use epoll;' to enable for Linux 2.6+
# 'use kqueue;' to enable for FreeBSD, OSX
}
http {
include mime.types;
# fallback in case we can't determine a type
default_type application/octet-stream;
access_log /var/log/nginx/access.log combined;
sendfile on;
proxy_connect_timeout 300;
proxy_send_timeout 300;
proxy_read_timeout 300;
send_timeout 300;
upstream app_server {
# fail_timeout=0 means we always retry an upstream even if it failed
# to return a good HTTP response
# for UNIX domain socket setups
# server unix:/tmp/gunicorn.sock fail_timeout=0;
# for a TCP configuration
server 127.0.0.1:8181 fail_timeout=0;
}
server {
listen 80;
listen [::]:80;
server_name www.miralytics.social;
return 301 https://www.miralytics.social$request_uri;
}
server {
# if no Host match, close the connection to prevent host spoofing
listen 443 default ssl;
ssl_certificate /certificates/fullchain1.pem;
ssl_certificate_key /certificates/privkey1.pem;
server_name www.miralytics.social;
gzip on;
gzip_vary on;
gzip_types text/plain text/html text/xml text/css application/x-javascript image/png image/jpeg application/javascript application/octet-stream application/json;
gzip_proxied any;
gzip_http_version 1.1;
gzip_min_length 0;
gzip_comp_level 9;
gzip_buffers 16 8k;
proxy_connect_timeout 600;
proxy_send_timeout 600;
proxy_read_timeout 600;
send_timeout 600;
keepalive_timeout 5;
# path for static files
root /home/edge7/UIBackend/dist;
location ~ ^/(images|javascript|js|css|flash|media|static)/ {
root /home/edge7/UIBackend/dist;
expires 1d;
}
location /auth/register {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://localhost:8181;
}
location /auth/login {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://localhost:8181;
}
location / {
# checks for static file, if not found proxy to app
try_files $uri #proxy_to_app;
}
location #proxy_to_app {
proxy_pass http://localhost:8181;
}
add_header Cache-Control no-cache; #(no cache for testing reasons)
}
}
Here the official prerender configuration for Nginx, but as you can see it does not fit my current configuration because I already have #proxy_to_app.
Has anyone experience with this?
You can just modify your config a little bit so where you have this:
location / {
# checks for static file, if not found proxy to app
try_files $uri #proxy_to_app;
}
You would want to change that to:
location / {
proxy_set_header X-Prerender-Token YOUR_TOKEN;
set $prerender 0;
if ($http_user_agent ~* "googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest|slackbot|vkShare|W3C_Validator") {
set $prerender 1;
}
if ($args ~ "_escaped_fragment_") {
set $prerender 1;
}
if ($http_user_agent ~ "Prerender") {
set $prerender 0;
}
if ($uri ~* "\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)") {
set $prerender 0;
}
#resolve using Google's DNS server to force DNS resolution and prevent caching of IPs
resolver 8.8.8.8;
if ($prerender = 1) {
#setting prerender as a variable forces DNS resolution since nginx caches IPs and doesnt play well with load balancing
set $prerender "service.prerender.io";
rewrite .* /$scheme://$host$request_uri? break;
proxy_pass http://$prerender;
}
# checks for static file, if not found proxy to app
try_files $uri #proxy_to_app;
}

Use prerender.io with my existing nginx configuration

I am trying to implement Dynamic Rendering for google.
I have a server on Laravel Forge for serving a Nuxt.js application and I want to use prerender.io
Prerender.io gives an nginx config to use but my current config seems a lot different from it. I don't have any experience with it so I am asking for help if there is anyone who can help.
This is my current nginx config
# FORGE CONFIG (DO NOT REMOVE!)
include forge-conf/beta.example.com/before/*;
map $sent_http_content_type $expires {
"text/html" epoch;
"text/html; charset=utf-8" epoch;
default off;
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name beta.example.com;
root /home/forge/beta.example.com/dist;
# FORGE SSL (DO NOT REMOVE!)
ssl_certificate /etc/nginx/ssl/beta.example.com/50331/server.crt;
ssl_certificate_key /etc/nginx/ssl/beta.example.com/50331/server.key;
ssl_protocols TLSv1.2;
ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/nginx/dhparams.pem;
add_header X-Frame-Options "SAMEORIGIN";
add_header X-XSS-Protection "1; mode=block";
add_header X-Content-Type-Options "nosniff";
charset utf-8;
gzip on;
gzip_types text/plain application/xml text/css application/javascript;
gzip_min_length 1000;
# FORGE CONFIG (DO NOT REMOVE!)
include forge-conf/beta.example.com/server/*;
location / {
expires $expires;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 1m;
proxy_connect_timeout 1m;
proxy_pass http://127.0.0.1:3000; # set the adress of the Node.js
}
access_log off;
error_log /var/log/nginx/beta.example.com-error.log error;
location ~ /\.(?!well-known).* {
deny all;
}
}
# FORGE CONFIG (DO NOT REMOVE!)
include forge-conf/beta.example.com/after/*;
This is how it should look like
https://gist.github.com/thoop/8165802
My main question basically is what happens to the current location / block.
You have two ways to achieve your goal:
Nuxt.js has its own pre-render solution. you could find it here.
If you still want to use prerender.io, here is an example:
I saw that your Nginx is proxying all traffic to http://127.0.0.1:3000, which is presumably a backend written in Node.js. So you could directly update your location / block to use prerender.io to catch everything.
...
location / {
proxy_set_header X-Prerender-Token YOUR_TOKEN;
set $prerender 0;
if ($http_user_agent ~* "googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest|slackbot|vkShare|W3C_Validator") {
set $prerender 1;
}
if ($args ~ "_escaped_fragment_") {
set $prerender 1;
}
if ($http_user_agent ~ "Prerender") {
set $prerender 0;
}
if ($uri ~* "\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)") {
set $prerender 0;
}
#resolve using Google's DNS server to force DNS resolution and prevent caching of IPs
resolver 8.8.8.8;
if ($prerender = 1) {
#setting prerender as a variable forces DNS resolution since nginx caches IPs and doesnt play well with load balancing
set $prerender "service.prerender.io";
rewrite .* /$scheme://$host$request_uri? break;
proxy_pass http://$prerender;
}
try_files #backend;
}
location #backend {
expires $expires;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 1m;
proxy_connect_timeout 1m;
proxy_pass http://127.0.0.1:3000; # set the adress of the Node.js
}
...

Senaite LIMS (Plone 4.3.18) css not working on Nginx with https enabled

I've installed and set up senaite.lims, which is a Plone extension, running on Plone 4.3.18 installed by the Unified Installer, and adding senaite.lims to the buildout.cfg eggs.
It's running fine on port 8080, and I can get Nginx to work redirecting / to :8080, but when I start using https, suddenly the css of the site doesn't work anymore.
I looked at the source, and the produced html page shows a link to the stylesheet with http://.... which I don't know if may cause problems, but if I actually try to open the .css file in the browser it works fine.
I set up and tried both with port 80 redirecting the https, and serving both a version of http and https, but neither one would get the page to render using .css. If anyone has any tips, or sees something wrongly configured in the nginx below, any help would be greatly appreciated.
Here is my nginx.conf:
user www-data;
worker_processes auto;
pid /run/nginx.pid;
events {
worker_connections 768;
}
http {
default_type application/octet-stream;
include /etc/nginx/mime.types;
sendfile on;
keepalive_timeout 75;
upstream plone {
server 127.0.0.1:8080;
}
server {
listen 80;
listen 443 ssl http2;
server_name 99.99.99.99; # changed for posting on SO
ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt;
ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key;
error_log /var/log/nginx/nginx.vhost.error.log;
location / {
proxy_pass http://localhost:8080/;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_buffer_size 128k;
proxy_buffers 8 128k;
proxy_busy_buffers_size 256k;
}
}
}
You missed to rewrite the URL, e.g:
rewrite ^(.*)$ /VirtualHostBase/$scheme/$host/senaite/VirtualHostRoot/$1 break;
Here is a complete working config for SENAITE:
server {
listen 80;
server_name senaite.mydomain.com;
return 301 https://$server_name$request_uri;
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name senaite.mydomain.com;
# https://www.digitalocean.com/community/tutorials/how-to-secure-nginx-with-let-s-encrypt-on-ubuntu-16-04
include snippets/ssl-senaite.mydomain.com.conf;
include snippets/ssl-params.conf;
include snippets/well-known.conf;
access_log /var/log/nginx/senaite.access.log;
error_log /var/log/nginx/senaite.error.log error;
# Allow Cross-Origin Resource Sharing from our HTTP domain
add_header "Access-Control-Allow-Origin" "http://senaite.ridingbytes.com";
add_header "Access-Control-Allow-Credentials" "true";
add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS";
add_header "X-Frame-Options" "SAMEORIGIN";
if ($http_cookie ~* "__ac=([^;]+)(?:;|$)" ) {
# prevent infinite recursions between http and https
break;
}
# rewrite ^(.*)(/logged_out)(.*) http://$server_name$1$2$3 redirect;
location / {
set $backend http://haproxy;
# API calls take a different backend w/o caching
if ($uri ~* "##API") {
set $backend http://api;
}
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
rewrite ^(.*)$ /VirtualHostBase/$scheme/$host/senaite/VirtualHostRoot/$1 break;
# proxy_pass $backend;
proxy_pass http://plone;
}
}

Resources