Problem : Nginx Server is buffring the Server sent events(SSE).
Setup : Node v12.13.1, Nginx 1.16.1, Chrome v80
Scenario:
I tried to turn off buffering with proxy_buffering off; and even added "X-Accel-Buffering": "no" in server resonse header however nginx is still buffering all SSE. if I close node server or restart nginx server then all the SSE message are delivered to client in bulk. I tried alot but dont know that I'm missing.
Nginx Config file :
events {
worker_connections 1024;
}
http {
include mime.types;
sendfile on;
keepalive_timeout 65;
server {
listen 4200;
server_name localhost;
location / {
proxy_set_header Connection '';
proxy_http_version 1.1;
chunked_transfer_encoding off;
proxy_buffering off;
proxy_cache off;
proxy_pass http://localhost:8700;
}
}
}
Node Server :
var express = require('express');
var app = express();
var template =
`<!DOCTYPE html> <html> <body>
<script type="text/javascript">
var source = new EventSource("/events/");
source.onmessage = function(e) {
document.body.innerHTML += e.data + "<br>";
};
</script>
</body> </html>`;
app.get('/', function (req, res) {
res.send(template); // <- Return the static template above
});
var clientId = 0;
var clients = {}; // <- Keep a map of attached clients
// Called once for each new client. Note, this response is left open!
app.get('/events/', function (req, res) {
req.socket.setTimeout(Number.MAX_VALUE);
res.writeHead(200, {
'Content-Type': 'text/event-stream', // <- Important headers
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'X-Accel-Buffering': 'no'
});
res.write('\n');
(function (clientId) {
clients[clientId] = res; // <- Add this client to those we consider "attached"
req.on("close", function () {
delete clients[clientId]
}); // <- Remove this client when he disconnects
})(++clientId)
});
setInterval(function () {
var msg = Math.random();
console.log("Clients: " + Object.keys(clients) + " <- " + msg);
for (clientId in clients) {
clients[clientId].write("data: " + msg + "\n\n"); // <- Push a message to a single attached client
};
}, 2000);
app.listen(process.env.PORT || 8700);
Related
Overview
I have a micro-service solution that includes a Blazor application, web API services, and Duende Identity Server. Also, I used Nginx as a reverse proxy for access to services. When I configure Blazor App to use Duende for authentication directly, it works correctly, but when I want to add Nginx between them, it does not work. I followed IdentityServer/IdentityServer4#1623 (comment) to solve this issue, but I couldn't succeed. Below, I describe the services configuration and what I am doing so far :
Nginx
The Nginx.conf has been configured as follows:
worker_processes 1;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
log_format main '[$time_local]-status :[$status] - $remote_addr - $host - UriAddr: [$request_uri] - XFF : ["$http_x_forwarded_for"] '
'- BodyByteSent : [$body_bytes_sent] "$http_referer" '
'"$request"'; #"$http_user_agent"
access_log logs/access.log main;
sendfile on;
keepalive_timeout 300;
include nginx_http.conf;
}
And nginx_http.conf has been configured as follows:
server {
listen 80;
server_name localhost;
server_tokens off;
gzip on;
gzip_buffers 16 8k;
gzip_comp_level 4;
gzip_http_version 1.0;
gzip_min_length 1280;
gzip_types *;
gzip_vary on;
gzip_disable msie6;
charset UTF-8;
include nginx_access_control.conf;
location / {
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
#proxy_set_header Referer $http_referer;
proxy_pass https://127.0.0.1:5001/;
}
location /Identity/ {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://127.0.0.1:12000/;
}
}
Duende Identity Service
this service runs on https://localhost:12001/ and HTTP://localhost:12000/ with dotnet 6
The Identity Server Configure is as follows:
builder.Services.AddIdentityServer(options =>
{
options.IssuerUri = "http://localhost/Identity/";
})
.AddDeveloperSigningCredential()
.AddInMemoryClients(new List<Client>
{
new Client
{
ClientName = "Web User",
ClientId = "web",/*"D2B8B5EC-9766-40B9-9D95-077B54245E6E",*/
ClientSecrets= {new Secret("123456".Sha256())},
AllowedGrantTypes = GrantTypes.Code,
// RedirectUris={"https://localhost:5001/signin-oidc"},
RedirectUris={"http://localhost/signin-oidc"},
PostLogoutRedirectUris={ "http://localhost/signout-callback-oidc" },
AllowedScopes=
{
"openid",
"profile"
}
}
}).AddInMemoryIdentityResources(new List<IdentityResource>
{
new IdentityResources.OpenId(),
new IdentityResources.Profile(),
})
.AddInMemoryApiScopes(new List<ApiScope>
{
new ApiScope("Management.fullaccess")
})
.AddInMemoryApiResources(new List<ApiResource>
{
new ApiResource("Management","Management Service")
{
Scopes = { "Management.fullaccess" }
}
})
.AddAspNetIdentity<User>();
app.UseHttpsRedirection();
app.UseStaticFiles();
app.UseRouting();
app.UseIdentityServer();
app.UseAuthorization();
app.MapRazorPages();
app.MapControllers();
app.Run();
Blazor App
This service runs on https://localhost:5001/ and HTTP://localhost:5000/ with dotnet 5
The Startup Configuration is as follows:
public void ConfigureServices(IServiceCollection services)
{
var blazorSevice = services.AddControllersWithViews();
services.Configure<ForwardedHeadersOptions>(options =>
{
options.ForwardedHeaders =
ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto;
});
if (environment.IsDevelopment())
{
blazorSevice.AddRazorRuntimeCompilation();
}
services.AddRazorPages();
services.AddServerSideBlazor(o => o.DetailedErrors = true);
services.AddAuthentication(p =>
{
p.DefaultScheme = CookieAuthenticationDefaults.AuthenticationScheme;
p.DefaultChallengeScheme = OpenIdConnectDefaults.AuthenticationScheme;
}).AddCookie(CookieAuthenticationDefaults.AuthenticationScheme)
.AddOpenIdConnect(OpenIdConnectDefaults.AuthenticationScheme, options =>
{
options.SignInScheme = CookieAuthenticationDefaults.AuthenticationScheme;
options.Authority = "http://localhost/Identity";//"http://localhost:12000";
options.MetadataAddress = "http://localhost/Identity/.well-known/openid-configuration";
options.ClientId = "web";
options.ClientSecret = "123456";
options.ResponseType = "code";
options.GetClaimsFromUserInfoEndpoint = true;
options.Scope.Add("profile");
options.Scope.Add("openid");
options.SaveTokens = true;
options.RequireHttpsMetadata = false;
});
services.AddTelerikBlazor();
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
app.UseForwardedHeaders();
app.Use(async (httpcontext, next) =>
{
await next();
string location = httpcontext.Response.Headers[Microsoft.Net.Http.Headers.HeaderNames.Location];
if ((httpcontext.Response.StatusCode == StatusCodes.Status302Found
|| httpcontext.Response.StatusCode == StatusCodes.Status307TemporaryRedirect)
&& location != "https://localhost:5001/signin-oidc"
&& location != "https://localhost:5001/")
{
location = location.Replace("https","http")
.Replace("://localhost:5001/", "://localhost/Identity/");
httpcontext.Response.Headers[Microsoft.Net.Http.Headers.HeaderNames.Location] = location;
}
if (httpcontext.Response.StatusCode == StatusCodes.Status302Found
|| httpcontext.Response.StatusCode == StatusCodes.Status307TemporaryRedirect)
{
if (location != "https://localhost:5001/"
&& location != "https://localhost:5001/signin-oidc"
&& !location.Contains("://localhost/Identity/"))
{
location = location.Replace("%3A5001", "")
.Replace("://localhost/", "://localhost/Identity/")
.Replace("://localhost:12001/", "://localhost/Identity/");
httpcontext.Response.Headers[Microsoft.Net.Http.Headers.HeaderNames.Location] = location;
}
}
});
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
else
{
app.UseExceptionHandler("/Error");
// The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
app.UseHsts();
}
app.UseHttpsRedirection();
app.UseStaticFiles();
app.UseRouting();
app.UseAuthentication();
app.UseAuthorization();
app.UseEndpoints(endpoints =>
{
endpoints.MapDefaultControllerRoute();
endpoints.MapControllers();
endpoints.MapBlazorHub(option =>
option.Transports = HttpTransportType.LongPolling);
endpoints.MapFallbackToPage("/_Host");
});
}
In the above code, I try to change the HTTP response location with middleware to redirect the requests belonging to Duende Identity Server, but I think it's not a common approach!
With all configurations that showed above, now, when I run the blazor app, it redirects to the login page in the Duende service, but when I click the login button, it redirects to /signin-oidc route and shows the error:
It seems that after redirecting to /signin-oidc, the blazor app tries to get a token from /connect/token in the Duende service, and it returns an error.
I searched for similar solutions, but I couldn't find any things! Can anyone help me?
The problem most like lies in the missing x-forwarded-for header. You need to use these headers to tell duende about the fact that you have a reverse proxy.
You can add the headers by adding the following middleware at the start of the pipeline:
app.UseForwardedHeaders(new ForwardedHeadersOptions
{
ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
});
I'm trying to get status code from response application and send it to external api bit in access_by_lua_file ngx.status or ngx.var.status always return 0 or 000. I use log_by_lua_file and get status code as my response application but I can't send it to external api because the API function is disabled is there another way to get status code and send it to external api
this my example code
logging.lua
local request_time = ngx.now() - ngx.req.start_time()
local data = {
request_method = ngx.var.request_method,
user_agent = ngx.var.http_user_agent,
request_time = request_time,
status = ngx.var.status,
url = "http://" .. ngx.var.host .. ngx.var.request_uri,
ip = ngx.var.remote_addr,
time = ngx.var.time_iso8601,
path = ngx.var.request_uri
}
local httpc = require("resty.http").new()
local output = cjson.encode(data)
ngx.log(ngx.ERR, ngx.status, ngx.var.status)
ngx.ctx.status_code = ngx.var.status
ngx.ctx.request_method = ngx.var.request_method
ngx.ctx.user_agent = ngx.var.user_agent
ngx.ctx.request_time = ngx.var.request_time
ngx.ctx.url = "http://" .. ngx.var.host .. ngx.var.request_uri
ngx.ctx.ip = ngx.var.remote_addr
-- ngx.location.capture("http://127.0.0.1:3232/api/create-log", { method = ngx.HTTP_POST, body = output })
-- ngx.location.capture_multi({"http://127.0.0.1:3232/api/create-log", { method = "POST", body = output}})
ngx.log(ngx.ERR, "dasdsadas ", ngx.var.status, " ", ngx.status, " ", ngx.var.upstream_status)
local res, err = httpc:request_uri("http://127.0.0.1:3232/api/create-log", {
method = "POST",
body = output,
headers = {
["Content-Type"] = "application/json",
},
})
if not res then
ngx.log(ngx.ERR, "request failed: ", err)
end
nginx.conf
worker_processes auto;
events {
worker_connections 1024;
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
default_type application/octet-stream;
log_format json_output '{"time_local": "$time_local", '
'"path": "$request_uri", '
'"url": "http://$host$request_uri",'
'"ip": "$remote_addr", '
'"time": "$time_iso8601", '
'"user_agent": "$http_user_agent", '
'"user_id_got": "$uid_got", '
'"user_id_set": "$uid_set", '
'"remote_user": "$remote_user", '
'"request": "$request", '
'"status": "$status", '
'"body_bytes_sent": "$body_bytes_sent", '
'"request_time": "$request_time", '
'"request_method": "$request_method",'
'"http_referrer": "$http_referer" }';
init_by_lua_block {
cjson = require("cjson")
}
error_log logs/error.log error;
server {
listen 8080;
access_log logs/access2.log json_output;
location /hello {
add_header Access-Control-Allow-Headers "Content-Type, Authorization, X-Requested-With, Cache-Control, Accept, Origin, X-Session-ID, Host";
add_header Access-Control-Allow-Methods "OPTIONS, GET, POST, PUT, HEAD, DELETE, OPTIONS, TRACE, CONNECT";
add_header Access-Control-Allow-Origin "*";
if ($request_method = OPTIONS) {
return 200;
}
if ($request_method = HEAD) {
return 200;
}
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
proxy_set_header Connection "upgrade";
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
access_by_lua_file lua/logging.lua;
proxy_pass http://127.0.0.1:3131;
}
location = /favicon.ico {
log_not_found off;
}
}
}
access_by_lua runs before the request is initiated to your upstream. Hence, the status codes won't be set and would be defaulted to 0.
https://stackoverflow.com/a/62202290/9090571
The response & its status code is available in content_by_lua, header_filter_by_lua, body_filter_by_lua, or log_by_lua but you won't be able to directly initiate a httpc request because the API function is disabled.
There is a known workaround that I have used earlier using
https://github.com/openresty/lua-nginx-module#ngxtimerat
local function push_data(request_method, http_user_agent, status, full_uri, args, remote_addr, time_iso8601, request_uri,request_time)
local data = {
request_method = request_method,
user_agent = http_user_agent,
status = status,
full_uri = full_uri,
args = args,
ip = remote_addr,
time = time_iso8601,
path = request_uri,
request_time = request_time,
}
local cjson = require "cjson"
local output = cjson.encode(data)
local httpc = require("/usr/local/openresty/nginx/resty/http").new()
local res, err = httpc:request_uri("http://127.0.0.1:3232/api/create-log", {
method = "POST",
body = output,
headers = {
["Content-Type"] = "application/json",
},
})
if not res then
ngx.log(ngx.ERR, "request failed: ", err)
end
end
local ok, err = ngx.timer.at(0, push_data,
ngx.var.request_method,
ngx.var.http_user_agent,
ngx.var.status,
"http://" .. ngx.var.host .. ngx.var.uri,
ngx.var.args,
ngx.var.remote_addr,
ngx.var.time_iso8601,
ngx.var.request_uri,
ngx.now() - ngx.req.start_time()
-- any other variables you need to set
)
if not ok then
ngx.log(ngx.ERR, "failed to create timer: ", err)
return
end
Or, you could also use https://github.com/cloudflare/lua-resty-logger-socket although it is probably not maintained and has a bunch of open issues.
I'm using a Fastify server to send SSE events to a React front-end.
While everything worked well locally, I'm having issues once deployed behind Nginx. The front-end and the server aren't on the same domain and although I set the cors origin to be "*" on the server, and the other call resolve without issue, for the server-sent-events endpoint only I get
Cross-Origin Request Blocked: The Same Origin Policy disallows reading the remote resource at https://example.com/events. (Reason: CORS request did not succeed). Status code: (null).
Here's how Fastify is configured, using #fastify/cors and fastify-sse-v2
import fastifyCors from "#fastify/cors";
import FastifySSEPlugin from "fastify-sse-v2";
// ...
await this.instance.register(FastifySSEPlugin, { logLevel: "debug" });
await this.instance.after();
await this.instance.register(fastifyCors, { origin: "*" });
Then sending events based on postgres pubsub with:
await pubsub.addChannel(TxUpdateChannel);
reply.sse(
(async function* () {
for await (const [event] of on(pubsub, TxUpdateChannel)) {
yield {
event: event.name,
data: JSON.stringify(event.data),
};
}
})()
);
On the front-end I use eventsource so that I can add Authorization headers:
import EventSource from "eventsource";
// ...
const source = new EventSource(`${SERVER_URL}/transaction/events`, {
headers: {
'Authorization': `Bearer ${jwt}`,
},
});
source.onmessage = (event) => {
console.log('got message', event)
getUserData()
}
source.onopen = (event) => {
console.log('---> open', event)
}
source.onerror = (event) => {
console.error('Event error', event)
}
The problem was in the nginx configuration. Thanks to EventSource / Server-Sent Events through Nginx I solved it using the following placed into the location of the nginx conf :
proxy_set_header Connection '';
proxy_http_version 1.1;
chunked_transfer_encoding off;
proxy_buffering off;
proxy_cache off;
I've asked this question before but decided to delete that old question and reformulate it along with a minimum reproducible example. The issue is that when I deploy my gunicorn webserver on nginx, my streamed responses from my go server via gRPC get truncated. All details can be found in the repository. My nginx configuration for this site looks like this:
server {
listen 80 default_server;
server_name example.com;
location / {
#include proxy_params;
proxy_pass http://localhost:5000;
proxy_buffering off;
chunked_transfer_encoding off;
}
}
The code receiving and parsing the response on the front end looks like this:
<script>
(async function(){
const response = await fetch("{{ url_for('realtimedata') }}");
const reader = response.body.pipeThrough(new TextDecoderStream()).getReader();
while (true) {
const {done, value} = await reader.read();
if (done) break;
try {
console.log('Received', value);
const rtd = JSON.parse(value);
console.log('Parsed', rtd);
} catch(err) {
console.log(err);
}
}
})()
</script>
Something to note regarding the data from the go server, one service is providing a data object with 96 fields and another service is providing data with 200 fields. Which makes the incoming stream response have variable length (in terms of bytes).
I want to use gunicorn because I may have multiple listeners at the same time. Using gunicorn solved an issue where all the responses were making it to the webserver but they were being distributed among the active clients. So each client would get a different response but not all of them.
EDIT:
I've tried changing the response object size on the goserver to be the same from both services but the truncating still happened. Having variable length doesn't seem to be the issue. I've also tried doing this with uWSGI instead of gunicorn and the issue persists. I even set uwsgi_buffering off; and the issue persists.
UPDATE:
I've ran the minimum reproducible example with Apache2 instead of Nginx and I'm getting the same issue. Maybe the issue is with something else.
Looking at your python code, it seems like pushing the data from the backend to the frontend would be done better with websockets. I've rewritten your backend to use FastAPI instead of Flask and modified the nginx configuration.
main.py
import asyncio
import dependencies.rpc_pb2 as r
import dependencies.rpc_pb2_grpc as rpc
from fastapi import FastAPI, WebSocket, Request
from fastapi.templating import Jinja2Templates
import grpc
import json
import os
os.environ["GRPC_SSL_CIPHER_SUITES"] = 'HIGH+ECDSA'
app = FastAPI()
templates = Jinja2Templates(directory="templates")
server_addr = "localhost"
server_port = 3567
#app.get("/")
def read_root(request: Request):
return templates.TemplateResponse("index.html", {"request": request})
def parseRtd(rtd):
rtdDict = {}
rtdDict["source"] = rtd.source
rtdDict["is_scanning"] = rtd.is_scanning
rtdDict["timestamp"] = int(rtd.timestamp)
rtdDict["data"] = {}
for key, v in rtd.data.items():
rtdDict["data"][int(key)] = {"name": v.name, "value": v.value}
return rtdDict
def get_rtd():
channel = grpc.insecure_channel(f"{server_addr}:{server_port}")
stub = rpc.RpcServiceStub(channel)
for rtd in stub.SubscribeDataStream(r.SubscribeDataRequest()):
yield parseRtd(rtd)
#app.websocket("/ws")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
await websocket.send_json({"test": "this is a test"})
it = get_rtd()
while True:
await asyncio.sleep(0.1)
payload = next(it)
await websocket.send_json(payload)
index.html
<html>
<head>
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.4.0/socket.io.js" integrity="sha512-nYuHvSAhY5lFZ4ixSViOwsEKFvlxHMU2NHts1ILuJgOS6ptUmAGt/0i5czIgMOahKZ6JN84YFDA+mCdky7dD8A==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
</head>
<body>
<script>
var ws = new WebSocket("ws://localhost:5000/ws");
ws.onopen = function () {
console.log("websocket was open");
};
ws.onclose = () => {
console.log("Websocket was closed!");
}
ws.onerror = (error) =>{
console.error("Websocket error: " + JSON.stringify(error));
};
ws.onmessage = (message) => {
console.log("MSG: " + message.data );
};
</script>
</body>
</html>
webserver.conf
server {
listen 80 default_server;
server_name example.com;
location / {
include proxy_params;
proxy_pass http://localhost:5000;
}
location /ws {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_pass http://localhost:5000;
}
}
I'm quite confused. I was testing this application on localhost with Mamp and everything was working fine but when moved to the development server the client stop receiving messages from server. I'm using it inside a Vuejs component.
On the client I've logged socket.on('connect') and the second check is returning true.
This is my code:
Server
var server = require ('http').Server();
var io = require ('socket.io')(server);
var Redis = require ('ioredis');
var redis = new Redis();
redis.subscribe('chat');
redis.on('message', (channel, message) => {
message = JSON.parse(message);
// channel:event:to_id:to_type - message.data
io.emit(channel + ':' + message.event + ':' + message.to_id + ':' + message.to_type, message.data);
console.log(message +' '+ channel);
});
server.listen('6001');
Client
var io = require('socket.io-client')
var socket = io.connect('http://localhost:6001', {reconnect: true});
...
mounted() {
console.log('check 1', socket.connected);
socket.on('connect', function() {
console.log('check 2', socket.connected);
});
socket.on('chat:newMessage:'+this.fromid+':'+this.fromtype, (data) => {
console.log('new message');
var message = {
'msg': data.message,
'type': 'received',
'color': 'green',
'pos': 'justify-content-start',
}
this.messages.push(message);
});
}
Nginx conf
upstream node1 {
server 127.0.0.1:6001;
}
server {
listen 80;
server_name localhost;
location / {
#Configure proxy to pass data to upstream node1
proxy_pass http://node1/socket.io/;
#HTTP version 1.1 is needed for sockets
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}
thanks a lot!
Ok I've found a solution.
I was using localhost
I've replaced
var socket = io.connect('http://localhost:6001', {reconnect: true});
With:
var socket = io.connect('http://'+ window.location.hostname +':6001', {reconnect: true});
And now everything is working fine.