I created a test source which should send a message to the client every x time. This is the ApiController:
public class TestSourceController : ApiController
{
private static readonly ConcurrentQueue<StreamWriter> ConnectedClients = new ConcurrentQueue<StreamWriter>();
[AllowAnonymous]
[Route("api/sources/test")]
public HttpResponseMessage Get()
{
var response = Request.CreateResponse();
response.Content = new PushStreamContent((Action<Stream, HttpContent, TransportContext>) OnStreamAvailable,
"text/event-stream");
return response;
}
private static void OnStreamAvailable(Stream stream, HttpContent headers, TransportContext context)
{
var clientStream = new StreamWriter(stream);
ConnectedClients.Enqueue(clientStream);
}
private static void DoThings()
{
const string outboundMessage = "Test";
foreach (var clientStream in ConnectedClients)
{
clientStream.WriteLine("data:" + JsonConvert.SerializeObject(outboundMessage));
clientStream.Flush();
}
}
}
The clientStream.Flush(); is called like expected and without exceptions.
I handle it in AngularJS like this:
$scope.handleServerCallback = function (data) {
console.log(data);
$scope.$apply(function() {
$scope.serverData = data;
});
};
$scope.listen = function () {
$scope.eventSource = new window.EventSource("http://localhost:18270/api/sources/test");
$scope.eventSource.onmessage = $scope.handleServerCallback;
$scope.eventSource.onopen = function() { console.log("Opened source"); };
$scope.eventSource.onerror = function (e) { console.error(e); };
};
$scope.listen();
My guess is it's a problem with the server since I can see the "EventStream" from the test call is empty in the chrome debugger.
Does anyone know how to make sure the messages arrive at the client?
The solution was quite easy, according to the spec every line has to end with "\n" and the very last line with "\n\n".
So:
clientStream.WriteLine("data:" + JsonConvert.SerializeObject(outboundMessage) + "\n\n");
Solves it.
Related
I have a service bus triggered Azure Function which listens for messages containing just blob URL strings of JSON data which each one of them is at least 10MB.
Message queue is near real-time(If I use the correct term) so producers keep putting messaging to the queue with a frequency so there is always data in the queue to be processed.
I have designed a solution but it gets OutOfMemoryException most of the time. The steps involved in the current solution sequentially are:
Consume a message
Download the file from the URL within the consumed message to a temporary folder
Read the whole file as a string
Deserialize it to an object
Partition into the chunks to supply Mongo bulk upsert limit
Bulk upsert to Mongo
I have tried to solve OutOfMemoryException and I thought that it's because my function/consumer don't have the same pace with the producer, so I think that at the time t1 when it gets the first message and process it and then while it's upserting to the mongo the function keeps getting the messages and they accumulate in the memory and waiting to be upserted.
Is my reasoning right?
Thus I think that If I could implement a streaming solution starting from #3, reading from file by chunking and putting it to a stream then I would prevent the memory keep growing and reduce time also. I have mostly Java background and I somehow know that with custom iterator/spliterator/iterable it is possible to do streaming and asynchronous processing.
How can I do asynchronous data streaming with .Net Core in an Azure Function?
Are there other approaches to solve this problem?
namespace x.y.Z
{
public class MyFunction
{
//...
[FunctionName("my-func")]
public async Task Run([ServiceBusTrigger("my-topic", "my-subscription", Connection = "AzureServiceBus")] string message, ILogger log, ExecutionContext context)
{
var data = new PredictionMessage();
try
{
data = myPredictionService.genericDeserialize(message);
await myPredictionService.ValidateAsync(data);
await myPredictionService.AddAsync(data);
}
catch (Exception ex)
{
//...
}
}
}
}
public class PredictionMessage
{
public string BlobURL { get; set; }
}
namespace x.y.z.Prediction
{
public abstract class BasePredictionService<T> : IBasePredictionService<T> where T : PredictionMessage, new()
{
protected readonly ILogger log;
private static JsonSerializer serializer;
public BasePredictionService(ILogger<BasePredictionService<T>> log)
{
this.log = log;
serializer = new JsonSerializer();
}
public async Task ValidateAsync(T message)
{
//...
}
public T genericDeserialize(string message)
{
return JsonConvert.DeserializeObject<T>(message);
}
public virtual Task AddAsync(T message)
{
throw new System.NotImplementedException();
}
public async Task<string> SerializePredictionResult(T message)
{
var result = string.Empty;
using (WebClient client = new WebClient())
{
var tempPath = Path.Combine(Path.GetTempPath(), DateTime.Now.Ticks + ".json");
Uri srcPath = new Uri(message.BlobURL);
await client.DownloadFileTaskAsync(srcPath, tempPath);
using (FileStream fs = File.Open(tempPath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (BufferedStream bs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bs))
{
result = sr.ReadToEnd();
}
}
Task.Run(() =>
{
File.Delete(tempPath);
});
return result;
}
}
protected TType StreamDataDeserialize<TType>(string streamResult)
{
var body = default(TType);
using (MemoryStream stream = new MemoryStream(Encoding.Default.GetBytes(streamResult)))
{
using (StreamReader streamReader = new StreamReader(stream))
{
body = (TType)serializer.Deserialize(streamReader, typeof(TType));
}
}
return body;
}
protected List<List<TType>> Split<TType>(List<TType> list, int chunkSize = 1000)
{
List<List<TType>> retVal = new List<List<TType>>();
while (list.Count > 0)
{
int count = list.Count > chunkSize ? chunkSize : list.Count;
retVal.Add(list.GetRange(0, count));
list.RemoveRange(0, count);
}
return retVal;
}
}
}
namespace x.y.z.Prediction
{
public class MyPredictionService : BasePredictionService<PredictionMessage>, IMyPredictionService
{
private readonly IMongoDBRepository<MyPrediction> repository;
public MyPredictionService(IMongoDBRepoFactory mongoDBRepoFactory, ILogger<MyPredictionService> log) : base(log)
{
repository = mongoDBRepoFactory.GetRepo<MyPrediction>();
}
public override async Task AddAsync(PredictionMessage message)
{
string streamResult = await base.SerializePredictionResult(message);
var body = base.StreamDataDeserialize<List<MyPrediction>>(streamResult);
if (body != null && body.Count > 0)
{
var chunkList = base.Split(body);
await BulkUpsertProcess(chunkList);
}
}
private async Task BulkUpsertProcess(List<List<MyPrediction>> chunkList)
{
foreach (var perChunk in chunkList)
{
var filterContainers = new List<IDictionary<string, object>>();
var updateContainer = new List<IDictionary<string, object>>();
foreach (var item in perChunk)
{
var filter = new Dictionary<string, object>();
var update = new Dictionary<string, object>();
filter.Add(/*...*/);
filterContainers.Add(filter);
update.Add(/*...*/);
updateContainer.Add(update);
}
await Task.Run(async () =>
{
await repository.BulkUpsertAsync(filterContainers, updateContainer);
});
}
}
}
}
In the Book "Introduction to Vala" by Dr Michael Lauer, he has mentioned that the lib Soup async api is broken. I'm struggling to write a simple example using session.queue_message that query radio stations using the service from radio-browser. Here is my code. I would appreciate any help form experienced Programmers like "Al Thomas". Thank you.
public class Station : Object {
// A globally unique identifier for the change of the station information
public string changeuuid { get; set; default = ""; }
// A globally unique identifier for the station
public string stationuuid { get; set; default = ""; }
// The name of the station
public string name { get; set; default = ""; }
// The stream URL provided by the user
public string url { get; set; default = ""; }
// and so on ... many properties
public string to_string () {
var builder = new StringBuilder ();
builder.append_printf ("\nchangeuuid = %s\n", changeuuid);
builder.append_printf ("stationuuid = %s\n", stationuuid);
builder.append_printf ("name = %s\n", name);
builder.append_printf ("url = %s\n", url);
return (owned) builder.str;
}
}
public class RadioBrowser : Object {
private static Soup.Session session;
// private static MainLoop main_loop;
public const string API_URL = "https://de1.api.radio-browser.info/json/stations";
public const string USER_AGENT = "github.aeldemery.radiolibrary";
public RadioBrowser (string user_agent = USER_AGENT, uint timeout = 50)
requires (timeout > 0)
{
Intl.setlocale ();
session = new Soup.Session ();
session.timeout = timeout;
session.user_agent = user_agent;
session.use_thread_context = true;
// main_loop = new MainLoop ();
}
private void check_response_status (Soup.Message msg) {
if (msg.status_code != 200) {
var str = "Error: Status message error %s.".printf (msg.reason_phrase);
error (str);
}
}
public Gee.ArrayList<Station> listStations () {
var stations = new Gee.ArrayList<Station> ();
var data_list = Datalist<string> ();
data_list.set_data ("limit", "100");
var parser = new Json.Parser ();
parser.array_element.connect ((pars, array, index) => {
var station = Json.gobject_deserialize (typeof (Station), array.get_element (index)) as Station;
assert_nonnull (station);
stations.add (station);
});
var msg = Soup.Form.request_new_from_datalist (
"POST",
API_URL,
data_list
);
// send_message works but not queue_message
// session.send_message (msg);
session.queue_message (msg, (sess, mess) => {
check_response_status (msg);
try {
parser.load_from_data ((string) msg.response_body.flatten ().data);
} catch (Error e) {
error ("Failed to parse data, error:" + e.message);
}
});
return stations;
}
}
int main (string[] args) {
var radio_browser = new RadioBrowser ();
var stations = radio_browser.listStations ();
assert_nonnull (stations);
foreach (var station in stations) {
print (station.to_string ());
}
return 0;
}
While I'm not Al Thomas, I still might be able to help. ;)
For async calls to work in there needs to be a main loop running, and typically from the main program thread. Thus you want to create and execute the main loop from your main() function, rather than in your application code:
int main (string[] args) {
var loop = new GLib.MainLoop ();
var radio_browser = new RadioBrowser ();
// set up async calls here
// then set the main loop running as the last thing
loop.run();
}
Also, if you want to wait for an async call to complete, you typically need to make the call using the yield keyword from another async function E.g:
public async Gee.ArrayList<Station> listStations () {
…
// When this call is made, execution of listStations() will be
// suspended until the soup response is received
yield session.send_async(msg);
// Execution then resumes normally
check_response_status (msg);
parser.load_from_data ((string) msg.response_body.flatten ().data);
…
return stations;
}
You can then call this from the (non-async) main function using the listStations.begin(…) notation:
int main (string[] args) {
var loop = new GLib.MainLoop ();
var radio_browser = new RadioBrowser ();
radio_browser.listStations.begin((obj, res) => {
var stations = radio_browser.listStations.end(res);
…
loop.quit();
});
loop.run();
}
As further reading, I would recommend the async section of the Vala Tutorial, and the asyc examples on the wiki as well.
I'm using Asp.Net Core as a Rest Api Service.
I need access to request and response in ActionFilter. Actually, I found the request in OnActionExcecuted but I can't read the response result.
I'm trying to return value as follow:
[HttpGet]
[ProducesResponseType(typeof(ResponseType), (int)HttpStatusCode.OK)]
[Route("[action]")]
public async Task<IActionResult> Get(CancellationToken cancellationToken)
{
var model = await _responseServices.Get(cancellationToken);
return Ok(model);
}
And in ActionFilter OnExcecuted method as follow:
_request = context.HttpContext.Request.ReadAsString().Result;
_response = context.HttpContext.Response.ReadAsString().Result; //?
I'm trying to get the response in ReadAsString as an Extension method as follow:
public static async Task<string> ReadAsString(this HttpResponse response)
{
var initialBody = response.Body;
var buffer = new byte[Convert.ToInt32(response.ContentLength)];
await response.Body.ReadAsync(buffer, 0, buffer.Length);
var body = Encoding.UTF8.GetString(buffer);
response.Body = initialBody;
return body;
}
But, there is no result!
How I can get the response in OnActionExcecuted?
Thanks, everyone for taking the time to try and help explain
If you're logging for json result/ view result , you don't need to read the whole response stream. Simply serialize the context.Result:
public class MyFilterAttribute : ActionFilterAttribute
{
private ILogger<MyFilterAttribute> logger;
public MyFilterAttribute(ILogger<MyFilterAttribute> logger){
this.logger = logger;
}
public override void OnActionExecuted(ActionExecutedContext context)
{
var result = context.Result;
if (result is JsonResult json)
{
var x = json.Value;
var status = json.StatusCode;
this.logger.LogInformation(JsonConvert.SerializeObject(x));
}
if(result is ViewResult view){
// I think it's better to log ViewData instead of the finally rendered template string
var status = view.StatusCode;
var x = view.ViewData;
var name = view.ViewName;
this.logger.LogInformation(JsonConvert.SerializeObject(x));
}
else{
this.logger.LogInformation("...");
}
}
I know there is already an answer but I want to also add that the problem is the MVC pipeline has not populated the Response.Body when running an ActionFilter so you cannot access it. The Response.Body is populated by the MVC middleware.
If you want to read Response.Body then you need to create your own custom middleware to intercept the call when the Response object has been populated. There are numerous websites that can show you how to do this. One example is here.
As discussed in the other answer, if you want to do it in an ActionFilter you can use the context.Result to access the information.
For logging whole request and response in the ASP.NET Core filter pipeline you can use Result filter attribute
public class LogRequestResponseAttribute : TypeFilterAttribute
{
public LogRequestResponseAttribute() : base(typeof(LogRequestResponseImplementation)) { }
private class LogRequestResponseImplementation : IAsyncResultFilter
{
public async Task OnResultExecutionAsync(ResultExecutingContext context, ResultExecutionDelegate next)
{
var requestHeadersText = CommonLoggingTools.SerializeHeaders(context.HttpContext.Request.Headers);
Log.Information("requestHeaders: " + requestHeadersText);
var requestBodyText = await CommonLoggingTools.FormatRequestBody(context.HttpContext.Request);
Log.Information("requestBody: " + requestBodyText);
await next();
var responseHeadersText = CommonLoggingTools.SerializeHeaders(context.HttpContext.Response.Headers);
Log.Information("responseHeaders: " + responseHeadersText);
var responseBodyText = await CommonLoggingTools.FormatResponseBody(context.HttpContext.Response);
Log.Information("responseBody: " + responseBodyText);
}
}
}
In Startup.cs add
app.UseMiddleware<ResponseRewindMiddleware>();
services.AddScoped<LogRequestResponseAttribute>();
Somewhere add static class
public static class CommonLoggingTools
{
public static async Task<string> FormatRequestBody(HttpRequest request)
{
//This line allows us to set the reader for the request back at the beginning of its stream.
request.EnableRewind();
//We now need to read the request stream. First, we create a new byte[] with the same length as the request stream...
var buffer = new byte[Convert.ToInt32(request.ContentLength)];
//...Then we copy the entire request stream into the new buffer.
await request.Body.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false);
//We convert the byte[] into a string using UTF8 encoding...
var bodyAsText = Encoding.UTF8.GetString(buffer);
//..and finally, assign the read body back to the request body, which is allowed because of EnableRewind()
request.Body.Position = 0;
return $"{request.Scheme} {request.Host}{request.Path} {request.QueryString} {bodyAsText}";
}
public static async Task<string> FormatResponseBody(HttpResponse response)
{
//We need to read the response stream from the beginning...
response.Body.Seek(0, SeekOrigin.Begin);
//...and copy it into a string
string text = await new StreamReader(response.Body).ReadToEndAsync();
//We need to reset the reader for the response so that the client can read it.
response.Body.Seek(0, SeekOrigin.Begin);
response.Body.Position = 0;
//Return the string for the response, including the status code (e.g. 200, 404, 401, etc.)
return $"{response.StatusCode}: {text}";
}
public static string SerializeHeaders(IHeaderDictionary headers)
{
var dict = new Dictionary<string, string>();
foreach (var item in headers.ToList())
{
//if (item.Value != null)
//{
var header = string.Empty;
foreach (var value in item.Value)
{
header += value + " ";
}
// Trim the trailing space and add item to the dictionary
header = header.TrimEnd(" ".ToCharArray());
dict.Add(item.Key, header);
//}
}
return JsonConvert.SerializeObject(dict, Formatting.Indented);
}
}
public class ResponseRewindMiddleware {
private readonly RequestDelegate next;
public ResponseRewindMiddleware(RequestDelegate next) {
this.next = next;
}
public async Task Invoke(HttpContext context) {
Stream originalBody = context.Response.Body;
try {
using (var memStream = new MemoryStream()) {
context.Response.Body = memStream;
await next(context);
//memStream.Position = 0;
//string responseBody = new StreamReader(memStream).ReadToEnd();
memStream.Position = 0;
await memStream.CopyToAsync(originalBody);
}
} finally {
context.Response.Body = originalBody;
}
}
You can also do...
string response = "Hello";
if (result is ObjectResult objectResult)
{
var status = objectResult.StatusCode;
var value = objectResult.Value;
var stringResult = objectResult.ToString();
responce = (JsonConvert.SerializeObject(value));
}
I used this in a .net core app.
Hope it helps.
I'm new to Xamarin.Forms and I'm making a Listview that needs to update every time I insert new information in the database, so far I can display the info of my list and add it via a PHP file but I can't make it refresh automatically.
namespace Proyect
{
[XamlCompilation(XamlCompilationOptions.Compile)]
public partial class Alarms : ContentPage
{
public Alarms ()
{
InitializeComponent();
AlarmsList.ItemTemplate = new DataTemplate(typeof(Cells.AlarmsCell)); //Template of the Alarms
this.LoadAlarms();
}
private async void LoadAlarms()
{
try
{
HttpClient client = new HttpClient();
client.BaseAddress = new Uri("Http://192.168.0.13");
string url = string.Format("/Proyect/alarmscode.php?");
var response = await client.GetAsync(url);
var result = response.Content.ReadAsStringAsync().Result;
var jsonalarms = JsonConvert.DeserializeObject<ObservableCollection<GetAlarms>>(result);
AlarmsList.ItemsSource = jsonalarms;
}
catch (Exception e)
{
await DisplayAlert("ERROR", e + "", "OK");
return;
}
}
}
}
Can you try to keep the same ObservableCollection and update its content instead of setting a new ObservableCollection every time?
namespace Proyect
{
[XamlCompilation(XamlCompilationOptions.Compile)]
public partial class Alarms : ContentPage
{
private ObservableCollection<GetAlarms> _itemsSource = null;
public Alarms()
{
InitializeComponent();
AlarmsList.ItemTemplate = new DataTemplate(typeof(Cells.AlarmsCell)); //Template of the Alarms
_itemsSource = new ObservableCollection<GetAlarms>();
AlarmsList.ItemsSource = _itemsSource;
this.LoadAlarms();
}
private async void LoadAlarms()
{
try
{
HttpClient client = new HttpClient();
client.BaseAddress = new Uri("Http://192.168.0.13");
string url = string.Format("/Proyect/alarmscode.php?");
var response = await client.GetAsync(url);
var result = response.Content.ReadAsStringAsync().Result;
var jsonalarms = JsonConvert.DeserializeObject<ObservableCollection<GetAlarms>>(result);
_itemsSource.Clear();
foreach (var alarm in jsonalarms)
{
_itemsSource.Add(alarm);
}
}
catch (Exception e)
{
await DisplayAlert("ERROR", e + "", "OK");
return;
}
}
}
}
Device.StartTimer (new TimeSpan (0, 0, 10), () => {
// do something every 10 seconds
return true; // runs again, or false to stop
});
If I have this controller:
public class SomeController : ApiController
{
public SomeInfoDto Get()
{
return new SomeInfoDto();
}
}
When I call /api/Some with a get request I will get JSON in the Content Body because JSON.NET has been kind enough to serialise it for me.
However, if I wanted to send a 500 HTTP code and some JSON I thought I could do this:
public HttpResponseMessage Get()
{
return new HttpResponseMessage(HttpStatusCode.InternalServerError)
{
Content = // not sure what to put here
}
}
So I googled to find out what to put for the Content and found this resource
public class JsonContent : HttpContent
{
private readonly JToken _value;
public JsonContent(JToken value)
{
_value = value;
Headers.ContentType = new MediaTypeHeaderValue("application/json");
}
protected override Task SerializeToStreamAsync(Stream stream,
TransportContext context)
{
var jw = new JsonTextWriter(new StreamWriter(stream))
{
Formatting = Formatting.Indented
};
_value.WriteTo(jw);
jw.Flush();
return Task.FromResult<object>(null);
}
protected override bool TryComputeLength(out long length)
{
length = -1;
return false;
}
}
Which I can easily modify to fit my purposes.
However, my question is, if I get SerializeToStreamAsync to use JsonConvert.SerializeObject(_value) further down the Web Api pipeline will it be serialised again?
I have set the GlobalConfiguration.Configuration like this:
var formatters = GlobalConfiguration.Configuration.Formatters;
var jsonFormatter = formatters.JsonFormatter;
var settings = jsonFormatter.SerializerSettings;
settings.Formatting = Formatting.Indented;
settings.ContractResolver = new CamelCasePropertyNamesContractResolver();
My Implementation for the SerializeToStreamAsync is this:
protected override Task SerializeToStreamAsync(Stream stream, TransportContext context)
{
using (var streamWriter = new StreamWriter(stream))
{
streamWriter.WriteAsync(JsonConvert.SerializeObject(this.value)).Wait();
streamWriter.FlushAsync().Wait();
}
return Task.FromResult<object>(null);
}
EDIT: Providing a concrete sample with formatter.
If you want to send JSON content with Http Status Code 500, you can use the extension method HttpRequestMessage.CreateResponse. There is no need for any advanced serialization and formatting. More info on that here.
https://msdn.microsoft.com/en-us/library/system.net.http.httprequestmessageextensions.createresponse%28v=vs.118%29.aspx
Request.CreateResponse(HttpStatusCode.InternalServerError, new string[] { "value1", "value2" }, new JsonMediaTypeFormatter())
(Or)
Request.CreateResponse(HttpStatusCode.InternalServerError, new string[] { "value1", "value2" }, 'application/json')