I have the following to read users:
public asaync Task<IGroupTransitiveMembersCollectionWithReferencesPage> GetGroupMembersPageByIdAsync(string groupId)
{
return await _graphServiceClient
.Groups[groupId]
.TransitiveMembers
.Request()
.Top(MaxResultCount)
.WithMaxRetry(MaxRetries)
.GetAsync();
}
public async Task<IGroupTransitiveMembersCollectionWithReferencesPage> GetGroupMembersNextPageAsnyc(
IGroupTransitiveMembersCollectionWithReferencesPage groupMembersRef,
string nextPageUrl)
{
groupMembersRef.InitializeNextPageRequest(_graphServiceClient, nextPageUrl);
return await groupMembersRef
.NextPageRequest
.GetAsync();
}
public async Task<(List<AzureADUser> users,
Dictionary<string, int> nonUserGraphObjects,
string nextPageUrl,
IGroupTransitiveMembersCollectionWithReferencesPage usersFromGroup)> GetFirstUsersPageAsync(Guid objectId)
{
var users = new List<AzureADUser>();
var nonUserGraphObjects = new Dictionary<string, int>();
var usersFromGroup = await GetGroupMembersPageByIdAsync(objectId.ToString());
TrackMetrics(usersFromGroup.AdditionalData);
usersFromGroup.AdditionalData.TryGetValue("#odata.nextLink", out object nextLink1);
var nextPageUrl = (nextLink1 == null) ? string.Empty : nextLink1.ToString();
users.AddRange(ToUsers(usersFromGroup, nonUserGraphObjects));
return (users, nonUserGraphObjects, nextPageUrl, usersFromGroup);
}
public async Task<(List<AzureADUser> users,
Dictionary<string, int> nonUserGraphObjects,
string nextPageUrl,
IGroupTransitiveMembersCollectionWithReferencesPage usersFromGroup)> GetNextUsersPageAsync(string nextPageUrl, IGroupTransitiveMembersCollectionWithReferencesPage usersFromGroup)
{
var users = new List<AzureADUser>();
var nonUserGraphObjects = new Dictionary<string, int>();
usersFromGroup = await GetGroupMembersNextPageAsnyc(usersFromGroup, nextPageUrl);
TrackMetrics(usersFromGroup.AdditionalData);
usersFromGroup.AdditionalData.TryGetValue("#odata.nextLink", out object nextLink2);
nextPageUrl = (nextLink2 == null) ? string.Empty : nextLink2.ToString();
users.AddRange(ToUsers(usersFromGroup, nonUserGraphObjects));
return (users, nonUserGraphObjects, nextPageUrl, usersFromGroup);
}
On running this code, I see the following exception from the function GetGroupMembersNextPageAsnyc:
Message: The request timed out.
The operation was canceled. Unable to read data from the transport connection: The I/O operation has been aborted because of either a thread exit or an application request.
What am I missing?
Related
I am unable to resolve why my Azure SignalR Hub method is not being triggered?
Environment: Xamarin.Forms client
Disclaimer:
My LocationHub class is in a separate project that is referenced by the project that hosts my Azure Function.
Can Azure SignalR invoke a hub method that's in a separate library?
Server: Here's the hub class:
type LocationHub() =
inherit Hub()
member x.SendLocation(v:SubjectLocation) =
async { do! (x :> Hub).Clients.All.SendAsync($"{v.SessionId}", v) |> Async.AwaitTask } |> Async.StartAsTask
Server: Here's the Azure function that is suppose to trigger the method on the hub class:
public static class LocationFn
{
[FunctionName(nameof(LocationFn))]
public static async Task<IActionResult> Run(
[HttpTrigger(
AuthorizationLevel.Anonymous,
"post",
Route = "locationfn")]
HttpRequest req,
[SignalR(HubName = "LocationHub")]
IAsyncCollector<SignalRMessage> signalRMessages,
ILogger log)
{
log.LogInformation($"{nameof(LocationFn)} has been invoked.");
try
{
using (var streamReader = new StreamReader(req.Body))
{
var json = await streamReader.ReadToEndAsync();
var subjectLocation = JsonConvert.DeserializeObject<SubjectLocation>(json);
await signalRMessages.AddAsync(
new SignalRMessage
{
Target = "SendLocation",
Arguments = new[] { subjectLocation }
});
var message = Log(log, subjectLocation);
return new OkObjectResult(message);
}
}
catch (Exception ex)
{
return new BadRequestObjectResult("There was an error: " + ex.Message);
}
}
static string Log(ILogger log, SubjectLocation subjectLocation)
{
var location = subjectLocation.Location;
var latitude = location.Latitude;
var longitude = location.Longitude;
var message = $"Received location: {subjectLocation.SubjectId} at '({latitude},{longitude})'";
log.LogInformation($"{nameof(LocationFn)} {message}");
return message;
}
}
Appendix:
Client: I have the following client request:
var sessionId = "some_session_id";
await CourierTracking.connectOn(sessionId, locationTracking(), "negotiatefn");
Client: The bird's-eye view of establishing a connection is implemented here:
open System.Diagnostics
open OrderRequest.SignalR.Client
module CourierTracking =
let private onConnectionChanged (_,_) = ()
let private onMessageReceived msg = Debug.WriteLine(sprintf "%A" msg)
let private signalR = SignalRService();
let connectOn(sessionId:string) (serviceHost:string) (resourceName:string) =
signalR.Connected .Add onConnectionChanged
signalR.ConnectionFailed .Add onConnectionChanged
signalR.MessageReceived .Add onMessageReceived
async {
do! signalR.ConnectOn(serviceHost, resourceName, sessionId) |> Async.AwaitTask
} |> Async.StartAsTask
Client: Here's the core implementation for connecting and receiving messages:
public class SignalRService
{
HttpClient _client = new HttpClient();
public delegate void MessageReceivedHandler(object sender, CourierLocation message);
public delegate void ConnectionHandler(object sender, bool successful, string message);
public event MessageReceivedHandler MessageReceived;
public event ConnectionHandler Connected;
public event ConnectionHandler ConnectionFailed;
public bool IsConnected { get; private set; }
public bool IsBusy { get; private set; }
public async Task ConnectOn(string host, string nameOfNegotiationFn, string sessionId)
{
try
{
IsBusy = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
var negotiateJson = await _client.GetStringAsync($"{host}{nameOfNegotiationFn}");
var negotiate = JsonConvert.DeserializeObject<NegotiateInfo>(negotiateJson);
var connection = new HubConnectionBuilder()
.AddNewtonsoftJsonProtocol()
.WithUrl(negotiate.Url, options => options.AccessTokenProvider = async () => negotiate.AccessToken)
.Build();
connection.Closed += Connection_Closed;
connection.On<JObject>(sessionId, OnIncomingMessage);
await connection.StartAsync();
IsConnected = true;
IsBusy = false;
Connected?.Invoke(this, true, "Connection successful.");
}
catch (Exception ex)
{
ConnectionFailed?.Invoke(this, false, ex.Message);
IsConnected = false;
IsBusy = false;
}
}
Task Connection_Closed(Exception arg)
{
ConnectionFailed?.Invoke(this, false, arg.Message);
IsConnected = false;
IsBusy = false;
return Task.CompletedTask;
}
void OnIncomingMessage(JObject message)
{
var courierId = message.GetValue("SubjectId").ToString();
var location = message.SelectToken("Location");
var latitude = double.Parse(location.SelectToken("Latitude").ToString());
var longitude = double.Parse(location.SelectToken("Longitude").ToString());
var courierLocation = new CourierLocation(courierId, new Coordinate(latitude, longitude));
MessageReceived?.Invoke(this, courierLocation);
}
}
I needed the client to pass in the exact name of the hub method that it subscribes to:
var hubMethodName = "LocationUpdate";
...
var connection = new HubConnectionBuilder()
.AddNewtonsoftJsonProtocol()
.WithUrl(negotiate.Url, options => options.AccessTokenProvider = async () => negotiate.AccessToken)
.Build();
connection.Closed -= Connection_Closed;
connection.Closed += Connection_Closed;
connection.On<JObject>(hubMethodName, OnIncomingMessage); // ** REF: HUB METHOD NAME **
await connection.StartAsync();
I have a service bus triggered Azure Function which listens for messages containing just blob URL strings of JSON data which each one of them is at least 10MB.
Message queue is near real-time(If I use the correct term) so producers keep putting messaging to the queue with a frequency so there is always data in the queue to be processed.
I have designed a solution but it gets OutOfMemoryException most of the time. The steps involved in the current solution sequentially are:
Consume a message
Download the file from the URL within the consumed message to a temporary folder
Read the whole file as a string
Deserialize it to an object
Partition into the chunks to supply Mongo bulk upsert limit
Bulk upsert to Mongo
I have tried to solve OutOfMemoryException and I thought that it's because my function/consumer don't have the same pace with the producer, so I think that at the time t1 when it gets the first message and process it and then while it's upserting to the mongo the function keeps getting the messages and they accumulate in the memory and waiting to be upserted.
Is my reasoning right?
Thus I think that If I could implement a streaming solution starting from #3, reading from file by chunking and putting it to a stream then I would prevent the memory keep growing and reduce time also. I have mostly Java background and I somehow know that with custom iterator/spliterator/iterable it is possible to do streaming and asynchronous processing.
How can I do asynchronous data streaming with .Net Core in an Azure Function?
Are there other approaches to solve this problem?
namespace x.y.Z
{
public class MyFunction
{
//...
[FunctionName("my-func")]
public async Task Run([ServiceBusTrigger("my-topic", "my-subscription", Connection = "AzureServiceBus")] string message, ILogger log, ExecutionContext context)
{
var data = new PredictionMessage();
try
{
data = myPredictionService.genericDeserialize(message);
await myPredictionService.ValidateAsync(data);
await myPredictionService.AddAsync(data);
}
catch (Exception ex)
{
//...
}
}
}
}
public class PredictionMessage
{
public string BlobURL { get; set; }
}
namespace x.y.z.Prediction
{
public abstract class BasePredictionService<T> : IBasePredictionService<T> where T : PredictionMessage, new()
{
protected readonly ILogger log;
private static JsonSerializer serializer;
public BasePredictionService(ILogger<BasePredictionService<T>> log)
{
this.log = log;
serializer = new JsonSerializer();
}
public async Task ValidateAsync(T message)
{
//...
}
public T genericDeserialize(string message)
{
return JsonConvert.DeserializeObject<T>(message);
}
public virtual Task AddAsync(T message)
{
throw new System.NotImplementedException();
}
public async Task<string> SerializePredictionResult(T message)
{
var result = string.Empty;
using (WebClient client = new WebClient())
{
var tempPath = Path.Combine(Path.GetTempPath(), DateTime.Now.Ticks + ".json");
Uri srcPath = new Uri(message.BlobURL);
await client.DownloadFileTaskAsync(srcPath, tempPath);
using (FileStream fs = File.Open(tempPath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (BufferedStream bs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bs))
{
result = sr.ReadToEnd();
}
}
Task.Run(() =>
{
File.Delete(tempPath);
});
return result;
}
}
protected TType StreamDataDeserialize<TType>(string streamResult)
{
var body = default(TType);
using (MemoryStream stream = new MemoryStream(Encoding.Default.GetBytes(streamResult)))
{
using (StreamReader streamReader = new StreamReader(stream))
{
body = (TType)serializer.Deserialize(streamReader, typeof(TType));
}
}
return body;
}
protected List<List<TType>> Split<TType>(List<TType> list, int chunkSize = 1000)
{
List<List<TType>> retVal = new List<List<TType>>();
while (list.Count > 0)
{
int count = list.Count > chunkSize ? chunkSize : list.Count;
retVal.Add(list.GetRange(0, count));
list.RemoveRange(0, count);
}
return retVal;
}
}
}
namespace x.y.z.Prediction
{
public class MyPredictionService : BasePredictionService<PredictionMessage>, IMyPredictionService
{
private readonly IMongoDBRepository<MyPrediction> repository;
public MyPredictionService(IMongoDBRepoFactory mongoDBRepoFactory, ILogger<MyPredictionService> log) : base(log)
{
repository = mongoDBRepoFactory.GetRepo<MyPrediction>();
}
public override async Task AddAsync(PredictionMessage message)
{
string streamResult = await base.SerializePredictionResult(message);
var body = base.StreamDataDeserialize<List<MyPrediction>>(streamResult);
if (body != null && body.Count > 0)
{
var chunkList = base.Split(body);
await BulkUpsertProcess(chunkList);
}
}
private async Task BulkUpsertProcess(List<List<MyPrediction>> chunkList)
{
foreach (var perChunk in chunkList)
{
var filterContainers = new List<IDictionary<string, object>>();
var updateContainer = new List<IDictionary<string, object>>();
foreach (var item in perChunk)
{
var filter = new Dictionary<string, object>();
var update = new Dictionary<string, object>();
filter.Add(/*...*/);
filterContainers.Add(filter);
update.Add(/*...*/);
updateContainer.Add(update);
}
await Task.Run(async () =>
{
await repository.BulkUpsertAsync(filterContainers, updateContainer);
});
}
}
}
}
I am trying to send some objects back to client after connecting, but somehow they are receiving them as empty arrays {} when List<T> is sent and as [object Object] for
here is my hub code:
public class MyHub : Hub
{
private static readonly List<User> Users = new List<User>();
private const string DateTimeFormat = "HH:mm:ss tt zzz";
public async Task Join(string username, UserType usertype, string locale, CallerInfo callerInfo)
{
// Add the new user
var user = Users.FirstOrDefault(e => e.Username == username);
if (user != null)
{
Users.Remove(user);
}
Users.Add(new User
{
Username = username,
ConnectionId = Context.ConnectionId,
Type = usertype,
Locale = locale,
ConnectionStartTime = DateTime.Now.ToString(DateTimeFormat),
CallerInfo = callerInfo
});
await SendUserListUpdate(StatusConstants.Join, usertype);
}
public async Task SendUserListUpdate(string status, UserType userType, string conID = "")
{
object[] args = { Users, status, conID == string.Empty ? Context.ConnectionId : conID, userType };
await Clients.All.SendCoreAsync(MethodNameConstants.UpdateUserList, args);
}
}
and here is my client code:
"use strict";
var locale = 'ar-SA';
var username = 'test-user';
var usersList;
var connection = new signalR.HubConnectionBuilder().withUrl("/MyHub").withAutomaticReconnect().build();
// register listeners
connection.on('UpdateUserList',
function(users,status, connectionId, userType) {
usersList = users;
console.log(users);
console.log(usersList);
console.log('status : '+ status);
console.log('connection id : '+ connectionId);
});
// join client list
connection.start().then(async function () {
console.log('trying to join users list');
await connection.invoke('Join',username,'guest', locale, null);
console.log('join successful');
}).catch(function (err) { console.error(err); });
I am trying to receive correct objects on client side so that I can handle them correctly. I couldn't figure out why it is received like this
After making an api call, if i input a wrong detail. My app keeps breaking with a null exception
I tried using the if-else to solve it. but it is still the same error
public class RemoteService
{
HttpClient httpClient;
public RemoteService()
{
httpClient = new HttpClient();
httpClient.BaseAddress = new Uri($"{App.BackendUrl}/");
}
public async Task<WeatherResponse> GetWeatherData(string query)
{
var weatherResponse = new WeatherResponse();
var response = await httpClient.GetAsync($"weather?q=" + query + App.AppID);
if (response.IsSuccessStatusCode)
{
var content = await response.Content.ReadAsStringAsync();
weatherResponse = JsonConvert.DeserializeObject<WeatherResponse>(content);
weatherResponse.Error = false;
return weatherResponse;
}
else
{
//await Application.Current.MainPage.DisplayAlert("Error", "City not found", "OK");
return new WeatherResponse { Error = true };
}
}
}
The problem was actually from the viewmodel class. Solved
I call a web Api that is returning me back a response. at first I define the response as var but not I return to define the response at the top of the method. Ia m not sure what type of variable I have to use for it. I used dictionary but I am getting this is in new keyword.
cannot implicitly convert type 'AnonymousType#1' to Dictionary.
This is my code:
[AllowAnonymous]
[Route("searchuser")]
[HttpPost]
public async Task<ActionResult> SearchUser(UserInfo userInfo)
{
object userObject = null;
Dictionary<string, object> respone = new Dictionary<string, object>();
if (userInfo.LastName != null && userInfo.Zip != null && userInfo.Ssn != null)
{
UserKey userKey = new UserKey();
userKey.AccountKey = accessKey;
var response = await httpClient.PostAsJsonAsync(string.Format("{0}{1}", LoanApiBaseUrlValue, "/verifyuser"), userKey);
if (response.IsSuccessStatusCode)
{
userObject = new JavaScriptSerializer().DeserializeObject(response.Content.ReadAsStringAsync().Result) as object;
var json = response.Content.ReadAsStringAsync().Result;
var userVerify = new JavaScriptSerializer().Deserialize<VerifyUser>(json);
}
}
respone = new // this is where I am getting the error before I was using var and it was working.
{
success = userObject != null,
data = userObject
};
return Json(respone, JsonRequestBehavior.AllowGet);
}
Sending a result in JSON you can use any type of variable (List, Dictionary, int, string, ...).
You can do something like this:
public async Task<HttpResponseMessage> SearchUser(UserInfo userInfo) {
HttpResponseMessage response = new HttpResponseMessage();
try
{
object userObject = null;
// Do something
object result = new { success = userObject != null, data = userObject };
response = Request.CreateResponse(HttpStatusCode.OK, result);
}
catch (Exception e)
{
// Do Log
response = Request.CreateResponse(HttpStatusCode.BadRequest, e.Message);
}
var tc = new TaskCompletionSource<HttpResponseMessage>();
tc.SetResult(response);
return tc.Task;
}
And you'll have your return in JSON, regardless of the type of the variable, which in this case is an anonymous object type.
The error message "cannot implicitly convert type 'AnonymousType#1' to Dictionary." is because you are trying to set a value of anonymous object type to a variable of type Dictionary:
// ...
Dictionary<string, object> respone = new Dictionary<string, object>();
// ...
respone = new // this is where I am getting the error before I was using var and it was working.
{
success = userObject != null,
data = userObject
};