I have some very heavy calculations for some statistics on my website that I want to update every 8 hours. I'm using a HostedService to update the cache every 8 hours. To disable the website, I create the app_offline.htm file at the root of the website.
This is a simplified version of the code:
public class CacheUpdaterHostedService : IHostedService
{
private readonly IServiceScopeFactory _scopeFactory;
private Timer _timer;
public CacheUpdaterHostedService(IServiceScopeFactory scopeFactory)
{
_scopeFactory = scopeFactory;
}
public async Task StartAsync(CancellationToken cancellationToken)
{
using var scope = _scopeFactory.CreateScope();
var contentRoot = scope.ServiceProvider.GetRequiredService<IWebHostEnvironment>().ContentRootPath;
var commissionService = scope.ServiceProvider.GetRequiredService<CommissionService>();
await commissionService.UpdateCacheAsync();
var interval = TimeSpan.FromHours(8);
var nextRunTime = GetNextRunTime(interval);
var firstInterval = nextRunTime.Subtract(DateTime.Now);
void action()
{
var t1 = Task.Delay(firstInterval, cancellationToken);
t1.Wait(cancellationToken);
_timer = new Timer(
async o =>
{
await FileHelper.CreateAppOfflineAsync(contentRoot);
using var scope = _scopeFactory.CreateScope();
var commissionService = scope.ServiceProvider.GetRequiredService<CommissionService>();
await commissionService.UpdateCacheAsync();
FileHelper.DeleteAppOffline(contentRoot);
},
null,
TimeSpan.Zero,
interval
);
}
Task.Run(action);
}
private static DateTime GetNextRunTime(TimeSpan interval)
{
var now = DateTime.Now;
var nextRunTime = DateTime.Today;
while(nextRunTime < now)
{
nextRunTime = nextRunTime.Add(interval);
}
return nextRunTime;
}
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
}
Unfortunately sometimes for some reason, this fails and I have to manually delete the app_offline.htm and restart the website again to get it to work. Is there a more robust approach to taking the website offline while there are some services running?
Related
When I call into my repository 10000 times, it either takes minutes (for a very simple keyed query, which does not take minutes to do on the database itself), or dies quickly with a connection pool exhaustion message. I know I am doing something wrong with some combination of disposing objects, creating objects, DI container lifespans, and so on. What am I doing wrong? I have tried a few permutations of .Singleton / .Scoped, a ThreadLocal cache of databases, etc.
Code is executing on Windows 10, framework is .NET Standard 2.1 (running on .NET Core 3.1), talking to SQL Server 2016.
My registration policy (Lamar):
public NPocoRegistry()
{
For<IDatabase>()
.Use(ctx => ctx.GetInstance<DatabaseFactory>().GetDatabase())
.Scoped();
For<DatabaseFactory>().Use(ctx =>
{
var configuration = ctx.GetInstance<IConfiguration>();
Database CreateDatabase()
{
return new Database(configuration.GetConnectionString("EdgeDev"),
DatabaseType.SqlServer2012,
SqlClientFactory.Instance)
{
KeepConnectionAlive = true
};
}
var configs = FluentMappingConfiguration.Configure(ctx.GetAllInstances<IMap>().ToArray());
return DatabaseFactory.Config(cfg => cfg
.UsingDatabase(CreateDatabase)
.WithFluentConfig(configs)
.WithMapper(new BooleanMapper())
.WithMapper(new BinaryStringMapper()));
}).Singleton();
Scan(scan =>
{
scan.TheCallingAssembly();
scan.AddAllTypesOf<IMap>();
});
}
My base repository:
public abstract class BaseNPocoRepository<T>
{
private readonly DatabaseFactory _dbFactory;
private readonly ThreadLocal<IDatabase> _databaseLocal;
protected BaseNPocoRepository(DatabaseFactory dbFactory)
{
_dbFactory = dbFactory;
_databaseLocal = new ThreadLocal<IDatabase>(_dbFactory.GetDatabase);
}
protected virtual IDatabase GetDatabase() => _databaseLocal.Value;
public virtual async Task CreateAsync(T item)
{
using var database = GetDatabase();
await database
.InsertAsync(item)
.ConfigureAwait(false);
}
public virtual async Task UpdateAsync(T item)
{
using var database = GetDatabase();
await database
.UpdateAsync(item)
.ConfigureAwait(false);
}
public virtual async Task DeleteAsync(T item)
{
using var database = GetDatabase();
await database
.DeleteAsync(item)
.ConfigureAwait(false);
}
public virtual async Task<IEnumerable<T>> RetrieveManyAsync()
{
using var database = GetDatabase();
return await database
.Query<T>()
.ToEnumerableAsync()
.ConfigureAwait(false);
}
}
A sample repository utilizing this pattern:
public class T_AccountRepository : BaseNPocoRepository<T_Account>
, IRetrieveMany<T_Account>
, IRetrieve<AccountId, T_Account>
{
public T_AccountRepository(DatabaseFactory dbFactory) : base(dbFactory)
{
}
public async Task<T_Account> RetrieveAsync(AccountId input)
{
using var database = GetDatabase();
return await database.Query<T_Account>()
.SingleAsync(x => x.AccountId == (int) input)
.ConfigureAwait(false);
}
}
How it's actually being called:
static async Task Main(string[] args)
{
Console.WriteLine("Booting up . . .");
var container = new Container(cfg =>
{
cfg.Scan(scan =>
{
scan.AssembliesFromApplicationBaseDirectory();
scan.AssemblyContainingType<NPocoRegistry>();
scan.LookForRegistries();
scan.With(new AllInterfacesConvention());
});
});
Console.WriteLine("Getting repository . . . ");
var repo = container.GetInstance<AccountRepository>();
Console.WriteLine("Starting benchmark . . .");
var sw = Stopwatch.StartNew();
for (int i = 0; i < 10000; i++)
{
await repo.RetrieveAsync(1253832471);
}
Console.WriteLine(sw.ElapsedMilliseconds + "ms");
}
I have a service bus triggered Azure Function which listens for messages containing just blob URL strings of JSON data which each one of them is at least 10MB.
Message queue is near real-time(If I use the correct term) so producers keep putting messaging to the queue with a frequency so there is always data in the queue to be processed.
I have designed a solution but it gets OutOfMemoryException most of the time. The steps involved in the current solution sequentially are:
Consume a message
Download the file from the URL within the consumed message to a temporary folder
Read the whole file as a string
Deserialize it to an object
Partition into the chunks to supply Mongo bulk upsert limit
Bulk upsert to Mongo
I have tried to solve OutOfMemoryException and I thought that it's because my function/consumer don't have the same pace with the producer, so I think that at the time t1 when it gets the first message and process it and then while it's upserting to the mongo the function keeps getting the messages and they accumulate in the memory and waiting to be upserted.
Is my reasoning right?
Thus I think that If I could implement a streaming solution starting from #3, reading from file by chunking and putting it to a stream then I would prevent the memory keep growing and reduce time also. I have mostly Java background and I somehow know that with custom iterator/spliterator/iterable it is possible to do streaming and asynchronous processing.
How can I do asynchronous data streaming with .Net Core in an Azure Function?
Are there other approaches to solve this problem?
namespace x.y.Z
{
public class MyFunction
{
//...
[FunctionName("my-func")]
public async Task Run([ServiceBusTrigger("my-topic", "my-subscription", Connection = "AzureServiceBus")] string message, ILogger log, ExecutionContext context)
{
var data = new PredictionMessage();
try
{
data = myPredictionService.genericDeserialize(message);
await myPredictionService.ValidateAsync(data);
await myPredictionService.AddAsync(data);
}
catch (Exception ex)
{
//...
}
}
}
}
public class PredictionMessage
{
public string BlobURL { get; set; }
}
namespace x.y.z.Prediction
{
public abstract class BasePredictionService<T> : IBasePredictionService<T> where T : PredictionMessage, new()
{
protected readonly ILogger log;
private static JsonSerializer serializer;
public BasePredictionService(ILogger<BasePredictionService<T>> log)
{
this.log = log;
serializer = new JsonSerializer();
}
public async Task ValidateAsync(T message)
{
//...
}
public T genericDeserialize(string message)
{
return JsonConvert.DeserializeObject<T>(message);
}
public virtual Task AddAsync(T message)
{
throw new System.NotImplementedException();
}
public async Task<string> SerializePredictionResult(T message)
{
var result = string.Empty;
using (WebClient client = new WebClient())
{
var tempPath = Path.Combine(Path.GetTempPath(), DateTime.Now.Ticks + ".json");
Uri srcPath = new Uri(message.BlobURL);
await client.DownloadFileTaskAsync(srcPath, tempPath);
using (FileStream fs = File.Open(tempPath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (BufferedStream bs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bs))
{
result = sr.ReadToEnd();
}
}
Task.Run(() =>
{
File.Delete(tempPath);
});
return result;
}
}
protected TType StreamDataDeserialize<TType>(string streamResult)
{
var body = default(TType);
using (MemoryStream stream = new MemoryStream(Encoding.Default.GetBytes(streamResult)))
{
using (StreamReader streamReader = new StreamReader(stream))
{
body = (TType)serializer.Deserialize(streamReader, typeof(TType));
}
}
return body;
}
protected List<List<TType>> Split<TType>(List<TType> list, int chunkSize = 1000)
{
List<List<TType>> retVal = new List<List<TType>>();
while (list.Count > 0)
{
int count = list.Count > chunkSize ? chunkSize : list.Count;
retVal.Add(list.GetRange(0, count));
list.RemoveRange(0, count);
}
return retVal;
}
}
}
namespace x.y.z.Prediction
{
public class MyPredictionService : BasePredictionService<PredictionMessage>, IMyPredictionService
{
private readonly IMongoDBRepository<MyPrediction> repository;
public MyPredictionService(IMongoDBRepoFactory mongoDBRepoFactory, ILogger<MyPredictionService> log) : base(log)
{
repository = mongoDBRepoFactory.GetRepo<MyPrediction>();
}
public override async Task AddAsync(PredictionMessage message)
{
string streamResult = await base.SerializePredictionResult(message);
var body = base.StreamDataDeserialize<List<MyPrediction>>(streamResult);
if (body != null && body.Count > 0)
{
var chunkList = base.Split(body);
await BulkUpsertProcess(chunkList);
}
}
private async Task BulkUpsertProcess(List<List<MyPrediction>> chunkList)
{
foreach (var perChunk in chunkList)
{
var filterContainers = new List<IDictionary<string, object>>();
var updateContainer = new List<IDictionary<string, object>>();
foreach (var item in perChunk)
{
var filter = new Dictionary<string, object>();
var update = new Dictionary<string, object>();
filter.Add(/*...*/);
filterContainers.Add(filter);
update.Add(/*...*/);
updateContainer.Add(update);
}
await Task.Run(async () =>
{
await repository.BulkUpsertAsync(filterContainers, updateContainer);
});
}
}
}
}
In the Book "Introduction to Vala" by Dr Michael Lauer, he has mentioned that the lib Soup async api is broken. I'm struggling to write a simple example using session.queue_message that query radio stations using the service from radio-browser. Here is my code. I would appreciate any help form experienced Programmers like "Al Thomas". Thank you.
public class Station : Object {
// A globally unique identifier for the change of the station information
public string changeuuid { get; set; default = ""; }
// A globally unique identifier for the station
public string stationuuid { get; set; default = ""; }
// The name of the station
public string name { get; set; default = ""; }
// The stream URL provided by the user
public string url { get; set; default = ""; }
// and so on ... many properties
public string to_string () {
var builder = new StringBuilder ();
builder.append_printf ("\nchangeuuid = %s\n", changeuuid);
builder.append_printf ("stationuuid = %s\n", stationuuid);
builder.append_printf ("name = %s\n", name);
builder.append_printf ("url = %s\n", url);
return (owned) builder.str;
}
}
public class RadioBrowser : Object {
private static Soup.Session session;
// private static MainLoop main_loop;
public const string API_URL = "https://de1.api.radio-browser.info/json/stations";
public const string USER_AGENT = "github.aeldemery.radiolibrary";
public RadioBrowser (string user_agent = USER_AGENT, uint timeout = 50)
requires (timeout > 0)
{
Intl.setlocale ();
session = new Soup.Session ();
session.timeout = timeout;
session.user_agent = user_agent;
session.use_thread_context = true;
// main_loop = new MainLoop ();
}
private void check_response_status (Soup.Message msg) {
if (msg.status_code != 200) {
var str = "Error: Status message error %s.".printf (msg.reason_phrase);
error (str);
}
}
public Gee.ArrayList<Station> listStations () {
var stations = new Gee.ArrayList<Station> ();
var data_list = Datalist<string> ();
data_list.set_data ("limit", "100");
var parser = new Json.Parser ();
parser.array_element.connect ((pars, array, index) => {
var station = Json.gobject_deserialize (typeof (Station), array.get_element (index)) as Station;
assert_nonnull (station);
stations.add (station);
});
var msg = Soup.Form.request_new_from_datalist (
"POST",
API_URL,
data_list
);
// send_message works but not queue_message
// session.send_message (msg);
session.queue_message (msg, (sess, mess) => {
check_response_status (msg);
try {
parser.load_from_data ((string) msg.response_body.flatten ().data);
} catch (Error e) {
error ("Failed to parse data, error:" + e.message);
}
});
return stations;
}
}
int main (string[] args) {
var radio_browser = new RadioBrowser ();
var stations = radio_browser.listStations ();
assert_nonnull (stations);
foreach (var station in stations) {
print (station.to_string ());
}
return 0;
}
While I'm not Al Thomas, I still might be able to help. ;)
For async calls to work in there needs to be a main loop running, and typically from the main program thread. Thus you want to create and execute the main loop from your main() function, rather than in your application code:
int main (string[] args) {
var loop = new GLib.MainLoop ();
var radio_browser = new RadioBrowser ();
// set up async calls here
// then set the main loop running as the last thing
loop.run();
}
Also, if you want to wait for an async call to complete, you typically need to make the call using the yield keyword from another async function E.g:
public async Gee.ArrayList<Station> listStations () {
…
// When this call is made, execution of listStations() will be
// suspended until the soup response is received
yield session.send_async(msg);
// Execution then resumes normally
check_response_status (msg);
parser.load_from_data ((string) msg.response_body.flatten ().data);
…
return stations;
}
You can then call this from the (non-async) main function using the listStations.begin(…) notation:
int main (string[] args) {
var loop = new GLib.MainLoop ();
var radio_browser = new RadioBrowser ();
radio_browser.listStations.begin((obj, res) => {
var stations = radio_browser.listStations.end(res);
…
loop.quit();
});
loop.run();
}
As further reading, I would recommend the async section of the Vala Tutorial, and the asyc examples on the wiki as well.
I am trying to implement Stripe.net into my Xamarin.Forms PCL using an ASP.NET Core MVC Web API. The goal is to process credit card payment from users. My web API runs locally on http://localhost:port for testing purposes.
In the PaymentPage, a user enters their credit card information into Entry objects and when they click the submit Button, a method in the PaymentPageViewModel is called to start the logic:
async void OnFinishBookingClicked(object sender, System.EventArgs e)
{
// TODO: Stripe integration
var viewModel = (PaymentPageViewModel)this.BindingContext;
await viewModel.ProcessPayment();
}
This is part of the PaymentPageViewModel:
private readonly IStripeRepository _repository;
private readonly IAPIRepository _api;
public PaymentPageViewModel(IStripeRepository repository, IAPIRepository api)
{
_repository = repository;
_api = api;
}
public async Task ProcessPayment()
{
try
{
if (string.IsNullOrEmpty(ExpirationDate))
ExpirationDate = "09/18";
var exp = ExpirationDate.Split('/');
var token = _repository.CreateToken(CreditCardNumber, exp[0], exp[1], SecurityCode);
await Application.Current.MainPage.DisplayAlert("Test Message", token, "OK");
await _api.ChargeCard(token, 5.00M);
}
catch (Exception ex)
{
await Application.Current.MainPage.DisplayAlert("Error", ex.Message, "OK");
}
}
This is what the APIRepository looks like:
public class APIRepository: IAPIRepository
{
const string Url = "http://localhost:5000";
private string authorizationKey;
private async Task<HttpClient> GetClient()
{
HttpClient client = new HttpClient();
if (string.IsNullOrEmpty(authorizationKey))
{
authorizationKey = await client.GetStringAsync(Url);
authorizationKey = JsonConvert.DeserializeObject<string>(authorizationKey);
}
client.DefaultRequestHeaders.Add("Authorization", authorizationKey);
client.DefaultRequestHeaders.Add("Accept", "application/json");
return client;
}
public async Task<string> ChargeCard(string token, decimal amount)
{
HttpClient client = await GetClient();
var json = JsonConvert.SerializeObject(new { token, amount });
var response = await client.PostAsync("/api/Stripe", new StringContent(json));
return await response.Content.ReadAsStringAsync();
}
}
The issue is that I get a series of errors during await _api.ChargeCard(token, 5.00M):
The first exception happens during authorizationKey = await client.GetStringAsync(Url); the exception message is the following:
{System.Net.Http.HttpRequestException: 404 (Not Found) at System.Net.Http.HttpResponseMessage.EnsureSuccessStatusCode () [0x0000a] in /Library/Frameworks/Xamarin.iOS.framework/Versions/11.2.0.11/src/mono/mcs/class/System.Net.Http/System.Net.Http/HttpResponseM…}
I get another exception during response = await client.PostAsync("/api/Stripe", new StringContent(json));
{System.InvalidOperationException: The request URI must either be an absolute URI or BaseAddress must be set at System.Net.Http.HttpClient.SendAsync (System.Net.Http.HttpRequestMessage request, System.Net.Http.HttpCompletionOption completionOption, System.Thr…}
The third exception happens at the catch block of the viewModel.ProcessPayment() method:
{System.NullReferenceException: Object reference not set to an instance of an object at Zwaby.Services.APIRepository+d__3.MoveNext () [0x00184] in /Users/carlos/Projects/Zwaby/Zwaby/Services/APIRepository.cs:57 --- End of stack trace from previou…}
In my Web API project, I have a StripeController, but my implementation may not be fully correct:
[Route("api/Stripe")]
public class StripeController : Controller
{
private readonly StripeContext _context;
public StripeController(StripeContext context)
{
_context = context;
if (_context.StripeCharges.Count() == 0)
{
_context.StripeCharges.Add(new StripeItem { });
_context.SaveChanges();
}
}
[HttpGet]
public IActionResult Get(string key)
{
// TODO: implement method that returns authorization key
}
[HttpPost]
public IActionResult Charge(string stripeToken, decimal amount)
{
var customers = new StripeCustomerService();
var charges = new StripeChargeService();
var customer = customers.Create(new StripeCustomerCreateOptions
{
SourceToken = stripeToken
});
var charge = charges.Create(new StripeChargeCreateOptions
{
Amount = (int)amount,
Description = "Sample Charge",
Currency = "usd",
CustomerId = customer.Id
});
return View();
}
}
For completeness, I am including the StripeRepository class, the other parameter of the PaymentPageViewModel:
public class StripeRepository: IStripeRepository
{
public string CreateToken(string cardNumber, string cardExpMonth, string cardExpYear, string cardCVC)
{
StripeConfiguration.SetApiKey("my_test_key");
//TODO: Wireup card information below
var tokenOptions = new StripeTokenCreateOptions()
{
Card = new StripeCreditCardOptions()
{
Number = "4242424242424242",
ExpirationYear = 2018,
ExpirationMonth = 10,
Cvc = "123"
}
};
var tokenService = new StripeTokenService();
StripeToken stripeToken = tokenService.Create(tokenOptions);
return stripeToken.Id;
}
}
Thank you so much!
Hi I am new to Windows Workflow. This may be very easy, but I am stuck on this from long.
I have a state machine workflow, in which i have a workflow host class.
Persistence is not working in this code. While debugging pointer never goes to application.persistableIdle event.
I use custom input argument, for which I have set as Serializable.
below is my code of the host class:
static InstanceStore instanceStore;
static AutoResetEvent instanceUnloaded = new AutoResetEvent(false);
static Activity activity = new Activity1();
static Guid id = new Guid();
static int intContractHeaderKey;
static Contract contract = new Contract();
public ContractActivityHost(Guid wfid, Int32 contractHeaderID)
{
SetupInstanceStore();
StartAndUnloadInstance(contractHeaderID);
if (intContractHeaderKey > 0)
{
LoadAndCompleteInstance(id, intContractHeaderKey);
}
}
static void StartAndUnloadInstance(Int32 contractHeaderID)
{
contract = new Contract();
//var objContract = new object();
var input = new Dictionary<string, object>
{
{"TheContract", contract}
};
input.Add("ContractHeaderKey", contractHeaderID);
WorkflowApplication application = new WorkflowApplication(activity, input);
application.InstanceStore = instanceStore;
//returning IdleAction.Unload instructs the WorkflowApplication to persists application state and remove it from memory
application.PersistableIdle = (e) =>
{
return PersistableIdleAction.Unload;
};
application.Unloaded = (e) =>
{
instanceUnloaded.Set();
};
//application.Idle = (e) =>
// {
// //application.Unload();
// instanceUnloaded.Set();
// };
//This call is not required
//Calling persist here captures the application durably before it has been started
application.Persist();
id = application.Id;
application.Run();
instanceUnloaded.WaitOne();
//application.Unload();
//contract = (Contract)objContract;
intContractHeaderKey = contract.ContractID;
}
static void LoadAndCompleteInstance(Guid wfid, Int32 contractHeaderID)
{
//string input = Console.ReadLine();
while (!contract.ContractWFPause)
{
contract.FireContract(contract.ContractID);
WorkflowApplication application = new WorkflowApplication(activity);
application.InstanceStore = instanceStore;
application.Completed = (workflowApplicationCompletedEventArgs) =>
{
//Console.WriteLine("\nWorkflowApplication has Completed in the {0} state.", workflowApplicationCompletedEventArgs.CompletionState);
strWFStatus = "Completed";
};
application.Unloaded = (workflowApplicationEventArgs) =>
{
//Console.WriteLine("WorkflowApplication has Unloaded\n");
strWFStatus = "Unloaded";
instanceUnloaded.Set();
};
application.Load(wfid);
instanceUnloaded.WaitOne();
}
}
private static void SetupInstanceStore()
{
instanceStore =
new SqlWorkflowInstanceStore(#"Data Source=.;Initial Catalog=WorkflowInstanceStore;Integrated Security=True;");
InstanceHandle handle = instanceStore.CreateInstanceHandle();
InstanceView view = instanceStore.Execute(handle, new CreateWorkflowOwnerCommand(), TimeSpan.FromSeconds(10));
handle.Free();
instanceStore.DefaultInstanceOwner = view.InstanceOwner;
}
I have been trying to resolve this from long time, but not sure where I am missing anything. I have gone through couple of sample applications and changed my code to match the flow and logic, but still it does not work.
After application.persist, record is inserted in [System.Activities.DurableInstancing].[InstancesTable] view.
But debug pointer does not move beyond instanceUnloaded.WaitOne();
it actually goes to idle state. if I uncomment application.idle event, it goes in that event code.
Any help to resolve this would be great.
Thanks.
Please check If you have added the below details
instanceStore = new SqlWorkflowInstanceStore(ConfigurationManager.ConnectionStrings["WFPersistenceDb"].ConnectionString);
StateMachineStateTracker.Promote(this.instanceStore);