NPoco exhausting connection pool on .NET Core 3.1 - .net-core

When I call into my repository 10000 times, it either takes minutes (for a very simple keyed query, which does not take minutes to do on the database itself), or dies quickly with a connection pool exhaustion message. I know I am doing something wrong with some combination of disposing objects, creating objects, DI container lifespans, and so on. What am I doing wrong? I have tried a few permutations of .Singleton / .Scoped, a ThreadLocal cache of databases, etc.
Code is executing on Windows 10, framework is .NET Standard 2.1 (running on .NET Core 3.1), talking to SQL Server 2016.
My registration policy (Lamar):
public NPocoRegistry()
{
For<IDatabase>()
.Use(ctx => ctx.GetInstance<DatabaseFactory>().GetDatabase())
.Scoped();
For<DatabaseFactory>().Use(ctx =>
{
var configuration = ctx.GetInstance<IConfiguration>();
Database CreateDatabase()
{
return new Database(configuration.GetConnectionString("EdgeDev"),
DatabaseType.SqlServer2012,
SqlClientFactory.Instance)
{
KeepConnectionAlive = true
};
}
var configs = FluentMappingConfiguration.Configure(ctx.GetAllInstances<IMap>().ToArray());
return DatabaseFactory.Config(cfg => cfg
.UsingDatabase(CreateDatabase)
.WithFluentConfig(configs)
.WithMapper(new BooleanMapper())
.WithMapper(new BinaryStringMapper()));
}).Singleton();
Scan(scan =>
{
scan.TheCallingAssembly();
scan.AddAllTypesOf<IMap>();
});
}
My base repository:
public abstract class BaseNPocoRepository<T>
{
private readonly DatabaseFactory _dbFactory;
private readonly ThreadLocal<IDatabase> _databaseLocal;
protected BaseNPocoRepository(DatabaseFactory dbFactory)
{
_dbFactory = dbFactory;
_databaseLocal = new ThreadLocal<IDatabase>(_dbFactory.GetDatabase);
}
protected virtual IDatabase GetDatabase() => _databaseLocal.Value;
public virtual async Task CreateAsync(T item)
{
using var database = GetDatabase();
await database
.InsertAsync(item)
.ConfigureAwait(false);
}
public virtual async Task UpdateAsync(T item)
{
using var database = GetDatabase();
await database
.UpdateAsync(item)
.ConfigureAwait(false);
}
public virtual async Task DeleteAsync(T item)
{
using var database = GetDatabase();
await database
.DeleteAsync(item)
.ConfigureAwait(false);
}
public virtual async Task<IEnumerable<T>> RetrieveManyAsync()
{
using var database = GetDatabase();
return await database
.Query<T>()
.ToEnumerableAsync()
.ConfigureAwait(false);
}
}
A sample repository utilizing this pattern:
public class T_AccountRepository : BaseNPocoRepository<T_Account>
, IRetrieveMany<T_Account>
, IRetrieve<AccountId, T_Account>
{
public T_AccountRepository(DatabaseFactory dbFactory) : base(dbFactory)
{
}
public async Task<T_Account> RetrieveAsync(AccountId input)
{
using var database = GetDatabase();
return await database.Query<T_Account>()
.SingleAsync(x => x.AccountId == (int) input)
.ConfigureAwait(false);
}
}
How it's actually being called:
static async Task Main(string[] args)
{
Console.WriteLine("Booting up . . .");
var container = new Container(cfg =>
{
cfg.Scan(scan =>
{
scan.AssembliesFromApplicationBaseDirectory();
scan.AssemblyContainingType<NPocoRegistry>();
scan.LookForRegistries();
scan.With(new AllInterfacesConvention());
});
});
Console.WriteLine("Getting repository . . . ");
var repo = container.GetInstance<AccountRepository>();
Console.WriteLine("Starting benchmark . . .");
var sw = Stopwatch.StartNew();
for (int i = 0; i < 10000; i++)
{
await repo.RetrieveAsync(1253832471);
}
Console.WriteLine(sw.ElapsedMilliseconds + "ms");
}

Related

Ardalis.Specification and Azure Cosmos Database : What is the place where the query is applied? Cosmos Database or Application Memory?

I am trying to implement Ardalis.Specification in .net core Web API application that uses Azure Cosmos Database
Below is my Specification definition
public class ToDoItemSearchSpecification : Specification<ToDoItem>
{
public ToDoItemSearchSpecification(
string title = "",
int pageStart = 0,
int pageSize = 50,
string sortColumn = "title",
SortDirection sortDirection = SortDirection.Ascending,
bool exactSearch = false
)
{
if (!string.IsNullOrWhiteSpace(title))
{
if (exactSearch)
{
Query.Where(item => item.Title.ToLower() == title.ToLower());
}
else
{
Query.Where(item => item.Title.ToLower().Contains(title.ToLower()));
}
}
// Pagination
if (pageSize != -1) // Display all entries and disable pagination
{
Query.Skip(pageStart).Take(pageSize);
}
// Sort
switch (sortColumn.ToLower())
{
case ("title"):
{
if (sortDirection == SortDirection.Ascending)
{
Query.OrderBy(x => x.Title);
}
else
{
Query.OrderByDescending(x => x.Title);
}
}
break;
default:
break;
}
}
}
and Evaluator
public class CosmosDbSpecificationEvaluator<T>: SpecificationEvaluatorBase<T> where T: class {}
//Below is the Repository method
private IQueryable<T> ApplySpecification(ISpecification<T> specification)
{
CosmosDbSpecificationEvaluator<T> evaluator = new CosmosDbSpecificationEvaluator<T>();
IOrderedQueryable<T> cosmosDBItems = _container.GetItemLinqQueryable<T>();
return evaluator.GetQuery(cosmosDBItems, specification);
}
public async Task < IEnumerable<T>> GetItemsAsync(ISpecification<T> specification) {
IQueryable <T> queryable = ApplySpecification(specification);
FeedIterator<T> iterator = queryable.ToFeedIterator();
List<T> results = new List<T>();
while (iterator.HasMoreResults) {
FeedResponse<T> response = await iterator.ReadNextAsync();
results.AddRange(response.ToList());
}
return results;
}
Below is the Controller implementation
[HttpGet("/search/title/{SearchString}", Name = "GetAllByTitle")]
[EnableQuery]
[ApiConventionMethod(typeof (DefaultApiConventions), nameof(DefaultApiConventions.Get))]
public async Task<IEnumerable<ToDoItem>> GetAllByTitle(string SearchString)
{
ToDoItemSearchSpecification specification = new ToDoItemSearchSpecification(SearchString);
IEnumerable<ToDoItem> result = await _repo.GetItemsAsync(specification);
return result;
}
And it is being called like mentioned below
https://localhost:7110/search/title/Get%200%20beers
I am trying to understand where the query is applied. Cosmos Database or Application Memory? and what does the below line do?
FeedIterator<T> iterator = queryable.ToFeedIterator();
The query is applied as an SQL query on the database side.
The Cosmos database will only return documents that satisfy the WHERE clause.

Disable Asp.Net Core website during cache operation

I have some very heavy calculations for some statistics on my website that I want to update every 8 hours. I'm using a HostedService to update the cache every 8 hours. To disable the website, I create the app_offline.htm file at the root of the website.
This is a simplified version of the code:
public class CacheUpdaterHostedService : IHostedService
{
private readonly IServiceScopeFactory _scopeFactory;
private Timer _timer;
public CacheUpdaterHostedService(IServiceScopeFactory scopeFactory)
{
_scopeFactory = scopeFactory;
}
public async Task StartAsync(CancellationToken cancellationToken)
{
using var scope = _scopeFactory.CreateScope();
var contentRoot = scope.ServiceProvider.GetRequiredService<IWebHostEnvironment>().ContentRootPath;
var commissionService = scope.ServiceProvider.GetRequiredService<CommissionService>();
await commissionService.UpdateCacheAsync();
var interval = TimeSpan.FromHours(8);
var nextRunTime = GetNextRunTime(interval);
var firstInterval = nextRunTime.Subtract(DateTime.Now);
void action()
{
var t1 = Task.Delay(firstInterval, cancellationToken);
t1.Wait(cancellationToken);
_timer = new Timer(
async o =>
{
await FileHelper.CreateAppOfflineAsync(contentRoot);
using var scope = _scopeFactory.CreateScope();
var commissionService = scope.ServiceProvider.GetRequiredService<CommissionService>();
await commissionService.UpdateCacheAsync();
FileHelper.DeleteAppOffline(contentRoot);
},
null,
TimeSpan.Zero,
interval
);
}
Task.Run(action);
}
private static DateTime GetNextRunTime(TimeSpan interval)
{
var now = DateTime.Now;
var nextRunTime = DateTime.Today;
while(nextRunTime < now)
{
nextRunTime = nextRunTime.Add(interval);
}
return nextRunTime;
}
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
}
Unfortunately sometimes for some reason, this fails and I have to manually delete the app_offline.htm and restart the website again to get it to work. Is there a more robust approach to taking the website offline while there are some services running?

How to fix 'A MessageReceived handler is blocking the gateway task.' in DiscordBot

I am trying to build my own Discord Bot, that soft bans people, who write racist or anti-Semitic words.
I try to do this with MessageReceivedAsync but it crashes all the time with the error 'A MessageReceived handler is blocking the gateway task.'
Here is the code of my Program.cs:
namespace NoNetworcc
{
class Program : ModuleBase<SocketCommandContext>
{
static void Main(string[] args)
=> new Program().MainAsync().GetAwaiter().GetResult();
public async Task MainAsync()
{
using (var services = ConfigureServices())
{
var client = services.GetRequiredService<DiscordSocketClient>();
client.Log += LogAsync;
client.MessageReceived += MessageReceivedAsync;
services.GetRequiredService<CommandService>().Log += LogAsync;
await client.LoginAsync(TokenType.Bot, "NzEyNDA2MDAxMjAxOTcxMjcw.XsRF4A.8YENNInx3D4kqJyK9N8xjTU3mcs");
await client.StartAsync();
await services.GetRequiredService<CommandHandlingService>().InitializeAsync();
await Task.Delay(Timeout.Infinite);
}
}
private Task LogAsync(LogMessage log)
{
Console.WriteLine(log.ToString());
return Task.CompletedTask;
}
private async Task MessageReceivedAsync(SocketMessage message)
{
using (BlacklistDatabaseContext lite = new BlacklistDatabaseContext())
{
var blacklistWords = lite.BlacklistWords;
foreach(var word in blacklistWords)
{
if(message.Content.Contains(word.Blacklistword.ToString()))
{
ulong roleID = 756500011331616840;
var role = Context.Guild.GetRole(roleID);
await ((IGuildUser)Context.User).AddRoleAsync(role);
await message.Channel.SendMessageAsync($"{Context.User} got softbanned for using the word '{word}'");
}
}
}
}
private ServiceProvider ConfigureServices()
{
return new ServiceCollection()
.AddSingleton<DiscordSocketClient>()
.AddSingleton<CommandService>()
.AddSingleton<CommandHandlingService>()
.AddSingleton<HttpClient>()
.AddSingleton<PictureService>()
.BuildServiceProvider();
}
}
}
And here is my Code for the HandlingService:
namespace NoNetworcc.Services
{
public class CommandHandlingService
{
private readonly CommandService _commands;
private readonly DiscordSocketClient _discord;
private readonly IServiceProvider _services;
public CommandHandlingService(IServiceProvider services)
{
_commands = services.GetRequiredService<CommandService>();
_discord = services.GetRequiredService<DiscordSocketClient>();
_services = services;
_commands.CommandExecuted += CommandExecutedAsync;
_discord.MessageReceived += MessageReceivedAsync;
}
public async Task InitializeAsync()
{
await _commands.AddModulesAsync(Assembly.GetEntryAssembly(), _services);
}
public async Task MessageReceivedAsync(SocketMessage rawMessage)
{
if (!(rawMessage is SocketUserMessage message)) return;
if (message.Source != MessageSource.User) return;
var argPos = 0;
var context = new SocketCommandContext(_discord, message);
await _commands.ExecuteAsync(context, argPos, _services);
}
public async Task CommandExecutedAsync(Optional<CommandInfo> command, ICommandContext context, IResult result)
{
if (!command.IsSpecified)
return;
if (result.IsSuccess)
return;
await context.Channel.SendMessageAsync($"error: {result}");
}
}
}
How can I fix this issue?
private Task MessageReceivedAsync(SocketMessage message) {
_ = Task.Run(async () => {
using (BlacklistDatabaseContext lite = new BlacklistDatabaseContext()) {
var blacklistWords = lite.BlacklistWords;
foreach (var word in blacklistWords) {
if(message.Content.Contains(word.Blacklistword.ToString())) {
ulong roleID = 756500011331616840;
var role = (message.Channel as ITextChannel)?.Guild.GetRole(roleID);
if (role != null) {
await (message.Author as SocketGuildUser)?.AddRoleAsync(role);
await message.Channel.SendMessageAsync($"{message.Author} got softbanned for using the word '{word}'");
}
}
}
}
});
return Task.CompletedTask;
}

How to implement asynchronous data streaming in .Net Core Service Bus triggered Azure Function processing huge data not to get OutOfMemoryException?

I have a service bus triggered Azure Function which listens for messages containing just blob URL strings of JSON data which each one of them is at least 10MB.
Message queue is near real-time(If I use the correct term) so producers keep putting messaging to the queue with a frequency so there is always data in the queue to be processed.
I have designed a solution but it gets OutOfMemoryException most of the time. The steps involved in the current solution sequentially are:
Consume a message
Download the file from the URL within the consumed message to a temporary folder
Read the whole file as a string
Deserialize it to an object
Partition into the chunks to supply Mongo bulk upsert limit
Bulk upsert to Mongo
I have tried to solve OutOfMemoryException and I thought that it's because my function/consumer don't have the same pace with the producer, so I think that at the time t1 when it gets the first message and process it and then while it's upserting to the mongo the function keeps getting the messages and they accumulate in the memory and waiting to be upserted.
Is my reasoning right?
Thus I think that If I could implement a streaming solution starting from #3, reading from file by chunking and putting it to a stream then I would prevent the memory keep growing and reduce time also. I have mostly Java background and I somehow know that with custom iterator/spliterator/iterable it is possible to do streaming and asynchronous processing.
How can I do asynchronous data streaming with .Net Core in an Azure Function?
Are there other approaches to solve this problem?
namespace x.y.Z
{
public class MyFunction
{
//...
[FunctionName("my-func")]
public async Task Run([ServiceBusTrigger("my-topic", "my-subscription", Connection = "AzureServiceBus")] string message, ILogger log, ExecutionContext context)
{
var data = new PredictionMessage();
try
{
data = myPredictionService.genericDeserialize(message);
await myPredictionService.ValidateAsync(data);
await myPredictionService.AddAsync(data);
}
catch (Exception ex)
{
//...
}
}
}
}
public class PredictionMessage
{
public string BlobURL { get; set; }
}
namespace x.y.z.Prediction
{
public abstract class BasePredictionService<T> : IBasePredictionService<T> where T : PredictionMessage, new()
{
protected readonly ILogger log;
private static JsonSerializer serializer;
public BasePredictionService(ILogger<BasePredictionService<T>> log)
{
this.log = log;
serializer = new JsonSerializer();
}
public async Task ValidateAsync(T message)
{
//...
}
public T genericDeserialize(string message)
{
return JsonConvert.DeserializeObject<T>(message);
}
public virtual Task AddAsync(T message)
{
throw new System.NotImplementedException();
}
public async Task<string> SerializePredictionResult(T message)
{
var result = string.Empty;
using (WebClient client = new WebClient())
{
var tempPath = Path.Combine(Path.GetTempPath(), DateTime.Now.Ticks + ".json");
Uri srcPath = new Uri(message.BlobURL);
await client.DownloadFileTaskAsync(srcPath, tempPath);
using (FileStream fs = File.Open(tempPath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (BufferedStream bs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bs))
{
result = sr.ReadToEnd();
}
}
Task.Run(() =>
{
File.Delete(tempPath);
});
return result;
}
}
protected TType StreamDataDeserialize<TType>(string streamResult)
{
var body = default(TType);
using (MemoryStream stream = new MemoryStream(Encoding.Default.GetBytes(streamResult)))
{
using (StreamReader streamReader = new StreamReader(stream))
{
body = (TType)serializer.Deserialize(streamReader, typeof(TType));
}
}
return body;
}
protected List<List<TType>> Split<TType>(List<TType> list, int chunkSize = 1000)
{
List<List<TType>> retVal = new List<List<TType>>();
while (list.Count > 0)
{
int count = list.Count > chunkSize ? chunkSize : list.Count;
retVal.Add(list.GetRange(0, count));
list.RemoveRange(0, count);
}
return retVal;
}
}
}
namespace x.y.z.Prediction
{
public class MyPredictionService : BasePredictionService<PredictionMessage>, IMyPredictionService
{
private readonly IMongoDBRepository<MyPrediction> repository;
public MyPredictionService(IMongoDBRepoFactory mongoDBRepoFactory, ILogger<MyPredictionService> log) : base(log)
{
repository = mongoDBRepoFactory.GetRepo<MyPrediction>();
}
public override async Task AddAsync(PredictionMessage message)
{
string streamResult = await base.SerializePredictionResult(message);
var body = base.StreamDataDeserialize<List<MyPrediction>>(streamResult);
if (body != null && body.Count > 0)
{
var chunkList = base.Split(body);
await BulkUpsertProcess(chunkList);
}
}
private async Task BulkUpsertProcess(List<List<MyPrediction>> chunkList)
{
foreach (var perChunk in chunkList)
{
var filterContainers = new List<IDictionary<string, object>>();
var updateContainer = new List<IDictionary<string, object>>();
foreach (var item in perChunk)
{
var filter = new Dictionary<string, object>();
var update = new Dictionary<string, object>();
filter.Add(/*...*/);
filterContainers.Add(filter);
update.Add(/*...*/);
updateContainer.Add(update);
}
await Task.Run(async () =>
{
await repository.BulkUpsertAsync(filterContainers, updateContainer);
});
}
}
}
}

How can i use engine object in my console application

"How can i use engine in my console application"
I shouldn't use the ITemplate-interface and Transform-Method.
I am using Tridion 2011
Could anyone please suggest me.
You can't. The Engine class is part of the TOM.NET and that API is explicitly reserved for use in:
Template Building Blocks
Event Handlers
For all other cases (such as console applications) you should use the Core Service.
There are many good questions (and articles on other web sites) already:
https://stackoverflow.com/search?q=%5Btridion%5D+core+service
http://www.google.com/#q=tridion+core+service
If you get stuck along the way, show us the relevant code+configuration you have and what error message your get (or at what step you are stuck) and we'll try to help from there.
From a console application you should use the Core Service. I wrote a small example using the Core Service to search for items in the content manager.
Console.WriteLine("FullTextQuery:");
var fullTextQuery = Console.ReadLine();
if (String.IsNullOrWhiteSpace(fullTextQuery) || fullTextQuery.Equals(":q", StringComparison.OrdinalIgnoreCase))
{
break;
}
Console.WriteLine("SearchIn IdRef:");
var searchInIdRef = Console.ReadLine();
var queryData = new SearchQueryData
{
FullTextQuery = fullTextQuery,
SearchIn = new LinkToIdentifiableObjectData
{
IdRef = searchInIdRef
}
};
var results = coreServiceClient.GetSearchResults(queryData);
results.ToList().ForEach(result => Console.WriteLine("{0} ({1})", result.Title, result.Id));
Add a reference to Tridion.ContentManager.CoreService.Client to your Visual Studio Project.
Code of the Core Service Client Provider:
public interface ICoreServiceProvider
{
CoreServiceClient GetCoreServiceClient();
}
public class CoreServiceDefaultProvider : ICoreServiceProvider
{
private CoreServiceClient _client;
public CoreServiceClient GetCoreServiceClient()
{
return _client ?? (_client = new CoreServiceClient());
}
}
And the client itself:
public class CoreServiceClient : IDisposable
{
public SessionAwareCoreServiceClient ProxyClient;
private const string DefaultEndpointName = "netTcp_2011";
public CoreServiceClient(string endPointName)
{
if(string.IsNullOrWhiteSpace(endPointName))
{
throw new ArgumentNullException("endPointName", "EndPointName is not specified.");
}
ProxyClient = new SessionAwareCoreServiceClient(endPointName);
}
public CoreServiceClient() : this(DefaultEndpointName) { }
public string GetApiVersionNumber()
{
return ProxyClient.GetApiVersion();
}
public IdentifiableObjectData[] GetSearchResults(SearchQueryData filter)
{
return ProxyClient.GetSearchResults(filter);
}
public IdentifiableObjectData Read(string id)
{
return ProxyClient.Read(id, new ReadOptions());
}
public ApplicationData ReadApplicationData(string subjectId, string applicationId)
{
return ProxyClient.ReadApplicationData(subjectId, applicationId);
}
public void Dispose()
{
if (ProxyClient.State == CommunicationState.Faulted)
{
ProxyClient.Abort();
}
else
{
ProxyClient.Close();
}
}
}
When you want to perform CRUD actions through the core service you can implement the following methods in the client:
public IdentifiableObjectData CreateItem(IdentifiableObjectData data)
{
data = ProxyClient.Create(data, new ReadOptions());
return data;
}
public IdentifiableObjectData UpdateItem(IdentifiableObjectData data)
{
data = ProxyClient.Update(data, new ReadOptions());
return data;
}
public IdentifiableObjectData ReadItem(string id)
{
return ProxyClient.Read(id, new ReadOptions());
}
To construct a data object of e.g. a Component you can implement a Component Builder class that implements a create method that does this for you:
public ComponentData Create(string folderUri, string title, string content)
{
var data = new ComponentData()
{
Id = "tcm:0-0-0",
Title = title,
Content = content,
LocationInfo = new LocationInfo()
};
data.LocationInfo.OrganizationalItem = new LinkToOrganizationalItemData
{
IdRef = folderUri
};
using (CoreServiceClient client = provider.GetCoreServiceClient())
{
data = (ComponentData)client.CreateItem(data);
}
return data;
}
Hope this gets you started.

Resources