I am trying to migrate between data between 2 containers with option StartFromBeginning =true. while migrating I am also making a small modification to the document also. When I add this logic I live sync between collections is not working. I had used Migrating data from old container to new partitioned container using change feed as reference which works. After deploying there seems to be no error but. How can I check what I am doing wrong. I have also enabled application insights.
namespace CosmosContainerMigration.Trigger
{
public class ContainerMigration
{
private IUpdatedDocument updatedDocument;
public ContainerMigration(IUpdatedDocument updatedDocument)
{
this.updatedDocument = updatedDocument;
}
[FunctionName("CosmosContainerMigration")]
public async Task Run([CosmosDBTrigger(
databaseName: "%SourceDatabaseName%",
collectionName: "%ContainerName%",
ConnectionStringSetting = "connectionString",
StartFromBeginning =true,
LeaseCollectionName ="%ContainerLeaseName%",
CreateLeaseCollectionIfNotExists = true)]IReadOnlyList<Document> source,
[CosmosDB(databaseName:"%TargetDatabaseName%",
collectionName:"%ContainerName%",
ConnectionStringSetting = "connectionString")]IAsyncCollector<Document> destination,
ILogger log)
{
log.LogInformation("Documents modified " + source.Count);
foreach (var item in source)
{
try
{
Document updatedItem = await this.updatedDocument.Update(item);
await destination.AddAsync(updatedItem);
}
catch (Exception)
{
log.LogInformation("Failed document ", item.Id);
throw;
}
}
}
}
}
Related
Has anyone had any success in setting up integration testing for web applications written in .NET 6 and EntityFramework Core 6, and using SQLite in-memory database? I am having issues with test setup/teardown, so tests which run fine in isolation, start randomly failing when running all tests together.
My test context is set up based on the Microsoft examples:
public class TestApplication : AutofacWebApplicationFactory<Startup>
{
protected override void ConfigureWebHost(IWebHostBuilder builder)
{
builder.ConfigureTestServices(services =>
{
services.AddAuthentication(options =>
{
options.DefaultAuthenticateScheme = FakeJwtBearerDefaults.AuthenticationScheme;
options.DefaultChallengeScheme = FakeJwtBearerDefaults.AuthenticationScheme;
}).AddFakeJwtBearer();
services.AddAuthorization(options =>
{
options.DefaultPolicy = new AuthorizationPolicyBuilder()
.AddAuthenticationSchemes(FakeJwtBearerDefaults.AuthenticationScheme)
.RequireAuthenticatedUser()
.Build();
});
var connectionString = new SqliteConnectionStringBuilder($"DataSource=file:{Guid.NewGuid()}?cache=shared&mode=memory");
var connection = new SqliteConnection(connectionString.ToString());
connection.Open();
var descriptor = services.SingleOrDefault(d => d.ServiceType == typeof(DbContextOptions<MyDbContext>));
if (descriptor != null)
{
services.Remove(descriptor);
}
services.AddDbContext<MyDbContext>(options =>
{
options.UseSqlite(connection);
});
var sp = services.BuildServiceProvider();
using var scope = sp.CreateScope();
var scopedServices = scope.ServiceProvider;
var db = scopedServices.GetRequiredService<MyDbContext>();
var logger = scopedServices.GetRequiredService<ILogger<TestApplication>>();
db.Database.EnsureDeleted();
db.Database.EnsureCreated();
try
{
logger.LogInformation("Initialising in-memory database with test data");
TestData.Initialise(db);
}
catch (Exception ex)
{
logger.LogError(ex, "An error occurred seeding the database with test messages. Error: {Message}", ex.Message);
}
});
}
}
which will be used in my test fixtures something like this:
public class WhenGettingPartners
{
[Fact]
public async Task ItShouldAcceptValidRequests()
{
await using var app = new TestApplication();
var client = app.CreateClient().WithRoles(Scopes.PartnerRead);
var result = await client.GetAsync("/Partners");
result.Should().BeSuccessful();
var data = await result.Content.ReadAsType<Partner[]>();
data.Should().NotBeEmpty();
}
[Fact]
public async Task ItShouldRejectUnauthorisedRequests()
{
await using var app = new TestApplication();
var client = app.CreateClient();
var result = await client.GetAsync("/Partners");
result.Should().HaveStatusCode(HttpStatusCode.Unauthorized);
}
}
If I select an individual test case and run it, it works fine; if I select the entire text fixture and run it, it works fine. But when I run the entire test project, then tests will fail randomly, usually when it tries to recreate the database and finds the tables already exist.
I've tried using xUnit's IClassFixture interface to share one instance of TestApplication across all tests in the fixture:
public abstract class ApiTestFixture : IClassFixture<TestApplication>
{
public ApiTestFixture(TestApplication application)
{
App = application;
}
public TestApplication App { get; }
}
public class WhenGettingClients : ApiTestFixture
{
public WhenGettingClients(TestApplication app) : base(app)
{
}
[Fact]
public async Task ItShouldAcceptValidRequests()
{
var client = App.CreateClient().WithRoles(Scopes.ClientRead);
var result = await client.GetAsync("/Clients");
result.Should().BeSuccessful();
var data = await result.Content.ReadAsType<Client[]>();
data.Should().HaveCount(2);
}
[Fact]
public async Task ItShouldRejectUnauthorisedRequests()
{
var client = App.CreateClient();
var result = await client.GetAsync("/Clients");
result.Should().HaveStatusCode(HttpStatusCode.Unauthorized);
}
}
but that fails in exactly the same way.
Update
I am developing using the Rider IDE from JetBrains; the same issue occurs if I run the tests in Visual Studio. However, I noticed that if I test with code coverage in Rider, then the tests all pass! Similarly, if I test with profiling in Rider, then the tests all pass; so I'm wondering if there's something more esoteric going on, that the code coverage runner somehow forces the tests to execute in a manner which is more deterministic or less likely to result in test pollution.
I've created this method to create a Team inside an Azure active directory.
public async Task CreateTeamTest()
{
...connection stuff...
var team = new Team
{
DisplayName = "0000My Sample Team",
Description = "My Sample Team’s Description",
AdditionalData = new Dictionary<string, object>()
{
{"template#odata.bind", "https://graph.microsoft.com/v1.0/teamsTemplates('standard')"}
}
};
await graphClient.Teams.Request().AddAsync(team);
}
After that I want to create a service I can call from an app sitting in an IIS, but I aint got a clue how to achieve that, I've started with something like that but it does nothing.
[WebMethod]
public async Task<string> createTeamDePruebaAsync()
{
try
{
//this is the class containing my method
TeamServices ts = new TeamServices();
await ts.CreateTeamTest();
return "OK";
}
catch(Exception e)
{
return e.Message;
}
}
My question is what route do you people use to transform a method unto a web service?
Maybe the problem is I'm using asmx with a microsoft graph request async, and they dont match
I'm using Confluent.Kafka(1.4.4) in a .netCore project as a message broker. In the startup of the project I set only "bootstrapservers" to the specific servers which were in the appSetting.json file and I produce messages in an API when necessary with the code below in related class:
public async Task WriteMessage<T>(string topicName, T message)
{
using (var p = new ProducerBuilder<Null, string>(_producerConfig).Build())
{
try
{
var serializedMessage= JsonConvert.SerializeObject(message);
var dr = await p.ProduceAsync(topicName, new Message<Null, string> { Value = serializedMessage });
logger.LogInformation($"Delivered '{dr.Value}' to '{dr.TopicPartitionOffset}'");
}
catch (ProduceException<Null, string> e)
{
logger.LogInformation($"Delivery failed: {e.Error.Reason}");
}
}
}
I have also added the following code In the consumer solution :
public async Task Run()
{
using (var consumerBuilder = new ConsumerBuilder<Ignore, string>(_consumerConfig).Build())
{
consumerBuilder.Subscribe(new List<string>() { "ActiveMemberCardForPanClubEvent", "CreatePanClubEvent", "RemovePanClubEvent"
});
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
while (true)
{
try
{
var consumer = consumerBuilder.Consume(cts.Token);
if (consumer.Message != null)
{
using (LogContext.PushProperty("RequestId", Guid.NewGuid()))
{
//Do something
logger.LogInformation($"Consumed message '{consumer.Message.Value}' at: '{consumer.TopicPartitionOffset}'.");
await DoJob(consumer.Topic, consumer.Message.Value);
consumer.Topic.Remove(0, consumer.Topic.Length);
}
}
else
{
logger.LogInformation($"message is null for topic '{consumer.Topic}'and partition : '{consumer.TopicPartitionOffset}' .");
consumer.Topic.Remove(0, consumer.Topic.Length);
}
}
catch (ConsumeException e)
{
logger.LogInformation($"Error occurred: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
consumerBuilder.Close();
}
}
}
I produce a message and when the consumer project is run everything goes perfectly and the message is being read in the consumer solution.
The problem is raised when the consumer project is not run and I queue a message in the API with the message producer in API. After running consumers there is not any valid message for that topic that it's message is being produced.
I am familiar with and have experiences with message brokers and I know that by sending a message it will be on the bus until it is being used but I don't understand why it doesn't work with Kafka in this project.
The default setting for the "auto.offset.reset" Consumer property is "latest".
That means (in the context of no offsets being written yet) if you write a message to some topic and then subsequently start the consumer, it will skip past any messages written before the consumer was started. This could be why your consumer is not seeing the messages queued by your producer.
The solution is to set "auto.offset.reset" to "earliest" which means that the consumer will start from the earliest offset on the topic.
https://docs.confluent.io/current/installation/configuration/consumer-configs.html#auto.offset.reset
I'm relatively new to mobile and async data access and I'm trying to build a line-of-business app from Xamarin starter "cross platform" template in VS2017. It seems that when I do database operations too frequently I get 'database is locked' (most questions deal with roll-your-own sqlite implementations). I had added pretty verbose logging (I have to support non-technical end mobile users).
I changed to (as suggested in other answers) a singleton model for database access which is producing non-traceable (meaning no exceptions are caught and no xamarin log entries) exceptions when calling table.ReadAsync (see below).
As a secondary question, having spent so much time on this and running into so many different roadblocks (no doubt of my own making) I'm wondering whether I'm not following some unspoken rule for mobile development such as "only one async object read per page and design UI for 100% async". Am I trying to do too much? Here is my current "singleton" data access class:
public static class MainDataStore
{
private static ReaderWriterLockSlim ReadLock = new ReaderWriterLockSlim();
public static bool IsInitialized { get; set; }
public static MobileServiceClient MobileService { get; set; }
public static bool UseAuthentication = true;
public static IMobileServiceSyncTable<User> UserTable;
public static IMobileServiceSyncTable<Showroom> ShowroomTable;
public static IEnumerable<User> Users { get; set; } //= new ObservableRangeCollection<User>();
public static IEnumerable<Showroom> Showrooms { get; set; }
public static void InitializeAsync()
{
try
{
if (IsInitialized)
return;
Logging.D("Starting to initialize main store.");
AuthenticationHandler handler = null;
handler = new AuthenticationHandler();
MobileService = new MobileServiceClient(App.AzureMobileAppUrl, handler)
{
SerializerSettings = new MobileServiceJsonSerializerSettings
{
CamelCasePropertyNames = true
}
};
var store = new MobileServiceSQLiteStore(Settings.DatabaseName);
store.DefineTable<User>();
store.DefineTable<Showroom>();
MobileService.SyncContext.InitializeAsync(store, new MobileServiceSyncHandler());
UserTable = MobileService.GetSyncTable<User>();
ShowroomTable = MobileService.GetSyncTable<Showroom>();
Logging.D("Finished initializing main store.");
IsInitialized = true;
}
catch (Exception ex)
{
Logging.E(ex); // Debug.WriteLine("EXCEPTION: " + ex.Message + ". Stack: " + ex.StackTrace);
}
}
public static async void Load(ECarnavalObjectType type)
{
Logging.D("Reading lock entering. Read count: " + ReadLock.CurrentReadCount.ToString());
// ReadLock.EnterReadLock();
switch (type)
{
case ECarnavalObjectType.Users:
await GetUsersAsync();
Users = await UserTable.ToEnumerableAsync();
break;
case ECarnavalObjectType.Showrooms:
await GetShowroomsAsync();
Showrooms = await ShowroomTable.ToEnumerableAsync();
break;
}
// ReadLock.ExitReadLock();
}
public static async Task GetUsersAsync()
{
if (CrossConnectivity.Current.IsConnected)
{
try
{
// await UserTable.ReadAsync<User>(UserTable.CreateQuery());
await UserTable.PullAsync($"all{typeof(User).Name}", UserTable.CreateQuery());
}
catch (Exception ex)
{
}
}
}
public static async Task GetShowroomsAsync()
{
await ShowroomTable.ReadAsync<Showroom>(ShowroomTable.CreateQuery());
}
}
In your code, you are not awaiting the InitializeAsync(), which means it is likely that the database is still locked and being set up when you go to synchronize it.
Arrange your code in a singleton, then have every single method (read/list/etc.) call await InitializeAsync() to initialize the database. Do an early return on the InitializeAsync() method if the database is already created (you've got some good code there for that).
For more info, see my book: https://adrianhall.github.io/develop-mobile-apps-with-csharp-and-azure/chapter3/client/
I have a method that does some work in a transaction:
public async Task<int> AddAsync(Item item)
{
int result;
using (var transaction = await _context.Database.BeginTransactionAsync())
{
_context.Add(item);
// Save the item so it has an ItemId
result = await _context.SaveChangesAsync();
// perform some actions using that new item's ItemId
_otherRepository.Execute(item.ItemId);
transaction.Commit();
}
return result;
}
I'd like to add unit tests to check that if _context.SaveChangesAsync or _otherRepository.Execute fail then the transaction is rolled back, is that possible?
I can't see a way to do that using InMemory or SQLite?
#Ilya Chumakov's excellent answer allowed me to unit test for the transaction. Our discussion in the comments then exposed some interesting points that I thought were worth moving into an answer so they'd be more permanent and easier to see:
The primary point is that the events logged by Entity Framework change dependent on the database provider, which surprised me. If using the InMemory provider you get just one event:
Id:1; ExecutedCommand
Whereas if you use Sqlite for the in-memory database you get four events:
Id:1; ExecutedCommand
Id:5; BeginningTransaction
Id:1; ExecutedCommand
Id:6; CommittingTransaction
I hadn't expected the events logged to change depending on the DB provider.
To anyone wanting to look into this more, I captured the event details by changing Ilya's logging code as follows:
public class FakeLogger : ILogger
{
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception,
Func<TState, Exception, string> formatter)
{
var record = new LogRecord
{
EventId = eventId.Id,
RelationalEventId = (RelationalEventId) eventId.Id,
Description = formatter(state, exception)
};
Events.Add(record);
}
public List<LogRecord> Events { get; set; } = new List<LogRecord>();
public bool IsEnabled(LogLevel logLevel) => true;
public IDisposable BeginScope<TState>(TState state) => null;
}
public class LogRecord
{
public EventId EventId { get; set; }
public RelationalEventId RelationalEventId { get; set; }
public string Description { get; set; }
}
And then I adjusted my code that returns an in-memory database so that I could switch in-memory DB provider as follows:
public class InMemoryDatabase
{
public FakeLogger EfLogger { get; private set; }
public MyDbContext GetContextWithData(bool useSqlite = false)
{
EfLogger = new FakeLogger();
var factoryMock = Substitute.For<ILoggerFactory>();
factoryMock.CreateLogger(Arg.Any<string>()).Returns(EfLogger);
DbContextOptions<MyDbContext> options;
if (useSqlite)
{
// In-memory database only exists while the connection is open
var connection = new SqliteConnection("DataSource=:memory:");
connection.Open();
options = new DbContextOptionsBuilder<MyDbContext>()
.UseSqlite(connection)
.UseLoggerFactory(factoryMock)
.Options;
}
else
{
options = new DbContextOptionsBuilder<MyDbContext>()
.UseInMemoryDatabase(Guid.NewGuid().ToString())
// don't raise the error warning us that the in memory db doesn't support transactions
.ConfigureWarnings(x => x.Ignore(InMemoryEventId.TransactionIgnoredWarning))
.UseLoggerFactory(factoryMock)
.Options;
}
var ctx = new MyDbContext(options);
if (useSqlite)
{
ctx.Database.EnsureCreated();
}
// code to populate the context with test data
ctx.SaveChanges();
return ctx;
}
}
Finally, in my unit test I made sure to clear the event log just before the assert part of my test to ensure I don't get a false positive due to events that were logged during the arrange part of my test:
public async Task Commits_transaction()
{
using (var context = _inMemoryDatabase.GetContextWithData(useSqlite: true))
{
// Arrange
// code to set up date for test
// make sure none of our setup added the event we are testing for
_inMemoryDatabase.EfLogger.Events.Clear();
// Act
// Call the method that has the transaction;
// Assert
var result = _inMemoryDatabase.EfLogger.Events
.Any(x => x.EventId.Id == (int) RelationalEventId.CommittingTransaction);
You could check EF Core logs for a RelationalEventId.RollingbackTransaction event type. I provided full details here:
How to trace an Entity Framework Core event for integration testing?
How it could look:
Assert.True(eventList.Contains((int)RelationalEventId.CommittingTransaction));
I think you are asking about how to rollback when a commit fails, EF core will auto rollback on if any of the statement failed
Read more here
, if you are asking for other reason or you want to do something when rollback happens, just to add try catch blocks,
using (var transaction = await
_context.Database.BeginTransactionAsync()){
try {
_context.Add(item);
// Save the item so it has an ItemId
result = await _context.SaveChangesAsync();
// perform some actions using that new item's ItemId
_otherRepository.Execute(item.ItemId);
transaction.Commit();
}
catch (Exception)
{
// failed, Do something
} }