How would you go about displaying Garbage Collection (GC) info in HealthChecksUI in .NET Core 3?
NuGet ref: https://github.com/Xabaril/AspNetCore.Diagnostics.HealthChecks
I've been looking at samples here and there, but can't really find what I'm looking for. What I'm trying to do is do present the GC memory allocation to HC-UI and report degraded if it surpasses some limit. And I got it working - but I believe the implementation can be a lot better due to heap allocation when checking it.
Concerned section marked with comment # startup.cs
Here's my example:
startup.cs
public void ConfigureServices(IServiceCollection services)
{
services.
.AddCustomHealthChecks()
.AddHealthChecksUI();
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
app
.UseRouting()
.UseEndpoints(endpoints =>
{
endpoints.MapHealthChecks("/health", new HealthCheckOptions
{
Predicate = (check) =>
check.Tags.Contains("self")
|| check.Tags.Contains("memory"),
ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
});
endpoints.MapHealthChecks("/liveness", new HealthCheckOptions
{
Predicate = (check) => check.Tags.Contains("self")
});
endpoints.MapHealthChecksUI(setup =>
{
setup.AddCustomStylesheet("healthcheck-ui.css");
});
});
}
public static IServiceCollection AddCustomHealthChecks(this IServiceCollection services)
{
services
.AddHealthChecks()
.AddCheck(
"Self", () =>
HealthCheckResult.Healthy("Dynamic Config is OK!"),
tags: new[] { "self" }
)
.AddCheck("Memory", () =>
new GCInfoHealthCheck() // This section right here
.CheckHealthAsync( // seems to be a very
new HealthCheckContext() // poor implementation due
).GetAwaiter().GetResult(), // to constant Heap allocation.
tags: new[] { "memory" }
);
return services;
}
GCInfoHealthCheck.cs
public class GCInfoHealthCheck : IHealthCheck
{
public string Name { get; } = "GCInfo";
public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default(CancellationToken))
{
var allocatedMegaBytes = GC.GetTotalMemory(forceFullCollection: false) / 1000000; // divided to get MB
var data = new Dictionary<string, object>()
{
{ "AllocatedMegaBytes", allocatedMegaBytes }
};
var status = (allocatedMegaBytes >= 20) ? HealthStatus.Unhealthy : HealthStatus.Healthy;
return Task.FromResult(
new HealthCheckResult(
status,
exception: null,
description: $"reports degraded status if allocated MB >= 20MB, current: {allocatedMegaBytes} MB",
data: data
)
);
}
}
Output
Here are some samples I've been looking at: https://github.com/aspnet/Diagnostics/tree/d1cba1f55bab1e3b206a46cee81eb1583d8732e2/samples/HealthChecksSample
I've been trying some other samples and register HC's as singletons, but I can't get it to work and report to HC-UI. So, is there better way to do it?
Seems like I've been a potato and probably misspelled the tag, because this works like a charm:
// register
.AddCheck<GCInfoHealthCheck>(
"memory",
failureStatus: HealthStatus.Unhealthy,
tags: new[] { "memory" }
)
// useage
.UseHealthChecks("/health", new HealthCheckOptions()
{
Predicate = (check) =>
|| check.Tags.Contains("memory"),
ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
})
Docs: https://learn.microsoft.com/en-us/aspnet/core/host-and-deploy/health-checks?view=aspnetcore-3.1
Related
I seem to be having an issue when using different rabbitmq hosts to publish / consume messages. Originally, the pub / sub happened on one rabbitmq host. A decision was made to create another host (sitting on a different server) and from there on out I have been running into some issues. I have read up on the multibus documentation (https://masstransit-project.com/usage/containers/multibus.html) and tried to implement it so that I can have one bus for producing messages and one bus for consuming messages. Please find the code below:
--- Startup.cs ---
services.AddMassTransit(x =>
{
var section = configuration.GetSection("Rabbit1");
x.AddConsumer<SomeConsumer>();
x.AddBus(context => Bus.Factory.CreateUsingRabbitMq(cfg =>
{
cfg.Host(section.GetValue<string>("Host"), host =>
{
host.Username(section.GetValue<string>("User"));
host.Password(section.GetValue<string>("Password"));
});
cfg.ReceiveEndpoint(QUEUE_NAME, e =>
{
e.ConfigureConsumer<SomeConsumer>(context);
EndpointConvention.Map<MessageObjectList>(e.InputAddress);
});
cfg.ConfigureEndpoints(context);
}));
});
services.AddMassTransit<ISecondBus>(x =>
{
var section = configuration.GetSection("Rabbit2");
x.AddConsumer<SomeOtherConsumer>();
x.AddBus(context => Bus.Factory.CreateUsingRabbitMq(cfg =>
{
cfg.Host(section.GetValue<string>("Host"), host =>
{
host.Username(section.GetValue<string>("User"));
host.Password(section.GetValue<string>("Password"));
});
cfg.ReceiveEndpoint(QUEUE_NAME, e =>
{
e.PrefetchCount = 4;
e.UseMessageRetry(r => r.Interval(10, 100));
e.ConfigureConsumer<SomeOtherConsumer>(context);
EndpointConvention.Map<MessageObjectList>(e.InputAddress);
});
cfg.ConfigureEndpoints(context);
}));
});
services.AddMassTransitHostedService();
--- ISecondBus.cs ---
public interface ISecondBus : IBus
{
}
--- Consumer.cs ---
public class SomeConsumer : IConsumer<MessageObjectList>
{
private readonly IBus _bus;
public SomeConsumer(IBus bus)
{
_bus = bus;
}
public async Task Consume(ConsumeContext<MessageObjectList> context)
{
var message = context.Message;
//Aggregate data
var newList = new MessageObjectList
{
Message = message
};
var endpoint = await _bus.GetSendEndpoint(new Uri(_configuration.GetConnectionString("Rabbit2") + "/" + QUEUE_NAME));
await endpoint.Send(newList);
}
}
The above code works fine and successfully consumes messages from rabbit1. However, when sending a message to Rabbit2, I can see that the queue is being created, but no data is being persisted to the queue.
See second consumer below:
--- Startup.cs ---
services.AddMassTransit(x =>
{
var section = configuration.GetSection("Rabbitmq2");
x.AddConsumer<SomeOtherConsumer>();
x.AddBus(context => Bus.Factory.CreateUsingRabbitMq(cfg =>
{
cfg.Host(section.GetValue<string>("Host"), host =>
{
host.Username(section.GetValue<string>("User"));
host.Password(section.GetValue<string>("Password"));
});
cfg.ReceiveEndpoint(QUEUE_NAME, e =>
{
e.ConfigureConsumer<SomeOtherConsumer>(context);
EndpointConvention.Map<MessageObjectList>(e.InputAddress);
});
cfg.ConfigureEndpoints(context);
}));
});
services.AddMassTransitHostedService();
--- SomeOtherConsumer.cs ---
public class SomeOtherConsumer: IConsumer<MessageObjectList>
{
public async Task Consume(ConsumeContext<MessageObjectList> context)
{
//Persist data to db
}
}
The problem here is that the queue for rabbit2 never gets populate (but it does get created) so no messages are being consumed from the second consumer.
Any assistance would be appreciated.
If you want to send to the second bus from the first bus, you need to use the second bus interface in the consumer on the first bus.
public class SomeConsumer : IConsumer<MessageObjectList>
{
private readonly ISecondBus _bus;
public SomeConsumer(ISecondBus bus)
{
_bus = bus;
}
public async Task Consume(ConsumeContext<MessageObjectList> context)
{
var message = context.Message;
//Aggregate data
var newList = new MessageObjectList
{
Message = message
};
var endpoint = await _bus.GetSendEndpoint(new Uri("queue:" + QUEUE_NAME));
await endpoint.Send(newList);
}
}
When I call into my repository 10000 times, it either takes minutes (for a very simple keyed query, which does not take minutes to do on the database itself), or dies quickly with a connection pool exhaustion message. I know I am doing something wrong with some combination of disposing objects, creating objects, DI container lifespans, and so on. What am I doing wrong? I have tried a few permutations of .Singleton / .Scoped, a ThreadLocal cache of databases, etc.
Code is executing on Windows 10, framework is .NET Standard 2.1 (running on .NET Core 3.1), talking to SQL Server 2016.
My registration policy (Lamar):
public NPocoRegistry()
{
For<IDatabase>()
.Use(ctx => ctx.GetInstance<DatabaseFactory>().GetDatabase())
.Scoped();
For<DatabaseFactory>().Use(ctx =>
{
var configuration = ctx.GetInstance<IConfiguration>();
Database CreateDatabase()
{
return new Database(configuration.GetConnectionString("EdgeDev"),
DatabaseType.SqlServer2012,
SqlClientFactory.Instance)
{
KeepConnectionAlive = true
};
}
var configs = FluentMappingConfiguration.Configure(ctx.GetAllInstances<IMap>().ToArray());
return DatabaseFactory.Config(cfg => cfg
.UsingDatabase(CreateDatabase)
.WithFluentConfig(configs)
.WithMapper(new BooleanMapper())
.WithMapper(new BinaryStringMapper()));
}).Singleton();
Scan(scan =>
{
scan.TheCallingAssembly();
scan.AddAllTypesOf<IMap>();
});
}
My base repository:
public abstract class BaseNPocoRepository<T>
{
private readonly DatabaseFactory _dbFactory;
private readonly ThreadLocal<IDatabase> _databaseLocal;
protected BaseNPocoRepository(DatabaseFactory dbFactory)
{
_dbFactory = dbFactory;
_databaseLocal = new ThreadLocal<IDatabase>(_dbFactory.GetDatabase);
}
protected virtual IDatabase GetDatabase() => _databaseLocal.Value;
public virtual async Task CreateAsync(T item)
{
using var database = GetDatabase();
await database
.InsertAsync(item)
.ConfigureAwait(false);
}
public virtual async Task UpdateAsync(T item)
{
using var database = GetDatabase();
await database
.UpdateAsync(item)
.ConfigureAwait(false);
}
public virtual async Task DeleteAsync(T item)
{
using var database = GetDatabase();
await database
.DeleteAsync(item)
.ConfigureAwait(false);
}
public virtual async Task<IEnumerable<T>> RetrieveManyAsync()
{
using var database = GetDatabase();
return await database
.Query<T>()
.ToEnumerableAsync()
.ConfigureAwait(false);
}
}
A sample repository utilizing this pattern:
public class T_AccountRepository : BaseNPocoRepository<T_Account>
, IRetrieveMany<T_Account>
, IRetrieve<AccountId, T_Account>
{
public T_AccountRepository(DatabaseFactory dbFactory) : base(dbFactory)
{
}
public async Task<T_Account> RetrieveAsync(AccountId input)
{
using var database = GetDatabase();
return await database.Query<T_Account>()
.SingleAsync(x => x.AccountId == (int) input)
.ConfigureAwait(false);
}
}
How it's actually being called:
static async Task Main(string[] args)
{
Console.WriteLine("Booting up . . .");
var container = new Container(cfg =>
{
cfg.Scan(scan =>
{
scan.AssembliesFromApplicationBaseDirectory();
scan.AssemblyContainingType<NPocoRegistry>();
scan.LookForRegistries();
scan.With(new AllInterfacesConvention());
});
});
Console.WriteLine("Getting repository . . . ");
var repo = container.GetInstance<AccountRepository>();
Console.WriteLine("Starting benchmark . . .");
var sw = Stopwatch.StartNew();
for (int i = 0; i < 10000; i++)
{
await repo.RetrieveAsync(1253832471);
}
Console.WriteLine(sw.ElapsedMilliseconds + "ms");
}
I'm hoping that somebody can help me. We are currently upgrading AutoMapper from v6 to v9 - we would go to v10 but the inability to create new ResolutionContext impacts our unit testing. That said with v9 we are still having the following issue with unit testing AutoMapper Converters...
A ConverterClass:
public class FooBarConverter :
ITypeConverter<FooSourceObject, BarDestinationObject>
{
/// <inheritdoc/>
public virtual BarDestinationObjectConvert(FooSourceObjectsource, BarDestinationObjectdestination, ResolutionContext context)
{
EnsureArg.IsNotNull(source, nameof(source));
switch (source.Type)
{
case SomeType.None:
return context.Mapper.Map<NoneBarDestinationObject>(source);
case SomeType.FixedAmount:
return context.Mapper.Map<FixedBarDestinationObject>(source);
case SomeType.Percentage:
return context.Mapper.Map<PercentageBarDestinationObject>(source);
default:
throw new ArgumentOutOfRangeException(nameof(source));
}
}
Before in AutoMapper 6 we had the following Unit Test (using Xunit):
public class FooBarConverterTests
{
private readonly FooBarConverter target;
private readonly Mock<IRuntimeMapper> mockMapper;
private readonly ResolutionContext resolutionContext;
public FooBarConverterTests()
{
this.mockMapper = new Mock<IRuntimeMapper>();
this.resolutionContext = new ResolutionContext(null, this.mockMapper.Object);
this.target = new FooBarConverter();
}
[Fact]
public void FixedAmountFooModel_ConvertsTo_FixedBarDomainModel()
{
// Arrange
var input = new Foo
{
Type = SomeType.FixedAmount
};
var expected = new DomainModels.FixedBarDestinationObject();
this.mockMapper.Setup(x => x.Map<FixedBarDestinationObject>(It.IsAny<FooSourceObjectsource>()))
.Returns(expected);
// Act
var actual = this.target.Convert(input, It.IsAny<BarDestinationObjectdestination>(), this.resolutionContext);
// Assert
actual.ShouldSatisfyAllConditions(
() => actual.ShouldNotBeNull(),
() => actual.ShouldBeAssignableTo<DomainModels.FixedBarDestinationObject>());
this.mockMapper.Verify(x => x.Map<DomainModels.FixedBarDestinationObject>(input));
}
}
Essentially, this was working fine, however since upgrading to v9, the mapping setup goes missing as it's passed through the resolution context. Meaning that the resulting call of Mapper.Map<FixedBarDestinationObject>() always returns null.
I understand that the ResolutionContext may have changed slightly, but I don't understand how to resolve this issue and ensure that the mock mapping is passed through to the underlying converter.
Thank you for any help or advice.
Thanks to Lucian I finally got my head around this:
public class FooBarConverterTests
{
private readonly FooBarConverter target;
private readonly IMapper mapper;
public FooBarConverterTests()
{
this.mapper = this.GetMapperConfiguration().CreateMapper();
this.target = new FooBarConverter();
}
[Fact]
public void FixedAmountFooModel_ConvertsTo_FixedBarDomainModel()
{
// Arrange
var input = new Foo
{
Type = SomeType.FixedAmount
};
var expected = new DomainModels.FixedBarDestinationObject();
// Act
var actual = this.Mapper.Map<BarDestinationObjectdestination>(input);
// Assert
actual.ShouldSatisfyAllConditions(
() => actual.ShouldNotBeNull(),
() => actual.ShouldBeAssignableTo<DomainModels.FixedBarDestinationObject>());
}
private MapperConfiguration GetMapperConfiguration()
{
return new MapperConfiguration(opt =>
{
opt.AddProfile<CustomAutomapperProfile>();
opt.ConstructServicesUsing(t =>
{
if (t == typeof(FooBarConverter))
{
return this.target;
}
return null;
});
});
}
}
So, I'm loading the mapper profile (which requires the converter) and call the converter through that, this ensures that the mapper profile is loaded.
As a bonus, this also means that I entirely do away with newing up the ResolutionContext and paves the way for upgrading to v10.
I have two Swagger docs generated by Swashbuckle, namely docs/v1 and docs/v2. However docs/v2 doesn't provide information on the action GetV2(). Please help if Swashbuckle has an option to address this.
1.Since the route template appears to be same for actions get() and getv2(), docs v2 doesn't show any info about getV2().
2. Swagger definition doesn't look v1.0/get while appears as v{version}/get in docs/v1
Note:I have refered on apiversioning samples, but not sure what am I missing. All samples refer to Swashbuckle.core while i use Swashbuckle.
[ApiVersion("1.0")]
[ApiVersion("2.0")]
public class HelloController : ApiControllerBase
{
[MapToApiVersion("1.0")]
[Route("v{version:apiVersion}/get")]
[HttpGet]
public ProjectSightActionResult Get()
{
return new Ok("Version 1.0");
}
[MapToApiVersion("2.0")]
[Route("v{version:apiVersion}/get")]
[HttpGet]
public ProjectSightActionResult GetV2()
{
return new Ok("Version 2.0");
}
}
This is my controller including two actions, one for version v1 and one for v2. Below is the webapi.config for the route constraint:
public static void Register(HttpConfiguration config)
{
// Web API configuration and services
// Version Start
// https://github.com/Microsoft/aspnet-api-versioning/wiki/Versioning-via-the-URL-Path
// added to the web api configuration in the application setup
var constraintResolver = new DefaultInlineConstraintResolver()
{
ConstraintMap = {["apiVersion"] = typeof( ApiVersionRouteConstraint )}
};
config.MapHttpAttributeRoutes(constraintResolver);
config.AddApiVersioning();
// Version End
// Web API routes
config.Routes.MapHttpRoute(
name: "DefaultApi",
routeTemplate: "{controller}/{id}",
defaults: new { id = RouteParameter.Optional }
);
config.Filters.Add(new AuthenticationFilter());
// This causes Web API to remove the IPrincipal from any request that enters the Web API pipeline. Effectively, it "un-authenticates" the request.
// https://learn.microsoft.com/en-us/aspnet/web-api/overview/security/authentication-filters
config.SuppressHostPrincipal();
}
My Swagger config has the code:
[assembly: PreApplicationStartMethod(typeof(SwaggerConfig), "Register")]
namespace sample.WebAPI
{
public class SwaggerConfig
{
public static void Register()
{
var thisAssembly = typeof(SwaggerConfig).Assembly;
GlobalConfiguration.Configuration
.EnableSwagger(c =>
{
c.MultipleApiVersions(
(apiDesc, targetApiVersion) => ResolveVersionSupportByRouteConstraint(apiDesc, targetApiVersion),
vc =>
{
vc.Version("v1", "sample.WebAPI");
vc.Version("v2", "sample.WebAPI");
});
}
)
.EnableSwaggerUi(c =>
{
c.EnableDiscoveryUrlSelector();
// If your API supports ApiKey, you can override the default values.
// "apiKeyIn" can either be "query" or "header"
c.EnableApiKeySupport("x-jwt-assertion", "header");
});
}
private static string GetXmlCommentsPath()
{
return string.Format(#"{0}\bin\XmlComments.xml", AppDomain.CurrentDomain.BaseDirectory);
}
private static bool ResolveVersionSupportByRouteConstraint(ApiDescription apiDesc, string targetApiVersion)
{
//check for deprecated versions
var controllerVersionAttributes = apiDesc.ActionDescriptor.ControllerDescriptor.GetCustomAttributes<ApiVersionAttribute>(true);
if (!controllerVersionAttributes.Any())
{
return true; // include when no attributes are defined
}
if (targetApiVersion.StartsWith("v"))
{
targetApiVersion = targetApiVersion.Substring(1); // remove the leading "v" in `v{x.x}`
}
var apiVersion = ApiVersion.Parse(targetApiVersion);
var controllerApiVersion = controllerVersionAttributes
.Where(x => x.Versions.Contains(apiVersion))
.FirstOrDefault();
// has a compatible version, now check the action for [MapToApiVersion]
if (controllerApiVersion != null)
{
var actionMapToAttributes = apiDesc.ActionDescriptor.GetCustomAttributes<MapToApiVersionAttribute>(false);
if (!actionMapToAttributes.Any())
{
return true; // no MapTo attributes matched, then include the action
}
if (actionMapToAttributes.Any(x => x.Versions.Contains(apiVersion)))
{
return true; // include mapped action
}
}
return false;
}
}
}
I'm not sure if you ever solved your problem, but you can now use the official API Explorer for API versioning, which makes Swagger integration simple. You can see a complete working example here.
Here's an abridged version that should work for you:
static void Register( HttpConfiguration configuration )
{
var constraintResolver = new DefaultInlineConstraintResolver() { ConstraintMap = { ["apiVersion"] = typeof( ApiVersionRouteConstraint ) } };
configuration.AddApiVersioning();
configuration.MapHttpAttributeRoutes( constraintResolver );
// note: this option is only necessary when versioning by url segment.
// the SubstitutionFormat property can be used to control the format of the API version
var apiExplorer = configuration.AddVersionedApiExplorer( options => options.SubstituteApiVersionInUrl = true );
configuration.EnableSwagger(
"{apiVersion}/swagger",
swagger =>
{
// build a swagger document and endpoint for each discovered API version
swagger.MultipleApiVersions(
( apiDescription, version ) => apiDescription.GetGroupName() == version,
info =>
{
foreach ( var group in apiExplorer.ApiDescriptions )
{
var description = "A sample application with Swagger, Swashbuckle, and API versioning.";
if ( group.IsDeprecated )
{
description += " This API version has been deprecated.";
}
info.Version( group.Name, $"Sample API {group.ApiVersion}" )
.Contact( c => c.Name( "Bill Mei" ).Email( "bill.mei#somewhere.com" ) )
.Description( description )
.License( l => l.Name( "MIT" ).Url( "https://opensource.org/licenses/MIT" ) )
.TermsOfService( "Shareware" );
}
} );
swagger.IncludeXmlComments( XmlCommentsFilePath );
} )
.EnableSwaggerUi( swagger => swagger.EnableDiscoveryUrlSelector() );
}
}
static string XmlCommentsFilePath
{
get
{
var basePath = System.AppDomain.CurrentDomain.RelativeSearchPath;
var fileName = typeof( Startup ).GetTypeInfo().Assembly.GetName().Name + ".xml";
return Path.Combine( basePath, fileName );
}
}
I am mocking a wrapper to an MSMQ. The wrapper simply allows an object instance to be created that directly calls static methods of the MessageQueue class.
I want to test reading the queue to exhaustion. To do this I would like the mocked wrapper to return some good results and throw an exception on the fourth call to the same method. The method accepts no parameters and returns a standard message object.
Can I set up this series of expectations on the method in Moq?
Yup, this is possible if you don't mind jumping through a few minor hoops. I've done this for one of my projects before. Alright here is the basic technique. I just tested it out in Visual Studio 2008, and this works:
var mockMessage1 = new Mock<IMessage>();
var mockMessage2 = new Mock<IMessage>();
var mockMessage3 = new Mock<IMessage>();
var messageQueue = new Queue<IMessage>(new [] { mockMessage1.Object, mockMessage2.Object, mockMessage3.Object });
var mockMsmqWrapper = new Mock<IMsmqWrapper>();
mockMsmqWrapper.Setup(x => x.GetMessage()).Returns(() => messageQueue.Dequeue()).Callback(() =>
{
if (messageQueue.Count == 0)
mockMsmqWrapper.Setup(x => x.GetMessage()).Throws<MyCustomException>();
});
A few notes:
You don't have to return mocked messages, but it's useful if you want to verify expectations on each message as well to see if certain methods were called or properties were set.
The queue idea is not my own, just a tip I got from a blog post.
The reason why I am throwing an exception of MyCustomException is because the Queue class automatically throws a InvalidOperationException. I wanted to make sure that the mocked MsmqWrapper object throws an exception because of Moq and not because of the queue running out of items.
Here's the complete code that works. Keep in mind that this code is ugly in some places, but I just wanted to show you how this could be tested:
public interface IMsmqWrapper
{
IMessage GetMessage();
}
public class MsmqWrapper : IMsmqWrapper
{
public IMessage GetMessage()
{
throw new NotImplementedException();
}
}
public class Processor
{
private IMsmqWrapper _wrapper;
public int MessagesProcessed { get; set; }
public bool ExceptionThrown { get; set; }
public Processor(IMsmqWrapper msmqWrapper)
{
_wrapper = msmqWrapper;
}
public virtual void ProcessMessages()
{
_wrapper.GetMessage();
MessagesProcessed++;
_wrapper.GetMessage();
MessagesProcessed++;
_wrapper.GetMessage();
MessagesProcessed++;
try
{
_wrapper.GetMessage();
}
catch (MyCustomException)
{
ExceptionThrown = true;
}
}
}
[Test]
public void TestMessageQueueGetsExhausted()
{
var mockMessage1 = new Mock<IMessage>();
var mockMessage2 = new Mock<IMessage>();
var mockMessage3 = new Mock<IMessage>();
var messageQueue = new Queue<IMessage>(new [] { mockMessage1.Object, mockMessage2.Object, mockMessage3.Object });
var mockMsmqWrapper = new Mock<IMsmqWrapper>();
mockMsmqWrapper.Setup(x => x.GetMessage()).Returns(() => messageQueue.Dequeue()).Callback(() =>
{
if (messageQueue.Count == 0)
mockMsmqWrapper.Setup(x => x.GetMessage()).Throws<InvalidProgramException>();
});
var processor = new Processor(mockMsmqWrapper.Object);
processor.ProcessMessages();
Assert.That(processor.MessagesProcessed, Is.EqualTo(3));
Assert.That(processor.ExceptionThrown, Is.EqualTo(true));
}