Merge pull request #78 from Abc-Arbitrage/persistence-add-standalone

Persistence add standalone

.gitignore

@@ -31,6 +31,7 @@ output/**
 .vs/
 .idea/
 local-*
+nCrunchTemp_*
 
 src/_NCrunch_Abc.Zebus/**
 lib/packages/**

src/Abc.Zebus.Contracts/Abc.Zebus.Contracts.v3.ncrunchproject

@@ -0,0 +1,7 @@
+<ProjectConfiguration>
+  <Settings>
+    <HiddenComponentWarnings>
+      <Value>NetCoreNetStandardLocalSystem</Value>
+    </HiddenComponentWarnings>
+  </Settings>
+</ProjectConfiguration>

src/Abc.Zebus.Directory.Runner/Abc.Zebus.Directory.Runner.csproj

@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <StartupObject>Abc.Zebus.Directory.Runner.Program</StartupObject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="log4net" Version="2.0.8" />
+    <PackageReference Include="System.Configuration.ConfigurationManager" Version="4.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Abc.Zebus.Contracts\Abc.Zebus.Contracts.csproj" />
+    <ProjectReference Include="..\Abc.Zebus.Directory.Cassandra\Abc.Zebus.Directory.Cassandra.csproj" />
+    <ProjectReference Include="..\Abc.Zebus.Directory\Abc.Zebus.Directory.csproj" />
+    <ProjectReference Include="..\Abc.Zebus\Abc.Zebus.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="log4net.config">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/Abc.Zebus.Directory.Runner/Abc.Zebus.Directory.Runner.v3.ncrunchproject

@@ -0,0 +1,7 @@
+<ProjectConfiguration>
+  <Settings>
+    <HiddenComponentWarnings>
+      <Value>NetCoreNetStandardLocalSystem</Value>
+    </HiddenComponentWarnings>
+  </Settings>
+</ProjectConfiguration>

src/Abc.Zebus.Directory/App.config → src/Abc.Zebus.Directory.Runner/App.config

@@ -5,7 +5,4 @@
     <add key="Environment" value="Demo"/>
     <add key="PeerId" value="Directory.0"/>
   </appSettings>
-  <startup>
-    <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6"/>
-  </startup>
 </configuration>

src/Abc.Zebus.Directory/Program.cs → src/Abc.Zebus.Directory.Runner/Program.cs

@@ -13,7 +13,7 @@ using log4net;
 using log4net.Config;
 using StructureMap;
 
-namespace Abc.Zebus.Directory
+namespace Abc.Zebus.Directory.Runner
 {
     internal class Program
     {
@@ -28,7 +28,7 @@ namespace Abc.Zebus.Directory
                 _cancelKeySignal.Set();
             };
 
-            XmlConfigurator.ConfigureAndWatch(new FileInfo(PathUtil.InBaseDirectory("log4net.config")));
+            XmlConfigurator.ConfigureAndWatch(LogManager.GetRepository(typeof(Program).Assembly), new FileInfo(PathUtil.InBaseDirectory("log4net.config")));
             _log.Info("Starting in memory directory");
 
             var busFactory = new BusFactory();
@@ -77,4 +77,4 @@ namespace Abc.Zebus.Directory
             });
         }
     }
-}
+}

src/Abc.Zebus.Directory.Runner/log4net.config

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<log4net>
+  <root>
+    <level value="INFO" />
+    <appender type="log4net.Appender.ManagedColoredConsoleAppender">
+      <mapping>
+        <level value="ERROR" />
+        <foreColor value="DarkRed" />
+        <backColor value="White" />
+      </mapping>
+      <mapping>
+        <level value="WARN" />
+        <foreColor value="Yellow" />
+      </mapping>
+      <mapping>
+        <level value="INFO" />
+        <foreColor value="White" />
+      </mapping>
+      <mapping>
+        <level value="DEBUG" />
+        <foreColor value="Blue" />
+      </mapping>
+      <layout type="log4net.Layout.PatternLayout">
+        <conversionPattern value="%date{HH:mm:ss.fff} - %-5level - %logger || %message%newline" />
+      </layout>
+    </appender>
+  </root>
+</log4net>

src/Abc.Zebus.Directory/Abc.Zebus.Directory.csproj

@@ -14,10 +14,4 @@
     <Reference Include="Microsoft.VisualBasic" />
   </ItemGroup>
 
-  <ItemGroup>
-    <None Update="log4net.config">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-
 </Project>

src/Abc.Zebus.Persistence.CQL.Testing/FakePeerStateRepository.cs

@@ -1,87 +0,0 @@
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Abc.Zebus.Persistence.CQL.Storage;
-
-namespace Abc.Zebus.Persistence.CQL.Testing
-{
-    public class FakePeerStateRepository : IPeerStateRepository
-    {
-        private readonly Dictionary<PeerId, PeerState> _peerStatesByPeerId = new Dictionary<PeerId, PeerState>();
-        private long _version;
-
-        public bool IsInitialized { get; set; }
-        public bool HasBeenSaved { get; set; }
-
-        public IEnumerator<PeerState> GetEnumerator()
-        {
-            return _peerStatesByPeerId.Values.GetEnumerator();
-        }
-
-        IEnumerator IEnumerable.GetEnumerator()
-        {
-            return GetEnumerator();
-        }
-
-        public void Initialize()
-        {
-            IsInitialized = true;
-        }
-
-        public PeerState GetPeerStateFor(PeerId peerId)
-        {
-            PeerState peerState;
-            return _peerStatesByPeerId.TryGetValue(peerId, out peerState) ? peerState : null;
-        }
-
-        public PeerState this[PeerId peerId] => _peerStatesByPeerId[peerId];
-
-        public void Add(PeerState state)
-        {
-            _peerStatesByPeerId.Add(state.PeerId, state);
-        }
-
-        public void UpdateNonAckMessageCount(PeerId peerId, int delta)
-        {
-            PeerState peerState;
-            if (!_peerStatesByPeerId.TryGetValue(peerId, out peerState))
-            {
-                peerState = new PeerState(peerId);
-                _peerStatesByPeerId.Add(peerId, peerState);
-            }
-
-            peerState.NonAckedMessageCount += delta;
-            peerState.LastNonAckedMessageCountVersion = Interlocked.Increment(ref _version);
-        }
-
-        public List<PeerState> GetUpdatedPeers(ref long version)
-        {
-            var previousVersion = version;
-            version = Interlocked.Increment(ref _version);
-
-            return _peerStatesByPeerId.Values
-                                      .Where(x => x.LastNonAckedMessageCountVersion >= previousVersion)
-                                      .ToList();
-        }
-
-        public Task RemovePeer(PeerId peerId)
-        {
-            var state = GetPeerStateFor(peerId);
-            if (state != null)
-            {
-                state.MarkAsRemoved();
-                _peerStatesByPeerId.Remove(peerId);
-            }
-
-            return Task.FromResult(0);
-        }
-
-        public Task Save()
-        {
-            HasBeenSaved = true;
-            return Task.FromResult(0);
-        }
-    }
-}

src/Abc.Zebus.Persistence.CQL.Tests/CqlMessageReaderTests.cs

@@ -5,6 +5,7 @@ using Abc.Zebus.Persistence.CQL.Data;
 using Abc.Zebus.Persistence.CQL.Storage;
 using Abc.Zebus.Persistence.CQL.Tests.Cql;
 using Abc.Zebus.Persistence.Messages;
+using Abc.Zebus.Persistence.Storage;
 using Abc.Zebus.Serialization;
 using Abc.Zebus.Testing.Comparison;
 using Abc.Zebus.Testing.Extensions;
@@ -60,7 +61,8 @@ namespace Abc.Zebus.Persistence.CQL.Tests
             nonAckedMessages.Count.ShouldEqual(3);
             for (var i = 0; i < nonAckedMessages.Count; i++)
             {
-                nonAckedMessages[i].DeepCompare(transportMessages[i]).ShouldBeTrue();
+                var transportMessage = TransportMessageDeserializer.Deserialize(nonAckedMessages[i]);
+                transportMessage.DeepCompare(transportMessages[i]).ShouldBeTrue();
             }
         }
 

src/Abc.Zebus.Persistence.CQL.Tests/CqlStorageTests.cs

@@ -4,7 +4,6 @@ using System.Linq;
 using System.Threading.Tasks;
 using Abc.Zebus.Persistence.CQL.Data;
 using Abc.Zebus.Persistence.CQL.Storage;
-using Abc.Zebus.Persistence.CQL.Testing;
 using Abc.Zebus.Persistence.CQL.Tests.Cql;
 using Abc.Zebus.Persistence.Matching;
 using Abc.Zebus.Persistence.Messages;
@@ -13,6 +12,7 @@ using Abc.Zebus.Testing;
 using Abc.Zebus.Testing.Extensions;
 using Abc.Zebus.Transport;
 using Abc.Zebus.Util;
+using Cassandra.Data.Linq;
 using Moq;
 using NUnit.Framework;
 using ProtoBuf;
@@ -22,7 +22,6 @@ namespace Abc.Zebus.Persistence.CQL.Tests
     public class CqlStorageTests : CqlTestFixture<PersistenceCqlDataContext, ICqlPersistenceConfiguration>
     {
         private CqlStorage _storage;
-        private FakePeerStateRepository _peerStateRepository;
         private Mock<IPersistenceConfiguration> _configurationMock;
         private Mock<IReporter> _reporterMock;
 
@@ -45,8 +44,7 @@ namespace Abc.Zebus.Persistence.CQL.Tests
         {
             _configurationMock = new Mock<IPersistenceConfiguration>();
             _reporterMock = new Mock<IReporter>();
-            _peerStateRepository = new FakePeerStateRepository();
-            _storage = new CqlStorage(DataContext, _peerStateRepository, _configurationMock.Object, _reporterMock.Object);
+            _storage = new CqlStorage(DataContext, _configurationMock.Object, _reporterMock.Object);
             _storage.Start();
         }
 
@@ -57,45 +55,42 @@ namespace Abc.Zebus.Persistence.CQL.Tests
         }
 
         [Test]
-        public void should_initialize_peer_state_repository_on_start()
+        public void should_initialize_peer_state_on_start()
         {
-            var peerStateRepository = new FakePeerStateRepository();
-            var storage = new CqlStorage(DataContext, peerStateRepository, _configurationMock.Object, _reporterMock.Object);
-            storage.Start();
-
-            peerStateRepository.IsInitialized.ShouldBeTrue();
-        }
+            DataContext.PeerStates.Insert(new CassandraPeerState(new PeerState(new PeerId("New")))).Execute();
 
-        [Test]
-        public void should_save_peer_state_repository_on_stop()
-        {
-            var peerStateRepository = new FakePeerStateRepository();
-            var storage = new CqlStorage(DataContext, peerStateRepository, _configurationMock.Object, _reporterMock.Object);
+            var storage = new CqlStorage(DataContext, _configurationMock.Object, _reporterMock.Object);
             storage.Start();
-            storage.Stop();
 
-            peerStateRepository.IsInitialized.ShouldBeTrue();
-            peerStateRepository.HasBeenSaved.ShouldBeTrue();
+            storage.GetAllKnownPeers().Count().ShouldEqual(1);
         }
 
         [Test]
         public async Task should_write_message_entry_fields_to_cassandra()
         {
-            var messageBytes = new byte[512];
-            new Random().NextBytes(messageBytes);
-            var messageId = MessageId.NextId();
-            var peerId = "Abc.Peer.0";
-
-            await _storage.Write(new List<MatcherEntry> { MatcherEntry.Message(new PeerId(peerId), messageId, MessageTypeId.PersistenceStopping, messageBytes) });
-
-            var retrievedMessage = DataContext.PersistentMessages.Execute().ExpectedSingle();
-            retrievedMessage.TransportMessage.ShouldBeEquivalentTo(messageBytes, true);
-            retrievedMessage.BucketId.ShouldEqual(GetBucketIdFromMessageId(messageId));
-            retrievedMessage.IsAcked.ShouldBeFalse();
-            retrievedMessage.PeerId.ShouldEqual(peerId);
-            retrievedMessage.UniqueTimestampInTicks.ShouldEqual(messageId.GetDateTime().Ticks);
-            var writeTimeRow = DataContext.Session.Execute("SELECT WRITETIME(\"IsAcked\") FROM \"PersistentMessage\";").ExpectedSingle();
-            writeTimeRow.GetValue<long>(0).ShouldEqual(ToUnixMicroSeconds(messageId.GetDateTime()));
+            using (SystemDateTime.PauseTime())
+            {
+                var messageBytes = new byte[512];
+                new Random().NextBytes(messageBytes);
+                var messageId = MessageId.NextId();
+                var peerId = "Abc.Peer.0";
+
+                await _storage.Write(new List<MatcherEntry> { MatcherEntry.Message(new PeerId(peerId), messageId, MessageTypeId.PersistenceStopping, messageBytes) });
+
+                var retrievedMessage = DataContext.PersistentMessages.Execute().ExpectedSingle();
+                retrievedMessage.TransportMessage.ShouldBeEquivalentTo(messageBytes, true);
+                retrievedMessage.BucketId.ShouldEqual(GetBucketIdFromMessageId(messageId));
+                retrievedMessage.IsAcked.ShouldBeFalse();
+                retrievedMessage.PeerId.ShouldEqual(peerId);
+                retrievedMessage.UniqueTimestampInTicks.ShouldEqual(messageId.GetDateTime().Ticks);
+                var writeTimeRow = DataContext.Session.Execute("SELECT WRITETIME(\"IsAcked\") FROM \"PersistentMessage\";").ExpectedSingle();
+                writeTimeRow.GetValue<long>(0).ShouldEqual(ToUnixMicroSeconds(messageId.GetDateTime()));
+
+                var peerState = DataContext.PeerStates.Execute().ExpectedSingle();
+                peerState.NonAckedMessageCount.ShouldEqual(1);
+                peerState.PeerId.ShouldEqual(peerId);
+                peerState.OldestNonAckedMessageTimestamp.ShouldEqual(messageId.GetDateTime().Ticks - CqlStorage.PersistentMessagesTimeToLive.Ticks);
+            }
         }
 
         [Test]
@@ -138,14 +133,6 @@ namespace Abc.Zebus.Persistence.CQL.Tests
             retrievedMessages.Count.ShouldEqual(2);
         }
 
-        private static long ToUnixMicroSeconds(DateTime timestamp)
-        {
-            var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0);
-            var diff = timestamp - origin;
-            var diffInMicroSeconds = diff.Ticks / 10;
-            return diffInMicroSeconds;
-        }
-
         [Test]
         public async Task should_write_ack_entry_fields_to_cassandra()
         {
@@ -225,41 +212,42 @@ namespace Abc.Zebus.Persistence.CQL.Tests
         }
 
         [Test]
-        public void should_call_peer_state_repository_when_asked_to_remove_peer()
+        public async Task should_remove_from_cassandra_when_asked_to_remove_peer()
         {
             var peerId = new PeerId("PeerId");
-            _peerStateRepository.Add(new PeerState(peerId));
-            var peerState =_peerStateRepository[peerId];
+            await _storage.Write(new List<MatcherEntry> { MatcherEntry.Message(peerId, MessageId.NextId(), MessageTypeId.PersistenceStopping, new byte[0]) });
 
-            _storage.RemovePeer(peerId);
-            
-            peerState.Removed.ShouldBeTrue();
-            _peerStateRepository.GetPeerStateFor(peerId).ShouldBeNull();
+            await _storage.RemovePeer(peerId);
+
+            DataContext.PeerStates.Execute().ShouldBeEmpty();
         }
 
         [Test]
-        public void should_return_cql_message_reader()
+        public async Task should_delete_all_buckets_for_peer_when_removed()
         {
             var peerId = new PeerId("PeerId");
-            _peerStateRepository.Add(new PeerState(peerId));
+            await _storage.Write(new List<MatcherEntry> { MatcherEntry.Message(peerId, MessageId.NextId(), MessageTypeId.PersistenceStopping, new byte[0]) });
 
-            _storage.CreateMessageReader(peerId).ShouldNotBeNull();
+            DataContext.PersistentMessages.Execute().Count().ShouldEqual(1);
+
+            await _storage.RemovePeer(new PeerId("PeerId"));
+
+            DataContext.PersistentMessages.Execute().Any().ShouldBeFalse();
         }
 
         [Test]
-        public void should_return_null_when_asked_for_a_message_reader_for_an_unknown_peer_id()
+        public async Task should_return_cql_message_reader()
         {
-            _storage.CreateMessageReader(new PeerId("UnknownPeerId")).ShouldBeNull();
-        }
+            var peerId = new PeerId("PeerId");
+            await _storage.Write(new List<MatcherEntry> { MatcherEntry.Message(peerId, MessageId.NextId(), MessageTypeId.PersistenceStopping, new byte[0]) });
 
-        private static long GetBucketIdFromMessageId(MessageId message)
-        {
-            return GetBucketIdFromDateTime(message.GetDateTime());
+            _storage.CreateMessageReader(peerId).ShouldNotBeNull();
         }
 
-        private static long GetBucketIdFromDateTime(DateTime timestamp)
+        [Test]
+        public void should_return_null_when_asked_for_a_message_reader_for_an_unknown_peer_id()
         {
-            return new DateTime(timestamp.Year, timestamp.Month, timestamp.Day, timestamp.Hour, 0, 0).Ticks;
+            _storage.CreateMessageReader(new PeerId("UnknownPeerId")).ShouldBeNull();
         }
 
         [Test]
@@ -306,24 +294,26 @@ namespace Abc.Zebus.Persistence.CQL.Tests
                 });
             }
         }
-        
+
         [Test]
-        public void should_update_non_ack_message_count()
+        public async Task should_update_non_ack_message_count()
         {
             var firstPeer = new PeerId("Abc.Testing.Target");
             var secondPeer = new PeerId("Abc.Testing.OtherTarget");
-            
-            _storage.Write(new[] { MatcherEntry.Message(firstPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x01, 0x02, 0x03 }) });
-            _storage.Write(new[] { MatcherEntry.Message(secondPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x04, 0x05, 0x06 }) });
-            _storage.Write(new[] { MatcherEntry.Message(firstPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x07, 0x08, 0x09 }) });
 
-            _peerStateRepository[firstPeer].NonAckedMessageCount.ShouldEqual(2);
-            _peerStateRepository[secondPeer].NonAckedMessageCount.ShouldEqual(1);
+            await _storage.Write(new[] { MatcherEntry.Message(firstPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x01, 0x02, 0x03 }) });
+            await _storage.Write(new[] { MatcherEntry.Message(secondPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x04, 0x05, 0x06 }) });
+            await _storage.Write(new[] { MatcherEntry.Message(firstPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x07, 0x08, 0x09 }) });
+
+            var nonAckedMessageCountsForUpdatedPeers = _storage.GetNonAckedMessageCounts();
+            nonAckedMessageCountsForUpdatedPeers[firstPeer].ShouldEqual(2);
+            nonAckedMessageCountsForUpdatedPeers[secondPeer].ShouldEqual(1);
 
-            _storage.Write(new[] { MatcherEntry.Ack(firstPeer, MessageId.NextId()) });
+            await _storage.Write(new[] { MatcherEntry.Ack(firstPeer, MessageId.NextId()) });
 
-            _peerStateRepository[firstPeer].NonAckedMessageCount.ShouldEqual(1);
-            _peerStateRepository[secondPeer].NonAckedMessageCount.ShouldEqual(1);
+            nonAckedMessageCountsForUpdatedPeers = _storage.GetNonAckedMessageCounts();
+            nonAckedMessageCountsForUpdatedPeers[firstPeer].ShouldEqual(1);
+            nonAckedMessageCountsForUpdatedPeers[secondPeer].ShouldEqual(1); 
         }
 
         [Test]
@@ -331,14 +321,12 @@ namespace Abc.Zebus.Persistence.CQL.Tests
         {
             var firstPeer = new PeerId("Abc.Testing.Target");
             var secondPeer = new PeerId("Abc.Testing.OtherTarget");
-            _peerStateRepository.Add(new PeerState(firstPeer, 0, SystemDateTime.UtcNow.Date.Ticks));
-            _peerStateRepository.Add(new PeerState(secondPeer, 0, SystemDateTime.UtcNow.Date.Ticks));
 
             using (MessageId.PauseIdGeneration())
             using (SystemDateTime.PauseTime())
             {
-                var expectedTransportMessages = Enumerable.Range(1, 100).Select(CreateTestTransportMessage).ToList();
-                var messages = expectedTransportMessages.SelectMany(x =>
+                var transportMessages = Enumerable.Range(1, 100).Select(CreateTestTransportMessage).ToList();
+                var messages = transportMessages.SelectMany(x =>
                                                         {
                                                             var transportMessageBytes = Serialization.Serializer.Serialize(x).ToArray();
                                                             return new[]
@@ -351,10 +339,12 @@ namespace Abc.Zebus.Persistence.CQL.Tests
 
                 await _storage.Write(messages);
 
-                _peerStateRepository[firstPeer].NonAckedMessageCount.ShouldEqual(100);
-                _peerStateRepository[secondPeer].NonAckedMessageCount.ShouldEqual(100);
+                var nonAckedMessageCountsForUpdatedPeers = _storage.GetNonAckedMessageCounts();
+                nonAckedMessageCountsForUpdatedPeers[firstPeer].ShouldEqual(100);
+                nonAckedMessageCountsForUpdatedPeers[secondPeer].ShouldEqual(100);
 
                 var readerForFirstPeer = (CqlMessageReader)_storage.CreateMessageReader(firstPeer);
+                var expectedTransportMessages = transportMessages.Select(Serialization.Serializer.Serialize).Select(x => x.ToArray()).ToList();
                 readerForFirstPeer.GetUnackedMessages().ToList().ShouldEqualDeeply(expectedTransportMessages);
 
                 var readerForSecondPeer = (CqlMessageReader)_storage.CreateMessageReader(secondPeer);
@@ -376,12 +366,171 @@ namespace Abc.Zebus.Persistence.CQL.Tests
             _reporterMock.Verify(r => r.AddStorageReport(2, 7, 4, "Abc.Message.Fat"));
         }
 
+        [Test]
+        public async Task should_update_oldest_non_acked_message_timestamp()
+        {
+            using (SystemDateTime.PauseTime())
+            {
+                var peerId = new PeerId("PeerId");
+                var now = SystemDateTime.UtcNow;
+                InsertPersistentMessage(peerId, now.AddMilliseconds(1), x => x.IsAcked = true);
+                InsertPersistentMessage(peerId, now.AddMilliseconds(2), x => x.IsAcked = false);
+                InsertPersistentMessage(peerId, now.AddMilliseconds(3), x => x.IsAcked = false);
+                InsertPersistentMessage(peerId, now.AddMilliseconds(4), x => x.IsAcked = true);
+                InsertPersistentMessage(peerId, now.AddMilliseconds(5), x => x.IsAcked = false);
+                var peerState = new PeerState(peerId, 0, now.AddMilliseconds(1).Ticks);
+                InsertPeerState(peerState);
+
+                await _storage.CleanBuckets(peerState);
+
+                GetPeerState(peerId).OldestNonAckedMessageTimestampInTicks.ShouldEqual(now.AddMilliseconds(2).Ticks);
+            }
+        }
+
+        [Test]
+        public async Task should_not_update_oldest_non_acked_message_timestamp_if_it_did_not_change()
+        {
+            var peerId = new PeerId("PeerId");
+            var now = SystemDateTime.UtcNow;
+            InsertPersistentMessage(peerId, now.AddMilliseconds(1), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddMilliseconds(2), x => x.IsAcked = false);
+            InsertPersistentMessage(peerId, now.AddMilliseconds(3), x => x.IsAcked = false);
+            InsertPersistentMessage(peerId, now.AddMilliseconds(4), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddMilliseconds(5), x => x.IsAcked = false);
+            var peerState = new PeerState(peerId, 0, now.AddMilliseconds(2).Ticks);
+            InsertPeerState(peerState);
+
+            await _storage.CleanBuckets(peerState);
+
+            GetPeerState(peerId).OldestNonAckedMessageTimestampInTicks.ShouldEqual(now.AddMilliseconds(2).Ticks);
+        }
+
+        [Test]
+        public async Task should_take_last_message_timestamp_plus_one_tick_as_oldest_non_acked_message_when_all_messages_are_acked()
+        {
+            var peerId = new PeerId("PeerId");
+            var now = SystemDateTime.UtcNow;
+            InsertPersistentMessage(peerId, now.AddMilliseconds(1), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddMilliseconds(2), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddMilliseconds(3), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddMilliseconds(4), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddMilliseconds(5), x => x.IsAcked = true);
+            var peerState = new PeerState(peerId, 0, now.AddMilliseconds(2).Ticks);
+            InsertPeerState(peerState);
+
+            await _storage.CleanBuckets(peerState);
+
+            GetPeerState(peerId).OldestNonAckedMessageTimestampInTicks.ShouldEqual(now.AddMilliseconds(5).Ticks + 1);
+        }
+
+        [Test]
+        public async Task should_take_utc_now_timestamp_as_oldest_non_acked_message_when_no_messages_are_acked()
+        {
+            using (SystemDateTime.PauseTime())
+            {
+                var peerId = new PeerId("PeerId");
+                var now = SystemDateTime.UtcNow;
+                var peerState = new PeerState(peerId, 0, now.AddMilliseconds(2).Ticks);
+                InsertPeerState(peerState);
+
+                await _storage.CleanBuckets(peerState);
+
+                GetPeerState(peerId).OldestNonAckedMessageTimestampInTicks.ShouldEqual(now.Ticks);
+            }
+        }
+
+        [Test]
+        public async Task should_delete_buckets_when_all_messages_are_acked_in_it()
+        {
+            var peerId = new PeerId("PeerId");
+            var now = DateTime.UtcNow.Date;
+            var peerState = new PeerState(peerId, 0, now.AddHours(-5).Ticks);
+            InsertPeerState(peerState);
+
+            // first bucket - 3 hours ago - all acked
+            InsertPersistentMessage(peerId, now.AddHours(-3), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddHours(-3).AddMinutes(1), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddHours(-3).AddMinutes(2), x => x.IsAcked = true);
+            // second bucket - 2 hours ago - all acked
+            InsertPersistentMessage(peerId, now.AddHours(-2), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddHours(-2).AddMinutes(1), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddHours(-2).AddMinutes(2), x => x.IsAcked = true);
+            // third bucket - 1 hours ago - with non acked
+            InsertPersistentMessage(peerId, now.AddHours(-1), x => x.IsAcked = true);
+            InsertPersistentMessage(peerId, now.AddHours(-1).AddMinutes(1), x => x.IsAcked = false); // <--- non acked !
+            InsertPersistentMessage(peerId, now.AddHours(-1).AddMinutes(2), x => x.IsAcked = true);
+            // current bucket - all acked
+            InsertPersistentMessage(peerId, now, x => x.IsAcked = true);
+
+            var messages = DataContext.PersistentMessages.Execute().ToList();
+            messages.Count.ShouldEqual(10);
+
+            await _storage.CleanBuckets(peerState);
+
+            Wait.Until(() => DataContext.PersistentMessages.Execute().Count() == 4, 2.Seconds());
+
+            GetPeerState(peerId).OldestNonAckedMessageTimestampInTicks.ShouldEqual(now.AddHours(-1).AddMinutes(1).Ticks);
+            var persistentMessagesFromDatabase = DataContext.PersistentMessages.Execute().ToList();
+            var storedMessages = persistentMessagesFromDatabase.Select(x => new { x.UniqueTimestampInTicks, x.IsAcked }).ToList();
+            storedMessages.ShouldBeEquivalentTo(new[]
+            {
+                new { UniqueTimestampInTicks = now.AddHours(-1).Ticks, IsAcked = true },
+                new { UniqueTimestampInTicks = now.AddHours(-1).AddMinutes(1).Ticks, IsAcked = false },
+                new { UniqueTimestampInTicks = now.AddHours(-1).AddMinutes(2).Ticks, IsAcked = true },
+                new { UniqueTimestampInTicks = now.Ticks, IsAcked = true },
+            });
+        }
+
+        private static long GetBucketIdFromMessageId(MessageId message)
+        {
+            return GetBucketIdFromDateTime(message.GetDateTime());
+        }
+
+        private static long GetBucketIdFromDateTime(DateTime timestamp)
+        {
+            return new DateTime(timestamp.Year, timestamp.Month, timestamp.Day, timestamp.Hour, 0, 0).Ticks;
+        }
+
+        private void InsertPersistentMessage(PeerId peerId, DateTime timestamp, Action<PersistentMessage> updateMessage = null)
+        {
+            var message = new PersistentMessage
+            {
+                PeerId = peerId.ToString(),
+                BucketId = BucketIdHelper.GetBucketId(timestamp),
+                IsAcked = true,
+                UniqueTimestampInTicks = timestamp.Ticks,
+                TransportMessage = new byte[0]
+            };
+            updateMessage?.Invoke(message);
+            DataContext.PersistentMessages.Insert(message).Execute();
+        }
+
+        private void InsertPeerState(PeerState peerState)
+        {
+            DataContext.PeerStates.Insert(new CassandraPeerState(peerState));
+        }
+
+        private PeerState GetPeerState(in PeerId peerId)
+        {
+            var peerString = peerId.ToString();
+            var state = DataContext.PeerStates.Where(x => x.PeerId == peerString).Execute().Single();
+            return new PeerState(new PeerId(state.PeerId), state.NonAckedMessageCount, state.OldestNonAckedMessageTimestamp);
+        }
+
         private static TransportMessage CreateTestTransportMessage(int i)
         {
             MessageId.PauseIdGenerationAtDate(SystemDateTime.UtcNow.Date.AddSeconds(i * 10));
             return new Message1(i).ToTransportMessage();
         }
 
+        private static long ToUnixMicroSeconds(DateTime timestamp)
+        {
+            var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0);
+            var diff = timestamp - origin;
+            var diffInMicroSeconds = diff.Ticks / 10;
+            return diffInMicroSeconds;
+        }
+
         [ProtoContract]
         private class Message1 : IEvent
         {

src/Abc.Zebus.Persistence.CQL.Tests/OldestNonAckedMessageUpdaterPeriodicActionTests.cs

@@ -1,11 +1,11 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Abc.Zebus.Persistence.CQL.Data;
 using Abc.Zebus.Persistence.CQL.PeriodicAction;
 using Abc.Zebus.Persistence.CQL.Storage;
-using Abc.Zebus.Persistence.CQL.Testing;
 using Abc.Zebus.Persistence.CQL.Tests.Cql;
-using Abc.Zebus.Persistence.Messages;
 using Abc.Zebus.Testing;
 using Abc.Zebus.Testing.Extensions;
 using Abc.Zebus.Util;
@@ -16,9 +16,9 @@ namespace Abc.Zebus.Persistence.CQL.Tests
 {
     public class OldestNonAckedMessageUpdaterPeriodicActionTests : CqlTestFixture<PersistenceCqlDataContext, ICqlPersistenceConfiguration>
     {
-        private FakePeerStateRepository _peerStateRepo;
         private TestBus _testBus;
         private OldestNonAckedMessageUpdaterPeriodicAction _oldestMessageUpdater;
+        private Mock<ICqlStorage> _cqlStorage;
 
         public override void CreateSchema()
         {
@@ -37,138 +37,64 @@ namespace Abc.Zebus.Persistence.CQL.Tests
         [SetUp]
         public void SetUp()
         {
-            _peerStateRepo = new FakePeerStateRepository();
             _testBus = new TestBus();
-            _oldestMessageUpdater = new OldestNonAckedMessageUpdaterPeriodicAction(_testBus, _peerStateRepo, DataContext, new Mock<ICqlPersistenceConfiguration>().Object);
+            var configurationMock = new Mock<ICqlPersistenceConfiguration>();
+            configurationMock.SetupGet(x => x.OldestMessagePerPeerCheckPeriod).Returns(30.Seconds());
+            configurationMock.SetupGet(x => x.OldestMessagePerPeerGlobalCheckPeriod).Returns(30.Seconds());
+            _cqlStorage = new Mock<ICqlStorage>();
+            _oldestMessageUpdater = new OldestNonAckedMessageUpdaterPeriodicAction(_testBus, configurationMock.Object, _cqlStorage.Object);
         }
 
         [Test]
-        public void should_update_oldest_non_acked_message_timestamp()
+        public void should_call_clean_up_buckets_global_check()
         {
-            var peerId = new PeerId("PeerId");
-            var now = SystemDateTime.UtcNow;
-            InsertPersistentMessage(peerId, now.AddMilliseconds(1), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(2), x => x.IsAcked = false);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(3), x => x.IsAcked = false);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(4), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(5), x => x.IsAcked = false);
-            _peerStateRepo.Add(new PeerState(peerId, 0, now.AddMilliseconds(1).Ticks));
+            var peerState = new PeerState(new PeerId("Peer"));
+            var otherPeerState = new PeerState(new PeerId("OtherPeer"));
+            _cqlStorage.Setup(s => s.GetAllKnownPeers()).Returns(new[] { peerState, otherPeerState, });
 
             _oldestMessageUpdater.DoPeriodicAction();
 
-            _peerStateRepo[peerId].OldestNonAckedMessageTimestampInTicks.ShouldEqual(now.AddMilliseconds(2).Ticks);
+            _cqlStorage.Verify(x => x.CleanBuckets(peerState), Times.Once);
+            _cqlStorage.Verify(x => x.CleanBuckets(otherPeerState), Times.Once);
         }
 
         [Test]
-        public void should_not_update_oldest_non_acked_message_timestamp_if_it_did_not_change()
+        public void should_call_clean_up_buckets()
         {
-            var peerId = new PeerId("PeerId");
-            var now = SystemDateTime.UtcNow;
-            InsertPersistentMessage(peerId, now.AddMilliseconds(1), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(2), x => x.IsAcked = false);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(3), x => x.IsAcked = false);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(4), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(5), x => x.IsAcked = false);
-            _peerStateRepo.Add(new PeerState(peerId, 0, now.AddMilliseconds(2).Ticks));
-
             _oldestMessageUpdater.DoPeriodicAction();
 
-            _peerStateRepo[peerId].OldestNonAckedMessageTimestampInTicks.ShouldEqual(now.AddMilliseconds(2).Ticks);
-        }
-
-        [Test]
-        public void should_take_last_message_timestamp_plus_one_tick_as_oldest_non_acked_message_when_all_messages_are_acked()
-        {
-            var peerId = new PeerId("PeerId");
-            var now = SystemDateTime.UtcNow;
-            InsertPersistentMessage(peerId, now.AddMilliseconds(1), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(2), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(3), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(4), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddMilliseconds(5), x => x.IsAcked = true);
-            _peerStateRepo.Add(new PeerState(peerId, 0, now.AddMilliseconds(2).Ticks));
+            var peerState = new PeerState(new PeerId("Peer"));
+            var otherPeerState = new PeerState(new PeerId("OtherPeer"));
+            _cqlStorage.Setup(s => s.GetAllKnownPeers()).Returns(new[] { peerState, otherPeerState, });
 
             _oldestMessageUpdater.DoPeriodicAction();
 
-            _peerStateRepo[peerId].OldestNonAckedMessageTimestampInTicks.ShouldEqual(now.AddMilliseconds(5).Ticks + 1);
+            _cqlStorage.Verify(x => x.CleanBuckets(peerState), Times.Once);
+            _cqlStorage.Verify(x => x.CleanBuckets(otherPeerState), Times.Once);
         }
 
         [Test]
-        public void should_take_utc_now_timestamp_as_oldest_non_acked_message_when_no_messages_are_acked()
+        public void should_only_call_clean_for_updated_peers()
         {
-            using (SystemDateTime.PauseTime())
+            var peerStates = new[]
             {
-                var peerId = new PeerId("PeerId");
-                var now = SystemDateTime.UtcNow;
-                _peerStateRepo.Add(new PeerState(peerId, 0, now.AddMilliseconds(2).Ticks));
-
-                _oldestMessageUpdater.DoPeriodicAction();
+                new PeerState(new PeerId("Peer")),
+                new PeerState(new PeerId("OtherPeer"))
+            };
 
-                _peerStateRepo[peerId].OldestNonAckedMessageTimestampInTicks.ShouldEqual(now.Ticks);
-            }
-        }
+            var cleanedPeerStates = new List<PeerState>();
 
-        [Test]
-        public void should_delete_buckets_when_all_messages_are_acked_in_it()
-        {
-            var peerId = new PeerId("PeerId");
-            var now = DateTime.UtcNow.Date;
-            _peerStateRepo.Add(new PeerState(peerId, 0, now.AddHours(-5).Ticks));
-
-            // first bucket - 3 hours ago - all acked
-            InsertPersistentMessage(peerId, now.AddHours(-3), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddHours(-3).AddMinutes(1), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddHours(-3).AddMinutes(2), x => x.IsAcked = true);
-            // second bucket - 2 hours ago - all acked
-            InsertPersistentMessage(peerId, now.AddHours(-2), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddHours(-2).AddMinutes(1), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddHours(-2).AddMinutes(2), x => x.IsAcked = true);
-            // third bucket - 1 hours ago - with non acked
-            InsertPersistentMessage(peerId, now.AddHours(-1), x => x.IsAcked = true);
-            InsertPersistentMessage(peerId, now.AddHours(-1).AddMinutes(1), x => x.IsAcked = false); // <--- non acked !
-            InsertPersistentMessage(peerId, now.AddHours(-1).AddMinutes(2), x => x.IsAcked = true);
-            // current bucket - all acked
-            InsertPersistentMessage(peerId, now, x => x.IsAcked = true);
-
-            var messages = DataContext.PersistentMessages.Execute().ToList();
-            messages.Count.ShouldEqual(10);
+            _cqlStorage.Setup(s => s.GetAllKnownPeers()).Returns(peerStates);
+            _cqlStorage.Setup(s => s.CleanBuckets(Capture.In(cleanedPeerStates))).Returns(Task.CompletedTask);
 
             _oldestMessageUpdater.DoPeriodicAction();
 
-            Wait.Until(()=>DataContext.PersistentMessages.Execute().Count() == 4, 2.Seconds());
-
-            _peerStateRepo[peerId].OldestNonAckedMessageTimestampInTicks.ShouldEqual(now.AddHours(-1).AddMinutes(1).Ticks);
-            var persistentMessagesFromDatabase = DataContext.PersistentMessages.Execute().ToList();
-            var storedMessages = persistentMessagesFromDatabase.Select(x => new { x.UniqueTimestampInTicks, x.IsAcked }).ToList();
-            storedMessages.ShouldBeEquivalentTo(new[]
-            {
-                new { UniqueTimestampInTicks = now.AddHours(-1).Ticks, IsAcked = true },
-                new { UniqueTimestampInTicks = now.AddHours(-1).AddMinutes(1).Ticks, IsAcked = false },
-                new { UniqueTimestampInTicks = now.AddHours(-1).AddMinutes(2).Ticks, IsAcked = true },
-                new { UniqueTimestampInTicks = now.Ticks, IsAcked = true },
-            });
-        }
-
-        [Test]
-        public void should_save_peer_state_repo()
-        {
+            peerStates[0] = peerStates[0].WithNonAckedMessageCountDelta(1);
+ 
             _oldestMessageUpdater.DoPeriodicAction();
 
-            _peerStateRepo.HasBeenSaved.ShouldBeTrue();
-        }
-
-        private void InsertPersistentMessage(PeerId peerId, DateTime timestamp, Action<PersistentMessage> updateMessage = null)
-        {
-            var message = new PersistentMessage
-            {
-                PeerId = peerId.ToString(),
-                BucketId = BucketIdHelper.GetBucketId(timestamp),
-                IsAcked = true,
-                UniqueTimestampInTicks = timestamp.Ticks,
-                TransportMessage = new byte[0]
-            };
-            updateMessage?.Invoke(message);
-            DataContext.PersistentMessages.Insert(message).Execute();
+            cleanedPeerStates.Select(x => x.PeerId)
+                             .ShouldBeEquivalentTo(peerStates[0].PeerId, peerStates[1].PeerId, peerStates[0].PeerId);
         }
     }
 }

src/Abc.Zebus.Persistence.CQL.Tests/PeerStateRepositoryTests.cs

@@ -1,170 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading.Tasks;
-using Abc.Zebus.Persistence.CQL.Data;
-using Abc.Zebus.Persistence.CQL.Storage;
-using Abc.Zebus.Persistence.CQL.Tests.Cql;
-using Abc.Zebus.Persistence.Messages;
-using Abc.Zebus.Testing.Extensions;
-using Abc.Zebus.Util;
-using NUnit.Framework;
-
-namespace Abc.Zebus.Persistence.CQL.Tests
-{
-    public class PeerStateRepositoryTests : CqlTestFixture<PersistenceCqlDataContext, ICqlPersistenceConfiguration>
-    {
-        private PeerStateRepository _peerStateRepository;
-
-        public override void CreateSchema()
-        {
-            IgnoreWhenSet("APPVEYOR");
-            IgnoreWhenSet("TF_BUILD");
-            base.CreateSchema();
-        }
-
-        private void IgnoreWhenSet(string environmentVariable)
-        {
-            var env = Environment.GetEnvironmentVariable(environmentVariable);
-            if (!string.IsNullOrEmpty(env) && bool.TryParse(env, out var isSet) && isSet)
-                Assert.Ignore("We need a cassandra node for this");
-        }
-
-        [SetUp]
-        public void SetUp()
-        {
-            _peerStateRepository = new PeerStateRepository(DataContext);
-        }
-
-        [Test]
-        public void update_non_acked_message_count_should_create_the_peer_state_if_not_exists_yet()
-        {
-            _peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId"), 10);
-
-            _peerStateRepository.ShouldNotBeEmpty();
-            _peerStateRepository[new PeerId("PeerId")].ShouldNotBeNull();
-        }
-
-        [Test]
-        public void should_update_non_acked_message_count()
-        {
-            _peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId"), 10);
-
-            _peerStateRepository[new PeerId("PeerId")].NonAckedMessageCount.ShouldEqual(10);
-        }
-
-        [Test]
-        public void should_update_last_count_change_when_changing_count()
-        {
-            _peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId"), 10);
-
-            _peerStateRepository[new PeerId("PeerId")].LastNonAckedMessageCountVersion.ShouldBeGreaterThan(0);
-        }
-
-        [Test]
-        public void peer_state_should_be_deleted_from_repository_when_removed()
-        {
-            var peerId = new PeerId("PeerId");
-            _peerStateRepository.UpdateNonAckMessageCount(peerId, 10);
-            _peerStateRepository.RemovePeer(peerId);
-
-            _peerStateRepository.ShouldBeEmpty();
-            _peerStateRepository.Any(x => x.PeerId == peerId).ShouldBeFalse();
-        }
-
-        [Test]
-        public async Task should_save_state_to_cassandra()
-        {
-            using (SystemDateTime.PauseTime())
-            {
-                _peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId"), 10);
-
-                await _peerStateRepository.Save();
-
-                var oldestNonAckedMessageTimestampCaptured = SystemDateTime.UtcNow - CqlStorage.PersistentMessagesTimeToLive;
-
-                var cassandraState = DataContext.PeerStates.Execute().ExpectedSingle();
-                cassandraState.PeerId.ShouldEqual("PeerId");
-                cassandraState.NonAckedMessageCount.ShouldEqual(10);
-                cassandraState.OldestNonAckedMessageTimestamp.ShouldEqual(oldestNonAckedMessageTimestampCaptured.Ticks);
-            }
-        }
-
-        [Test]
-        public async Task should_reload_state_from_cassandra_on_initialize()
-        {
-            using (SystemDateTime.PauseTime())
-            {
-                _peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId"), 10);
-
-                await _peerStateRepository.Save();
-
-                var oldestNonAckedMessageTimestampCaptured = SystemDateTime.UtcNow - CqlStorage.PersistentMessagesTimeToLive;
-
-                using (SystemDateTime.Set(utcNow: SystemDateTime.UtcNow.Add(2.Hours())))
-                {
-                    var newRepo = new PeerStateRepository(DataContext);
-                    newRepo.Initialize();
-
-                    var cassandraState = newRepo.ExpectedSingle();
-                    cassandraState.PeerId.ShouldEqual(new PeerId("PeerId"));
-                    cassandraState.NonAckedMessageCount.ShouldEqual(10);
-                    cassandraState.OldestNonAckedMessageTimestampInTicks.ShouldEqual(oldestNonAckedMessageTimestampCaptured.Ticks);
-                }
-            }
-        }
-
-        [Test]
-        public async Task should_delete_peer_from_cassandra_when_removed()
-        {
-            using (SystemDateTime.PauseTime())
-            {
-                _peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId"), 10);
-                await _peerStateRepository.Save();
-
-                DataContext.PeerStates.Execute().ShouldNotBeEmpty();
-
-                await _peerStateRepository.RemovePeer(new PeerId("PeerId"));
-
-                DataContext.PeerStates.Execute().ShouldBeEmpty();
-            }
-        }
-
-        [Test]
-        public void should_get_updated_peers_only_for_the_peers_that_actually_changed_since_the_last_publication()
-        {
-            var version = 0L;
-
-            _peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId.1"), 10);
-            _peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId.2"), 20);
-
-            var peers1 = _peerStateRepository.GetUpdatedPeers(ref version).Select(x => x.PeerId.ToString()).ToList();
-            peers1.ShouldBeEquivalentTo(new[] { "PeerId.1", "PeerId.2" });
-
-            _peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId.1"), 2);
-
-            var peers2 = _peerStateRepository.GetUpdatedPeers(ref version).Select(x => x.PeerId.ToString()).ToList();
-            peers2.ShouldBeEquivalentTo(new[] { "PeerId.1" });
-        }
-
-        [Test]
-        public async Task should_delete_all_buckets_for_peer_when_removed()
-        {
-            var now = SystemDateTime.UtcNow;
-            _peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId"), 0);
-            DataContext.PersistentMessages.Insert(new PersistentMessage
-            {
-                BucketId = BucketIdHelper.GetBucketId(now),
-                IsAcked = true,
-                PeerId = "PeerId",
-                TransportMessage = new byte[0],
-                UniqueTimestampInTicks = now.Ticks
-            }).Execute();
-
-            DataContext.PersistentMessages.Execute().Count().ShouldEqual(1);
-
-            await _peerStateRepository.RemovePeer(new PeerId("PeerId"));
-
-            DataContext.PersistentMessages.Execute().Any().ShouldBeFalse();
-        }
-    }
-}

src/Abc.Zebus.Persistence.CQL/IPeerStateRepository.cs

@@ -1,20 +0,0 @@
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using Abc.Zebus.Persistence.CQL.Storage;
-
-namespace Abc.Zebus.Persistence.CQL
-{
-    public interface IPeerStateRepository : IEnumerable<PeerState>
-    {
-        void Initialize();
-
-        PeerState GetPeerStateFor(PeerId peerId);
-
-        void UpdateNonAckMessageCount(PeerId peerId, int delta);
-
-        List<PeerState> GetUpdatedPeers(ref long version);
-
-        Task Save();
-        Task RemovePeer(PeerId peerId);
-    }
-}

src/Abc.Zebus.Persistence.CQL/PeriodicAction/OldestNonAckedMessageUpdaterPeriodicAction.cs

@@ -1,45 +1,44 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Abc.Zebus.Hosting;
-using Abc.Zebus.Persistence.CQL.Data;
 using Abc.Zebus.Persistence.CQL.Storage;
-using Abc.Zebus.Persistence.Messages;
+using Abc.Zebus.Persistence.Storage;
 using Abc.Zebus.Util;
-using Cassandra.Data.Linq;
 
 namespace Abc.Zebus.Persistence.CQL.PeriodicAction
 {
     public class OldestNonAckedMessageUpdaterPeriodicAction : PeriodicActionHostInitializer
     {
-        private readonly IPeerStateRepository _peerStateRepository;
-        private readonly PersistenceCqlDataContext _dataContext;
         private readonly ICqlPersistenceConfiguration _configuration;
-        private long _lastCheckVersion;
+        private readonly ICqlStorage _cqlStorage;
         private DateTime _lastGlobalCheck;
+        private readonly NonAckedCountCache _nonAckedCountCache = new NonAckedCountCache();
 
-        public OldestNonAckedMessageUpdaterPeriodicAction(IBus bus, IPeerStateRepository peerStateRepository, PersistenceCqlDataContext dataContext, ICqlPersistenceConfiguration configuration) 
+        public OldestNonAckedMessageUpdaterPeriodicAction(IBus bus, ICqlPersistenceConfiguration configuration, ICqlStorage cqlStorage)
             : base(bus, configuration.OldestMessagePerPeerCheckPeriod)
         {
-            _peerStateRepository = peerStateRepository;
-            _dataContext = dataContext;
             _configuration = configuration;
+            _cqlStorage = cqlStorage;
         }
 
         public override void DoPeriodicAction()
         {
             var isGlobalCheck = SystemDateTime.UtcNow >= _lastGlobalCheck.Add(_configuration.OldestMessagePerPeerGlobalCheckPeriod);
+            var allPeersDictionary = _cqlStorage.GetAllKnownPeers().ToDictionary(state => state.PeerId);
+            IEnumerable<PeerState> peersToCheck = allPeersDictionary.Values;
+            var updatedPeers = _nonAckedCountCache.GetForUpdatedPeers(peersToCheck.Select(x => (x.PeerId, x.NonAckedMessageCount)).ToList());
             if (isGlobalCheck)
             {
-                _lastCheckVersion = 0;
                 _lastGlobalCheck = SystemDateTime.UtcNow;
             }
-
-            var peersToCheck = _peerStateRepository.GetUpdatedPeers(ref _lastCheckVersion);
+            else
+            {
+                peersToCheck = updatedPeers.Select(x => allPeersDictionary[x.PeerId]);
+            }
 
             Parallel.ForEach(peersToCheck, new ParallelOptions { MaxDegreeOfParallelism = 10 }, UpdateOldestNonAckedMessage);
-
-            _peerStateRepository.Save();
         }
 
         private void UpdateOldestNonAckedMessage(PeerState peer)
@@ -47,53 +46,8 @@ namespace Abc.Zebus.Persistence.CQL.PeriodicAction
             if (peer.Removed)
                 return;
 
-            var newOldest = GetOldestNonAckedMessageTimestamp(peer);
-
-            CleanBuckets(peer.PeerId, peer.OldestNonAckedMessageTimestampInTicks, newOldest);
-
-            peer.UpdateOldestNonAckedMessageTimestamp(newOldest);
-        }
-
-        private long GetOldestNonAckedMessageTimestamp(PeerState peer)
-        {
-            var peerId = peer.PeerId.ToString();
-            var lastAckedMessageTimestamp = 0L;
-
-            foreach (var currentBucketId in BucketIdHelper.GetBucketsCollection(peer.OldestNonAckedMessageTimestampInTicks))
-            {
-                var messagesInBucket = _dataContext.PersistentMessages
-                                                   .Where(x => x.PeerId == peerId
-                                                               && x.BucketId == currentBucketId
-                                                               && x.UniqueTimestampInTicks >= peer.OldestNonAckedMessageTimestampInTicks)
-                                                   .OrderBy(x => x.UniqueTimestampInTicks)
-                                                   .Select(x => new { x.IsAcked, x.UniqueTimestampInTicks })
-                                                   .Execute();
-
-                foreach (var message in messagesInBucket)
-                {
-                    if (!message.IsAcked)
-                        return message.UniqueTimestampInTicks;
-
-                    lastAckedMessageTimestamp = message.UniqueTimestampInTicks;
-                    
-                }
-            }
-
-            return lastAckedMessageTimestamp == 0 ? SystemDateTime.UtcNow.Ticks : lastAckedMessageTimestamp + 1;
-        }
-
-        private void CleanBuckets(PeerId peerId, long previousOldestMessageTimestamp, long newOldestMessageTimestamp)
-        {
-            var firstBucketToDelete = BucketIdHelper.GetBucketId(previousOldestMessageTimestamp);
-            var lastBucketToDelete = BucketIdHelper.GetPreviousBucketId(newOldestMessageTimestamp);
-            if (firstBucketToDelete == lastBucketToDelete)
-                return;
-
-            var bucketsToDelete = BucketIdHelper.GetBucketsCollection(firstBucketToDelete, lastBucketToDelete).ToArray();
-            _dataContext.PersistentMessages
-                        .Where(x => x.PeerId == peerId.ToString() && bucketsToDelete.Contains(x.BucketId))
-                        .Delete()
-                        .ExecuteAsync();
+            _cqlStorage.CleanBuckets(peer)
+                       .Wait(_configuration.OldestMessagePerPeerCheckPeriod);
         }
     }
 }

src/Abc.Zebus.Persistence.CQL/Properties/AssemblyInfo.cs

@@ -7,3 +7,4 @@ using System.Runtime.InteropServices;
 [assembly: InternalsVisibleTo("Abc.Zebus.Integration")]
 [assembly: InternalsVisibleTo("Abc.Zebus.PersistenceService.Tests")]
 [assembly: InternalsVisibleTo("Abc.Zebus.Persistence.CQL.Testing")]
+[assembly: InternalsVisibleTo("Abc.Zebus.Persistence.CQL.Tests")]

src/Abc.Zebus.Persistence.CQL/Storage/CqlMessageReader.cs

@@ -4,7 +4,6 @@ using System.Linq;
 using Abc.Zebus.Persistence.CQL.Data;
 using Abc.Zebus.Persistence.Messages;
 using Abc.Zebus.Persistence.Storage;
-using Abc.Zebus.Transport;
 using Cassandra;
 using Cassandra.Data.Linq;
 using log4net;
@@ -31,7 +30,7 @@ namespace Abc.Zebus.Persistence.CQL.Storage
                                                                           .ToString());
         }
 
-        public IEnumerable<TransportMessage> GetUnackedMessages()
+        public IEnumerable<byte[]> GetUnackedMessages()
         {
             var oldestNonAckedMessageTimestampInTicks = _peerState.OldestNonAckedMessageTimestampInTicks;
             _log.Info($"Reading messages for peer {_peerState.PeerId} from {oldestNonAckedMessageTimestampInTicks} ({new DateTime(oldestNonAckedMessageTimestampInTicks).ToLongTimeString()})");
@@ -52,16 +51,12 @@ namespace Abc.Zebus.Persistence.CQL.Storage
             _log.Info($"{nonAckedMessageRead} non acked messages replayed for peer {_peerState.PeerId}");
         }
 
-        private IEnumerable<TransportMessage> GetNonAckedMessagesInBucket(long oldestNonAckedMessageTimestampInTicks, long bucketId)
+        private IEnumerable<byte[]> GetNonAckedMessagesInBucket(long oldestNonAckedMessageTimestampInTicks, long bucketId)
         {
-            return _dataContext.Session.Execute(_preparedStatement.Bind(_peerState.PeerId.ToString(), bucketId, oldestNonAckedMessageTimestampInTicks).SetPageSize(10 * 1000))
+            return _dataContext.Session
+                               .Execute(_preparedStatement.Bind(_peerState.PeerId.ToString(), bucketId, oldestNonAckedMessageTimestampInTicks).SetPageSize(10 * 1000))
                                .Where(x => !x.GetValue<bool>("IsAcked"))
-                               .Select(CreatePersistentMessageFromRow);
-        }
-
-        private static TransportMessage CreatePersistentMessageFromRow(Row row)
-        {
-            return TransportMessageDeserializer.Deserialize(row.GetValue<byte[]>("TransportMessage"));
+                               .Select(row => row.GetValue<byte[]>("TransportMessage"));
         }
 
         public void Dispose()

src/Abc.Zebus.Persistence.CQL/Storage/CqlStorage.cs

@@ -11,27 +11,27 @@ using Abc.Zebus.Persistence.Storage;
 using Abc.Zebus.Persistence.Util;
 using Abc.Zebus.Util;
 using Cassandra;
+using Cassandra.Data.Linq;
 using log4net;
 
 namespace Abc.Zebus.Persistence.CQL.Storage
 {
-    public class CqlStorage : IStorage, IDisposable
+    public class CqlStorage : ICqlStorage, IDisposable
     {
         private static readonly ILog _log = LogManager.GetLogger(typeof(CqlStorage));
         private static readonly DateTime _unixOrigin = new DateTime(1970, 1, 1, 0, 0, 0, 0);
 
         private readonly PersistenceCqlDataContext _dataContext;
-        private readonly IPeerStateRepository _peerStateRepository;
+        private readonly PeerStateRepository _peerStateRepository;
         private readonly IPersistenceConfiguration _configuration;
         private readonly IReporter _reporter;
         private readonly ParallelPersistor _parallelPersistor;
         private readonly PreparedStatement _preparedStatement;
-        private long _lastNonAckedMessageCountsVersion;
 
-        public CqlStorage(PersistenceCqlDataContext dataContext, IPeerStateRepository peerStateRepository, IPersistenceConfiguration configuration, IReporter reporter)
+        public CqlStorage(PersistenceCqlDataContext dataContext, IPersistenceConfiguration configuration, IReporter reporter)
         {
             _dataContext = dataContext;
-            _peerStateRepository = peerStateRepository;
+            _peerStateRepository = new PeerStateRepository(dataContext);
             _configuration = configuration;
             _reporter = reporter;
 
@@ -43,9 +43,9 @@ namespace Abc.Zebus.Persistence.CQL.Storage
 
         public int PersistenceQueueSize => _parallelPersistor.QueueSize;
 
-        public Dictionary<PeerId, int> GetNonAckedMessageCountsForUpdatedPeers()
+        public Dictionary<PeerId, int> GetNonAckedMessageCounts()
         {
-            return _peerStateRepository.GetUpdatedPeers(ref _lastNonAckedMessageCountsVersion)
+            return _peerStateRepository.GetAllKnownPeers()
                                        .ToDictionary(x => x.PeerId, x => x.NonAckedMessageCount);
         }
 
@@ -58,7 +58,6 @@ namespace Abc.Zebus.Persistence.CQL.Storage
         public void Stop()
         {
             Dispose();
-            _peerStateRepository.Save().Wait(2.Seconds());
         }
 
         public Task Write(IList<MatcherEntry> entriesToPersist)
@@ -75,7 +74,7 @@ namespace Abc.Zebus.Persistence.CQL.Storage
             {
                 var shouldInvestigatePeer = _configuration.PeerIdsToInvestigate != null && _configuration.PeerIdsToInvestigate.Contains(matcherEntry.PeerId.ToString());
                 if (shouldInvestigatePeer)
-                    _log.Info($"Storage requested for peer {matcherEntry.PeerId}, Type: {matcherEntry.Type}, Message Id: {matcherEntry.MessageId}"); 
+                    _log.Info($"Storage requested for peer {matcherEntry.PeerId}, Type: {matcherEntry.Type}, Message Id: {matcherEntry.MessageId}");
 
                 var messageDateTime = matcherEntry.MessageId.GetDateTimeForV2OrV3();
                 var rowTimestamp = matcherEntry.IsAck ? messageDateTime.AddTicks(10) : messageDateTime;
@@ -99,17 +98,18 @@ namespace Abc.Zebus.Persistence.CQL.Storage
                     _log.Info($"Count delta computed for peer {matcherEntry.PeerId}, will increment: {countDelta}");
             }
 
+            var updateNonAckedCountTasks = new List<Task>();
             foreach (var countForPeer in countByPeer)
             {
-                _peerStateRepository.UpdateNonAckMessageCount(countForPeer.Key, countForPeer.Value);
+                updateNonAckedCountTasks.Add(_peerStateRepository.UpdateNonAckMessageCount(countForPeer.Key, countForPeer.Value));
             }
 
-            return Task.WhenAll(insertTasks);
+            return Task.WhenAll(insertTasks.Concat(updateNonAckedCountTasks));
         }
 
-        public void RemovePeer(PeerId peerId)
+        public Task RemovePeer(PeerId peerId)
         {
-            _peerStateRepository.RemovePeer(peerId);
+            return _peerStateRepository.RemovePeer(peerId);
         }
 
         public IMessageReader CreateMessageReader(PeerId peerId)
@@ -133,6 +133,58 @@ namespace Abc.Zebus.Persistence.CQL.Storage
             _parallelPersistor.Dispose();
         }
 
+        public Task CleanBuckets(PeerState peer)
+        {
+            var peerId = peer.PeerId;
+            var newOldestMessageTimestamp = GetOldestNonAckedMessageTimestampInTicks(peer);
+
+            var firstBucketToDelete = BucketIdHelper.GetBucketId(peer.OldestNonAckedMessageTimestampInTicks);
+            var lastBucketToDelete = BucketIdHelper.GetPreviousBucketId(newOldestMessageTimestamp);
+            if (firstBucketToDelete == lastBucketToDelete)
+                return Task.CompletedTask;
+
+            var bucketsToDelete = BucketIdHelper.GetBucketsCollection(firstBucketToDelete, lastBucketToDelete).ToArray();
+            var peerIdString = peerId.ToString();
+            _dataContext.PersistentMessages
+                        .Where(x => x.PeerId == peerIdString && bucketsToDelete.Contains(x.BucketId))
+                        .Delete()
+                        .ExecuteAsync();
+
+            return _peerStateRepository.UpdateNewOldestMessageTimestamp(peer, newOldestMessageTimestamp);
+        }
+
+        private long GetOldestNonAckedMessageTimestampInTicks(PeerState peer)
+        {
+            var peerId = peer.PeerId.ToString();
+            var lastAckedMessageTimestamp = 0L;
+
+            foreach (var currentBucketId in BucketIdHelper.GetBucketsCollection(peer.OldestNonAckedMessageTimestampInTicks))
+            {
+                var messagesInBucket = _dataContext.PersistentMessages
+                                                   .Where(x => x.PeerId == peerId &&
+                                                               x.BucketId == currentBucketId &&
+                                                               x.UniqueTimestampInTicks >= peer.OldestNonAckedMessageTimestampInTicks)
+                                                   .OrderBy(x => x.UniqueTimestampInTicks)
+                                                   .Select(x => new { x.IsAcked, x.UniqueTimestampInTicks })
+                                                   .Execute();
+
+                foreach (var message in messagesInBucket)
+                {
+                    if (!message.IsAcked)
+                        return message.UniqueTimestampInTicks;
+
+                    lastAckedMessageTimestamp = message.UniqueTimestampInTicks;
+                }
+            }
+
+            return lastAckedMessageTimestamp == 0 ? SystemDateTime.UtcNow.Ticks : lastAckedMessageTimestamp + 1;
+        }
+
+        public IEnumerable<PeerState> GetAllKnownPeers()
+        {
+            return _peerStateRepository.GetAllKnownPeers();
+        }
+
         private static long ToUnixMicroSeconds(DateTime timestamp)
         {
             var diff = timestamp - _unixOrigin;

src/Abc.Zebus.Persistence.CQL/Storage/ICqlStorage.cs

@@ -0,0 +1,12 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Abc.Zebus.Persistence.Storage;
+
+namespace Abc.Zebus.Persistence.CQL.Storage
+{
+    public interface ICqlStorage : IStorage
+    {
+        Task CleanBuckets(PeerState peer);
+        IEnumerable<PeerState> GetAllKnownPeers();
+    }
+}

src/Abc.Zebus.Persistence.CQL/Storage/PeerState.cs

@@ -4,31 +4,35 @@ namespace Abc.Zebus.Persistence.CQL.Storage
 {
     public class PeerState
     {
-        public PeerState(PeerId peerId, int nonAckMessageCount = 0, long oldestNonAckedMessageTimestamp = 0)
+        public PeerState(PeerId peerId, int nonAckMessageCount = 0, long oldestNonAckedMessageTimestamp = 0, bool removed = false)
         {
             PeerId = peerId;
             NonAckedMessageCount = nonAckMessageCount;
             OldestNonAckedMessageTimestampInTicks = oldestNonAckedMessageTimestamp > 0 ? oldestNonAckedMessageTimestamp : SystemDateTime.UtcNow.Ticks - CqlStorage.PersistentMessagesTimeToLive.Ticks;
+            Removed = removed;
         }
 
         public PeerId PeerId { get; }
 
         public bool Removed { get; private set; }
         
-        public long OldestNonAckedMessageTimestampInTicks { get; private set; }
+        public long OldestNonAckedMessageTimestampInTicks { get; }
 
-        public long LastNonAckedMessageCountVersion { get; internal set; }
+        public int NonAckedMessageCount { get; }
 
-        public int NonAckedMessageCount { get; internal set; }
+        public void MarkAsRemoved()
+        {
+            Removed = true;
+        }
 
-        public void UpdateOldestNonAckedMessageTimestamp(long uniqueTimestampInTicks)
+        public PeerState WithNonAckedMessageCountDelta(int delta)
         {
-            OldestNonAckedMessageTimestampInTicks = uniqueTimestampInTicks;
+            return new PeerState(PeerId, NonAckedMessageCount + delta, OldestNonAckedMessageTimestampInTicks, Removed);
         }
 
-        public void MarkAsRemoved()
+        public PeerState WithOldestNonAckedMessageTimestampInTicks(long value)
         {
-            Removed = true;
+            return new PeerState(PeerId, NonAckedMessageCount, value, Removed);
         }
     }
 }

src/Abc.Zebus.Persistence.CQL/Storage/PeerStateRepository.cs

@@ -1,8 +1,6 @@
-using System.Collections;
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
-using System.Threading;
 using System.Threading.Tasks;
 using Abc.Zebus.Persistence.CQL.Data;
 using Abc.Zebus.Persistence.Messages;
@@ -14,13 +12,12 @@ using log4net;
 
 namespace Abc.Zebus.Persistence.CQL.Storage
 {
-    public class PeerStateRepository : IPeerStateRepository
+    public class PeerStateRepository
     {
         private static readonly ILog _log = LogManager.GetLogger(typeof(PeerStateRepository));
 
         private readonly PersistenceCqlDataContext _dataContext;
         private readonly ConcurrentDictionary<PeerId, PeerState> _statesByPeerId = new ConcurrentDictionary<PeerId, PeerState>();
-        private long _version;
 
         public PeerStateRepository(PersistenceCqlDataContext dataContext)
         {
@@ -45,32 +42,17 @@ namespace Abc.Zebus.Persistence.CQL.Storage
             return _statesByPeerId.GetValueOrDefault(peerId);
         }
 
-        public PeerState this[PeerId peerId] => _statesByPeerId[peerId];
-
-        public void UpdateNonAckMessageCount(PeerId peerId, int delta)
-        {
-            var peerState = _statesByPeerId.GetOrAdd(peerId, p =>
-            {
-                _log.Info($"Create new state for peer {p}");
-                return new PeerState(p);
-            });
-
-            peerState.NonAckedMessageCount += delta;
-            peerState.LastNonAckedMessageCountVersion = Interlocked.Increment(ref _version);
-        }
-
-        public List<PeerState> GetUpdatedPeers(ref long version)
+        public Task UpdateNonAckMessageCount(PeerId peerId, int delta)
         {
-            var previousVersion = version;
-            var nextVersion = Interlocked.Increment(ref _version);
-
-            var peers = _statesByPeerId.Values
-                                       .Where(x => x.LastNonAckedMessageCountVersion >= previousVersion)
-                                       .ToList();
-
-            version = nextVersion;
-
-            return peers;
+            var peerState = _statesByPeerId.AddOrUpdate(peerId,
+                                                          p =>
+                                                          {
+                                                              _log.Info($"Created new state for peer {p}");
+                                                              return new PeerState(p, delta);
+                                                          },
+                                                          (id, state) => state.WithNonAckedMessageCountDelta(delta));
+
+            return UpdatePeerState(peerState);
         }
 
         public Task RemovePeer(PeerId peerId)
@@ -91,6 +73,20 @@ namespace Abc.Zebus.Persistence.CQL.Storage
             return removeTask;
         }
 
+        public IEnumerable<PeerState> GetAllKnownPeers()
+        {
+            return _statesByPeerId.Values;
+        }
+
+        public Task UpdateNewOldestMessageTimestamp(PeerState peer, long newOldestMessageTimestamp)
+        {
+            var updatedPeer = _statesByPeerId.AddOrUpdate(peer.PeerId,
+                                                          id => new PeerState(id, 0, newOldestMessageTimestamp),
+                                                          (id, state) => state.WithOldestNonAckedMessageTimestampInTicks(newOldestMessageTimestamp));
+
+            return UpdatePeerState(updatedPeer);
+        }
+
         private Task<RowSet> DeletePeerState(PeerId peerId)
         {
             return _dataContext.PeerStates.Where(x => x.PeerId == peerId.ToString()).Delete().ExecuteAsync();
@@ -106,24 +102,9 @@ namespace Abc.Zebus.Persistence.CQL.Storage
                                .ExecuteAsync();
         }
 
-        public Task Save()
-        {
-            _log.Info("Saving state");
-            var tasks = _statesByPeerId.Values
-                                       .Select(p => _dataContext.PeerStates.Insert(new CassandraPeerState(p)).ExecuteAsync())
-                                       .ToArray();
-
-            return Task.WhenAll(tasks).ContinueWith(t => _log.Info("State saved"));
-        }
-
-        public IEnumerator<PeerState> GetEnumerator()
-        {
-            return _statesByPeerId.Values.GetEnumerator();
-        }
-
-        IEnumerator IEnumerable.GetEnumerator()
+        private Task<RowSet> UpdatePeerState(PeerState p)
         {
-            return GetEnumerator();
+            return _dataContext.PeerStates.Insert(new CassandraPeerState(p)).ExecuteAsync();
         }
     }
 }

src/Abc.Zebus.Persistence.RocksDb.Tests/Abc.Zebus.Persistence.RocksDb.Tests.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net471</TargetFramework>
+    <Version>$(ZebusPersistenceVersion)</Version>
+    <Platform>x64</Platform>
+    <Platforms>AnyCPU;x64</Platforms>
+  </PropertyGroup>
+
+  <Import Project="..\Abc.Zebus.Tests.props" />
+
+  <ItemGroup>
+    <ProjectReference Include="..\Abc.Zebus.Persistence.RocksDb\Abc.Zebus.Persistence.RocksDb.csproj" />
+    <ProjectReference Include="..\Abc.Zebus.Persistence.Messages\Abc.Zebus.Persistence.Messages.csproj" PrivateAssets="All" />
+    <ProjectReference Include="..\Abc.Zebus.Testing\Abc.Zebus.Testing.csproj" />
+  </ItemGroup>
+
+</Project>

src/Abc.Zebus.Persistence.RocksDb.Tests/Abc.Zebus.Persistence.RocksDb.Tests.v3.ncrunchproject

@@ -0,0 +1,9 @@
+<ProjectConfiguration>
+  <Settings>
+    <CopyReferencedAssembliesToWorkspace>True</CopyReferencedAssembliesToWorkspace>
+    <HiddenComponentWarnings>
+      <Value>CopyReferencedAssembliesToWorkspaceIsOn</Value>
+    </HiddenComponentWarnings>
+    <UseCPUArchitecture>x64</UseCPUArchitecture>
+  </Settings>
+</ProjectConfiguration>

src/Abc.Zebus.Persistence.RocksDb.Tests/PerformanceTests.cs

@@ -0,0 +1,160 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Abc.Zebus.Persistence.Matching;
+using Abc.Zebus.Testing;
+using Abc.Zebus.Transport;
+using Abc.Zebus.Util;
+using NUnit.Framework;
+using ProtoBuf;
+
+namespace Abc.Zebus.Persistence.RocksDb.Tests
+{
+    [TestFixture, Explicit]
+    public class PerformanceTests
+    {
+        private RocksDbStorage _storage;
+
+        [SetUp]
+        public void SetUp()
+        {
+            _storage = new RocksDbStorage(Guid.NewGuid().ToString());
+            _storage.Start();
+        }
+
+        [TearDown]
+        public void Teardown()
+        {
+            _storage.Stop();
+        }
+
+        [Test]
+        public async Task should_do_this()
+        {
+            var messageBytes = MessageBytes();
+
+            var startTime = DateTime.UtcNow;
+            var testDuration = 30.Seconds();
+            // Thread 1 - write messages
+            var writeTask = Task.Run(async () =>
+            {
+                var count = 0;
+                while (DateTime.UtcNow - startTime < testDuration)
+                {
+                    var entriesToPersist = GetEntriesToPersist(messageBytes, count);
+                    await _storage.Write(entriesToPersist);
+
+                    // Thread.Sleep(2.Seconds());
+
+                    await _storage.Write(ToAckEntries(entriesToPersist));
+
+                    count++;
+                }
+
+                Console.WriteLine($"Wrote {count * 100:N0} messages");
+            });
+
+            // Thread 2 - ask for all unacked messages
+            /*
+            var updatesTask = Task.Run(() =>
+            {
+                var count = 0;
+                while (DateTime.UtcNow - startTime < testDuration)
+                {
+                    _storage.GetNonAckedMessageCountsForUpdatedPeers();
+                    count++;
+                    // Thread.Sleep(100.Milliseconds());
+                }
+
+                Console.WriteLine($"Got non acked message counts {count:N0} times");
+            });
+            */
+
+            await Task.WhenAll(writeTask/*, updatesTask*/);
+        }
+
+        [Test]
+        public void should_test_replay()
+        {
+            var messageBytes = MessageBytes();
+
+            // Fill with lots of unacked messages
+            var entriesToPersist = new List<MatcherEntry>();
+            var peerId = new PeerId("Peer");
+            for (int i = 0; i < 10_000; i++)
+            {
+                MessageId.PauseIdGenerationAtDate(SystemDateTime.UtcNow.Date.AddSeconds(i * 10));
+                entriesToPersist.Add(MatcherEntry.Message(peerId, MessageId.NextId(), new MessageTypeId("SomeEvent"), messageBytes));
+            }
+
+            _storage.Write(entriesToPersist);
+
+            // Read all unacked messages 
+            var messageReader = _storage.CreateMessageReader(peerId);
+            var startTime = DateTime.UtcNow;
+            var testDuration = 30.Seconds();
+            var count = 0;
+            while (DateTime.UtcNow - startTime < testDuration)
+            {
+                foreach (var transportMessage in messageReader.GetUnackedMessages()) { }
+
+                count++;
+            }
+
+            Console.WriteLine($"Replayed {count:N0} times ({count*entriesToPersist.Count:N0} messages) in {testDuration.TotalSeconds:N0}s");
+        }
+
+        private List<MatcherEntry> ToAckEntries(List<MatcherEntry> entriesToPersist)
+        {
+            return entriesToPersist.Select(x => MatcherEntry.Ack(x.PeerId, x.MessageId)).ToList();
+        }
+
+        private static List<MatcherEntry> GetEntriesToPersist(byte[] messageBytes, int offset, int count = 100)
+        {
+            var entriesToPersist = new List<MatcherEntry>();
+            for (int i = 0; i < count; i++)
+            {
+                MessageId.PauseIdGenerationAtDate(SystemDateTime.UtcNow.Date.AddSeconds(i * 10));
+                entriesToPersist.Add(MatcherEntry.Message(new PeerId("Peer" + (i + offset)), MessageId.NextId(), new MessageTypeId("SomeEvent"), messageBytes));
+            }
+            return entriesToPersist;
+        }
+
+        private static byte[] MessageBytes()
+        {
+            var message = CreateTestTransportMessage(1);
+            var messageBytes = Serialization.Serializer.Serialize(message).ToArray();
+            return messageBytes;
+        }
+
+        private static TransportMessage CreateTestTransportMessage(int i)
+        {
+            MessageId.PauseIdGenerationAtDate(SystemDateTime.UtcNow.Date.AddSeconds(i * 10));
+            return new Message1(i).ToTransportMessage();
+        }
+
+        [ProtoContract]
+        private class Message1 : IEvent
+        {
+            [ProtoMember(1, IsRequired = true)]
+            public int Id { get; private set; }
+
+            [ProtoMember(2, IsRequired = true)]
+            public long Data1 { get; set; }
+
+            [ProtoMember(3, IsRequired = true)]
+            public Guid Data2 { get; set; }
+
+            [ProtoMember(4, IsRequired = true)]
+            public DateTime Data3 { get; set; }
+
+            public Message1(int id)
+            {
+                Id = id;
+            }
+        }
+    }
+}

src/Abc.Zebus.Persistence.RocksDb.Tests/Properties/AssemblyInfo.cs

@@ -0,0 +1,6 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("123c2076-de3c-4f41-ac58-1ac5210f1ccd")]

src/Abc.Zebus.Persistence.RocksDb.Tests/RocksDbStorageTests.cs

@@ -0,0 +1,272 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Abc.Zebus.Persistence.Matching;
+using Abc.Zebus.Persistence.Reporter;
+using Abc.Zebus.Serialization.Protobuf;
+using Abc.Zebus.Testing;
+using Abc.Zebus.Testing.Extensions;
+using Abc.Zebus.Transport;
+using Abc.Zebus.Util;
+using Moq;
+using NUnit.Framework;
+using ProtoBuf;
+
+namespace Abc.Zebus.Persistence.RocksDb.Tests
+{
+    [TestFixture]
+    public class RocksDbStorageTests
+    {
+        private RocksDbStorage _storage;
+        private Mock<IReporter> _reporterMock;
+        private string _databaseDirectoryPath;
+
+        [SetUp]
+        public void SetUp()
+        {
+            _databaseDirectoryPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+
+            _reporterMock = new Mock<IReporter>();
+            _storage = new RocksDbStorage(_databaseDirectoryPath);
+            _storage.Start();
+        }
+
+        [TearDown]
+        public void Teardown()
+        {
+            _storage.Stop();
+            System.IO.Directory.Delete(_databaseDirectoryPath, true);
+        }
+
+        [Test]
+        public async Task should_write_message_entry_fields_to_cassandra()
+        {
+            var inputMessage = CreateTestTransportMessage(1);
+            var messageBytes = Serialization.Serializer.Serialize(inputMessage).ToArray();
+            var messageId = MessageId.NextId();
+
+            var peerId = new PeerId("Abc.Peer.0");
+            await _storage.Write(new List<MatcherEntry> { MatcherEntry.Message(peerId, messageId, MessageTypeId.PersistenceStopping, messageBytes) });
+
+            var messages = _storage.CreateMessageReader(peerId).GetUnackedMessages();
+            var retrievedMessage = messages.Single();
+            retrievedMessage.ShouldEqualDeeply(messageBytes);
+        }
+
+        [Test]
+        public async Task should_not_overwrite_messages_with_same_time_component_and_different_message_id()
+        {
+            var messageBytes = Serialization.Serializer.Serialize(CreateTestTransportMessage(1)).ToArray();
+            var messageId = new MessageId(Guid.Parse("0000c399-1ab0-e511-9706-ae1ea5dcf365"));      // Time component @2016-01-01 00:00:00Z
+            var otherMessageId = new MessageId(Guid.Parse("0000c399-1ab0-e511-9806-f1ef55aac8e9")); // Time component @2016-01-01 00:00:00Z
+
+            var peerId = new PeerId("Abc.Peer.0");
+            await _storage.Write(new List<MatcherEntry>
+            {
+                MatcherEntry.Message(peerId, messageId, MessageTypeId.PersistenceStopping, messageBytes),
+                MatcherEntry.Message(peerId, otherMessageId, MessageTypeId.PersistenceStopping, messageBytes),
+            });
+
+            var messages = _storage.CreateMessageReader(peerId).GetUnackedMessages();
+            messages.ToList().Count.ShouldEqual(2);
+        }
+
+        [Test]
+        public async Task should_support_out_of_order_acks_and_messages()
+        {
+            var inputMessage = CreateTestTransportMessage(1);
+            var messageBytes = Serialization.Serializer.Serialize(inputMessage).ToArray();
+            var messageId = MessageId.NextId();
+
+            var peerId = new PeerId("Abc.Peer.0");
+            await _storage.Write(new List<MatcherEntry> { MatcherEntry.Ack(peerId, messageId) });
+            await Task.Delay(50);
+            await _storage.Write(new List<MatcherEntry> { MatcherEntry.Message(peerId, messageId, MessageTypeId.PersistenceStopping, messageBytes) });
+
+            var messageReader = _storage.CreateMessageReader(peerId);
+            messageReader.ShouldNotBeNull();
+            var messages = messageReader.GetUnackedMessages().ToList();
+            messages.ShouldBeEmpty();
+        }
+
+        [Test]
+        public void should_return_null_when_asked_for_a_message_reader_for_an_unknown_peer_id()
+        {
+            _storage.CreateMessageReader(new PeerId("UnknownPeerId")).ShouldBeNull();
+        }
+
+        [Test]
+        public async Task should_remove_peer()
+        {
+            var inputMessage = CreateTestTransportMessage(1);
+            var messageBytes = Serialization.Serializer.Serialize(inputMessage).ToArray();
+            var messageId = MessageId.NextId();
+
+            var peerId = new PeerId("Abc.Peer.0");
+            await _storage.Write(new List<MatcherEntry> { MatcherEntry.Message(peerId, messageId, MessageTypeId.PersistenceStopping, messageBytes) });
+
+            await _storage.RemovePeer(peerId);
+
+            _storage.CreateMessageReader(peerId).ShouldBeNull();
+            _storage.GetNonAckedMessageCounts().ContainsKey(peerId).ShouldBeFalse();
+        }
+
+        [Test]
+        public void should_update_non_ack_message_count()
+        {
+            var firstPeer = new PeerId("Abc.Testing.Target");
+            var secondPeer = new PeerId("Abc.Testing.OtherTarget");
+
+            _storage.Write(new[] { MatcherEntry.Message(firstPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x01, 0x02, 0x03 }) });
+            _storage.Write(new[] { MatcherEntry.Message(secondPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x04, 0x05, 0x06 }) });
+            _storage.Write(new[] { MatcherEntry.Message(firstPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x07, 0x08, 0x09 }) });
+
+            var nonAckedMessageCounts = _storage.GetNonAckedMessageCounts();
+            nonAckedMessageCounts[firstPeer].ShouldEqual(2);
+            nonAckedMessageCounts[secondPeer].ShouldEqual(1);
+
+            _storage.Write(new[] { MatcherEntry.Ack(firstPeer, MessageId.NextId()) });
+
+            nonAckedMessageCounts = _storage.GetNonAckedMessageCounts();
+            nonAckedMessageCounts[firstPeer].ShouldEqual(1);
+            nonAckedMessageCounts[secondPeer].ShouldEqual(1);
+        }
+
+        [Test]
+        public async Task should_persist_messages_in_order()
+        {
+            var firstPeer = new PeerId("Abc.Testing.Target");
+            var secondPeer = new PeerId("Abc.Testing.OtherTarget");
+
+            using (MessageId.PauseIdGeneration())
+            using (SystemDateTime.PauseTime())
+            {
+                var inputMessages = Enumerable.Range(1, 100).Select(CreateTestTransportMessage).ToList();
+                var messages = inputMessages.SelectMany(x =>
+                                                        {
+                                                            var transportMessageBytes = Serialization.Serializer.Serialize(x).ToArray();
+                                                            return new[]
+                                                            {
+                                                                MatcherEntry.Message(firstPeer, x.Id, x.MessageTypeId, transportMessageBytes),
+                                                                MatcherEntry.Message(secondPeer, x.Id, x.MessageTypeId, transportMessageBytes),
+                                                            };
+                                                        })
+                                                        .ToList();
+
+                await _storage.Write(messages);
+
+                var expectedTransportMessages = inputMessages.Select(Serialization.Serializer.Serialize).Select(x => x.ToArray()).ToList();
+                using (var readerForFirstPeer = _storage.CreateMessageReader(firstPeer))
+                {
+                    var transportMessages = readerForFirstPeer.GetUnackedMessages().ToList();
+                    transportMessages.Count.ShouldEqual(100);
+                    transportMessages.Last().ShouldEqualDeeply(expectedTransportMessages.Last());
+                }
+
+                using (var readerForSecondPeer = _storage.CreateMessageReader(secondPeer))
+                {
+                    var transportMessages = readerForSecondPeer.GetUnackedMessages().ToList();
+                    transportMessages.Count.ShouldEqual(100);
+                    transportMessages.Last().ShouldEqualDeeply(expectedTransportMessages.Last());
+                }
+            }
+        }
+
+        [Test]
+        public async Task should_not_get_acked_message()
+        {
+            var peer = new PeerId("Abc.Testing.Target");
+
+            var message1 = GetMatcherEntryWithValidTransportMessage(peer, 1);
+            var message2 = GetMatcherEntryWithValidTransportMessage(peer, 2);
+
+            await _storage.Write(new[] { message1 });
+            await _storage.Write(new[] { message2 });
+            await _storage.Write(new[] { MatcherEntry.Ack(peer, message2.MessageId) });
+
+            using (var reader = _storage.CreateMessageReader(peer))
+            {
+                reader.GetUnackedMessages()
+                      .Select(TransportMessageDeserializer.Deserialize)
+                      .Select(x => x.Id)
+                      .ToList()
+                      .ShouldBeEquivalentTo(message1.MessageId);
+            }
+        }
+
+        private static MatcherEntry GetMatcherEntryWithValidTransportMessage(PeerId peer, int i)
+        {
+            var inputMessage = CreateTestTransportMessage(i);
+            var messageBytes = Serialization.Serializer.Serialize(inputMessage).ToArray();
+            var message1 = MatcherEntry.Message(peer, inputMessage.Id, MessageUtil.TypeId<Message1>(), messageBytes);
+            return message1;
+        }
+
+        [Test]
+        public async Task should_load_previous_out_of_order_acks()
+        {
+            var peer = new PeerId("Abc.Testing.Target");
+
+            var messageId = MessageId.NextId();
+            await _storage.Write(new[] { MatcherEntry.Ack(peer, messageId) });
+            _storage.Stop();
+
+            _storage = new RocksDbStorage(_databaseDirectoryPath);
+            _storage.Start();
+
+            var message = MatcherEntry.Message(peer, messageId, MessageUtil.TypeId<Message1>(), Array.Empty<byte>());
+            await _storage.Write(new[] { message });
+
+            using (var messageReader = _storage.CreateMessageReader(peer))
+            {
+                messageReader.GetUnackedMessages()
+                             .Count()
+                             .ShouldEqual(0);
+            }
+        }
+
+        [Test, Explicit]
+        public void should_report_storage_informations()
+        {
+            var peer = new PeerId("peer");
+
+            _storage.Write(new[]
+            {
+                MatcherEntry.Message(peer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x01, 0x02, 0x03 }),
+                MatcherEntry.Message(peer, MessageId.NextId(), new MessageTypeId("Abc.Message.Fat"), new byte[] { 0x01, 0x02, 0x03, 0x04 }),
+            });
+
+            _reporterMock.Verify(r => r.AddStorageReport(2, 7, 4, "Abc.Message.Fat"));
+        }
+
+        private static TransportMessage CreateTestTransportMessage(int i)
+        {
+            MessageId.PauseIdGenerationAtDate(SystemDateTime.UtcNow.Date.AddSeconds(i * 10));
+            return new Message1(i).ToTransportMessage();
+        }
+
+        [ProtoContract]
+        private class Message1 : IEvent
+        {
+            [ProtoMember(1, IsRequired = true)]
+            public int Id { get; private set; }
+
+            public Message1(int id)
+            {
+                Id = id;
+            }
+        }
+
+        private static class TransportMessageDeserializer
+        {
+            public static TransportMessage Deserialize(byte[] bytes)
+            {
+                var inputStream = new CodedInputStream(bytes, 0, bytes.Length);
+                var readTransportMessage = inputStream.ReadTransportMessage();
+                return readTransportMessage;
+            }
+        }
+    }
+}

src/Abc.Zebus.Persistence.RocksDb/Abc.Zebus.Persistence.RocksDb.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="RocksDbNative" Version="5.4.6.10" />
+    <PackageReference Include="RocksDbSharp" Version="5.4.6.11" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Abc.Zebus.Contracts\Abc.Zebus.Contracts.csproj" />
+    <ProjectReference Include="..\Abc.Zebus.Persistence\Abc.Zebus.Persistence.csproj" />
+    <ProjectReference Include="..\Abc.Zebus\Abc.Zebus.csproj" />
+  </ItemGroup>
+
+</Project>

src/Abc.Zebus.Persistence.RocksDb/Abc.Zebus.Persistence.RocksDb.v3.ncrunchproject

@@ -0,0 +1,7 @@
+<ProjectConfiguration>
+  <Settings>
+    <HiddenComponentWarnings>
+      <Value>NetCoreNetStandardLocalSystem</Value>
+    </HiddenComponentWarnings>
+  </Settings>
+</ProjectConfiguration>

src/Abc.Zebus.Persistence.RocksDb/RocksDbMessageReader.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Abc.Zebus.Persistence.Storage;
+using RocksDbSharp;
+
+namespace Abc.Zebus.Persistence.RocksDb
+{
+    public class RocksDbMessageReader : IMessageReader
+    {
+        private readonly RocksDbSharp.RocksDb _db;
+        private readonly PeerId _peerId;
+        private readonly ColumnFamilyHandle _messagesColumnFamily;
+        private Iterator _iterator;
+
+        public RocksDbMessageReader(RocksDbSharp.RocksDb db, in PeerId peerId, ColumnFamilyHandle messagesColumnFamily)
+        {
+            _db = db;
+            _peerId = peerId;
+            _messagesColumnFamily = messagesColumnFamily;
+        }
+
+        public IEnumerable<byte[]> GetUnackedMessages()
+        {
+            var key = RocksDbStorage.CreateKeyBuffer(_peerId);
+            RocksDbStorage.FillKey(key, _peerId, 0, Guid.Empty);
+
+            _iterator?.Dispose();
+            _iterator = _db.NewIterator(_messagesColumnFamily);
+            if (!_iterator.Seek(key).Valid())
+                return Enumerable.Empty<byte[]>();
+
+            return TransportMessages(_iterator, key, _peerId);
+        }
+
+        private static IEnumerable<byte[]> TransportMessages(Iterator iterator, byte[] key, PeerId peerId)
+        {
+            var found = true;
+            var peerPartLength = GetPeerPartLength(peerId);
+            while (found)
+            {
+                var currentKey = iterator.Key();
+                if (!RocksDbStorage.CompareStart(currentKey, key, peerPartLength))
+                    break;
+
+                yield return iterator.Value();
+
+                found = iterator.Next().Valid();
+            }
+        }
+
+        private static int GetPeerPartLength(PeerId peer) => Encoding.UTF8.GetByteCount(peer.ToString());
+
+        public void Dispose()
+        {
+            _iterator?.Dispose();
+        }
+    }
+}

src/Abc.Zebus.Persistence.RocksDb/RocksDbStorage.cs

@@ -0,0 +1,244 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Abc.Zebus.Persistence.Matching;
+using Abc.Zebus.Persistence.Storage;
+using RocksDbSharp;
+using StructureMap;
+
+namespace Abc.Zebus.Persistence.RocksDb
+{
+    /// <summary>
+    /// Key structure:
+    /// -------------------------------------------------------------------
+    /// |  PeerId (n bytes)  |  Ticks (8 bytes)  |  MessageId (16 bytes)  | 
+    /// -------------------------------------------------------------------
+    /// </summary>
+    public class RocksDbStorage : IStorage, IDisposable
+    {
+        private static readonly int _guidLength = Guid.Empty.ToByteArray().Length;
+
+        private readonly ConcurrentDictionary<MessageId, bool> _outOfOrderAcks = new ConcurrentDictionary<MessageId, bool>();
+
+        private RocksDbSharp.RocksDb _db;
+        private readonly string _databaseDirectoryPath;
+        private ColumnFamilyHandle _messagesColumnFamily;
+        private ColumnFamilyHandle _peersColumnFamily;
+        private ColumnFamilyHandle _acksColumnFamily;
+
+        [DefaultConstructor]
+        public RocksDbStorage()
+            : this(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "database"))
+        {
+        }
+
+        public RocksDbStorage(string databaseDirectoryPath)
+        {
+            _databaseDirectoryPath = databaseDirectoryPath;
+        }
+
+        public int PersistenceQueueSize { get; } = 0;
+
+        public void Start()
+        {
+            var options = new DbOptions().SetCreateIfMissing()
+                                         .SetCreateMissingColumnFamilies()
+                                         .SetMaxBackgroundCompactions(4)
+                                         .SetMaxBackgroundFlushes(2)
+                                         .SetBytesPerSync(1024 * 1024);
+
+            var columnFamilies = new ColumnFamilies
+            {
+                { "Messages", ColumnFamilyOptions() },
+                { "Peers", ColumnFamilyOptions() },
+                { "Acks", ColumnFamilyOptions() }
+            };
+
+            _db = RocksDbSharp.RocksDb.Open(options, _databaseDirectoryPath, columnFamilies);
+
+            _messagesColumnFamily = _db.GetColumnFamily("Messages");
+            _peersColumnFamily = _db.GetColumnFamily("Peers");
+            _acksColumnFamily = _db.GetColumnFamily("Acks");
+
+            ColumnFamilyOptions ColumnFamilyOptions() => new ColumnFamilyOptions().SetCompression(CompressionTypeEnum.rocksdb_no_compression)
+                                                                                  .SetLevelCompactionDynamicLevelBytes(true)
+                                                                                  .SetArenaBlockSize(16 * 1024);
+
+            LoadAllOutOfOrderAcks();
+        }
+
+        public void Stop() => Dispose();
+
+        public void Dispose() => _db?.Dispose();
+
+        public Task Write(IList<MatcherEntry> entriesToPersist)
+        {
+            foreach (var entry in entriesToPersist)
+            {
+                var key = CreateKeyBuffer(entry.PeerId);
+                FillKey(key, entry.PeerId, entry.MessageId.GetDateTime().Ticks, entry.MessageId.Value);
+                if (entry.IsAck)
+                {
+                    // Ack
+                    var bytes = _db.Get(key, _messagesColumnFamily);
+                    if (bytes != null)
+                    {
+                        // Acked message
+                        _db.Remove(key, _messagesColumnFamily);
+                    }
+                    else
+                    {
+                        // Ack before message
+                        _outOfOrderAcks.TryAdd(entry.MessageId, default);
+                        _db.Put(key, Array.Empty<byte>(), _acksColumnFamily);
+                    }
+                }
+                else
+                {
+                    // Message 
+                    if (!_outOfOrderAcks.TryRemove(entry.MessageId, out _))
+                        // Message before ack
+                        _db.Put(key, entry.MessageBytes, _messagesColumnFamily);
+                    else
+                        // Otherwise ignore the message and remove the ack as it has already been acked
+                        _db.Remove(key, _acksColumnFamily);
+                }
+            }
+
+            foreach (var entry in entriesToPersist.GroupBy(x => x.PeerId))
+            {
+                UpdateNonAckedCounts(entry);
+            }
+
+            return Task.CompletedTask;
+        }
+
+        private void UpdateNonAckedCounts(IGrouping<PeerId, MatcherEntry> entry)
+        {
+            var nonAcked = entry.Aggregate(0, (s, e) => s + (e.IsAck ? -1 : 1));
+            var peerKey = GetPeerKey(entry.Key);
+            using (var iterator = _db.NewIterator(_peersColumnFamily))//, new ReadOptions().SetTotalOrderSeek(true)))
+            {
+                // TODO: figure out why Seek() returns true for a different key
+                var alreadyExists = iterator.Seek(peerKey).Valid() && CompareStart(iterator.Key(), peerKey, peerKey.Length);
+                var currentNonAcked = alreadyExists ? BitConverter.ToInt32(iterator.Value(), 0) : 0;
+
+                var value = currentNonAcked + nonAcked;
+                _db.Put(peerKey, BitConverter.GetBytes(value), _peersColumnFamily);
+            }
+        }
+
+        public IMessageReader CreateMessageReader(PeerId peerId)
+        {
+            using (var iterator = _db.NewIterator(_peersColumnFamily))
+            {
+                if (!iterator.Seek(GetPeerKey(peerId)).Valid())
+                    return null;
+            }
+
+            return new RocksDbMessageReader(_db, peerId, _messagesColumnFamily);
+        }
+
+        public Task RemovePeer(PeerId peerId)
+        {
+            var key = CreateKeyBuffer(peerId);
+            FillKey(key, peerId, 0, Guid.Empty);
+
+            using (var cursor = _db.NewIterator(_messagesColumnFamily))
+            {
+                if (!cursor.Seek(key).Valid())
+                    return Task.CompletedTask;
+
+                var peerIdLength = peerId.ToString().Length;
+                byte[] currentKey;
+                do
+                {
+                    currentKey = cursor.Key();
+                    _db.Remove(currentKey);
+                    cursor.Next();
+                } while (cursor.Valid() && CompareStart(currentKey, key, peerIdLength));
+            }
+
+            _db.Remove(GetPeerKey(peerId), _peersColumnFamily);
+
+            // TODO: remove out of order acks
+            return Task.CompletedTask;
+        }
+
+        public Dictionary<PeerId, int> GetNonAckedMessageCounts()
+        {
+            var nonAckedCounts = new Dictionary<PeerId, int>();
+            using (var cursor = _db.NewIterator(_peersColumnFamily))
+            {
+                for (cursor.SeekToFirst(); cursor.Valid(); cursor.Next())
+                {
+                    var peerId = ReadPeerKey(cursor.Key());
+                    nonAckedCounts[peerId] = BitConverter.ToInt32(cursor.Value(), 0);
+                }
+            }
+
+            return nonAckedCounts;
+        }
+
+        public static void FillKey(byte[] key, PeerId peerId, long ticks, Guid messageId)
+        {
+            var peerPart = Encoding.UTF8.GetBytes(peerId.ToString());
+            Buffer.BlockCopy(peerPart, 0, key, 0, peerPart.Length);
+
+            var tickPart = BitConverter.GetBytes(ticks);
+            if (BitConverter.IsLittleEndian)
+                Array.Reverse(tickPart); // change endianness so sorting will be correct
+
+            Buffer.BlockCopy(tickPart, 0, key, peerPart.Length, sizeof(long));
+
+            var messageIdPart = messageId.ToByteArray();
+            Buffer.BlockCopy(messageIdPart, 0, key, peerPart.Length + sizeof(long), _guidLength);
+        }
+
+        private void LoadAllOutOfOrderAcks()
+        {
+            using (var newIterator = _db.NewIterator(_acksColumnFamily))
+            {
+                newIterator.SeekToFirst();
+                while (newIterator.Valid())
+                {
+                    var keyBytes = newIterator.Key();
+                    var messageId = ReadMessageIdFromKey(keyBytes);
+                    _outOfOrderAcks.TryAdd(new MessageId(messageId), default);
+
+                    newIterator.Next();
+                }
+            }
+        }
+
+        private static Guid ReadMessageIdFromKey(byte[] keyBytes)
+        {
+            var messageIdBytes = new byte[_guidLength];
+            Buffer.BlockCopy(keyBytes, keyBytes.Length - _guidLength, messageIdBytes, 0, _guidLength);
+            var messageId = new Guid(messageIdBytes);
+            return messageId;
+        }
+
+        private static PeerId ReadPeerKey(byte[] keyBytes) => new PeerId(Encoding.UTF8.GetString(keyBytes));
+        private static byte[] GetPeerKey(PeerId peerId) => Encoding.UTF8.GetBytes(peerId.ToString());
+        public static byte[] CreateKeyBuffer(PeerId entryPeerId) => new byte[Encoding.UTF8.GetByteCount(entryPeerId.ToString()) + sizeof(long) + _guidLength];
+
+        public static bool CompareStart(byte[] x, byte[] y, int length)
+        {
+            if (x.Length < length || y.Length < length)
+                return false;
+
+            for (var index = length - 1; index >= 0; index--)
+            {
+                if (x[index] != y[index])
+                    return false;
+            }
+
+            return true;
+        }
+    }
+}

src/Abc.Zebus.Persistence.Runner/Abc.Zebus.Persistence.Runner.csproj

@@ -4,14 +4,24 @@
     <TargetFramework>net471</TargetFramework>
     <Version>$(ZebusPersistenceVersion)</Version>
     <IsPackable>false</IsPackable>
+    <ApplicationIcon />
+    <OutputType>Exe</OutputType>
+    <StartupObject />
   </PropertyGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Abc.Zebus.Persistence.CQL\Abc.Zebus.Persistence.CQL.csproj" />
+    <ProjectReference Include="..\Abc.Zebus.Persistence.RocksDb\Abc.Zebus.Persistence.RocksDb.csproj" />
   </ItemGroup>
 
   <ItemGroup>
     <PackageReference Include="FluentDateTime" Version="1.14.0" />
   </ItemGroup>
 
+  <ItemGroup>
+    <None Update="log4net.config">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 </Project>

src/Abc.Zebus.Persistence.Runner/App.config

@@ -1,8 +1,10 @@
 <?xml version="1.0" encoding="utf-8" ?>
 <configuration>
   <appSettings>
-    <add key="Endpoint" value="tcp://*:129" />
+    <add key="Bus.Directory.EndPoints" value="tcp://localhost:129"/>
+    <add key="Endpoint" value="tcp://*:130" />
     <add key="Environment" value="Demo" />
     <add key="PeerId" value="Persistence.0" />
+    <add key="PersistenceStorage" value="RocksDb"/>
   </appSettings>
-</configuration>
+</configuration>

src/Abc.Zebus.Persistence.Runner/Program.cs

@@ -13,6 +13,7 @@ using Abc.Zebus.Persistence.CQL.Util;
 using Abc.Zebus.Persistence.Initialization;
 using Abc.Zebus.Persistence.Matching;
 using Abc.Zebus.Persistence.Reporter;
+using Abc.Zebus.Persistence.RocksDb;
 using Abc.Zebus.Persistence.Storage;
 using Abc.Zebus.Persistence.Transport;
 using Abc.Zebus.Transport;
@@ -38,15 +39,14 @@ namespace Abc.Zebus.Persistence.Runner
             XmlConfigurator.ConfigureAndWatch(new FileInfo(InBaseDirectory("log4net.config")));
             _log.Info("Starting persistence");
 
-            var busFactory = new BusFactory();
             var appSettingsConfiguration = new AppSettingsConfiguration();
-            InjectPersistenceServiceSpecificConfiguration(busFactory, appSettingsConfiguration);
-
-            busFactory
-                .WithConfiguration(appSettingsConfiguration, ConfigurationManager.AppSettings["Environment"])
-                .WithScan()
-                .WithEndpoint(ConfigurationManager.AppSettings["Endpoint"])
-                .WithPeerId(ConfigurationManager.AppSettings["PeerId"]);
+            var useCassandraStorage = ConfigurationManager.AppSettings["PersistenceStorage"] == "Cassandra";
+            var busFactory = new BusFactory().WithConfiguration(appSettingsConfiguration, ConfigurationManager.AppSettings["Environment"])
+                                             .WithScan()
+                                             .WithEndpoint(ConfigurationManager.AppSettings["Endpoint"])
+                                             .WithPeerId(ConfigurationManager.AppSettings["PeerId"]);
+            
+            InjectPersistenceServiceSpecificConfiguration(busFactory, appSettingsConfiguration, useCassandraStorage);
 
             using (busFactory.CreateAndStartBus())
             {
@@ -54,16 +54,20 @@ namespace Abc.Zebus.Persistence.Runner
                 var inMemoryMessageMatcherInitializer = busFactory.Container.GetInstance<InMemoryMessageMatcherInitializer>();
                 inMemoryMessageMatcherInitializer.BeforeStart();
 
-                var oldestNonAckedMessageUpdaterPeriodicAction = busFactory.Container.GetInstance<OldestNonAckedMessageUpdaterPeriodicAction>();
-                oldestNonAckedMessageUpdaterPeriodicAction.AfterStart();
+                OldestNonAckedMessageUpdaterPeriodicAction oldestNonAckedMessageUpdaterPeriodicAction = null;
+                if (useCassandraStorage)
+                {
+                    oldestNonAckedMessageUpdaterPeriodicAction = busFactory.Container.GetInstance<OldestNonAckedMessageUpdaterPeriodicAction>();
+                    oldestNonAckedMessageUpdaterPeriodicAction.AfterStart();
+                }
 
                 _log.Info("Persistence started");
-                
+
                 _cancelKeySignal.WaitOne();
 
                 _log.Info("Stopping initialisers");
-                oldestNonAckedMessageUpdaterPeriodicAction.BeforeStop();
-                
+                oldestNonAckedMessageUpdaterPeriodicAction?.BeforeStop();
+
                 var messageReplayerInitializer = busFactory.Container.GetInstance<MessageReplayerInitializer>();
                 messageReplayerInitializer.BeforeStop();
 
@@ -78,18 +82,17 @@ namespace Abc.Zebus.Persistence.Runner
             return Path.Combine(AppDomain.CurrentDomain.BaseDirectory, path);
         }
 
-        private static void InjectPersistenceServiceSpecificConfiguration(BusFactory busFactory, AppSettingsConfiguration configuration)
+        private static void InjectPersistenceServiceSpecificConfiguration(BusFactory busFactory, AppSettingsConfiguration configuration, bool useCassandraStorage)
         {
             busFactory.ConfigureContainer(c =>
             {
                 c.ForSingletonOf<IPersistenceConfiguration>().Use(configuration);
 
-                // TODO: Add InMemoryStorage
-                c.ForSingletonOf<IStorage>().Use<CqlStorage>();
+                c.ForSingletonOf<IStorage>().Use(ctx => useCassandraStorage ? (IStorage)ctx.GetInstance<CqlStorage>() : ctx.GetInstance<RocksDbStorage>());
 
                 c.ForSingletonOf<IMessageReplayerRepository>().Use<MessageReplayerRepository>();
                 c.ForSingletonOf<IMessageReplayer>().Use<MessageReplayer>();
-                
+
                 c.ForSingletonOf<IMessageDispatcher>().Use(typeof(Func<IContext, MessageDispatcher>).Name,
                                                            ctx =>
                                                            {
@@ -107,10 +110,12 @@ namespace Abc.Zebus.Persistence.Runner
                 c.ForSingletonOf<IReporter>().Use<NoopReporter>();
 
                 // Cassandra specific
-                c.ForSingletonOf<PeerStateRepository>().Use<PeerStateRepository>();
-                c.ForSingletonOf<CassandraCqlSessionManager>().Use(() => CassandraCqlSessionManager.Create());
-                c.Forward<PeerStateRepository, IPeerStateRepository>();
-                c.ForSingletonOf<ICqlPersistenceConfiguration>().Use<CassandraAppSettingsConfiguration>();
+                if (useCassandraStorage)
+                {
+                    c.ForSingletonOf<ICqlStorage>().Use<CqlStorage>();
+                    c.ForSingletonOf<CassandraCqlSessionManager>().Use(() => CassandraCqlSessionManager.Create());
+                    c.ForSingletonOf<ICqlPersistenceConfiguration>().Use<CassandraAppSettingsConfiguration>();
+                }
             });
         }
     }

src/Abc.Zebus.Directory/log4net.config → src/Abc.Zebus.Persistence.Runner/log4net.config

@@ -1,11 +1,11 @@
 <?xml version="1.0" encoding="utf-8"?>
 <log4net>
-	<root>
-		<level value="INFO" />
+  <root>
+    <level value="INFO" />
     <appender type="log4net.Appender.ColoredConsoleAppender">
       <layout type="log4net.Layout.PatternLayout">
         <conversionPattern value="%date{HH:mm:ss.fff} - %-5level - %logger || %message%newline" />
       </layout>
     </appender>
-	</root>
-</log4net>
+  </root>
+</log4net>

src/Abc.Zebus.Persistence.Tests/Handlers/PublishNonAckMessagesCountCommandHandlerTests.cs

@@ -26,12 +26,32 @@ namespace Abc.Zebus.Persistence.Tests.Handlers
         [Test]
         public void should_publish_messages_count()
         {
-            _storage.Setup(x => x.GetNonAckedMessageCountsForUpdatedPeers())
+            _storage.Setup(x => x.GetNonAckedMessageCounts())
                     .Returns(new Dictionary<PeerId, int> { { new PeerId("Abc.Peer.0"), 42 } });
 
             _handler.Handle(new PublishNonAckMessagesCountCommand());
 
             _bus.ExpectExactly(new NonAckMessagesCountChanged(new[] { new NonAckMessage("Abc.Peer.0", 42) }));
         }
+
+        [Test]
+        public void should_publish_messages_for_updated_peers()
+        {
+            _storage.Setup(x => x.GetNonAckedMessageCounts())
+                    .Returns(new Dictionary<PeerId, int>());
+            _handler.Handle(new PublishNonAckMessagesCountCommand());
+
+            _storage.Setup(x => x.GetNonAckedMessageCounts())
+                    .Returns(new Dictionary<PeerId, int> { { new PeerId("Abc.Peer.0"), 42 } }); 
+            _handler.Handle(new PublishNonAckMessagesCountCommand());
+
+            _storage.Setup(x => x.GetNonAckedMessageCounts())
+                    .Returns(new Dictionary<PeerId, int> { { new PeerId("Abc.Peer.0"), 43 } }); 
+            _handler.Handle(new PublishNonAckMessagesCountCommand());
+
+            _bus.ExpectExactly(new NonAckMessagesCountChanged(new NonAckMessage[0]),
+                               new NonAckMessagesCountChanged(new[] { new NonAckMessage("Abc.Peer.0", 42) }),
+                               new NonAckMessagesCountChanged(new[] { new NonAckMessage("Abc.Peer.0", 43) }));
+        }
     }
 }

src/Abc.Zebus.Persistence.Tests/MessageReplayerTests.cs

@@ -32,12 +32,14 @@ namespace Abc.Zebus.Persistence.Tests
         private Peer _targetPeer;
         private Peer _anotherPeer;
         private Guid _replayId;
-        private List<TransportMessage> _insertedMessages;
+        private List<byte[]> _insertedMessages;
         private Mock<IStorage> _storageMock;
+        private TransportMessageSerializer _transportMessageSerializer;
 
         [SetUp]
         public void Setup()
         {
+            _transportMessageSerializer = new TransportMessageSerializer();
             _configurationMock = new Mock<IPersistenceConfiguration>();
             _configurationMock.Setup(conf => conf.SafetyPhaseDuration).Returns(500.Milliseconds());
             _configurationMock.SetupGet(x => x.ReplayBatchSize).Returns(_replayBatchSize);
@@ -53,7 +55,7 @@ namespace Abc.Zebus.Persistence.Tests
 
             _replayId = Guid.NewGuid();
 
-            _insertedMessages = new List<TransportMessage>();
+            _insertedMessages = new List<byte[]>();
             var readerMock = new Mock<IMessageReader>();
             _storageMock = new Mock<IStorage>();
             _storageMock.Setup(x => x.CreateMessageReader(It.IsAny<PeerId>())).Returns(readerMock.Object);
@@ -109,11 +111,11 @@ namespace Abc.Zebus.Persistence.Tests
         public void should_not_replay_messages_if_peer_does_not_exist()
         {
             _storageMock.Setup(x => x.CreateMessageReader(It.IsAny<PeerId>())).Returns((IMessageReader)null);
-            
+
             using (MessageId.PauseIdGeneration())
             {
                 _replayer.Run(new CancellationToken());
-                
+
                 _bus.Expect(new ReplaySessionStarted(_targetPeer.Id, _replayId));
             }
         }
@@ -221,7 +223,7 @@ namespace Abc.Zebus.Persistence.Tests
             _replayer.UnackedMessageCountThatReleasesNextBatch = 1;
 
             var unackedTransportMessages = InsertMessagesInThePast(DateTime.Now, messageCount: 10);
-            
+
             using (MessageId.PauseIdGeneration())
             {
                 _replayer.Start();
@@ -237,7 +239,7 @@ namespace Abc.Zebus.Persistence.Tests
 
                     Thread.Sleep(10);
                 }
-                
+
                 unackedTransportMessages = unackedTransportMessages.OrderBy(msg => msg.Id.GetDateTime()).ToList();
                 messageIndex = 0;
                 foreach (var unackedTransportMessage in unackedTransportMessages)
@@ -267,7 +269,7 @@ namespace Abc.Zebus.Persistence.Tests
 
             var refTime = refDateTime.AddHours(-messageCount);
             var transportMessages = new List<TransportMessage>();
-            
+
             for (var i = 0; i < messageCount; ++i)
             {
                 TransportMessage transportMessage;
@@ -276,7 +278,8 @@ namespace Abc.Zebus.Persistence.Tests
                     transportMessage = new FakeCommand(i).ToTransportMessage(_anotherPeer);
                 }
                 transportMessages.Add(transportMessage);
-                _insertedMessages.AddRange(transportMessage);
+
+                _insertedMessages.AddRange(_transportMessageSerializer.Serialize(transportMessage));
                 refTime = refTime.AddHours(1);
             }
 
@@ -290,8 +293,8 @@ namespace Abc.Zebus.Persistence.Tests
             for (var i = 0; i < messageCount; ++i)
             {
                 var transportMessage = new FakeCommand(i).ToTransportMessage(_anotherPeer);
-                _insertedMessages.Add(transportMessage);
+                _insertedMessages.Add(_transportMessageSerializer.Serialize(transportMessage));
             }
         }
     }
-}
+}

src/Abc.Zebus.Persistence/Handlers/PublishNonAckMessagesCountCommandHandler.cs

@@ -8,6 +8,7 @@ namespace Abc.Zebus.Persistence.Handlers
     {
         private readonly IStorage _storage;
         private readonly IBus _bus;
+        private readonly NonAckedCountCache _nonAckedCountCache = new NonAckedCountCache();
 
         public PublishNonAckMessagesCountCommandHandler(IStorage storage, IBus bus)
         {
@@ -17,9 +18,10 @@ namespace Abc.Zebus.Persistence.Handlers
 
         public void Handle(PublishNonAckMessagesCountCommand message)
         {
-            var messagesCount = _storage.GetNonAckedMessageCountsForUpdatedPeers()
-                                        .Select(x => new NonAckMessage(x.Key.ToString(), x.Value))
-                                        .ToArray();
+            var allNonAckedCounts = _storage.GetNonAckedMessageCounts();
+            var updatedNonAckedCounts = _nonAckedCountCache.GetForUpdatedPeers(allNonAckedCounts.Select(x => (x.Key, x.Value)).ToList());
+            var messagesCount = updatedNonAckedCounts.Select(x => new NonAckMessage(x.PeerId.ToString(), x.Count))
+                                                     .ToArray();
 
             _bus.Publish(new NonAckMessagesCountChanged(messagesCount));
         }

src/Abc.Zebus.Persistence/Handlers/PurgeMessageQueueCommandHandler.cs

@@ -1,5 +1,6 @@
 using Abc.Zebus.Persistence.Messages;
 using Abc.Zebus.Persistence.Storage;
+using Abc.Zebus.Util;
 
 namespace Abc.Zebus.Persistence.Handlers
 {
@@ -17,7 +18,7 @@ namespace Abc.Zebus.Persistence.Handlers
         public void Handle(PurgeMessageQueueCommand message)
         {
             var peerId = new PeerId(message.InstanceName);
-            _storage.RemovePeer(peerId);
+            _storage.RemovePeer(peerId).Wait(10.Seconds());
 
             _bus.Publish(new NonAckMessagesCountChanged(new[] { new NonAckMessage(peerId.ToString(), 0) }));
         }

src/Abc.Zebus.Persistence/MessageReplayer.cs

@@ -35,7 +35,7 @@ namespace Abc.Zebus.Persistence
         private readonly int _replayBatchSize;
         private readonly SendContext _emptySendContext = new SendContext();
 
-        public MessageReplayer(IPersistenceConfiguration persistenceConfiguration, IStorage storage,  IBus bus, ITransport transport,
+        public MessageReplayer(IPersistenceConfiguration persistenceConfiguration, IStorage storage, IBus bus, ITransport transport,
                                IInMemoryMessageMatcher inMemoryMessageMatcher, Peer peer, Guid replayId, IReporter reporter)
         {
             _persistenceConfiguration = persistenceConfiguration;
@@ -120,7 +120,7 @@ namespace Abc.Zebus.Persistence
             var replayDuration = MeasureDuration();
             var totalReplayedCount = ReplayUnackedMessages(cancellationToken);
             _logger.Info($"Replay phase ended for {_peer.Id}. {totalReplayedCount} messages replayed in {replayDuration.Value} ({totalReplayedCount / replayDuration.Value.TotalSeconds} msg/s)");
-            
+
             if (cancellationToken.IsCancellationRequested)
                 return;
 
@@ -143,13 +143,13 @@ namespace Abc.Zebus.Persistence
                 if (reader == null)
                     return 0;
                 var totalMessageCount = 0;
-                
+
                 foreach (var partition in reader.GetUnackedMessages().TakeWhile(m => !cancellationToken.IsCancellationRequested).Partition(_replayBatchSize, true))
                 {
                     var messageSentCount = 0;
                     var batchDuration = MeasureDuration();
                     var readAndSendDuration = MeasureDuration();
-                    foreach (var message in partition)
+                    foreach (var message in partition.Select(DeserializeTransportMessage))
                     {
                         _unackedIds.Add(message.Id);
                         ReplayMessage(message);
@@ -169,6 +169,8 @@ namespace Abc.Zebus.Persistence
             }
         }
 
+        private static TransportMessage DeserializeTransportMessage(byte[] row) => TransportMessageDeserializer.Deserialize(row);
+
         private void WaitForAcks(CancellationToken cancellationToken)
         {
             if (_unackedIds.Count <= UnackedMessageCountThatReleasesNextBatch)
@@ -214,7 +216,7 @@ namespace Abc.Zebus.Persistence
 
         private TransportMessage ToTransportMessage(IMessage message, bool wasPersisted = false)
         {
-            return new TransportMessage(message.TypeId(), Serializer.Serialize(message), _self) {  WasPersisted = wasPersisted };
+            return new TransportMessage(message.TypeId(), Serializer.Serialize(message), _self) { WasPersisted = wasPersisted };
         }
 
         public void Handle(MessageHandled messageHandled)

src/Abc.Zebus.Persistence/Storage/IMessageReader.cs

@@ -6,6 +6,6 @@ namespace Abc.Zebus.Persistence.Storage
 {
     public interface IMessageReader : IDisposable
     {
-        IEnumerable<TransportMessage> GetUnackedMessages();
+        IEnumerable<byte[]> GetUnackedMessages();
     }
-}
+}

src/Abc.Zebus.Persistence/Storage/IStorage.cs

@@ -29,9 +29,13 @@ namespace Abc.Zebus.Persistence.Storage
         /// <summary>
         /// Remove the specified peer.
         /// </summary>
-        void RemovePeer(PeerId peerId);
+        Task RemovePeer(PeerId peerId);
 
-        Dictionary<PeerId, int> GetNonAckedMessageCountsForUpdatedPeers();
+        /// <summary>
+        /// Returns the count of all non-acked messages for peers updated since the last call to this method
+        /// </summary>
+        /// <returns></returns>
+        Dictionary<PeerId, int> GetNonAckedMessageCounts();
 
         void Start();
 

src/Abc.Zebus.Persistence/Storage/NonAckedCountCache.cs

@@ -0,0 +1,38 @@
+using System.Collections.Generic;
+using System.Linq;
+using Abc.Zebus.Util.Extensions;
+
+namespace Abc.Zebus.Persistence.Storage
+{
+    public class NonAckedCountCache
+    {
+        private readonly Dictionary<PeerId, NonAckedCount> _nonAckedCounts = new Dictionary<PeerId, NonAckedCount>();
+
+        public IEnumerable<NonAckedCount> GetForUpdatedPeers(ICollection<(PeerId, int)> allPeerStates)
+        {
+            var updatedPeers = (from peerState in allPeerStates
+                                let count = _nonAckedCounts.GetValueOrDefault(peerState.Item1, id => new NonAckedCount(id, -42))
+                                where count.Count != peerState.Item2
+                                select new NonAckedCount(peerState.Item1, peerState.Item2)).ToList();
+
+            foreach (var peerState in allPeerStates)
+            {
+                _nonAckedCounts[peerState.Item1] = new NonAckedCount(peerState.Item1, peerState.Item2);
+            }
+
+            return updatedPeers;
+        }
+    }
+
+    public readonly struct NonAckedCount
+    {
+        public readonly PeerId PeerId;
+        public readonly int Count;
+
+        public NonAckedCount(PeerId peerId, int count)
+        {
+            PeerId = peerId;
+            Count = count;
+        } 
+    }
+}

src/Abc.Zebus.Testing/Abc.Zebus.Testing.csproj

@@ -15,7 +15,7 @@
     <PackageReference Include="CompareNETObjects" Version="4.55.0" />
     <PackageReference Include="Moq" Version="4.9.0" />
     <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
-    <PackageReference Include="NUnit" Version="3.10.1" />
+    <PackageReference Include="NUnit" Version="3.11.0" />
     <PackageReference Include="protobuf-net" Version="2.3.13" />
     <PackageReference Include="structuremap" Version="4.7.0" />
     <PackageReference Include="System.Management" Version="4.5.0" />

src/Abc.Zebus.Testing/Abc.Zebus.Testing.netcoreapp2.1.v3.ncrunchproject

@@ -0,0 +1,8 @@
+<ProjectConfiguration>
+  <Settings>
+    <HiddenComponentWarnings>
+      <Value>NetCoreNetStandardLocalSystem</Value>
+      <Value>NCrunchAssemblyDepsResolutionFailure</Value>
+    </HiddenComponentWarnings>
+  </Settings>
+</ProjectConfiguration>

src/Abc.Zebus.Testing/Extensions/NUnitExtensions.cs

@@ -7,7 +7,6 @@ using System.Linq;
 using System.Linq.Expressions;
 using System.Reflection;
 using Abc.Zebus.Testing.Comparison;
-using Abc.Zebus.Util.Extensions;
 using KellermanSoftware.CompareNetObjects;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
@@ -182,6 +181,11 @@ namespace Abc.Zebus.Testing.Extensions
             }
         }
 
+        public static void ShouldBeEquivalentTo<T>(this IEnumerable<T> collection, params T[] expected)
+        {
+            ShouldBeEquivalentTo((IEnumerable)collection, expected);
+        }
+
         public static void ShouldBeEquivalentTo(this IEnumerable collection, IEnumerable expected, bool compareDeeply = false)
         {
             if (compareDeeply)

src/Abc.Zebus.Testing/Properties/AssemblyInfo.cs

@@ -12,3 +12,4 @@ using System.Runtime.InteropServices;
 [assembly: InternalsVisibleTo("Abc.Zebus.Directory.Cassandra.Tests")]
 [assembly: InternalsVisibleTo("Abc.Zebus.Persistence.Tests")]
 [assembly: InternalsVisibleTo("Abc.Zebus.Persistence.CQL.Tests")]
+[assembly: InternalsVisibleTo("Abc.Zebus.Persistence.RocksDb.Tests")]

src/Abc.Zebus.Tests/Abc.Zebus.Tests.netcoreapp2.1.v3.ncrunchproject

@@ -0,0 +1,8 @@
+<ProjectConfiguration>
+  <Settings>
+    <HiddenComponentWarnings>
+      <Value>NCrunchAssemblyDepsResolutionFailure</Value>
+      <Value>NetCoreNetStandardLocalSystem</Value>
+    </HiddenComponentWarnings>
+  </Settings>
+</ProjectConfiguration>

src/Abc.Zebus.Tests/Core/BusManualTests.cs

@@ -6,6 +6,7 @@ using Abc.Zebus.Core;
 using Abc.Zebus.Dispatch;
 using Abc.Zebus.Testing;
 using Abc.Zebus.Util;
+using Moq;
 using NUnit.Framework;
 using ProtoBuf;
 
@@ -78,9 +79,36 @@ namespace Abc.Zebus.Tests.Core
             }
         }
 
+        [Test]
+        public void should_generate_unacked_messages()
+        {
+            var targetConfig = new Mock<IBusConfiguration>();
+            targetConfig.SetupGet(x => x.DirectoryServiceEndPoints).Returns(new[] { _directoryEndPoint });
+            targetConfig.SetupGet(x => x.IsPersistent).Returns(true);
+            targetConfig.SetupGet(x => x.RegistrationTimeout).Returns(30.Seconds());
+            targetConfig.SetupGet(x => x.StartReplayTimeout).Returns(30.Seconds());
+
+            var target = CreateBusFactory().WithHandlers(typeof(ManualEventHandler))
+                                                  .WithConfiguration(targetConfig.Object, "Demo")
+                                                  .WithPeerId("Some.Random.Persistent.Peer.0")
+                                                  .CreateAndStartBus();
+            using (var source = CreateBusFactory().CreateAndStartBus())
+            {
+                source.Publish(new ManualEvent(42));
+                Thread.Sleep(2000);
+
+                target.Dispose();
+
+                for (int i = 0; i < 1_000; i++)
+                {
+                    source.Publish(new ManualEvent(42));
+                }
+            }
+        }
+
         private static BusFactory CreateBusFactory()
         {
-            return new BusFactory().WithConfiguration(_directoryEndPoint, "Dev");
+            return new BusFactory().WithConfiguration(_directoryEndPoint, "Demo");
         }
 
         [ProtoContract]

src/Abc.Zebus.sln

@@ -44,196 +44,90 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Abc.Zebus.Persistence.Tests
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Abc.Zebus.Persistence", "Abc.Zebus.Persistence\Abc.Zebus.Persistence.csproj", "{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Abc.Zebus.Persistence.RocksDb", "Abc.Zebus.Persistence.RocksDb\Abc.Zebus.Persistence.RocksDb.csproj", "{03C805D6-0217-4E94-8DF5-1C0ACD4CF82A}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Abc.Zebus.Persistence.RocksDb.Tests", "Abc.Zebus.Persistence.RocksDb.Tests\Abc.Zebus.Persistence.RocksDb.Tests.csproj", "{123C2076-DE3C-4F41-AC58-1AC5210F1CCD}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Abc.Zebus.Directory.Runner", "Abc.Zebus.Directory.Runner\Abc.Zebus.Directory.Runner.csproj", "{F9B2BCDB-401F-4A13-B235-375F06B0D206}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
-		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
 		Release|Any CPU = Release|Any CPU
-		Release|x64 = Release|x64
-		Release|x86 = Release|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Debug|x64.Build.0 = Debug|Any CPU
-		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Debug|x86.Build.0 = Debug|Any CPU
 		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Release|Any CPU.Build.0 = Release|Any CPU
-		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Release|x64.ActiveCfg = Release|Any CPU
-		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Release|x64.Build.0 = Release|Any CPU
-		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Release|x86.ActiveCfg = Release|Any CPU
-		{1F4C6307-6113-40D5-BF42-4B6BF5DF13B2}.Release|x86.Build.0 = Release|Any CPU
 		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Debug|x64.Build.0 = Debug|Any CPU
-		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Debug|x86.Build.0 = Debug|Any CPU
 		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Release|Any CPU.Build.0 = Release|Any CPU
-		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Release|x64.ActiveCfg = Release|Any CPU
-		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Release|x64.Build.0 = Release|Any CPU
-		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Release|x86.ActiveCfg = Release|Any CPU
-		{7BF5AC28-91DB-40EF-8CB7-024E518FA799}.Release|x86.Build.0 = Release|Any CPU
 		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Debug|x64.Build.0 = Debug|Any CPU
-		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Debug|x86.Build.0 = Debug|Any CPU
 		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Release|x64.ActiveCfg = Release|Any CPU
-		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Release|x64.Build.0 = Release|Any CPU
-		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Release|x86.ActiveCfg = Release|Any CPU
-		{4CE123DF-8021-411C-929B-53E5C5FC1E04}.Release|x86.Build.0 = Release|Any CPU
 		{507A4411-DB8A-4663-A491-CCA29854B890}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{507A4411-DB8A-4663-A491-CCA29854B890}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{507A4411-DB8A-4663-A491-CCA29854B890}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{507A4411-DB8A-4663-A491-CCA29854B890}.Debug|x64.Build.0 = Debug|Any CPU
-		{507A4411-DB8A-4663-A491-CCA29854B890}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{507A4411-DB8A-4663-A491-CCA29854B890}.Debug|x86.Build.0 = Debug|Any CPU
 		{507A4411-DB8A-4663-A491-CCA29854B890}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{507A4411-DB8A-4663-A491-CCA29854B890}.Release|Any CPU.Build.0 = Release|Any CPU
-		{507A4411-DB8A-4663-A491-CCA29854B890}.Release|x64.ActiveCfg = Release|Any CPU
-		{507A4411-DB8A-4663-A491-CCA29854B890}.Release|x64.Build.0 = Release|Any CPU
-		{507A4411-DB8A-4663-A491-CCA29854B890}.Release|x86.ActiveCfg = Release|Any CPU
-		{507A4411-DB8A-4663-A491-CCA29854B890}.Release|x86.Build.0 = Release|Any CPU
 		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Debug|x64.Build.0 = Debug|Any CPU
-		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Debug|x86.Build.0 = Debug|Any CPU
 		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Release|x64.ActiveCfg = Release|Any CPU
-		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Release|x64.Build.0 = Release|Any CPU
-		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Release|x86.ActiveCfg = Release|Any CPU
-		{EE6730D9-6A21-46A5-AE3F-AF78060B6EA3}.Release|x86.Build.0 = Release|Any CPU
 		{8F786F88-CB54-43DC-927F-65B75795EC60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{8F786F88-CB54-43DC-927F-65B75795EC60}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{8F786F88-CB54-43DC-927F-65B75795EC60}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{8F786F88-CB54-43DC-927F-65B75795EC60}.Debug|x64.Build.0 = Debug|Any CPU
-		{8F786F88-CB54-43DC-927F-65B75795EC60}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{8F786F88-CB54-43DC-927F-65B75795EC60}.Debug|x86.Build.0 = Debug|Any CPU
 		{8F786F88-CB54-43DC-927F-65B75795EC60}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8F786F88-CB54-43DC-927F-65B75795EC60}.Release|Any CPU.Build.0 = Release|Any CPU
-		{8F786F88-CB54-43DC-927F-65B75795EC60}.Release|x64.ActiveCfg = Release|Any CPU
-		{8F786F88-CB54-43DC-927F-65B75795EC60}.Release|x64.Build.0 = Release|Any CPU
-		{8F786F88-CB54-43DC-927F-65B75795EC60}.Release|x86.ActiveCfg = Release|Any CPU
-		{8F786F88-CB54-43DC-927F-65B75795EC60}.Release|x86.Build.0 = Release|Any CPU
 		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Debug|x64.Build.0 = Debug|Any CPU
-		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Debug|x86.Build.0 = Debug|Any CPU
 		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Release|x64.ActiveCfg = Release|Any CPU
-		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Release|x64.Build.0 = Release|Any CPU
-		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Release|x86.ActiveCfg = Release|Any CPU
-		{549C137C-9CE4-4765-9E1D-E348E96A08D6}.Release|x86.Build.0 = Release|Any CPU
 		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Debug|x64.Build.0 = Debug|Any CPU
-		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Debug|x86.Build.0 = Debug|Any CPU
 		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Release|Any CPU.Build.0 = Release|Any CPU
-		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Release|x64.ActiveCfg = Release|Any CPU
-		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Release|x64.Build.0 = Release|Any CPU
-		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Release|x86.ActiveCfg = Release|Any CPU
-		{7C004E59-3B24-4359-8E45-1CD93F03FD80}.Release|x86.Build.0 = Release|Any CPU
 		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Debug|x64.Build.0 = Debug|Any CPU
-		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Debug|x86.Build.0 = Debug|Any CPU
 		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Release|Any CPU.Build.0 = Release|Any CPU
-		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Release|x64.ActiveCfg = Release|Any CPU
-		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Release|x64.Build.0 = Release|Any CPU
-		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Release|x86.ActiveCfg = Release|Any CPU
-		{C59CFDDA-987B-4948-B016-95982BBE2CFA}.Release|x86.Build.0 = Release|Any CPU
 		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Debug|x64.Build.0 = Debug|Any CPU
-		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Debug|x86.Build.0 = Debug|Any CPU
 		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Release|x64.ActiveCfg = Release|Any CPU
-		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Release|x64.Build.0 = Release|Any CPU
-		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Release|x86.ActiveCfg = Release|Any CPU
-		{1059B7D5-1C8F-4702-A6DA-58986FB042F6}.Release|x86.Build.0 = Release|Any CPU
 		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Debug|x64.Build.0 = Debug|Any CPU
-		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Debug|x86.Build.0 = Debug|Any CPU
 		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Release|x64.ActiveCfg = Release|Any CPU
-		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Release|x64.Build.0 = Release|Any CPU
-		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Release|x86.ActiveCfg = Release|Any CPU
-		{A9B86B14-A1D5-46F8-9F2D-777C204D7C0B}.Release|x86.Build.0 = Release|Any CPU
 		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Debug|x64.Build.0 = Debug|Any CPU
-		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Debug|x86.Build.0 = Debug|Any CPU
 		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Release|Any CPU.Build.0 = Release|Any CPU
-		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Release|x64.ActiveCfg = Release|Any CPU
-		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Release|x64.Build.0 = Release|Any CPU
-		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Release|x86.ActiveCfg = Release|Any CPU
-		{8FC5C46E-9C7F-4905-B8A8-89E154F576CF}.Release|x86.Build.0 = Release|Any CPU
 		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Debug|x64.Build.0 = Debug|Any CPU
-		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Debug|x86.Build.0 = Debug|Any CPU
 		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Release|x64.ActiveCfg = Release|Any CPU
-		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Release|x64.Build.0 = Release|Any CPU
-		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Release|x86.ActiveCfg = Release|Any CPU
-		{04F7E288-7EFB-4074-88BC-04FFF99C424B}.Release|x86.Build.0 = Release|Any CPU
 		{32140735-1E96-4E64-8C15-7B0D547791F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{32140735-1E96-4E64-8C15-7B0D547791F1}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{32140735-1E96-4E64-8C15-7B0D547791F1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{32140735-1E96-4E64-8C15-7B0D547791F1}.Debug|x64.Build.0 = Debug|Any CPU
-		{32140735-1E96-4E64-8C15-7B0D547791F1}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{32140735-1E96-4E64-8C15-7B0D547791F1}.Debug|x86.Build.0 = Debug|Any CPU
 		{32140735-1E96-4E64-8C15-7B0D547791F1}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{32140735-1E96-4E64-8C15-7B0D547791F1}.Release|Any CPU.Build.0 = Release|Any CPU
-		{32140735-1E96-4E64-8C15-7B0D547791F1}.Release|x64.ActiveCfg = Release|Any CPU
-		{32140735-1E96-4E64-8C15-7B0D547791F1}.Release|x64.Build.0 = Release|Any CPU
-		{32140735-1E96-4E64-8C15-7B0D547791F1}.Release|x86.ActiveCfg = Release|Any CPU
-		{32140735-1E96-4E64-8C15-7B0D547791F1}.Release|x86.Build.0 = Release|Any CPU
 		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Debug|x64.Build.0 = Debug|Any CPU
-		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Debug|x86.Build.0 = Debug|Any CPU
 		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Release|x64.ActiveCfg = Release|Any CPU
-		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Release|x64.Build.0 = Release|Any CPU
-		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Release|x86.ActiveCfg = Release|Any CPU
-		{A8117C95-B3CA-447C-BCDF-9B6F7A17F290}.Release|x86.Build.0 = Release|Any CPU
+		{03C805D6-0217-4E94-8DF5-1C0ACD4CF82A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{03C805D6-0217-4E94-8DF5-1C0ACD4CF82A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{03C805D6-0217-4E94-8DF5-1C0ACD4CF82A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{03C805D6-0217-4E94-8DF5-1C0ACD4CF82A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{123C2076-DE3C-4F41-AC58-1AC5210F1CCD}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{123C2076-DE3C-4F41-AC58-1AC5210F1CCD}.Debug|Any CPU.Build.0 = Debug|x64
+		{123C2076-DE3C-4F41-AC58-1AC5210F1CCD}.Release|Any CPU.ActiveCfg = Release|x64
+		{123C2076-DE3C-4F41-AC58-1AC5210F1CCD}.Release|Any CPU.Build.0 = Release|x64
+		{F9B2BCDB-401F-4A13-B235-375F06B0D206}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F9B2BCDB-401F-4A13-B235-375F06B0D206}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F9B2BCDB-401F-4A13-B235-375F06B0D206}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F9B2BCDB-401F-4A13-B235-375F06B0D206}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/Abc.Zebus/Abc.Zebus.v3.ncrunchproject

@@ -0,0 +1,7 @@
+<ProjectConfiguration>
+  <Settings>
+    <HiddenComponentWarnings>
+      <Value>NetCoreNetStandardLocalSystem</Value>
+    </HiddenComponentWarnings>
+  </Settings>
+</ProjectConfiguration>

src/Abc.Zebus/MessageId.cs

@@ -45,17 +45,13 @@ namespace Abc.Zebus
             return new DisposableAction(() => _pausedGuid = null);
         }
 
-        public static MessageId NextId()
-            => new MessageId(NewGuid());
+        public static MessageId NextId() => new MessageId(NewGuid());
 
-        private static Guid NewGuid()
-            => _pausedGuid ?? _generator.NewGuid();
+        private static Guid NewGuid() => _pausedGuid ?? _generator.NewGuid();
 
-        public DateTime GetDateTime()
-            => TimeGuidGenerator.ExtractDateTime(Value);
+        public DateTime GetDateTime() => TimeGuidGenerator.ExtractDateTime(Value);
 
-        public static void ResetLastTimestamp()
-            => _generator.Reset();
+        public static void ResetLastTimestamp() => _generator.Reset();
 
         /// <summary>
         /// Time-based Guid generator.

src/Abc.Zebus/Properties/AssemblyInfo.cs

@@ -6,6 +6,7 @@ using System.Runtime.InteropServices;
 
 [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")]
 [assembly: InternalsVisibleTo("Abc.Zebus.Directory")]
+[assembly: InternalsVisibleTo("Abc.Zebus.Directory.Runner")]
 [assembly: InternalsVisibleTo("Abc.Zebus.Directory.Tests")]
 [assembly: InternalsVisibleTo("Abc.Zebus.Directory.Cassandra.Tests")]
 [assembly: InternalsVisibleTo("Abc.Zebus.Testing")]
@@ -26,3 +27,5 @@ using System.Runtime.InteropServices;
 [assembly: InternalsVisibleTo("Abc.Zebus.Persistence")]
 [assembly: InternalsVisibleTo("Abc.Zebus.Persistence.CQL")]
 [assembly: InternalsVisibleTo("Abc.Zebus.Persistence.CQL.Tests")]
+[assembly: InternalsVisibleTo("Abc.Zebus.Persistence.RocksDb")]
+[assembly: InternalsVisibleTo("Abc.Zebus.Persistence.RocksDb.Tests")]