Azure#105 Added IConnectionMultiplexerFactory as an extension point f…

…or handling custom lifecycle of StackEchange.IConnectionMultiplexer.

.gitignore

@@ -1,6 +1,8 @@
 bin
 obj
 csx
+.vs
+_ReSharper.Caches
 
 *.exe
 *.dll

src/OutputCacheProvider_net452/RedisOutputCacheProvider_net452.csproj

@@ -73,6 +73,12 @@
     <Compile Include="..\Shared\ChangeTrackingSessionStateItemCollection.cs">
       <Link>ChangeTrackingSessionStateItemCollection.cs</Link>
     </Compile>
+    <Compile Include="..\Shared\ConnectionMultiplexerFactory.cs">
+      <Link>ConnectionMultiplexerFactory.cs</Link>
+    </Compile>
+    <Compile Include="..\Shared\IConnectionMultiplexerFactory.cs">
+      <Link>IConnectionMultiplexerFactory.cs</Link>
+    </Compile>
     <Compile Include="..\Shared\IRedisClientConnection.cs">
       <Link>IRedisClientConnection.cs</Link>
     </Compile>

src/OutputCacheProvider_net462/RedisOutputCacheProvider_net462.csproj

@@ -77,6 +77,12 @@
     <Compile Include="..\Shared\ChangeTrackingSessionStateItemCollection.cs">
       <Link>ChangeTrackingSessionStateItemCollection.cs</Link>
     </Compile>
+    <Compile Include="..\Shared\ConnectionMultiplexerFactory.cs">
+      <Link>ConnectionMultiplexerFactory.cs</Link>
+    </Compile>
+    <Compile Include="..\Shared\IConnectionMultiplexerFactory.cs">
+      <Link>IConnectionMultiplexerFactory.cs</Link>
+    </Compile>
     <Compile Include="..\Shared\IRedisClientConnection.cs">
       <Link>IRedisClientConnection.cs</Link>
     </Compile>

src/RedisSessionStateProvider/RedisConnectionWrapper.cs

@@ -11,19 +11,16 @@ namespace Microsoft.Web.Redis
 {
     internal class RedisConnectionWrapper : ICacheConnection
     {
-        internal static RedisSharedConnection sharedConnection;
-        static object lockForSharedConnection = new object();
+        internal static RedisSharedConnection sharedConnection; // todo remove field
+        static readonly object lockForSharedConnection = new object();
         internal static RedisUtility redisUtility;
 
         public KeyGenerator Keys { set; get; }
 
         internal IRedisClientConnection redisConnection;
-        ProviderConfiguration configuration;
-
 
         public RedisConnectionWrapper(ProviderConfiguration configuration, string id)
         {
-            this.configuration = configuration;
             Keys = new KeyGenerator(id, configuration.ApplicationName);
 
             // only single object of RedisSharedConnection will be created and then reused
@@ -62,7 +59,8 @@ public TimeSpan GetLockAge(object lockId)
         // ARGV[1] = session-timeout 
         // this order should not change LUA script depends on it
         // if data doesn't exists then do nothing
-        static readonly string updateExpiryTimeScript = (@" 
+        // TODO TrimScript
+        static readonly string updateExpiryTimeScript = (@"
                 local dataExists = redis.call('EXISTS', KEYS[1])
                 if dataExists == 0 then
                     return 1;

src/RedisSessionStateProvider_net452/RedisSessionStateProvider_net452.csproj

@@ -73,6 +73,12 @@
     <Compile Include="..\Shared\ChangeTrackingSessionStateItemCollection.cs">
       <Link>ChangeTrackingSessionStateItemCollection.cs</Link>
     </Compile>
+    <Compile Include="..\Shared\ConnectionMultiplexerFactory.cs">
+      <Link>ConnectionMultiplexerFactory.cs</Link>
+    </Compile>
+    <Compile Include="..\Shared\IConnectionMultiplexerFactory.cs">
+      <Link>IConnectionMultiplexerFactory.cs</Link>
+    </Compile>
     <Compile Include="..\Shared\IRedisClientConnection.cs">
       <Link>IRedisClientConnection.cs</Link>
     </Compile>

src/RedisSessionStateProvider_net462/RedisSessionStateProvider_net462.csproj

@@ -77,6 +77,12 @@
     <Compile Include="..\Shared\ChangeTrackingSessionStateItemCollection.cs">
       <Link>ChangeTrackingSessionStateItemCollection.cs</Link>
     </Compile>
+    <Compile Include="..\Shared\ConnectionMultiplexerFactory.cs">
+      <Link>ConnectionMultiplexerFactory.cs</Link>
+    </Compile>
+    <Compile Include="..\Shared\IConnectionMultiplexerFactory.cs">
+      <Link>IConnectionMultiplexerFactory.cs</Link>
+    </Compile>
     <Compile Include="..\Shared\IRedisClientConnection.cs">
       <Link>IRedisClientConnection.cs</Link>
     </Compile>

src/Shared/ChangeTrackingSessionStateItemCollection.cs

@@ -192,7 +192,7 @@ internal object GetDataWithoutUpdatingModifiedKeys(string name)
             {
                 return valueWrapper.GetActualValue(_utility);
             }
-            return null;;
+            return null;
         }
 
         public override IEnumerator GetEnumerator()

src/Shared/ConnectionMultiplexerFactory.cs

@@ -0,0 +1,125 @@
+using System;
+using StackExchange.Redis;
+
+namespace Microsoft.Web.Redis
+{
+    internal class ConnectionMultiplexerFactory : IConnectionMultiplexerFactory
+    {
+        private readonly ConfigurationOptions _configOption;
+
+        internal static DateTimeOffset lastReconnectTime = DateTimeOffset.MinValue;
+        internal static DateTimeOffset firstErrorTime = DateTimeOffset.MinValue;
+        internal static DateTimeOffset previousErrorTime = DateTimeOffset.MinValue;
+        static readonly object reconnectLock = new object();
+        internal static TimeSpan ReconnectFrequency = TimeSpan.FromSeconds(60);
+        internal static TimeSpan ReconnectErrorThreshold = TimeSpan.FromSeconds(30);
+
+        public ConnectionMultiplexerFactory(ProviderConfiguration configuration)
+        {
+            // If connection string is given then use it otherwise use individual options
+            if (!string.IsNullOrEmpty(configuration.ConnectionString))
+            {
+                _configOption = ConfigurationOptions.Parse(configuration.ConnectionString);
+                // Setting explicitly 'abortconnect' to false. It will overwrite customer provided value for 'abortconnect'
+                // As it doesn't make sense to allow to customer to set it to true as we don't give them access to ConnectionMultiplexer
+                // in case of failure customer can not create ConnectionMultiplexer so right choice is to automatically create it by providing AbortOnConnectFail = false
+                _configOption.AbortOnConnectFail = false;
+            }
+            else
+            {
+                _configOption = new ConfigurationOptions();
+                if (configuration.Port == 0)
+                {
+                    _configOption.EndPoints.Add(configuration.Host);
+                }
+                else
+                {
+                    _configOption.EndPoints.Add(configuration.Host + ":" + configuration.Port);
+                }
+                _configOption.Password = configuration.AccessKey;
+                _configOption.Ssl = configuration.UseSsl;
+                _configOption.AbortOnConnectFail = false;
+
+                if (configuration.ConnectionTimeoutInMilliSec != 0)
+                {
+                    _configOption.ConnectTimeout = configuration.ConnectionTimeoutInMilliSec;
+                }
+
+                if (configuration.OperationTimeoutInMilliSec != 0)
+                {
+                    _configOption.SyncTimeout = configuration.OperationTimeoutInMilliSec;
+                }
+            }
+        }
+
+        public IConnectionMultiplexer CreateMultiplexer()
+        {
+            lastReconnectTime = DateTimeOffset.UtcNow;
+            return LogUtility.logger == null
+                ? ConnectionMultiplexer.Connect(_configOption)
+                : ConnectionMultiplexer.Connect(_configOption, LogUtility.logger);
+        }
+
+        private static void CloseMultiplexer(IConnectionMultiplexer oldMultiplexer)
+        {
+            try
+            {
+                oldMultiplexer.Close();
+            }
+            catch (Exception)
+            {
+                // Example error condition: if accessing old.Value causes a connection attempt and that fails. 
+            }
+        }
+
+        public IConnectionMultiplexer RestartMultiplexer(IConnectionMultiplexer connectionMultiplexer)
+        {
+            var previousReconnect = lastReconnectTime;
+            var elapsedSinceLastReconnect = DateTimeOffset.UtcNow - previousReconnect;
+
+            // If mulitple threads call RestartMultiplexer at the same time, we only want to honor one of them. 
+            if (elapsedSinceLastReconnect > ReconnectFrequency)
+            {
+                lock (reconnectLock)
+                {
+                    var utcNow = DateTimeOffset.UtcNow;
+                    elapsedSinceLastReconnect = utcNow - lastReconnectTime;
+
+                    if (elapsedSinceLastReconnect < ReconnectFrequency)
+                    {
+                        return connectionMultiplexer; // Some other thread made it through the check and the lock, so nothing to do. 
+                    }
+
+                    if (firstErrorTime == DateTimeOffset.MinValue)
+                    {
+                        // We got error first time after last reconnect
+                        firstErrorTime = utcNow;
+                        previousErrorTime = utcNow;
+                        return connectionMultiplexer;
+                    }
+
+                    var elapsedSinceFirstError = utcNow - firstErrorTime;
+                    var elapsedSinceMostRecentError = utcNow - previousErrorTime;
+                    previousErrorTime = utcNow;
+
+                    if ((elapsedSinceFirstError >= ReconnectErrorThreshold) && (elapsedSinceMostRecentError <= ReconnectErrorThreshold))
+                    {
+                        LogUtility.LogInfo($"RestartMultiplexer: now: {utcNow.ToString()}");
+                        LogUtility.LogInfo($"RestartMultiplexer: elapsedSinceLastReconnect: {elapsedSinceLastReconnect.ToString()}, ReconnectFrequency: {ReconnectFrequency.ToString()}");
+                        LogUtility.LogInfo($"RestartMultiplexer: elapsedSinceFirstError: {elapsedSinceFirstError.ToString()}, elapsedSinceMostRecentError: {elapsedSinceMostRecentError.ToString()}, ReconnectErrorThreshold: {ReconnectErrorThreshold.ToString()}");
+
+                        firstErrorTime = DateTimeOffset.MinValue;
+                        previousErrorTime = DateTimeOffset.MinValue;
+
+                        //var oldMultiplexer = _redisMultiplexer;
+                        CloseMultiplexer(connectionMultiplexer);
+                        return CreateMultiplexer();
+                    }
+
+                    return connectionMultiplexer;
+                }
+            }
+            return connectionMultiplexer;
+        }
+    }
+}

src/Shared/IConnectionMultiplexerFactory.cs

@@ -0,0 +1,22 @@
+using StackExchange.Redis;
+
+namespace Microsoft.Web.Redis
+{
+    public interface IConnectionMultiplexerFactory
+    {
+        /// <summary>
+        /// This method provides either new or already existing instance of <see cref="IConnectionMultiplexer"/>.
+        /// </summary>
+        /// <returns>Fully configured connection multiplexer</returns>
+        IConnectionMultiplexer CreateMultiplexer();
+
+        /// <summary>
+        /// When <see cref="RedisSessionStateProvider"/> fails with <see cref="RedisConnectionException"/>
+        /// then it sends to <see cref="IConnectionMultiplexerFactory"/> and attempt to cleanup a failed multiplexer.
+        /// Additionally, the factory itself is responsible for providing a clean, fresh instance of IConnectionMultiplexer (it can be the same instance).
+        /// </summary>
+        /// <param name="connectionMultiplexer">Instance that failed with <see cref="RedisConnectionException"/></param>
+        /// <returns></returns>
+        IConnectionMultiplexer RestartMultiplexer(IConnectionMultiplexer connectionMultiplexer);
+    }
+}

src/Shared/ProviderConfiguration.cs

@@ -29,6 +29,7 @@ internal class ProviderConfiguration
         public int OperationTimeoutInMilliSec { get; set; }
         public string ConnectionString { get; set; }
         public string RedisSerializerType { get; set; }
+        public string ConnectionMultiplexerFactoryType { get; internal set; }
 
         /* Empty constructor required for testing */
         internal ProviderConfiguration()
@@ -83,6 +84,7 @@ private ProviderConfiguration(NameValueCollection config)
             AccessKey = GetStringSettings(config, "accessKey", null);
             UseSsl = GetBoolSettings(config, "ssl", true);
             RedisSerializerType = GetStringSettings(config, "redisSerializerType", null);
+            ConnectionMultiplexerFactoryType = GetStringSettings(config, "connectionMultiplexerFactoryType", null);
             // All below parameters are only fetched from web.config
             DatabaseId = GetIntSettings(config, "databaseId", 0);
             ApplicationName = GetStringSettings(config, "applicationName", null);