POC sql connection pool

Emby.Server.Implementations/ApplicationHost.cs

@@ -627,6 +627,9 @@ namespace Emby.Server.Implementations
                 }
             }
 
+            ((SqliteItemRepository)Resolve<IItemRepository>()).Initialize();
+            ((SqliteUserDataRepository)Resolve<IUserDataRepository>()).Initialize();
+
             var localizationManager = (LocalizationManager)Resolve<ILocalizationManager>();
             await localizationManager.LoadAll().ConfigureAwait(false);
 
@@ -634,9 +637,6 @@ namespace Emby.Server.Implementations
 
             SetStaticProperties();
 
-            var userDataRepo = (SqliteUserDataRepository)Resolve<IUserDataRepository>();
-            ((SqliteItemRepository)Resolve<IItemRepository>()).Initialize(userDataRepo, Resolve<IUserManager>());
-
             FindParts();
         }
 

Emby.Server.Implementations/Data/BaseSqliteRepository.cs

@@ -4,7 +4,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Threading;
 using Jellyfin.Extensions;
 using Microsoft.Extensions.Logging;
 using SQLitePCL.pretty;
@@ -27,9 +26,19 @@ namespace Emby.Server.Implementations.Data
         /// <summary>
         /// Gets or sets the path to the DB file.
         /// </summary>
-        /// <value>Path to the DB file.</value>
         protected string DbFilePath { get; set; }
 
+        /// <summary>
+        /// Gets or sets the number of write connections to create.
+        /// </summary>
+        /// <value>Path to the DB file.</value>
+        protected int WriteConnectionsCount { get; set; } = 1;
+
+        /// <summary>
+        /// Gets or sets the number of read connections to create.
+        /// </summary>
+        protected int ReadConnectionsCount { get; set; } = 1;
+
         /// <summary>
         /// Gets the logger.
         /// </summary>
@@ -63,7 +72,7 @@ namespace Emby.Server.Implementations.Data
         /// <summary>
         /// Gets the locking mode. <see href="https://www.sqlite.org/pragma.html#pragma_locking_mode" />.
         /// </summary>
-        protected virtual string LockingMode => "EXCLUSIVE";
+        protected virtual string LockingMode => "NORMAL";
 
         /// <summary>
         /// Gets the journal mode. <see href="https://www.sqlite.org/pragma.html#pragma_journal_mode" />.
@@ -88,7 +97,7 @@ namespace Emby.Server.Implementations.Data
         /// </summary>
         /// <value>The temp store mode.</value>
         /// <see cref="TempStoreMode"/>
-        protected virtual TempStoreMode TempStore => TempStoreMode.Default;
+        protected virtual TempStoreMode TempStore => TempStoreMode.Memory;
 
         /// <summary>
         /// Gets the synchronous mode.
@@ -101,63 +110,115 @@ namespace Emby.Server.Implementations.Data
         /// Gets or sets the write lock.
         /// </summary>
         /// <value>The write lock.</value>
-        protected SemaphoreSlim WriteLock { get; set; } = new SemaphoreSlim(1, 1);
+        protected ConnectionPool WriteConnections { get; set; }
 
         /// <summary>
         /// Gets or sets the write connection.
         /// </summary>
         /// <value>The write connection.</value>
-        protected SQLiteDatabaseConnection WriteConnection { get; set; }
+        protected ConnectionPool ReadConnections { get; set; }
+
+        public virtual void Initialize()
+        {
+            WriteConnections = new ConnectionPool(WriteConnectionsCount, CreateWriteConnection);
+            ReadConnections = new ConnectionPool(ReadConnectionsCount, CreateReadConnection);
+        }
 
         protected ManagedConnection GetConnection(bool readOnly = false)
         {
-            WriteLock.Wait();
-            if (WriteConnection is not null)
+            if (readOnly)
             {
-                return new ManagedConnection(WriteConnection, WriteLock);
+                return ReadConnections.GetConnection();
             }
 
-            WriteConnection = SQLite3.Open(
+            return WriteConnections.GetConnection();
+        }
+
+        protected SQLiteDatabaseConnection CreateWriteConnection()
+        {
+            var writeConnection = SQLite3.Open(
                 DbFilePath,
                 DefaultConnectionFlags | ConnectionFlags.Create | ConnectionFlags.ReadWrite,
                 null);
 
             if (CacheSize.HasValue)
             {
-                WriteConnection.Execute("PRAGMA cache_size=" + CacheSize.Value);
+                writeConnection.Execute("PRAGMA cache_size=" + CacheSize.Value);
             }
 
             if (!string.IsNullOrWhiteSpace(LockingMode))
             {
-                WriteConnection.Execute("PRAGMA locking_mode=" + LockingMode);
+                writeConnection.Execute("PRAGMA locking_mode=" + LockingMode);
             }
 
             if (!string.IsNullOrWhiteSpace(JournalMode))
             {
-                WriteConnection.Execute("PRAGMA journal_mode=" + JournalMode);
+                writeConnection.Execute("PRAGMA journal_mode=" + JournalMode);
             }
 
             if (JournalSizeLimit.HasValue)
             {
-                WriteConnection.Execute("PRAGMA journal_size_limit=" + JournalSizeLimit.Value);
+                writeConnection.Execute("PRAGMA journal_size_limit=" + JournalSizeLimit.Value);
             }
 
             if (Synchronous.HasValue)
             {
-                WriteConnection.Execute("PRAGMA synchronous=" + (int)Synchronous.Value);
+                writeConnection.Execute("PRAGMA synchronous=" + (int)Synchronous.Value);
             }
 
             if (PageSize.HasValue)
             {
-                WriteConnection.Execute("PRAGMA page_size=" + PageSize.Value);
+                writeConnection.Execute("PRAGMA page_size=" + PageSize.Value);
             }
 
-            WriteConnection.Execute("PRAGMA temp_store=" + (int)TempStore);
+            writeConnection.Execute("PRAGMA temp_store=" + (int)TempStore);
 
             // Configuration and pragmas can affect VACUUM so it needs to be last.
-            WriteConnection.Execute("VACUUM");
+            writeConnection.Execute("VACUUM");
 
-            return new ManagedConnection(WriteConnection, WriteLock);
+            return writeConnection;
+        }
+
+        protected SQLiteDatabaseConnection CreateReadConnection()
+        {
+            var writeConnection = SQLite3.Open(
+                DbFilePath,
+                DefaultConnectionFlags | ConnectionFlags.ReadOnly,
+                null);
+
+            if (CacheSize.HasValue)
+            {
+                writeConnection.Execute("PRAGMA cache_size=" + CacheSize.Value);
+            }
+
+            if (!string.IsNullOrWhiteSpace(LockingMode))
+            {
+                writeConnection.Execute("PRAGMA locking_mode=" + LockingMode);
+            }
+
+            if (!string.IsNullOrWhiteSpace(JournalMode))
+            {
+                writeConnection.Execute("PRAGMA journal_mode=" + JournalMode);
+            }
+
+            if (JournalSizeLimit.HasValue)
+            {
+                writeConnection.Execute("PRAGMA journal_size_limit=" + JournalSizeLimit.Value);
+            }
+
+            if (Synchronous.HasValue)
+            {
+                writeConnection.Execute("PRAGMA synchronous=" + (int)Synchronous.Value);
+            }
+
+            if (PageSize.HasValue)
+            {
+                writeConnection.Execute("PRAGMA page_size=" + PageSize.Value);
+            }
+
+            writeConnection.Execute("PRAGMA temp_store=" + (int)TempStore);
+
+            return writeConnection;
         }
 
         public IStatement PrepareStatement(ManagedConnection connection, string sql)
@@ -240,22 +301,10 @@ namespace Emby.Server.Implementations.Data
 
             if (dispose)
             {
-                WriteLock.Wait();
-                try
-                {
-                    WriteConnection?.Dispose();
-                }
-                finally
-                {
-                    WriteLock.Release();
-                }
-
-                WriteLock.Dispose();
+                WriteConnections.Dispose();
+                ReadConnections.Dispose();
             }
 
-            WriteConnection = null;
-            WriteLock = null;
-
             _disposed = true;
         }
     }

Emby.Server.Implementations/Data/ConnectionPool.cs

@@ -0,0 +1,68 @@
+#pragma warning disable CS1591
+
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+using SQLitePCL.pretty;
+
+namespace Emby.Server.Implementations.Data;
+
+public sealed class ConnectionPool : IDisposable
+{
+    private readonly int _count;
+    private readonly SemaphoreSlim _lock;
+    private readonly ConcurrentQueue<SQLiteDatabaseConnection> _connections = new ConcurrentQueue<SQLiteDatabaseConnection>();
+    private bool _disposed;
+
+    public ConnectionPool(int count, Func<SQLiteDatabaseConnection> factory)
+    {
+        _count = count;
+        _lock = new SemaphoreSlim(count, count);
+        for (int i = 0; i < count; i++)
+        {
+            _connections.Enqueue(factory.Invoke());
+        }
+    }
+
+    public ManagedConnection GetConnection()
+    {
+        _lock.Wait();
+        if (!_connections.TryDequeue(out var connection))
+        {
+            _lock.Release();
+            throw new InvalidOperationException();
+        }
+
+        return new ManagedConnection(connection, this);
+    }
+
+    public void Return(SQLiteDatabaseConnection connection)
+    {
+        _connections.Enqueue(connection);
+        _lock.Release();
+    }
+
+    public void Dispose()
+    {
+        if (_disposed)
+        {
+            return;
+        }
+
+        for (int i = 0; i < _count; i++)
+        {
+            _lock.Wait();
+            if (!_connections.TryDequeue(out var connection))
+            {
+                _lock.Release();
+                throw new InvalidOperationException();
+            }
+
+            connection.Dispose();
+        }
+
+        _lock.Dispose();
+
+        _disposed = true;
+    }
+}

Emby.Server.Implementations/Data/ManagedConnection.cs

@@ -2,23 +2,22 @@
 
 using System;
 using System.Collections.Generic;
-using System.Threading;
 using SQLitePCL.pretty;
 
 namespace Emby.Server.Implementations.Data
 {
     public sealed class ManagedConnection : IDisposable
     {
-        private readonly SemaphoreSlim _writeLock;
+        private readonly ConnectionPool _pool;
 
-        private SQLiteDatabaseConnection? _db;
+        private SQLiteDatabaseConnection _db;
 
         private bool _disposed = false;
 
-        public ManagedConnection(SQLiteDatabaseConnection db, SemaphoreSlim writeLock)
+        public ManagedConnection(SQLiteDatabaseConnection db, ConnectionPool pool)
         {
             _db = db;
-            _writeLock = writeLock;
+            _pool = pool;
         }
 
         public IStatement PrepareStatement(string sql)
@@ -73,9 +72,9 @@ namespace Emby.Server.Implementations.Data
                 return;
             }
 
-            _writeLock.Release();
+            _pool.Return(_db);
 
-            _db = null; // Don't dispose it
+            _db = null!; // Don't dispose it
             _disposed = true;
         }
     }

Emby.Server.Implementations/Data/SqliteItemRepository.cs

@@ -336,6 +336,7 @@ namespace Emby.Server.Implementations.Data
             _jsonOptions = JsonDefaults.Options;
 
             DbFilePath = Path.Combine(_config.ApplicationPaths.DataPath, "library.db");
+            ReadConnectionsCount = 5;
         }
 
         /// <inheritdoc />
@@ -347,10 +348,10 @@ namespace Emby.Server.Implementations.Data
         /// <summary>
         /// Opens the connection to the database.
         /// </summary>
-        /// <param name="userDataRepo">The user data repository.</param>
-        /// <param name="userManager">The user manager.</param>
-        public void Initialize(SqliteUserDataRepository userDataRepo, IUserManager userManager)
+        public override void Initialize()
         {
+            base.Initialize();
+
             const string CreateMediaStreamsTableCommand
                     = "create table if not exists mediastreams (ItemId GUID, StreamIndex INT, StreamType TEXT, Codec TEXT, Language TEXT, ChannelLayout TEXT, Profile TEXT, AspectRatio TEXT, Path TEXT, IsInterlaced BIT, BitRate INT NULL, Channels INT NULL, SampleRate INT NULL, IsDefault BIT, IsForced BIT, IsExternal BIT, Height INT NULL, Width INT NULL, AverageFrameRate FLOAT NULL, RealFrameRate FLOAT NULL, Level FLOAT NULL, PixelFormat TEXT, BitDepth INT NULL, IsAnamorphic BIT NULL, RefFrames INT NULL, CodecTag TEXT NULL, Comment TEXT NULL, NalLengthSize TEXT NULL, IsAvc BIT NULL, Title TEXT NULL, TimeBase TEXT NULL, CodecTimeBase TEXT NULL, ColorPrimaries TEXT NULL, ColorSpace TEXT NULL, ColorTransfer TEXT NULL, DvVersionMajor INT NULL, DvVersionMinor INT NULL, DvProfile INT NULL, DvLevel INT NULL, RpuPresentFlag INT NULL, ElPresentFlag INT NULL, BlPresentFlag INT NULL, DvBlSignalCompatibilityId INT NULL, IsHearingImpaired BIT NULL, PRIMARY KEY (ItemId, StreamIndex))";
 
@@ -551,8 +552,6 @@ namespace Emby.Server.Implementations.Data
 
                 connection.RunQueries(postQueries);
             }
-
-            userDataRepo.Initialize(userManager, WriteLock, WriteConnection);
         }
 
         public void SaveImages(BaseItem item)

Emby.Server.Implementations/Data/SqliteUserDataRepository.cs

@@ -7,7 +7,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using Jellyfin.Data.Entities;
-using MediaBrowser.Common.Configuration;
+using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.Library;
 using MediaBrowser.Controller.Persistence;
@@ -18,33 +18,32 @@ namespace Emby.Server.Implementations.Data
 {
     public class SqliteUserDataRepository : BaseSqliteRepository, IUserDataRepository
     {
+        private readonly IUserManager _userManager;
+
         public SqliteUserDataRepository(
             ILogger<SqliteUserDataRepository> logger,
-            IApplicationPaths appPaths)
+            IServerConfigurationManager config,
+            IUserManager userManager)
             : base(logger)
         {
-            DbFilePath = Path.Combine(appPaths.DataPath, "library.db");
+            _userManager = userManager;
+
+            DbFilePath = Path.Combine(config.ApplicationPaths.DataPath, "library.db");
         }
 
         /// <summary>
         /// Opens the connection to the database.
         /// </summary>
-        /// <param name="userManager">The user manager.</param>
-        /// <param name="dbLock">The lock to use for database IO.</param>
-        /// <param name="dbConnection">The connection to use for database IO.</param>
-        public void Initialize(IUserManager userManager, SemaphoreSlim dbLock, SQLiteDatabaseConnection dbConnection)
+        public override void Initialize()
         {
-            WriteLock.Dispose();
-            WriteLock = dbLock;
-            WriteConnection?.Dispose();
-            WriteConnection = dbConnection;
+            base.Initialize();
 
             using (var connection = GetConnection())
             {
                 var userDatasTableExists = TableExists(connection, "UserDatas");
                 var userDataTableExists = TableExists(connection, "userdata");
 
-                var users = userDatasTableExists ? null : userManager.Users;
+                var users = userDatasTableExists ? null : _userManager.Users;
 
                 connection.RunInTransaction(
                     db =>
@@ -371,20 +370,5 @@ namespace Emby.Server.Implementations.Data
 
             return userData;
         }
-
-#pragma warning disable CA2215
-        /// <inheritdoc/>
-        /// <remarks>
-        /// There is nothing to dispose here since <see cref="BaseSqliteRepository.WriteLock"/> and
-        /// <see cref="BaseSqliteRepository.WriteConnection"/> are managed by <see cref="SqliteItemRepository"/>.
-        /// See <see cref="Initialize(IUserManager, SemaphoreSlim, SQLiteDatabaseConnection)"/>.
-        /// </remarks>
-        protected override void Dispose(bool dispose)
-        {
-            // The write lock and connection for the item repository are shared with the user data repository
-            // since they point to the same database. The item repo has responsibility for disposing these two objects,
-            // so the user data repo should not attempt to dispose them as well
-        }
-#pragma warning restore CA2215
     }
 }