Merge pull request #9643 from Bond-009/sqlconnectionpool

Emby.Server.Implementations/ApplicationHost.cs

@@ -627,6 +627,9 @@ namespace Emby.Server.Implementations
                 }
             }
 
+            ((SqliteItemRepository)Resolve<IItemRepository>()).Initialize();
+            ((SqliteUserDataRepository)Resolve<IUserDataRepository>()).Initialize();
+
             var localizationManager = (LocalizationManager)Resolve<ILocalizationManager>();
             await localizationManager.LoadAll().ConfigureAwait(false);
 
@@ -634,9 +637,6 @@ namespace Emby.Server.Implementations
 
             SetStaticProperties();
 
-            var userDataRepo = (SqliteUserDataRepository)Resolve<IUserDataRepository>();
-            ((SqliteItemRepository)Resolve<IItemRepository>()).Initialize(userDataRepo, Resolve<IUserManager>());
-
             FindParts();
         }
 

Emby.Server.Implementations/Data/BaseSqliteRepository.cs

@@ -4,7 +4,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Threading;
 using Jellyfin.Extensions;
 using Microsoft.Extensions.Logging;
 using SQLitePCL.pretty;
@@ -27,9 +26,19 @@ namespace Emby.Server.Implementations.Data
         /// <summary>
         /// Gets or sets the path to the DB file.
         /// </summary>
-        /// <value>Path to the DB file.</value>
         protected string DbFilePath { get; set; }
 
+        /// <summary>
+        /// Gets or sets the number of write connections to create.
+        /// </summary>
+        /// <value>Path to the DB file.</value>
+        protected int WriteConnectionsCount { get; set; } = 1;
+
+        /// <summary>
+        /// Gets or sets the number of read connections to create.
+        /// </summary>
+        protected int ReadConnectionsCount { get; set; } = 1;
+
         /// <summary>
         /// Gets the logger.
         /// </summary>
@@ -63,7 +72,7 @@ namespace Emby.Server.Implementations.Data
         /// <summary>
         /// Gets the locking mode. <see href="https://www.sqlite.org/pragma.html#pragma_locking_mode" />.
         /// </summary>
-        protected virtual string LockingMode => "EXCLUSIVE";
+        protected virtual string LockingMode => "NORMAL";
 
         /// <summary>
         /// Gets the journal mode. <see href="https://www.sqlite.org/pragma.html#pragma_journal_mode" />.
@@ -88,7 +97,7 @@ namespace Emby.Server.Implementations.Data
         /// </summary>
         /// <value>The temp store mode.</value>
         /// <see cref="TempStoreMode"/>
-        protected virtual TempStoreMode TempStore => TempStoreMode.Default;
+        protected virtual TempStoreMode TempStore => TempStoreMode.Memory;
 
         /// <summary>
         /// Gets the synchronous mode.
@@ -101,63 +110,106 @@ namespace Emby.Server.Implementations.Data
         /// Gets or sets the write lock.
         /// </summary>
         /// <value>The write lock.</value>
-        protected SemaphoreSlim WriteLock { get; set; } = new SemaphoreSlim(1, 1);
+        protected ConnectionPool WriteConnections { get; set; }
 
         /// <summary>
         /// Gets or sets the write connection.
         /// </summary>
         /// <value>The write connection.</value>
-        protected SQLiteDatabaseConnection WriteConnection { get; set; }
+        protected ConnectionPool ReadConnections { get; set; }
 
-        protected ManagedConnection GetConnection(bool readOnly = false)
+        public virtual void Initialize()
         {
-            WriteLock.Wait();
+            WriteConnections = new ConnectionPool(WriteConnectionsCount, CreateWriteConnection);
-            if (WriteConnection is not null)
+            ReadConnections = new ConnectionPool(ReadConnectionsCount, CreateReadConnection);
+
+            // Configuration and pragmas can affect VACUUM so it needs to be last.
+            using (var connection = GetConnection())
             {
-                return new ManagedConnection(WriteConnection, WriteLock);
+                connection.Execute("VACUUM");
             }
+        }
 
-            WriteConnection = SQLite3.Open(
+        protected ManagedConnection GetConnection(bool readOnly = false)
+            => readOnly ? ReadConnections.GetConnection() : WriteConnections.GetConnection();
+
+        protected SQLiteDatabaseConnection CreateWriteConnection()
+        {
+            var writeConnection = SQLite3.Open(
                 DbFilePath,
                 DefaultConnectionFlags | ConnectionFlags.Create | ConnectionFlags.ReadWrite,
                 null);
 
             if (CacheSize.HasValue)
             {
-                WriteConnection.Execute("PRAGMA cache_size=" + CacheSize.Value);
+                writeConnection.Execute("PRAGMA cache_size=" + CacheSize.Value);
             }
 
             if (!string.IsNullOrWhiteSpace(LockingMode))
             {
-                WriteConnection.Execute("PRAGMA locking_mode=" + LockingMode);
+                writeConnection.Execute("PRAGMA locking_mode=" + LockingMode);
             }
 
             if (!string.IsNullOrWhiteSpace(JournalMode))
             {
-                WriteConnection.Execute("PRAGMA journal_mode=" + JournalMode);
+                writeConnection.Execute("PRAGMA journal_mode=" + JournalMode);
             }
 
             if (JournalSizeLimit.HasValue)
             {
-                WriteConnection.Execute("PRAGMA journal_size_limit=" + JournalSizeLimit.Value);
+                writeConnection.Execute("PRAGMA journal_size_limit=" + JournalSizeLimit.Value);
             }
 
             if (Synchronous.HasValue)
             {
-                WriteConnection.Execute("PRAGMA synchronous=" + (int)Synchronous.Value);
+                writeConnection.Execute("PRAGMA synchronous=" + (int)Synchronous.Value);
             }
 
             if (PageSize.HasValue)
             {
-                WriteConnection.Execute("PRAGMA page_size=" + PageSize.Value);
+                writeConnection.Execute("PRAGMA page_size=" + PageSize.Value);
             }
 
-            WriteConnection.Execute("PRAGMA temp_store=" + (int)TempStore);
+            writeConnection.Execute("PRAGMA temp_store=" + (int)TempStore);
 
-            // Configuration and pragmas can affect VACUUM so it needs to be last.
+            return writeConnection;
-            WriteConnection.Execute("VACUUM");
+        }
+
+        protected SQLiteDatabaseConnection CreateReadConnection()
+        {
+            var connection = SQLite3.Open(
+                DbFilePath,
+                DefaultConnectionFlags | ConnectionFlags.ReadOnly,
+                null);
+
+            if (CacheSize.HasValue)
+            {
+                connection.Execute("PRAGMA cache_size=" + CacheSize.Value);
+            }
+
+            if (!string.IsNullOrWhiteSpace(LockingMode))
+            {
+                connection.Execute("PRAGMA locking_mode=" + LockingMode);
+            }
+
+            if (!string.IsNullOrWhiteSpace(JournalMode))
+            {
+                connection.Execute("PRAGMA journal_mode=" + JournalMode);
+            }
+
+            if (JournalSizeLimit.HasValue)
+            {
+                connection.Execute("PRAGMA journal_size_limit=" + JournalSizeLimit.Value);
+            }
+
+            if (Synchronous.HasValue)
+            {
+                connection.Execute("PRAGMA synchronous=" + (int)Synchronous.Value);
+            }
+
+            connection.Execute("PRAGMA temp_store=" + (int)TempStore);
 
-            return new ManagedConnection(WriteConnection, WriteLock);
+            return connection;
         }
 
         public IStatement PrepareStatement(ManagedConnection connection, string sql)
@@ -240,22 +292,10 @@ namespace Emby.Server.Implementations.Data
 
             if (dispose)
             {
-                WriteLock.Wait();
+                WriteConnections.Dispose();
-                try
+                ReadConnections.Dispose();
-                {
-                    WriteConnection?.Dispose();
-                }
-                finally
-                {
-                    WriteLock.Release();
-                }
-
-                WriteLock.Dispose();
             }
 
-            WriteConnection = null;
-            WriteLock = null;
-
             _disposed = true;
         }
     }

Emby.Server.Implementations/Data/ConnectionPool.cs

@@ -0,0 +1,79 @@
+using System;
+using System.Collections.Concurrent;
+using SQLitePCL.pretty;
+
+namespace Emby.Server.Implementations.Data;
+
+/// <summary>
+/// A pool of SQLite Database connections.
+/// </summary>
+public sealed class ConnectionPool : IDisposable
+{
+    private readonly BlockingCollection<SQLiteDatabaseConnection> _connections = new();
+    private bool _disposed;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ConnectionPool" /> class.
+    /// </summary>
+    /// <param name="count">The number of database connection to create.</param>
+    /// <param name="factory">Factory function to create the database connections.</param>
+    public ConnectionPool(int count, Func<SQLiteDatabaseConnection> factory)
+    {
+        for (int i = 0; i < count; i++)
+        {
+            _connections.Add(factory.Invoke());
+        }
+    }
+
+    /// <summary>
+    /// Gets a database connection from the pool if one is available, otherwise blocks.
+    /// </summary>
+    /// <returns>A database connection.</returns>
+    public ManagedConnection GetConnection()
+    {
+        if (_disposed)
+        {
+            ThrowObjectDisposedException();
+        }
+
+        return new ManagedConnection(_connections.Take(), this);
+
+        static void ThrowObjectDisposedException()
+        {
+            throw new ObjectDisposedException(nameof(ConnectionPool));
+        }
+    }
+
+    /// <summary>
+    /// Return a database connection to the pool.
+    /// </summary>
+    /// <param name="connection">The database connection to return.</param>
+    public void Return(SQLiteDatabaseConnection connection)
+    {
+        if (_disposed)
+        {
+            connection.Dispose();
+            return;
+        }
+
+        _connections.Add(connection);
+    }
+
+    /// <inheritdoc />
+    public void Dispose()
+    {
+        if (_disposed)
+        {
+            return;
+        }
+
+        foreach (var connection in _connections)
+        {
+            connection.Dispose();
+        }
+
+        _connections.Dispose();
+
+        _disposed = true;
+    }
+}

Emby.Server.Implementations/Data/ManagedConnection.cs

@@ -2,23 +2,22 @@
 
 using System;
 using System.Collections.Generic;
-using System.Threading;
 using SQLitePCL.pretty;
 
 namespace Emby.Server.Implementations.Data
 {
     public sealed class ManagedConnection : IDisposable
     {
-        private readonly SemaphoreSlim _writeLock;
+        private readonly ConnectionPool _pool;
 
-        private SQLiteDatabaseConnection? _db;
+        private SQLiteDatabaseConnection _db;
 
         private bool _disposed = false;
 
-        public ManagedConnection(SQLiteDatabaseConnection db, SemaphoreSlim writeLock)
+        public ManagedConnection(SQLiteDatabaseConnection db, ConnectionPool pool)
         {
             _db = db;
-            _writeLock = writeLock;
+            _pool = pool;
         }
 
         public IStatement PrepareStatement(string sql)
@@ -73,9 +72,9 @@ namespace Emby.Server.Implementations.Data
                 return;
             }
 
-            _writeLock.Release();
+            _pool.Return(_db);
 
-            _db = null; // Don't dispose it
+            _db = null!; // Don't dispose it
             _disposed = true;
         }
     }

Emby.Server.Implementations/Data/SqliteItemRepository.cs

@@ -336,6 +336,7 @@ namespace Emby.Server.Implementations.Data
             _jsonOptions = JsonDefaults.Options;
 
             DbFilePath = Path.Combine(_config.ApplicationPaths.DataPath, "library.db");
+            ReadConnectionsCount = Environment.ProcessorCount * 2;
         }
 
         /// <inheritdoc />
@@ -347,10 +348,10 @@ namespace Emby.Server.Implementations.Data
         /// <summary>
         /// Opens the connection to the database.
         /// </summary>
-        /// <param name="userDataRepo">The user data repository.</param>
+        public override void Initialize()
-        /// <param name="userManager">The user manager.</param>
-        public void Initialize(SqliteUserDataRepository userDataRepo, IUserManager userManager)
         {
+            base.Initialize();
+
             const string CreateMediaStreamsTableCommand
                     = "create table if not exists mediastreams (ItemId GUID, StreamIndex INT, StreamType TEXT, Codec TEXT, Language TEXT, ChannelLayout TEXT, Profile TEXT, AspectRatio TEXT, Path TEXT, IsInterlaced BIT, BitRate INT NULL, Channels INT NULL, SampleRate INT NULL, IsDefault BIT, IsForced BIT, IsExternal BIT, Height INT NULL, Width INT NULL, AverageFrameRate FLOAT NULL, RealFrameRate FLOAT NULL, Level FLOAT NULL, PixelFormat TEXT, BitDepth INT NULL, IsAnamorphic BIT NULL, RefFrames INT NULL, CodecTag TEXT NULL, Comment TEXT NULL, NalLengthSize TEXT NULL, IsAvc BIT NULL, Title TEXT NULL, TimeBase TEXT NULL, CodecTimeBase TEXT NULL, ColorPrimaries TEXT NULL, ColorSpace TEXT NULL, ColorTransfer TEXT NULL, DvVersionMajor INT NULL, DvVersionMinor INT NULL, DvProfile INT NULL, DvLevel INT NULL, RpuPresentFlag INT NULL, ElPresentFlag INT NULL, BlPresentFlag INT NULL, DvBlSignalCompatibilityId INT NULL, IsHearingImpaired BIT NULL, PRIMARY KEY (ItemId, StreamIndex))";
 
@@ -551,8 +552,6 @@ namespace Emby.Server.Implementations.Data
 
                 connection.RunQueries(postQueries);
             }
-
-            userDataRepo.Initialize(userManager, WriteLock, WriteConnection);
         }
 
         public void SaveImages(BaseItem item)

Emby.Server.Implementations/Data/SqliteUserDataRepository.cs

@@ -7,7 +7,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using Jellyfin.Data.Entities;
-using MediaBrowser.Common.Configuration;
+using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.Library;
 using MediaBrowser.Controller.Persistence;
@@ -18,33 +18,32 @@ namespace Emby.Server.Implementations.Data
 {
     public class SqliteUserDataRepository : BaseSqliteRepository, IUserDataRepository
     {
+        private readonly IUserManager _userManager;
+
         public SqliteUserDataRepository(
             ILogger<SqliteUserDataRepository> logger,
-            IApplicationPaths appPaths)
+            IServerConfigurationManager config,
+            IUserManager userManager)
             : base(logger)
         {
-            DbFilePath = Path.Combine(appPaths.DataPath, "library.db");
+            _userManager = userManager;
+
+            DbFilePath = Path.Combine(config.ApplicationPaths.DataPath, "library.db");
         }
 
         /// <summary>
         /// Opens the connection to the database.
         /// </summary>
-        /// <param name="userManager">The user manager.</param>
+        public override void Initialize()
-        /// <param name="dbLock">The lock to use for database IO.</param>
-        /// <param name="dbConnection">The connection to use for database IO.</param>
-        public void Initialize(IUserManager userManager, SemaphoreSlim dbLock, SQLiteDatabaseConnection dbConnection)
         {
-            WriteLock.Dispose();
+            base.Initialize();
-            WriteLock = dbLock;
-            WriteConnection?.Dispose();
-            WriteConnection = dbConnection;
 
             using (var connection = GetConnection())
             {
                 var userDatasTableExists = TableExists(connection, "UserDatas");
                 var userDataTableExists = TableExists(connection, "userdata");
 
-                var users = userDatasTableExists ? null : userManager.Users;
+                var users = userDatasTableExists ? null : _userManager.Users;
 
                 connection.RunInTransaction(
                     db =>
@@ -371,20 +370,5 @@ namespace Emby.Server.Implementations.Data
 
             return userData;
         }
-
-#pragma warning disable CA2215
-        /// <inheritdoc/>
-        /// <remarks>
-        /// There is nothing to dispose here since <see cref="BaseSqliteRepository.WriteLock"/> and
-        /// <see cref="BaseSqliteRepository.WriteConnection"/> are managed by <see cref="SqliteItemRepository"/>.
-        /// See <see cref="Initialize(IUserManager, SemaphoreSlim, SQLiteDatabaseConnection)"/>.
-        /// </remarks>
-        protected override void Dispose(bool dispose)
-        {
-            // The write lock and connection for the item repository are shared with the user data repository
-            // since they point to the same database. The item repo has responsibility for disposing these two objects,
-            // so the user data repo should not attempt to dispose them as well
-        }
-#pragma warning restore CA2215
     }
 }