diff --git a/MediaBrowser.Controller/Entities/CollectionFolder.cs b/MediaBrowser.Controller/Entities/CollectionFolder.cs
index 9682ea8c21..6d619ce048 100644
--- a/MediaBrowser.Controller/Entities/CollectionFolder.cs
+++ b/MediaBrowser.Controller/Entities/CollectionFolder.cs
@@ -61,7 +61,10 @@ namespace MediaBrowser.Controller.Entities
         {
             //we don't directly validate our children
             //but we do need to clear out the index cache...
-            IndexCache = new ConcurrentDictionary<string, List<BaseItem>>(StringComparer.OrdinalIgnoreCase);
+            if (IndexCache != null)
+            {
+                IndexCache.Clear();
+            }
 
             return NullTaskResult;
         }
@@ -102,7 +105,7 @@ namespace MediaBrowser.Controller.Entities
         /// Our children are actually just references to the ones in the physical root...
         /// </summary>
         /// <value>The actual children.</value>
-        protected override ConcurrentDictionary<Guid, BaseItem> ActualChildren
+        protected override IEnumerable<BaseItem> ActualChildren
         {
             get
             {
@@ -115,16 +118,14 @@ namespace MediaBrowser.Controller.Entities
                 catch (IOException ex)
                 {
                     Logger.ErrorException("Error getting ResolveArgs for {0}", ex, Path);
-                    return new ConcurrentDictionary<Guid, BaseItem>();
+                    return new BaseItem[] { };
                 }
 
-                var ourChildren =
+                return
                     LibraryManager.RootFolder.Children
                     .OfType<Folder>()
                     .Where(i => i.Path != null && locationsDicionary.ContainsKey(i.Path))
                     .SelectMany(c => c.Children);
-
-                return new ConcurrentDictionary<Guid, BaseItem>(ourChildren.ToDictionary(i => i.Id));
             }
         }
     }
diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs
index d14ee22284..80c29f6245 100644
--- a/MediaBrowser.Controller/Entities/Folder.cs
+++ b/MediaBrowser.Controller/Entities/Folder.cs
@@ -99,14 +99,11 @@ namespace MediaBrowser.Controller.Entities
                 item.DateModified = DateTime.UtcNow;
             }
 
-            if (!_children.TryAdd(item.Id, item))
-            {
-                throw new InvalidOperationException("Unable to add " + item.Name);
-            }
+            _children.Add(item);
 
             await LibraryManager.CreateItem(item, cancellationToken).ConfigureAwait(false);
 
-            await ItemRepository.SaveChildren(Id, _children.Values.ToList().Select(i => i.Id), cancellationToken).ConfigureAwait(false);
+            await ItemRepository.SaveChildren(Id, _children.ToList().Select(i => i.Id), cancellationToken).ConfigureAwait(false);
         }
 
         /// <summary>
@@ -135,18 +132,22 @@ namespace MediaBrowser.Controller.Entities
         /// <exception cref="System.InvalidOperationException">Unable to remove  + item.Name</exception>
         public Task RemoveChild(BaseItem item, CancellationToken cancellationToken)
         {
-            BaseItem removed;
+            List<BaseItem> newChildren;
 
-            if (!_children.TryRemove(item.Id, out removed))
+            lock (_childrenSyncLock)
             {
-                throw new InvalidOperationException("Unable to remove " + item.Name);
+                newChildren = _children.ToList();
+
+                newChildren.Remove(item);
+
+                _children = new ConcurrentBag<BaseItem>(newChildren);
             }
 
             item.Parent = null;
 
             LibraryManager.ReportItemRemoved(item);
 
-            return ItemRepository.SaveChildren(Id, _children.Values.ToList().Select(i => i.Id), cancellationToken);
+            return ItemRepository.SaveChildren(Id, newChildren.Select(i => i.Id), cancellationToken);
         }
 
         #region Indexing
@@ -411,9 +412,13 @@ namespace MediaBrowser.Controller.Entities
         /// <returns>IEnumerable{BaseItem}.</returns>
         private IEnumerable<BaseItem> GetIndexedChildren(User user, string indexBy)
         {
-            List<BaseItem> result;
+            List<BaseItem> result = null;
             var cacheKey = user.Name + indexBy;
-            IndexCache.TryGetValue(cacheKey, out result);
+
+            if (IndexCache != null)
+            {
+                IndexCache.TryGetValue(cacheKey, out result);
+            }
 
             if (result == null)
             {
@@ -438,7 +443,7 @@ namespace MediaBrowser.Controller.Entities
         /// <summary>
         /// The index cache
         /// </summary>
-        protected ConcurrentDictionary<string, List<BaseItem>> IndexCache = new ConcurrentDictionary<string, List<BaseItem>>(StringComparer.OrdinalIgnoreCase);
+        protected ConcurrentDictionary<string, List<BaseItem>> IndexCache;
 
         /// <summary>
         /// Builds the index.
@@ -449,6 +454,11 @@ namespace MediaBrowser.Controller.Entities
         /// <returns>List{BaseItem}.</returns>
         protected virtual List<BaseItem> BuildIndex(string indexKey, Func<User, IEnumerable<BaseItem>> indexFunction, User user)
         {
+            if (IndexCache == null)
+            {
+                IndexCache = new ConcurrentDictionary<string, List<BaseItem>>();
+            }
+
             return indexFunction != null
                        ? IndexCache[user.Name + indexKey] = indexFunction(user).ToList()
                        : null;
@@ -459,7 +469,7 @@ namespace MediaBrowser.Controller.Entities
         /// <summary>
         /// The children
         /// </summary>
-        private ConcurrentDictionary<Guid, BaseItem> _children;
+        private ConcurrentBag<BaseItem> _children;
         /// <summary>
         /// The _children initialized
         /// </summary>
@@ -472,22 +482,13 @@ namespace MediaBrowser.Controller.Entities
         /// Gets or sets the actual children.
         /// </summary>
         /// <value>The actual children.</value>
-        protected virtual ConcurrentDictionary<Guid, BaseItem> ActualChildren
+        protected virtual IEnumerable<BaseItem> ActualChildren
         {
             get
             {
                 LazyInitializer.EnsureInitialized(ref _children, ref _childrenInitialized, ref _childrenSyncLock, LoadChildrenInternal);
                 return _children;
             }
-            private set
-            {
-                _children = value;
-
-                if (value == null)
-                {
-                    _childrenInitialized = false;
-                }
-            }
         }
 
         /// <summary>
@@ -497,10 +498,7 @@ namespace MediaBrowser.Controller.Entities
         [IgnoreDataMember]
         public IEnumerable<BaseItem> Children
         {
-            get
-            {
-                return ActualChildren.Values.ToArray();
-            }
+            get { return ActualChildren; }
         }
 
         /// <summary>
@@ -529,9 +527,9 @@ namespace MediaBrowser.Controller.Entities
             }
         }
 
-        private ConcurrentDictionary<Guid, BaseItem> LoadChildrenInternal()
+        private ConcurrentBag<BaseItem> LoadChildrenInternal()
         {
-            return new ConcurrentDictionary<Guid, BaseItem>(LoadChildren().ToDictionary(i => i.Id));
+            return new ConcurrentBag<BaseItem>(LoadChildren());
         }
 
         /// <summary>
@@ -642,7 +640,7 @@ namespace MediaBrowser.Controller.Entities
             progress.Report(5);
 
             //build a dictionary of the current children we have now by Id so we can compare quickly and easily
-            var currentChildren = ActualChildren;
+            var currentChildren = ActualChildren.ToDictionary(i => i.Id);
 
             //create a list for our validated children
             var validChildren = new ConcurrentBag<Tuple<BaseItem, bool>>();
@@ -694,22 +692,14 @@ namespace MediaBrowser.Controller.Entities
                 //that's all the new and changed ones - now see if there are any that are missing
                 var itemsRemoved = currentChildren.Values.Except(newChildren).ToList();
 
+                var actualRemovals = new List<BaseItem>();
+
                 foreach (var item in itemsRemoved)
                 {
                     if (IsRootPathAvailable(item.Path))
                     {
                         item.IsOffline = false;
-
-                        BaseItem removed;
-
-                        if (!_children.TryRemove(item.Id, out removed))
-                        {
-                            Logger.Error("Failed to remove {0}", item.Name);
-                        }
-                        else
-                        {
-                            LibraryManager.ReportItemRemoved(item);
-                        }
+                        actualRemovals.Add(item);
                     }
                     else
                     {
@@ -719,24 +709,30 @@ namespace MediaBrowser.Controller.Entities
                     }
                 }
 
+                if (actualRemovals.Count > 0)
+                {
+                    lock (_childrenSyncLock)
+                    {
+                        _children = new ConcurrentBag<BaseItem>(_children.Except(actualRemovals));
+                    }
+                }
+
                 await LibraryManager.CreateItems(newItems, cancellationToken).ConfigureAwait(false);
 
                 foreach (var item in newItems)
                 {
-                    if (!_children.TryAdd(item.Id, item))
-                    {
-                        Logger.Error("Failed to add {0}", item.Name);
-                    }
-                    else
-                    {
-                        Logger.Debug("** " + item.Name + " Added to library.");
-                    }
+                    _children.Add(item);
+
+                    Logger.Debug("** " + item.Name + " Added to library.");
                 }
 
-                await ItemRepository.SaveChildren(Id, _children.Values.ToList().Select(i => i.Id), cancellationToken).ConfigureAwait(false);
+                await ItemRepository.SaveChildren(Id, _children.ToList().Select(i => i.Id), cancellationToken).ConfigureAwait(false);
 
                 //force the indexes to rebuild next time
-                IndexCache.Clear();
+                if (IndexCache != null)
+                {
+                    IndexCache.Clear();
+                }
             }
 
             progress.Report(10);
@@ -949,7 +945,7 @@ namespace MediaBrowser.Controller.Entities
             }
 
             // If indexed is false or the indexing function is null
-            return children.Where(c => c.IsVisible(user));
+            return children.AsParallel().Where(c => c.IsVisible(user)).AsEnumerable();
         }
 
         /// <summary>
@@ -970,7 +966,7 @@ namespace MediaBrowser.Controller.Entities
 
             if (includeLinkedChildren)
             {
-                children = children.DistinctBy(i => i.Id);
+                children = children.Distinct();
             }
 
             return children;
@@ -1028,7 +1024,7 @@ namespace MediaBrowser.Controller.Entities
             {
                 throw new ArgumentException("Encountered linked child with empty path.");
             }
-                    
+
             var item = LibraryManager.RootFolder.FindByPath(info.Path);
 
             if (item == null)
diff --git a/MediaBrowser.Server.Implementations/Library/LibraryManager.cs b/MediaBrowser.Server.Implementations/Library/LibraryManager.cs
index 9b78f39805..27d6953d79 100644
--- a/MediaBrowser.Server.Implementations/Library/LibraryManager.cs
+++ b/MediaBrowser.Server.Implementations/Library/LibraryManager.cs
@@ -341,7 +341,7 @@ namespace MediaBrowser.Server.Implementations.Library
 
             items.Add(RootFolder);
 
-            // Need to use DistinctBy Id because there could be multiple instances with the same id
+            // Need to use Distinct because there could be multiple instances with the same id
             // due to sharing the default library
             var userRootFolders = _userManager.Users.Select(i => i.RootFolder)
                 .Distinct()
@@ -357,9 +357,14 @@ namespace MediaBrowser.Server.Implementations.Library
 
             items.AddRange(userFolders);
 
-            var disctinctItems = items.DistinctBy(i => i.Id).ToList();
+            var dictionary = new ConcurrentDictionary<Guid, BaseItem>();
 
-            return new ConcurrentDictionary<Guid, BaseItem>(disctinctItems.ToDictionary(i => i.Id));
+            foreach (var item in items)
+            {
+                dictionary[item.Id] = item;
+            }
+
+            return dictionary;
         }
 
         /// <summary>
diff --git a/MediaBrowser.Server.Implementations/Library/Validators/ArtistsValidator.cs b/MediaBrowser.Server.Implementations/Library/Validators/ArtistsValidator.cs
index e3faea0719..767df9c796 100644
--- a/MediaBrowser.Server.Implementations/Library/Validators/ArtistsValidator.cs
+++ b/MediaBrowser.Server.Implementations/Library/Validators/ArtistsValidator.cs
@@ -72,6 +72,12 @@ namespace MediaBrowser.Server.Implementations.Library.Validators
 
             var numComplete = 0;
 
+            var userLibraries = _userManager.Users
+                .Select(i => new Tuple<Guid, IHasArtist[]>(i.Id, i.RootFolder.GetRecursiveChildren(i).OfType<IHasArtist>().ToArray()))
+                .ToArray();
+
+            var numArtists = allArtists.Count;
+
             foreach (var artist in allArtists)
             {
                 cancellationToken.ThrowIfCancellationRequested();
@@ -106,14 +112,14 @@ namespace MediaBrowser.Server.Implementations.Library.Validators
                 // Populate counts of items
                 //SetItemCounts(artist, null, allItems.OfType<IHasArtist>());
 
-                foreach (var user in _userManager.Users.ToArray())
+                foreach (var lib in userLibraries)
                 {
-                    SetItemCounts(artist, user.Id, user.RootFolder.GetRecursiveChildren(user).OfType<IHasArtist>().ToArray());
+                    SetItemCounts(artist, lib.Item1, lib.Item2);
                 }
 
                 numComplete++;
                 double percent = numComplete;
-                percent /= allArtists.Length;
+                percent /= numArtists;
                 percent *= 20;
 
                 progress.Report(80 + percent);
@@ -180,7 +186,7 @@ namespace MediaBrowser.Server.Implementations.Library.Validators
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <param name="progress">The progress.</param>
         /// <returns>Task{Artist[]}.</returns>
-        private async Task<Artist[]> GetAllArtists(IEnumerable<Audio> allSongs, CancellationToken cancellationToken, IProgress<double> progress)
+        private async Task<ConcurrentBag<Artist>> GetAllArtists(IEnumerable<Audio> allSongs, CancellationToken cancellationToken, IProgress<double> progress)
         {
             var allArtists = allSongs
                 .SelectMany(i =>
@@ -251,7 +257,7 @@ namespace MediaBrowser.Server.Implementations.Library.Validators
 
             await Task.WhenAll(tasks).ConfigureAwait(false);
 
-            return returnArtists.ToArray();
+            return returnArtists;
         }
 
         /// <summary>