New: Import list clean library option

Closes #5201

frontend/src/App/State/SettingsAppState.ts

@@ -7,6 +7,7 @@ import AppSectionState, {
 import Language from 'Language/Language';
 import DownloadClient from 'typings/DownloadClient';
 import ImportList from 'typings/ImportList';
+import ImportListOptionsSettings from 'typings/ImportListOptionsSettings';
 import Indexer from 'typings/Indexer';
 import Notification from 'typings/Notification';
 import QualityProfile from 'typings/QualityProfile';
@@ -35,10 +36,15 @@ export interface QualityProfilesAppState
   extends AppSectionState<QualityProfile>,
     AppSectionSchemaState<QualityProfile> {}
 
+export interface ImportListOptionsSettingsAppState
+  extends AppSectionItemState<ImportListOptionsSettings>,
+    AppSectionSaveState {}
+
 export type LanguageSettingsAppState = AppSectionState<Language>;
 export type UiSettingsAppState = AppSectionItemState<UiSettings>;
 
 interface SettingsAppState {
+  advancedSettings: boolean;
   downloadClients: DownloadClientAppState;
   importLists: ImportListAppState;
   indexers: IndexerAppState;
@@ -46,6 +52,7 @@ interface SettingsAppState {
   notifications: NotificationAppState;
   qualityProfiles: QualityProfilesAppState;
   ui: UiSettingsAppState;
+  importListOptions: ImportListOptionsSettingsAppState;
 }
 
 export default SettingsAppState;

frontend/src/Settings/ImportLists/ImportListSettings.js

@@ -10,6 +10,7 @@ import translate from 'Utilities/String/translate';
 import ImportListsExclusionsConnector from './ImportListExclusions/ImportListExclusionsConnector';
 import ImportListsConnector from './ImportLists/ImportListsConnector';
 import ManageImportListsModal from './ImportLists/Manage/ManageImportListsModal';
+import ImportListOptions from './Options/ImportListOptions';
 
 class ImportListSettings extends Component {
 
@@ -19,7 +20,10 @@ class ImportListSettings extends Component {
   constructor(props, context) {
     super(props, context);
 
+    this._saveCallback = null;
+
     this.state = {
+      isSaving: false,
       hasPendingChanges: false,
       isManageImportListsOpen: false
     };
@@ -28,6 +32,14 @@ class ImportListSettings extends Component {
   //
   // Listeners
 
+  setChildSave = (saveCallback) => {
+    this._saveCallback = saveCallback;
+  };
+
+  onChildStateChange = (payload) => {
+    this.setState(payload);
+  };
+
   setListOptionsRef = (ref) => {
     this._listOptions = ref;
   };
@@ -47,7 +59,9 @@ class ImportListSettings extends Component {
   };
 
   onSavePress = () => {
-    this._listOptions.getWrappedInstance().save();
+    if (this._saveCallback) {
+      this._saveCallback();
+    }
   };
 
   //
@@ -93,6 +107,12 @@ class ImportListSettings extends Component {
 
         <PageContentBody>
           <ImportListsConnector />
+
+          <ImportListOptions
+            setChildSave={this.setChildSave}
+            onChildStateChange={this.onChildStateChange}
+          />
+
           <ImportListsExclusionsConnector />
           <ManageImportListsModal
             isOpen={isManageImportListsOpen}

frontend/src/Settings/ImportLists/Options/ImportListOptions.tsx

@@ -0,0 +1,148 @@
+import React, { useCallback, useEffect } from 'react';
+import { useDispatch, useSelector } from 'react-redux';
+import { createSelector } from 'reselect';
+import AppState from 'App/State/AppState';
+import FieldSet from 'Components/FieldSet';
+import Form from 'Components/Form/Form';
+import FormGroup from 'Components/Form/FormGroup';
+import FormInputGroup from 'Components/Form/FormInputGroup';
+import FormLabel from 'Components/Form/FormLabel';
+import LoadingIndicator from 'Components/Loading/LoadingIndicator';
+import { inputTypes } from 'Helpers/Props';
+import { clearPendingChanges } from 'Store/Actions/baseActions';
+import {
+  fetchImportListOptions,
+  saveImportListOptions,
+  setImportListOptionsValue,
+} from 'Store/Actions/settingsActions';
+import createSettingsSectionSelector from 'Store/Selectors/createSettingsSectionSelector';
+import translate from 'Utilities/String/translate';
+
+const SECTION = 'importListOptions';
+const cleanLibraryLevelOptions = [
+  { key: 'disabled', value: () => translate('Disabled') },
+  { key: 'logOnly', value: () => translate('LogOnly') },
+  { key: 'keepAndUnmonitor', value: () => translate('KeepAndUnmonitorSeries') },
+  { key: 'keepAndTag', value: () => translate('KeepAndTagSeries') },
+];
+
+function createImportListOptionsSelector() {
+  return createSelector(
+    (state: AppState) => state.settings.advancedSettings,
+    createSettingsSectionSelector(SECTION),
+    (advancedSettings, sectionSettings) => {
+      return {
+        advancedSettings,
+        save: sectionSettings.isSaving,
+        ...sectionSettings,
+      };
+    }
+  );
+}
+
+interface ImportListOptionsPageProps {
+  setChildSave(saveCallback: () => void): void;
+  onChildStateChange(payload: unknown): void;
+}
+
+function ImportListOptions(props: ImportListOptionsPageProps) {
+  const { setChildSave, onChildStateChange } = props;
+  const selected = useSelector(createImportListOptionsSelector());
+
+  const {
+    isSaving,
+    hasPendingChanges,
+    advancedSettings,
+    isFetching,
+    error,
+    settings,
+    hasSettings,
+  } = selected;
+
+  const { listSyncLevel, listSyncTag } = settings;
+
+  const dispatch = useDispatch();
+
+  const onInputChange = useCallback(
+    ({ name, value }: { name: string; value: unknown }) => {
+      // @ts-expect-error 'setImportListOptionsValue' isn't typed yet
+      dispatch(setImportListOptionsValue({ name, value }));
+    },
+    [dispatch]
+  );
+
+  const onTagChange = useCallback(
+    ({ name, value }: { name: string; value: number[] }) => {
+      const id = value.length === 0 ? 0 : value.pop();
+      // @ts-expect-error 'setImportListOptionsValue' isn't typed yet
+      dispatch(setImportListOptionsValue({ name, value: id }));
+    },
+    [dispatch]
+  );
+
+  useEffect(() => {
+    dispatch(fetchImportListOptions());
+    setChildSave(() => dispatch(saveImportListOptions()));
+
+    return () => {
+      dispatch(clearPendingChanges({ section: SECTION }));
+    };
+  }, [dispatch, setChildSave]);
+
+  useEffect(() => {
+    onChildStateChange({
+      isSaving,
+      hasPendingChanges,
+    });
+  }, [onChildStateChange, isSaving, hasPendingChanges]);
+
+  const translatedLevelOptions = cleanLibraryLevelOptions.map(
+    ({ key, value }) => {
+      return {
+        key,
+        value: value(),
+      };
+    }
+  );
+
+  return advancedSettings ? (
+    <FieldSet legend={translate('Options')}>
+      {isFetching ? <LoadingIndicator /> : null}
+
+      {!isFetching && error ? (
+        <div>{translate('UnableToLoadListOptions')}</div>
+      ) : null}
+
+      {hasSettings && !isFetching && !error ? (
+        <Form>
+          <FormGroup advancedSettings={advancedSettings} isAdvanced={true}>
+            <FormLabel>{translate('CleanLibraryLevel')}</FormLabel>
+            <FormInputGroup
+              type={inputTypes.SELECT}
+              name="listSyncLevel"
+              values={translatedLevelOptions}
+              helpText={translate('ListSyncLevelHelpText')}
+              onChange={onInputChange}
+              {...listSyncLevel}
+            />
+          </FormGroup>
+          {listSyncLevel.value === 'keepAndTag' ? (
+            <FormGroup advancedSettings={advancedSettings} isAdvanced={true}>
+              <FormLabel>{translate('ListSyncTag')}</FormLabel>
+              <FormInputGroup
+                {...listSyncTag}
+                type={inputTypes.TAG}
+                name="listSyncTag"
+                value={listSyncTag.value === 0 ? [] : [listSyncTag.value]}
+                helpText={translate('ListSyncTagHelpText')}
+                onChange={onTagChange}
+              />
+            </FormGroup>
+          ) : null}
+        </Form>
+      ) : null}
+    </FieldSet>
+  ) : null;
+}
+
+export default ImportListOptions;

frontend/src/Store/Actions/Settings/importListOptions.js

@@ -0,0 +1,64 @@
+import { createAction } from 'redux-actions';
+import createFetchHandler from 'Store/Actions/Creators/createFetchHandler';
+import createSaveHandler from 'Store/Actions/Creators/createSaveHandler';
+import createSetSettingValueReducer from 'Store/Actions/Creators/Reducers/createSetSettingValueReducer';
+import { createThunk } from 'Store/thunks';
+
+//
+// Variables
+
+const section = 'settings.importListOptions';
+
+//
+// Actions Types
+
+export const FETCH_IMPORT_LIST_OPTIONS = 'settings/importListOptions/fetchImportListOptions';
+export const SAVE_IMPORT_LIST_OPTIONS = 'settings/importListOptions/saveImportListOptions';
+export const SET_IMPORT_LIST_OPTIONS_VALUE = 'settings/importListOptions/setImportListOptionsValue';
+
+//
+// Action Creators
+
+export const fetchImportListOptions = createThunk(FETCH_IMPORT_LIST_OPTIONS);
+export const saveImportListOptions = createThunk(SAVE_IMPORT_LIST_OPTIONS);
+export const setImportListOptionsValue = createAction(SET_IMPORT_LIST_OPTIONS_VALUE, (payload) => {
+  return {
+    section,
+    ...payload
+  };
+});
+
+//
+// Details
+
+export default {
+
+  //
+  // State
+
+  defaultState: {
+    isFetching: false,
+    isPopulated: false,
+    error: null,
+    pendingChanges: {},
+    isSaving: false,
+    saveError: null,
+    item: {}
+  },
+
+  //
+  // Action Handlers
+
+  actionHandlers: {
+    [FETCH_IMPORT_LIST_OPTIONS]: createFetchHandler(section, '/config/importlist'),
+    [SAVE_IMPORT_LIST_OPTIONS]: createSaveHandler(section, '/config/importlist')
+  },
+
+  //
+  // Reducers
+
+  reducers: {
+    [SET_IMPORT_LIST_OPTIONS_VALUE]: createSetSettingValueReducer(section)
+  }
+
+};

frontend/src/Store/Actions/settingsActions.js

@@ -10,6 +10,7 @@ import downloadClientOptions from './Settings/downloadClientOptions';
 import downloadClients from './Settings/downloadClients';
 import general from './Settings/general';
 import importListExclusions from './Settings/importListExclusions';
+import importListOptions from './Settings/importListOptions';
 import importLists from './Settings/importLists';
 import indexerOptions from './Settings/indexerOptions';
 import indexers from './Settings/indexers';
@@ -33,6 +34,7 @@ export * from './Settings/delayProfiles';
 export * from './Settings/downloadClients';
 export * from './Settings/downloadClientOptions';
 export * from './Settings/general';
+export * from './Settings/importListOptions';
 export * from './Settings/importLists';
 export * from './Settings/importListExclusions';
 export * from './Settings/indexerOptions';
@@ -69,6 +71,7 @@ export const defaultState = {
   general: general.defaultState,
   importLists: importLists.defaultState,
   importListExclusions: importListExclusions.defaultState,
+  importListOptions: importListOptions.defaultState,
   indexerOptions: indexerOptions.defaultState,
   indexers: indexers.defaultState,
   languages: languages.defaultState,
@@ -112,6 +115,7 @@ export const actionHandlers = handleThunks({
   ...general.actionHandlers,
   ...importLists.actionHandlers,
   ...importListExclusions.actionHandlers,
+  ...importListOptions.actionHandlers,
   ...indexerOptions.actionHandlers,
   ...indexers.actionHandlers,
   ...languages.actionHandlers,
@@ -146,6 +150,7 @@ export const reducers = createHandleActions({
   ...general.reducers,
   ...importLists.reducers,
   ...importListExclusions.reducers,
+  ...importListOptions.reducers,
   ...indexerOptions.reducers,
   ...indexers.reducers,
   ...languages.reducers,

frontend/src/Store/Selectors/createSettingsSectionSelector.js

@@ -1,32 +0,0 @@
-import { createSelector } from 'reselect';
-import selectSettings from 'Store/Selectors/selectSettings';
-
-function createSettingsSectionSelector(section) {
-  return createSelector(
-    (state) => state.settings[section],
-    (sectionSettings) => {
-      const {
-        isFetching,
-        isPopulated,
-        error,
-        item,
-        pendingChanges,
-        isSaving,
-        saveError
-      } = sectionSettings;
-
-      const settings = selectSettings(item, pendingChanges, saveError);
-
-      return {
-        isFetching,
-        isPopulated,
-        error,
-        isSaving,
-        saveError,
-        ...settings
-      };
-    }
-  );
-}
-
-export default createSettingsSectionSelector;

frontend/src/Store/Selectors/createSettingsSectionSelector.ts

@@ -0,0 +1,49 @@
+import { createSelector } from 'reselect';
+import AppSectionState, {
+  AppSectionItemState,
+} from 'App/State/AppSectionState';
+import AppState from 'App/State/AppState';
+import selectSettings from 'Store/Selectors/selectSettings';
+import { PendingSection } from 'typings/pending';
+
+type SettingNames = keyof Omit<AppState['settings'], 'advancedSettings'>;
+type GetSectionState<Name extends SettingNames> = AppState['settings'][Name];
+type GetSettingsSectionItemType<Name extends SettingNames> =
+  GetSectionState<Name> extends AppSectionItemState<infer R>
+    ? R
+    : GetSectionState<Name> extends AppSectionState<infer R>
+    ? R
+    : never;
+
+type AppStateWithPending<Name extends SettingNames> = {
+  item?: GetSettingsSectionItemType<Name>;
+  pendingChanges?: Partial<GetSettingsSectionItemType<Name>>;
+  saveError?: Error;
+} & GetSectionState<Name>;
+
+function createSettingsSectionSelector<Name extends SettingNames>(
+  section: Name
+) {
+  return createSelector(
+    (state: AppState) => state.settings[section],
+    (sectionSettings) => {
+      const { item, pendingChanges, saveError, ...other } =
+        sectionSettings as AppStateWithPending<Name>;
+
+      const { settings, ...rest } = selectSettings(
+        item,
+        pendingChanges,
+        saveError
+      );
+
+      return {
+        ...other,
+        saveError,
+        settings: settings as PendingSection<GetSettingsSectionItemType<Name>>,
+        ...rest,
+      };
+    }
+  );
+}
+
+export default createSettingsSectionSelector;

frontend/src/typings/ImportListOptionsSettings.ts

@@ -0,0 +1,10 @@
+export type ListSyncLevel =
+  | 'disabled'
+  | 'logOnly'
+  | 'keepAndUnmonitor'
+  | 'keepAndTag';
+
+export default interface ImportListOptionsSettings {
+  listSyncLevel: ListSyncLevel;
+  listSyncTag: number;
+}

frontend/src/typings/pending.ts

@@ -0,0 +1,9 @@
+export interface Pending<T> {
+  value: T;
+  errors: any[];
+  warnings: any[];
+}
+
+export type PendingSection<T> = {
+  [K in keyof T]: Pending<T[K]>;
+};

src/NzbDrone.Core.Test/ImportListTests/FetchAndParseImportListFixture.cs

@@ -0,0 +1,219 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using FizzWare.NBuilder;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.ImportLists;
+using NzbDrone.Core.ImportLists.ImportListItems;
+using NzbDrone.Core.MetadataSource;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.ImportListTests
+{
+    [TestFixture]
+    public class FetchAndParseImportListServiceFixture : CoreTest<FetchAndParseImportListService>
+    {
+        private List<IImportList> _importLists;
+        private List<ImportListItemInfo> _listSeries;
+
+        [SetUp]
+        public void Setup()
+        {
+            _importLists = new List<IImportList>();
+
+            Mocker.GetMock<IImportListFactory>()
+                  .Setup(v => v.AutomaticAddEnabled(It.IsAny<bool>()))
+                  .Returns(_importLists);
+
+            _listSeries = Builder<ImportListItemInfo>.CreateListOfSize(5)
+                .Build().ToList();
+
+            Mocker.GetMock<ISearchForNewSeries>()
+                .Setup(v => v.SearchForNewSeriesByImdbId(It.IsAny<string>()))
+                .Returns((string value) => new List<Tv.Series>() { new Tv.Series() { ImdbId = value } });
+        }
+
+        private Mock<IImportList> WithList(int id, bool enabled, bool enabledAuto, ImportListFetchResult fetchResult, TimeSpan? minRefresh = null, int? lastSyncOffset = null, int? syncDeletedCount = null)
+        {
+            return CreateListResult(id, enabled, enabledAuto, fetchResult, minRefresh, lastSyncOffset, syncDeletedCount);
+        }
+
+        private Mock<IImportList> CreateListResult(int id, bool enabled, bool enabledAuto, ImportListFetchResult fetchResult, TimeSpan? minRefresh = null, int? lastSyncOffset = null, int? syncDeletedCount = null)
+        {
+            var refreshInterval = minRefresh ?? TimeSpan.FromHours(12);
+            var importListDefinition = new ImportListDefinition { Id = id, Enable = enabled, EnableAutomaticAdd = enabledAuto, MinRefreshInterval = refreshInterval };
+
+            var mockImportList = new Mock<IImportList>();
+            mockImportList.SetupGet(s => s.Definition).Returns(importListDefinition);
+            mockImportList.Setup(s => s.Fetch()).Returns(fetchResult);
+            mockImportList.SetupGet(s => s.MinRefreshInterval).Returns(refreshInterval);
+
+            DateTime? lastSync = lastSyncOffset.HasValue ? DateTime.UtcNow.AddHours(lastSyncOffset.Value) : null;
+            Mocker.GetMock<IImportListStatusService>()
+                .Setup(v => v.GetListStatus(id))
+                .Returns(new ImportListStatus() { LastInfoSync = lastSync });
+
+            if (syncDeletedCount.HasValue)
+            {
+                Mocker.GetMock<IImportListItemService>()
+                    .Setup(v => v.SyncSeriesForList(It.IsAny<List<ImportListItemInfo>>(), id))
+                    .Returns(syncDeletedCount.Value);
+            }
+
+            _importLists.Add(mockImportList.Object);
+
+            return mockImportList;
+        }
+
+        [Test]
+        public void should_skip_recently_fetched_list()
+        {
+            var fetchResult = new ImportListFetchResult();
+            var list = WithList(1, true, true, fetchResult, lastSyncOffset: 0);
+
+            var result = Subject.Fetch();
+
+            list.Verify(f => f.Fetch(), Times.Never());
+            result.Series.Count.Should().Be(0);
+            result.AnyFailure.Should().BeFalse();
+        }
+
+        [Test]
+        public void should_skip_recent_and_fetch_good()
+        {
+            var fetchResult = new ImportListFetchResult();
+            var recent = WithList(1, true, true, fetchResult, lastSyncOffset: 0);
+            var old = WithList(2, true, true, fetchResult);
+
+            var result = Subject.Fetch();
+
+            recent.Verify(f => f.Fetch(), Times.Never());
+            old.Verify(f => f.Fetch(), Times.Once());
+            result.AnyFailure.Should().BeFalse();
+        }
+
+        [Test]
+        public void should_return_failure_if_single_list_fails()
+        {
+            var fetchResult = new ImportListFetchResult { Series = _listSeries, AnyFailure = true };
+            WithList(1, true, true, fetchResult);
+
+            var listResult = Subject.Fetch();
+            listResult.AnyFailure.Should().BeTrue();
+
+            Mocker.GetMock<IImportListStatusService>()
+                .Verify(v => v.UpdateListSyncStatus(It.IsAny<int>(), It.IsAny<bool>()), Times.Never());
+        }
+
+        [Test]
+        public void should_return_failure_if_any_list_fails()
+        {
+            var fetchResult1 = new ImportListFetchResult { Series = _listSeries, AnyFailure = true };
+            WithList(1, true, true, fetchResult1);
+            var fetchResult2 = new ImportListFetchResult { Series = _listSeries, AnyFailure = false };
+            WithList(2, true, true, fetchResult2);
+
+            var listResult = Subject.Fetch();
+            listResult.AnyFailure.Should().BeTrue();
+        }
+
+        [Test]
+        public void should_return_early_if_no_available_lists()
+        {
+            var listResult = Subject.Fetch();
+
+            Mocker.GetMock<IImportListStatusService>()
+                  .Verify(v => v.GetListStatus(It.IsAny<int>()), Times.Never());
+
+            listResult.Series.Count.Should().Be(0);
+            listResult.AnyFailure.Should().BeFalse();
+        }
+
+        [Test]
+        public void should_store_series_if_list_doesnt_fail()
+        {
+            var listId = 1;
+            var fetchResult = new ImportListFetchResult { Series = _listSeries, AnyFailure = false };
+            WithList(listId, true, true, fetchResult);
+
+            var listResult = Subject.Fetch();
+            listResult.AnyFailure.Should().BeFalse();
+
+            Mocker.GetMock<IImportListStatusService>()
+                .Verify(v => v.UpdateListSyncStatus(listId, false), Times.Once());
+            Mocker.GetMock<IImportListItemService>()
+                .Verify(v => v.SyncSeriesForList(_listSeries, listId), Times.Once());
+        }
+
+        [Test]
+        public void should_not_store_series_if_list_fails()
+        {
+            var listId = 1;
+            var fetchResult = new ImportListFetchResult { Series = _listSeries, AnyFailure = true };
+            WithList(listId, true, true, fetchResult);
+
+            var listResult = Subject.Fetch();
+            listResult.AnyFailure.Should().BeTrue();
+
+            Mocker.GetMock<IImportListStatusService>()
+                .Verify(v => v.UpdateListSyncStatus(listId, false), Times.Never());
+            Mocker.GetMock<IImportListItemService>()
+                .Verify(v => v.SyncSeriesForList(It.IsAny<List<ImportListItemInfo>>(), listId), Times.Never());
+        }
+
+        [Test]
+        public void should_only_store_series_for_lists_that_dont_fail()
+        {
+            var passedListId = 1;
+            var fetchResult1 = new ImportListFetchResult { Series = _listSeries, AnyFailure = false };
+            WithList(passedListId, true, true, fetchResult1);
+            var failedListId = 2;
+            var fetchResult2 = new ImportListFetchResult { Series = _listSeries, AnyFailure = true };
+            WithList(failedListId, true, true, fetchResult2);
+
+            var listResult = Subject.Fetch();
+            listResult.AnyFailure.Should().BeTrue();
+
+            Mocker.GetMock<IImportListStatusService>()
+                .Verify(v => v.UpdateListSyncStatus(passedListId, false), Times.Once());
+            Mocker.GetMock<IImportListItemService>()
+                .Verify(v => v.SyncSeriesForList(_listSeries, passedListId), Times.Once());
+            Mocker.GetMock<IImportListStatusService>()
+                .Verify(v => v.UpdateListSyncStatus(failedListId, false), Times.Never());
+            Mocker.GetMock<IImportListItemService>()
+                .Verify(v => v.SyncSeriesForList(It.IsAny<List<ImportListItemInfo>>(), failedListId), Times.Never());
+        }
+
+        [Test]
+        public void should_return_all_results_for_all_lists()
+        {
+            var passedListId = 1;
+            var fetchResult1 = new ImportListFetchResult { Series = _listSeries, AnyFailure = false };
+            WithList(passedListId, true, true, fetchResult1);
+            var secondListId = 2;
+            var fetchResult2 = new ImportListFetchResult { Series = _listSeries, AnyFailure = false };
+            WithList(secondListId, true, true, fetchResult2);
+
+            var listResult = Subject.Fetch();
+            listResult.AnyFailure.Should().BeFalse();
+            listResult.Series.Count.Should().Be(5);
+        }
+
+        [Test]
+        public void should_set_removed_flag_if_list_has_removed_items()
+        {
+            var listId = 1;
+            var fetchResult = new ImportListFetchResult { Series = _listSeries, AnyFailure = false };
+            WithList(listId, true, true, fetchResult, syncDeletedCount: 500);
+
+            var result = Subject.Fetch();
+            result.AnyFailure.Should().BeFalse();
+
+            Mocker.GetMock<IImportListStatusService>()
+                .Verify(v => v.UpdateListSyncStatus(listId, true), Times.Once());
+        }
+    }
+}

src/NzbDrone.Core.Test/ImportListTests/ImportListItemServiceFixture.cs

@@ -0,0 +1,57 @@
+using System.Collections.Generic;
+using System.Linq;
+using FizzWare.NBuilder;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.ImportLists.ImportListItems;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.ImportListTests
+{
+    public class ImportListItemServiceFixture : CoreTest<ImportListItemService>
+    {
+        [SetUp]
+        public void SetUp()
+        {
+            var existing = Builder<ImportListItemInfo>.CreateListOfSize(3)
+                .TheFirst(1)
+                .With(s => s.TvdbId = 6)
+                .With(s => s.ImdbId = "6")
+                .TheNext(1)
+                .With(s => s.TvdbId = 7)
+                .With(s => s.ImdbId = "7")
+                .TheNext(1)
+                .With(s => s.TvdbId = 8)
+                .With(s => s.ImdbId = "8")
+                .Build().ToList();
+            Mocker.GetMock<IImportListItemInfoRepository>()
+                .Setup(v => v.GetAllForLists(It.IsAny<List<int>>()))
+                .Returns(existing);
+        }
+
+        [Test]
+        public void should_insert_new_update_existing_and_delete_missing()
+        {
+            var newItems = Builder<ImportListItemInfo>.CreateListOfSize(3)
+                .TheFirst(1)
+                .With(s => s.TvdbId = 5)
+                .TheNext(1)
+                .With(s => s.TvdbId = 6)
+                .TheNext(1)
+                .With(s => s.TvdbId = 7)
+                .Build().ToList();
+
+            var numDeleted = Subject.SyncSeriesForList(newItems, 1);
+
+            numDeleted.Should().Be(1);
+            Mocker.GetMock<IImportListItemInfoRepository>()
+                .Verify(v => v.InsertMany(It.Is<List<ImportListItemInfo>>(s => s.Count == 1 && s[0].TvdbId == 5)), Times.Once());
+            Mocker.GetMock<IImportListItemInfoRepository>()
+                .Verify(v => v.UpdateMany(It.Is<List<ImportListItemInfo>>(s => s.Count == 2 && s[0].TvdbId == 6 && s[1].TvdbId == 7)), Times.Once());
+            Mocker.GetMock<IImportListItemInfoRepository>()
+                .Verify(v => v.DeleteMany(It.Is<List<ImportListItemInfo>>(s => s.Count == 1 && s[0].TvdbId == 8)), Times.Once());
+        }
+    }
+}

src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs

@@ -1,9 +1,13 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
+using FizzWare.NBuilder;
 using Moq;
 using NUnit.Framework;
+using NzbDrone.Core.Configuration;
 using NzbDrone.Core.ImportLists;
 using NzbDrone.Core.ImportLists.Exclusions;
+using NzbDrone.Core.ImportLists.ImportListItems;
 using NzbDrone.Core.MetadataSource;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.Test.Framework;
@@ -13,17 +17,61 @@ namespace NzbDrone.Core.Test.ImportListTests
 {
     public class ImportListSyncServiceFixture : CoreTest<ImportListSyncService>
     {
-        private List<ImportListItemInfo> _importListReports;
+        private ImportListFetchResult _importListFetch;
+        private List<ImportListItemInfo> _list1Series;
+        private List<ImportListItemInfo> _list2Series;
+
+        private List<Series> _existingSeries;
+        private List<IImportList> _importLists;
+        private ImportListSyncCommand _commandAll;
+        private ImportListSyncCommand _commandSingle;
 
         [SetUp]
         public void SetUp()
         {
-            var importListItem1 = new ImportListItemInfo
+            _importLists = new List<IImportList>();
+
+            var item1 = new ImportListItemInfo()
             {
                 Title = "Breaking Bad"
             };
 
-            _importListReports = new List<ImportListItemInfo> { importListItem1 };
+            _list1Series = new List<ImportListItemInfo>() { item1 };
+
+            _existingSeries = Builder<Series>.CreateListOfSize(3)
+                .TheFirst(1)
+                .With(s => s.TvdbId = 6)
+                .With(s => s.ImdbId = "6")
+                .TheNext(1)
+                .With(s => s.TvdbId = 7)
+                .With(s => s.ImdbId = "7")
+                .TheNext(1)
+                .With(s => s.TvdbId = 8)
+                .With(s => s.ImdbId = "8")
+                .Build().ToList();
+
+            _list2Series = Builder<ImportListItemInfo>.CreateListOfSize(3)
+                .TheFirst(1)
+                .With(s => s.TvdbId = 6)
+                .With(s => s.ImdbId = "6")
+                .TheNext(1)
+                .With(s => s.TvdbId = 7)
+                .With(s => s.ImdbId = "7")
+                .TheNext(1)
+                .With(s => s.TvdbId = 8)
+                .With(s => s.ImdbId = "8")
+                .Build().ToList();
+
+            _importListFetch = new ImportListFetchResult(_list1Series, false);
+
+            _commandAll = new ImportListSyncCommand
+            {
+            };
+
+            _commandSingle = new ImportListSyncCommand
+            {
+                DefinitionId = 1
+            };
 
             var mockImportList = new Mock<IImportList>();
 
@@ -31,6 +79,10 @@ namespace NzbDrone.Core.Test.ImportListTests
                   .Setup(v => v.AllSeriesTvdbIds())
                   .Returns(new List<int>());
 
+            Mocker.GetMock<ISeriesService>()
+                .Setup(v => v.GetAllSeries())
+                .Returns(_existingSeries);
+
             Mocker.GetMock<ISearchForNewSeries>()
                   .Setup(v => v.SearchForNewSeries(It.IsAny<string>()))
                   .Returns(new List<Series>());
@@ -41,15 +93,19 @@ namespace NzbDrone.Core.Test.ImportListTests
 
             Mocker.GetMock<IImportListFactory>()
                   .Setup(v => v.All())
-                  .Returns(new List<ImportListDefinition> { new ImportListDefinition { ShouldMonitor = MonitorTypes.All } });
+                  .Returns(() => _importLists.Select(x => x.Definition as ImportListDefinition).ToList());
+
+            Mocker.GetMock<IImportListFactory>()
+                .Setup(v => v.GetAvailableProviders())
+                .Returns(_importLists);
 
             Mocker.GetMock<IImportListFactory>()
                 .Setup(v => v.AutomaticAddEnabled(It.IsAny<bool>()))
-                .Returns(new List<IImportList> { mockImportList.Object });
+                .Returns(() => _importLists.Where(x => (x.Definition as ImportListDefinition).EnableAutomaticAdd).ToList());
 
             Mocker.GetMock<IFetchAndParseImportList>()
                   .Setup(v => v.Fetch())
-                  .Returns(_importListReports);
+                  .Returns(_importListFetch);
 
             Mocker.GetMock<IImportListExclusionService>()
                   .Setup(v => v.All())
@@ -58,19 +114,19 @@ namespace NzbDrone.Core.Test.ImportListTests
 
         private void WithTvdbId()
         {
-            _importListReports.First().TvdbId = 81189;
+            _list1Series.First().TvdbId = 81189;
         }
 
         private void WithImdbId()
         {
-            _importListReports.First().ImdbId = "tt0496424";
+            _list1Series.First().ImdbId = "tt0496424";
         }
 
         private void WithExistingSeries()
         {
             Mocker.GetMock<ISeriesService>()
                   .Setup(v => v.AllSeriesTvdbIds())
-                  .Returns(new List<int> { _importListReports.First().TvdbId });
+                  .Returns(new List<int> { _list1Series.First().TvdbId });
         }
 
         private void WithExcludedSeries()
@@ -81,22 +137,281 @@ namespace NzbDrone.Core.Test.ImportListTests
                     {
                       new ImportListExclusion
                         {
-                          TvdbId = 81189
+                          TvdbId = _list1Series.First().TvdbId
                         }
                     });
         }
 
         private void WithMonitorType(MonitorTypes monitor)
         {
+            _importLists.ForEach(li => (li.Definition as ImportListDefinition).ShouldMonitor = monitor);
+        }
+
+        private void WithCleanLevel(ListSyncLevelType cleanLevel, int? tagId = null)
+        {
+            Mocker.GetMock<IConfigService>()
+                  .SetupGet(v => v.ListSyncLevel)
+                  .Returns(cleanLevel);
+            if (tagId.HasValue)
+            {
+                Mocker.GetMock<IConfigService>()
+                    .SetupGet(v => v.ListSyncTag)
+                    .Returns(tagId.Value);
+            }
+        }
+
+        private void WithList(int id, bool enabledAuto, int lastSyncHoursOffset = 0, bool pendingRemovals = true, DateTime? disabledTill = null)
+        {
+            var importListDefinition = new ImportListDefinition { Id = id, EnableAutomaticAdd = enabledAuto };
+
             Mocker.GetMock<IImportListFactory>()
-                  .Setup(v => v.All())
+                  .Setup(v => v.Get(id))
-                  .Returns(new List<ImportListDefinition> { new ImportListDefinition { ShouldMonitor = monitor } });
+                  .Returns(importListDefinition);
+
+            var mockImportList = new Mock<IImportList>();
+            mockImportList.SetupGet(s => s.Definition).Returns(importListDefinition);
+            mockImportList.SetupGet(s => s.MinRefreshInterval).Returns(TimeSpan.FromHours(12));
+
+            var status = new ImportListStatus()
+            {
+                LastInfoSync = DateTime.UtcNow.AddHours(lastSyncHoursOffset),
+                HasRemovedItemSinceLastClean = pendingRemovals,
+                DisabledTill = disabledTill
+            };
+
+            if (disabledTill.HasValue)
+            {
+                _importListFetch.AnyFailure = true;
+            }
+
+            Mocker.GetMock<IImportListStatusService>()
+                .Setup(v => v.GetListStatus(id))
+                .Returns(status);
+
+            _importLists.Add(mockImportList.Object);
+        }
+
+        private void VerifyDidAddTag(int expectedSeriesCount, int expectedTagId)
+        {
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.UpdateSeries(It.Is<List<Series>>(x => x.Count == expectedSeriesCount && x.All(series => series.Tags.Contains(expectedTagId))), true), Times.Once());
+        }
+
+        [Test]
+        public void should_not_clean_library_if_lists_have_not_removed_any_items()
+        {
+            _importListFetch.Series = _existingSeries.Select(x => new ImportListItemInfo() { TvdbId = x.TvdbId }).ToList();
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true, pendingRemovals: false);
+            WithCleanLevel(ListSyncLevelType.KeepAndUnmonitor);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.GetAllSeries(), Times.Never());
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.UpdateSeries(It.IsAny<List<Series>>(), true), Times.Never());
+        }
+
+        [Test]
+        public void should_not_clean_library_if_config_value_disable()
+        {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
+            WithCleanLevel(ListSyncLevelType.Disabled);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.GetAllSeries(), Times.Never());
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.UpdateSeries(new List<Series>(), true), Times.Never());
+        }
+
+        [Test]
+        public void should_log_only_on_clean_library_if_config_value_logonly()
+        {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
+            WithCleanLevel(ListSyncLevelType.LogOnly);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.GetAllSeries(), Times.Once());
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.DeleteSeries(It.IsAny<List<int>>(), It.IsAny<bool>(), It.IsAny<bool>()), Times.Never());
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.UpdateSeries(new List<Series>(), true), Times.Once());
+        }
+
+        [Test]
+        public void should_unmonitor_on_clean_library_if_config_value_keepAndUnmonitor()
+        {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
+            WithCleanLevel(ListSyncLevelType.KeepAndUnmonitor);
+            var monitored = _existingSeries.Count(x => x.Monitored);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.GetAllSeries(), Times.Once());
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.DeleteSeries(It.IsAny<List<int>>(), It.IsAny<bool>(), It.IsAny<bool>()), Times.Never());
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.UpdateSeries(It.Is<List<Series>>(s => s.Count == monitored && s.All(m => !m.Monitored)), true), Times.Once());
+        }
+
+        [Test]
+        public void should_not_clean_on_clean_library_if_tvdb_match()
+        {
+            WithList(1, true);
+            WithCleanLevel(ListSyncLevelType.KeepAndUnmonitor);
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+
+            Mocker.GetMock<IImportListItemService>()
+                .Setup(v => v.Exists(6, It.IsAny<string>()))
+                .Returns(true);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.UpdateSeries(It.Is<List<Series>>(s => s.Count > 0 && s.All(m => !m.Monitored)), true), Times.Once());
+        }
+
+        [Test]
+        public void should_not_clean_on_clean_library_if_imdb_match()
+        {
+            WithList(1, true);
+            WithCleanLevel(ListSyncLevelType.KeepAndUnmonitor);
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+
+            var x = _importLists;
+
+            Mocker.GetMock<IImportListItemService>()
+                .Setup(v => v.Exists(It.IsAny<int>(), "6"))
+                .Returns(true);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.UpdateSeries(It.Is<List<Series>>(s => s.Count > 0 && s.All(m => !m.Monitored)), true), Times.Once());
+        }
+
+        [Test]
+        public void should_tag_series_on_clean_library_if_config_value_keepAndTag()
+        {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
+            WithCleanLevel(ListSyncLevelType.KeepAndTag, 1);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.GetAllSeries(), Times.Once());
+
+            VerifyDidAddTag(_existingSeries.Count, 1);
+        }
+
+        [Test]
+        public void should_not_clean_if_list_failures()
+        {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true, disabledTill: DateTime.UtcNow.AddHours(1));
+            WithCleanLevel(ListSyncLevelType.LogOnly);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.GetAllSeries(), Times.Never());
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.UpdateSeries(It.IsAny<List<Series>>(), It.IsAny<bool>()), Times.Never());
+            Mocker.GetMock<ISeriesService>()
+                .Verify(v => v.DeleteSeries(It.IsAny<List<int>>(), It.IsAny<bool>(), It.IsAny<bool>()), Times.Never());
+        }
+
+        [Test]
+        public void should_add_new_series_from_single_list_to_library()
+        {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
+            WithCleanLevel(ListSyncLevelType.Disabled);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<IAddSeriesService>()
+                  .Verify(v => v.AddSeries(It.Is<List<Series>>(s => s.Count == 1), true), Times.Once());
+        }
+
+        [Test]
+        public void should_add_new_series_from_multiple_list_to_library()
+        {
+            _list2Series.ForEach(m => m.ImportListId = 2);
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            _importListFetch.Series.AddRange(_list2Series);
+
+            WithList(1, true);
+            WithList(2, true);
+
+            WithCleanLevel(ListSyncLevelType.Disabled);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<IAddSeriesService>()
+                  .Verify(v => v.AddSeries(It.Is<List<Series>>(s => s.Count == 4), true), Times.Once());
+        }
+
+        [Test]
+        public void should_add_new_series_to_library_only_from_enabled_lists()
+        {
+            _list2Series.ForEach(m => m.ImportListId = 2);
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            _importListFetch.Series.AddRange(_list2Series);
+
+            WithList(1, true);
+            WithList(2, false);
+
+            WithCleanLevel(ListSyncLevelType.Disabled);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<IAddSeriesService>()
+                  .Verify(v => v.AddSeries(It.Is<List<Series>>(s => s.Count == 1), true), Times.Once());
+        }
+
+        [Test]
+        public void should_not_add_duplicate_series_from_seperate_lists()
+        {
+            _list2Series.ForEach(m => m.ImportListId = 2);
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            _importListFetch.Series.AddRange(_list2Series);
+            _importListFetch.Series[0].TvdbId = 6;
+
+            WithList(1, true);
+            WithList(2, true);
+
+            WithCleanLevel(ListSyncLevelType.Disabled);
+
+            Subject.Execute(_commandAll);
+
+            Mocker.GetMock<IAddSeriesService>()
+                  .Verify(v => v.AddSeries(It.Is<List<Series>>(s => s.Count == 3), true), Times.Once());
         }
 
         [Test]
         public void should_search_if_series_title_and_no_series_id()
         {
-            Subject.Execute(new ImportListSyncCommand());
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
+            Subject.Execute(_commandAll);
 
             Mocker.GetMock<ISearchForNewSeries>()
                   .Verify(v => v.SearchForNewSeries(It.IsAny<string>()), Times.Once());
@@ -105,8 +420,10 @@ namespace NzbDrone.Core.Test.ImportListTests
         [Test]
         public void should_not_search_if_series_title_and_series_id()
         {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
             WithTvdbId();
-            Subject.Execute(new ImportListSyncCommand());
+            Subject.Execute(_commandAll);
 
             Mocker.GetMock<ISearchForNewSeries>()
                   .Verify(v => v.SearchForNewSeries(It.IsAny<string>()), Times.Never());
@@ -115,8 +432,10 @@ namespace NzbDrone.Core.Test.ImportListTests
         [Test]
         public void should_search_by_imdb_if_series_title_and_series_imdb()
         {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
             WithImdbId();
-            Subject.Execute(new ImportListSyncCommand());
+            Subject.Execute(_commandAll);
 
             Mocker.GetMock<ISearchForNewSeries>()
                   .Verify(v => v.SearchForNewSeriesByImdbId(It.IsAny<string>()), Times.Once());
@@ -125,10 +444,12 @@ namespace NzbDrone.Core.Test.ImportListTests
         [Test]
         public void should_not_add_if_existing_series()
         {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
             WithTvdbId();
             WithExistingSeries();
 
-            Subject.Execute(new ImportListSyncCommand());
+            Subject.Execute(_commandAll);
 
             Mocker.GetMock<IAddSeriesService>()
                   .Verify(v => v.AddSeries(It.Is<List<Series>>(t => t.Count == 0), It.IsAny<bool>()));
@@ -138,10 +459,12 @@ namespace NzbDrone.Core.Test.ImportListTests
         [TestCase(MonitorTypes.All, true)]
         public void should_add_if_not_existing_series(MonitorTypes monitor, bool expectedSeriesMonitored)
         {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
             WithTvdbId();
             WithMonitorType(monitor);
 
-            Subject.Execute(new ImportListSyncCommand());
+            Subject.Execute(_commandAll);
 
             Mocker.GetMock<IAddSeriesService>()
                   .Verify(v => v.AddSeries(It.Is<List<Series>>(t => t.Count == 1 && t.First().Monitored == expectedSeriesMonitored), It.IsAny<bool>()));
@@ -150,10 +473,12 @@ namespace NzbDrone.Core.Test.ImportListTests
         [Test]
         public void should_not_add_if_excluded_series()
         {
+            _importListFetch.Series.ForEach(m => m.ImportListId = 1);
+            WithList(1, true);
             WithTvdbId();
             WithExcludedSeries();
 
-            Subject.Execute(new ImportListSyncCommand());
+            Subject.Execute(_commandAll);
 
             Mocker.GetMock<IAddSeriesService>()
                   .Verify(v => v.AddSeries(It.Is<List<Series>>(t => t.Count == 0), It.IsAny<bool>()));
@@ -177,7 +502,7 @@ namespace NzbDrone.Core.Test.ImportListTests
         {
             Mocker.GetMock<IFetchAndParseImportList>()
                 .Setup(v => v.Fetch())
-                .Returns(new List<ImportListItemInfo>());
+                .Returns(new ImportListFetchResult());
 
             Subject.Execute(new ImportListSyncCommand());
 

src/NzbDrone.Core/Configuration/ConfigService.cs

@@ -6,6 +6,7 @@ using NLog;
 using NzbDrone.Common.EnsureThat;
 using NzbDrone.Common.Http.Proxy;
 using NzbDrone.Core.Configuration.Events;
+using NzbDrone.Core.ImportLists;
 using NzbDrone.Core.Languages;
 using NzbDrone.Core.MediaFiles;
 using NzbDrone.Core.MediaFiles.EpisodeImport;
@@ -276,6 +277,18 @@ namespace NzbDrone.Core.Configuration
             set { SetValue("ChownGroup", value); }
         }
 
+        public ListSyncLevelType ListSyncLevel
+        {
+            get { return GetValueEnum("ListSyncLevel", ListSyncLevelType.Disabled); }
+            set { SetValue("ListSyncLevel", value); }
+        }
+
+        public int ListSyncTag
+        {
+            get { return GetValueInt("ListSyncTag"); }
+            set { SetValue("ListSyncTag", value); }
+        }
+
         public int FirstDayOfWeek
         {
             get { return GetValueInt("FirstDayOfWeek", (int)CultureInfo.CurrentCulture.DateTimeFormat.FirstDayOfWeek); }

src/NzbDrone.Core/Configuration/IConfigService.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using NzbDrone.Common.Http.Proxy;
+using NzbDrone.Core.ImportLists;
 using NzbDrone.Core.MediaFiles;
 using NzbDrone.Core.MediaFiles.EpisodeImport;
 using NzbDrone.Core.Qualities;
@@ -52,6 +53,9 @@ namespace NzbDrone.Core.Configuration
         int MaximumSize { get; set; }
         int MinimumAge { get; set; }
 
+        ListSyncLevelType ListSyncLevel { get; set; }
+        int ListSyncTag { get; set; }
+
         // UI
         int FirstDayOfWeek { get; set; }
         string CalendarWeekColumnHeader { get; set; }

src/NzbDrone.Core/Datastore/Migration/193_add_import_list_items.cs

@@ -0,0 +1,26 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(193)]
+    public class add_import_list_items : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Create.TableForModel("ImportListItems")
+                .WithColumn("ImportListId").AsInt32()
+                .WithColumn("Title").AsString()
+                .WithColumn("TvdbId").AsInt32()
+                .WithColumn("Year").AsInt32().Nullable()
+                .WithColumn("TmdbId").AsInt32().Nullable()
+                .WithColumn("ImdbId").AsString().Nullable()
+                .WithColumn("MalId").AsInt32().Nullable()
+                .WithColumn("AniListId").AsInt32().Nullable()
+                .WithColumn("ReleaseDate").AsDateTimeOffset().Nullable();
+
+            Alter.Table("ImportListStatus")
+                .AddColumn("HasRemovedItemSinceLastClean").AsBoolean().WithDefaultValue(false);
+        }
+    }
+}

src/NzbDrone.Core/Datastore/TableMapping.cs

@@ -81,6 +81,9 @@ namespace NzbDrone.Core.Datastore
                   .Ignore(i => i.MinRefreshInterval)
                   .Ignore(i => i.Enable);
 
+            Mapper.Entity<ImportListItemInfo>("ImportListItems").RegisterModel()
+                   .Ignore(i => i.ImportList);
+
             Mapper.Entity<NotificationDefinition>("Notifications").RegisterModel()
                   .Ignore(x => x.ImplementationName)
                   .Ignore(i => i.SupportsOnGrab)

src/NzbDrone.Core/ImportLists/AniList/AniListImportBase.cs

@@ -5,7 +5,6 @@ using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Localization;
 using NzbDrone.Core.Parser;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.Validation;
 
 namespace NzbDrone.Core.ImportLists.AniList
@@ -65,7 +64,7 @@ namespace NzbDrone.Core.ImportLists.AniList
             return new { };
         }
 
-        public override IList<ImportListItemInfo> Fetch()
+        public override ImportListFetchResult Fetch()
         {
             CheckToken();
             return base.Fetch();

src/NzbDrone.Core/ImportLists/AniList/List/AniListImport.cs

@@ -44,10 +44,11 @@ namespace NzbDrone.Core.ImportLists.AniList.List
             return new AniListParser(Settings);
         }
 
-        protected override IList<ImportListItemInfo> FetchItems(Func<IImportListRequestGenerator, ImportListPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
+        protected override ImportListFetchResult FetchItems(Func<IImportListRequestGenerator, ImportListPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
         {
             var releases = new List<ImportListItemInfo>();
             var url = string.Empty;
+            var anyFailure = true;
 
             try
             {
@@ -77,6 +78,7 @@ namespace NzbDrone.Core.ImportLists.AniList.List
                 while (hasNextPage);
 
                 _importListStatusService.RecordSuccess(Definition.Id);
+                anyFailure = false;
             }
             catch (WebException webException)
             {
@@ -149,7 +151,7 @@ namespace NzbDrone.Core.ImportLists.AniList.List
                 _logger.Error(ex, "An error occurred while processing feed. {0}", url);
             }
 
-            return CleanupListItems(releases);
+            return new ImportListFetchResult(CleanupListItems(releases), anyFailure);
         }
     }
 }

src/NzbDrone.Core/ImportLists/Custom/CustomImport.cs

@@ -30,9 +30,10 @@ namespace NzbDrone.Core.ImportLists.Custom
             _customProxy = customProxy;
         }
 
-        public override IList<ImportListItemInfo> Fetch()
+        public override ImportListFetchResult Fetch()
         {
             var series = new List<ImportListItemInfo>();
+            var anyFailure = false;
 
             try
             {
@@ -50,12 +51,13 @@ namespace NzbDrone.Core.ImportLists.Custom
             }
             catch (Exception ex)
             {
+                anyFailure = true;
                 _logger.Debug(ex, "Failed to fetch data for list {0} ({1})", Definition.Name, Name);
 
                 _importListStatusService.RecordFailure(Definition.Id);
             }
 
-            return CleanupListItems(series);
+            return new ImportListFetchResult(CleanupListItems(series), anyFailure);
         }
 
         public override object RequestAction(string action, IDictionary<string, string> query)

src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs

@@ -4,32 +4,34 @@ using System.Linq;
 using System.Threading.Tasks;
 using NLog;
 using NzbDrone.Common.TPL;
-using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.ImportLists.ImportListItems;
 
 namespace NzbDrone.Core.ImportLists
 {
     public interface IFetchAndParseImportList
     {
-        List<ImportListItemInfo> Fetch();
+        ImportListFetchResult Fetch();
-        List<ImportListItemInfo> FetchSingleList(ImportListDefinition definition);
+        ImportListFetchResult FetchSingleList(ImportListDefinition definition);
     }
 
     public class FetchAndParseImportListService : IFetchAndParseImportList
     {
         private readonly IImportListFactory _importListFactory;
         private readonly IImportListStatusService _importListStatusService;
+        private readonly IImportListItemService _importListItemService;
         private readonly Logger _logger;
 
-        public FetchAndParseImportListService(IImportListFactory importListFactory, IImportListStatusService importListStatusService, Logger logger)
+        public FetchAndParseImportListService(IImportListFactory importListFactory, IImportListStatusService importListStatusService, IImportListItemService importListItemService, Logger logger)
         {
             _importListFactory = importListFactory;
             _importListStatusService = importListStatusService;
+            _importListItemService = importListItemService;
             _logger = logger;
         }
 
-        public List<ImportListItemInfo> Fetch()
+        public ImportListFetchResult Fetch()
         {
-            var result = new List<ImportListItemInfo>();
+            var result = new ImportListFetchResult();
 
             var importLists = _importListFactory.AutomaticAddEnabled();
 
@@ -47,7 +49,7 @@ namespace NzbDrone.Core.ImportLists
             foreach (var importList in importLists)
             {
                 var importListLocal = importList;
-                var importListStatus = _importListStatusService.GetLastSyncListInfo(importListLocal.Definition.Id);
+                var importListStatus = _importListStatusService.GetListStatus(importListLocal.Definition.Id).LastInfoSync;
 
                 if (importListStatus.HasValue)
                 {
@@ -64,16 +66,23 @@ namespace NzbDrone.Core.ImportLists
                      {
                          try
                          {
-                             var importListReports = importListLocal.Fetch();
+                             var fetchResult = importListLocal.Fetch();
+                             var importListReports = fetchResult.Series;
 
                              lock (result)
                              {
                                  _logger.Debug("Found {0} reports from {1} ({2})", importListReports.Count, importList.Name, importListLocal.Definition.Name);
 
-                                 result.AddRange(importListReports);
+                                 if (!fetchResult.AnyFailure)
-                             }
+                                 {
+                                     importListReports.ForEach(s => s.ImportListId = importList.Definition.Id);
+                                     result.Series.AddRange(importListReports);
+                                     var removed = _importListItemService.SyncSeriesForList(importListReports, importList.Definition.Id);
+                                     _importListStatusService.UpdateListSyncStatus(importList.Definition.Id, removed > 0);
+                                 }
 
-                             _importListStatusService.UpdateListSyncStatus(importList.Definition.Id);
+                                 result.AnyFailure |= fetchResult.AnyFailure;
+                             }
                          }
                          catch (Exception e)
                          {
@@ -86,16 +95,16 @@ namespace NzbDrone.Core.ImportLists
 
             Task.WaitAll(taskList.ToArray());
 
-            result = result.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList();
+            result.Series = result.Series.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList();
 
-            _logger.Debug("Found {0} total reports from {1} lists", result.Count, importLists.Count);
+            _logger.Debug("Found {0} total reports from {1} lists", result.Series.Count, importLists.Count);
 
             return result;
         }
 
-        public List<ImportListItemInfo> FetchSingleList(ImportListDefinition definition)
+        public ImportListFetchResult FetchSingleList(ImportListDefinition definition)
         {
-            var result = new List<ImportListItemInfo>();
+            var result = new ImportListFetchResult();
 
             var importList = _importListFactory.GetInstance(definition);
 
@@ -114,16 +123,25 @@ namespace NzbDrone.Core.ImportLists
             {
                 try
                 {
-                    var importListReports = importListLocal.Fetch();
+                    var fetchResult = importListLocal.Fetch();
+                    var importListReports = fetchResult.Series;
 
                     lock (result)
                     {
                         _logger.Debug("Found {0} reports from {1} ({2})", importListReports.Count, importList.Name, importListLocal.Definition.Name);
 
-                        result.AddRange(importListReports);
+                        if (!fetchResult.AnyFailure)
+                        {
+                            importListReports.ForEach(s => s.ImportListId = importList.Definition.Id);
+                            result.Series.AddRange(importListReports);
+                            var removed = _importListItemService.SyncSeriesForList(importListReports, importList.Definition.Id);
+                            _importListStatusService.UpdateListSyncStatus(importList.Definition.Id, removed > 0);
+                        }
+
+                        result.AnyFailure |= fetchResult.AnyFailure;
                     }
 
-                    _importListStatusService.UpdateListSyncStatus(importList.Definition.Id);
+                    result.AnyFailure |= fetchResult.AnyFailure;
                 }
                 catch (Exception e)
                 {
@@ -135,8 +153,6 @@ namespace NzbDrone.Core.ImportLists
 
             Task.WaitAll(taskList.ToArray());
 
-            result = result.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList();
-
             return result;
         }
     }

src/NzbDrone.Core/ImportLists/HttpImportListBase.cs

@@ -38,15 +38,16 @@ namespace NzbDrone.Core.ImportLists
             _httpClient = httpClient;
         }
 
-        public override IList<ImportListItemInfo> Fetch()
+        public override ImportListFetchResult Fetch()
         {
             return FetchItems(g => g.GetListItems(), true);
         }
 
-        protected virtual IList<ImportListItemInfo> FetchItems(Func<IImportListRequestGenerator, ImportListPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
+        protected virtual ImportListFetchResult FetchItems(Func<IImportListRequestGenerator, ImportListPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
         {
             var releases = new List<ImportListItemInfo>();
             var url = string.Empty;
+            var anyFailure = true;
 
             try
             {
@@ -92,6 +93,7 @@ namespace NzbDrone.Core.ImportLists
                 }
 
                 _importListStatusService.RecordSuccess(Definition.Id);
+                anyFailure = false;
             }
             catch (WebException webException)
             {
@@ -163,7 +165,7 @@ namespace NzbDrone.Core.ImportLists
                 _logger.Error(ex, "An error occurred while processing feed. {0}", url);
             }
 
-            return CleanupListItems(releases);
+            return new ImportListFetchResult(CleanupListItems(releases), anyFailure);
         }
 
         protected virtual bool IsValidItem(ImportListItemInfo listItem)

src/NzbDrone.Core/ImportLists/IImportList.cs

@@ -1,6 +1,4 @@
 using System;
-using System.Collections.Generic;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 
 namespace NzbDrone.Core.ImportLists
@@ -9,6 +7,6 @@ namespace NzbDrone.Core.ImportLists
     {
         ImportListType ListType { get; }
         TimeSpan MinRefreshInterval { get; }
-        IList<ImportListItemInfo> Fetch();
+        ImportListFetchResult Fetch();
     }
 }

src/NzbDrone.Core/ImportLists/ImportListBase.cs

@@ -11,6 +11,23 @@ using NzbDrone.Core.ThingiProvider;
 
 namespace NzbDrone.Core.ImportLists
 {
+    public class ImportListFetchResult
+    {
+        public ImportListFetchResult()
+        {
+            Series = new List<ImportListItemInfo>();
+        }
+
+        public ImportListFetchResult(IEnumerable<ImportListItemInfo> series, bool anyFailure)
+        {
+            Series = series.ToList();
+            AnyFailure = anyFailure;
+        }
+
+        public List<ImportListItemInfo> Series { get; set; }
+        public bool AnyFailure { get; set; }
+    }
+
     public abstract class ImportListBase<TSettings> : IImportList
         where TSettings : IImportListSettings, new()
     {
@@ -63,7 +80,7 @@ namespace NzbDrone.Core.ImportLists
 
         protected TSettings Settings => (TSettings)Definition.Settings;
 
-        public abstract IList<ImportListItemInfo> Fetch();
+        public abstract ImportListFetchResult Fetch();
 
         protected virtual IList<ImportListItemInfo> CleanupListItems(IEnumerable<ImportListItemInfo> releases)
         {

src/NzbDrone.Core/ImportLists/ImportListItems/ImportListItemRepository.cs

@@ -0,0 +1,43 @@
+using System.Collections.Generic;
+using System.Linq;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.ImportLists.ImportListItems
+{
+    public interface IImportListItemInfoRepository : IBasicRepository<ImportListItemInfo>
+    {
+        List<ImportListItemInfo> GetAllForLists(List<int> listIds);
+        bool Exists(int tvdbId, string imdbId);
+    }
+
+    public class ImportListItemRepository : BasicRepository<ImportListItemInfo>, IImportListItemInfoRepository
+    {
+        public ImportListItemRepository(IMainDatabase database, IEventAggregator eventAggregator)
+            : base(database, eventAggregator)
+        {
+        }
+
+        public List<ImportListItemInfo> GetAllForLists(List<int> listIds)
+        {
+            return Query(x => listIds.Contains(x.ImportListId));
+        }
+
+        public bool Exists(int tvdbId, string imdbId)
+        {
+            List<ImportListItemInfo> items;
+
+            if (string.IsNullOrWhiteSpace(imdbId))
+            {
+                items = Query(x => x.TvdbId == tvdbId);
+            }
+            else
+            {
+                items = Query(x => x.TvdbId == tvdbId || x.ImdbId == imdbId);
+            }
+
+            return items.Any();
+        }
+    }
+}

src/NzbDrone.Core/ImportLists/ImportListItems/ImportListItemService.cs

@@ -0,0 +1,59 @@
+using System.Collections.Generic;
+using System.Linq;
+using NLog;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.ThingiProvider.Events;
+
+namespace NzbDrone.Core.ImportLists.ImportListItems
+{
+    public interface IImportListItemService
+    {
+        List<ImportListItemInfo> GetAllForLists(List<int> listIds);
+        int SyncSeriesForList(List<ImportListItemInfo> listSeries, int listId);
+        bool Exists(int tvdbId, string imdbId);
+    }
+
+    public class ImportListItemService : IImportListItemService, IHandleAsync<ProviderDeletedEvent<IImportList>>
+    {
+        private readonly IImportListItemInfoRepository _importListSeriesRepository;
+        private readonly Logger _logger;
+
+        public ImportListItemService(IImportListItemInfoRepository importListSeriesRepository,
+                             Logger logger)
+        {
+            _importListSeriesRepository = importListSeriesRepository;
+            _logger = logger;
+        }
+
+        public int SyncSeriesForList(List<ImportListItemInfo> listSeries, int listId)
+        {
+            var existingListSeries = GetAllForLists(new List<int> { listId });
+
+            listSeries.ForEach(l => l.Id = existingListSeries.FirstOrDefault(e => e.TvdbId == l.TvdbId)?.Id ?? 0);
+
+            _importListSeriesRepository.InsertMany(listSeries.Where(l => l.Id == 0).ToList());
+            _importListSeriesRepository.UpdateMany(listSeries.Where(l => l.Id > 0).ToList());
+            var toDelete = existingListSeries.Where(l => !listSeries.Any(x => x.TvdbId == l.TvdbId)).ToList();
+            _importListSeriesRepository.DeleteMany(toDelete);
+
+            return toDelete.Count;
+        }
+
+        public List<ImportListItemInfo> GetAllForLists(List<int> listIds)
+        {
+            return _importListSeriesRepository.GetAllForLists(listIds).ToList();
+        }
+
+        public void HandleAsync(ProviderDeletedEvent<IImportList> message)
+        {
+            var seriesOnList = _importListSeriesRepository.GetAllForLists(new List<int> { message.ProviderId });
+            _importListSeriesRepository.DeleteMany(seriesOnList);
+        }
+
+        public bool Exists(int tvdbId, string imdbId)
+        {
+            return _importListSeriesRepository.Exists(tvdbId, imdbId);
+        }
+    }
+}

src/NzbDrone.Core/ImportLists/ImportListStatus.cs

@@ -6,5 +6,6 @@ namespace NzbDrone.Core.ImportLists
     public class ImportListStatus : ProviderStatusBase
     {
         public DateTime? LastInfoSync { get; set; }
+        public bool HasRemovedItemSinceLastClean { get; set; }
     }
 }

src/NzbDrone.Core/ImportLists/ImportListStatusService.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using NLog;
 using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Messaging.Events;
@@ -8,9 +9,10 @@ namespace NzbDrone.Core.ImportLists
 {
     public interface IImportListStatusService : IProviderStatusServiceBase<ImportListStatus>
     {
-        DateTime? GetLastSyncListInfo(int importListId);
+        ImportListStatus GetListStatus(int importListId);
 
-        void UpdateListSyncStatus(int importListId);
+        void UpdateListSyncStatus(int importListId, bool removedItems);
+        void MarkListsAsCleaned();
     }
 
     public class ImportListStatusService : ProviderStatusServiceBase<IImportList, ImportListStatus>, IImportListStatusService
@@ -20,21 +22,38 @@ namespace NzbDrone.Core.ImportLists
         {
         }
 
-        public DateTime? GetLastSyncListInfo(int importListId)
+        public ImportListStatus GetListStatus(int importListId)
         {
-            return GetProviderStatus(importListId).LastInfoSync;
+            return GetProviderStatus(importListId);
         }
 
-        public void UpdateListSyncStatus(int importListId)
+        public void UpdateListSyncStatus(int importListId, bool removedItems)
         {
             lock (_syncRoot)
             {
                 var status = GetProviderStatus(importListId);
 
                 status.LastInfoSync = DateTime.UtcNow;
+                status.HasRemovedItemSinceLastClean |= removedItems;
 
                 _providerStatusRepository.Upsert(status);
             }
         }
+
+        public void MarkListsAsCleaned()
+        {
+            lock (_syncRoot)
+            {
+                var toUpdate = new List<ImportListStatus>();
+
+                foreach (var status in _providerStatusRepository.All())
+                {
+                    status.HasRemovedItemSinceLastClean = false;
+                    toUpdate.Add(status);
+                }
+
+                _providerStatusRepository.UpdateMany(toUpdate);
+            }
+        }
     }
 }

src/NzbDrone.Core/ImportLists/ImportListSyncService.cs

@@ -3,41 +3,85 @@ using System.Linq;
 using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Instrumentation.Extensions;
+using NzbDrone.Core.Configuration;
 using NzbDrone.Core.ImportLists.Exclusions;
+using NzbDrone.Core.ImportLists.ImportListItems;
+using NzbDrone.Core.Jobs;
 using NzbDrone.Core.Messaging.Commands;
+using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.MetadataSource;
 using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.ThingiProvider.Events;
 using NzbDrone.Core.Tv;
 
 namespace NzbDrone.Core.ImportLists
 {
-    public class ImportListSyncService : IExecute<ImportListSyncCommand>
+    public class ImportListSyncService : IExecute<ImportListSyncCommand>, IHandleAsync<ProviderDeletedEvent<IImportList>>
     {
         private readonly IImportListFactory _importListFactory;
+        private readonly IImportListStatusService _importListStatusService;
         private readonly IImportListExclusionService _importListExclusionService;
+        private readonly IImportListItemService _importListItemService;
         private readonly IFetchAndParseImportList _listFetcherAndParser;
         private readonly ISearchForNewSeries _seriesSearchService;
         private readonly ISeriesService _seriesService;
         private readonly IAddSeriesService _addSeriesService;
+        private readonly IConfigService _configService;
+        private readonly ITaskManager _taskManager;
         private readonly Logger _logger;
 
         public ImportListSyncService(IImportListFactory importListFactory,
+                              IImportListStatusService importListStatusService,
                               IImportListExclusionService importListExclusionService,
+                              IImportListItemService importListItemService,
                               IFetchAndParseImportList listFetcherAndParser,
                               ISearchForNewSeries seriesSearchService,
                               ISeriesService seriesService,
                               IAddSeriesService addSeriesService,
+                              IConfigService configService,
+                              ITaskManager taskManager,
                               Logger logger)
         {
             _importListFactory = importListFactory;
+            _importListStatusService = importListStatusService;
             _importListExclusionService = importListExclusionService;
+            _importListItemService = importListItemService;
             _listFetcherAndParser = listFetcherAndParser;
             _seriesSearchService = seriesSearchService;
             _seriesService = seriesService;
             _addSeriesService = addSeriesService;
+            _configService = configService;
+            _taskManager = taskManager;
             _logger = logger;
         }
 
+        private bool AllListsSuccessfulWithAPendingClean()
+        {
+            var lists = _importListFactory.AutomaticAddEnabled(false);
+            var anyRemoved = false;
+
+            foreach (var list in lists)
+            {
+                var status = _importListStatusService.GetListStatus(list.Definition.Id);
+
+                if (status.DisabledTill.HasValue)
+                {
+                    // list failed the last time it was synced.
+                    return false;
+                }
+
+                if (!status.LastInfoSync.HasValue)
+                {
+                    // list has never been synced.
+                    return false;
+                }
+
+                anyRemoved |= status.HasRemovedItemSinceLastClean;
+            }
+
+            return anyRemoved;
+        }
+
         private void SyncAll()
         {
             if (_importListFactory.AutomaticAddEnabled().Empty())
@@ -49,18 +93,26 @@ namespace NzbDrone.Core.ImportLists
 
             _logger.ProgressInfo("Starting Import List Sync");
 
-            var listItems = _listFetcherAndParser.Fetch().ToList();
+            var result = _listFetcherAndParser.Fetch();
+
+            var listItems = result.Series.ToList();
 
             ProcessListItems(listItems);
+
+            TryCleanLibrary();
         }
 
         private void SyncList(ImportListDefinition definition)
         {
             _logger.ProgressInfo(string.Format("Starting Import List Refresh for List {0}", definition.Name));
 
-            var listItems = _listFetcherAndParser.FetchSingleList(definition).ToList();
+            var result = _listFetcherAndParser.FetchSingleList(definition);
+
+            var listItems = result.Series.ToList();
 
             ProcessListItems(listItems);
+
+            TryCleanLibrary();
         }
 
         private void ProcessListItems(List<ImportListItemInfo> items)
@@ -90,6 +142,11 @@ namespace NzbDrone.Core.ImportLists
 
                 var importList = importLists.Single(x => x.Id == item.ImportListId);
 
+                if (!importList.EnableAutomaticAdd)
+                {
+                    continue;
+                }
+
                 // Map by IMDb ID if we have it
                 if (item.TvdbId <= 0 && item.ImdbId.IsNotNullOrWhiteSpace())
                 {
@@ -180,10 +237,10 @@ namespace NzbDrone.Core.ImportLists
                         SeasonFolder = importList.SeasonFolder,
                         Tags = importList.Tags,
                         AddOptions = new AddSeriesOptions
-                                     {
+                        {
-                                         SearchForMissingEpisodes = importList.SearchForMissingEpisodes,
+                            SearchForMissingEpisodes = importList.SearchForMissingEpisodes,
-                                         Monitor = importList.ShouldMonitor
+                            Monitor = importList.ShouldMonitor
-                                     }
+                        }
                     });
                 }
             }
@@ -206,5 +263,64 @@ namespace NzbDrone.Core.ImportLists
                 SyncAll();
             }
         }
+
+        private void TryCleanLibrary()
+        {
+            if (_configService.ListSyncLevel == ListSyncLevelType.Disabled)
+            {
+                return;
+            }
+
+            if (AllListsSuccessfulWithAPendingClean())
+            {
+                CleanLibrary();
+            }
+        }
+
+        private void CleanLibrary()
+        {
+            if (_configService.ListSyncLevel == ListSyncLevelType.Disabled)
+            {
+                return;
+            }
+
+            var seriesToUpdate = new List<Series>();
+            var seriesInLibrary = _seriesService.GetAllSeries();
+
+            foreach (var series in seriesInLibrary)
+            {
+                var seriesExists = _importListItemService.Exists(series.TvdbId, series.ImdbId);
+
+                if (!seriesExists)
+                {
+                    switch (_configService.ListSyncLevel)
+                    {
+                        case ListSyncLevelType.LogOnly:
+                            _logger.Info("{0} was in your library, but not found in your lists --> You might want to unmonitor or remove it", series);
+                            break;
+                        case ListSyncLevelType.KeepAndUnmonitor when series.Monitored:
+                            _logger.Info("{0} was in your library, but not found in your lists --> Keeping in library but unmonitoring it", series);
+                            series.Monitored = false;
+                            seriesToUpdate.Add(series);
+                            break;
+                        case ListSyncLevelType.KeepAndTag when !series.Tags.Contains(_configService.ListSyncTag):
+                            _logger.Info("{0} was in your library, but not found in your lists --> Keeping in library but tagging it", series);
+                            series.Tags.Add(_configService.ListSyncTag);
+                            seriesToUpdate.Add(series);
+                            break;
+                        default:
+                            break;
+                    }
+                }
+            }
+
+            _seriesService.UpdateSeries(seriesToUpdate, true);
+            _importListStatusService.MarkListsAsCleaned();
+        }
+
+        public void HandleAsync(ProviderDeletedEvent<IImportList> message)
+        {
+            TryCleanLibrary();
+        }
     }
 }

src/NzbDrone.Core/ImportLists/ListSyncLevelType.cs

@@ -0,0 +1,10 @@
+namespace NzbDrone.Core.ImportLists
+{
+    public enum ListSyncLevelType
+    {
+        Disabled,
+        LogOnly,
+        KeepAndUnmonitor,
+        KeepAndTag
+    }
+}

src/NzbDrone.Core/ImportLists/Plex/PlexImport.cs

@@ -8,7 +8,6 @@ using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Localization;
 using NzbDrone.Core.Notifications.Plex.PlexTv;
 using NzbDrone.Core.Parser;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.Validation;
 
 namespace NzbDrone.Core.ImportLists.Plex
@@ -35,7 +34,7 @@ namespace NzbDrone.Core.ImportLists.Plex
         public override string Name => _localizationService.GetLocalizedString("ImportListsPlexSettingsWatchlistName");
         public override int PageSize => 50;
 
-        public override IList<ImportListItemInfo> Fetch()
+        public override ImportListFetchResult Fetch()
         {
             Settings.Validate().Filter("AccessToken").ThrowOnError();
 

src/NzbDrone.Core/ImportLists/Rss/RssImportBase.cs

@@ -1,11 +1,9 @@
 using System;
-using System.Collections.Generic;
 using NLog;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Localization;
 using NzbDrone.Core.Parser;
-using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.ImportLists.Rss
 {
@@ -26,7 +24,7 @@ namespace NzbDrone.Core.ImportLists.Rss
         {
         }
 
-        public override IList<ImportListItemInfo> Fetch()
+        public override ImportListFetchResult Fetch()
         {
             return FetchItems(g => g.GetListItems());
         }

src/NzbDrone.Core/ImportLists/Simkl/SimklImportBase.cs

@@ -36,7 +36,7 @@ namespace NzbDrone.Core.ImportLists.Simkl
             _importListRepository = netImportRepository;
         }
 
-        public override IList<ImportListItemInfo> Fetch()
+        public override ImportListFetchResult Fetch()
         {
             Settings.Validate().Filter("AccessToken", "RefreshToken").ThrowOnError();
             _logger.Trace($"Access token expires at {Settings.Expires}");
@@ -47,13 +47,14 @@ namespace NzbDrone.Core.ImportLists.Simkl
                 RefreshToken();
             }
 
-            var lastFetch = _importListStatusService.GetLastSyncListInfo(Definition.Id);
+            var lastFetch = _importListStatusService.GetListStatus(Definition.Id).LastInfoSync;
             var lastActivity = GetLastActivity();
 
             // Check to see if user has any activity since last sync, if not return empty to avoid work
             if (lastFetch.HasValue && lastActivity < lastFetch.Value.AddHours(-2))
             {
-                return Array.Empty<ImportListItemInfo>();
+                // mark failure to avoid deleting series due to emptyness
+                return new ImportListFetchResult(new List<ImportListItemInfo>(), true);
             }
 
             var generator = GetRequestGenerator();

src/NzbDrone.Core/ImportLists/Sonarr/SonarrImport.cs

@@ -31,10 +31,10 @@ namespace NzbDrone.Core.ImportLists.Sonarr
             _sonarrV3Proxy = sonarrV3Proxy;
         }
 
-        public override IList<ImportListItemInfo> Fetch()
+        public override ImportListFetchResult Fetch()
         {
             var series = new List<ImportListItemInfo>();
-
+            var anyFailure = false;
             try
             {
                 var remoteSeries = _sonarrV3Proxy.GetSeries(Settings);
@@ -75,9 +75,10 @@ namespace NzbDrone.Core.ImportLists.Sonarr
                 _logger.Debug(ex, "Failed to fetch data for list {0} ({1})", Definition.Name, Name);
 
                 _importListStatusService.RecordFailure(Definition.Id);
+                anyFailure = true;
             }
 
-            return CleanupListItems(series);
+            return new ImportListFetchResult(CleanupListItems(series), anyFailure);
         }
 
         public override object RequestAction(string action, IDictionary<string, string> query)

src/NzbDrone.Core/ImportLists/Trakt/TraktImportBase.cs

@@ -6,7 +6,6 @@ using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Localization;
 using NzbDrone.Core.Parser;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.Validation;
 
 namespace NzbDrone.Core.ImportLists.Trakt
@@ -36,7 +35,7 @@ namespace NzbDrone.Core.ImportLists.Trakt
             _importListRepository = netImportRepository;
         }
 
-        public override IList<ImportListItemInfo> Fetch()
+        public override ImportListFetchResult Fetch()
         {
             Settings.Validate().Filter("AccessToken", "RefreshToken").ThrowOnError();
             _logger.Trace($"Access token expires at {Settings.Expires}");

src/NzbDrone.Core/ImportLists/Trakt/TraktParser.cs

@@ -35,7 +35,8 @@ namespace NzbDrone.Core.ImportLists.Trakt
                 series.AddIfNotNull(new ImportListItemInfo()
                 {
                     Title = traktResponse.Show.Title,
-                    TvdbId = traktResponse.Show.Ids.Tvdb.GetValueOrDefault()
+                    TvdbId = traktResponse.Show.Ids.Tvdb.GetValueOrDefault(),
+                    ImdbId = traktResponse.Show.Ids.Imdb
                 });
             }
 

src/NzbDrone.Core/Localization/Core/en.json

@@ -208,6 +208,7 @@
   "ChownGroup": "chown Group",
   "ChownGroupHelpText": "Group name or gid. Use gid for remote file systems.",
   "ChownGroupHelpTextWarning": "This only works if the user running {appName} is the owner of the file. It's better to ensure the download client uses the same group as {appName}.",
+  "CleanLibraryLevel": "Clean Library Level",
   "Clear": "Clear",
   "ClearBlocklist": "Clear blocklist",
   "ClearBlocklistMessageText": "Are you sure you want to clear all items from the blocklist?",
@@ -790,6 +791,7 @@
   "ImportListSearchForMissingEpisodes": "Search for Missing Episodes",
   "ImportListSearchForMissingEpisodesHelpText": "After series is added to {appName} automatically search for missing episodes",
   "ImportListSettings": "Import List Settings",
+  "ImportListStatusAllPossiblePartialFetchHealthCheckMessage": "All lists require manual interaction due to possible partial fetches",
   "ImportListStatusAllUnavailableHealthCheckMessage": "All lists are unavailable due to failures",
   "ImportListStatusUnavailableHealthCheckMessage": "Lists unavailable due to failures: {importListNames}",
   "ImportLists": "Import Lists",
@@ -1010,6 +1012,8 @@
   "Interval": "Interval",
   "InvalidFormat": "Invalid Format",
   "InvalidUILanguage": "Your UI is set to an invalid language, correct it and save your settings",
+  "KeepAndTagSeries": "Keep and Tag Series",
+  "KeepAndUnmonitorSeries": "Keep and Unmonitor Series",
   "KeyboardShortcuts": "Keyboard Shortcuts",
   "KeyboardShortcutsCloseModal": "Close Current Modal",
   "KeyboardShortcutsConfirmModal": "Accept Confirmation Modal",
@@ -1038,6 +1042,9 @@
   "ListOptionsLoadError": "Unable to load list options",
   "ListQualityProfileHelpText": "Quality Profile list items will be added with",
   "ListRootFolderHelpText": "Root Folder list items will be added to",
+  "ListSyncLevelHelpText": "Series in library will be handled based on your selection if they fall off or do not appear on your list(s)",
+  "ListSyncTag": "List Sync Tag",
+  "ListSyncTagHelpText": "This tag will be added when a series falls off or is no longer on your list(s)",
   "ListTagsHelpText": "Tags that will be added on import from this list",
   "ListWillRefreshEveryInterval": "List will refresh every {refreshInterval}",
   "ListsLoadError": "Unable to load Lists",
@@ -1050,6 +1057,7 @@
   "LogFilesLocation": "Log files are located in: {location}",
   "LogLevel": "Log Level",
   "LogLevelTraceHelpTextWarning": "Trace logging should only be enabled temporarily",
+  "LogOnly": "Log Only",
   "Logging": "Logging",
   "Logout": "Logout",
   "Logs": "Logs",
@@ -1946,6 +1954,7 @@
   "Umask777Description": "{octal} - Everyone write",
   "UnableToLoadAutoTagging": "Unable to load auto tagging",
   "UnableToLoadBackups": "Unable to load backups",
+  "UnableToLoadListOptions": "Unable to load list options",
   "UnableToUpdateSonarrDirectly": "Unable to update {appName} directly,",
   "Unavailable": "Unavailable",
   "Underscore": "Underscore",

src/NzbDrone.Core/Parser/Model/ImportListItemInfo.cs

@@ -1,8 +1,9 @@
 using System;
+using NzbDrone.Core.Datastore;
 
 namespace NzbDrone.Core.Parser.Model
 {
-    public class ImportListItemInfo
+    public class ImportListItemInfo : ModelBase
     {
         public int ImportListId { get; set; }
         public string ImportList { get; set; }

src/Sonarr.Api.V3/Config/ImportListConfigController.cs

@@ -0,0 +1,27 @@
+using FluentValidation;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.ImportLists;
+using NzbDrone.Core.Validation;
+using Sonarr.Http;
+
+namespace Sonarr.Api.V3.Config
+{
+    [V3ApiController("config/importlist")]
+
+    public class ImportListConfigController : ConfigController<ImportListConfigResource>
+    {
+        public ImportListConfigController(IConfigService configService)
+            : base(configService)
+        {
+            SharedValidator.RuleFor(x => x.ListSyncTag)
+               .ValidId()
+               .WithMessage("Tag must be specified")
+               .When(x => x.ListSyncLevel == ListSyncLevelType.KeepAndTag);
+        }
+
+        protected override ImportListConfigResource ToResource(IConfigService model)
+        {
+            return ImportListConfigResourceMapper.ToResource(model);
+        }
+    }
+}

src/Sonarr.Api.V3/Config/ImportListConfigResource.cs

@@ -0,0 +1,24 @@
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.ImportLists;
+using Sonarr.Http.REST;
+
+namespace Sonarr.Api.V3.Config
+{
+    public class ImportListConfigResource : RestResource
+    {
+        public ListSyncLevelType ListSyncLevel { get; set; }
+        public int ListSyncTag { get; set; }
+    }
+
+    public static class ImportListConfigResourceMapper
+    {
+        public static ImportListConfigResource ToResource(IConfigService model)
+        {
+            return new ImportListConfigResource
+            {
+                ListSyncLevel = model.ListSyncLevel,
+                ListSyncTag = model.ListSyncTag,
+            };
+        }
+    }
+}