From c105c9a65e4ce23d25d3540ffebb8b7f324d1fdc Mon Sep 17 00:00:00 2001 From: Qstick Date: Tue, 6 Feb 2018 18:08:36 -0500 Subject: [PATCH] New: Import Lists Base (#196) * New: Import Lists Base --- frontend/src/App/AppRoutes.js | 6 + .../Components/Page/Sidebar/PageSidebar.js | 4 + .../ImportLists/ImportListSettings.js | 56 ++++ .../ImportLists/AddImportListItem.css | 44 ++++ .../ImportLists/AddImportListItem.js | 110 ++++++++ .../ImportLists/AddImportListModal.js | 25 ++ .../ImportLists/AddImportListModalContent.css | 5 + .../ImportLists/AddImportListModalContent.js | 96 +++++++ .../AddImportListModalContentConnector.js | 72 +++++ .../AddImportListPresetMenuItem.js | 49 ++++ .../ImportLists/EditImportListModal.js | 25 ++ .../EditImportListModalConnector.js | 65 +++++ .../EditImportListModalContent.css | 12 + .../ImportLists/EditImportListModalContent.js | 234 +++++++++++++++++ .../EditImportListModalContentConnector.js | 98 +++++++ .../ImportLists/ImportLists/ImportList.css | 19 ++ .../ImportLists/ImportLists/ImportList.js | 119 +++++++++ .../ImportLists/ImportLists/ImportLists.css | 20 ++ .../ImportLists/ImportLists/ImportLists.js | 117 +++++++++ .../ImportLists/ImportListsConnector.js | 62 +++++ frontend/src/Settings/Settings.js | 11 + .../src/Store/Actions/Settings/importLists.js | 113 ++++++++ frontend/src/Store/Actions/settingsActions.js | 5 + .../ImportLists/ImportListModule.cs | 23 ++ .../ImportLists/ImportListResource.cs | 55 ++++ src/Lidarr.Api.V1/Lidarr.Api.V1.csproj | 2 + .../Checks/ImportListStatusCheckFixture.cs | 87 +++++++ .../CleanupOrphanedImportListStatusFixture.cs | 54 ++++ .../FixFutureImportListStatusTimesFixture.cs | 119 +++++++++ .../ImportListServiceFixture.cs | 43 +++ .../ImportListStatusServiceFixture.cs | 67 +++++ .../ImportListSyncServiceFixture.cs | 177 +++++++++++++ .../NzbDrone.Core.Test.csproj | 6 + .../Datastore/Migration/011_import_lists.cs | 32 +++ src/NzbDrone.Core/Datastore/TableMapping.cs | 6 + .../Checks/ImportListStatusCheck.cs | 45 ++++ .../CleanupOrphanedImportListStatus.cs | 26 ++ .../FixFutureImportListStatusTimes.cs | 12 + .../Exceptions/ImportListException.cs | 23 ++ .../FetchAndParseImportListService.cs | 80 ++++++ .../HeadphonesImport/HeadphonesImport.cs | 30 +++ .../HeadphonesImport/HeadphonesImportApi.cs | 9 + .../HeadphonesImportParser.cs | 64 +++++ .../HeadphonesImportRequestGenerator.cs | 34 +++ .../HeadphonesImportSettings.cs | 35 +++ .../ImportLists/HttpImportListBase.cs | 246 ++++++++++++++++++ src/NzbDrone.Core/ImportLists/IImportList.cs | 11 + .../IImportListRequestGenerator.cs | 7 + .../ImportLists/IImportListSettings.cs | 9 + .../ImportLists/IProcessImportListResponse.cs | 10 + .../ImportLists/ImportListBase.cs | 97 +++++++ .../ImportLists/ImportListDefinition.cs | 19 ++ .../ImportLists/ImportListFactory.cs | 79 ++++++ .../ImportLists/ImportListPageableRequest.cs | 25 ++ .../ImportListPageableRequestChain.cs | 54 ++++ .../ImportLists/ImportListRepository.cs | 18 ++ .../ImportLists/ImportListRequest.cs | 21 ++ .../ImportLists/ImportListResponse.cs | 24 ++ .../ImportLists/ImportListStatus.cs | 10 + .../ImportLists/ImportListStatusRepository.cs | 19 ++ .../ImportLists/ImportListStatusService.cs | 40 +++ .../ImportLists/ImportListSyncCommand.cs | 9 + .../ImportListSyncCompleteEvent.cs | 16 ++ .../ImportLists/ImportListSyncService.cs | 136 ++++++++++ .../ImportLists/LidarrLists/LidarrLists.cs | 63 +++++ .../ImportLists/LidarrLists/LidarrListsApi.cs | 13 + .../LidarrLists/LidarrListsParser.cs | 73 ++++++ .../LidarrListsRequestGenerator.cs | 34 +++ .../LidarrLists/LidarrListsSettings.cs | 34 +++ src/NzbDrone.Core/Jobs/TaskManager.cs | 7 + .../Music/AlbumMonitoredService.cs | 8 +- src/NzbDrone.Core/Music/MonitoringOptions.cs | 7 + src/NzbDrone.Core/NzbDrone.Core.csproj | 36 +++ .../Parser/Model/ImportListItemInfo.cs | 21 ++ 74 files changed, 3538 insertions(+), 4 deletions(-) create mode 100644 frontend/src/Settings/ImportLists/ImportListSettings.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/AddImportListItem.css create mode 100644 frontend/src/Settings/ImportLists/ImportLists/AddImportListItem.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/AddImportListModal.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContent.css create mode 100644 frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContent.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContentConnector.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/AddImportListPresetMenuItem.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/EditImportListModal.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/EditImportListModalConnector.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContent.css create mode 100644 frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContent.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContentConnector.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/ImportList.css create mode 100644 frontend/src/Settings/ImportLists/ImportLists/ImportList.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/ImportLists.css create mode 100644 frontend/src/Settings/ImportLists/ImportLists/ImportLists.js create mode 100644 frontend/src/Settings/ImportLists/ImportLists/ImportListsConnector.js create mode 100644 frontend/src/Store/Actions/Settings/importLists.js create mode 100644 src/Lidarr.Api.V1/ImportLists/ImportListModule.cs create mode 100644 src/Lidarr.Api.V1/ImportLists/ImportListResource.cs create mode 100644 src/NzbDrone.Core.Test/HealthCheck/Checks/ImportListStatusCheckFixture.cs create mode 100644 src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedImportListStatusFixture.cs create mode 100644 src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureImportListStatusTimesFixture.cs create mode 100644 src/NzbDrone.Core.Test/ImportListTests/ImportListServiceFixture.cs create mode 100644 src/NzbDrone.Core.Test/ImportListTests/ImportListStatusServiceFixture.cs create mode 100644 src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/011_import_lists.cs create mode 100644 src/NzbDrone.Core/HealthCheck/Checks/ImportListStatusCheck.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedImportListStatus.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureImportListStatusTimes.cs create mode 100644 src/NzbDrone.Core/ImportLists/Exceptions/ImportListException.cs create mode 100644 src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs create mode 100644 src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImport.cs create mode 100644 src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportApi.cs create mode 100644 src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportParser.cs create mode 100644 src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportRequestGenerator.cs create mode 100644 src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportSettings.cs create mode 100644 src/NzbDrone.Core/ImportLists/HttpImportListBase.cs create mode 100644 src/NzbDrone.Core/ImportLists/IImportList.cs create mode 100644 src/NzbDrone.Core/ImportLists/IImportListRequestGenerator.cs create mode 100644 src/NzbDrone.Core/ImportLists/IImportListSettings.cs create mode 100644 src/NzbDrone.Core/ImportLists/IProcessImportListResponse.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListBase.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListDefinition.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListFactory.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListPageableRequest.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListPageableRequestChain.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListRepository.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListRequest.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListResponse.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListStatus.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListStatusRepository.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListStatusService.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListSyncCommand.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListSyncCompleteEvent.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListSyncService.cs create mode 100644 src/NzbDrone.Core/ImportLists/LidarrLists/LidarrLists.cs create mode 100644 src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsApi.cs create mode 100644 src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsParser.cs create mode 100644 src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsRequestGenerator.cs create mode 100644 src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsSettings.cs create mode 100644 src/NzbDrone.Core/Parser/Model/ImportListItemInfo.cs diff --git a/frontend/src/App/AppRoutes.js b/frontend/src/App/AppRoutes.js index df0216f1f..b36b27fbf 100644 --- a/frontend/src/App/AppRoutes.js +++ b/frontend/src/App/AppRoutes.js @@ -22,6 +22,7 @@ import MediaManagementConnector from 'Settings/MediaManagement/MediaManagementCo import Profiles from 'Settings/Profiles/Profiles'; import Quality from 'Settings/Quality/Quality'; import IndexerSettings from 'Settings/Indexers/IndexerSettings'; +import ImportListSettings from 'Settings/ImportLists/ImportListSettings'; import DownloadClientSettings from 'Settings/DownloadClients/DownloadClientSettings'; import NotificationSettings from 'Settings/Notifications/NotificationSettings'; import MetadataSettings from 'Settings/Metadata/MetadataSettings'; @@ -175,6 +176,11 @@ function AppRoutes(props) { component={DownloadClientSettings} /> + + { + this._listOptions = ref; + } + + onHasPendingChange = (hasPendingChanges) => { + this.setState({ + hasPendingChanges + }); + } + + onSavePress = () => { + this._listOptions.getWrappedInstance().save(); + } + + // + // Render + + render() { + return ( + + + + + + + + ); + } +} + +export default ImportListSettings; diff --git a/frontend/src/Settings/ImportLists/ImportLists/AddImportListItem.css b/frontend/src/Settings/ImportLists/ImportLists/AddImportListItem.css new file mode 100644 index 000000000..3ce8ab9d5 --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/AddImportListItem.css @@ -0,0 +1,44 @@ +.list { + composes: card from 'Components/Card.css'; + + position: relative; + width: 300px; + height: 100px; +} + +.underlay { + @add-mixin cover; +} + +.overlay { + @add-mixin linkOverlay; + + padding: 10px; +} + +.name { + text-align: center; + font-weight: lighter; + font-size: 24px; +} + +.actions { + margin-top: 20px; + text-align: right; +} + +.presetsMenu { + composes: menu from 'Components/Menu/Menu.css'; + + display: inline-block; + margin: 0 5px; +} + +.presetsMenuButton { + composes: button from 'Components/Link/Button.css'; + + &::after { + margin-left: 5px; + content: '\25BE'; + } +} diff --git a/frontend/src/Settings/ImportLists/ImportLists/AddImportListItem.js b/frontend/src/Settings/ImportLists/ImportLists/AddImportListItem.js new file mode 100644 index 000000000..21058636c --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/AddImportListItem.js @@ -0,0 +1,110 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import { sizes } from 'Helpers/Props'; +import Button from 'Components/Link/Button'; +import Link from 'Components/Link/Link'; +import Menu from 'Components/Menu/Menu'; +import MenuContent from 'Components/Menu/MenuContent'; +import AddImportListPresetMenuItem from './AddImportListPresetMenuItem'; +import styles from './AddImportListItem.css'; + +class AddImportListItem extends Component { + + // + // Listeners + + onImportListSelect = () => { + const { + implementation + } = this.props; + + this.props.onImportListSelect({ implementation }); + } + + // + // Render + + render() { + const { + implementation, + implementationName, + infoLink, + presets, + onImportListSelect + } = this.props; + + const hasPresets = !!presets && !!presets.length; + + return ( +
+ + +
+
+ {implementationName} +
+ +
+ { + hasPresets && + + + + + + + + { + presets.map((preset) => { + return ( + + ); + }) + } + + + + } + + +
+
+
+ ); + } +} + +AddImportListItem.propTypes = { + implementation: PropTypes.string.isRequired, + implementationName: PropTypes.string.isRequired, + infoLink: PropTypes.string.isRequired, + presets: PropTypes.arrayOf(PropTypes.object), + onImportListSelect: PropTypes.func.isRequired +}; + +export default AddImportListItem; diff --git a/frontend/src/Settings/ImportLists/ImportLists/AddImportListModal.js b/frontend/src/Settings/ImportLists/ImportLists/AddImportListModal.js new file mode 100644 index 000000000..a188d6b4a --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/AddImportListModal.js @@ -0,0 +1,25 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import Modal from 'Components/Modal/Modal'; +import AddImportListModalContentConnector from './AddImportListModalContentConnector'; + +function AddImportListModal({ isOpen, onModalClose, ...otherProps }) { + return ( + + + + ); +} + +AddImportListModal.propTypes = { + isOpen: PropTypes.bool.isRequired, + onModalClose: PropTypes.func.isRequired +}; + +export default AddImportListModal; diff --git a/frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContent.css b/frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContent.css new file mode 100644 index 000000000..8454ca79f --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContent.css @@ -0,0 +1,5 @@ +.lists { + display: flex; + justify-content: center; + flex-wrap: wrap; +} diff --git a/frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContent.js b/frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContent.js new file mode 100644 index 000000000..66844e399 --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContent.js @@ -0,0 +1,96 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import { kinds } from 'Helpers/Props'; +import Alert from 'Components/Alert'; +import Button from 'Components/Link/Button'; +import FieldSet from 'Components/FieldSet'; +import LoadingIndicator from 'Components/Loading/LoadingIndicator'; +import ModalContent from 'Components/Modal/ModalContent'; +import ModalHeader from 'Components/Modal/ModalHeader'; +import ModalBody from 'Components/Modal/ModalBody'; +import ModalFooter from 'Components/Modal/ModalFooter'; +import AddImportListItem from './AddImportListItem'; +import styles from './AddImportListModalContent.css'; + +class AddImportListModalContent extends Component { + + // + // Render + + render() { + const { + isFetching, + isPopulated, + error, + allLists, + onImportListSelect, + onModalClose + } = this.props; + + return ( + + + Add List + + + + { + isFetching && + + } + + { + !isFetching && !!error && +
Unable to add a new list, please try again.
+ } + + { + isPopulated && !error && +
+ + +
Lidarr supports multiple lists for importing Albums and Artists into the database.
+
For more information on the individual lists, clink on the info buttons.
+ + +
+
+ { + allLists.map((list) => { + return ( + + ); + }) + } +
+
+
+ } + + + + + + ); + } +} + +AddImportListModalContent.propTypes = { + isFetching: PropTypes.bool.isRequired, + isPopulated: PropTypes.bool.isRequired, + error: PropTypes.object, + allLists: PropTypes.arrayOf(PropTypes.object).isRequired, + onImportListSelect: PropTypes.func.isRequired, + onModalClose: PropTypes.func.isRequired +}; + +export default AddImportListModalContent; diff --git a/frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContentConnector.js b/frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContentConnector.js new file mode 100644 index 000000000..6985076ab --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/AddImportListModalContentConnector.js @@ -0,0 +1,72 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import { connect } from 'react-redux'; +import { createSelector } from 'reselect'; +import { fetchImportListSchema, selectImportListSchema } from 'Store/Actions/settingsActions'; +import AddImportListModalContent from './AddImportListModalContent'; + +function createMapStateToProps() { + return createSelector( + (state) => state.settings.importLists, + (importLists) => { + const { + isFetching, + error, + isPopulated, + schema + } = importLists; + + const allLists = schema; + + return { + isFetching, + error, + isPopulated, + allLists + }; + } + ); +} + +const mapDispatchToProps = { + fetchImportListSchema, + selectImportListSchema +}; + +class AddImportListModalContentConnector extends Component { + + // + // Lifecycle + + componentDidMount() { + this.props.fetchImportListSchema(); + } + + // + // Listeners + + onImportListSelect = ({ implementation, name }) => { + this.props.selectImportListSchema({ implementation, presetName: name }); + this.props.onModalClose({ listSelected: true }); + } + + // + // Render + + render() { + return ( + + ); + } +} + +AddImportListModalContentConnector.propTypes = { + fetchImportListSchema: PropTypes.func.isRequired, + selectImportListSchema: PropTypes.func.isRequired, + onModalClose: PropTypes.func.isRequired +}; + +export default connect(createMapStateToProps, mapDispatchToProps)(AddImportListModalContentConnector); diff --git a/frontend/src/Settings/ImportLists/ImportLists/AddImportListPresetMenuItem.js b/frontend/src/Settings/ImportLists/ImportLists/AddImportListPresetMenuItem.js new file mode 100644 index 000000000..477044ae0 --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/AddImportListPresetMenuItem.js @@ -0,0 +1,49 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import MenuItem from 'Components/Menu/MenuItem'; + +class AddImportListPresetMenuItem extends Component { + + // + // Listeners + + onPress = () => { + const { + name, + implementation + } = this.props; + + this.props.onPress({ + name, + implementation + }); + } + + // + // Render + + render() { + const { + name, + implementation, + ...otherProps + } = this.props; + + return ( + + {name} + + ); + } +} + +AddImportListPresetMenuItem.propTypes = { + name: PropTypes.string.isRequired, + implementation: PropTypes.string.isRequired, + onPress: PropTypes.func.isRequired +}; + +export default AddImportListPresetMenuItem; diff --git a/frontend/src/Settings/ImportLists/ImportLists/EditImportListModal.js b/frontend/src/Settings/ImportLists/ImportLists/EditImportListModal.js new file mode 100644 index 000000000..b673ae9a4 --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/EditImportListModal.js @@ -0,0 +1,25 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import Modal from 'Components/Modal/Modal'; +import EditImportListModalContentConnector from './EditImportListModalContentConnector'; + +function EditImportListModal({ isOpen, onModalClose, ...otherProps }) { + return ( + + + + ); +} + +EditImportListModal.propTypes = { + isOpen: PropTypes.bool.isRequired, + onModalClose: PropTypes.func.isRequired +}; + +export default EditImportListModal; diff --git a/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalConnector.js b/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalConnector.js new file mode 100644 index 000000000..72d39817b --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalConnector.js @@ -0,0 +1,65 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import { connect } from 'react-redux'; +import { clearPendingChanges } from 'Store/Actions/baseActions'; +import { cancelTestImportList, cancelSaveImportList } from 'Store/Actions/settingsActions'; +import EditImportListModal from './EditImportListModal'; + +function createMapDispatchToProps(dispatch, props) { + const section = 'settings.importLists'; + + return { + dispatchClearPendingChanges() { + dispatch(clearPendingChanges({ section })); + }, + + dispatchCancelTestImportList() { + dispatch(cancelTestImportList({ section })); + }, + + dispatchCancelSaveImportList() { + dispatch(cancelSaveImportList({ section })); + } + }; +} + +class EditImportListModalConnector extends Component { + + // + // Listeners + + onModalClose = () => { + this.props.dispatchClearPendingChanges(); + this.props.dispatchCancelTestImportList(); + this.props.dispatchCancelSaveImportList(); + this.props.onModalClose(); + } + + // + // Render + + render() { + const { + dispatchClearPendingChanges, + dispatchCancelTestImportList, + dispatchCancelSaveImportList, + ...otherProps + } = this.props; + + return ( + + ); + } +} + +EditImportListModalConnector.propTypes = { + onModalClose: PropTypes.func.isRequired, + dispatchClearPendingChanges: PropTypes.func.isRequired, + dispatchCancelTestImportList: PropTypes.func.isRequired, + dispatchCancelSaveImportList: PropTypes.func.isRequired +}; + +export default connect(null, createMapDispatchToProps)(EditImportListModalConnector); diff --git a/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContent.css b/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContent.css new file mode 100644 index 000000000..00080a3cb --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContent.css @@ -0,0 +1,12 @@ +.deleteButton { + composes: button from 'Components/Link/Button.css'; + + margin-right: auto; +} + +.hideLanguageProfile, +.hideMetadataProfile { + composes: group from 'Components/Form/FormGroup.css'; + + display: none; +} diff --git a/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContent.js b/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContent.js new file mode 100644 index 000000000..438f0e15b --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContent.js @@ -0,0 +1,234 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import { inputTypes, kinds } from 'Helpers/Props'; +import Button from 'Components/Link/Button'; +import SpinnerErrorButton from 'Components/Link/SpinnerErrorButton'; +import LoadingIndicator from 'Components/Loading/LoadingIndicator'; +import ModalContent from 'Components/Modal/ModalContent'; +import ModalHeader from 'Components/Modal/ModalHeader'; +import ModalBody from 'Components/Modal/ModalBody'; +import ModalFooter from 'Components/Modal/ModalFooter'; +import Form from 'Components/Form/Form'; +import FormGroup from 'Components/Form/FormGroup'; +import FormLabel from 'Components/Form/FormLabel'; +import FormInputGroup from 'Components/Form/FormInputGroup'; +import ProviderFieldFormGroup from 'Components/Form/ProviderFieldFormGroup'; +import styles from './EditImportListModalContent.css'; + +function EditImportListModalContent(props) { + const { + advancedSettings, + isFetching, + error, + isSaving, + isTesting, + saveError, + item, + onInputChange, + onFieldChange, + onModalClose, + onSavePress, + onTestPress, + onDeleteImportListPress, + showLanguageProfile, + showMetadataProfile, + ...otherProps + } = props; + + const { + id, + name, + enableAutomaticAdd, + shouldMonitor, + rootFolderPath, + profileId, + languageProfileId, + metadataProfileId, + fields + } = item; + + return ( + + + {id ? 'Edit List' : 'Add List'} + + + + { + isFetching && + + } + + { + !isFetching && !!error && +
Unable to add a new list, please try again.
+ } + + { + !isFetching && !error && +
+ + Name + + + + + + Enable Automatic Add + + + + + + Monitor + + + + + + Root Folder + + + + + + Quality Profile + + + + + + Language Profile + + + + + + Metadata Profile + + + + + { + !!fields && !!fields.length && +
+ { + fields.map((field) => { + return ( + + ); + }) + } +
+ } + +
+ } + + + { + id && + + } + + + Test + + + + + + Save + + + + ); +} + +EditImportListModalContent.propTypes = { + advancedSettings: PropTypes.bool.isRequired, + isFetching: PropTypes.bool.isRequired, + error: PropTypes.object, + isSaving: PropTypes.bool.isRequired, + isTesting: PropTypes.bool.isRequired, + saveError: PropTypes.object, + item: PropTypes.object.isRequired, + showLanguageProfile: PropTypes.bool.isRequired, + showMetadataProfile: PropTypes.bool.isRequired, + onInputChange: PropTypes.func.isRequired, + onFieldChange: PropTypes.func.isRequired, + onModalClose: PropTypes.func.isRequired, + onSavePress: PropTypes.func.isRequired, + onTestPress: PropTypes.func.isRequired, + onDeleteImportListPress: PropTypes.func +}; + +export default EditImportListModalContent; diff --git a/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContentConnector.js b/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContentConnector.js new file mode 100644 index 000000000..e98b1dbf2 --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/EditImportListModalContentConnector.js @@ -0,0 +1,98 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import { createSelector } from 'reselect'; +import createProviderSettingsSelector from 'Store/Selectors/createProviderSettingsSelector'; +import { setImportListValue, setImportListFieldValue, saveImportList, testImportList } from 'Store/Actions/settingsActions'; +import connectSection from 'Store/connectSection'; +import EditImportListModalContent from './EditImportListModalContent'; + +function createMapStateToProps() { + return createSelector( + (state) => state.settings.advancedSettings, + (state) => state.settings.languageProfiles, + (state) => state.settings.metadataProfiles, + createProviderSettingsSelector(), + (advancedSettings, languageProfiles, metadataProfiles, importList) => { + return { + advancedSettings, + showLanguageProfile: languageProfiles.items.length > 1, + showMetadataProfile: metadataProfiles.items.length > 1, + ...importList + }; + } + ); +} + +const mapDispatchToProps = { + setImportListValue, + setImportListFieldValue, + saveImportList, + testImportList +}; + +class EditImportListModalContentConnector extends Component { + + // + // Lifecycle + + componentDidUpdate(prevProps, prevState) { + if (prevProps.isSaving && !this.props.isSaving && !this.props.saveError) { + this.props.onModalClose(); + } + } + + // + // Listeners + + onInputChange = ({ name, value }) => { + this.props.setImportListValue({ name, value }); + } + + onFieldChange = ({ name, value }) => { + this.props.setImportListFieldValue({ name, value }); + } + + onSavePress = () => { + this.props.saveImportList({ id: this.props.id }); + } + + onTestPress = () => { + this.props.testImportList({ id: this.props.id }); + } + + // + // Render + + render() { + return ( + + ); + } +} + +EditImportListModalContentConnector.propTypes = { + id: PropTypes.number, + isFetching: PropTypes.bool.isRequired, + isSaving: PropTypes.bool.isRequired, + saveError: PropTypes.object, + item: PropTypes.object.isRequired, + setImportListValue: PropTypes.func.isRequired, + setImportListFieldValue: PropTypes.func.isRequired, + saveImportList: PropTypes.func.isRequired, + testImportList: PropTypes.func.isRequired, + onModalClose: PropTypes.func.isRequired +}; + +export default connectSection( + createMapStateToProps, + mapDispatchToProps, + undefined, + undefined, + { section: 'importLists' } +)(EditImportListModalContentConnector); diff --git a/frontend/src/Settings/ImportLists/ImportLists/ImportList.css b/frontend/src/Settings/ImportLists/ImportLists/ImportList.css new file mode 100644 index 000000000..4b7a5cc84 --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/ImportList.css @@ -0,0 +1,19 @@ +.list { + composes: card from 'Components/Card.css'; + + width: 290px; +} + +.name { + @add-mixin truncate; + + margin-bottom: 20px; + font-weight: 300; + font-size: 24px; +} + +.enabled { + display: flex; + flex-wrap: wrap; + margin-top: 5px; +} diff --git a/frontend/src/Settings/ImportLists/ImportLists/ImportList.js b/frontend/src/Settings/ImportLists/ImportLists/ImportList.js new file mode 100644 index 000000000..407486e3a --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/ImportList.js @@ -0,0 +1,119 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import { kinds } from 'Helpers/Props'; +import Card from 'Components/Card'; +import Label from 'Components/Label'; +import ConfirmModal from 'Components/Modal/ConfirmModal'; +import EditImportListModalConnector from './EditImportListModalConnector'; +import styles from './ImportList.css'; + +function getLabelKind(supports, enabled) { + if (!supports) { + return kinds.DEFAULT; + } + + if (!enabled) { + return kinds.DANGER; + } + + return kinds.SUCCESS; +} + +class ImportList extends Component { + + // + // Lifecycle + + constructor(props, context) { + super(props, context); + + this.state = { + isEditImportListModalOpen: false, + isDeleteImportListModalOpen: false + }; + } + + // + // Listeners + + onEditImportListPress = () => { + this.setState({ isEditImportListModalOpen: true }); + } + + onEditImportListModalClose = () => { + this.setState({ isEditImportListModalOpen: false }); + } + + onDeleteImportListPress = () => { + this.setState({ + isEditImportListModalOpen: false, + isDeleteImportListModalOpen: true + }); + } + + onDeleteImportListModalClose= () => { + this.setState({ isDeleteImportListModalOpen: false }); + } + + onConfirmDeleteImportList = () => { + this.props.onConfirmDeleteImportList(this.props.id); + } + + // + // Render + + render() { + const { + id, + name, + enableAutomaticAdd + } = this.props; + + return ( + +
+ {name} +
+ +
+ +
+ + + + + + ); + } +} + +ImportList.propTypes = { + id: PropTypes.number.isRequired, + name: PropTypes.string.isRequired, + enableAutomaticAdd: PropTypes.bool.isRequired, + onConfirmDeleteImportList: PropTypes.func.isRequired +}; + +export default ImportList; diff --git a/frontend/src/Settings/ImportLists/ImportLists/ImportLists.css b/frontend/src/Settings/ImportLists/ImportLists/ImportLists.css new file mode 100644 index 000000000..3913ac52e --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/ImportLists.css @@ -0,0 +1,20 @@ +.lists { + display: flex; + flex-wrap: wrap; +} + +.addList { + composes: list from './ImportList.css'; + + background-color: $cardAlternateBackgroundColor; + color: $gray; + text-align: center; +} + +.center { + display: inline-block; + padding: 5px 20px 0; + border: 1px solid $borderColor; + border-radius: 4px; + background-color: $white; +} diff --git a/frontend/src/Settings/ImportLists/ImportLists/ImportLists.js b/frontend/src/Settings/ImportLists/ImportLists/ImportLists.js new file mode 100644 index 000000000..7f654050e --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/ImportLists.js @@ -0,0 +1,117 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import sortByName from 'Utilities/Array/sortByName'; +import { icons } from 'Helpers/Props'; +import FieldSet from 'Components/FieldSet'; +import Card from 'Components/Card'; +import Icon from 'Components/Icon'; +import PageSectionContent from 'Components/Page/PageSectionContent'; +import ImportList from './ImportList'; +import AddImportListModal from './AddImportListModal'; +import EditImportListModalConnector from './EditImportListModalConnector'; +import styles from './ImportLists.css'; + +class ImportLists extends Component { + + // + // Lifecycle + + constructor(props, context) { + super(props, context); + + this.state = { + isAddImportListModalOpen: false, + isEditImportListModalOpen: false + }; + } + + // + // Listeners + + onAddImportListPress = () => { + this.setState({ isAddImportListModalOpen: true }); + } + + onAddImportListModalClose = ({ listSelected = false } = {}) => { + this.setState({ + isAddImportListModalOpen: false, + isEditImportListModalOpen: listSelected + }); + } + + onEditImportListModalClose = () => { + this.setState({ isEditImportListModalOpen: false }); + } + + // + // Render + + render() { + const { + items, + onConfirmDeleteImportList, + ...otherProps + } = this.props; + + const { + isAddImportListModalOpen, + isEditImportListModalOpen + } = this.state; + + return ( +
+ +
+ { + items.sort(sortByName).map((item) => { + return ( + + ); + }) + } + + +
+ +
+ +
+ + + + + +
+ ); + } +} + +ImportLists.propTypes = { + isFetching: PropTypes.bool.isRequired, + error: PropTypes.object, + items: PropTypes.arrayOf(PropTypes.object).isRequired, + onConfirmDeleteImportList: PropTypes.func.isRequired +}; + +export default ImportLists; diff --git a/frontend/src/Settings/ImportLists/ImportLists/ImportListsConnector.js b/frontend/src/Settings/ImportLists/ImportLists/ImportListsConnector.js new file mode 100644 index 000000000..3938da4ae --- /dev/null +++ b/frontend/src/Settings/ImportLists/ImportLists/ImportListsConnector.js @@ -0,0 +1,62 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import { connect } from 'react-redux'; +import { createSelector } from 'reselect'; +import { fetchImportLists, deleteImportList } from 'Store/Actions/settingsActions'; +import { fetchRootFolders } from 'Store/Actions/rootFolderActions'; +import ImportLists from './ImportLists'; + +function createMapStateToProps() { + return createSelector( + (state) => state.settings.importLists, + (importLists) => { + return { + ...importLists + }; + } + ); +} + +const mapDispatchToProps = { + fetchImportLists, + deleteImportList, + fetchRootFolders +}; + +class ListsConnector extends Component { + + // + // Lifecycle + + componentDidMount() { + this.props.fetchImportLists(); + this.props.fetchRootFolders(); + } + + // + // Listeners + + onConfirmDeleteImportList = (id) => { + this.props.deleteImportList({ id }); + } + + // + // Render + + render() { + return ( + + ); + } +} + +ListsConnector.propTypes = { + fetchImportLists: PropTypes.func.isRequired, + deleteImportList: PropTypes.func.isRequired, + fetchRootFolders: PropTypes.func.isRequired +}; + +export default connect(createMapStateToProps, mapDispatchToProps)(ListsConnector); diff --git a/frontend/src/Settings/Settings.js b/frontend/src/Settings/Settings.js index c516c7eb2..96c878d18 100644 --- a/frontend/src/Settings/Settings.js +++ b/frontend/src/Settings/Settings.js @@ -68,6 +68,17 @@ function Settings() { Download clients, download handling and remote path mappings + + Import Lists + + +
+ Import Lists +
+ { + return { + section, + ...payload + }; +}); + +export const setImportListFieldValue = createAction(SET_IMPORT_LIST_FIELD_VALUE, (payload) => { + return { + section, + ...payload + }; +}); + +// +// Details + +export default { + + // + // State + + defaultState: { + isFetching: false, + isPopulated: false, + error: null, + isFetchingSchema: false, + isSchemaPopulated: false, + schemaError: null, + schema: [], + selectedSchema: {}, + isSaving: false, + saveError: null, + isTesting: false, + items: [], + pendingChanges: {} + }, + + // + // Action Handlers + + actionHandlers: { + [FETCH_IMPORT_LISTS]: createFetchHandler(section, '/importlist'), + [FETCH_IMPORT_LIST_SCHEMA]: createFetchSchemaHandler(section, '/importlist/schema'), + + [SAVE_IMPORT_LIST]: createSaveProviderHandler(section, '/importlist'), + [CANCEL_SAVE_IMPORT_LIST]: createCancelSaveProviderHandler(section), + [DELETE_IMPORT_LIST]: createRemoveItemHandler(section, '/importlist'), + [TEST_IMPORT_LIST]: createTestProviderHandler(section, '/importlist'), + [CANCEL_TEST_IMPORT_LIST]: createCancelTestProviderHandler(section) + }, + + // + // Reducers + + reducers: { + [SET_IMPORT_LIST_VALUE]: createSetSettingValueReducer(section), + [SET_IMPORT_LIST_FIELD_VALUE]: createSetProviderFieldValueReducer(section), + + [SELECT_IMPORT_LIST_SCHEMA]: (state, { payload }) => { + return selectProviderSchema(state, section, payload, (selectedSchema) => { + selectedSchema.enableRss = selectedSchema.supportsRss; + selectedSchema.enableSearch = selectedSchema.supportsSearch; + + return selectedSchema; + }); + } + } + +}; diff --git a/frontend/src/Store/Actions/settingsActions.js b/frontend/src/Store/Actions/settingsActions.js index 07b6620dc..5eaa1b81c 100644 --- a/frontend/src/Store/Actions/settingsActions.js +++ b/frontend/src/Store/Actions/settingsActions.js @@ -8,6 +8,7 @@ import general from './Settings/general'; import indexerOptions from './Settings/indexerOptions'; import indexers from './Settings/indexers'; import languageProfiles from './Settings/languageProfiles'; +import importLists from './Settings/importLists'; import metadataProfiles from './Settings/metadataProfiles'; import mediaManagement from './Settings/mediaManagement'; import metadata from './Settings/metadata'; @@ -25,6 +26,7 @@ export * from './Settings/delayProfiles'; export * from './Settings/downloadClients'; export * from './Settings/downloadClientOptions'; export * from './Settings/general'; +export * from './Settings/importLists'; export * from './Settings/indexerOptions'; export * from './Settings/indexers'; export * from './Settings/languageProfiles'; @@ -59,6 +61,7 @@ export const defaultState = { indexerOptions: indexerOptions.defaultState, indexers: indexers.defaultState, languageProfiles: languageProfiles.defaultState, + importLists: importLists.defaultState, metadataProfiles: metadataProfiles.defaultState, mediaManagement: mediaManagement.defaultState, metadata: metadata.defaultState, @@ -98,6 +101,7 @@ export const actionHandlers = handleThunks({ ...indexerOptions.actionHandlers, ...indexers.actionHandlers, ...languageProfiles.actionHandlers, + ...importLists.actionHandlers, ...metadataProfiles.actionHandlers, ...mediaManagement.actionHandlers, ...metadata.actionHandlers, @@ -128,6 +132,7 @@ export const reducers = createHandleActions({ ...indexerOptions.reducers, ...indexers.reducers, ...languageProfiles.reducers, + ...importLists.reducers, ...metadataProfiles.reducers, ...mediaManagement.reducers, ...metadata.reducers, diff --git a/src/Lidarr.Api.V1/ImportLists/ImportListModule.cs b/src/Lidarr.Api.V1/ImportLists/ImportListModule.cs new file mode 100644 index 000000000..122dbef7c --- /dev/null +++ b/src/Lidarr.Api.V1/ImportLists/ImportListModule.cs @@ -0,0 +1,23 @@ +using NzbDrone.Core.ImportLists; + +namespace Lidarr.Api.V1.ImportLists +{ + public class ImportListModule : ProviderModuleBase + { + public static readonly ImportListResourceMapper ResourceMapper = new ImportListResourceMapper(); + + public ImportListModule(ImportListFactory importListFactory) + : base(importListFactory, "importlist", ResourceMapper) + { + } + + protected override void Validate(ImportListDefinition definition, bool includeWarnings) + { + if (!definition.Enable) + { + return; + } + base.Validate(definition, includeWarnings); + } + } +} diff --git a/src/Lidarr.Api.V1/ImportLists/ImportListResource.cs b/src/Lidarr.Api.V1/ImportLists/ImportListResource.cs new file mode 100644 index 000000000..6ea6fee9e --- /dev/null +++ b/src/Lidarr.Api.V1/ImportLists/ImportListResource.cs @@ -0,0 +1,55 @@ +using NzbDrone.Core.ImportLists; + +namespace Lidarr.Api.V1.ImportLists +{ + public class ImportListResource : ProviderResource + { + public bool EnableAutomaticAdd { get; set; } + public bool ShouldMonitor { get; set; } + public string RootFolderPath { get; set; } + public int ProfileId { get; set; } + public int LanguageProfileId { get; set; } + public int MetadataProfileId { get; set; } + } + + public class ImportListResourceMapper : ProviderResourceMapper + { + public override ImportListResource ToResource(ImportListDefinition definition) + { + if (definition == null) + { + return null; + } + + var resource = base.ToResource(definition); + + resource.EnableAutomaticAdd = definition.EnableAutomaticAdd; + resource.ShouldMonitor = definition.ShouldMonitor; + resource.RootFolderPath = definition.RootFolderPath; + resource.ProfileId = definition.ProfileId; + resource.LanguageProfileId = definition.LanguageProfileId; + resource.MetadataProfileId = definition.MetadataProfileId; + + return resource; + } + + public override ImportListDefinition ToModel(ImportListResource resource) + { + if (resource == null) + { + return null; + } + + var definition = base.ToModel(resource); + + definition.EnableAutomaticAdd = resource.EnableAutomaticAdd; + definition.ShouldMonitor = resource.ShouldMonitor; + definition.RootFolderPath = resource.RootFolderPath; + definition.ProfileId = resource.ProfileId; + definition.LanguageProfileId = resource.LanguageProfileId; + definition.MetadataProfileId = resource.MetadataProfileId; + + return definition; + } + } +} diff --git a/src/Lidarr.Api.V1/Lidarr.Api.V1.csproj b/src/Lidarr.Api.V1/Lidarr.Api.V1.csproj index 954b6e39f..d22faf472 100644 --- a/src/Lidarr.Api.V1/Lidarr.Api.V1.csproj +++ b/src/Lidarr.Api.V1/Lidarr.Api.V1.csproj @@ -96,6 +96,8 @@ + + diff --git a/src/NzbDrone.Core.Test/HealthCheck/Checks/ImportListStatusCheckFixture.cs b/src/NzbDrone.Core.Test/HealthCheck/Checks/ImportListStatusCheckFixture.cs new file mode 100644 index 000000000..a69efdfb1 --- /dev/null +++ b/src/NzbDrone.Core.Test/HealthCheck/Checks/ImportListStatusCheckFixture.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.HealthCheck.Checks; +using NzbDrone.Core.ImportLists; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.HealthCheck.Checks +{ + [TestFixture] + public class ImportListStatusCheckFixture : CoreTest + { + private List _importLists = new List(); + private List _blockedImportLists = new List(); + + [SetUp] + public void SetUp() + { + Mocker.GetMock() + .Setup(v => v.GetAvailableProviders()) + .Returns(_importLists); + + Mocker.GetMock() + .Setup(v => v.GetBlockedProviders()) + .Returns(_blockedImportLists); + } + + private Mock GivenImportList(int i, double backoffHours, double failureHours) + { + var id = i; + + var mockImportList = new Mock(); + mockImportList.SetupGet(s => s.Definition).Returns(new ImportListDefinition { Id = id }); + + _importLists.Add(mockImportList.Object); + + if (backoffHours != 0.0) + { + _blockedImportLists.Add(new ImportListStatus + { + ProviderId = id, + InitialFailure = DateTime.UtcNow.AddHours(-failureHours), + MostRecentFailure = DateTime.UtcNow.AddHours(-0.1), + EscalationLevel = 5, + DisabledTill = DateTime.UtcNow.AddHours(backoffHours) + }); + } + + return mockImportList; + } + + + [Test] + public void should_not_return_error_when_no_import_lists() + { + Subject.Check().ShouldBeOk(); + } + + [Test] + public void should_return_warning_if_import_list_unavailable() + { + GivenImportList(1, 10.0, 24.0); + GivenImportList(2, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + + [Test] + public void should_return_error_if_all_import_lists_unavailable() + { + GivenImportList(1, 10.0, 24.0); + + Subject.Check().ShouldBeError(); + } + + [Test] + public void should_return_warning_if_few_import_lists_unavailable() + { + GivenImportList(1, 10.0, 24.0); + GivenImportList(2, 10.0, 24.0); + GivenImportList(3, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + } +} diff --git a/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedImportListStatusFixture.cs b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedImportListStatusFixture.cs new file mode 100644 index 000000000..860834cdb --- /dev/null +++ b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedImportListStatusFixture.cs @@ -0,0 +1,54 @@ +using FizzWare.NBuilder; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Housekeeping.Housekeepers; +using NzbDrone.Core.ImportLists; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Housekeeping.Housekeepers +{ + [TestFixture] + public class CleanupOrphanedImportListStatusFixture : DbTest + { + private ImportListDefinition _importList; + + [SetUp] + public void Setup() + { + _importList = Builder.CreateNew() + .BuildNew(); + } + + private void GivenImportList() + { + Db.Insert(_importList); + } + + [Test] + public void should_delete_orphaned_importliststatus() + { + var status = Builder.CreateNew() + .With(h => h.ProviderId = _importList.Id) + .BuildNew(); + Db.Insert(status); + + Subject.Clean(); + AllStoredModels.Should().BeEmpty(); + } + + [Test] + public void should_not_delete_unorphaned_importliststatus() + { + GivenImportList(); + + var status = Builder.CreateNew() + .With(h => h.ProviderId = _importList.Id) + .BuildNew(); + Db.Insert(status); + + Subject.Clean(); + AllStoredModels.Should().HaveCount(1); + AllStoredModels.Should().Contain(h => h.ProviderId == _importList.Id); + } + } +} diff --git a/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureImportListStatusTimesFixture.cs b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureImportListStatusTimesFixture.cs new file mode 100644 index 000000000..77197ad34 --- /dev/null +++ b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureImportListStatusTimesFixture.cs @@ -0,0 +1,119 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using FizzWare.NBuilder; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Housekeeping.Housekeepers; +using NzbDrone.Core.ImportLists; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Test.Housekeeping.Housekeepers +{ + [TestFixture] + public class FixFutureImportListStatusTimesFixture : CoreTest + { + [Test] + public void should_set_disabled_till_when_its_too_far_in_the_future() + { + var disabledTillTime = EscalationBackOff.Periods[1]; + var importListStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.EscalationLevel = 1) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(importListStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.All( + s => s.DisabledTill.Value <= DateTime.UtcNow.AddMinutes(disabledTillTime))) + ) + ); + } + + [Test] + public void should_set_initial_failure_when_its_in_the_future() + { + var importListStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.EscalationLevel = 1) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(importListStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.All( + s => s.InitialFailure.Value <= DateTime.UtcNow)) + ) + ); + } + + [Test] + public void should_set_most_recent_failure_when_its_in_the_future() + { + var importListStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(5)) + .With(t => t.EscalationLevel = 1) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(importListStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.All( + s => s.MostRecentFailure.Value <= DateTime.UtcNow)) + ) + ); + } + + [Test] + public void should_not_change_statuses_when_times_are_in_the_past() + { + var importListStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.EscalationLevel = 0) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(importListStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.Count == 0) + ) + ); + } + + + } +} diff --git a/src/NzbDrone.Core.Test/ImportListTests/ImportListServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/ImportListServiceFixture.cs new file mode 100644 index 000000000..086821da0 --- /dev/null +++ b/src/NzbDrone.Core.Test/ImportListTests/ImportListServiceFixture.cs @@ -0,0 +1,43 @@ +using System.Collections.Generic; +using FizzWare.NBuilder; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.ImportLists; +using NzbDrone.Core.ImportLists.LidarrLists; +using NzbDrone.Core.Lifecycle; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.ImportListTests +{ + public class ImportListServiceFixture : DbTest + { + private List _importLists; + + [SetUp] + public void Setup() + { + _importLists = new List(); + + _importLists.Add(Mocker.Resolve()); + + Mocker.SetConstant>(_importLists); + } + + [Test] + public void should_remove_missing_import_lists_on_startup() + { + var repo = Mocker.Resolve(); + + Mocker.SetConstant(repo); + + var existingImportLists = Builder.CreateNew().BuildNew(); + existingImportLists.ConfigContract = typeof (LidarrListsSettings).Name; + + repo.Insert(existingImportLists); + + Subject.Handle(new ApplicationStartedEvent()); + + AllStoredModels.Should().NotContain(c => c.Id == existingImportLists.Id); + } + } +} diff --git a/src/NzbDrone.Core.Test/ImportListTests/ImportListStatusServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/ImportListStatusServiceFixture.cs new file mode 100644 index 000000000..92ca88a83 --- /dev/null +++ b/src/NzbDrone.Core.Test/ImportListTests/ImportListStatusServiceFixture.cs @@ -0,0 +1,67 @@ +using System; +using System.Linq; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.ImportLists; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.ImportListTests +{ + public class ImportListStatusServiceFixture : CoreTest + { + private DateTime _epoch; + + [SetUp] + public void SetUp() + { + _epoch = DateTime.UtcNow; + } + + private void WithStatus(ImportListStatus status) + { + Mocker.GetMock() + .Setup(v => v.FindByProviderId(1)) + .Returns(status); + + Mocker.GetMock() + .Setup(v => v.All()) + .Returns(new[] { status }); + } + + private void VerifyUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Once()); + } + + private void VerifyNoUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Never()); + } + + [Test] + public void should_cancel_backoff_on_success() + { + WithStatus(new ImportListStatus { EscalationLevel = 2 }); + + Subject.RecordSuccess(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().BeNull(); + } + + [Test] + public void should_not_store_update_if_already_okay() + { + WithStatus(new ImportListStatus { EscalationLevel = 0 }); + + Subject.RecordSuccess(1); + + VerifyNoUpdate(); + } + } +} diff --git a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs new file mode 100644 index 000000000..2a125fee3 --- /dev/null +++ b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs @@ -0,0 +1,177 @@ +using System.Linq; +using System.Collections.Generic; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.ImportLists; +using NzbDrone.Core.MetadataSource; +using NzbDrone.Core.Music; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.ImportListTests +{ + public class ImportListSyncServiceFixture : CoreTest + { + private List _importListReports; + + [SetUp] + public void SetUp() + { + var importListItem1 = new ImportListItemInfo + { + Artist = "Linkin Park" + }; + + _importListReports = new List{importListItem1}; + + Mocker.GetMock() + .Setup(v => v.Fetch()) + .Returns(_importListReports); + + Mocker.GetMock() + .Setup(v => v.SearchForNewArtist(It.IsAny())) + .Returns(new List()); + + Mocker.GetMock() + .Setup(v => v.SearchForNewAlbum(It.IsAny(), It.IsAny())) + .Returns(new List()); + + Mocker.GetMock() + .Setup(v => v.Get(It.IsAny())) + .Returns(new ImportListDefinition()); + + Mocker.GetMock() + .Setup(v => v.Fetch()) + .Returns(_importListReports); + } + + private void WithAlbum() + { + _importListReports.First().Album = "Meteora"; + } + + private void WithArtistId() + { + _importListReports.First().ArtistMusicBrainzId = "f59c5520-5f46-4d2c-b2c4-822eabf53419"; + } + + private void WithAlbumId() + { + _importListReports.First().AlbumMusicBrainzId = "09474d62-17dd-3a4f-98fb-04c65f38a479"; + } + + private void WithExistingArtist() + { + Mocker.GetMock() + .Setup(v => v.FindById(_importListReports.First().ArtistMusicBrainzId)) + .Returns(new Artist{ForeignArtistId = _importListReports.First().ArtistMusicBrainzId }); + } + + [Test] + public void should_search_if_artist_title_and_no_artist_id() + { + Subject.Execute(new ImportListSyncCommand()); + + Mocker.GetMock() + .Verify(v => v.SearchForNewArtist(It.IsAny()), Times.Once()); + } + + [Test] + public void should_not_search_if_artist_title_and_artist_id() + { + WithArtistId(); + Subject.Execute(new ImportListSyncCommand()); + + Mocker.GetMock() + .Verify(v => v.SearchForNewArtist(It.IsAny()), Times.Never()); + } + + [Test] + public void should_search_if_album_title_and_no_album_id() + { + WithAlbum(); + Subject.Execute(new ImportListSyncCommand()); + + Mocker.GetMock() + .Verify(v => v.SearchForNewAlbum(It.IsAny(), It.IsAny()), Times.Once()); + } + + [Test] + public void should_not_search_if_album_title_and_album_id() + { + WithAlbum(); + WithAlbumId(); + Subject.Execute(new ImportListSyncCommand()); + + Mocker.GetMock() + .Verify(v => v.SearchForNewAlbum(It.IsAny(), It.IsAny()), Times.Never()); + } + + [Test] + public void should_not_search_if_all_info() + { + WithArtistId(); + WithAlbum(); + WithAlbumId(); + Subject.Execute(new ImportListSyncCommand()); + + Mocker.GetMock() + .Verify(v => v.SearchForNewArtist(It.IsAny()), Times.Never()); + + Mocker.GetMock() + .Verify(v => v.SearchForNewAlbum(It.IsAny(), It.IsAny()), Times.Never()); + } + + [Test] + public void should_not_try_add_if_existing_artist() + { + WithArtistId(); + WithAlbum(); + WithAlbumId(); + WithExistingArtist(); + + Subject.Execute(new ImportListSyncCommand()); + + Mocker.GetMock() + .Verify(v => v.AddArtists(It.Is>(t=>t.Count == 0))); + } + + [Test] + public void should_add_if_not_existing_artist() + { + WithArtistId(); + WithAlbum(); + WithAlbumId(); + + Subject.Execute(new ImportListSyncCommand()); + + Mocker.GetMock() + .Verify(v => v.AddArtists(It.Is>(t => t.Count == 1))); + } + + [Test] + public void should_mark_album_for_monitor_if_album_id() + { + WithArtistId(); + WithAlbum(); + WithAlbumId(); + + Subject.Execute(new ImportListSyncCommand()); + + Mocker.GetMock() + .Verify(v => v.AddArtists(It.Is>(t => t.Count == 1 && t.First().AddOptions.AlbumsToMonitor.Contains("09474d62-17dd-3a4f-98fb-04c65f38a479")))); + } + + [Test] + public void should_not_mark_album_for_monitor_if_no_album_id() + { + WithArtistId(); + + Subject.Execute(new ImportListSyncCommand()); + + Mocker.GetMock() + .Verify(v => v.AddArtists(It.Is>(t => t.Count == 1 && t.First().AddOptions.AlbumsToMonitor.Count == 0))); + } + + } +} diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index fa1d16aca..89912fe14 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -205,6 +205,7 @@ + @@ -219,6 +220,7 @@ + @@ -227,9 +229,13 @@ + + + + diff --git a/src/NzbDrone.Core/Datastore/Migration/011_import_lists.cs b/src/NzbDrone.Core/Datastore/Migration/011_import_lists.cs new file mode 100644 index 000000000..91de5e5f3 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/011_import_lists.cs @@ -0,0 +1,32 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(11)] + public class import_lists : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Create.TableForModel("ImportLists") + .WithColumn("Name").AsString().Unique() + .WithColumn("Implementation").AsString() + .WithColumn("Settings").AsString().Nullable() + .WithColumn("ConfigContract").AsString().Nullable() + .WithColumn("EnableAutomaticAdd").AsBoolean().Nullable() + .WithColumn("RootFolderPath").AsString() + .WithColumn("ShouldMonitor").AsInt32() + .WithColumn("ProfileId").AsInt32() + .WithColumn("LanguageProfileId").AsInt32() + .WithColumn("MetadataProfileId").AsInt32(); + + Create.TableForModel("ImportListStatus") + .WithColumn("ProviderId").AsInt32().NotNullable().Unique() + .WithColumn("InitialFailure").AsDateTime().Nullable() + .WithColumn("MostRecentFailure").AsDateTime().Nullable() + .WithColumn("EscalationLevel").AsInt32().NotNullable() + .WithColumn("DisabledTill").AsDateTime().Nullable() + .WithColumn("LastSyncListInfo").AsString().Nullable(); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 7609ee825..72a2706bb 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -10,6 +10,7 @@ using NzbDrone.Core.Datastore.Extensions; using NzbDrone.Core.Download; using NzbDrone.Core.Download.Pending; using NzbDrone.Core.Indexers; +using NzbDrone.Core.ImportLists; using NzbDrone.Core.Instrumentation; using NzbDrone.Core.Jobs; using NzbDrone.Core.MediaFiles; @@ -59,6 +60,10 @@ namespace NzbDrone.Core.Datastore .Ignore(i => i.SupportsSearch) .Ignore(d => d.Tags); + Mapper.Entity().RegisterDefinition("ImportLists") + .Ignore(i => i.Enable) + .Ignore(d => d.Tags); + Mapper.Entity().RegisterDefinition("Notifications") .Ignore(i => i.SupportsOnGrab) .Ignore(i => i.SupportsOnDownload) @@ -130,6 +135,7 @@ namespace NzbDrone.Core.Datastore Mapper.Entity().RegisterModel("IndexerStatus"); Mapper.Entity().RegisterModel("DownloadClientStatus"); + Mapper.Entity().RegisterModel("ImportListStatus"); } private static void RegisterMappers() diff --git a/src/NzbDrone.Core/HealthCheck/Checks/ImportListStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/ImportListStatusCheck.cs new file mode 100644 index 000000000..4a3855146 --- /dev/null +++ b/src/NzbDrone.Core/HealthCheck/Checks/ImportListStatusCheck.cs @@ -0,0 +1,45 @@ +using System; +using System.Linq; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.ImportLists; +using NzbDrone.Core.ThingiProvider.Events; + +namespace NzbDrone.Core.HealthCheck.Checks +{ + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ProviderStatusChangedEvent))] + public class ImportListStatusCheck : HealthCheckBase + { + private readonly IImportListFactory _providerFactory; + private readonly IImportListStatusService _providerStatusService; + + public ImportListStatusCheck(IImportListFactory providerFactory, IImportListStatusService providerStatusService) + { + _providerFactory = providerFactory; + _providerStatusService = providerStatusService; + } + + public override HealthCheck Check() + { + var enabledProviders = _providerFactory.GetAvailableProviders(); + var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), + i => i.Definition.Id, + s => s.ProviderId, + (i, s) => new { ImportList = i, Status = s }) + .ToList(); + + if (backOffProviders.Empty()) + { + return new HealthCheck(GetType()); + } + + if (backOffProviders.Count == enabledProviders.Count) + { + return new HealthCheck(GetType(), HealthCheckResult.Error, "All import lists are unavailable due to failures", "#import-lists-are-unavailable-due-to-failures"); + } + + return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Import lists unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.ImportList.Definition.Name))), "#import-lsits-are-unavailable-due-to-failures"); + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedImportListStatus.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedImportListStatus.cs new file mode 100644 index 000000000..9a267ee81 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedImportListStatus.cs @@ -0,0 +1,26 @@ +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class CleanupOrphanedImportListStatus : IHousekeepingTask + { + private readonly IMainDatabase _database; + + public CleanupOrphanedImportListStatus(IMainDatabase database) + { + _database = database; + } + + public void Clean() + { + var mapper = _database.GetDataMapper(); + + mapper.ExecuteNonQuery(@"DELETE FROM ImportListStatus + WHERE Id IN ( + SELECT ImportListStatus.Id FROM ImportListStatus + LEFT OUTER JOIN ImportLists + ON ImportListStatus.ProviderId = ImportLists.Id + WHERE ImportLists.Id IS NULL)"); + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureImportListStatusTimes.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureImportListStatusTimes.cs new file mode 100644 index 000000000..2c0ce3fa0 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureImportListStatusTimes.cs @@ -0,0 +1,12 @@ +using NzbDrone.Core.ImportLists; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class FixFutureImportListStatusTimes : FixFutureProviderStatusTimes, IHousekeepingTask + { + public FixFutureImportListStatusTimes(IImportListStatusRepository importListStatusRepository) + : base(importListStatusRepository) + { + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Exceptions/ImportListException.cs b/src/NzbDrone.Core/ImportLists/Exceptions/ImportListException.cs new file mode 100644 index 000000000..cdc61c095 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Exceptions/ImportListException.cs @@ -0,0 +1,23 @@ +using NzbDrone.Common.Exceptions; + +namespace NzbDrone.Core.ImportLists.Exceptions +{ + public class ImportListException : NzbDroneException + { + private readonly ImportListResponse _importListResponse; + + public ImportListException(ImportListResponse response, string message, params object[] args) + : base(message, args) + { + _importListResponse = response; + } + + public ImportListException(ImportListResponse response, string message) + : base(message) + { + _importListResponse = response; + } + + public ImportListResponse Response => _importListResponse; + } +} diff --git a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs new file mode 100644 index 000000000..4092879f5 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs @@ -0,0 +1,80 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NLog; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Common.TPL; +using System; +using NzbDrone.Common.Extensions; + +namespace NzbDrone.Core.ImportLists +{ + public interface IFetchAndParseImportList + { + List Fetch(); + } + + public class FetchAndParseImportListService : IFetchAndParseImportList + { + private readonly IImportListFactory _importListFactory; + private readonly Logger _logger; + + public FetchAndParseImportListService(IImportListFactory importListFactory, Logger logger) + { + _importListFactory = importListFactory; + _logger = logger; + } + + public List Fetch() + { + var result = new List(); + + var importLists = _importListFactory.AutomaticAddEnabled(); + + if (!importLists.Any()) + { + _logger.Warn("No available import lists. check your configuration."); + return result; + } + + _logger.Debug("Available import lists {0}", importLists.Count); + + var taskList = new List(); + var taskFactory = new TaskFactory(TaskCreationOptions.LongRunning, TaskContinuationOptions.None); + + foreach (var importList in importLists) + { + var importListLocal = importList; + + var task = taskFactory.StartNew(() => + { + try + { + var importListReports = importListLocal.Fetch(); + + lock (result) + { + _logger.Debug("Found {0} from {1}", importListReports.Count, importList.Name); + + result.AddRange(importListReports); + } + } + catch (Exception e) + { + _logger.Error(e, "Error during Import List Sync"); + } + }).LogExceptions(); + + taskList.Add(task); + } + + Task.WaitAll(taskList.ToArray()); + + result = result.DistinctBy(r => new {r.Artist, r.Album}).ToList(); + + _logger.Debug("Found {0} reports", result.Count); + + return result; + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImport.cs b/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImport.cs new file mode 100644 index 000000000..ec22977f2 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImport.cs @@ -0,0 +1,30 @@ +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; + +namespace NzbDrone.Core.ImportLists.HeadphonesImport +{ + public class HeadphonesImport : HttpImportListBase + { + public override string Name => "Headphones"; + + public override int PageSize => 1000; + + public HeadphonesImport(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, importListStatusService, configService, parsingService, logger) + { + } + + public override IImportListRequestGenerator GetRequestGenerator() + { + return new HeadphonesImportRequestGenerator { Settings = Settings}; + } + + public override IParseImportListResponse GetParser() + { + return new HeadphonesImportParser(); + } + + } +} diff --git a/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportApi.cs b/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportApi.cs new file mode 100644 index 000000000..ce9d5341f --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportApi.cs @@ -0,0 +1,9 @@ + +namespace NzbDrone.Core.ImportLists.HeadphonesImport +{ + public class HeadphonesImportArtist + { + public string ArtistName { get; set; } + public string ArtistId { get; set; } + } +} diff --git a/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportParser.cs b/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportParser.cs new file mode 100644 index 000000000..b83ccb030 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportParser.cs @@ -0,0 +1,64 @@ +using Newtonsoft.Json; +using NzbDrone.Core.ImportLists.Exceptions; +using NzbDrone.Core.Parser.Model; +using System.Collections.Generic; +using System.Net; +using NLog; +using NzbDrone.Common.Extensions; + +namespace NzbDrone.Core.ImportLists.HeadphonesImport +{ + public class HeadphonesImportParser : IParseImportListResponse + { + private ImportListResponse _importListResponse; + private readonly Logger _logger; + + public IList ParseResponse(ImportListResponse importListResponse) + { + _importListResponse = importListResponse; + + var items = new List(); + + if (!PreProcess(_importListResponse)) + { + return items; + } + + var jsonResponse = JsonConvert.DeserializeObject>(_importListResponse.Content); + + // no albums were return + if (jsonResponse == null) + { + return items; + } + + foreach (var item in jsonResponse) + { + items.AddIfNotNull(new ImportListItemInfo + { + Artist = item.ArtistName, + ArtistMusicBrainzId = item.ArtistId + }); + } + + return items; + } + + protected virtual bool PreProcess(ImportListResponse importListResponse) + { + if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new ImportListException(importListResponse, "Import List API call resulted in an unexpected StatusCode [{0}]", importListResponse.HttpResponse.StatusCode); + } + + if (importListResponse.HttpResponse.Headers.ContentType != null && importListResponse.HttpResponse.Headers.ContentType.Contains("text/json") && + importListResponse.HttpRequest.Headers.Accept != null && !importListResponse.HttpRequest.Headers.Accept.Contains("text/json")) + { + throw new ImportListException(importListResponse, "Import List responded with html content. Site is likely blocked or unavailable."); + } + + return true; + } + + } +} diff --git a/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportRequestGenerator.cs new file mode 100644 index 000000000..664c32273 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportRequestGenerator.cs @@ -0,0 +1,34 @@ +using System.Collections.Generic; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.ImportLists.HeadphonesImport +{ + public class HeadphonesImportRequestGenerator : IImportListRequestGenerator + { + public HeadphonesImportSettings Settings { get; set; } + + public int MaxPages { get; set; } + public int PageSize { get; set; } + + public HeadphonesImportRequestGenerator() + { + MaxPages = 1; + PageSize = 1000; + } + + public virtual ImportListPageableRequestChain GetListItems() + { + var pageableRequests = new ImportListPageableRequestChain(); + + pageableRequests.Add(GetPagedRequests()); + + return pageableRequests; + } + + private IEnumerable GetPagedRequests() + { + yield return new ImportListRequest(string.Format("{0}/api?cmd=getIndex&apikey={1}", Settings.BaseUrl.TrimEnd('/'), Settings.ApiKey), HttpAccept.Json); + } + + } +} diff --git a/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportSettings.cs b/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportSettings.cs new file mode 100644 index 000000000..456b10399 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/HeadphonesImport/HeadphonesImportSettings.cs @@ -0,0 +1,35 @@ +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.ImportLists.HeadphonesImport +{ + public class HeadphonesImportSettingsValidator : AbstractValidator + { + public HeadphonesImportSettingsValidator() + { + RuleFor(c => c.BaseUrl).ValidRootUrl(); + } + } + + public class HeadphonesImportSettings : IImportListSettings + { + private static readonly HeadphonesImportSettingsValidator Validator = new HeadphonesImportSettingsValidator(); + + public HeadphonesImportSettings() + { + BaseUrl = "http://localhost:8181/"; + } + + [FieldDefinition(0, Label = "Headphones URL")] + public string BaseUrl { get; set; } + + [FieldDefinition(1, Label = "API Key")] + public string ApiKey { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs b/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs new file mode 100644 index 000000000..98942d450 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using FluentValidation.Results; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Http.CloudFlare; +using NzbDrone.Core.ImportLists.Exceptions; +using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.ImportLists +{ + public abstract class HttpImportListBase : ImportListBase + where TSettings : IImportListSettings, new() + { + protected const int MaxNumResultsPerQuery = 1000; + + protected readonly IHttpClient _httpClient; + + public bool SupportsPaging => PageSize > 0; + + public virtual int PageSize => 0; + public virtual TimeSpan RateLimit => TimeSpan.FromSeconds(2); + + public abstract IImportListRequestGenerator GetRequestGenerator(); + public abstract IParseImportListResponse GetParser(); + + public HttpImportListBase(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(importListStatusService, configService, parsingService, logger) + { + _httpClient = httpClient; + } + + public override IList Fetch() + { + return FetchReleases(g => g.GetListItems(), true); + } + + protected virtual IList FetchReleases(Func pageableRequestChainSelector, bool isRecent = false) + { + var releases = new List(); + var url = string.Empty; + + try + { + var generator = GetRequestGenerator(); + var parser = GetParser(); + + var pageableRequestChain = pageableRequestChainSelector(generator); + + for (int i = 0; i < pageableRequestChain.Tiers; i++) + { + var pageableRequests = pageableRequestChain.GetTier(i); + + foreach (var pageableRequest in pageableRequests) + { + var pagedReleases = new List(); + + foreach (var request in pageableRequest) + { + url = request.Url.FullUri; + + var page = FetchPage(request, parser); + + pagedReleases.AddRange(page); + + if (pagedReleases.Count >= MaxNumResultsPerQuery) + { + break; + } + + if (!IsFullPage(page)) + { + break; + } + } + + releases.AddRange(pagedReleases.Where(IsValidRelease)); + } + + if (releases.Any()) + { + break; + } + } + + _importListStatusService.RecordSuccess(Definition.Id); + } + catch (WebException webException) + { + if (webException.Status == WebExceptionStatus.NameResolutionFailure || + webException.Status == WebExceptionStatus.ConnectFailure) + { + _importListStatusService.RecordConnectionFailure(Definition.Id); + } + else + { + _importListStatusService.RecordFailure(Definition.Id); + } + + if (webException.Message.Contains("502") || webException.Message.Contains("503") || + webException.Message.Contains("timed out")) + { + _logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message); + } + else + { + _logger.Warn("{0} {1} {2}", this, url, webException.Message); + } + } + catch (TooManyRequestsException ex) + { + if (ex.RetryAfter != TimeSpan.Zero) + { + _importListStatusService.RecordFailure(Definition.Id, ex.RetryAfter); + } + else + { + _importListStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); + } + _logger.Warn("API Request Limit reached for {0}", this); + } + catch (HttpException ex) + { + _importListStatusService.RecordFailure(Definition.Id); + _logger.Warn("{0} {1}", this, ex.Message); + } + catch (RequestLimitReachedException) + { + _importListStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); + _logger.Warn("API Request Limit reached for {0}", this); + } + catch (CloudFlareCaptchaException ex) + { + _importListStatusService.RecordFailure(Definition.Id); + ex.WithData("FeedUrl", url); + if (ex.IsExpired) + { + _logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in import list settings.", this); + } + else + { + _logger.Error(ex, "CAPTCHA token required for {0}, check import list settings.", this); + } + } + catch (ImportListException ex) + { + _importListStatusService.RecordFailure(Definition.Id); + _logger.Warn(ex, "{0}", url); + } + catch (Exception ex) + { + _importListStatusService.RecordFailure(Definition.Id); + ex.WithData("FeedUrl", url); + _logger.Error(ex, "An error occurred while processing feed. {0}", url); + } + + return CleanupListItems(releases); + } + + protected virtual bool IsValidRelease(ImportListItemInfo release) + { + if (release.Album.IsNullOrWhiteSpace() && release.Artist.IsNullOrWhiteSpace()) + { + return false; + } + + return true; + } + + protected virtual bool IsFullPage(IList page) + { + return PageSize != 0 && page.Count >= PageSize; + } + + protected virtual IList FetchPage(ImportListRequest request, IParseImportListResponse parser) + { + var response = FetchImportListResponse(request); + + return parser.ParseResponse(response).ToList(); + } + + protected virtual ImportListResponse FetchImportListResponse(ImportListRequest request) + { + _logger.Debug("Downloading Feed " + request.HttpRequest.ToString(false)); + + if (request.HttpRequest.RateLimit < RateLimit) + { + request.HttpRequest.RateLimit = RateLimit; + } + + return new ImportListResponse(request, _httpClient.Execute(request.HttpRequest)); + } + + protected override void Test(List failures) + { + failures.AddIfNotNull(TestConnection()); + } + + protected virtual ValidationFailure TestConnection() + { + try + { + var parser = GetParser(); + var generator = GetRequestGenerator(); + var releases = FetchPage(generator.GetListItems().GetAllTiers().First().First(), parser); + + if (releases.Empty()) + { + return new ValidationFailure(string.Empty, "No results were returned from your import list, please check your settings."); + } + } + catch (RequestLimitReachedException) + { + _logger.Warn("Request limit reached"); + } + catch (UnsupportedFeedException ex) + { + _logger.Warn(ex, "Import list feed is not supported"); + + return new ValidationFailure(string.Empty, "Import list feed is not supported: " + ex.Message); + } + catch (ImportListException ex) + { + _logger.Warn(ex, "Unable to connect to import list"); + + return new ValidationFailure(string.Empty, "Unable to connect to import list. " + ex.Message); + } + catch (Exception ex) + { + _logger.Warn(ex, "Unable to connect to import list"); + + return new ValidationFailure(string.Empty, "Unable to connect to import list, check the log for more details"); + } + + return null; + } + } + +} diff --git a/src/NzbDrone.Core/ImportLists/IImportList.cs b/src/NzbDrone.Core/ImportLists/IImportList.cs new file mode 100644 index 000000000..04eb45c07 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/IImportList.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.ImportLists +{ + public interface IImportList : IProvider + { + IList Fetch(); + } +} diff --git a/src/NzbDrone.Core/ImportLists/IImportListRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/IImportListRequestGenerator.cs new file mode 100644 index 000000000..b0cbe0410 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/IImportListRequestGenerator.cs @@ -0,0 +1,7 @@ +namespace NzbDrone.Core.ImportLists +{ + public interface IImportListRequestGenerator + { + ImportListPageableRequestChain GetListItems(); + } +} diff --git a/src/NzbDrone.Core/ImportLists/IImportListSettings.cs b/src/NzbDrone.Core/ImportLists/IImportListSettings.cs new file mode 100644 index 000000000..3fbd7b7e3 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/IImportListSettings.cs @@ -0,0 +1,9 @@ +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.ImportLists +{ + public interface IImportListSettings : IProviderConfig + { + string BaseUrl { get; set; } + } +} diff --git a/src/NzbDrone.Core/ImportLists/IProcessImportListResponse.cs b/src/NzbDrone.Core/ImportLists/IProcessImportListResponse.cs new file mode 100644 index 000000000..fef025282 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/IProcessImportListResponse.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.ImportLists +{ + public interface IParseImportListResponse + { + IList ParseResponse(ImportListResponse importListResponse); + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListBase.cs b/src/NzbDrone.Core/ImportLists/ImportListBase.cs new file mode 100644 index 000000000..8864830d7 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListBase.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using FluentValidation.Results; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.ImportLists +{ + public abstract class ImportListBase : IImportList + where TSettings : IImportListSettings, new() + { + protected readonly IImportListStatusService _importListStatusService; + protected readonly IConfigService _configService; + protected readonly IParsingService _parsingService; + protected readonly Logger _logger; + + public abstract string Name { get; } + + public ImportListBase(IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + { + _importListStatusService = importListStatusService; + _configService = configService; + _parsingService = parsingService; + _logger = logger; + } + + public Type ConfigContract => typeof(TSettings); + + public virtual ProviderMessage Message => null; + + public virtual IEnumerable DefaultDefinitions + { + get + { + var config = (IProviderConfig)new TSettings(); + + yield return new ImportListDefinition + { + Name = GetType().Name, + EnableAutomaticAdd = config.Validate().IsValid, + Implementation = GetType().Name, + Settings = config + }; + } + } + + public virtual ProviderDefinition Definition { get; set; } + + public virtual object RequestAction(string action, IDictionary query) { return null; } + + protected TSettings Settings => (TSettings)Definition.Settings; + + public abstract IList Fetch(); + + protected virtual IList CleanupListItems(IEnumerable releases) + { + var result = releases.DistinctBy(r => new {r.Artist, r.Album}).ToList(); + + result.ForEach(c => + { + c.ImportListId = Definition.Id; + c.ImportList = Definition.Name; + }); + + return result; + } + + public ValidationResult Test() + { + var failures = new List(); + + try + { + Test(failures); + } + catch (Exception ex) + { + _logger.Error(ex, "Test aborted due to exception"); + failures.Add(new ValidationFailure(string.Empty, "Test was aborted due to an error: " + ex.Message)); + } + + return new ValidationResult(failures); + } + + protected abstract void Test(List failures); + + public override string ToString() + { + return Definition.Name; + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListDefinition.cs b/src/NzbDrone.Core/ImportLists/ImportListDefinition.cs new file mode 100644 index 000000000..6ffcffd4b --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListDefinition.cs @@ -0,0 +1,19 @@ +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.ImportLists +{ + public class ImportListDefinition : ProviderDefinition + { + public bool EnableAutomaticAdd { get; set; } + + public bool ShouldMonitor { get; set; } + public int ProfileId { get; set; } + public int LanguageProfileId { get; set; } + public int MetadataProfileId { get; set; } + public string RootFolderPath { get; set; } + + public override bool Enable => EnableAutomaticAdd; + + public ImportListStatus Status { get; set; } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListFactory.cs b/src/NzbDrone.Core/ImportLists/ImportListFactory.cs new file mode 100644 index 000000000..a2a793a66 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListFactory.cs @@ -0,0 +1,79 @@ +using System.Collections.Generic; +using System.Linq; +using FluentValidation.Results; +using NLog; +using NzbDrone.Common.Composition; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.ImportLists +{ + public interface IImportListFactory : IProviderFactory + { + List AutomaticAddEnabled(bool filterBlockedImportLists = true); + } + + public class ImportListFactory : ProviderFactory, IImportListFactory + { + private readonly IImportListStatusService _importListStatusService; + private readonly Logger _logger; + + public ImportListFactory(IImportListStatusService importListStatusService, + IImportListRepository providerRepository, + IEnumerable providers, + IContainer container, + IEventAggregator eventAggregator, + Logger logger) + : base(providerRepository, providers, container, eventAggregator, logger) + { + _importListStatusService = importListStatusService; + _logger = logger; + } + + protected override List Active() + { + return base.Active().Where(c => c.Enable).ToList(); + } + + public List AutomaticAddEnabled(bool filterBlockedImportLists = true) + { + var enabledImportLists = GetAvailableProviders().Where(n => ((ImportListDefinition)n.Definition).EnableAutomaticAdd); + + if (filterBlockedImportLists) + { + return FilterBlockedImportLists(enabledImportLists).ToList(); + } + + return enabledImportLists.ToList(); + } + + private IEnumerable FilterBlockedImportLists(IEnumerable importLists) + { + var blockedImportLists = _importListStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v); + + foreach (var importList in importLists) + { + ImportListStatus blockedImportListStatus; + if (blockedImportLists.TryGetValue(importList.Definition.Id, out blockedImportListStatus)) + { + _logger.Debug("Temporarily ignoring import list {0} till {1} due to recent failures.", importList.Definition.Name, blockedImportListStatus.DisabledTill.Value.ToLocalTime()); + continue; + } + + yield return importList; + } + } + + public override ValidationResult Test(ImportListDefinition definition) + { + var result = base.Test(definition); + + if ((result == null || result.IsValid) && definition.Id != 0) + { + _importListStatusService.RecordSuccess(definition.Id); + } + + return result; + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListPageableRequest.cs b/src/NzbDrone.Core/ImportLists/ImportListPageableRequest.cs new file mode 100644 index 000000000..64b4fbe81 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListPageableRequest.cs @@ -0,0 +1,25 @@ +using System.Collections; +using System.Collections.Generic; + +namespace NzbDrone.Core.ImportLists +{ + public class ImportListPageableRequest : IEnumerable + { + private readonly IEnumerable _enumerable; + + public ImportListPageableRequest(IEnumerable enumerable) + { + _enumerable = enumerable; + } + + public IEnumerator GetEnumerator() + { + return _enumerable.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return _enumerable.GetEnumerator(); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListPageableRequestChain.cs b/src/NzbDrone.Core/ImportLists/ImportListPageableRequestChain.cs new file mode 100644 index 000000000..9b2503c35 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListPageableRequestChain.cs @@ -0,0 +1,54 @@ +using System.Collections.Generic; +using System.Linq; + +namespace NzbDrone.Core.ImportLists +{ + public class ImportListPageableRequestChain + { + private List> _chains; + + public ImportListPageableRequestChain() + { + _chains = new List>(); + _chains.Add(new List()); + } + + public int Tiers => _chains.Count; + + public IEnumerable GetAllTiers() + { + return _chains.SelectMany(v => v); + } + + public IEnumerable GetTier(int index) + { + return _chains[index]; + } + + public void Add(IEnumerable request) + { + if (request == null) + { + return; + } + + _chains.Last().Add(new ImportListPageableRequest(request)); + } + + public void AddTier(IEnumerable request) + { + AddTier(); + Add(request); + } + + public void AddTier() + { + if (_chains.Last().Count == 0) + { + return; + } + + _chains.Add(new List()); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListRepository.cs b/src/NzbDrone.Core/ImportLists/ImportListRepository.cs new file mode 100644 index 000000000..8de00d5e6 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListRepository.cs @@ -0,0 +1,18 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.ImportLists +{ + public interface IImportListRepository : IProviderRepository + { + } + + public class ImportListRepository : ProviderRepository, IImportListRepository + { + public ImportListRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListRequest.cs b/src/NzbDrone.Core/ImportLists/ImportListRequest.cs new file mode 100644 index 000000000..f95b9b95e --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListRequest.cs @@ -0,0 +1,21 @@ +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.ImportLists +{ + public class ImportListRequest + { + public HttpRequest HttpRequest { get; private set; } + + public ImportListRequest(string url, HttpAccept httpAccept) + { + HttpRequest = new HttpRequest(url, httpAccept); + } + + public ImportListRequest(HttpRequest httpRequest) + { + HttpRequest = httpRequest; + } + + public HttpUri Url => HttpRequest.Url; + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListResponse.cs b/src/NzbDrone.Core/ImportLists/ImportListResponse.cs new file mode 100644 index 000000000..2dc465b8d --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListResponse.cs @@ -0,0 +1,24 @@ +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.ImportLists +{ + public class ImportListResponse + { + private readonly ImportListRequest _importListRequest; + private readonly HttpResponse _httpResponse; + + public ImportListResponse(ImportListRequest importListRequest, HttpResponse httpResponse) + { + _importListRequest = importListRequest; + _httpResponse = httpResponse; + } + + public ImportListRequest Request => _importListRequest; + + public HttpRequest HttpRequest => _httpResponse.Request; + + public HttpResponse HttpResponse => _httpResponse; + + public string Content => _httpResponse.Content; + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListStatus.cs b/src/NzbDrone.Core/ImportLists/ImportListStatus.cs new file mode 100644 index 000000000..e58976744 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListStatus.cs @@ -0,0 +1,10 @@ +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.ImportLists +{ + public class ImportListStatus : ProviderStatusBase + { + public ImportListItemInfo LastSyncListInfo { get; set; } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListStatusRepository.cs b/src/NzbDrone.Core/ImportLists/ImportListStatusRepository.cs new file mode 100644 index 000000000..8bf500a7a --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListStatusRepository.cs @@ -0,0 +1,19 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.ImportLists +{ + public interface IImportListStatusRepository : IProviderStatusRepository + { + } + + public class ImportListStatusRepository : ProviderStatusRepository, IImportListStatusRepository + + { + public ImportListStatusRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListStatusService.cs b/src/NzbDrone.Core/ImportLists/ImportListStatusService.cs new file mode 100644 index 000000000..14b51b101 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListStatusService.cs @@ -0,0 +1,40 @@ +using NLog; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.ImportLists +{ + public interface IImportListStatusService : IProviderStatusServiceBase + { + ImportListItemInfo GetLastSyncListInfo(int importListId); + + void UpdateListSyncStatus(int importListId, ImportListItemInfo listItemInfo); + } + + public class ImportListStatusService : ProviderStatusServiceBase, IImportListStatusService + { + public ImportListStatusService(IImportListStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) + : base(providerStatusRepository, eventAggregator, logger) + { + } + + public ImportListItemInfo GetLastSyncListInfo(int importListId) + { + return GetProviderStatus(importListId).LastSyncListInfo; + } + + + public void UpdateListSyncStatus(int importListId, ImportListItemInfo listItemInfo) + { + lock (_syncRoot) + { + var status = GetProviderStatus(importListId); + + status.LastSyncListInfo = listItemInfo; + + _providerStatusRepository.Upsert(status); + } + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListSyncCommand.cs b/src/NzbDrone.Core/ImportLists/ImportListSyncCommand.cs new file mode 100644 index 000000000..fafb73dde --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListSyncCommand.cs @@ -0,0 +1,9 @@ +using NzbDrone.Core.Messaging.Commands; + +namespace NzbDrone.Core.ImportLists +{ + public class ImportListSyncCommand : Command + { + public override bool SendUpdatesToClient => true; + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListSyncCompleteEvent.cs b/src/NzbDrone.Core/ImportLists/ImportListSyncCompleteEvent.cs new file mode 100644 index 000000000..b055a0383 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListSyncCompleteEvent.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using NzbDrone.Common.Messaging; +using NzbDrone.Core.Music; + +namespace NzbDrone.Core.ImportLists +{ + public class ImportListSyncCompleteEvent : IEvent + { + public List ProcessedDecisions { get; private set; } + + public ImportListSyncCompleteEvent(List processedDecisions) + { + ProcessedDecisions = processedDecisions; + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs new file mode 100644 index 000000000..3cdbc15b2 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs @@ -0,0 +1,136 @@ +using System.Collections.Generic; +using System.Linq; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Instrumentation.Extensions; +using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.MetadataSource; +using NzbDrone.Core.Music; + +namespace NzbDrone.Core.ImportLists +{ + public class ImportListSyncService : IExecute + { + private readonly IImportListStatusService _importListStatusService; + private readonly IImportListFactory _importListFactory; + private readonly IFetchAndParseImportList _listFetcherAndParser; + private readonly ISearchForNewAlbum _albumSearchService; + private readonly ISearchForNewArtist _artistSearchService; + private readonly IArtistService _artistService; + private readonly IAddArtistService _addArtistService; + private readonly IEventAggregator _eventAggregator; + private readonly Logger _logger; + + public ImportListSyncService(IImportListStatusService importListStatusService, + IImportListFactory importListFactory, + IFetchAndParseImportList listFetcherAndParser, + ISearchForNewAlbum albumSearchService, + ISearchForNewArtist artistSearchService, + IArtistService artistService, + IAddArtistService addArtistService, + IEventAggregator eventAggregator, + Logger logger) + { + _importListStatusService = importListStatusService; + _importListFactory = importListFactory; + _listFetcherAndParser = listFetcherAndParser; + _albumSearchService = albumSearchService; + _artistSearchService = artistSearchService; + _artistService = artistService; + _addArtistService = addArtistService; + _eventAggregator = eventAggregator; + _logger = logger; + } + + + private List Sync() + { + _logger.ProgressInfo("Starting Import List Sync"); + + var rssReleases = _listFetcherAndParser.Fetch(); + + var reports = rssReleases.ToList(); + var processed = new List(); + var artistsToAdd = new List(); + + _logger.ProgressInfo("Processing {0} list items", reports.Count); + + var reportNumber = 1; + + foreach (var report in reports) + { + _logger.ProgressTrace("Processing list item {0}/{1}", reportNumber, reports.Count); + + reportNumber++; + + var importList = _importListFactory.Get(report.ImportListId); + + // Map MBid if we only have an album title + if (report.AlbumMusicBrainzId.IsNullOrWhiteSpace() && report.Album.IsNotNullOrWhiteSpace()) + { + var mappedAlbum = _albumSearchService.SearchForNewAlbum(report.Album, report.Artist) + .FirstOrDefault(); + + if (mappedAlbum == null) continue; // Break if we are looking for an album and cant find it. This will avoid us from adding the artist and possibly getting it wrong. + + report.AlbumMusicBrainzId = mappedAlbum.ForeignAlbumId; + report.Album = mappedAlbum.Title; + report.Artist = mappedAlbum.Artist?.Name; + report.ArtistMusicBrainzId = mappedAlbum?.Artist?.ForeignArtistId; + + } + + // Map MBid if we only have a artist name + if (report.ArtistMusicBrainzId.IsNullOrWhiteSpace() && report.Artist.IsNotNullOrWhiteSpace()) + { + var mappedArtist = _artistSearchService.SearchForNewArtist(report.Artist) + .FirstOrDefault(); + report.ArtistMusicBrainzId = mappedArtist?.ForeignArtistId; + report.Artist = mappedArtist?.Name; + } + + // Check to see if artist in DB + var existingArtist = _artistService.FindById(report.ArtistMusicBrainzId); + + // Append Artist if not already in DB or already on add list + if (existingArtist == null && artistsToAdd.All(s => s.ForeignArtistId != report.ArtistMusicBrainzId)) + { + artistsToAdd.Add(new Artist + { + ForeignArtistId = report.ArtistMusicBrainzId, + Name = report.Artist, + Monitored = importList.ShouldMonitor, + RootFolderPath = importList.RootFolderPath, + ProfileId = importList.ProfileId, + LanguageProfileId = importList.LanguageProfileId, + MetadataProfileId = importList.MetadataProfileId, + AlbumFolder = true, + AddOptions = new AddArtistOptions{SearchForMissingAlbums = true, Monitored = importList.ShouldMonitor } + }); + } + + // Add Album so we know what to monitor + if (report.AlbumMusicBrainzId.IsNotNullOrWhiteSpace() && artistsToAdd.Any(s=>s.ForeignArtistId == report.ArtistMusicBrainzId) && importList.ShouldMonitor) + { + artistsToAdd.Find(s => s.ForeignArtistId == report.ArtistMusicBrainzId).AddOptions.AlbumsToMonitor.Add(report.AlbumMusicBrainzId); + } + } + + _addArtistService.AddArtists(artistsToAdd); + + var message = string.Format("Import List Sync Completed. Reports found: {0}, Reports grabbed: {1}", reports.Count, processed.Count); + + _logger.ProgressInfo(message); + + return processed; + } + + public void Execute(ImportListSyncCommand message) + { + var processed = Sync(); + + _eventAggregator.PublishEvent(new ImportListSyncCompleteEvent(processed)); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrLists.cs b/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrLists.cs new file mode 100644 index 000000000..8a073a0e3 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrLists.cs @@ -0,0 +1,63 @@ +using System.Collections.Generic; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.ImportLists.LidarrLists +{ + public class LidarrLists : HttpImportListBase + { + public override string Name => "Lidarr Lists"; + + public override int PageSize => 10; + + public LidarrLists(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, importListStatusService, configService, parsingService, logger) + { + + } + + public override IEnumerable DefaultDefinitions + { + get + { + yield return GetDefinition("iTunes Top Albums", GetSettings("itunes/album/top")); + yield return GetDefinition("Billboard Top Albums", GetSettings("billboard/album/top")); + yield return GetDefinition("Billboard Top Artists", GetSettings("billboard/artist/top")); + yield return GetDefinition("Last.fm Top Albums", GetSettings("lastfm/album/top")); + yield return GetDefinition("Last.fm Top Artists", GetSettings("lastfm/artist/top")); + } + } + + private ImportListDefinition GetDefinition(string name, LidarrListsSettings settings) + { + return new ImportListDefinition + { + EnableAutomaticAdd = false, + Name = name, + Implementation = GetType().Name, + Settings = settings + }; + } + + private LidarrListsSettings GetSettings(string url) + { + var settings = new LidarrListsSettings { ListId = url }; + + return settings; + } + + public override IImportListRequestGenerator GetRequestGenerator() + { + return new LidarrListsRequestGenerator { Settings = Settings, PageSize = PageSize }; + } + + public override IParseImportListResponse GetParser() + { + return new LidarrListsParser(Settings); + } + + } +} diff --git a/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsApi.cs b/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsApi.cs new file mode 100644 index 000000000..b153d5011 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsApi.cs @@ -0,0 +1,13 @@ +using System; + +namespace NzbDrone.Core.ImportLists.LidarrLists +{ + public class LidarrListsAlbum + { + public string ArtistName { get; set; } + public string AlbumTitle { get; set; } + public string ArtistId { get; set; } + public string AlbumId { get; set; } + public DateTime? ReleaseDate { get; set; } + } +} diff --git a/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsParser.cs b/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsParser.cs new file mode 100644 index 000000000..5e3e363c2 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsParser.cs @@ -0,0 +1,73 @@ +using Newtonsoft.Json; +using NzbDrone.Core.ImportLists.Exceptions; +using NzbDrone.Core.Parser.Model; +using System.Collections.Generic; +using System.Net; +using NLog; +using NzbDrone.Common.Extensions; + +namespace NzbDrone.Core.ImportLists.LidarrLists +{ + public class LidarrListsParser : IParseImportListResponse + { + private readonly LidarrListsSettings _settings; + private ImportListResponse _importListResponse; + private readonly Logger _logger; + + public LidarrListsParser(LidarrListsSettings settings) + { + _settings = settings; + } + + public IList ParseResponse(ImportListResponse importListResponse) + { + _importListResponse = importListResponse; + + var items = new List(); + + if (!PreProcess(_importListResponse)) + { + return items; + } + + var jsonResponse = JsonConvert.DeserializeObject>(_importListResponse.Content); + + // no albums were return + if (jsonResponse == null) + { + return items; + } + + foreach (var item in jsonResponse) + { + items.AddIfNotNull(new ImportListItemInfo + { + Artist = item.ArtistName, + Album = item.AlbumTitle, + ArtistMusicBrainzId = item.ArtistId, + AlbumMusicBrainzId = item.AlbumId, + ReleaseDate = item.ReleaseDate.GetValueOrDefault() + }); + } + + return items; + } + + protected virtual bool PreProcess(ImportListResponse importListResponse) + { + if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new ImportListException(importListResponse, "Import List API call resulted in an unexpected StatusCode [{0}]", importListResponse.HttpResponse.StatusCode); + } + + if (importListResponse.HttpResponse.Headers.ContentType != null && importListResponse.HttpResponse.Headers.ContentType.Contains("text/json") && + importListResponse.HttpRequest.Headers.Accept != null && !importListResponse.HttpRequest.Headers.Accept.Contains("text/json")) + { + throw new ImportListException(importListResponse, "Import List responded with html content. Site is likely blocked or unavailable."); + } + + return true; + } + + } +} diff --git a/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsRequestGenerator.cs new file mode 100644 index 000000000..d296f16fe --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsRequestGenerator.cs @@ -0,0 +1,34 @@ +using System.Collections.Generic; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.ImportLists.LidarrLists +{ + public class LidarrListsRequestGenerator : IImportListRequestGenerator + { + public LidarrListsSettings Settings { get; set; } + + public int MaxPages { get; set; } + public int PageSize { get; set; } + + public LidarrListsRequestGenerator() + { + MaxPages = 1; + PageSize = 10; + } + + public virtual ImportListPageableRequestChain GetListItems() + { + var pageableRequests = new ImportListPageableRequestChain(); + + pageableRequests.Add(GetPagedRequests()); + + return pageableRequests; + } + + private IEnumerable GetPagedRequests() + { + yield return new ImportListRequest(string.Format("{0}{1}", Settings.BaseUrl, Settings.ListId), HttpAccept.Json); + } + + } +} diff --git a/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsSettings.cs b/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsSettings.cs new file mode 100644 index 000000000..c4f70eada --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/LidarrLists/LidarrListsSettings.cs @@ -0,0 +1,34 @@ +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.ImportLists.LidarrLists +{ + public class LidarrListsSettingsValidator : AbstractValidator + { + public LidarrListsSettingsValidator() + { + RuleFor(c => c.BaseUrl).ValidRootUrl(); + } + } + + public class LidarrListsSettings : IImportListSettings + { + private static readonly LidarrListsSettingsValidator Validator = new LidarrListsSettingsValidator(); + + public LidarrListsSettings() + { + BaseUrl = "https://api.lidarr.audio/api/v0.3/chart/"; + } + + public string BaseUrl { get; set; } + + [FieldDefinition(0, Label = "List Id")] + public string ListId { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} diff --git a/src/NzbDrone.Core/Jobs/TaskManager.cs b/src/NzbDrone.Core/Jobs/TaskManager.cs index 1fa1a8a76..c5b03b1bd 100644 --- a/src/NzbDrone.Core/Jobs/TaskManager.cs +++ b/src/NzbDrone.Core/Jobs/TaskManager.cs @@ -9,6 +9,7 @@ using NzbDrone.Core.Download; using NzbDrone.Core.HealthCheck; using NzbDrone.Core.Housekeeping; using NzbDrone.Core.Indexers; +using NzbDrone.Core.ImportLists; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; @@ -72,6 +73,12 @@ namespace NzbDrone.Core.Jobs TypeName = typeof(BackupCommand).FullName }, + new ScheduledTask + { + Interval = 24 * 60, // TODO: Add a setting? + TypeName = typeof(ImportListSyncCommand).FullName + }, + new ScheduledTask { Interval = GetRssSyncInterval(), diff --git a/src/NzbDrone.Core/Music/AlbumMonitoredService.cs b/src/NzbDrone.Core/Music/AlbumMonitoredService.cs index a33c78ac2..bcf146dd6 100644 --- a/src/NzbDrone.Core/Music/AlbumMonitoredService.cs +++ b/src/NzbDrone.Core/Music/AlbumMonitoredService.cs @@ -34,14 +34,14 @@ namespace NzbDrone.Core.Music var albums = _albumService.GetAlbumsByArtist(artist.Id); - var monitoredAlbums = artist.Albums; + var monitoredAlbums = monitoringOptions.AlbumsToMonitor; - if (monitoredAlbums != null) + if (monitoredAlbums.Any()) { ToggleAlbumsMonitoredState( - albums.Where(s => monitoredAlbums.Any(t => t.ForeignAlbumId == s.ForeignAlbumId)), true); + albums.Where(s => monitoredAlbums.Any(t => t == s.ForeignAlbumId)), true); ToggleAlbumsMonitoredState( - albums.Where(s => monitoredAlbums.Any(t => t.ForeignAlbumId != s.ForeignAlbumId)), false); + albums.Where(s => monitoredAlbums.Any(t => t != s.ForeignAlbumId)), false); } else { diff --git a/src/NzbDrone.Core/Music/MonitoringOptions.cs b/src/NzbDrone.Core/Music/MonitoringOptions.cs index 63bab1cd7..aa824975d 100644 --- a/src/NzbDrone.Core/Music/MonitoringOptions.cs +++ b/src/NzbDrone.Core/Music/MonitoringOptions.cs @@ -1,11 +1,18 @@ using NzbDrone.Core.Datastore; +using System.Collections.Generic; namespace NzbDrone.Core.Music { public class MonitoringOptions : IEmbeddedDocument { + public MonitoringOptions() + { + AlbumsToMonitor = new List(); + } + public bool IgnoreAlbumsWithFiles { get; set; } public bool IgnoreAlbumsWithoutFiles { get; set; } + public List AlbumsToMonitor { get; set; } public bool Monitored { get; set; } } } diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 35fc775fe..9afb45949 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -183,6 +183,7 @@ + @@ -464,6 +465,7 @@ + @@ -493,6 +495,7 @@ + @@ -502,6 +505,7 @@ + @@ -515,6 +519,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -883,6 +918,7 @@ + diff --git a/src/NzbDrone.Core/Parser/Model/ImportListItemInfo.cs b/src/NzbDrone.Core/Parser/Model/ImportListItemInfo.cs new file mode 100644 index 000000000..97353c564 --- /dev/null +++ b/src/NzbDrone.Core/Parser/Model/ImportListItemInfo.cs @@ -0,0 +1,21 @@ +using System; +using System.Text; + +namespace NzbDrone.Core.Parser.Model +{ + public class ImportListItemInfo + { + public int ImportListId { get; set; } + public string ImportList { get; set; } + public string Artist { get; set; } + public string ArtistMusicBrainzId { get; set; } + public string Album { get; set; } + public string AlbumMusicBrainzId { get; set; } + public DateTime ReleaseDate { get; set; } + + public override string ToString() + { + return string.Format("[{0}] {1} [{2}]", ReleaseDate, Artist, Album); + } + } +}