open-source-mirrors
/
Lidarr
mirror of https://github.com/Lidarr/Lidarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed: Removed hardlink-based transactional file transfer logic (instead relying on explicit copy+delete for cifs)

Browse Source
pull/1807/head
Taloth Saldono 4 years ago committed by Qstick
parent d61a6852b2
commit 4f220d9532
8 changed files with 93 additions and 617 deletions
Show Stats Download Patch File Download Diff File

416
src/NzbDrone.Common.Test/DiskTests/DiskTransferServiceFixture.cs
Unescape View File

@ -17,8 +17,6 @@ namespace NzbDrone.Common.Test.DiskTests
{ {
private readonly string _sourcePath = @"C:\source\my.video.mkv".AsOsAgnostic(); private readonly string _sourcePath = @"C:\source\my.video.mkv".AsOsAgnostic();
private readonly string _targetPath = @"C:\target\my.video.mkv".AsOsAgnostic(); private readonly string _targetPath = @"C:\target\my.video.mkv".AsOsAgnostic();
private readonly string _backupPath = @"C:\source\my.video.mkv.backup~".AsOsAgnostic();
private readonly string _tempTargetPath = @"C:\target\my.video.mkv.partial~".AsOsAgnostic();
private readonly string _nfsFile = ".nfs01231232"; private readonly string _nfsFile = ".nfs01231232";
private MockMount _sourceMount; private MockMount _sourceMount;
@ -46,65 +44,17 @@ namespace NzbDrone.Common.Test.DiskTests
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.GetMount(It.Is<string>(p => p.StartsWith(_sourceMount.RootDirectory)))) .Setup(v => v.GetMount(It.Is<string>(p => p.StartsWith(_sourceMount.RootDirectory))))
.Returns(_sourceMount); .Returns<string>(s => _sourceMount);
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.GetMount(It.Is<string>(p => p.StartsWith(_targetMount.RootDirectory)))) .Setup(v => v.GetMount(It.Is<string>(p => p.StartsWith(_targetMount.RootDirectory))))
.Returns(_targetMount); .Returns<string>(s => _targetMount);
WithEmulatedDiskProvider(); WithEmulatedDiskProvider();
WithExistingFile(_sourcePath); WithExistingFile(_sourcePath);
} }
[Test]
public void should_use_verified_transfer_on_mono()
{
PosixOnly();
Subject.VerificationMode.Should().Be(DiskTransferVerificationMode.TryTransactional);
}
[Test]
public void should_not_use_verified_transfer_on_windows()
{
WindowsOnly();
Subject.VerificationMode.Should().Be(DiskTransferVerificationMode.VerifyOnly);
var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.TryCreateHardLink(_sourcePath, _backupPath), Times.Never());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Once());
}
[TestCase("fuse.mergerfs", "")]
[TestCase("fuse.rclone", "")]
[TestCase("mergerfs", "")]
[TestCase("rclone", "")]
[TestCase("", "fuse.mergerfs")]
[TestCase("", "fuse.rclone")]
[TestCase("", "mergerfs")]
[TestCase("", "rclone")]
public void should_not_use_verified_transfer_on_specific_filesystems(string fsSource, string fsTarget)
{
MonoOnly();
_sourceMount.DriveFormat = fsSource;
_targetMount.DriveFormat = fsTarget;
var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.TryCreateHardLink(_sourcePath, _backupPath), Times.Never());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Once());
}
[Test] [Test]
public void should_throw_if_path_is_the_same() public void should_throw_if_path_is_the_same()
{ {
@ -125,72 +75,75 @@ namespace NzbDrone.Common.Test.DiskTests
[Test] [Test]
public void should_rename_via_temp_if_different_casing() public void should_rename_via_temp_if_different_casing()
{ {
var backupPath = _sourcePath + ".backup~";
var targetPath = Path.Combine(Path.GetDirectoryName(_sourcePath), Path.GetFileName(_sourcePath).ToUpper()); var targetPath = Path.Combine(Path.GetDirectoryName(_sourcePath), Path.GetFileName(_sourcePath).ToUpper());
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_sourcePath, _backupPath, true)) .Setup(v => v.MoveFile(_sourcePath, backupPath, true))
.Callback(() => .Callback(() =>
{ {
WithExistingFile(_backupPath, true); WithExistingFile(backupPath, true);
WithExistingFile(_sourcePath, false); WithExistingFile(_sourcePath, false);
}); });
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_backupPath, targetPath, false)) .Setup(v => v.MoveFile(backupPath, targetPath, false))
.Callback(() => .Callback(() =>
{ {
WithExistingFile(targetPath, true); WithExistingFile(targetPath, true);
WithExistingFile(_backupPath, false); WithExistingFile(backupPath, false);
}); });
var result = Subject.TransferFile(_sourcePath, targetPath, TransferMode.Move); var result = Subject.TransferFile(_sourcePath, targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_backupPath, targetPath, false), Times.Once()); .Verify(v => v.MoveFile(backupPath, targetPath, false), Times.Once());
} }
[Test] [Test]
public void should_rollback_rename_via_temp_on_exception() public void should_rollback_rename_via_temp_on_exception()
{ {
var backupPath = _sourcePath + ".backup~";
var targetPath = Path.Combine(Path.GetDirectoryName(_sourcePath), Path.GetFileName(_sourcePath).ToUpper()); var targetPath = Path.Combine(Path.GetDirectoryName(_sourcePath), Path.GetFileName(_sourcePath).ToUpper());
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_sourcePath, _backupPath, true)) .Setup(v => v.MoveFile(_sourcePath, backupPath, true))
.Callback(() => .Callback(() =>
{ {
WithExistingFile(_backupPath, true); WithExistingFile(backupPath, true);
WithExistingFile(_sourcePath, false); WithExistingFile(_sourcePath, false);
}); });
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_backupPath, targetPath, false)) .Setup(v => v.MoveFile(backupPath, targetPath, false))
.Throws(new IOException("Access Violation")); .Throws(new IOException("Access Violation"));
Assert.Throws<IOException>(() => Subject.TransferFile(_sourcePath, targetPath, TransferMode.Move)); Assert.Throws<IOException>(() => Subject.TransferFile(_sourcePath, targetPath, TransferMode.Move));
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_backupPath, _sourcePath, false), Times.Once()); .Verify(v => v.MoveFile(backupPath, _sourcePath, false), Times.Once());
} }
[Test] [Test]
public void should_log_error_if_rollback_move_fails() public void should_log_error_if_rollback_move_fails()
{ {
var backupPath = _sourcePath + ".backup~";
var targetPath = Path.Combine(Path.GetDirectoryName(_sourcePath), Path.GetFileName(_sourcePath).ToUpper()); var targetPath = Path.Combine(Path.GetDirectoryName(_sourcePath), Path.GetFileName(_sourcePath).ToUpper());
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_sourcePath, _backupPath, true)) .Setup(v => v.MoveFile(_sourcePath, backupPath, true))
.Callback(() => .Callback(() =>
{ {
WithExistingFile(_backupPath, true); WithExistingFile(backupPath, true);
WithExistingFile(_sourcePath, false); WithExistingFile(_sourcePath, false);
}); });
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_backupPath, targetPath, false)) .Setup(v => v.MoveFile(backupPath, targetPath, false))
.Throws(new IOException("Access Violation")); .Throws(new IOException("Access Violation"));
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_backupPath, _sourcePath, false)) .Setup(v => v.MoveFile(backupPath, _sourcePath, false))
.Throws(new IOException("Access Violation")); .Throws(new IOException("Access Violation"));
Assert.Throws<IOException>(() => Subject.TransferFile(_sourcePath, targetPath, TransferMode.Move)); Assert.Throws<IOException>(() => Subject.TransferFile(_sourcePath, targetPath, TransferMode.Move));
@ -225,28 +178,44 @@ namespace NzbDrone.Common.Test.DiskTests
} }
[Test] [Test]
public void should_fallback_to_copy_if_hardlink_failed() public void should_use_copy_delete_on_cifs()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.Transactional; _sourceMount.DriveFormat = "ext4";
_targetMount.DriveFormat = "cifs";
WithFailedHardlink();
var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move); var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Verify(v => v.CopyFile(_sourcePath, _tempTargetPath, false), Times.Once()); .Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Never());
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_tempTargetPath, _targetPath, false), Times.Once()); .Verify(v => v.CopyFile(_sourcePath, _targetPath, false), Times.Once());
VerifyDeletedFile(_sourcePath); Mocker.GetMock<IDiskProvider>()
.Verify(v => v.DeleteFile(_sourcePath), Times.Once());
} }
[Test] [Test]
public void mode_none_should_not_verify_copy() public void should_use_move_on_cifs_if_same_mount()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.None; _sourceMount.DriveFormat = "cifs";
_targetMount = _sourceMount;
var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Once());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.CopyFile(_sourcePath, _targetPath, false), Times.Never());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.DeleteFile(_sourcePath), Times.Never());
}
[Test]
public void should_not_verify_copy()
{
Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Copy); Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Copy);
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
@ -254,10 +223,8 @@ namespace NzbDrone.Common.Test.DiskTests
} }
[Test] [Test]
public void mode_none_should_not_verify_move() public void should_not_verify_move()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.None;
Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move); Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
@ -265,10 +232,8 @@ namespace NzbDrone.Common.Test.DiskTests
} }
[Test] [Test]
public void mode_none_should_delete_existing_target_when_overwriting() public void should_delete_existing_target_when_overwriting()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.None;
WithExistingFile(_targetPath); WithExistingFile(_targetPath);
Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move, true); Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move, true);
@ -281,10 +246,8 @@ namespace NzbDrone.Common.Test.DiskTests
} }
[Test] [Test]
public void mode_none_should_throw_if_existing_target_when_not_overwriting() public void should_throw_if_existing_target_when_not_overwriting()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.None;
WithExistingFile(_targetPath); WithExistingFile(_targetPath);
Assert.Throws<DestinationAlreadyExistsException>(() => Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move, false)); Assert.Throws<DestinationAlreadyExistsException>(() => Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move, false));
@ -297,10 +260,8 @@ namespace NzbDrone.Common.Test.DiskTests
} }
[Test] [Test]
public void mode_verifyonly_should_verify_copy() public void should_verify_copy()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.VerifyOnly;
Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Copy); Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Copy);
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
@ -311,10 +272,8 @@ namespace NzbDrone.Common.Test.DiskTests
} }
[Test] [Test]
public void mode_verifyonly_should_rollback_copy_on_partial_and_throw() public void should_rollback_copy_on_partial_and_throw()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.VerifyOnly;
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.CopyFile(_sourcePath, _targetPath, false)) .Setup(v => v.CopyFile(_sourcePath, _targetPath, false))
.Callback(() => .Callback(() =>
@ -331,8 +290,6 @@ namespace NzbDrone.Common.Test.DiskTests
[Test] [Test]
public void should_log_error_if_rollback_copy_fails() public void should_log_error_if_rollback_copy_fails()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.VerifyOnly;
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.CopyFile(_sourcePath, _targetPath, false)) .Setup(v => v.CopyFile(_sourcePath, _targetPath, false))
.Callback(() => .Callback(() =>
@ -350,10 +307,8 @@ namespace NzbDrone.Common.Test.DiskTests
} }
[Test] [Test]
public void mode_verifyonly_should_verify_move() public void should_verify_move()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.VerifyOnly;
Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move); Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
@ -364,10 +319,8 @@ namespace NzbDrone.Common.Test.DiskTests
} }
[Test] [Test]
public void mode_verifyonly_should_not_rollback_move_on_partial_and_throw() public void should_not_rollback_move_on_partial_and_throw_if_source_gone()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.VerifyOnly;
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_sourcePath, _targetPath, false)) .Setup(v => v.MoveFile(_sourcePath, _targetPath, false))
.Callback(() => .Callback(() =>
@ -385,10 +338,8 @@ namespace NzbDrone.Common.Test.DiskTests
} }
[Test] [Test]
public void mode_verifyonly_should_rollback_move_on_partial_if_source_remains() public void should_rollback_move_on_partial_if_source_remains()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.VerifyOnly;
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_sourcePath, _targetPath, false)) .Setup(v => v.MoveFile(_sourcePath, _targetPath, false))
.Callback(() => .Callback(() =>
@ -405,8 +356,6 @@ namespace NzbDrone.Common.Test.DiskTests
[Test] [Test]
public void should_log_error_if_rollback_partialmove_fails() public void should_log_error_if_rollback_partialmove_fails()
{ {
Subject.VerificationMode = DiskTransferVerificationMode.VerifyOnly;
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_sourcePath, _targetPath, false)) .Setup(v => v.MoveFile(_sourcePath, _targetPath, false))
.Callback(() => .Callback(() =>
@ -424,251 +373,7 @@ namespace NzbDrone.Common.Test.DiskTests
} }
[Test] [Test]
public void mode_transactional_should_move_and_verify_if_same_folder() [Retry(5)]
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
var targetPath = _sourcePath + ".test";
var result = Subject.TransferFile(_sourcePath, targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.TryCreateHardLink(_sourcePath, _backupPath), Times.Never());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.CopyFile(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>()), Times.Never());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_sourcePath, targetPath, false), Times.Once());
}
[Test]
public void mode_trytransactional_should_revert_to_verifyonly_if_hardlink_fails()
{
Subject.VerificationMode = DiskTransferVerificationMode.TryTransactional;
WithFailedHardlink();
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_sourcePath, _targetPath, false))
.Callback(() =>
{
WithExistingFile(_sourcePath, false);
WithExistingFile(_targetPath, true);
});
var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.TryCreateHardLink(_sourcePath, _backupPath), Times.Once());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.CopyFile(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>()), Times.Never());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Once());
}
[Test]
public void mode_transactional_should_delete_old_backup_on_move()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
WithExistingFile(_backupPath);
WithSuccessfulHardlink(_sourcePath, _backupPath);
Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.DeleteFile(_backupPath), Times.Once());
}
[Test]
public void mode_transactional_should_delete_old_partial_on_move()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
WithExistingFile(_tempTargetPath);
WithSuccessfulHardlink(_sourcePath, _backupPath);
Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.DeleteFile(_tempTargetPath), Times.Once());
}
[Test]
public void mode_transactional_should_delete_old_partial_on_copy()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
WithExistingFile(_tempTargetPath);
Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Copy);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.DeleteFile(_tempTargetPath), Times.Once());
}
[Test]
public void mode_transactional_should_hardlink_before_move()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
WithSuccessfulHardlink(_sourcePath, _backupPath);
var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.TryCreateHardLink(_sourcePath, _backupPath), Times.Once());
}
[Test]
public void mode_transactional_should_retry_if_partial_copy()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
var retry = 0;
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.CopyFile(_sourcePath, _tempTargetPath, false))
.Callback(() =>
{
WithExistingFile(_tempTargetPath, true, 900);
if (retry++ == 1)
{
WithExistingFile(_tempTargetPath, true, 1000);
}
});
var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Copy);
ExceptionVerification.ExpectedWarns(1);
}
[Test]
public void mode_transactional_should_retry_twice_if_partial_copy()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
var retry = 0;
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.CopyFile(_sourcePath, _tempTargetPath, false))
.Callback(() =>
{
WithExistingFile(_tempTargetPath, true, 900);
if (retry++ == 3)
{
throw new Exception("Test Failed, retried too many times.");
}
});
Assert.Throws<IOException>(() => Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Copy));
ExceptionVerification.ExpectedWarns(2);
ExceptionVerification.ExpectedErrors(1);
}
[Test]
public void mode_transactional_should_remove_source_after_move()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
WithSuccessfulHardlink(_sourcePath, _backupPath);
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_backupPath, _tempTargetPath, false))
.Callback(() => WithExistingFile(_tempTargetPath, true));
var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
VerifyDeletedFile(_sourcePath);
}
[Test]
public void mode_transactional_should_not_remove_source_if_partial_still_exists()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
var targetPath = Path.Combine(Path.GetDirectoryName(_targetPath), Path.GetFileName(_targetPath).ToUpper());
var tempTargetPath = targetPath + ".partial~";
WithSuccessfulHardlink(_sourcePath, _backupPath);
WithExistingFile(_targetPath);
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_backupPath, tempTargetPath, false))
.Callback(() => WithExistingFile(tempTargetPath, true));
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(tempTargetPath, targetPath, false))
.Callback(() => { });
Assert.Throws<IOException>(() => Subject.TransferFile(_sourcePath, targetPath, TransferMode.Move));
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.DeleteFile(_sourcePath), Times.Never());
}
[Test]
public void mode_transactional_should_remove_partial_if_copy_fails()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
WithSuccessfulHardlink(_sourcePath, _backupPath);
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.CopyFile(_sourcePath, _tempTargetPath, false))
.Callback(() =>
{
WithExistingFile(_tempTargetPath, true, 900);
})
.Throws(new IOException("Blackbox IO error"));
Assert.Throws<IOException>(() => Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Copy));
VerifyDeletedFile(_tempTargetPath);
}
[Test]
public void mode_transactional_should_remove_backup_if_move_throws()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
WithSuccessfulHardlink(_sourcePath, _backupPath);
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_backupPath, _tempTargetPath, false))
.Throws(new IOException("Blackbox IO error"));
Assert.Throws<IOException>(() => Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move));
VerifyDeletedFile(_backupPath);
}
[Test]
public void mode_transactional_should_remove_partial_if_move_fails()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
WithSuccessfulHardlink(_sourcePath, _backupPath);
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_backupPath, _tempTargetPath, false))
.Callback(() =>
{
WithExistingFile(_backupPath, false);
WithExistingFile(_tempTargetPath, true, 900);
});
Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
VerifyDeletedFile(_tempTargetPath);
}
[Test]
public void CopyFolder_should_copy_folder() public void CopyFolder_should_copy_folder()
{ {
WithRealDiskProvider(); WithRealDiskProvider();
@ -884,23 +589,6 @@ namespace NzbDrone.Common.Test.DiskTests
.Verify(v => v.MoveFolder(src, dst), Times.Never()); .Verify(v => v.MoveFolder(src, dst), Times.Never());
} }
[Test]
public void TransferFolder_should_not_use_movefolder_if_on_same_mount_but_transactional()
{
WithEmulatedDiskProvider();
var src = @"C:\Base1\TestDir1".AsOsAgnostic();
var dst = @"C:\Base1\TestDir2".AsOsAgnostic();
WithMockMount(@"C:\Base1".AsOsAgnostic());
WithExistingFile(@"C:\Base1\TestDir1\test.file.txt".AsOsAgnostic());
Subject.TransferFolder(src, dst, TransferMode.Move, DiskTransferVerificationMode.Transactional);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFolder(src, dst), Times.Never());
}
[Test] [Test]
public void TransferFolder_should_not_use_movefolder_if_on_different_mount() public void TransferFolder_should_not_use_movefolder_if_on_different_mount()
{ {

267
src/NzbDrone.Common/Disk/DiskTransferService.cs
Unescape View File

@ -12,64 +12,38 @@ namespace NzbDrone.Common.Disk
{ {
public interface IDiskTransferService public interface IDiskTransferService
{ {
TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, bool verified = true); TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode);
TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite = false, bool verified = true); TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite = false);
int MirrorFolder(string sourcePath, string targetPath); int MirrorFolder(string sourcePath, string targetPath);
} }
public enum DiskTransferVerificationMode
{
None,
VerifyOnly,
TryTransactional,
Transactional
}
public class DiskTransferService : IDiskTransferService public class DiskTransferService : IDiskTransferService
{ {
private const int RetryCount = 2;
private readonly IDiskProvider _diskProvider; private readonly IDiskProvider _diskProvider;
private readonly Logger _logger; private readonly Logger _logger;
public DiskTransferVerificationMode VerificationMode { get; set; }
public DiskTransferService(IDiskProvider diskProvider, Logger logger) public DiskTransferService(IDiskProvider diskProvider, Logger logger)
{ {
_diskProvider = diskProvider; _diskProvider = diskProvider;
_logger = logger; _logger = logger;
// TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer.
// (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.)
VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.TryTransactional;
} }
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, bool verified = true) public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode)
{
var verificationMode = verified ? VerificationMode : DiskTransferVerificationMode.VerifyOnly;
return TransferFolder(sourcePath, targetPath, mode, verificationMode);
}
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, DiskTransferVerificationMode verificationMode)
{ {
Ensure.That(sourcePath, () => sourcePath).IsValidPath(); Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath(); Ensure.That(targetPath, () => targetPath).IsValidPath();
if (mode == TransferMode.Move && !_diskProvider.FolderExists(targetPath)) if (mode == TransferMode.Move && !_diskProvider.FolderExists(targetPath))
{ {
if (verificationMode == DiskTransferVerificationMode.TryTransactional || verificationMode == DiskTransferVerificationMode.VerifyOnly) var sourceMount = _diskProvider.GetMount(sourcePath);
{ var targetMount = _diskProvider.GetMount(targetPath);
var sourceMount = _diskProvider.GetMount(sourcePath);
var targetMount = _diskProvider.GetMount(targetPath);
// If we're on the same mount, do a simple folder move. // If we're on the same mount, do a simple folder move.
if (sourceMount != null && targetMount != null && sourceMount.RootDirectory == targetMount.RootDirectory) if (sourceMount != null && targetMount != null && sourceMount.RootDirectory == targetMount.RootDirectory)
{ {
_logger.Debug("Move Directory [{0}] > [{1}]", sourcePath, targetPath); _logger.Debug("Move Directory [{0}] > [{1}]", sourcePath, targetPath);
_diskProvider.MoveFolder(sourcePath, targetPath); _diskProvider.MoveFolder(sourcePath, targetPath);
return mode; return mode;
}
} }
} }
@ -89,7 +63,7 @@ namespace NzbDrone.Common.Disk
continue; continue;
} }
result &= TransferFolder(subDir.FullName, Path.Combine(targetPath, subDir.Name), mode, verificationMode); result &= TransferFolder(subDir.FullName, Path.Combine(targetPath, subDir.Name), mode);
} }
foreach (var sourceFile in _diskProvider.GetFileInfos(sourcePath)) foreach (var sourceFile in _diskProvider.GetFileInfos(sourcePath))
@ -101,7 +75,7 @@ namespace NzbDrone.Common.Disk
var destFile = Path.Combine(targetPath, sourceFile.Name); var destFile = Path.Combine(targetPath, sourceFile.Name);
result &= TransferFile(sourceFile.FullName, destFile, mode, true, verificationMode); result &= TransferFile(sourceFile.FullName, destFile, mode, true);
} }
if (mode.HasFlag(TransferMode.Move)) if (mode.HasFlag(TransferMode.Move))
@ -176,7 +150,7 @@ namespace NzbDrone.Common.Disk
continue; continue;
} }
TransferFile(sourceFile.FullName, targetFile, TransferMode.Copy, true, true); TransferFile(sourceFile.FullName, targetFile, TransferMode.Copy, true);
filesCopied++; filesCopied++;
} }
@ -226,14 +200,7 @@ namespace NzbDrone.Common.Disk
} }
} }
public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite = false, bool verified = true) public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite = false)
{
var verificationMode = verified ? VerificationMode : DiskTransferVerificationMode.None;
return TransferFile(sourcePath, targetPath, mode, overwrite, verificationMode);
}
public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite, DiskTransferVerificationMode verificationMode)
{ {
Ensure.That(sourcePath, () => sourcePath).IsValidPath(); Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath(); Ensure.That(targetPath, () => targetPath).IsValidPath();
@ -308,78 +275,33 @@ namespace NzbDrone.Common.Disk
} }
// Adjust the transfer mode depending on the filesystems // Adjust the transfer mode depending on the filesystems
if (verificationMode == DiskTransferVerificationMode.TryTransactional) var sourceMount = _diskProvider.GetMount(sourcePath);
{ var targetMount = _diskProvider.GetMount(targetPath);
var sourceMount = _diskProvider.GetMount(sourcePath);
var targetMount = _diskProvider.GetMount(targetPath);
var isSameMount = sourceMount != null && targetMount != null && sourceMount.RootDirectory == targetMount.RootDirectory; var isSameMount = sourceMount != null && targetMount != null && sourceMount.RootDirectory == targetMount.RootDirectory;
var sourceDriveFormat = sourceMount?.DriveFormat ?? string.Empty; var sourceDriveFormat = sourceMount?.DriveFormat ?? string.Empty;
var targetDriveFormat = targetMount?.DriveFormat ?? string.Empty; var targetDriveFormat = targetMount?.DriveFormat ?? string.Empty;
if (isSameMount) var isCifs = targetDriveFormat == "cifs";
{
// No transaction needed for operations on same mount, force VerifyOnly
verificationMode = DiskTransferVerificationMode.VerifyOnly;
}
else if (sourceDriveFormat.Contains("mergerfs") || sourceDriveFormat.Contains("rclone") ||
targetDriveFormat.Contains("mergerfs") || targetDriveFormat.Contains("rclone"))
{
// Cloud storage filesystems don't need any Transactional stuff and it hurts performance, force VerifyOnly
verificationMode = DiskTransferVerificationMode.VerifyOnly;
}
else if ((sourceDriveFormat == "cifs" || targetDriveFormat == "cifs") && OsInfo.IsNotWindows)
{
// Force Transactional on a cifs mount due to the likeliness of move failures on certain scenario's on mono
verificationMode = DiskTransferVerificationMode.Transactional;
}
}
if (mode.HasFlag(TransferMode.Copy)) if (mode.HasFlag(TransferMode.Copy))
{ {
if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional) TryCopyFileVerified(sourcePath, targetPath, originalSize);
{ return TransferMode.Copy;
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{
return TransferMode.Copy;
}
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
TryCopyFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Copy;
}
else
{
_diskProvider.CopyFile(sourcePath, targetPath);
return TransferMode.Copy;
}
} }
if (mode.HasFlag(TransferMode.Move)) if (mode.HasFlag(TransferMode.Move))
{ {
if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional) if (isCifs && !isSameMount)
{
if (TryMoveFileTransactional(sourcePath, targetPath, originalSize, verificationMode))
{
return TransferMode.Move;
}
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Move;
}
else
{ {
_diskProvider.MoveFile(sourcePath, targetPath); TryCopyFileVerified(sourcePath, targetPath, originalSize);
_diskProvider.DeleteFile(sourcePath);
return TransferMode.Move; return TransferMode.Move;
} }
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Move;
} }
return TransferMode.None; return TransferMode.None;
@ -464,137 +386,6 @@ namespace NzbDrone.Common.Disk
Thread.Sleep(3000); Thread.Sleep(3000);
} }
private bool TryCopyFileTransactional(string sourcePath, string targetPath, long originalSize)
{
var tempTargetPath = targetPath + ".partial~";
if (_diskProvider.FileExists(tempTargetPath))
{
_logger.Trace("Removing old partial.");
_diskProvider.DeleteFile(tempTargetPath);
}
try
{
for (var i = 0; i <= RetryCount; i++)
{
_diskProvider.CopyFile(sourcePath, tempTargetPath);
if (_diskProvider.FileExists(tempTargetPath))
{
var targetSize = _diskProvider.GetFileSize(tempTargetPath);
if (targetSize == originalSize)
{
_diskProvider.MoveFile(tempTargetPath, targetPath);
return true;
}
}
WaitForIO();
_diskProvider.DeleteFile(tempTargetPath);
if (i == RetryCount)
{
_logger.Error("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath);
}
else
{
_logger.Warn("Failed to completely transfer [{0}] to [{1}], retrying [{2}/{3}].", sourcePath, targetPath, i + 1, RetryCount);
}
}
}
catch
{
WaitForIO();
if (_diskProvider.FileExists(tempTargetPath))
{
_diskProvider.DeleteFile(tempTargetPath);
}
throw;
}
return false;
}
private bool TryMoveFileTransactional(string sourcePath, string targetPath, long originalSize, DiskTransferVerificationMode verificationMode)
{
var backupPath = sourcePath + ".backup~";
var tempTargetPath = targetPath + ".partial~";
if (_diskProvider.FileExists(backupPath))
{
_logger.Trace("Removing old backup.");
_diskProvider.DeleteFile(backupPath);
}
if (_diskProvider.FileExists(tempTargetPath))
{
_logger.Trace("Removing old partial.");
_diskProvider.DeleteFile(tempTargetPath);
}
try
{
_logger.Trace("Attempting to move hardlinked backup.");
if (_diskProvider.TryCreateHardLink(sourcePath, backupPath))
{
_diskProvider.MoveFile(backupPath, tempTargetPath);
if (_diskProvider.FileExists(tempTargetPath))
{
var targetSize = _diskProvider.GetFileSize(tempTargetPath);
if (targetSize == originalSize)
{
_diskProvider.MoveFile(tempTargetPath, targetPath);
if (_diskProvider.FileExists(tempTargetPath))
{
throw new IOException(string.Format("Temporary file '{0}' still exists, aborting.", tempTargetPath));
}
_logger.Trace("Hardlink move succeeded, deleting source.");
_diskProvider.DeleteFile(sourcePath);
return true;
}
}
Thread.Sleep(5000);
_diskProvider.DeleteFile(tempTargetPath);
}
}
finally
{
if (_diskProvider.FileExists(backupPath))
{
_diskProvider.DeleteFile(backupPath);
}
}
if (verificationMode == DiskTransferVerificationMode.Transactional)
{
_logger.Trace("Hardlink move failed, reverting to copy.");
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{
_logger.Trace("Copy succeeded, deleting source.");
_diskProvider.DeleteFile(sourcePath);
return true;
}
}
else
{
_logger.Trace("Hardlink move failed, reverting to move.");
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return true;
}
_logger.Trace("Move failed.");
return false;
}
private void TryCopyFileVerified(string sourcePath, string targetPath, long originalSize) private void TryCopyFileVerified(string sourcePath, string targetPath, long originalSize)
{ {
try try

11
src/NzbDrone.Core.Test/MusicTests/MoveArtistServiceFixture.cs
Unescape View File

@ -63,7 +63,7 @@ namespace NzbDrone.Core.Test.MusicTests
private void GivenFailedMove() private void GivenFailedMove()
{ {
Mocker.GetMock<IDiskTransferService>() Mocker.GetMock<IDiskTransferService>()
.Setup(s => s.TransferFolder(It.IsAny<string>(), It.IsAny<string>(), TransferMode.Move, true)) .Setup(s => s.TransferFolder(It.IsAny<string>(), It.IsAny<string>(), TransferMode.Move))
.Throws<IOException>(); .Throws<IOException>();
} }
@ -99,8 +99,7 @@ namespace NzbDrone.Core.Test.MusicTests
.Verify( .Verify(
v => v.TransferFolder(_command.SourcePath, v => v.TransferFolder(_command.SourcePath,
_command.DestinationPath, _command.DestinationPath,
TransferMode.Move, TransferMode.Move),
It.IsAny<bool>()),
Times.Once()); Times.Once());
Mocker.GetMock<IBuildFileNames>() Mocker.GetMock<IBuildFileNames>()
@ -123,8 +122,7 @@ namespace NzbDrone.Core.Test.MusicTests
.Verify( .Verify(
v => v.TransferFolder(_bulkCommand.Artist.First().SourcePath, v => v.TransferFolder(_bulkCommand.Artist.First().SourcePath,
expectedPath, expectedPath,
TransferMode.Move, TransferMode.Move),
It.IsAny<bool>()),
Times.Once()); Times.Once());
} }
@ -141,8 +139,7 @@ namespace NzbDrone.Core.Test.MusicTests
.Verify( .Verify(
v => v.TransferFolder(_command.SourcePath, v => v.TransferFolder(_command.SourcePath,
_command.DestinationPath, _command.DestinationPath,
TransferMode.Move, TransferMode.Move), Times.Never());
It.IsAny<bool>()), Times.Never());
Mocker.GetMock<IBuildFileNames>() Mocker.GetMock<IBuildFileNames>()
.Verify(v => v.GetArtistFolder(It.IsAny<Artist>(), null), Times.Never()); .Verify(v => v.GetArtistFolder(It.IsAny<Artist>(), null), Times.Never());

2
src/NzbDrone.Core.Test/ProviderTests/RecycleBinProviderTests/DeleteDirectoryFixture.cs
Unescape View File

@ -46,7 +46,7 @@ namespace NzbDrone.Core.Test.ProviderTests.RecycleBinProviderTests
Mocker.Resolve<RecycleBinProvider>().DeleteFolder(path); Mocker.Resolve<RecycleBinProvider>().DeleteFolder(path);
Mocker.GetMock<IDiskTransferService>() Mocker.GetMock<IDiskTransferService>()
.Verify(v => v.TransferFolder(path, @"C:\Test\Recycle Bin\30 Rock".AsOsAgnostic(), TransferMode.Move, true), Times.Once()); .Verify(v => v.TransferFolder(path, @"C:\Test\Recycle Bin\30 Rock".AsOsAgnostic(), TransferMode.Move), Times.Once());
} }
[Test] [Test]

8
src/NzbDrone.Core.Test/ProviderTests/RecycleBinProviderTests/DeleteFileFixture.cs
Unescape View File

@ -1,4 +1,4 @@
using System; using System;
using Moq; using Moq;
using NUnit.Framework; using NUnit.Framework;
using NzbDrone.Common.Disk; using NzbDrone.Common.Disk;
@ -44,7 +44,7 @@ namespace NzbDrone.Core.Test.ProviderTests.RecycleBinProviderTests
Mocker.Resolve<RecycleBinProvider>().DeleteFile(path); Mocker.Resolve<RecycleBinProvider>().DeleteFile(path);
Mocker.GetMock<IDiskTransferService>().Verify(v => v.TransferFile(path, @"C:\Test\Recycle Bin\S01E01.avi".AsOsAgnostic(), TransferMode.Move, false, true), Times.Once()); Mocker.GetMock<IDiskTransferService>().Verify(v => v.TransferFile(path, @"C:\Test\Recycle Bin\S01E01.avi".AsOsAgnostic(), TransferMode.Move, false), Times.Once());
} }
[Test] [Test]
@ -60,7 +60,7 @@ namespace NzbDrone.Core.Test.ProviderTests.RecycleBinProviderTests
Mocker.Resolve<RecycleBinProvider>().DeleteFile(path); Mocker.Resolve<RecycleBinProvider>().DeleteFile(path);
Mocker.GetMock<IDiskTransferService>().Verify(v => v.TransferFile(path, @"C:\Test\Recycle Bin\S01E01_2.avi".AsOsAgnostic(), TransferMode.Move, false, true), Times.Once()); Mocker.GetMock<IDiskTransferService>().Verify(v => v.TransferFile(path, @"C:\Test\Recycle Bin\S01E01_2.avi".AsOsAgnostic(), TransferMode.Move, false), Times.Once());
} }
[Test] [Test]
@ -84,7 +84,7 @@ namespace NzbDrone.Core.Test.ProviderTests.RecycleBinProviderTests
Mocker.Resolve<RecycleBinProvider>().DeleteFile(path, "30 Rock"); Mocker.Resolve<RecycleBinProvider>().DeleteFile(path, "30 Rock");
Mocker.GetMock<IDiskTransferService>().Verify(v => v.TransferFile(path, @"C:\Test\Recycle Bin\30 Rock\S01E01.avi".AsOsAgnostic(), TransferMode.Move, false, true), Times.Once()); Mocker.GetMock<IDiskTransferService>().Verify(v => v.TransferFile(path, @"C:\Test\Recycle Bin\30 Rock\S01E01.avi".AsOsAgnostic(), TransferMode.Move, false), Times.Once());
} }
} }
} }

2
src/NzbDrone.Core.Test/UpdateTests/UpdateServiceFixture.cs
Unescape View File

@ -145,7 +145,7 @@ namespace NzbDrone.Core.Test.UpdateTests
Subject.Execute(new ApplicationUpdateCommand()); Subject.Execute(new ApplicationUpdateCommand());
Mocker.GetMock<IDiskTransferService>() Mocker.GetMock<IDiskTransferService>()
.Verify(c => c.TransferFolder(updateClientFolder, _sandboxFolder, TransferMode.Move, false)); .Verify(c => c.TransferFolder(updateClientFolder, _sandboxFolder, TransferMode.Move));
} }
[Test] [Test]

2
src/NzbDrone.Core/Extras/Files/ExtraFileManager.cs
Unescape View File

@ -69,7 +69,7 @@ namespace NzbDrone.Core.Extras.Files
transferMode = _configService.CopyUsingHardlinks ? TransferMode.HardLinkOrCopy : TransferMode.Copy; transferMode = _configService.CopyUsingHardlinks ? TransferMode.HardLinkOrCopy : TransferMode.Copy;
} }
_diskTransferService.TransferFile(path, newFileName, transferMode, true, false); _diskTransferService.TransferFile(path, newFileName, transferMode, true);
return new TExtraFile return new TExtraFile
{ {

2
src/NzbDrone.Core/Update/InstallUpdateService.cs
Unescape View File

@ -144,7 +144,7 @@ namespace NzbDrone.Core.Update
} }
_logger.Info("Preparing client"); _logger.Info("Preparing client");
_diskTransferService.TransferFolder(_appFolderInfo.GetUpdateClientFolder(), updateSandboxFolder, TransferMode.Move, false); _diskTransferService.TransferFolder(_appFolderInfo.GetUpdateClientFolder(), updateSandboxFolder, TransferMode.Move);
// Set executable flag on update app // Set executable flag on update app
if (OsInfo.IsOsx || (OsInfo.IsLinux && PlatformInfo.IsNetCore)) if (OsInfo.IsOsx || (OsInfo.IsLinux && PlatformInfo.IsNetCore))

Write Preview
Loading…
Cancel
Save