From 9eb5b335f32d51a839bafb7035a27f8abbca067d Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Wed, 10 Jun 2020 20:48:54 +0200 Subject: [PATCH] New: Fast copy using reflink on btrfs volumes --- src/NzbDrone.Common/Disk/DiskProviderBase.cs | 28 +++++++ .../Disk/DiskTransferService.cs | 27 +++++++ src/NzbDrone.Common/Disk/IDiskProvider.cs | 3 + src/NzbDrone.Mono/Disk/DiskProvider.cs | 31 +++++++- src/NzbDrone.Mono/Disk/RefLinkCreator.cs | 75 +++++++++++++++++++ src/NzbDrone.Mono/Interop/NativeMethods.cs | 28 +++++++ src/NzbDrone.Mono/Interop/SafeUnixHandle.cs | 37 +++++++++ 7 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 src/NzbDrone.Mono/Disk/RefLinkCreator.cs create mode 100644 src/NzbDrone.Mono/Interop/NativeMethods.cs create mode 100644 src/NzbDrone.Mono/Interop/SafeUnixHandle.cs diff --git a/src/NzbDrone.Common/Disk/DiskProviderBase.cs b/src/NzbDrone.Common/Disk/DiskProviderBase.cs index de2d45edf..584b15bce 100644 --- a/src/NzbDrone.Common/Disk/DiskProviderBase.cs +++ b/src/NzbDrone.Common/Disk/DiskProviderBase.cs @@ -212,6 +212,24 @@ namespace NzbDrone.Common.Disk _fileSystem.File.Delete(path); } + public void CloneFile(string source, string destination, bool overwrite = false) + { + Ensure.That(source, () => source).IsValidPath(); + Ensure.That(destination, () => destination).IsValidPath(); + + if (source.PathEquals(destination)) + { + throw new IOException(string.Format("Source and destination can't be the same {0}", source)); + } + + CloneFileInternal(source, destination, overwrite); + } + + protected virtual void CloneFileInternal(string source, string destination, bool overwrite = false) + { + CopyFileInternal(source, destination, overwrite); + } + public void CopyFile(string source, string destination, bool overwrite = false) { Ensure.That(source, () => source).IsValidPath(); @@ -262,8 +280,18 @@ namespace NzbDrone.Common.Disk _fileSystem.File.Move(source, destination); } + public virtual bool TryRenameFile(string source, string destination) + { + return false; + } + public abstract bool TryCreateHardLink(string source, string destination); + public virtual bool TryCreateRefLink(string source, string destination) + { + return false; + } + public void DeleteFolder(string path, bool recursive) { Ensure.That(path, () => path).IsValidPath(); diff --git a/src/NzbDrone.Common/Disk/DiskTransferService.cs b/src/NzbDrone.Common/Disk/DiskTransferService.cs index 01281da4d..8a75ed34d 100644 --- a/src/NzbDrone.Common/Disk/DiskTransferService.cs +++ b/src/NzbDrone.Common/Disk/DiskTransferService.cs @@ -284,18 +284,45 @@ namespace NzbDrone.Common.Disk var targetDriveFormat = targetMount?.DriveFormat ?? string.Empty; var isCifs = targetDriveFormat == "cifs"; + var isBtrfs = sourceDriveFormat == "btrfs" && targetDriveFormat == "btrfs"; if (mode.HasFlag(TransferMode.Copy)) { + if (isBtrfs) + { + if (_diskProvider.TryCreateRefLink(sourcePath, targetPath)) + { + return TransferMode.Copy; + } + } + TryCopyFileVerified(sourcePath, targetPath, originalSize); return TransferMode.Copy; } if (mode.HasFlag(TransferMode.Move)) { + if (isBtrfs) + { + if (isSameMount && _diskProvider.TryRenameFile(sourcePath, targetPath)) + { + _logger.Trace("Renamed [{0}] to [{1}].", sourcePath, targetPath); + return TransferMode.Move; + } + + if (_diskProvider.TryCreateRefLink(sourcePath, targetPath)) + { + _logger.Trace("Reflink successful, deleting source [{0}].", sourcePath); + _diskProvider.DeleteFile(sourcePath); + return TransferMode.Move; + } + } + if (isCifs && !isSameMount) { + _logger.Trace("On cifs mount. Starting verified copy [{0}] to [{1}].", sourcePath, targetPath); TryCopyFileVerified(sourcePath, targetPath, originalSize); + _logger.Trace("Copy successful, deleting source [{0}].", sourcePath); _diskProvider.DeleteFile(sourcePath); return TransferMode.Move; } diff --git a/src/NzbDrone.Common/Disk/IDiskProvider.cs b/src/NzbDrone.Common/Disk/IDiskProvider.cs index 7d40550f1..1c94daad3 100644 --- a/src/NzbDrone.Common/Disk/IDiskProvider.cs +++ b/src/NzbDrone.Common/Disk/IDiskProvider.cs @@ -30,10 +30,13 @@ namespace NzbDrone.Common.Disk long GetFileSize(string path); void CreateFolder(string path); void DeleteFile(string path); + void CloneFile(string source, string destination, bool overwrite = false); void CopyFile(string source, string destination, bool overwrite = false); void MoveFile(string source, string destination, bool overwrite = false); void MoveFolder(string source, string destination); + bool TryRenameFile(string source, string destination); bool TryCreateHardLink(string source, string destination); + bool TryCreateRefLink(string source, string destination); void DeleteFolder(string path, bool recursive); string ReadAllText(string filePath); void WriteAllText(string filename, string contents); diff --git a/src/NzbDrone.Mono/Disk/DiskProvider.cs b/src/NzbDrone.Mono/Disk/DiskProvider.cs index bed42bfaa..3125a9215 100644 --- a/src/NzbDrone.Mono/Disk/DiskProvider.cs +++ b/src/NzbDrone.Mono/Disk/DiskProvider.cs @@ -22,22 +22,26 @@ namespace NzbDrone.Mono.Disk private readonly Logger _logger; private readonly IProcMountProvider _procMountProvider; private readonly ISymbolicLinkResolver _symLinkResolver; + private readonly IRefLinkCreator _createRefLink; public DiskProvider(IProcMountProvider procMountProvider, ISymbolicLinkResolver symLinkResolver, + IRefLinkCreator createRefLink, Logger logger) - : this(new FileSystem(), procMountProvider, symLinkResolver, logger) + : this(new FileSystem(), procMountProvider, symLinkResolver, createRefLink, logger) { } public DiskProvider(IFileSystem fileSystem, IProcMountProvider procMountProvider, ISymbolicLinkResolver symLinkResolver, + IRefLinkCreator createRefLink, Logger logger) : base(fileSystem) { _procMountProvider = procMountProvider; _symLinkResolver = symLinkResolver; + _createRefLink = createRefLink; _logger = logger; } @@ -79,7 +83,7 @@ namespace NzbDrone.Mono.Disk var permissions = NativeConvert.FromOctalPermissionString(mask); - if (Directory.Exists(path)) + if (_fileSystem.Directory.Exists(path)) { permissions = GetFolderPermissions(permissions); } @@ -186,6 +190,19 @@ namespace NzbDrone.Mono.Disk return mount?.TotalSize; } + protected override void CloneFileInternal(string source, string destination, bool overwrite) + { + if (!FileExists(destination) && !UnixFileSystemInfo.GetFileSystemEntry(source).IsSymbolicLink) + { + if (_createRefLink.TryCreateRefLink(source, destination)) + { + return; + } + } + + CopyFileInternal(source, destination, overwrite); + } + protected override void CopyFileInternal(string source, string destination, bool overwrite) { var sourceInfo = UnixFileSystemInfo.GetFileSystemEntry(source); @@ -398,6 +415,11 @@ namespace NzbDrone.Mono.Disk } } + public override bool TryRenameFile(string source, string destination) + { + return Syscall.rename(source, destination) == 0; + } + public override bool TryCreateHardLink(string source, string destination) { try @@ -419,6 +441,11 @@ namespace NzbDrone.Mono.Disk } } + public override bool TryCreateRefLink(string source, string destination) + { + return _createRefLink.TryCreateRefLink(source, destination); + } + private uint GetUserId(string user) { if (user.IsNullOrWhiteSpace()) diff --git a/src/NzbDrone.Mono/Disk/RefLinkCreator.cs b/src/NzbDrone.Mono/Disk/RefLinkCreator.cs new file mode 100644 index 000000000..7a1e9692a --- /dev/null +++ b/src/NzbDrone.Mono/Disk/RefLinkCreator.cs @@ -0,0 +1,75 @@ +using System; +using Mono.Unix; +using Mono.Unix.Native; +using NLog; +using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Mono.Interop; + +namespace NzbDrone.Mono.Disk +{ + public interface IRefLinkCreator + { + bool TryCreateRefLink(string srcPath, string linkPath); + } + + public class RefLinkCreator : IRefLinkCreator + { + private readonly Logger _logger; + private readonly bool _supported; + + public RefLinkCreator(Logger logger) + { + _logger = logger; + + // Only support x86_64 because we know the FICLONE value is valid for it + _supported = OsInfo.IsLinux && (Syscall.uname(out var results) == 0 && results.machine == "x86_64"); + } + + public bool TryCreateRefLink(string srcPath, string linkPath) + { + if (!_supported) + { + return false; + } + + try + { + using (var srcHandle = NativeMethods.open(srcPath, OpenFlags.O_RDONLY)) + { + if (srcHandle.IsInvalid) + { + _logger.Trace("Failed to create reflink at '{0}' to '{1}': Couldn't open source file", linkPath, srcPath); + return false; + } + + using (var linkHandle = NativeMethods.open(linkPath, OpenFlags.O_WRONLY | OpenFlags.O_CREAT | OpenFlags.O_TRUNC)) + { + if (linkHandle.IsInvalid) + { + _logger.Trace("Failed to create reflink at '{0}' to '{1}': Couldn't create new link file", linkPath, srcPath); + return false; + } + + if (NativeMethods.clone_file(linkHandle, srcHandle) == -1) + { + var error = new UnixIOException(); + linkHandle.Dispose(); + Syscall.unlink(linkPath); + _logger.Trace("Failed to create reflink at '{0}' to '{1}': {2}", linkPath, srcPath, error.Message); + return false; + } + + _logger.Trace("Created reflink at '{0}' to '{1}'", linkPath, srcPath); + return true; + } + } + } + catch (Exception ex) + { + Syscall.unlink(linkPath); + _logger.Trace(ex, "Failed to create reflink at '{0}' to '{1}'", linkPath, srcPath); + return false; + } + } + } +} diff --git a/src/NzbDrone.Mono/Interop/NativeMethods.cs b/src/NzbDrone.Mono/Interop/NativeMethods.cs new file mode 100644 index 000000000..a92c78a2f --- /dev/null +++ b/src/NzbDrone.Mono/Interop/NativeMethods.cs @@ -0,0 +1,28 @@ +using System.Runtime.InteropServices; +using Mono.Unix.Native; + +namespace NzbDrone.Mono.Interop +{ + internal enum IoctlRequest : uint + { + // Hardcoded ioctl for FICLONE on a typical linux system + // #define FICLONE _IOW(0x94, 9, int) + FICLONE = 0x40049409 + } + + internal static class NativeMethods + { + [DllImport("libc", EntryPoint = "ioctl", SetLastError = true)] + private static extern int Ioctl(SafeUnixHandle dst_fd, IoctlRequest request, SafeUnixHandle src_fd); + + public static SafeUnixHandle open(string pathname, OpenFlags flags) + { + return new SafeUnixHandle(Syscall.open(pathname, flags)); + } + + internal static int clone_file(SafeUnixHandle link_fd, SafeUnixHandle src_fd) + { + return Ioctl(link_fd, IoctlRequest.FICLONE, src_fd); + } + } +} diff --git a/src/NzbDrone.Mono/Interop/SafeUnixHandle.cs b/src/NzbDrone.Mono/Interop/SafeUnixHandle.cs new file mode 100644 index 000000000..22b8f167d --- /dev/null +++ b/src/NzbDrone.Mono/Interop/SafeUnixHandle.cs @@ -0,0 +1,37 @@ +using System; +using System.Runtime.ConstrainedExecution; +using System.Runtime.InteropServices; +using System.Security.Permissions; +using Mono.Unix.Native; + +namespace NzbDrone.Mono.Interop +{ + [SecurityPermission(SecurityAction.InheritanceDemand, UnmanagedCode = true)] + [SecurityPermission(SecurityAction.Demand, UnmanagedCode = true)] + internal sealed class SafeUnixHandle : SafeHandle + { + [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)] + private SafeUnixHandle() + : base(new IntPtr(-1), true) + { + } + + [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)] + public SafeUnixHandle(int fd) + : base(new IntPtr(-1), true) + { + handle = new IntPtr(fd); + } + + public override bool IsInvalid + { + get { return handle == new IntPtr(-1); } + } + + [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)] + protected override bool ReleaseHandle() + { + return Syscall.close(handle.ToInt32()) != -1; + } + } +}