From 7002628514a84d83a1882310e7f11c10905eb1a7 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Wed, 10 Jun 2020 20:48:54 +0200 Subject: [PATCH] New: Fast copy using reflink on btrfs volumes Signed-off-by: Robin Dadswell --- src/NzbDrone.Common/Disk/DiskProviderBase.cs | 28 +++ .../Disk/DiskTransferService.cs | 27 +++ src/NzbDrone.Common/Disk/IDiskProvider.cs | 3 + src/NzbDrone.Mono/Disk/DiskProvider.cs | 164 +++++++++++++++++- src/NzbDrone.Mono/Disk/RefLinkCreator.cs | 75 ++++++++ src/NzbDrone.Mono/Interop/NativeMethods.cs | 28 +++ src/NzbDrone.Mono/Interop/SafeUnixHandle.cs | 37 ++++ 7 files changed, 358 insertions(+), 4 deletions(-) create mode 100644 src/NzbDrone.Mono/Disk/RefLinkCreator.cs create mode 100644 src/NzbDrone.Mono/Interop/NativeMethods.cs create mode 100644 src/NzbDrone.Mono/Interop/SafeUnixHandle.cs diff --git a/src/NzbDrone.Common/Disk/DiskProviderBase.cs b/src/NzbDrone.Common/Disk/DiskProviderBase.cs index d866db663..cc1e407ac 100644 --- a/src/NzbDrone.Common/Disk/DiskProviderBase.cs +++ b/src/NzbDrone.Common/Disk/DiskProviderBase.cs @@ -212,6 +212,24 @@ namespace NzbDrone.Common.Disk _fileSystem.File.Delete(path); } + public void CloneFile(string source, string destination, bool overwrite = false) + { + Ensure.That(source, () => source).IsValidPath(); + Ensure.That(destination, () => destination).IsValidPath(); + + if (source.PathEquals(destination)) + { + throw new IOException(string.Format("Source and destination can't be the same {0}", source)); + } + + CloneFileInternal(source, destination, overwrite); + } + + protected virtual void CloneFileInternal(string source, string destination, bool overwrite = false) + { + CopyFileInternal(source, destination, overwrite); + } + public void CopyFile(string source, string destination, bool overwrite = false) { Ensure.That(source, () => source).IsValidPath(); @@ -262,8 +280,18 @@ namespace NzbDrone.Common.Disk _fileSystem.File.Move(source, destination); } + public virtual bool TryRenameFile(string source, string destination) + { + return false; + } + public abstract bool TryCreateHardLink(string source, string destination); + public virtual bool TryCreateRefLink(string source, string destination) + { + return false; + } + public void DeleteFolder(string path, bool recursive) { Ensure.That(path, () => path).IsValidPath(); diff --git a/src/NzbDrone.Common/Disk/DiskTransferService.cs b/src/NzbDrone.Common/Disk/DiskTransferService.cs index 01281da4d..8a75ed34d 100644 --- a/src/NzbDrone.Common/Disk/DiskTransferService.cs +++ b/src/NzbDrone.Common/Disk/DiskTransferService.cs @@ -284,18 +284,45 @@ namespace NzbDrone.Common.Disk var targetDriveFormat = targetMount?.DriveFormat ?? string.Empty; var isCifs = targetDriveFormat == "cifs"; + var isBtrfs = sourceDriveFormat == "btrfs" && targetDriveFormat == "btrfs"; if (mode.HasFlag(TransferMode.Copy)) { + if (isBtrfs) + { + if (_diskProvider.TryCreateRefLink(sourcePath, targetPath)) + { + return TransferMode.Copy; + } + } + TryCopyFileVerified(sourcePath, targetPath, originalSize); return TransferMode.Copy; } if (mode.HasFlag(TransferMode.Move)) { + if (isBtrfs) + { + if (isSameMount && _diskProvider.TryRenameFile(sourcePath, targetPath)) + { + _logger.Trace("Renamed [{0}] to [{1}].", sourcePath, targetPath); + return TransferMode.Move; + } + + if (_diskProvider.TryCreateRefLink(sourcePath, targetPath)) + { + _logger.Trace("Reflink successful, deleting source [{0}].", sourcePath); + _diskProvider.DeleteFile(sourcePath); + return TransferMode.Move; + } + } + if (isCifs && !isSameMount) { + _logger.Trace("On cifs mount. Starting verified copy [{0}] to [{1}].", sourcePath, targetPath); TryCopyFileVerified(sourcePath, targetPath, originalSize); + _logger.Trace("Copy successful, deleting source [{0}].", sourcePath); _diskProvider.DeleteFile(sourcePath); return TransferMode.Move; } diff --git a/src/NzbDrone.Common/Disk/IDiskProvider.cs b/src/NzbDrone.Common/Disk/IDiskProvider.cs index 7d40550f1..1c94daad3 100644 --- a/src/NzbDrone.Common/Disk/IDiskProvider.cs +++ b/src/NzbDrone.Common/Disk/IDiskProvider.cs @@ -30,10 +30,13 @@ namespace NzbDrone.Common.Disk long GetFileSize(string path); void CreateFolder(string path); void DeleteFile(string path); + void CloneFile(string source, string destination, bool overwrite = false); void CopyFile(string source, string destination, bool overwrite = false); void MoveFile(string source, string destination, bool overwrite = false); void MoveFolder(string source, string destination); + bool TryRenameFile(string source, string destination); bool TryCreateHardLink(string source, string destination); + bool TryCreateRefLink(string source, string destination); void DeleteFolder(string path, bool recursive); string ReadAllText(string filePath); void WriteAllText(string filename, string contents); diff --git a/src/NzbDrone.Mono/Disk/DiskProvider.cs b/src/NzbDrone.Mono/Disk/DiskProvider.cs index 8136ac19c..b6da9ac0c 100644 --- a/src/NzbDrone.Mono/Disk/DiskProvider.cs +++ b/src/NzbDrone.Mono/Disk/DiskProvider.cs @@ -23,20 +23,27 @@ namespace NzbDrone.Mono.Disk private readonly IProcMountProvider _procMountProvider; private readonly ISymbolicLinkResolver _symLinkResolver; + private readonly IRefLinkCreator _createRefLink; public DiskProvider(IProcMountProvider procMountProvider, - ISymbolicLinkResolver symLinkResolver) - : this(new FileSystem(), procMountProvider, symLinkResolver) + ISymbolicLinkResolver symLinkResolver, + IRefLinkCreator createRefLink, + Logger logger) + : this(new FileSystem(), procMountProvider, symLinkResolver, createRefLink, logger) { } public DiskProvider(IFileSystem fileSystem, IProcMountProvider procMountProvider, - ISymbolicLinkResolver symLinkResolver) + ISymbolicLinkResolver symLinkResolver, + IRefLinkCreator createRefLink, + Logger logger) : base(fileSystem) { _procMountProvider = procMountProvider; _symLinkResolver = symLinkResolver; + _createRefLink = createRefLink; + _logger = logger; } public override IMount GetMount(string path) @@ -77,7 +84,7 @@ namespace NzbDrone.Mono.Disk var permissions = NativeConvert.FromOctalPermissionString(mask); - if (Directory.Exists(path)) + if (_fileSystem.Directory.Exists(path)) { permissions = GetFolderPermissions(permissions); } @@ -184,6 +191,19 @@ namespace NzbDrone.Mono.Disk return mount?.TotalSize; } + protected override void CloneFileInternal(string source, string destination, bool overwrite) + { + if (!FileExists(destination) && !UnixFileSystemInfo.GetFileSystemEntry(source).IsSymbolicLink) + { + if (_createRefLink.TryCreateRefLink(source, destination)) + { + return; + } + } + + CopyFileInternal(source, destination, overwrite); + } + protected override void CopyFileInternal(string source, string destination, bool overwrite) { var sourceInfo = UnixFileSystemInfo.GetFileSystemEntry(source); @@ -259,6 +279,137 @@ namespace NzbDrone.Mono.Disk } } + private void TransferFilePatched(string source, string destination, bool overwrite, bool move) + { + // Mono 6.x throws errors if permissions or timestamps cannot be set + // - In 6.0 it'll leave a full length file + // - In 6.6 it'll leave a zero length file + // Catch the exception and attempt to handle these edgecases + + // Mono 6.x till 6.10 doesn't properly try use rename first. + if (move && + ((PlatformInfo.Platform == PlatformType.Mono && PlatformInfo.GetVersion() < new Version(6, 10)) || + (PlatformInfo.Platform == PlatformType.NetCore))) + { + if (Syscall.lstat(source, out var sourcestat) == 0 && + Syscall.lstat(destination, out var deststat) != 0 && + Syscall.rename(source, destination) == 0) + { + _logger.Trace("Moved '{0}' -> '{1}' using Syscall.rename", source, destination); + return; + } + } + + try + { + if (move) + { + base.MoveFileInternal(source, destination); + } + else + { + base.CopyFileInternal(source, destination); + } + } + catch (UnauthorizedAccessException) + { + var srcInfo = new FileInfo(source); + var dstInfo = new FileInfo(destination); + var exists = dstInfo.Exists && srcInfo.Exists; + + if (PlatformInfo.Platform == PlatformType.Mono && PlatformInfo.GetVersion() >= new Version(6, 6) && + exists && dstInfo.Length == 0 && srcInfo.Length != 0) + { + // mono >=6.6 bug: zero length file since chmod happens at the start + _logger.Debug("{3} failed to {2} file likely due to known {3} bug, attempting to {2} directly. '{0}' -> '{1}'", source, destination, move ? "move" : "copy", PlatformInfo.PlatformName); + + try + { + _logger.Trace("Copying content from {0} to {1} ({2} bytes)", source, destination, srcInfo.Length); + using (var srcStream = new FileStream(source, FileMode.Open, FileAccess.Read)) + using (var dstStream = new FileStream(destination, FileMode.Create, FileAccess.Write)) + { + srcStream.CopyTo(dstStream); + } + } + catch + { + // If it fails again then bail + throw; + } + } + else if (((PlatformInfo.Platform == PlatformType.Mono && + PlatformInfo.GetVersion() >= new Version(6, 0) && + PlatformInfo.GetVersion() < new Version(6, 6)) || + PlatformInfo.Platform == PlatformType.NetCore) && + exists && dstInfo.Length == srcInfo.Length) + { + // mono 6.0, mono 6.4 and netcore 3.1 bug: full length file since utime and chmod happens at the end + _logger.Debug("{3} failed to {2} file likely due to known {3} bug, attempting to {2} directly. '{0}' -> '{1}'", source, destination, move ? "move" : "copy", PlatformInfo.PlatformName); + + // Check at least part of the file since UnauthorizedAccess can happen due to legitimate reasons too + var checkLength = (int)Math.Min(64 * 1024, dstInfo.Length); + if (checkLength > 0) + { + var srcData = new byte[checkLength]; + var dstData = new byte[checkLength]; + + _logger.Trace("Check last {0} bytes from {1}", checkLength, destination); + + using (var srcStream = new FileStream(source, FileMode.Open, FileAccess.Read)) + using (var dstStream = new FileStream(destination, FileMode.Open, FileAccess.Read)) + { + srcStream.Position = srcInfo.Length - checkLength; + dstStream.Position = dstInfo.Length - checkLength; + + srcStream.Read(srcData, 0, checkLength); + dstStream.Read(dstData, 0, checkLength); + } + + for (var i = 0; i < checkLength; i++) + { + if (srcData[i] != dstData[i]) + { + // Files aren't the same, the UnauthorizedAccess was unrelated + _logger.Trace("Copy was incomplete, rethrowing original error"); + throw; + } + } + + _logger.Trace("Copy was complete, finishing {0} operation", move ? "move" : "copy"); + } + } + else + { + // Unrecognized situation, the UnauthorizedAccess was unrelated + throw; + } + + if (exists) + { + try + { + dstInfo.LastWriteTimeUtc = srcInfo.LastWriteTimeUtc; + } + catch + { + _logger.Debug("Unable to change last modified date for {0}, skipping.", destination); + } + + if (move) + { + _logger.Trace("Removing source file {0}", source); + File.Delete(source); + } + } + } + } + + public override bool TryRenameFile(string source, string destination) + { + return Syscall.rename(source, destination) == 0; + } + public override bool TryCreateHardLink(string source, string destination) { try @@ -280,6 +431,11 @@ namespace NzbDrone.Mono.Disk } } + public override bool TryCreateRefLink(string source, string destination) + { + return _createRefLink.TryCreateRefLink(source, destination); + } + private uint GetUserId(string user) { if (user.IsNullOrWhiteSpace()) diff --git a/src/NzbDrone.Mono/Disk/RefLinkCreator.cs b/src/NzbDrone.Mono/Disk/RefLinkCreator.cs new file mode 100644 index 000000000..7a1e9692a --- /dev/null +++ b/src/NzbDrone.Mono/Disk/RefLinkCreator.cs @@ -0,0 +1,75 @@ +using System; +using Mono.Unix; +using Mono.Unix.Native; +using NLog; +using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Mono.Interop; + +namespace NzbDrone.Mono.Disk +{ + public interface IRefLinkCreator + { + bool TryCreateRefLink(string srcPath, string linkPath); + } + + public class RefLinkCreator : IRefLinkCreator + { + private readonly Logger _logger; + private readonly bool _supported; + + public RefLinkCreator(Logger logger) + { + _logger = logger; + + // Only support x86_64 because we know the FICLONE value is valid for it + _supported = OsInfo.IsLinux && (Syscall.uname(out var results) == 0 && results.machine == "x86_64"); + } + + public bool TryCreateRefLink(string srcPath, string linkPath) + { + if (!_supported) + { + return false; + } + + try + { + using (var srcHandle = NativeMethods.open(srcPath, OpenFlags.O_RDONLY)) + { + if (srcHandle.IsInvalid) + { + _logger.Trace("Failed to create reflink at '{0}' to '{1}': Couldn't open source file", linkPath, srcPath); + return false; + } + + using (var linkHandle = NativeMethods.open(linkPath, OpenFlags.O_WRONLY | OpenFlags.O_CREAT | OpenFlags.O_TRUNC)) + { + if (linkHandle.IsInvalid) + { + _logger.Trace("Failed to create reflink at '{0}' to '{1}': Couldn't create new link file", linkPath, srcPath); + return false; + } + + if (NativeMethods.clone_file(linkHandle, srcHandle) == -1) + { + var error = new UnixIOException(); + linkHandle.Dispose(); + Syscall.unlink(linkPath); + _logger.Trace("Failed to create reflink at '{0}' to '{1}': {2}", linkPath, srcPath, error.Message); + return false; + } + + _logger.Trace("Created reflink at '{0}' to '{1}'", linkPath, srcPath); + return true; + } + } + } + catch (Exception ex) + { + Syscall.unlink(linkPath); + _logger.Trace(ex, "Failed to create reflink at '{0}' to '{1}'", linkPath, srcPath); + return false; + } + } + } +} diff --git a/src/NzbDrone.Mono/Interop/NativeMethods.cs b/src/NzbDrone.Mono/Interop/NativeMethods.cs new file mode 100644 index 000000000..a92c78a2f --- /dev/null +++ b/src/NzbDrone.Mono/Interop/NativeMethods.cs @@ -0,0 +1,28 @@ +using System.Runtime.InteropServices; +using Mono.Unix.Native; + +namespace NzbDrone.Mono.Interop +{ + internal enum IoctlRequest : uint + { + // Hardcoded ioctl for FICLONE on a typical linux system + // #define FICLONE _IOW(0x94, 9, int) + FICLONE = 0x40049409 + } + + internal static class NativeMethods + { + [DllImport("libc", EntryPoint = "ioctl", SetLastError = true)] + private static extern int Ioctl(SafeUnixHandle dst_fd, IoctlRequest request, SafeUnixHandle src_fd); + + public static SafeUnixHandle open(string pathname, OpenFlags flags) + { + return new SafeUnixHandle(Syscall.open(pathname, flags)); + } + + internal static int clone_file(SafeUnixHandle link_fd, SafeUnixHandle src_fd) + { + return Ioctl(link_fd, IoctlRequest.FICLONE, src_fd); + } + } +} diff --git a/src/NzbDrone.Mono/Interop/SafeUnixHandle.cs b/src/NzbDrone.Mono/Interop/SafeUnixHandle.cs new file mode 100644 index 000000000..22b8f167d --- /dev/null +++ b/src/NzbDrone.Mono/Interop/SafeUnixHandle.cs @@ -0,0 +1,37 @@ +using System; +using System.Runtime.ConstrainedExecution; +using System.Runtime.InteropServices; +using System.Security.Permissions; +using Mono.Unix.Native; + +namespace NzbDrone.Mono.Interop +{ + [SecurityPermission(SecurityAction.InheritanceDemand, UnmanagedCode = true)] + [SecurityPermission(SecurityAction.Demand, UnmanagedCode = true)] + internal sealed class SafeUnixHandle : SafeHandle + { + [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)] + private SafeUnixHandle() + : base(new IntPtr(-1), true) + { + } + + [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)] + public SafeUnixHandle(int fd) + : base(new IntPtr(-1), true) + { + handle = new IntPtr(fd); + } + + public override bool IsInvalid + { + get { return handle == new IntPtr(-1); } + } + + [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)] + protected override bool ReleaseHandle() + { + return Syscall.close(handle.ToInt32()) != -1; + } + } +}