New: Fast copy using reflink on btrfs volumes

src/NzbDrone.Common/Disk/DiskProviderBase.cs

@@ -198,6 +198,24 @@ namespace NzbDrone.Common.Disk
             File.Delete(path);
         }
 
+        public void CloneFile(string source, string destination, bool overwrite = false)
+        {
+            Ensure.That(source, () => source).IsValidPath();
+            Ensure.That(destination, () => destination).IsValidPath();
+
+            if (source.PathEquals(destination))
+            {
+                throw new IOException(string.Format("Source and destination can't be the same {0}", source));
+            }
+
+            CloneFileInternal(source, destination, overwrite);
+        }
+
+        protected virtual void CloneFileInternal(string source, string destination, bool overwrite = false)
+        {
+            CopyFileInternal(source, destination, overwrite);
+        }
+
         public void CopyFile(string source, string destination, bool overwrite = false)
         {
             Ensure.That(source, () => source).IsValidPath();
@@ -259,8 +277,18 @@ namespace NzbDrone.Common.Disk
             File.Move(source, destination);
         }
 
+        public virtual bool TryRenameFile(string source, string destination)
+        {
+            return false;
+        }
+
         public abstract bool TryCreateHardLink(string source, string destination);
 
+        public virtual bool TryCreateRefLink(string source, string destination)
+        {
+            return false;
+        }
+
         public void DeleteFolder(string path, bool recursive)
         {
             Ensure.That(path, () => path).IsValidPath();

src/NzbDrone.Common/Disk/DiskTransferService.cs

@@ -283,18 +283,45 @@ namespace NzbDrone.Common.Disk
             var targetDriveFormat = targetMount?.DriveFormat ?? string.Empty;
 
             var isCifs = targetDriveFormat == "cifs";
+            var isBtrfs = sourceDriveFormat == "btrfs" && targetDriveFormat == "btrfs";
 
             if (mode.HasFlag(TransferMode.Copy))
             {
+                if (isBtrfs)
+                {
+                    if (_diskProvider.TryCreateRefLink(sourcePath, targetPath))
+                    {
+                        return TransferMode.Copy;
+                    }
+                }
+
                 TryCopyFileVerified(sourcePath, targetPath, originalSize);
                 return TransferMode.Copy;
             }
 
             if (mode.HasFlag(TransferMode.Move))
             {
+                if (isBtrfs)
+                {
+                    if (isSameMount && _diskProvider.TryRenameFile(sourcePath, targetPath))
+                    {
+                        _logger.Trace("Renamed [{0}] to [{1}].", sourcePath, targetPath);
+                        return TransferMode.Move;
+                    }
+
+                    if (_diskProvider.TryCreateRefLink(sourcePath, targetPath))
+                    {
+                        _logger.Trace("Reflink successful, deleting source [{0}].", sourcePath);
+                        _diskProvider.DeleteFile(sourcePath);
+                        return TransferMode.Move;
+                    }
+                }
+
                 if (isCifs && !isSameMount)
                 {
+                    _logger.Trace("On cifs mount. Starting verified copy [{0}] to [{1}].", sourcePath, targetPath);
                     TryCopyFileVerified(sourcePath, targetPath, originalSize);
+                    _logger.Trace("Copy successful, deleting source [{0}].", sourcePath);
                     _diskProvider.DeleteFile(sourcePath);
                     return TransferMode.Move;
                 }

src/NzbDrone.Common/Disk/IDiskProvider.cs

@@ -29,10 +29,13 @@ namespace NzbDrone.Common.Disk
         long GetFileSize(string path);
         void CreateFolder(string path);
         void DeleteFile(string path);
+        void CloneFile(string source, string destination, bool overwrite = false);
         void CopyFile(string source, string destination, bool overwrite = false);
         void MoveFile(string source, string destination, bool overwrite = false);
         void MoveFolder(string source, string destination, bool overwrite = false);
+        bool TryRenameFile(string source, string destination);
         bool TryCreateHardLink(string source, string destination);
+        bool TryCreateRefLink(string source, string destination);
         void DeleteFolder(string path, bool recursive);
         string ReadAllText(string filePath);
         void WriteAllText(string filename, string contents);

src/NzbDrone.Mono/Disk/DiskProvider.cs

@@ -21,11 +21,13 @@ namespace NzbDrone.Mono.Disk
         private readonly Logger _logger;
         private readonly IProcMountProvider _procMountProvider;
         private readonly ISymbolicLinkResolver _symLinkResolver;
+        private readonly ICreateRefLink _createRefLink;
 
-        public DiskProvider(IProcMountProvider procMountProvider, ISymbolicLinkResolver symLinkResolver, Logger logger)
+        public DiskProvider(IProcMountProvider procMountProvider, ISymbolicLinkResolver symLinkResolver, ICreateRefLink createRefLink, Logger logger)
         {
             _procMountProvider = procMountProvider;
             _symLinkResolver = symLinkResolver;
+            _createRefLink = createRefLink;
             _logger = logger;
         }
 
@@ -172,6 +174,19 @@ namespace NzbDrone.Mono.Disk
             return mount?.TotalSize;
         }
 
+        protected override void CloneFileInternal(string source, string destination, bool overwrite)
+        {
+            if (!File.Exists(destination) && !UnixFileSystemInfo.GetFileSystemEntry(source).IsSymbolicLink)
+            {
+                if (_createRefLink.TryCreateRefLink(source, destination))
+                {
+                    return;
+                }
+            }
+
+            CopyFileInternal(source, destination, overwrite);
+        }
+
         protected override void CopyFileInternal(string source, string destination, bool overwrite)
         {
             var sourceInfo = UnixFileSystemInfo.GetFileSystemEntry(source);
@@ -384,6 +399,11 @@ namespace NzbDrone.Mono.Disk
             }
         }
 
+        public override bool TryRenameFile(string source, string destination)
+        {
+            return Syscall.rename(source, destination) == 0;
+        }
+
         public override bool TryCreateHardLink(string source, string destination)
         {
             try
@@ -405,6 +425,11 @@ namespace NzbDrone.Mono.Disk
             }
         }
 
+        public override bool TryCreateRefLink(string source, string destination)
+        {
+            return _createRefLink.TryCreateRefLink(source, destination);
+        }
+
         private uint GetUserId(string user)
         {
             if (user.IsNullOrWhiteSpace())

src/NzbDrone.Mono/Disk/RefLinkCreator.cs

@@ -0,0 +1,75 @@
+using System;
+using Mono.Unix;
+using Mono.Unix.Native;
+using NLog;
+using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Mono.Interop;
+
+namespace NzbDrone.Mono.Disk
+{
+    public interface ICreateRefLink
+    {
+        bool TryCreateRefLink(string srcPath, string linkPath);
+    }
+
+    public class RefLinkCreator : ICreateRefLink
+    {
+        private readonly Logger _logger;
+        private readonly bool _supported;
+
+        public RefLinkCreator(Logger logger)
+        {
+            _logger = logger;
+
+            // Only support x86_64 because we know the FICLONE value is valid for it
+            _supported = OsInfo.IsLinux && (Syscall.uname(out var results) == 0 && results.machine == "x86_64");
+        }
+
+        public bool TryCreateRefLink(string srcPath, string linkPath)
+        {
+            if (!_supported)
+            {
+                return false;
+            }
+
+            try
+            {
+                using (var srcHandle = NativeMethods.open(srcPath, OpenFlags.O_RDONLY))
+                {
+                    if (srcHandle.IsInvalid)
+                    {
+                        _logger.Trace("Failed to create reflink at '{0}' to '{1}': Couldn't open source file", linkPath, srcPath);
+                        return false;
+                    }
+
+                    using (var linkHandle = NativeMethods.open(linkPath, OpenFlags.O_WRONLY | OpenFlags.O_CREAT | OpenFlags.O_TRUNC))
+                    {
+                        if (linkHandle.IsInvalid)
+                        {
+                            _logger.Trace("Failed to create reflink at '{0}' to '{1}': Couldn't create new link file", linkPath, srcPath);
+                            return false;
+                        }
+
+                        if (NativeMethods.clone_file(linkHandle, srcHandle) == -1)
+                        {
+                            var error = new UnixIOException();
+                            linkHandle.Dispose();
+                            Syscall.unlink(linkPath);
+                            _logger.Trace("Failed to create reflink at '{0}' to '{1}': {2}", linkPath, srcPath, error.Message);
+                            return false;
+                        }
+
+                        _logger.Trace("Created reflink at '{0}' to '{1}'", linkPath, srcPath);
+                        return true;
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Syscall.unlink(linkPath);
+                _logger.Trace(ex, "Failed to create reflink at '{0}' to '{1}'", linkPath, srcPath);
+                return false;
+            }
+        }
+    }
+}

src/NzbDrone.Mono/Interop/NativeMethods.cs

@@ -0,0 +1,28 @@
+using System.Runtime.InteropServices;
+using Mono.Unix.Native;
+
+namespace NzbDrone.Mono.Interop
+{
+    internal enum IoctlRequest : uint
+    {
+        // Hardcoded ioctl for FICLONE on a typical linux system
+        // #define FICLONE _IOW(0x94, 9, int)
+        FICLONE = 0x40049409
+    }
+
+    internal static class NativeMethods
+    {
+        [DllImport("libc", EntryPoint = "ioctl", SetLastError = true)]
+        private static extern int Ioctl(SafeUnixHandle dst_fd, IoctlRequest request, SafeUnixHandle src_fd);
+
+        public static SafeUnixHandle open(string pathname, OpenFlags flags)
+        {
+            return new SafeUnixHandle(Syscall.open(pathname, flags));
+        }
+
+        internal static int clone_file(SafeUnixHandle link_fd, SafeUnixHandle src_fd)
+        {
+            return Ioctl(link_fd, IoctlRequest.FICLONE, src_fd);
+        }
+    }
+}

src/NzbDrone.Mono/Interop/SafeUnixHandle.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Runtime.ConstrainedExecution;
+using System.Runtime.InteropServices;
+using System.Security.Permissions;
+using Mono.Unix.Native;
+
+namespace NzbDrone.Mono.Interop
+{
+    [SecurityPermission(SecurityAction.InheritanceDemand, UnmanagedCode = true)]
+    [SecurityPermission(SecurityAction.Demand, UnmanagedCode = true)]
+    internal sealed class SafeUnixHandle : SafeHandle
+    {
+        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
+        private SafeUnixHandle()
+            : base(new IntPtr(-1), true)
+        {
+        }
+
+        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
+        public SafeUnixHandle(int fd)
+            : base(new IntPtr(-1), true)
+        {
+            handle = new IntPtr(fd);
+        }
+
+        public override bool IsInvalid
+        {
+            get { return this.handle == new IntPtr(-1); }
+        }
+
+        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
+        protected override bool ReleaseHandle()
+        {
+            return Syscall.close(this.handle.ToInt32()) != -1;
+        }
+    }
+}