New: Fast copy using reflink on btrfs volumes

pull/1807/head
Taloth Saldono 4 years ago committed by Qstick
parent 4f220d9532
commit 9eb5b335f3

@ -212,6 +212,24 @@ namespace NzbDrone.Common.Disk
_fileSystem.File.Delete(path); _fileSystem.File.Delete(path);
} }
public void CloneFile(string source, string destination, bool overwrite = false)
{
Ensure.That(source, () => source).IsValidPath();
Ensure.That(destination, () => destination).IsValidPath();
if (source.PathEquals(destination))
{
throw new IOException(string.Format("Source and destination can't be the same {0}", source));
}
CloneFileInternal(source, destination, overwrite);
}
protected virtual void CloneFileInternal(string source, string destination, bool overwrite = false)
{
CopyFileInternal(source, destination, overwrite);
}
public void CopyFile(string source, string destination, bool overwrite = false) public void CopyFile(string source, string destination, bool overwrite = false)
{ {
Ensure.That(source, () => source).IsValidPath(); Ensure.That(source, () => source).IsValidPath();
@ -262,8 +280,18 @@ namespace NzbDrone.Common.Disk
_fileSystem.File.Move(source, destination); _fileSystem.File.Move(source, destination);
} }
public virtual bool TryRenameFile(string source, string destination)
{
return false;
}
public abstract bool TryCreateHardLink(string source, string destination); public abstract bool TryCreateHardLink(string source, string destination);
public virtual bool TryCreateRefLink(string source, string destination)
{
return false;
}
public void DeleteFolder(string path, bool recursive) public void DeleteFolder(string path, bool recursive)
{ {
Ensure.That(path, () => path).IsValidPath(); Ensure.That(path, () => path).IsValidPath();

@ -284,18 +284,45 @@ namespace NzbDrone.Common.Disk
var targetDriveFormat = targetMount?.DriveFormat ?? string.Empty; var targetDriveFormat = targetMount?.DriveFormat ?? string.Empty;
var isCifs = targetDriveFormat == "cifs"; var isCifs = targetDriveFormat == "cifs";
var isBtrfs = sourceDriveFormat == "btrfs" && targetDriveFormat == "btrfs";
if (mode.HasFlag(TransferMode.Copy)) if (mode.HasFlag(TransferMode.Copy))
{ {
if (isBtrfs)
{
if (_diskProvider.TryCreateRefLink(sourcePath, targetPath))
{
return TransferMode.Copy;
}
}
TryCopyFileVerified(sourcePath, targetPath, originalSize); TryCopyFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Copy; return TransferMode.Copy;
} }
if (mode.HasFlag(TransferMode.Move)) if (mode.HasFlag(TransferMode.Move))
{ {
if (isBtrfs)
{
if (isSameMount && _diskProvider.TryRenameFile(sourcePath, targetPath))
{
_logger.Trace("Renamed [{0}] to [{1}].", sourcePath, targetPath);
return TransferMode.Move;
}
if (_diskProvider.TryCreateRefLink(sourcePath, targetPath))
{
_logger.Trace("Reflink successful, deleting source [{0}].", sourcePath);
_diskProvider.DeleteFile(sourcePath);
return TransferMode.Move;
}
}
if (isCifs && !isSameMount) if (isCifs && !isSameMount)
{ {
_logger.Trace("On cifs mount. Starting verified copy [{0}] to [{1}].", sourcePath, targetPath);
TryCopyFileVerified(sourcePath, targetPath, originalSize); TryCopyFileVerified(sourcePath, targetPath, originalSize);
_logger.Trace("Copy successful, deleting source [{0}].", sourcePath);
_diskProvider.DeleteFile(sourcePath); _diskProvider.DeleteFile(sourcePath);
return TransferMode.Move; return TransferMode.Move;
} }

@ -30,10 +30,13 @@ namespace NzbDrone.Common.Disk
long GetFileSize(string path); long GetFileSize(string path);
void CreateFolder(string path); void CreateFolder(string path);
void DeleteFile(string path); void DeleteFile(string path);
void CloneFile(string source, string destination, bool overwrite = false);
void CopyFile(string source, string destination, bool overwrite = false); void CopyFile(string source, string destination, bool overwrite = false);
void MoveFile(string source, string destination, bool overwrite = false); void MoveFile(string source, string destination, bool overwrite = false);
void MoveFolder(string source, string destination); void MoveFolder(string source, string destination);
bool TryRenameFile(string source, string destination);
bool TryCreateHardLink(string source, string destination); bool TryCreateHardLink(string source, string destination);
bool TryCreateRefLink(string source, string destination);
void DeleteFolder(string path, bool recursive); void DeleteFolder(string path, bool recursive);
string ReadAllText(string filePath); string ReadAllText(string filePath);
void WriteAllText(string filename, string contents); void WriteAllText(string filename, string contents);

@ -22,22 +22,26 @@ namespace NzbDrone.Mono.Disk
private readonly Logger _logger; private readonly Logger _logger;
private readonly IProcMountProvider _procMountProvider; private readonly IProcMountProvider _procMountProvider;
private readonly ISymbolicLinkResolver _symLinkResolver; private readonly ISymbolicLinkResolver _symLinkResolver;
private readonly IRefLinkCreator _createRefLink;
public DiskProvider(IProcMountProvider procMountProvider, public DiskProvider(IProcMountProvider procMountProvider,
ISymbolicLinkResolver symLinkResolver, ISymbolicLinkResolver symLinkResolver,
IRefLinkCreator createRefLink,
Logger logger) Logger logger)
: this(new FileSystem(), procMountProvider, symLinkResolver, logger) : this(new FileSystem(), procMountProvider, symLinkResolver, createRefLink, logger)
{ {
} }
public DiskProvider(IFileSystem fileSystem, public DiskProvider(IFileSystem fileSystem,
IProcMountProvider procMountProvider, IProcMountProvider procMountProvider,
ISymbolicLinkResolver symLinkResolver, ISymbolicLinkResolver symLinkResolver,
IRefLinkCreator createRefLink,
Logger logger) Logger logger)
: base(fileSystem) : base(fileSystem)
{ {
_procMountProvider = procMountProvider; _procMountProvider = procMountProvider;
_symLinkResolver = symLinkResolver; _symLinkResolver = symLinkResolver;
_createRefLink = createRefLink;
_logger = logger; _logger = logger;
} }
@ -79,7 +83,7 @@ namespace NzbDrone.Mono.Disk
var permissions = NativeConvert.FromOctalPermissionString(mask); var permissions = NativeConvert.FromOctalPermissionString(mask);
if (Directory.Exists(path)) if (_fileSystem.Directory.Exists(path))
{ {
permissions = GetFolderPermissions(permissions); permissions = GetFolderPermissions(permissions);
} }
@ -186,6 +190,19 @@ namespace NzbDrone.Mono.Disk
return mount?.TotalSize; return mount?.TotalSize;
} }
protected override void CloneFileInternal(string source, string destination, bool overwrite)
{
if (!FileExists(destination) && !UnixFileSystemInfo.GetFileSystemEntry(source).IsSymbolicLink)
{
if (_createRefLink.TryCreateRefLink(source, destination))
{
return;
}
}
CopyFileInternal(source, destination, overwrite);
}
protected override void CopyFileInternal(string source, string destination, bool overwrite) protected override void CopyFileInternal(string source, string destination, bool overwrite)
{ {
var sourceInfo = UnixFileSystemInfo.GetFileSystemEntry(source); var sourceInfo = UnixFileSystemInfo.GetFileSystemEntry(source);
@ -398,6 +415,11 @@ namespace NzbDrone.Mono.Disk
} }
} }
public override bool TryRenameFile(string source, string destination)
{
return Syscall.rename(source, destination) == 0;
}
public override bool TryCreateHardLink(string source, string destination) public override bool TryCreateHardLink(string source, string destination)
{ {
try try
@ -419,6 +441,11 @@ namespace NzbDrone.Mono.Disk
} }
} }
public override bool TryCreateRefLink(string source, string destination)
{
return _createRefLink.TryCreateRefLink(source, destination);
}
private uint GetUserId(string user) private uint GetUserId(string user)
{ {
if (user.IsNullOrWhiteSpace()) if (user.IsNullOrWhiteSpace())

@ -0,0 +1,75 @@
using System;
using Mono.Unix;
using Mono.Unix.Native;
using NLog;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Mono.Interop;
namespace NzbDrone.Mono.Disk
{
public interface IRefLinkCreator
{
bool TryCreateRefLink(string srcPath, string linkPath);
}
public class RefLinkCreator : IRefLinkCreator
{
private readonly Logger _logger;
private readonly bool _supported;
public RefLinkCreator(Logger logger)
{
_logger = logger;
// Only support x86_64 because we know the FICLONE value is valid for it
_supported = OsInfo.IsLinux && (Syscall.uname(out var results) == 0 && results.machine == "x86_64");
}
public bool TryCreateRefLink(string srcPath, string linkPath)
{
if (!_supported)
{
return false;
}
try
{
using (var srcHandle = NativeMethods.open(srcPath, OpenFlags.O_RDONLY))
{
if (srcHandle.IsInvalid)
{
_logger.Trace("Failed to create reflink at '{0}' to '{1}': Couldn't open source file", linkPath, srcPath);
return false;
}
using (var linkHandle = NativeMethods.open(linkPath, OpenFlags.O_WRONLY | OpenFlags.O_CREAT | OpenFlags.O_TRUNC))
{
if (linkHandle.IsInvalid)
{
_logger.Trace("Failed to create reflink at '{0}' to '{1}': Couldn't create new link file", linkPath, srcPath);
return false;
}
if (NativeMethods.clone_file(linkHandle, srcHandle) == -1)
{
var error = new UnixIOException();
linkHandle.Dispose();
Syscall.unlink(linkPath);
_logger.Trace("Failed to create reflink at '{0}' to '{1}': {2}", linkPath, srcPath, error.Message);
return false;
}
_logger.Trace("Created reflink at '{0}' to '{1}'", linkPath, srcPath);
return true;
}
}
}
catch (Exception ex)
{
Syscall.unlink(linkPath);
_logger.Trace(ex, "Failed to create reflink at '{0}' to '{1}'", linkPath, srcPath);
return false;
}
}
}
}

@ -0,0 +1,28 @@
using System.Runtime.InteropServices;
using Mono.Unix.Native;
namespace NzbDrone.Mono.Interop
{
internal enum IoctlRequest : uint
{
// Hardcoded ioctl for FICLONE on a typical linux system
// #define FICLONE _IOW(0x94, 9, int)
FICLONE = 0x40049409
}
internal static class NativeMethods
{
[DllImport("libc", EntryPoint = "ioctl", SetLastError = true)]
private static extern int Ioctl(SafeUnixHandle dst_fd, IoctlRequest request, SafeUnixHandle src_fd);
public static SafeUnixHandle open(string pathname, OpenFlags flags)
{
return new SafeUnixHandle(Syscall.open(pathname, flags));
}
internal static int clone_file(SafeUnixHandle link_fd, SafeUnixHandle src_fd)
{
return Ioctl(link_fd, IoctlRequest.FICLONE, src_fd);
}
}
}

@ -0,0 +1,37 @@
using System;
using System.Runtime.ConstrainedExecution;
using System.Runtime.InteropServices;
using System.Security.Permissions;
using Mono.Unix.Native;
namespace NzbDrone.Mono.Interop
{
[SecurityPermission(SecurityAction.InheritanceDemand, UnmanagedCode = true)]
[SecurityPermission(SecurityAction.Demand, UnmanagedCode = true)]
internal sealed class SafeUnixHandle : SafeHandle
{
[ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
private SafeUnixHandle()
: base(new IntPtr(-1), true)
{
}
[ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
public SafeUnixHandle(int fd)
: base(new IntPtr(-1), true)
{
handle = new IntPtr(fd);
}
public override bool IsInvalid
{
get { return handle == new IntPtr(-1); }
}
[ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
protected override bool ReleaseHandle()
{
return Syscall.close(handle.ToInt32()) != -1;
}
}
}
Loading…
Cancel
Save