Fixed: ZFS and other mounts now listed in the System page.

Will now also automatically revert to a fully transactional move/copy if the move is in our out of a cifs mount. (assuming the cifs mount can be detected)
pull/1037/merge
Taloth Saldono 9 years ago
parent f5b3d70641
commit 97cdb6a4a5

@ -24,6 +24,10 @@ namespace NzbDrone.Common.Test.DiskTests
{
Mocker.GetMock<IDiskProvider>(MockBehavior.Strict);
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.GetMount(It.IsAny<string>()))
.Returns((IMount)null);
WithEmulatedDiskProvider();
WithExistingFile(_sourcePath);

@ -346,12 +346,12 @@ namespace NzbDrone.Common.Disk
public string[] GetFixedDrives()
{
return (DriveInfo.GetDrives().Where(x => x.DriveType == DriveType.Fixed).Select(x => x.Name)).ToArray();
return GetMounts().Where(x => x.DriveType == DriveType.Fixed).Select(x => x.RootDirectory).ToArray();
}
public string GetVolumeLabel(string path)
{
var driveInfo = DriveInfo.GetDrives().SingleOrDefault(d => d.Name == path);
var driveInfo = GetMounts().SingleOrDefault(d => d.RootDirectory.PathEquals(path));
if (driveInfo == null)
{
@ -376,11 +376,28 @@ namespace NzbDrone.Common.Disk
return new FileStream(path, FileMode.Create);
}
public List<DriveInfo> GetDrives()
public virtual List<IMount> GetMounts()
{
return GetDriveInfoMounts();
}
public virtual IMount GetMount(string path)
{
var mounts = GetMounts();
return mounts.Where(drive => drive.RootDirectory.PathEquals(path) ||
drive.RootDirectory.IsParentPath(path))
.OrderByDescending(drive => drive.RootDirectory.Length)
.FirstOrDefault();
}
protected List<IMount> GetDriveInfoMounts()
{
return DriveInfo.GetDrives()
.Where(d => d.DriveType == DriveType.Fixed || d.DriveType == DriveType.Network)
.Where(d => d.DriveType == DriveType.Fixed || d.DriveType == DriveType.Network || d.DriveType == DriveType.Removable)
.Where(d => d.IsReady)
.Select(d => new DriveInfoMount(d))
.Cast<IMount>()
.ToList();
}

@ -45,6 +45,13 @@ namespace NzbDrone.Common.Disk
}
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, bool verified = true)
{
var verificationMode = verified ? VerificationMode : DiskTransferVerificationMode.VerifyOnly;
return TransferFolder(sourcePath, targetPath, mode, verificationMode);
}
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, DiskTransferVerificationMode verificationMode)
{
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath();
@ -58,14 +65,14 @@ namespace NzbDrone.Common.Disk
foreach (var subDir in _diskProvider.GetDirectoryInfos(sourcePath))
{
result &= TransferFolder(subDir.FullName, Path.Combine(targetPath, subDir.Name), mode, verified);
result &= TransferFolder(subDir.FullName, Path.Combine(targetPath, subDir.Name), mode, verificationMode);
}
foreach (var sourceFile in _diskProvider.GetFileInfos(sourcePath))
{
var destFile = Path.Combine(targetPath, sourceFile.Name);
result &= TransferFile(sourceFile.FullName, destFile, mode, true, verified);
result &= TransferFile(sourceFile.FullName, destFile, mode, true, verificationMode);
}
if (mode.HasFlag(TransferMode.Move))
@ -77,15 +84,17 @@ namespace NzbDrone.Common.Disk
}
public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite = false, bool verified = true)
{
var verificationMode = verified ? VerificationMode : DiskTransferVerificationMode.None;
return TransferFile(sourcePath, targetPath, mode, overwrite, verificationMode);
}
public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite, DiskTransferVerificationMode verificationMode)
{
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath();
if (VerificationMode != DiskTransferVerificationMode.Transactional && VerificationMode != DiskTransferVerificationMode.TryTransactional)
{
verified = false;
}
_logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath);
var originalSize = _diskProvider.GetFileSize(sourcePath);
@ -154,49 +163,59 @@ namespace NzbDrone.Common.Disk
}
}
if (verified)
// We force a transactional transfer if the transfer occurs between mounts and one of the mounts is cifs, it would be a copy anyway.
if (verificationMode == DiskTransferVerificationMode.TryTransactional && OsInfo.IsNotWindows)
{
if (mode.HasFlag(TransferMode.Copy))
var sourceMount = _diskProvider.GetMount(sourcePath);
var targetMount = _diskProvider.GetMount(targetPath);
if (sourceMount != null && targetMount != null && sourceMount.RootDirectory != targetMount.RootDirectory &&
(sourceMount.DriveFormat == "cifs" || targetMount.DriveFormat == "cifs"))
{
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{
return TransferMode.Copy;
}
verificationMode = DiskTransferVerificationMode.Transactional;
}
}
if (mode.HasFlag(TransferMode.Move))
if (mode.HasFlag(TransferMode.Copy))
{
if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional)
{
if (TryMoveFileTransactional(sourcePath, targetPath, originalSize))
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{
return TransferMode.Move;
return TransferMode.Copy;
}
}
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (VerificationMode != DiskTransferVerificationMode.None)
{
if (mode.HasFlag(TransferMode.Copy))
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
TryCopyFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Copy;
}
if (mode.HasFlag(TransferMode.Move))
else
{
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Move;
_diskProvider.CopyFile(sourcePath, targetPath);
return TransferMode.Copy;
}
}
else
if (mode.HasFlag(TransferMode.Move))
{
if (mode.HasFlag(TransferMode.Copy))
if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional)
{
_diskProvider.CopyFile(sourcePath, targetPath);
return TransferMode.Copy;
}
if (TryMoveFileTransactional(sourcePath, targetPath, originalSize, verificationMode))
{
return TransferMode.Move;
}
if (mode.HasFlag(TransferMode.Move))
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Move;
}
else
{
_diskProvider.MoveFile(sourcePath, targetPath);
return TransferMode.Move;
@ -340,7 +359,7 @@ namespace NzbDrone.Common.Disk
return false;
}
private bool TryMoveFileTransactional(string sourcePath, string targetPath, long originalSize)
private bool TryMoveFileTransactional(string sourcePath, string targetPath, long originalSize, DiskTransferVerificationMode verificationMode)
{
var backupPath = sourcePath + ".backup~";
var tempTargetPath = targetPath + ".partial~";
@ -394,7 +413,7 @@ namespace NzbDrone.Common.Disk
}
}
if (VerificationMode == DiskTransferVerificationMode.Transactional)
if (verificationMode == DiskTransferVerificationMode.Transactional)
{
_logger.Trace("Hardlink move failed, reverting to copy.");
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))

@ -0,0 +1,76 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using NzbDrone.Common.Extensions;
namespace NzbDrone.Common.Disk
{
public class DriveInfoMount : IMount
{
private readonly DriveInfo _driveInfo;
public DriveInfoMount(DriveInfo driveInfo)
{
_driveInfo = driveInfo;
}
public long AvailableFreeSpace
{
get { return _driveInfo.AvailableFreeSpace; }
}
public string DriveFormat
{
get { return _driveInfo.DriveFormat; }
}
public DriveType DriveType
{
get { return _driveInfo.DriveType; }
}
public bool IsReady
{
get { return _driveInfo.IsReady; }
}
public string Name
{
get { return _driveInfo.Name; }
}
public string RootDirectory
{
get { return _driveInfo.RootDirectory.FullName; }
}
public long TotalFreeSpace
{
get { return _driveInfo.TotalFreeSpace; }
}
public long TotalSize
{
get { return _driveInfo.TotalSize; }
}
public string VolumeLabel
{
get { return _driveInfo.VolumeLabel; }
}
public string VolumeName
{
get
{
if (VolumeLabel.IsNullOrWhiteSpace())
{
return Name;
}
return string.Format("{0} ({1})", Name, VolumeLabel);
}
}
}
}

@ -103,12 +103,12 @@ namespace NzbDrone.Common.Disk
private List<FileSystemModel> GetDrives()
{
return _diskProvider.GetDrives()
return _diskProvider.GetMounts()
.Select(d => new FileSystemModel
{
Type = FileSystemEntityType.Drive,
Name = GetVolumeName(d),
Path = d.Name,
Name = d.VolumeLabel,
Path = d.RootDirectory,
LastModified = null
})
.ToList();
@ -158,16 +158,6 @@ namespace NzbDrone.Common.Disk
return path;
}
private string GetVolumeName(DriveInfo driveInfo)
{
if (driveInfo.VolumeLabel.IsNullOrWhiteSpace())
{
return driveInfo.Name;
}
return string.Format("{0} ({1})", driveInfo.Name, driveInfo.VolumeLabel);
}
private string GetParent(string path)
{
var di = new DirectoryInfo(path);

@ -40,11 +40,11 @@ namespace NzbDrone.Common.Disk
void SetPermissions(string filename, WellKnownSidType accountSid, FileSystemRights rights, AccessControlType controlType);
FileAttributes GetFileAttributes(string path);
void EmptyFolder(string path);
string[] GetFixedDrives();
string GetVolumeLabel(string path);
FileStream OpenReadStream(string path);
FileStream OpenWriteStream(string path);
List<DriveInfo> GetDrives();
List<IMount> GetMounts();
IMount GetMount(string path);
List<DirectoryInfo> GetDirectoryInfos(string path);
List<FileInfo> GetFileInfos(string path);
}

@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace NzbDrone.Common.Disk
{
public interface IMount
{
long AvailableFreeSpace { get; }
string DriveFormat { get; }
DriveType DriveType { get; }
bool IsReady { get; }
string Name { get; }
string RootDirectory { get; }
long TotalFreeSpace { get; }
long TotalSize { get; }
string VolumeLabel { get; }
}
}

@ -73,8 +73,14 @@ namespace NzbDrone.Common.Extensions
public static bool IsParentPath(this string parentPath, string childPath)
{
parentPath = parentPath.TrimEnd(Path.DirectorySeparatorChar);
childPath = childPath.TrimEnd(Path.DirectorySeparatorChar);
if (parentPath != "/")
{
parentPath = parentPath.TrimEnd(Path.DirectorySeparatorChar);
}
if (childPath != "/")
{
childPath = childPath.TrimEnd(Path.DirectorySeparatorChar);
}
var parent = new DirectoryInfo(parentPath);
var child = new DirectoryInfo(childPath);

@ -72,6 +72,8 @@
<Compile Include="ConvertBase32.cs" />
<Compile Include="Crypto\HashProvider.cs" />
<Compile Include="Disk\FileSystemLookupService.cs" />
<Compile Include="Disk\DriveInfoMount.cs" />
<Compile Include="Disk\IMount.cs" />
<Compile Include="Disk\RelativeFileSystemModel.cs" />
<Compile Include="Disk\FileSystemModel.cs" />
<Compile Include="Disk\FileSystemResult.cs" />

@ -1,4 +1,5 @@
using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using NLog;
@ -58,7 +59,7 @@ namespace NzbDrone.Core.DiskSpace
private IEnumerable<DiskSpace> GetFixedDisksFreeSpace()
{
return GetDiskSpace(_diskProvider.GetFixedDrives(), true);
return GetDiskSpace(_diskProvider.GetMounts().Where(d => d.DriveType == DriveType.Fixed).Select(d => d.RootDirectory), true);
}
private IEnumerable<DiskSpace> GetDiskSpace(IEnumerable<string> paths, bool suppressWarnings = false)

@ -4,6 +4,7 @@ using System.Text.RegularExpressions;
using FluentValidation.Validators;
using NzbDrone.Common.Disk;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
namespace NzbDrone.Core.Validation.Paths
{
@ -31,17 +32,11 @@ namespace NzbDrone.Core.Validation.Paths
if (!DriveRegex.IsMatch(path)) return true;
var drives = _diskProvider.GetDrives();
var mount = _diskProvider.GetMount(path);
foreach (var drive in drives)
if (mount != null && mount.DriveType == DriveType.Network)
{
if (path.StartsWith(drive.Name, StringComparison.InvariantCultureIgnoreCase))
{
if (drive.DriveType == DriveType.Network)
{
return false;
}
}
return false;
}
return true;

@ -1,13 +1,13 @@
using System;
using System.IO;
using System.Linq;
using Mono.Unix;
using Mono.Unix.Native;
using NLog;
using NzbDrone.Common.Disk;
using NzbDrone.Common.EnsureThat;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Instrumentation;
using Mono.Unix;
namespace NzbDrone.Mono
{
@ -15,26 +15,28 @@ namespace NzbDrone.Mono
{
private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(DiskProvider));
private readonly IProcMountProvider _procMountProvider;
public DiskProvider(IProcMountProvider procMountProvider)
{
_procMountProvider = procMountProvider;
}
public override long? GetAvailableSpace(string path)
{
Ensure.That(path, () => path).IsValidPath();
var root = GetPathRoot(path);
if (!FolderExists(root))
throw new DirectoryNotFoundException(root);
try
{
var driveInfo = GetDriveInfo(path);
var mount = GetMount(path);
if (driveInfo == null)
if (mount == null)
{
Logger.Debug("Unable to get free space for '{0}', unable to find suitable drive", path);
return null;
}
return driveInfo.AvailableFreeSpace;
return mount.AvailableFreeSpace;
}
catch (InvalidOperationException ex)
{
@ -116,22 +118,25 @@ namespace NzbDrone.Mono
}
}
public override System.Collections.Generic.List<IMount> GetMounts()
{
return base.GetMounts()
.Concat(_procMountProvider.GetMounts())
.DistinctBy(v => v.RootDirectory)
.ToList();
}
public override long? GetTotalSize(string path)
{
Ensure.That(path, () => path).IsValidPath();
var root = GetPathRoot(path);
if (!FolderExists(root))
throw new DirectoryNotFoundException(root);
try
{
var driveInfo = GetDriveInfo(path);
var mount = GetMount(path);
if (driveInfo == null) return null;
if (mount == null) return null;
return driveInfo.TotalSize;
return mount.TotalSize;
}
catch (InvalidOperationException e)
{
@ -141,18 +146,6 @@ namespace NzbDrone.Mono
return null;
}
private DriveInfo GetDriveInfo(string path)
{
var drives = DriveInfo.GetDrives();
return
drives.Where(drive => drive.IsReady &&
drive.Name.IsNotNullOrWhiteSpace() &&
path.StartsWith(drive.Name, StringComparison.CurrentCultureIgnoreCase))
.OrderByDescending(drive => drive.Name.Length)
.FirstOrDefault();
}
public override bool TryCreateHardLink(string source, string destination)
{
try

@ -71,6 +71,8 @@
<Compile Include="DiskProvider.cs" />
<Compile Include="LinuxPermissionsException.cs" />
<Compile Include="MonoRuntimeProvider.cs" />
<Compile Include="ProcMount.cs" />
<Compile Include="ProcMountProvider.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>

@ -0,0 +1,60 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using NLog;
using NzbDrone.Common.Disk;
using NzbDrone.Common.Extensions;
using Mono.Unix;
namespace NzbDrone.Mono
{
public class ProcMount : IMount
{
private readonly UnixDriveInfo _unixDriveInfo;
public ProcMount(DriveType driveType, string name, string mount, string type, Dictionary<string, string> options)
{
DriveType = driveType;
Name = name;
RootDirectory = mount;
DriveFormat = type;
_unixDriveInfo = new UnixDriveInfo(mount);
}
public long AvailableFreeSpace
{
get { return _unixDriveInfo.AvailableFreeSpace; }
}
public string DriveFormat { get; private set; }
public DriveType DriveType { get; private set; }
public bool IsReady
{
get { return _unixDriveInfo.IsReady; }
}
public string Name { get; private set; }
public string RootDirectory { get; private set; }
public long TotalFreeSpace
{
get { return _unixDriveInfo.TotalFreeSpace; }
}
public long TotalSize
{
get { return _unixDriveInfo.TotalSize; }
}
public string VolumeLabel
{
get { return _unixDriveInfo.VolumeLabel; }
}
}
}

@ -0,0 +1,138 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using NLog;
using NzbDrone.Common.Disk;
using NzbDrone.Common.Extensions;
using Mono.Unix;
namespace NzbDrone.Mono
{
public interface IProcMountProvider
{
List<IMount> GetMounts();
}
public class ProcMountProvider : IProcMountProvider
{
private static string[] _fixedTypes = new [] { "ext3", "ext2", "ext4", "vfat", "fuseblk", "xfs", "jfs", "msdos", "ntfs", "minix", "hfs", "hfsplus", "qnx4", "ufs", "btrfs" };
private static string[] _networkDriveTypes = new [] { "cifs", "nfs", "nfs4", "nfsd", "sshfs" };
private static Dictionary<string, bool> _fileSystems;
private readonly Logger _logger;
public ProcMountProvider(Logger logger)
{
_logger = logger;
}
public List<IMount> GetMounts()
{
try
{
if (File.Exists(@"/proc/mounts"))
{
var lines = File.ReadAllLines(@"/proc/mounts");
return lines.Select(ParseLine).OfType<IMount>().ToList();
}
}
catch (Exception ex)
{
_logger.DebugException("Failed to retrieve mounts from /proc/mounts", ex);
}
return new List<IMount>();
}
private Dictionary<string, bool> GetFileSystems()
{
if (_fileSystems == null)
{
var result = new Dictionary<string, bool>();
try
{
if (File.Exists(@"/proc/filesystems"))
{
var lines = File.ReadAllLines(@"/proc/filesystems");
foreach (var line in lines)
{
var split = line.Split('\t');
result.Add(split[1], split[0] != "nodev");
}
}
}
catch (Exception ex)
{
_logger.DebugException("Failed to get filesystem types from /proc/filesystems, using default set.", ex);
}
if (result.Empty())
{
foreach (var type in _fixedTypes)
{
result.Add(type, true);
}
}
_fileSystems = result;
}
return _fileSystems;
}
private IMount ParseLine(string line)
{
var split = line.Split(' ');
if (split.Length != 6)
{
_logger.Debug("Unable to parser /proc/mount line: {0}", line);
}
var name = split[0];
var mount = split[1];
var type = split[2];
var options = ParseOptions(split[3]);
var driveType = DriveType.Unknown;
if (name.StartsWith("/dev/") || GetFileSystems().GetValueOrDefault(type, false))
{
// Not always fixed, but lets assume it.
driveType = DriveType.Fixed;
}
if (_networkDriveTypes.Contains(type))
{
driveType = DriveType.Network;
}
if (type == "zfs")
{
driveType = DriveType.Fixed;
}
return new ProcMount(driveType, name, mount, type, options);
}
private Dictionary<string, string> ParseOptions(string options)
{
var result = new Dictionary<string, string>();
foreach (var option in options.Split(','))
{
var split = option.Split(new[] { '=' }, 2);
result.Add(split[0], split.Length == 2 ? split[1] : string.Empty);
}
return result;
}
}
}
Loading…
Cancel
Save