Merge pull request #2778 from nextcloud/windows_cfapi_backend_for_vfs

Windows cfapi backend for vfs

src/gui/socketapi.cpp

@@ -1178,8 +1178,11 @@ DirectEditor* SocketApi::getDirectEditorForLocalFile(const QString &localFile)
     auto capabilities = fileData.folder->accountState()->account()->capabilities();
 
     if (fileData.folder && fileData.folder->accountState()->isConnected()) {
+        const auto record = fileData.journalRecord();
+        const auto mimeMatchMode = record.isVirtualFile() ? QMimeDatabase::MatchExtension : QMimeDatabase::MatchDefault;
+
         QMimeDatabase db;
-        QMimeType type = db.mimeTypeForFile(localFile);
+        QMimeType type = db.mimeTypeForFile(localFile, mimeMatchMode);
 
         DirectEditor* editor = capabilities.getDirectEditorForMimetype(type);
         if (!editor) {

src/libsync/CMakeLists.txt

@@ -61,6 +61,16 @@ set(libsync_SRCS
     vfs/suffix/vfs_suffix.cpp
 )
 
+if (WIN32)
+    set(libsync_SRCS ${libsync_SRCS}
+        vfs/cfapi/cfapiwrapper.cpp
+        vfs/cfapi/hydrationjob.cpp
+        vfs/cfapi/vfs_cfapi.cpp
+    )
+    add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_WIN10)
+    list(APPEND OS_SPECIFIC_LINK_LIBRARIES cldapi)
+endif()
+
 if(TOKEN_AUTH_ONLY)
     set (libsync_SRCS ${libsync_SRCS} creds/tokencredentials.cpp)
 else()

src/libsync/discovery.cpp

@@ -860,8 +860,8 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo(
     auto postProcessLocalNew = [item, localEntry, this]() {
         if (localEntry.isVirtualFile) {
             // Remove the spurious file if it looks like a placeholder file
-            // (we know placeholder files contain " ")
-            if (localEntry.size <= 1) {
+            // (we know placeholder files contain " ", but only in the suffix case)
+            if (localEntry.size <= 1 || !isVfsWithSuffix()) {
                 qCWarning(lcDisco) << "Wiping virtual file without db entry for" << _currentFolder._local + "/" + localEntry.name;
                 item->_instruction = CSYNC_INSTRUCTION_REMOVE;
                 item->_direction = SyncFileItem::Down;

src/libsync/theme.cpp

@@ -738,7 +738,8 @@ QPixmap Theme::createColorAwarePixmap(const QString &name)
 
 bool Theme::showVirtualFilesOption() const
 {
-    return ConfigFile().showExperimentalOptions();
+    const auto vfsMode = bestAvailableVfsMode();
+    return ConfigFile().showExperimentalOptions() || vfsMode == Vfs::WindowsCfApi;
 }
 
 } // end namespace client

src/libsync/vfs/cfapi/cfapiwrapper.cpp

@@ -0,0 +1,501 @@
+/*
+ * Copyright (C) by Kevin Ottens <kevin.ottens@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include "cfapiwrapper.h"
+
+#include "common/utility.h"
+#include "hydrationjob.h"
+#include "vfs_cfapi.h"
+
+#include <QDir>
+#include <QFileInfo>
+#include <QLocalSocket>
+#include <QLoggingCategory>
+
+#include <cfapi.h>
+#include <comdef.h>
+#include <ntstatus.h>
+
+Q_LOGGING_CATEGORY(lcCfApiWrapper, "nextcloud.sync.vfs.cfapi.wrapper", QtInfoMsg)
+
+#define FIELD_SIZE( type, field ) ( sizeof( ( (type*)0 )->field ) )
+#define CF_SIZE_OF_OP_PARAM( field )                                           \
+    ( FIELD_OFFSET( CF_OPERATION_PARAMETERS, field ) +                         \
+      FIELD_SIZE( CF_OPERATION_PARAMETERS, field ) )
+
+namespace {
+void cfApiSendTransferInfo(const CF_CONNECTION_KEY &connectionKey, const CF_TRANSFER_KEY &transferKey, NTSTATUS status, void *buffer, qint64 offset, qint64 length)
+{
+
+    CF_OPERATION_INFO opInfo = { 0 };
+    CF_OPERATION_PARAMETERS opParams = { 0 };
+
+    opInfo.StructSize = sizeof(opInfo);
+    opInfo.Type = CF_OPERATION_TYPE_TRANSFER_DATA;
+    opInfo.ConnectionKey = connectionKey;
+    opInfo.TransferKey = transferKey;
+    opParams.ParamSize = CF_SIZE_OF_OP_PARAM(TransferData);
+    opParams.TransferData.CompletionStatus = status;
+    opParams.TransferData.Buffer = buffer;
+    opParams.TransferData.Offset.QuadPart = offset;
+    opParams.TransferData.Length.QuadPart = length;
+
+    const qint64 result = CfExecute(&opInfo, &opParams);
+    Q_ASSERT(result == S_OK);
+    if (result != S_OK) {
+        qCCritical(lcCfApiWrapper) << "Couldn't send transfer info" << QString::number(transferKey.QuadPart, 16) << ":" << _com_error(result).ErrorMessage();
+    }
+}
+
+
+void CALLBACK cfApiFetchDataCallback(const CF_CALLBACK_INFO *callbackInfo, const CF_CALLBACK_PARAMETERS *callbackParameters)
+{
+    const auto sendTransferError = [=] {
+        cfApiSendTransferInfo(callbackInfo->ConnectionKey,
+                              callbackInfo->TransferKey,
+                              STATUS_UNSUCCESSFUL,
+                              nullptr,
+                              callbackParameters->FetchData.RequiredFileOffset.QuadPart,
+                              callbackParameters->FetchData.RequiredLength.QuadPart);
+    };
+
+    const auto sendTransferInfo = [=](QByteArray &data, qint64 offset) {
+        cfApiSendTransferInfo(callbackInfo->ConnectionKey,
+                              callbackInfo->TransferKey,
+                              STATUS_SUCCESS,
+                              data.data(),
+                              offset,
+                              data.length());
+    };
+
+    auto vfs = reinterpret_cast<OCC::VfsCfApi *>(callbackInfo->CallbackContext);
+    Q_ASSERT(vfs->metaObject()->className() == QByteArrayLiteral("OCC::VfsCfApi"));
+    const auto path = QString(QString::fromWCharArray(callbackInfo->VolumeDosName) + QString::fromWCharArray(callbackInfo->NormalizedPath));
+    const auto requestId = QString::number(callbackInfo->TransferKey.QuadPart, 16);
+
+    const auto invokeResult = QMetaObject::invokeMethod(vfs, [=] { vfs->requestHydration(requestId, path); }, Qt::QueuedConnection);
+    if (!invokeResult) {
+        qCCritical(lcCfApiWrapper) << "Failed to trigger hydration for" << path << requestId;
+        sendTransferError();
+        return;
+    }
+
+    // Block and wait for vfs to signal back the hydration is ready
+    bool hydrationRequestResult = false;
+    QEventLoop loop;
+    QObject::connect(vfs, &OCC::VfsCfApi::hydrationRequestReady, &loop, [&](const QString &id) {
+        if (requestId == id) {
+            hydrationRequestResult = true;
+            loop.quit();
+        }
+    });
+    QObject::connect(vfs, &OCC::VfsCfApi::hydrationRequestFailed, &loop, [&](const QString &id) {
+        if (requestId == id) {
+            hydrationRequestResult = false;
+            loop.quit();
+        }
+    });
+    loop.exec();
+    QObject::disconnect(vfs, nullptr, &loop, nullptr);
+    qCInfo(lcCfApiWrapper) << "VFS replied for hydration of" << path << requestId << "status was:" << hydrationRequestResult;
+
+    if (!hydrationRequestResult) {
+        sendTransferError();
+        return;
+    }
+
+    QLocalSocket socket;
+    socket.connectToServer(requestId);
+    const auto connectResult = socket.waitForConnected();
+    if (!connectResult) {
+        qCWarning(lcCfApiWrapper) << "Couldn't connect the socket" << requestId << socket.error() << socket.errorString();
+        sendTransferError();
+        return;
+    }
+
+    qint64 offset = 0;
+
+    QObject::connect(&socket, &QLocalSocket::readyRead, &loop, [&] {
+        auto data = socket.readAll();
+        if (data.isEmpty()) {
+            qCWarning(lcCfApiWrapper) << "Unexpected empty data received" << requestId;
+            sendTransferError();
+            loop.quit();
+            return;
+        }
+        sendTransferInfo(data, offset);
+        offset += data.length();
+    });
+
+    QObject::connect(vfs, &OCC::VfsCfApi::hydrationRequestFinished, &loop, [&](const QString &id, int s) {
+        if (requestId == id) {
+            const auto status = static_cast<OCC::HydrationJob::Status>(s);
+            qCInfo(lcCfApiWrapper) << "Hydration done for" << path << requestId << status;
+            if (status != OCC::HydrationJob::Success) {
+                sendTransferError();
+            }
+            loop.quit();
+        }
+    });
+
+    loop.exec();
+}
+
+CF_CALLBACK_REGISTRATION cfApiCallbacks[] = {
+    { CF_CALLBACK_TYPE_FETCH_DATA, cfApiFetchDataCallback },
+    CF_CALLBACK_REGISTRATION_END
+};
+
+DWORD sizeToDWORD(size_t size)
+{
+    return OCC::Utility::convertSizeToDWORD(size);
+}
+
+void deletePlaceholderInfo(CF_PLACEHOLDER_BASIC_INFO *info)
+{
+    auto byte = reinterpret_cast<char *>(info);
+    delete[] byte;
+}
+
+std::wstring pathForHandle(const OCC::CfApiWrapper::FileHandle &handle)
+{
+    wchar_t buffer[MAX_PATH];
+    const qint64 result = GetFinalPathNameByHandle(handle.get(), buffer, MAX_PATH, VOLUME_NAME_DOS);
+    Q_ASSERT(result < MAX_PATH);
+    return std::wstring(buffer);
+}
+
+OCC::PinState cfPinStateToPinState(CF_PIN_STATE state)
+{
+    switch (state) {
+    case CF_PIN_STATE_UNSPECIFIED:
+        return OCC::PinState::Unspecified;
+    case CF_PIN_STATE_PINNED:
+        return OCC::PinState::AlwaysLocal;
+    case CF_PIN_STATE_UNPINNED:
+        return OCC::PinState::OnlineOnly;
+    case CF_PIN_STATE_INHERIT:
+        return OCC::PinState::Inherited;
+    default:
+        Q_UNREACHABLE();
+        return OCC::PinState::Inherited;
+    }
+}
+
+CF_PIN_STATE pinStateToCfPinState(OCC::PinState state)
+{
+    switch (state) {
+    case OCC::PinState::Inherited:
+        return CF_PIN_STATE_INHERIT;
+    case OCC::PinState::AlwaysLocal:
+        return CF_PIN_STATE_PINNED;
+    case OCC::PinState::OnlineOnly:
+        return CF_PIN_STATE_UNPINNED;
+    case OCC::PinState::Unspecified:
+        return CF_PIN_STATE_UNSPECIFIED;
+    default:
+        Q_UNREACHABLE();
+        return CF_PIN_STATE_UNSPECIFIED;
+    }
+}
+
+CF_SET_PIN_FLAGS pinRecurseModeToCfSetPinFlags(OCC::CfApiWrapper::SetPinRecurseMode mode)
+{
+    switch (mode) {
+    case OCC::CfApiWrapper::NoRecurse:
+        return CF_SET_PIN_FLAG_NONE;
+    case OCC::CfApiWrapper::Recurse:
+        return CF_SET_PIN_FLAG_RECURSE;
+    case OCC::CfApiWrapper::ChildrenOnly:
+        return CF_SET_PIN_FLAG_RECURSE_ONLY;
+    default:
+        Q_UNREACHABLE();
+        return CF_SET_PIN_FLAG_NONE;
+    }
+}
+
+}
+
+OCC::CfApiWrapper::ConnectionKey::ConnectionKey()
+    : _data(new CF_CONNECTION_KEY, [](void *p) { delete reinterpret_cast<CF_CONNECTION_KEY *>(p); })
+{
+}
+
+OCC::CfApiWrapper::FileHandle::FileHandle()
+    : _data(nullptr, [](void *) {})
+{
+}
+
+OCC::CfApiWrapper::FileHandle::FileHandle(void *data, Deleter deleter)
+    : _data(data, deleter)
+{
+}
+
+OCC::CfApiWrapper::PlaceHolderInfo::PlaceHolderInfo()
+    : _data(nullptr, [](CF_PLACEHOLDER_BASIC_INFO *) {})
+{
+}
+
+OCC::CfApiWrapper::PlaceHolderInfo::PlaceHolderInfo(CF_PLACEHOLDER_BASIC_INFO *data, Deleter deleter)
+    : _data(data, deleter)
+{
+}
+
+OCC::Optional<OCC::PinStateEnums::PinState> OCC::CfApiWrapper::PlaceHolderInfo::pinState() const
+{
+    Q_ASSERT(_data);
+    if (!_data) {
+        return {};
+    }
+
+    return cfPinStateToPinState(_data->PinState);
+}
+
+OCC::Result<void, QString> OCC::CfApiWrapper::registerSyncRoot(const QString &path, const QString &providerName, const QString &providerVersion)
+{
+    const auto p = path.toStdWString();
+    const auto name = providerName.toStdWString();
+    const auto version = providerVersion.toStdWString();
+
+    CF_SYNC_REGISTRATION info;
+    info.ProviderName = name.data();
+    info.ProviderVersion = version.data();
+    info.SyncRootIdentity = nullptr;
+    info.SyncRootIdentityLength = 0;
+    info.FileIdentity = nullptr;
+    info.FileIdentityLength = 0;
+
+    CF_SYNC_POLICIES policies;
+    policies.Hydration.Primary = CF_HYDRATION_POLICY_FULL;
+    policies.Hydration.Modifier = CF_HYDRATION_POLICY_MODIFIER_NONE;
+    policies.Population.Primary = CF_POPULATION_POLICY_ALWAYS_FULL;
+    policies.Population.Modifier = CF_POPULATION_POLICY_MODIFIER_NONE;
+    policies.InSync = CF_INSYNC_POLICY_PRESERVE_INSYNC_FOR_SYNC_ENGINE;
+    policies.HardLink = CF_HARDLINK_POLICY_NONE;
+
+    const qint64 result = CfRegisterSyncRoot(p.data(), &info, &policies, CF_REGISTER_FLAG_UPDATE);
+    Q_ASSERT(result == S_OK);
+    if (result != S_OK) {
+        return QString::fromWCharArray(_com_error(result).ErrorMessage());
+    } else {
+        return {};
+    }
+}
+
+OCC::Result<void, QString> OCC::CfApiWrapper::unegisterSyncRoot(const QString &path)
+{
+    const auto p = path.toStdWString();
+    const qint64 result = CfUnregisterSyncRoot(p.data());
+    Q_ASSERT(result == S_OK);
+    if (result != S_OK) {
+        return QString::fromWCharArray(_com_error(result).ErrorMessage());
+    } else {
+        return {};
+    }
+}
+
+OCC::Result<OCC::CfApiWrapper::ConnectionKey, QString> OCC::CfApiWrapper::connectSyncRoot(const QString &path, OCC::VfsCfApi *context)
+{
+    auto key = ConnectionKey();
+    const auto p = path.toStdWString();
+    const qint64 result = CfConnectSyncRoot(p.data(),
+                                            cfApiCallbacks,
+                                            context,
+                                            CF_CONNECT_FLAG_REQUIRE_PROCESS_INFO | CF_CONNECT_FLAG_REQUIRE_FULL_FILE_PATH,
+                                            static_cast<CF_CONNECTION_KEY *>(key.get()));
+    Q_ASSERT(result == S_OK);
+    if (result != S_OK) {
+        return QString::fromWCharArray(_com_error(result).ErrorMessage());
+    } else {
+        return key;
+    }
+}
+
+OCC::Result<void, QString> OCC::CfApiWrapper::disconnectSyncRoot(ConnectionKey &&key)
+{
+    const qint64 result = CfDisconnectSyncRoot(*static_cast<CF_CONNECTION_KEY *>(key.get()));
+    Q_ASSERT(result == S_OK);
+    if (result != S_OK) {
+        return QString::fromWCharArray(_com_error(result).ErrorMessage());
+    } else {
+        return {};
+    }
+}
+
+bool OCC::CfApiWrapper::isSparseFile(const QString &path)
+{
+    const auto p = path.toStdWString();
+    const auto attributes = GetFileAttributes(p.data());
+    return (attributes & FILE_ATTRIBUTE_SPARSE_FILE) != 0;
+}
+
+OCC::CfApiWrapper::FileHandle OCC::CfApiWrapper::handleForPath(const QString &path)
+{
+    if (QFileInfo(path).isDir()) {
+        HANDLE handle = nullptr;
+        const qint64 openResult = CfOpenFileWithOplock(path.toStdWString().data(), CF_OPEN_FILE_FLAG_NONE, &handle);
+        if (openResult == S_OK) {
+            return {handle, CfCloseHandle};
+        }
+    } else {
+        const auto handle = CreateFile(path.toStdWString().data(), 0, 0, nullptr,
+                                       OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
+        if (handle != INVALID_HANDLE_VALUE) {
+            return {handle, [](HANDLE h) { CloseHandle(h); }};
+        }
+    }
+
+    return {};
+}
+
+OCC::CfApiWrapper::PlaceHolderInfo OCC::CfApiWrapper::findPlaceholderInfo(const FileHandle &handle)
+{
+    Q_ASSERT(handle);
+
+    constexpr auto fileIdMaxLength = 128;
+    const auto infoSize = sizeof(CF_PLACEHOLDER_BASIC_INFO) + fileIdMaxLength;
+    auto info = PlaceHolderInfo(reinterpret_cast<CF_PLACEHOLDER_BASIC_INFO *>(new char[infoSize]), deletePlaceholderInfo);
+    const qint64 result = CfGetPlaceholderInfo(handle.get(), CF_PLACEHOLDER_INFO_BASIC, info.get(), sizeToDWORD(infoSize), nullptr);
+
+    if (result == S_OK) {
+        return info;
+    } else {
+        return {};
+    }
+}
+
+OCC::Result<void, QString> OCC::CfApiWrapper::setPinState(const FileHandle &handle, PinState state, SetPinRecurseMode mode)
+{
+    const auto cfState = pinStateToCfPinState(state);
+    const auto flags = pinRecurseModeToCfSetPinFlags(mode);
+
+    const qint64 result = CfSetPinState(handle.get(), cfState, flags, nullptr);
+    if (result == S_OK) {
+        return {};
+    } else {
+        qCWarning(lcCfApiWrapper) << "Couldn't set pin state" << state << "for" << pathForHandle(handle) << "with recurse mode" << mode << ":" << _com_error(result).ErrorMessage();
+        return "Couldn't set pin state";
+    }
+}
+
+OCC::Result<void, QString> OCC::CfApiWrapper::createPlaceholderInfo(const QString &path, time_t modtime, qint64 size, const QByteArray &fileId)
+{
+    const auto fileInfo = QFileInfo(path);
+    const auto localBasePath = QDir::toNativeSeparators(fileInfo.path()).toStdWString();
+    const auto relativePath = fileInfo.fileName().toStdWString();
+
+    const auto fileIdentity = QString::fromUtf8(fileId).toStdWString();
+
+    CF_PLACEHOLDER_CREATE_INFO cloudEntry;
+    cloudEntry.FileIdentity = fileIdentity.data();
+    const auto fileIdentitySize = (fileIdentity.length() + 1) * sizeof(wchar_t);
+    cloudEntry.FileIdentityLength = sizeToDWORD(fileIdentitySize);
+
+    cloudEntry.RelativeFileName = relativePath.data();
+    cloudEntry.Flags = CF_PLACEHOLDER_CREATE_FLAG_MARK_IN_SYNC;
+    cloudEntry.FsMetadata.FileSize.QuadPart = size;
+    cloudEntry.FsMetadata.BasicInfo.FileAttributes = FILE_ATTRIBUTE_NORMAL;
+    OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &cloudEntry.FsMetadata.BasicInfo.CreationTime);
+    OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &cloudEntry.FsMetadata.BasicInfo.LastWriteTime);
+    OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &cloudEntry.FsMetadata.BasicInfo.LastAccessTime);
+    OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &cloudEntry.FsMetadata.BasicInfo.ChangeTime);
+
+    if (fileInfo.isDir()) {
+        cloudEntry.Flags |= CF_PLACEHOLDER_CREATE_FLAG_DISABLE_ON_DEMAND_POPULATION;
+        cloudEntry.FsMetadata.BasicInfo.FileAttributes = FILE_ATTRIBUTE_DIRECTORY;
+        cloudEntry.FsMetadata.FileSize.QuadPart = 0;
+    }
+
+    const qint64 result = CfCreatePlaceholders(localBasePath.data(), &cloudEntry, 1, CF_CREATE_FLAG_NONE, nullptr);
+    if (result != S_OK) {
+        qCWarning(lcCfApiWrapper) << "Couldn't create placeholder info for" << path << ":" << _com_error(result).ErrorMessage();
+        return "Couldn't create placeholder info";
+    }
+
+    const auto parentHandle = handleForPath(QDir::toNativeSeparators(QFileInfo(path).absolutePath()));
+    const auto parentInfo = findPlaceholderInfo(parentHandle);
+    const auto state = parentInfo && parentInfo->PinState == CF_PIN_STATE_UNPINNED ? CF_PIN_STATE_UNPINNED : CF_PIN_STATE_INHERIT;
+
+    const auto handle = handleForPath(path);
+    if (!setPinState(handle, cfPinStateToPinState(state), NoRecurse)) {
+        return "Couldn't set the default inherit pin state";
+    }
+
+    return {};
+}
+
+OCC::Result<void, QString> OCC::CfApiWrapper::updatePlaceholderInfo(const FileHandle &handle, time_t modtime, qint64 size, const QByteArray &fileId, const QString &replacesPath)
+{
+    Q_ASSERT(handle);
+
+    const auto info = replacesPath.isEmpty() ? findPlaceholderInfo(handle)
+                                             : findPlaceholderInfo(handleForPath(replacesPath));
+    if (!info) {
+        return "Can't update non existing placeholder info";
+    }
+
+    const auto previousPinState = cfPinStateToPinState(info->PinState);
+    const auto fileIdentity = QString::fromUtf8(fileId).toStdWString();
+    const auto fileIdentitySize = (fileIdentity.length() + 1) * sizeof(wchar_t);
+
+    CF_FS_METADATA metadata;
+    metadata.FileSize.QuadPart = size;
+    OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &metadata.BasicInfo.CreationTime);
+    OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &metadata.BasicInfo.LastWriteTime);
+    OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &metadata.BasicInfo.LastAccessTime);
+    OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &metadata.BasicInfo.ChangeTime);
+
+    const qint64 result = CfUpdatePlaceholder(handle.get(), &metadata,
+                                              fileIdentity.data(), sizeToDWORD(fileIdentitySize),
+                                              nullptr, 0, CF_UPDATE_FLAG_NONE, nullptr, nullptr);
+
+    if (result != S_OK) {
+        qCWarning(lcCfApiWrapper) << "Couldn't update placeholder info for" << pathForHandle(handle) << ":" << _com_error(result).ErrorMessage();
+        return "Couldn't update placeholder info";
+    }
+
+    // Pin state tends to be lost on updates, so restore it every time
+    if (!setPinState(handle, previousPinState, NoRecurse)) {
+        return "Couldn't restore pin state";
+    }
+
+    return {};
+}
+
+OCC::Result<void, QString> OCC::CfApiWrapper::convertToPlaceholder(const FileHandle &handle, time_t modtime, qint64 size, const QByteArray &fileId, const QString &replacesPath)
+{
+    Q_ASSERT(handle);
+
+    const auto fileIdentity = QString::fromUtf8(fileId).toStdWString();
+    const auto fileIdentitySize = (fileIdentity.length() + 1) * sizeof(wchar_t);
+    const qint64 result = CfConvertToPlaceholder(handle.get(), fileIdentity.data(), sizeToDWORD(fileIdentitySize), CF_CONVERT_FLAG_NONE, nullptr, nullptr);
+    Q_ASSERT(result == S_OK);
+    if (result != S_OK) {
+        qCCritical(lcCfApiWrapper) << "Couldn't convert to placeholder" << pathForHandle(handle) << ":" << _com_error(result).ErrorMessage();
+        return "Couldn't convert to placeholder";
+    }
+
+    const auto originalHandle = handleForPath(replacesPath);
+    const auto originalInfo = originalHandle ? findPlaceholderInfo(originalHandle) : PlaceHolderInfo(nullptr, deletePlaceholderInfo);
+    if (!originalInfo) {
+        const auto stateResult = setPinState(handle, PinState::Inherited, NoRecurse);
+        Q_ASSERT(stateResult);
+        return stateResult;
+    } else {
+        const auto state = cfPinStateToPinState(originalInfo->PinState);
+        const auto stateResult = setPinState(handle, state, NoRecurse);
+        Q_ASSERT(stateResult);
+        return stateResult;
+    }
+}

src/libsync/vfs/cfapi/cfapiwrapper.h

@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) by Kevin Ottens <kevin.ottens@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+#pragma once
+
+#include <memory>
+
+#include "owncloudlib.h"
+#include "common/pinstate.h"
+#include "common/result.h"
+
+struct CF_PLACEHOLDER_BASIC_INFO;
+
+namespace OCC {
+
+class VfsCfApi;
+
+namespace CfApiWrapper
+{
+
+class OWNCLOUDSYNC_EXPORT ConnectionKey
+{
+public:
+    ConnectionKey();
+    inline void *get() const { return _data.get(); }
+
+private:
+    std::unique_ptr<void, void(*)(void *)> _data;
+};
+
+class OWNCLOUDSYNC_EXPORT FileHandle
+{
+public:
+    using Deleter = void (*)(void *);
+
+    FileHandle();
+    FileHandle(void *data, Deleter deleter);
+
+    inline void *get() const { return _data.get(); }
+    inline explicit operator bool() const noexcept { return static_cast<bool>(_data); }
+
+private:
+    std::unique_ptr<void, void(*)(void *)> _data;
+};
+
+class OWNCLOUDSYNC_EXPORT PlaceHolderInfo
+{
+public:
+    using Deleter = void (*)(CF_PLACEHOLDER_BASIC_INFO *);
+
+    PlaceHolderInfo();
+    PlaceHolderInfo(CF_PLACEHOLDER_BASIC_INFO *data, Deleter deleter);
+
+    inline CF_PLACEHOLDER_BASIC_INFO *get() const noexcept { return _data.get(); }
+    inline CF_PLACEHOLDER_BASIC_INFO *operator->() const noexcept { return _data.get(); }
+    inline explicit operator bool() const noexcept { return static_cast<bool>(_data); }
+
+    Optional<PinState> pinState() const;
+
+private:
+    std::unique_ptr<CF_PLACEHOLDER_BASIC_INFO, Deleter> _data;
+};
+
+OWNCLOUDSYNC_EXPORT Result<void, QString> registerSyncRoot(const QString &path, const QString &providerName, const QString &providerVersion);
+OWNCLOUDSYNC_EXPORT Result<void, QString> unegisterSyncRoot(const QString &path);
+
+OWNCLOUDSYNC_EXPORT Result<ConnectionKey, QString> connectSyncRoot(const QString &path, VfsCfApi *context);
+OWNCLOUDSYNC_EXPORT Result<void, QString> disconnectSyncRoot(ConnectionKey &&key);
+
+OWNCLOUDSYNC_EXPORT bool isSparseFile(const QString &path);
+
+OWNCLOUDSYNC_EXPORT FileHandle handleForPath(const QString &path);
+
+OWNCLOUDSYNC_EXPORT PlaceHolderInfo findPlaceholderInfo(const FileHandle &handle);
+
+enum SetPinRecurseMode {
+    NoRecurse = 0,
+    Recurse,
+    ChildrenOnly
+};
+
+OWNCLOUDSYNC_EXPORT Result<void, QString> setPinState(const FileHandle &handle, PinState state, SetPinRecurseMode mode);
+OWNCLOUDSYNC_EXPORT Result<void, QString> createPlaceholderInfo(const QString &path, time_t modtime, qint64 size, const QByteArray &fileId);
+OWNCLOUDSYNC_EXPORT Result<void, QString> updatePlaceholderInfo(const FileHandle &handle, time_t modtime, qint64 size, const QByteArray &fileId, const QString &replacesPath = QString());
+OWNCLOUDSYNC_EXPORT Result<void, QString> convertToPlaceholder(const FileHandle &handle, time_t modtime, qint64 size, const QByteArray &fileId, const QString &replacesPath);
+
+}
+
+} // namespace OCC

src/libsync/vfs/cfapi/hydrationjob.cpp

@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) by Kevin Ottens <kevin.ottens@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include "hydrationjob.h"
+
+#include "common/syncjournaldb.h"
+#include "propagatedownload.h"
+
+#include <QLocalServer>
+#include <QLocalSocket>
+
+Q_LOGGING_CATEGORY(lcHydration, "nextcloud.sync.vfs.hydrationjob", QtInfoMsg)
+
+OCC::HydrationJob::HydrationJob(QObject *parent)
+    : QObject(parent)
+{
+    connect(this, &HydrationJob::finished, this, &HydrationJob::deleteLater);
+}
+
+OCC::AccountPtr OCC::HydrationJob::account() const
+{
+    return _account;
+}
+
+void OCC::HydrationJob::setAccount(const AccountPtr &account)
+{
+    _account = account;
+}
+
+QString OCC::HydrationJob::remotePath() const
+{
+    return _remotePath;
+}
+
+void OCC::HydrationJob::setRemotePath(const QString &remotePath)
+{
+    _remotePath = remotePath;
+}
+
+QString OCC::HydrationJob::localPath() const
+{
+    return _localPath;
+}
+
+void OCC::HydrationJob::setLocalPath(const QString &localPath)
+{
+    _localPath = localPath;
+}
+
+OCC::SyncJournalDb *OCC::HydrationJob::journal() const
+{
+    return _journal;
+}
+
+void OCC::HydrationJob::setJournal(SyncJournalDb *journal)
+{
+    _journal = journal;
+}
+
+QString OCC::HydrationJob::requestId() const
+{
+    return _requestId;
+}
+
+void OCC::HydrationJob::setRequestId(const QString &requestId)
+{
+    _requestId = requestId;
+}
+
+QString OCC::HydrationJob::folderPath() const
+{
+    return _folderPath;
+}
+
+void OCC::HydrationJob::setFolderPath(const QString &folderPath)
+{
+    _folderPath = folderPath;
+}
+
+OCC::HydrationJob::Status OCC::HydrationJob::status() const
+{
+    return _status;
+}
+
+void OCC::HydrationJob::start()
+{
+    Q_ASSERT(_account);
+    Q_ASSERT(_journal);
+    Q_ASSERT(!_remotePath.isEmpty() && !_localPath.isEmpty());
+    Q_ASSERT(!_requestId.isEmpty() && !_folderPath.isEmpty());
+
+    Q_ASSERT(_remotePath.endsWith('/'));
+    Q_ASSERT(_localPath.endsWith('/'));
+    Q_ASSERT(!_folderPath.startsWith('/'));
+
+    _server = new QLocalServer(this);
+    const auto listenResult = _server->listen(_requestId);
+    if (!listenResult) {
+        qCCritical(lcHydration) << "Couldn't get server to listen" << _requestId << _localPath << _folderPath;
+        emitFinished(Error);
+        return;
+    }
+
+    qCInfo(lcHydration) << "Server ready, waiting for connections" << _requestId << _localPath << _folderPath;
+    connect(_server, &QLocalServer::newConnection, this, &HydrationJob::onNewConnection);
+}
+
+void OCC::HydrationJob::emitFinished(Status status)
+{
+    _status = status;
+    if (status == Success) {
+        _socket->disconnectFromServer();
+        connect(_socket, &QLocalSocket::disconnected, this, [=]{
+            _socket->close();
+            emit finished(this);
+        });
+    } else {
+        _socket->close();
+        emit finished(this);
+    }
+}
+
+void OCC::HydrationJob::onNewConnection()
+{
+    Q_ASSERT(!_socket);
+    Q_ASSERT(!_job);
+
+    qCInfo(lcHydration) << "Got new connection starting GETFileJob" << _requestId << _folderPath;
+    _socket = _server->nextPendingConnection();
+    _job = new GETFileJob(_account, _remotePath + _folderPath, _socket, {}, {}, 0, this);
+    connect(_job, &GETFileJob::finishedSignal, this, &HydrationJob::onGetFinished);
+    _job->start();
+}
+
+void OCC::HydrationJob::onGetFinished()
+{
+    qCInfo(lcHydration) << "GETFileJob finished" << _requestId << _folderPath << _job->reply()->error();
+
+    if (_job->reply()->error()) {
+        emitFinished(Error);
+        return;
+    }
+
+    SyncJournalFileRecord record;
+    _journal->getFileRecord(_folderPath, &record);
+    Q_ASSERT(record.isValid());
+    if (!record.isValid()) {
+        qCWarning(lcHydration) << "Couldn't find record to update after hydration" << _requestId << _folderPath;
+        emitFinished(Error);
+        return;
+    }
+
+    record._type = ItemTypeFile;
+    _journal->setFileRecord(record);
+    emitFinished(Success);
+}

src/libsync/vfs/cfapi/hydrationjob.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) by Kevin Ottens <kevin.ottens@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+#pragma once
+
+#include <QObject>
+
+#include "account.h"
+
+class QLocalServer;
+class QLocalSocket;
+
+namespace OCC {
+class GETFileJob;
+class SyncJournalDb;
+
+class OWNCLOUDSYNC_EXPORT HydrationJob : public QObject
+{
+    Q_OBJECT
+public:
+    enum Status {
+        Success = 0,
+        Error,
+    };
+    Q_ENUM(Status)
+
+    explicit HydrationJob(QObject *parent = nullptr);
+
+    AccountPtr account() const;
+    void setAccount(const AccountPtr &account);
+
+    QString remotePath() const;
+    void setRemotePath(const QString &remotePath);
+
+    QString localPath() const;
+    void setLocalPath(const QString &localPath);
+
+    SyncJournalDb *journal() const;
+    void setJournal(SyncJournalDb *journal);
+
+    QString requestId() const;
+    void setRequestId(const QString &requestId);
+
+    QString folderPath() const;
+    void setFolderPath(const QString &folderPath);
+
+    Status status() const;
+
+    void start();
+
+signals:
+    void finished(HydrationJob *job);
+
+private:
+    void emitFinished(Status status);
+
+    void onNewConnection();
+    void onGetFinished();
+
+    AccountPtr _account;
+    QString _remotePath;
+    QString _localPath;
+    SyncJournalDb *_journal = nullptr;
+
+    QString _requestId;
+    QString _folderPath;
+
+    QLocalServer *_server = nullptr;
+    QLocalSocket *_socket = nullptr;
+    GETFileJob *_job = nullptr;
+    Status _status = Success;
+};
+
+} // namespace OCC

src/libsync/vfs/cfapi/vfs_cfapi.cpp

@@ -0,0 +1,394 @@
+/*
+ * Copyright (C) by Kevin Ottens <kevin.ottens@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include "vfs_cfapi.h"
+
+#include <QDir>
+#include <QFile>
+
+#include "cfapiwrapper.h"
+#include "hydrationjob.h"
+#include "syncfileitem.h"
+#include "filesystem.h"
+#include "common/syncjournaldb.h"
+
+#include <cfapi.h>
+#include <comdef.h>
+
+Q_LOGGING_CATEGORY(lcCfApi, "nextcloud.sync.vfs.cfapi", QtInfoMsg)
+
+namespace cfapi {
+using namespace OCC::CfApiWrapper;
+}
+
+namespace OCC {
+
+class VfsCfApiPrivate
+{
+public:
+    QList<HydrationJob *> hydrationJobs;
+    cfapi::ConnectionKey connectionKey;
+};
+
+VfsCfApi::VfsCfApi(QObject *parent)
+    : Vfs(parent)
+    , d(new VfsCfApiPrivate)
+{
+}
+
+VfsCfApi::~VfsCfApi() = default;
+
+Vfs::Mode VfsCfApi::mode() const
+{
+    return WindowsCfApi;
+}
+
+QString VfsCfApi::fileSuffix() const
+{
+    return {};
+}
+
+void VfsCfApi::startImpl(const VfsSetupParams &params)
+{
+    const auto localPath = QDir::toNativeSeparators(params.filesystemPath);
+
+    const auto registerResult = cfapi::registerSyncRoot(localPath, params.providerName, params.providerVersion);
+    if (!registerResult) {
+        qCCritical(lcCfApi) << "Initialization failed, couldn't register sync root:" << registerResult.error();
+        return;
+    }
+
+    auto connectResult = cfapi::connectSyncRoot(localPath, this);
+    if (!connectResult) {
+        qCCritical(lcCfApi) << "Initialization failed, couldn't connect sync root:" << connectResult.error();
+        return;
+    }
+
+    d->connectionKey = *std::move(connectResult);
+}
+
+void VfsCfApi::stop()
+{
+    const auto result = cfapi::disconnectSyncRoot(std::move(d->connectionKey));
+    if (!result) {
+        qCCritical(lcCfApi) << "Disconnect failed for" << QDir::toNativeSeparators(params().filesystemPath) << ":" << result.error();
+    }
+}
+
+void VfsCfApi::unregisterFolder()
+{
+    const auto localPath = QDir::toNativeSeparators(params().filesystemPath);
+    const auto result = cfapi::unegisterSyncRoot(localPath);
+    if (!result) {
+        qCCritical(lcCfApi) << "Unregistration failed for" << localPath << ":" << result.error();
+    }
+}
+
+bool VfsCfApi::socketApiPinStateActionsShown() const
+{
+    return true;
+}
+
+bool VfsCfApi::isHydrating() const
+{
+    return !d->hydrationJobs.isEmpty();
+}
+
+Result<void, QString> VfsCfApi::updateMetadata(const QString &filePath, time_t modtime, qint64 size, const QByteArray &fileId)
+{
+    const auto localPath = QDir::toNativeSeparators(filePath);
+    const auto handle = cfapi::handleForPath(localPath);
+    if (handle) {
+        return cfapi::updatePlaceholderInfo(handle, modtime, size, fileId);
+    } else {
+        qCWarning(lcCfApi) << "Couldn't update metadata for non existing file" << localPath;
+        return "Couldn't update metadata";
+    }
+}
+
+Result<void, QString> VfsCfApi::createPlaceholder(const SyncFileItem &item)
+{
+    Q_ASSERT(params().filesystemPath.endsWith('/'));
+    const auto localPath = QDir::toNativeSeparators(params().filesystemPath + item._file);
+    const auto result = cfapi::createPlaceholderInfo(localPath, item._modtime, item._size, item._fileId);
+    return result;
+}
+
+Result<void, QString> VfsCfApi::dehydratePlaceholder(const SyncFileItem &item)
+{
+    const auto previousPin = pinState(item._file);
+
+    if (!QFile::remove(_setupParams.filesystemPath + item._file)) {
+        return QStringLiteral("Couldn't remove %1 to fulfill dehydration").arg(item._file);
+    }
+
+    const auto r = createPlaceholder(item);
+    if (!r) {
+        return r;
+    }
+
+    if (previousPin) {
+        if (*previousPin == PinState::AlwaysLocal) {
+            setPinState(item._file, PinState::Unspecified);
+        } else {
+            setPinState(item._file, *previousPin);
+        }
+    }
+
+    return {};
+}
+
+void VfsCfApi::convertToPlaceholder(const QString &filename, const SyncFileItem &item, const QString &replacesFile)
+{
+    const auto localPath = QDir::toNativeSeparators(filename);
+    const auto replacesPath = QDir::toNativeSeparators(replacesFile);
+
+    const auto handle = cfapi::handleForPath(localPath);
+    if (cfapi::findPlaceholderInfo(handle)) {
+        cfapi::updatePlaceholderInfo(handle, item._modtime, item._size, item._fileId, replacesPath);
+    } else {
+        cfapi::convertToPlaceholder(handle, item._modtime, item._size, item._fileId, replacesPath);
+    }
+}
+
+bool VfsCfApi::needsMetadataUpdate(const SyncFileItem &item)
+{
+    return false;
+}
+
+bool VfsCfApi::isDehydratedPlaceholder(const QString &filePath)
+{
+    const auto path = QDir::toNativeSeparators(filePath);
+    return cfapi::isSparseFile(path);
+}
+
+bool VfsCfApi::statTypeVirtualFile(csync_file_stat_t *stat, void *statData)
+{
+    const auto ffd = static_cast<WIN32_FIND_DATA *>(statData);
+
+    const auto isDirectory = (ffd->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
+    const auto isSparseFile = (ffd->dwFileAttributes & FILE_ATTRIBUTE_SPARSE_FILE) != 0;
+    const auto isPinned = (ffd->dwFileAttributes & FILE_ATTRIBUTE_PINNED) != 0;
+    const auto isUnpinned = (ffd->dwFileAttributes & FILE_ATTRIBUTE_UNPINNED) != 0;
+    const auto hasReparsePoint = (ffd->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0;
+    const auto hasCloudTag = (ffd->dwReserved0 & IO_REPARSE_TAG_CLOUD) != 0;
+
+    // It's a dir with a reparse point due to the placeholder info (hence the cloud tag)
+    // if we don't remove the reparse point flag the discovery will end up thinking
+    // it is a file... let's prevent it
+    if (isDirectory && hasReparsePoint && hasCloudTag) {
+        ffd->dwFileAttributes &= ~FILE_ATTRIBUTE_REPARSE_POINT;
+        return false;
+    } else if (isSparseFile && isPinned) {
+        stat->type = ItemTypeVirtualFileDownload;
+        return true;
+    } else if (!isSparseFile && isUnpinned){
+        stat->type = ItemTypeVirtualFileDehydration;
+        return true;
+    } else if (isSparseFile) {
+        stat->type = ItemTypeVirtualFile;
+        return true;
+    }
+
+    return false;
+}
+
+bool VfsCfApi::setPinState(const QString &folderPath, PinState state)
+{
+    const auto localPath = QDir::toNativeSeparators(params().filesystemPath + folderPath);
+    const auto handle = cfapi::handleForPath(localPath);
+    if (handle) {
+        if (cfapi::setPinState(handle, state, cfapi::Recurse)) {
+            return true;
+        } else {
+            return false;
+        }
+    } else {
+        qCWarning(lcCfApi) << "Couldn't update pin state for non existing file" << localPath;
+        return false;
+    }
+}
+
+Optional<PinState> VfsCfApi::pinState(const QString &folderPath)
+{
+    const auto localPath = QDir::toNativeSeparators(params().filesystemPath + folderPath);
+    const auto handle = cfapi::handleForPath(localPath);
+    if (!handle) {
+        qCWarning(lcCfApi) << "Couldn't find pin state for non existing file" << localPath;
+        return {};
+    }
+
+    const auto info = cfapi::findPlaceholderInfo(handle);
+    if (!info) {
+        qCWarning(lcCfApi) << "Couldn't find pin state for regular non-placeholder file" << localPath;
+        return {};
+    }
+
+    return info.pinState();
+}
+
+Vfs::AvailabilityResult VfsCfApi::availability(const QString &folderPath)
+{
+    const auto basePinState = pinState(folderPath);
+    const auto hydrationAndPinStates = computeRecursiveHydrationAndPinStates(folderPath, basePinState);
+
+    const auto pin = hydrationAndPinStates.pinState;
+    const auto hydrationStatus = hydrationAndPinStates.hydrationStatus;
+
+    if (hydrationStatus.hasDehydrated) {
+        if (hydrationStatus.hasHydrated)
+            return VfsItemAvailability::Mixed;
+        if (pin && *pin == PinState::OnlineOnly)
+            return VfsItemAvailability::OnlineOnly;
+        else
+            return VfsItemAvailability::AllDehydrated;
+    } else if (hydrationStatus.hasHydrated) {
+        if (pin && *pin == PinState::AlwaysLocal)
+            return VfsItemAvailability::AlwaysLocal;
+        else
+            return VfsItemAvailability::AllHydrated;
+    }
+    return AvailabilityError::NoSuchItem;
+}
+
+void VfsCfApi::requestHydration(const QString &requestId, const QString &path)
+{
+    qCInfo(lcCfApi) << "Received request to hydrate" << path << requestId;
+    const auto root = QDir::toNativeSeparators(params().filesystemPath);
+    Q_ASSERT(path.startsWith(root));
+
+    const auto relativePath = QDir::fromNativeSeparators(path.mid(root.length()));
+    const auto journal = params().journal;
+
+    // Set in the database that we should download the file
+    SyncJournalFileRecord record;
+    journal->getFileRecord(relativePath, &record);
+    if (!record.isValid()) {
+        qCInfo(lcCfApi) << "Couldn't hydrate, did not find file in db";
+        emit hydrationRequestFailed(requestId);
+        return;
+    }
+
+    if (!record.isVirtualFile()) {
+        qCInfo(lcCfApi) << "Couldn't hydrate, the file is not virtual";
+        emit hydrationRequestFailed(requestId);
+        return;
+    }
+
+    // This is impossible to handle with CfAPI since the file size is generally different
+    // between the encrypted and the decrypted file which would make CfAPI reject the hydration
+    // of the placeholder with decrypted data
+    if (record._isE2eEncrypted || !record._e2eMangledName.isEmpty()) {
+        qCInfo(lcCfApi) << "Couldn't hydrate, the file is E2EE this is not supported";
+        emit hydrationRequestFailed(requestId);
+        return;
+    }
+
+    // All good, let's hydrate now
+    scheduleHydrationJob(requestId, relativePath);
+}
+
+void VfsCfApi::fileStatusChanged(const QString &systemFileName, SyncFileStatus fileStatus)
+{
+    Q_UNUSED(systemFileName);
+    Q_UNUSED(fileStatus);
+}
+
+void VfsCfApi::scheduleHydrationJob(const QString &requestId, const QString &folderPath)
+{
+    Q_ASSERT(std::none_of(std::cbegin(d->hydrationJobs), std::cend(d->hydrationJobs), [=](HydrationJob *job) {
+        return job->requestId() == requestId || job->folderPath() == folderPath;
+    }));
+
+    if (d->hydrationJobs.isEmpty()) {
+        emit beginHydrating();
+    }
+
+    auto job = new HydrationJob(this);
+    job->setAccount(params().account);
+    job->setRemotePath(params().remotePath);
+    job->setLocalPath(params().filesystemPath);
+    job->setJournal(params().journal);
+    job->setRequestId(requestId);
+    job->setFolderPath(folderPath);
+    connect(job, &HydrationJob::finished, this, &VfsCfApi::onHydrationJobFinished);
+    d->hydrationJobs << job;
+    job->start();
+    emit hydrationRequestReady(requestId);
+}
+
+void VfsCfApi::onHydrationJobFinished(HydrationJob *job)
+{
+    Q_ASSERT(d->hydrationJobs.contains(job));
+    qCInfo(lcCfApi) << "Hydration job finished" << job->requestId() << job->folderPath() << job->status();
+    emit hydrationRequestFinished(job->requestId(), job->status());
+    d->hydrationJobs.removeAll(job);
+    if (d->hydrationJobs.isEmpty()) {
+        emit doneHydrating();
+    }
+}
+
+VfsCfApi::HydratationAndPinStates VfsCfApi::computeRecursiveHydrationAndPinStates(const QString &folderPath, const Optional<PinState> &basePinState)
+{
+    Q_ASSERT(!folderPath.endsWith('/'));
+    QFileInfo info(params().filesystemPath + folderPath);
+
+    if (!info.exists()) {
+        return {};
+    }
+
+    const auto effectivePin = pinState(folderPath);
+    const auto pinResult = (!effectivePin && !basePinState) ? Optional<PinState>()
+                         : (!effectivePin || !basePinState) ? PinState::Inherited
+                         : (*effectivePin == *basePinState) ? *effectivePin
+                         : PinState::Inherited;
+
+    if (info.isDir()) {
+        const auto dirState = HydratationAndPinStates {
+            pinResult,
+            {}
+        };
+        const auto dir = QDir(info.absoluteFilePath());
+        Q_ASSERT(dir.exists());
+        const auto children = dir.entryList();
+        return std::accumulate(std::cbegin(children), std::cend(children), dirState, [=](const HydratationAndPinStates &currentState, const QString &name) {
+            if (name == QStringLiteral("..") || name == QStringLiteral(".")) {
+                return currentState;
+            }
+
+            const auto path = folderPath + '/' + name;
+            const auto states = computeRecursiveHydrationAndPinStates(path, currentState.pinState);
+            return HydratationAndPinStates {
+                states.pinState,
+                {
+                    states.hydrationStatus.hasHydrated || currentState.hydrationStatus.hasHydrated,
+                    states.hydrationStatus.hasDehydrated || currentState.hydrationStatus.hasDehydrated,
+                }
+            };
+        });
+    } else { // file case
+        const auto isDehydrated = isDehydratedPlaceholder(info.absoluteFilePath());
+        return {
+            pinResult,
+            {
+                !isDehydrated,
+                isDehydrated
+            }
+        };
+    }
+}
+
+} // namespace OCC
+
+OCC_DEFINE_VFS_FACTORY("win", OCC::VfsCfApi)

src/libsync/vfs/cfapi/vfs_cfapi.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) by Kevin Ottens <kevin.ottens@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+#pragma once
+
+#include <QObject>
+#include <QScopedPointer>
+
+#include "common/vfs.h"
+
+namespace OCC {
+class HydrationJob;
+class VfsCfApiPrivate;
+
+class VfsCfApi : public Vfs
+{
+    Q_OBJECT
+
+public:
+    explicit VfsCfApi(QObject *parent = nullptr);
+    ~VfsCfApi();
+
+    Mode mode() const override;
+    QString fileSuffix() const override;
+
+    void stop() override;
+    void unregisterFolder() override;
+
+    bool socketApiPinStateActionsShown() const override;
+    bool isHydrating() const override;
+
+    Result<void, QString> updateMetadata(const QString &filePath, time_t modtime, qint64 size, const QByteArray &fileId) override;
+
+    Result<void, QString> createPlaceholder(const SyncFileItem &item) override;
+    Result<void, QString> dehydratePlaceholder(const SyncFileItem &item) override;
+    void convertToPlaceholder(const QString &filename, const SyncFileItem &item, const QString &replacesFile) override;
+
+    bool needsMetadataUpdate(const SyncFileItem &) override;
+    bool isDehydratedPlaceholder(const QString &filePath) override;
+    bool statTypeVirtualFile(csync_file_stat_t *stat, void *statData) override;
+
+    bool setPinState(const QString &folderPath, PinState state) override;
+    Optional<PinState> pinState(const QString &folderPath) override;
+    AvailabilityResult availability(const QString &folderPath) override;
+
+public slots:
+    void requestHydration(const QString &requestId, const QString &path);
+    void fileStatusChanged(const QString &systemFileName, SyncFileStatus fileStatus) override;
+
+signals:
+    void hydrationRequestReady(const QString &requestId);
+    void hydrationRequestFailed(const QString &requestId);
+    void hydrationRequestFinished(const QString &requestId, int status);
+
+protected:
+    void startImpl(const VfsSetupParams &params) override;
+
+private:
+    void scheduleHydrationJob(const QString &requestId, const QString &folderPath);
+    void onHydrationJobFinished(HydrationJob *job);
+
+    struct HasHydratedDehydrated {
+        bool hasHydrated = false;
+        bool hasDehydrated = false;
+    };
+    struct HydratationAndPinStates {
+        Optional<PinState> pinState;
+        HasHydratedDehydrated hydrationStatus;
+    };
+    HydratationAndPinStates computeRecursiveHydrationAndPinStates(const QString &path, const Optional<PinState> &basePinState);
+
+    QScopedPointer<VfsCfApiPrivate> d;
+};
+
+} // namespace OCC

test/CMakeLists.txt

@@ -73,6 +73,10 @@ if( UNIX AND NOT APPLE )
     nextcloud_add_test(InotifyWatcher "${FolderWatcher_SRC}")
 endif(UNIX AND NOT APPLE)
 
+if (WIN32)
+    nextcloud_add_test(SyncCfApi "")
+endif()
+
 nextcloud_add_benchmark(LargeSync "")
 
 SET(FolderMan_SRC ../src/gui/folderman.cpp)

test/testsynccfapi.cpp

@@ -0,0 +1,1189 @@
+/*
+ *    This software is in the public domain, furnished "as is", without technical
+ *    support, and with no warranty, express or implied, as to its usefulness for
+ *    any purpose.
+ *
+ */
+
+#include <QtTest>
+#include "syncenginetestutils.h"
+#include "common/vfs.h"
+#include "config.h"
+#include <syncengine.h>
+
+#include "vfs/cfapi/cfapiwrapper.h"
+
+namespace cfapi {
+using namespace OCC::CfApiWrapper;
+}
+
+#define CFVERIFY_VIRTUAL(folder, path) \
+    QVERIFY(QFileInfo((folder).localPath() + (path)).exists()); \
+    QVERIFY(cfapi::isSparseFile((folder).localPath() + (path))); \
+    QVERIFY(dbRecord((folder), (path)).isValid()); \
+    QCOMPARE(dbRecord((folder), (path))._type, ItemTypeVirtualFile);
+
+#define CFVERIFY_NONVIRTUAL(folder, path) \
+    QVERIFY(QFileInfo((folder).localPath() + (path)).exists()); \
+    QVERIFY(!cfapi::isSparseFile((folder).localPath() + (path))); \
+    QVERIFY(dbRecord((folder), (path)).isValid()); \
+    QCOMPARE(dbRecord((folder), (path))._type, ItemTypeFile);
+
+#define CFVERIFY_GONE(folder, path) \
+    QVERIFY(!QFileInfo((folder).localPath() + (path)).exists()); \
+    QVERIFY(!dbRecord((folder), (path)).isValid());
+
+using namespace OCC;
+
+enum ErrorKind : int {
+    NoError = 0,
+    // Lower code are corresponding to HTTP error code
+    Timeout = 1000,
+};
+
+void setPinState(const QString &path, PinState state, cfapi::SetPinRecurseMode mode)
+{
+    Q_ASSERT(mode == cfapi::Recurse || mode == cfapi::NoRecurse);
+
+    const auto p = QDir::toNativeSeparators(path);
+    const auto handle = cfapi::handleForPath(p);
+    Q_ASSERT(handle);
+
+    const auto result = cfapi::setPinState(handle, state, mode);
+    Q_ASSERT(result);
+
+    if (mode == cfapi::NoRecurse) {
+        const auto result = cfapi::setPinState(handle, PinState::Inherited, cfapi::ChildrenOnly);
+        Q_ASSERT(result);
+    }
+}
+
+bool itemInstruction(const ItemCompletedSpy &spy, const QString &path, const SyncInstructions instr)
+{
+    auto item = spy.findItem(path);
+    return item->_instruction == instr;
+}
+
+SyncJournalFileRecord dbRecord(FakeFolder &folder, const QString &path)
+{
+    SyncJournalFileRecord record;
+    folder.syncJournal().getFileRecord(path, &record);
+    return record;
+}
+
+void triggerDownload(FakeFolder &folder, const QByteArray &path)
+{
+    auto &journal = folder.syncJournal();
+    SyncJournalFileRecord record;
+    journal.getFileRecord(path, &record);
+    if (!record.isValid())
+        return;
+    record._type = ItemTypeVirtualFileDownload;
+    journal.setFileRecord(record);
+    journal.schedulePathForRemoteDiscovery(record._path);
+}
+
+void markForDehydration(FakeFolder &folder, const QByteArray &path)
+{
+    auto &journal = folder.syncJournal();
+    SyncJournalFileRecord record;
+    journal.getFileRecord(path, &record);
+    if (!record.isValid())
+        return;
+    record._type = ItemTypeVirtualFileDehydration;
+    journal.setFileRecord(record);
+    journal.schedulePathForRemoteDiscovery(record._path);
+}
+
+QSharedPointer<Vfs> setupVfs(FakeFolder &folder)
+{
+    auto cfapiVfs = QSharedPointer<Vfs>(createVfsFromPlugin(Vfs::WindowsCfApi).release());
+    QObject::connect(&folder.syncEngine().syncFileStatusTracker(), &SyncFileStatusTracker::fileStatusChanged,
+                     cfapiVfs.data(), &Vfs::fileStatusChanged);
+    folder.switchToVfs(cfapiVfs);
+
+    setPinState(folder.localPath(), PinState::Unspecified, cfapi::NoRecurse);
+
+    return cfapiVfs;
+}
+
+class TestSyncCfApi : public QObject
+{
+    Q_OBJECT
+
+private slots:
+    void testVirtualFileLifecycle_data()
+    {
+        QTest::addColumn<bool>("doLocalDiscovery");
+
+        QTest::newRow("full local discovery") << true;
+        QTest::newRow("skip local discovery") << false;
+    }
+
+    void testVirtualFileLifecycle()
+    {
+        QFETCH(bool, doLocalDiscovery);
+
+        FakeFolder fakeFolder{ FileInfo() };
+        setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+        ItemCompletedSpy completeSpy(fakeFolder);
+
+        auto cleanup = [&]() {
+            completeSpy.clear();
+            if (!doLocalDiscovery)
+                fakeFolder.syncEngine().setLocalDiscoveryOptions(LocalDiscoveryStyle::DatabaseAndFilesystem);
+        };
+        cleanup();
+
+        // Create a virtual file for a new remote file
+        fakeFolder.remoteModifier().mkdir("A");
+        fakeFolder.remoteModifier().insert("A/a1", 64);
+        auto someDate = QDateTime(QDate(1984, 07, 30), QTime(1,3,2));
+        fakeFolder.remoteModifier().setModTime("A/a1", someDate);
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 64);
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").lastModified(), someDate);
+        QVERIFY(fakeFolder.currentRemoteState().find("A/a1"));
+        QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_NEW));
+        cleanup();
+
+        // Another sync doesn't actually lead to changes
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 64);
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").lastModified(), someDate);
+        QVERIFY(fakeFolder.currentRemoteState().find("A/a1"));
+        QVERIFY(completeSpy.isEmpty());
+        cleanup();
+
+        // Not even when the remote is rediscovered
+        fakeFolder.syncJournal().forceRemoteDiscoveryNextSync();
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 64);
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").lastModified(), someDate);
+        QVERIFY(fakeFolder.currentRemoteState().find("A/a1"));
+        QVERIFY(completeSpy.isEmpty());
+        cleanup();
+
+        // Neither does a remote change
+        fakeFolder.remoteModifier().appendByte("A/a1");
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 65);
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").lastModified(), someDate);
+        QVERIFY(fakeFolder.currentRemoteState().find("A/a1"));
+        QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_UPDATE_METADATA));
+        QCOMPARE(dbRecord(fakeFolder, "A/a1")._fileSize, 65);
+        cleanup();
+
+        // If the local virtual file is removed, this will be propagated remotely
+        if (!doLocalDiscovery)
+            fakeFolder.syncEngine().setLocalDiscoveryOptions(LocalDiscoveryStyle::DatabaseAndFilesystem, { "A" });
+        fakeFolder.localModifier().remove("A/a1");
+        QVERIFY(fakeFolder.syncOnce());
+        QVERIFY(!fakeFolder.currentLocalState().find("A/a1"));
+        QVERIFY(!fakeFolder.currentRemoteState().find("A/a1"));
+        QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_REMOVE));
+        QVERIFY(!dbRecord(fakeFolder, "A/a1").isValid());
+        cleanup();
+
+        // Recreate a1 before carrying on with the other tests
+        fakeFolder.remoteModifier().insert("A/a1", 65);
+        fakeFolder.remoteModifier().setModTime("A/a1", someDate);
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 65);
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").lastModified(), someDate);
+        QVERIFY(fakeFolder.currentRemoteState().find("A/a1"));
+        QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_NEW));
+        cleanup();
+
+        // Remote rename is propagated
+        fakeFolder.remoteModifier().rename("A/a1", "A/a1m");
+        QVERIFY(fakeFolder.syncOnce());
+        QVERIFY(!QFileInfo(fakeFolder.localPath() + "A/a1").exists());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1m");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1m").size(), 65);
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1m").lastModified(), someDate);
+        QVERIFY(!fakeFolder.currentRemoteState().find("A/a1"));
+        QVERIFY(fakeFolder.currentRemoteState().find("A/a1m"));
+        QVERIFY(
+            itemInstruction(completeSpy, "A/a1m", CSYNC_INSTRUCTION_RENAME)
+            || (itemInstruction(completeSpy, "A/a1m", CSYNC_INSTRUCTION_NEW)
+                && itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_REMOVE)));
+        QVERIFY(!dbRecord(fakeFolder, "A/a1").isValid());
+        cleanup();
+
+        // Remote remove is propagated
+        fakeFolder.remoteModifier().remove("A/a1m");
+        QVERIFY(fakeFolder.syncOnce());
+        QVERIFY(!QFileInfo(fakeFolder.localPath() + "A/a1m").exists());
+        QVERIFY(!fakeFolder.currentRemoteState().find("A/a1m"));
+        QVERIFY(itemInstruction(completeSpy, "A/a1m", CSYNC_INSTRUCTION_REMOVE));
+        QVERIFY(!dbRecord(fakeFolder, "A/a1").isValid());
+        QVERIFY(!dbRecord(fakeFolder, "A/a1m").isValid());
+        cleanup();
+
+        // Edge case: Local virtual file but no db entry for some reason
+        fakeFolder.remoteModifier().insert("A/a2", 32);
+        fakeFolder.remoteModifier().insert("A/a3", 33);
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a2");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a2").size(), 32);
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a3");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a3").size(), 33);
+        cleanup();
+
+        fakeFolder.syncEngine().journal()->deleteFileRecord("A/a2");
+        fakeFolder.syncEngine().journal()->deleteFileRecord("A/a3");
+        fakeFolder.remoteModifier().remove("A/a3");
+        fakeFolder.syncEngine().setLocalDiscoveryOptions(LocalDiscoveryStyle::FilesystemOnly);
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a2");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a2").size(), 32);
+        QVERIFY(itemInstruction(completeSpy, "A/a2", CSYNC_INSTRUCTION_UPDATE_METADATA));
+        QVERIFY(!QFileInfo(fakeFolder.localPath() + "A/a3").exists());
+        QVERIFY(itemInstruction(completeSpy, "A/a3", CSYNC_INSTRUCTION_REMOVE));
+        QVERIFY(!dbRecord(fakeFolder, "A/a3").isValid());
+        cleanup();
+    }
+
+    void testVirtualFileConflict()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+        ItemCompletedSpy completeSpy(fakeFolder);
+
+        auto cleanup = [&]() {
+            completeSpy.clear();
+        };
+        cleanup();
+
+        // Create a virtual file for a new remote file
+        fakeFolder.remoteModifier().mkdir("A");
+        fakeFolder.remoteModifier().insert("A/a1", 11);
+        fakeFolder.remoteModifier().insert("A/a2", 12);
+        fakeFolder.remoteModifier().mkdir("B");
+        fakeFolder.remoteModifier().insert("B/b1", 21);
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a2");
+        CFVERIFY_VIRTUAL(fakeFolder, "B/b1");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 11);
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a2").size(), 12);
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "B/b1").size(), 21);
+        cleanup();
+
+        // All the files are touched on the server
+        fakeFolder.remoteModifier().appendByte("A/a1");
+        fakeFolder.remoteModifier().appendByte("A/a2");
+        fakeFolder.remoteModifier().appendByte("B/b1");
+
+        // A: the correct file and a conflicting file are added
+        // B: user adds a *directory* locally
+        fakeFolder.localModifier().remove("A/a1");
+        fakeFolder.localModifier().insert("A/a1", 12);
+        fakeFolder.localModifier().remove("A/a2");
+        fakeFolder.localModifier().insert("A/a2", 10);
+        fakeFolder.localModifier().remove("B/b1");
+        fakeFolder.localModifier().mkdir("B/b1");
+        fakeFolder.localModifier().insert("B/b1/foo");
+        QVERIFY(fakeFolder.syncOnce());
+
+        // Everything is CONFLICT
+        QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_CONFLICT));
+        QVERIFY(itemInstruction(completeSpy, "A/a2", CSYNC_INSTRUCTION_CONFLICT));
+        QVERIFY(itemInstruction(completeSpy, "B/b1", CSYNC_INSTRUCTION_CONFLICT));
+
+        // conflict files should exist
+        QCOMPARE(fakeFolder.syncJournal().conflictRecordPaths().size(), 2);
+
+        // nothing should have the virtual file tag
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a1");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a2");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "B/b1");
+
+        cleanup();
+    }
+
+    void testWithNormalSync()
+    {
+        FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() };
+        setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+        ItemCompletedSpy completeSpy(fakeFolder);
+
+        auto cleanup = [&]() {
+            completeSpy.clear();
+        };
+        cleanup();
+
+        // No effect sync
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+        cleanup();
+
+        // Existing files are propagated just fine in both directions
+        fakeFolder.localModifier().appendByte("A/a1");
+        fakeFolder.localModifier().insert("A/a3");
+        fakeFolder.remoteModifier().appendByte("A/a2");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+        cleanup();
+
+        // New files on the remote create virtual files
+        fakeFolder.remoteModifier().insert("A/new", 42);
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/new");
+        QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/new").size(), 42);
+        QVERIFY(fakeFolder.currentRemoteState().find("A/new"));
+        QVERIFY(itemInstruction(completeSpy, "A/new", CSYNC_INSTRUCTION_NEW));
+        cleanup();
+    }
+
+    void testVirtualFileDownload()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+        ItemCompletedSpy completeSpy(fakeFolder);
+
+        auto cleanup = [&]() {
+            completeSpy.clear();
+        };
+        cleanup();
+
+        // Create a virtual file for remote files
+        fakeFolder.remoteModifier().mkdir("A");
+        fakeFolder.remoteModifier().insert("A/a1");
+        fakeFolder.remoteModifier().insert("A/a2");
+        fakeFolder.remoteModifier().insert("A/a3");
+        fakeFolder.remoteModifier().insert("A/a4");
+        fakeFolder.remoteModifier().insert("A/a5");
+        fakeFolder.remoteModifier().insert("A/a6");
+        fakeFolder.remoteModifier().insert("A/a7");
+        fakeFolder.remoteModifier().insert("A/b1");
+        fakeFolder.remoteModifier().insert("A/b2");
+        fakeFolder.remoteModifier().insert("A/b3");
+        fakeFolder.remoteModifier().insert("A/b4");
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a2");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a3");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a4");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a5");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a6");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a7");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/b1");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/b2");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/b3");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/b4");
+
+        cleanup();
+
+        // Download by changing the db entry
+        triggerDownload(fakeFolder, "A/a1");
+        triggerDownload(fakeFolder, "A/a2");
+        triggerDownload(fakeFolder, "A/a3");
+        triggerDownload(fakeFolder, "A/a4");
+        triggerDownload(fakeFolder, "A/a5");
+        triggerDownload(fakeFolder, "A/a6");
+        triggerDownload(fakeFolder, "A/a7");
+        triggerDownload(fakeFolder, "A/b1");
+        triggerDownload(fakeFolder, "A/b2");
+        triggerDownload(fakeFolder, "A/b3");
+        triggerDownload(fakeFolder, "A/b4");
+
+        // Remote complications
+        fakeFolder.remoteModifier().appendByte("A/a2");
+        fakeFolder.remoteModifier().remove("A/a3");
+        fakeFolder.remoteModifier().rename("A/a4", "A/a4m");
+        fakeFolder.remoteModifier().appendByte("A/b2");
+        fakeFolder.remoteModifier().remove("A/b3");
+        fakeFolder.remoteModifier().rename("A/b4", "A/b4m");
+
+        // Local complications
+        fakeFolder.localModifier().remove("A/a5");
+        fakeFolder.localModifier().insert("A/a5");
+        fakeFolder.localModifier().remove("A/a6");
+        fakeFolder.localModifier().insert("A/a6");
+
+        QVERIFY(fakeFolder.syncOnce());
+        QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_SYNC));
+        QCOMPARE(completeSpy.findItem("A/a1")->_type, ItemTypeVirtualFileDownload);
+        QVERIFY(itemInstruction(completeSpy, "A/a2", CSYNC_INSTRUCTION_SYNC));
+        QCOMPARE(completeSpy.findItem("A/a2")->_type, ItemTypeVirtualFileDownload);
+        QVERIFY(itemInstruction(completeSpy, "A/a3", CSYNC_INSTRUCTION_REMOVE));
+        QVERIFY(itemInstruction(completeSpy, "A/a4m", CSYNC_INSTRUCTION_NEW));
+        QVERIFY(itemInstruction(completeSpy, "A/a4", CSYNC_INSTRUCTION_REMOVE));
+        QVERIFY(itemInstruction(completeSpy, "A/a5", CSYNC_INSTRUCTION_CONFLICT));
+        QVERIFY(itemInstruction(completeSpy, "A/a6", CSYNC_INSTRUCTION_CONFLICT));
+        QVERIFY(itemInstruction(completeSpy, "A/a7", CSYNC_INSTRUCTION_SYNC));
+        QVERIFY(itemInstruction(completeSpy, "A/b1", CSYNC_INSTRUCTION_SYNC));
+        QVERIFY(itemInstruction(completeSpy, "A/b2", CSYNC_INSTRUCTION_SYNC));
+        QVERIFY(itemInstruction(completeSpy, "A/b3", CSYNC_INSTRUCTION_REMOVE));
+        QVERIFY(itemInstruction(completeSpy, "A/b4m", CSYNC_INSTRUCTION_NEW));
+        QVERIFY(itemInstruction(completeSpy, "A/b4", CSYNC_INSTRUCTION_REMOVE));
+
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a1");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a2");
+        CFVERIFY_GONE(fakeFolder, "A/a3");
+        CFVERIFY_GONE(fakeFolder, "A/a4");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a4m");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a5");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a6");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a7");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/b1");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/b2");
+        CFVERIFY_GONE(fakeFolder, "A/b3");
+        CFVERIFY_GONE(fakeFolder, "A/b4");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/b4m");
+
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+    }
+
+    void testVirtualFileDownloadResume()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+        ItemCompletedSpy completeSpy(fakeFolder);
+
+        auto cleanup = [&]() {
+            completeSpy.clear();
+            fakeFolder.syncJournal().wipeErrorBlacklist();
+        };
+        cleanup();
+
+        // Create a virtual file for remote files
+        fakeFolder.remoteModifier().mkdir("A");
+        fakeFolder.remoteModifier().insert("A/a1");
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        cleanup();
+
+        // Download by changing the db entry
+        triggerDownload(fakeFolder, "A/a1");
+        fakeFolder.serverErrorPaths().append("A/a1", 500);
+        QVERIFY(!fakeFolder.syncOnce());
+        QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_SYNC));
+        QVERIFY(cfapi::isSparseFile(fakeFolder.localPath() + "A/a1"));
+        QVERIFY(QFileInfo(fakeFolder.localPath() + "A/a1").exists());
+        QCOMPARE(dbRecord(fakeFolder, "A/a1")._type, ItemTypeVirtualFileDownload);
+        cleanup();
+
+        fakeFolder.serverErrorPaths().clear();
+        QVERIFY(fakeFolder.syncOnce());
+        QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_SYNC));
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a1");
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+    }
+
+    void testNewFilesNotVirtual()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        fakeFolder.remoteModifier().mkdir("A");
+        fakeFolder.remoteModifier().insert("A/a1");
+        QVERIFY(fakeFolder.syncOnce());
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+
+        setPinState(fakeFolder.localPath(), PinState::AlwaysLocal, cfapi::NoRecurse);
+
+        // Create a new remote file, it'll not be virtual
+        fakeFolder.remoteModifier().insert("A/a2");
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a2");
+    }
+
+    void testDownloadRecursive()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        // Create a virtual file for remote files
+        fakeFolder.remoteModifier().mkdir("A");
+        fakeFolder.remoteModifier().mkdir("A/Sub");
+        fakeFolder.remoteModifier().mkdir("A/Sub/SubSub");
+        fakeFolder.remoteModifier().mkdir("A/Sub2");
+        fakeFolder.remoteModifier().mkdir("B");
+        fakeFolder.remoteModifier().mkdir("B/Sub");
+        fakeFolder.remoteModifier().insert("A/a1");
+        fakeFolder.remoteModifier().insert("A/a2");
+        fakeFolder.remoteModifier().insert("A/Sub/a3");
+        fakeFolder.remoteModifier().insert("A/Sub/a4");
+        fakeFolder.remoteModifier().insert("A/Sub/SubSub/a5");
+        fakeFolder.remoteModifier().insert("A/Sub2/a6");
+        fakeFolder.remoteModifier().insert("B/b1");
+        fakeFolder.remoteModifier().insert("B/Sub/b2");
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a2");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/Sub/a3");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/Sub/a4");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/Sub/SubSub/a5");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/Sub2/a6");
+        CFVERIFY_VIRTUAL(fakeFolder, "B/b1");
+        CFVERIFY_VIRTUAL(fakeFolder, "B/Sub/b2");
+
+        // Download All file in the directory A/Sub
+        // (as in Folder::downloadVirtualFile)
+        fakeFolder.syncJournal().markVirtualFileForDownloadRecursively("A/Sub");
+
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a2");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/a3");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/a4");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/SubSub/a5");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/Sub2/a6");
+        CFVERIFY_VIRTUAL(fakeFolder, "B/b1");
+        CFVERIFY_VIRTUAL(fakeFolder, "B/Sub/b2");
+
+        // Add a file in a subfolder that was downloaded
+        // Currently, this continue to add it as a virtual file.
+        fakeFolder.remoteModifier().insert("A/Sub/SubSub/a7");
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "A/Sub/SubSub/a7");
+
+        // Now download all files in "A"
+        fakeFolder.syncJournal().markVirtualFileForDownloadRecursively("A");
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a1");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/a2");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/a3");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/a4");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/SubSub/a5");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/SubSub/a7");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub2/a6");
+        CFVERIFY_VIRTUAL(fakeFolder, "B/b1");
+        CFVERIFY_VIRTUAL(fakeFolder, "B/Sub/b2");
+
+        // Now download remaining files in "B"
+        fakeFolder.syncJournal().markVirtualFileForDownloadRecursively("B");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+    }
+
+    void testRenameVirtual()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+        ItemCompletedSpy completeSpy(fakeFolder);
+
+        auto cleanup = [&]() {
+            completeSpy.clear();
+        };
+        cleanup();
+
+        fakeFolder.remoteModifier().insert("file1", 128, 'C');
+        fakeFolder.remoteModifier().insert("file2", 256, 'C');
+        fakeFolder.remoteModifier().insert("file3", 256, 'C');
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "file1");
+        CFVERIFY_VIRTUAL(fakeFolder, "file2");
+        CFVERIFY_VIRTUAL(fakeFolder, "file3");
+
+        cleanup();
+
+        fakeFolder.localModifier().rename("file1", "renamed1");
+        fakeFolder.localModifier().rename("file2", "renamed2");
+        triggerDownload(fakeFolder, "file2");
+        triggerDownload(fakeFolder, "file3");
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_GONE(fakeFolder, "file1");
+        CFVERIFY_VIRTUAL(fakeFolder, "renamed1");
+
+        QVERIFY(fakeFolder.currentRemoteState().find("renamed1"));
+        QVERIFY(itemInstruction(completeSpy, "renamed1", CSYNC_INSTRUCTION_RENAME));
+
+        // file2 has a conflict between the download request and the rename:
+        // the rename wins, the download is ignored
+
+        CFVERIFY_GONE(fakeFolder, "file2");
+        CFVERIFY_VIRTUAL(fakeFolder, "renamed2");
+
+        QVERIFY(fakeFolder.currentRemoteState().find("renamed2"));
+        QVERIFY(itemInstruction(completeSpy, "renamed2", CSYNC_INSTRUCTION_RENAME));
+
+        QVERIFY(itemInstruction(completeSpy, "file3", CSYNC_INSTRUCTION_SYNC));
+        CFVERIFY_NONVIRTUAL(fakeFolder, "file3");
+        cleanup();
+    }
+
+    void testRenameVirtual2()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        setupVfs(fakeFolder);
+        ItemCompletedSpy completeSpy(fakeFolder);
+        auto cleanup = [&]() {
+            completeSpy.clear();
+        };
+        cleanup();
+
+        fakeFolder.remoteModifier().insert("case3", 128, 'C');
+        fakeFolder.remoteModifier().insert("case4", 256, 'C');
+        QVERIFY(fakeFolder.syncOnce());
+
+        triggerDownload(fakeFolder, "case4");
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "case3");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "case4");
+
+        cleanup();
+
+        // Case 1: non-virtual, foo -> bar (tested elsewhere)
+        // Case 2: virtual, foo -> bar (tested elsewhere)
+
+        // Case 3: virtual, foo.oc -> bar.oc (db hydrate)
+        fakeFolder.localModifier().rename("case3", "case3-rename");
+        triggerDownload(fakeFolder, "case3");
+
+        // Case 4: non-virtual foo -> bar (db dehydrate)
+        fakeFolder.localModifier().rename("case4", "case4-rename");
+        markForDehydration(fakeFolder, "case4");
+
+        QVERIFY(fakeFolder.syncOnce());
+
+        // Case 3: the rename went though, hydration is forgotten
+        CFVERIFY_GONE(fakeFolder, "case3");
+        CFVERIFY_VIRTUAL(fakeFolder, "case3-rename");
+        QVERIFY(!fakeFolder.currentRemoteState().find("case3"));
+        QVERIFY(fakeFolder.currentRemoteState().find("case3-rename"));
+        QVERIFY(itemInstruction(completeSpy, "case3-rename", CSYNC_INSTRUCTION_RENAME));
+
+        // Case 4: the rename went though, dehydration is forgotten
+        CFVERIFY_GONE(fakeFolder, "case4");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "case4-rename");
+        QVERIFY(!fakeFolder.currentRemoteState().find("case4"));
+        QVERIFY(fakeFolder.currentRemoteState().find("case4-rename"));
+        QVERIFY(itemInstruction(completeSpy, "case4-rename", CSYNC_INSTRUCTION_RENAME));
+    }
+
+    // Dehydration via sync works
+    void testSyncDehydration()
+    {
+        FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() };
+        setupVfs(fakeFolder);
+
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        ItemCompletedSpy completeSpy(fakeFolder);
+        auto cleanup = [&]() {
+            completeSpy.clear();
+        };
+        cleanup();
+
+        //
+        // Mark for dehydration and check
+        //
+
+        markForDehydration(fakeFolder, "A/a1");
+
+        markForDehydration(fakeFolder, "A/a2");
+        fakeFolder.remoteModifier().appendByte("A/a2");
+        // expect: normal dehydration
+
+        markForDehydration(fakeFolder, "B/b1");
+        fakeFolder.remoteModifier().remove("B/b1");
+        // expect: local removal
+
+        markForDehydration(fakeFolder, "B/b2");
+        fakeFolder.remoteModifier().rename("B/b2", "B/b3");
+        // expect: B/b2 is gone, B/b3 is NEW placeholder
+
+        markForDehydration(fakeFolder, "C/c1");
+        fakeFolder.localModifier().appendByte("C/c1");
+        // expect: no dehydration, upload of c1
+
+        markForDehydration(fakeFolder, "C/c2");
+        fakeFolder.localModifier().appendByte("C/c2");
+        fakeFolder.remoteModifier().appendByte("C/c2");
+        fakeFolder.remoteModifier().appendByte("C/c2");
+        // expect: no dehydration, conflict
+
+        QVERIFY(fakeFolder.syncOnce());
+
+        auto isDehydrated = [&](const QString &path) {
+            return cfapi::isSparseFile(fakeFolder.localPath() + path)
+                && QFileInfo(fakeFolder.localPath() + path).exists();
+        };
+        auto hasDehydratedDbEntries = [&](const QString &path) {
+            SyncJournalFileRecord rec;
+            fakeFolder.syncJournal().getFileRecord(path, &rec);
+            return rec.isValid() && rec._type == ItemTypeVirtualFile;
+        };
+
+        QVERIFY(isDehydrated("A/a1"));
+        QVERIFY(hasDehydratedDbEntries("A/a1"));
+        QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_SYNC));
+        QCOMPARE(completeSpy.findItem("A/a1")->_type, ItemTypeVirtualFileDehydration);
+        QCOMPARE(completeSpy.findItem("A/a1")->_file, QStringLiteral("A/a1"));
+        QVERIFY(isDehydrated("A/a2"));
+        QVERIFY(hasDehydratedDbEntries("A/a2"));
+        QVERIFY(itemInstruction(completeSpy, "A/a2", CSYNC_INSTRUCTION_SYNC));
+        QCOMPARE(completeSpy.findItem("A/a2")->_type, ItemTypeVirtualFileDehydration);
+
+        QVERIFY(!QFileInfo(fakeFolder.localPath() + "B/b1").exists());
+        QVERIFY(!fakeFolder.currentRemoteState().find("B/b1"));
+        QVERIFY(itemInstruction(completeSpy, "B/b1", CSYNC_INSTRUCTION_REMOVE));
+
+        QVERIFY(!QFileInfo(fakeFolder.localPath() + "B/b2").exists());
+        QVERIFY(!fakeFolder.currentRemoteState().find("B/b2"));
+        QVERIFY(isDehydrated("B/b3"));
+        QVERIFY(hasDehydratedDbEntries("B/b3"));
+        QVERIFY(itemInstruction(completeSpy, "B/b2", CSYNC_INSTRUCTION_REMOVE));
+        QVERIFY(itemInstruction(completeSpy, "B/b3", CSYNC_INSTRUCTION_NEW));
+
+        QCOMPARE(fakeFolder.currentRemoteState().find("C/c1")->size, 25);
+        QVERIFY(itemInstruction(completeSpy, "C/c1", CSYNC_INSTRUCTION_SYNC));
+
+        QCOMPARE(fakeFolder.currentRemoteState().find("C/c2")->size, 26);
+        QVERIFY(itemInstruction(completeSpy, "C/c2", CSYNC_INSTRUCTION_CONFLICT));
+        cleanup();
+
+        auto expectedRemoteState = fakeFolder.currentRemoteState();
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentRemoteState(), expectedRemoteState);
+
+        QVERIFY(isDehydrated("A/a1"));
+        QVERIFY(hasDehydratedDbEntries("A/a1"));
+        QVERIFY(isDehydrated("A/a2"));
+        QVERIFY(hasDehydratedDbEntries("A/a2"));
+
+        QVERIFY(!QFileInfo(fakeFolder.localPath() + "B/b1").exists());
+        QVERIFY(!QFileInfo(fakeFolder.localPath() + "B/b2").exists());
+        QVERIFY(isDehydrated("B/b3"));
+        QVERIFY(hasDehydratedDbEntries("B/b3"));
+
+        QVERIFY(QFileInfo(fakeFolder.localPath() + "C/c1").exists());
+        QVERIFY(dbRecord(fakeFolder, "C/c1").isValid());
+        QVERIFY(!isDehydrated("C/c1"));
+        QVERIFY(!hasDehydratedDbEntries("C/c1"));
+
+        QVERIFY(QFileInfo(fakeFolder.localPath() + "C/c2").exists());
+        QVERIFY(dbRecord(fakeFolder, "C/c2").isValid());
+        QVERIFY(!isDehydrated("C/c2"));
+        QVERIFY(!hasDehydratedDbEntries("C/c2"));
+    }
+
+    void testWipeVirtualSuffixFiles()
+    {
+        FakeFolder fakeFolder{ FileInfo{} };
+        setupVfs(fakeFolder);
+
+        // Create a suffix-vfs baseline
+
+        fakeFolder.remoteModifier().mkdir("A");
+        fakeFolder.remoteModifier().mkdir("A/B");
+        fakeFolder.remoteModifier().insert("f1");
+        fakeFolder.remoteModifier().insert("A/a1");
+        fakeFolder.remoteModifier().insert("A/a3");
+        fakeFolder.remoteModifier().insert("A/B/b1");
+        fakeFolder.localModifier().mkdir("A");
+        fakeFolder.localModifier().mkdir("A/B");
+        fakeFolder.localModifier().insert("f2");
+        fakeFolder.localModifier().insert("A/a2");
+        fakeFolder.localModifier().insert("A/B/b2");
+
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "f1");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a1");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/a3");
+        CFVERIFY_VIRTUAL(fakeFolder, "A/B/b1");
+
+        // Make local changes to a3
+        fakeFolder.localModifier().remove("A/a3");
+        fakeFolder.localModifier().insert("A/a3", 100);
+
+        // Now wipe the virtuals
+        SyncEngine::wipeVirtualFiles(fakeFolder.localPath(), fakeFolder.syncJournal(), *fakeFolder.syncEngine().syncOptions()._vfs);
+
+        CFVERIFY_GONE(fakeFolder, "f1");
+        CFVERIFY_GONE(fakeFolder, "A/a1");
+        QVERIFY(QFileInfo(fakeFolder.localPath() + "A/a3").exists());
+        QVERIFY(!dbRecord(fakeFolder, "A/a3").isValid());
+        CFVERIFY_GONE(fakeFolder, "A/B/b1");
+
+        fakeFolder.switchToVfs(QSharedPointer<Vfs>(new VfsOff));
+        ItemCompletedSpy completeSpy(fakeFolder);
+        QVERIFY(fakeFolder.syncOnce());
+
+        QVERIFY(fakeFolder.currentLocalState().find("A"));
+        QVERIFY(fakeFolder.currentLocalState().find("A/B"));
+        QVERIFY(fakeFolder.currentLocalState().find("A/B/b1"));
+        QVERIFY(fakeFolder.currentLocalState().find("A/B/b2"));
+        QVERIFY(fakeFolder.currentLocalState().find("A/a1"));
+        QVERIFY(fakeFolder.currentLocalState().find("A/a2"));
+        QVERIFY(fakeFolder.currentLocalState().find("A/a3"));
+        QVERIFY(fakeFolder.currentLocalState().find("f1"));
+        QVERIFY(fakeFolder.currentLocalState().find("f2"));
+
+        // a3 has a conflict
+        QVERIFY(itemInstruction(completeSpy, "A/a3", CSYNC_INSTRUCTION_CONFLICT));
+
+        // conflict files should exist
+        QCOMPARE(fakeFolder.syncJournal().conflictRecordPaths().size(), 1);
+    }
+
+    void testNewVirtuals()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        fakeFolder.remoteModifier().mkdir("local");
+        fakeFolder.remoteModifier().mkdir("online");
+        fakeFolder.remoteModifier().mkdir("unspec");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::Recurse);
+        setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::Recurse);
+        setPinState(fakeFolder.localPath() + "unspec", PinState::Unspecified, cfapi::Recurse);
+
+        // Test 1: root is Unspecified
+        fakeFolder.remoteModifier().insert("file1");
+        fakeFolder.remoteModifier().insert("online/file1");
+        fakeFolder.remoteModifier().insert("local/file1");
+        fakeFolder.remoteModifier().insert("unspec/file1");
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "file1");
+        CFVERIFY_VIRTUAL(fakeFolder, "online/file1");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "local/file1");
+        CFVERIFY_VIRTUAL(fakeFolder, "unspec/file1");
+
+        // Test 2: change root to AlwaysLocal
+        setPinState(fakeFolder.localPath(), PinState::AlwaysLocal, cfapi::Recurse);
+        // Need to force pin state for the subfolders again
+        setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::Recurse);
+        setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::Recurse);
+        setPinState(fakeFolder.localPath() + "unspec", PinState::Unspecified, cfapi::Recurse);
+
+        fakeFolder.remoteModifier().insert("file2");
+        fakeFolder.remoteModifier().insert("online/file2");
+        fakeFolder.remoteModifier().insert("local/file2");
+        fakeFolder.remoteModifier().insert("unspec/file2");
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_NONVIRTUAL(fakeFolder, "file2");
+        CFVERIFY_VIRTUAL(fakeFolder, "online/file2");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "local/file2");
+        CFVERIFY_VIRTUAL(fakeFolder, "unspec/file2");
+
+        // root file1 was hydrated due to its new pin state
+        CFVERIFY_NONVIRTUAL(fakeFolder, "file1");
+
+        // file1 is unchanged in the explicitly pinned subfolders
+        CFVERIFY_VIRTUAL(fakeFolder, "online/file1");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "local/file1");
+        CFVERIFY_VIRTUAL(fakeFolder, "unspec/file1");
+
+        // Test 3: change root to OnlineOnly
+        setPinState(fakeFolder.localPath(), PinState::OnlineOnly, cfapi::Recurse);
+        // Need to force pin state for the subfolders again
+        setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::Recurse);
+        setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::Recurse);
+        setPinState(fakeFolder.localPath() + "unspec", PinState::Unspecified, cfapi::Recurse);
+
+        fakeFolder.remoteModifier().insert("file3");
+        fakeFolder.remoteModifier().insert("online/file3");
+        fakeFolder.remoteModifier().insert("local/file3");
+        fakeFolder.remoteModifier().insert("unspec/file3");
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "file3");
+        CFVERIFY_VIRTUAL(fakeFolder, "online/file3");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "local/file3");
+        CFVERIFY_VIRTUAL(fakeFolder, "unspec/file3");
+
+        // root file1 was dehydrated due to its new pin state
+        CFVERIFY_VIRTUAL(fakeFolder, "file1");
+
+        // file1 is unchanged in the explicitly pinned subfolders
+        CFVERIFY_VIRTUAL(fakeFolder, "online/file1");
+        CFVERIFY_NONVIRTUAL(fakeFolder, "local/file1");
+        CFVERIFY_VIRTUAL(fakeFolder, "unspec/file1");
+    }
+
+    void testAvailability()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        auto vfs = setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        fakeFolder.remoteModifier().mkdir("local");
+        fakeFolder.remoteModifier().mkdir("local/sub");
+        fakeFolder.remoteModifier().mkdir("online");
+        fakeFolder.remoteModifier().mkdir("online/sub");
+        fakeFolder.remoteModifier().mkdir("unspec");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::Recurse);
+        setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::Recurse);
+        setPinState(fakeFolder.localPath() + "unspec", PinState::Unspecified, cfapi::Recurse);
+
+        fakeFolder.remoteModifier().insert("file1");
+        fakeFolder.remoteModifier().insert("online/file1");
+        fakeFolder.remoteModifier().insert("online/file2");
+        fakeFolder.remoteModifier().insert("local/file1");
+        fakeFolder.remoteModifier().insert("local/file2");
+        fakeFolder.remoteModifier().insert("unspec/file1");
+        QVERIFY(fakeFolder.syncOnce());
+
+        // root is unspecified
+        QCOMPARE(*vfs->availability("file1"), VfsItemAvailability::AllDehydrated);
+        QCOMPARE(*vfs->availability("local"), VfsItemAvailability::AlwaysLocal);
+        QCOMPARE(*vfs->availability("local/file1"), VfsItemAvailability::AlwaysLocal);
+        QCOMPARE(*vfs->availability("online"), VfsItemAvailability::OnlineOnly);
+        QCOMPARE(*vfs->availability("online/file1"), VfsItemAvailability::OnlineOnly);
+        QCOMPARE(*vfs->availability("unspec"), VfsItemAvailability::AllDehydrated);
+        QCOMPARE(*vfs->availability("unspec/file1"), VfsItemAvailability::AllDehydrated);
+
+        // Subitem pin states can ruin "pure" availabilities
+        setPinState(fakeFolder.localPath() + "local/sub", PinState::OnlineOnly, cfapi::NoRecurse);
+        QCOMPARE(*vfs->availability("local"), VfsItemAvailability::AllHydrated);
+        setPinState(fakeFolder.localPath() + "online/sub", PinState::Unspecified, cfapi::NoRecurse);
+        QCOMPARE(*vfs->availability("online"), VfsItemAvailability::AllDehydrated);
+
+        triggerDownload(fakeFolder, "unspec/file1");
+        setPinState(fakeFolder.localPath() + "local/file2", PinState::OnlineOnly, cfapi::NoRecurse);
+        setPinState(fakeFolder.localPath() + "online/file2", PinState::AlwaysLocal, cfapi::NoRecurse);
+        QVERIFY(fakeFolder.syncOnce());
+
+        QCOMPARE(*vfs->availability("unspec"), VfsItemAvailability::AllHydrated);
+        QCOMPARE(*vfs->availability("local"), VfsItemAvailability::Mixed);
+        QCOMPARE(*vfs->availability("online"), VfsItemAvailability::Mixed);
+
+        vfs->setPinState("local", PinState::AlwaysLocal);
+        vfs->setPinState("online", PinState::OnlineOnly);
+        QVERIFY(fakeFolder.syncOnce());
+
+        QCOMPARE(*vfs->availability("online"), VfsItemAvailability::OnlineOnly);
+        QCOMPARE(*vfs->availability("local"), VfsItemAvailability::AlwaysLocal);
+
+        auto r = vfs->availability("nonexistant");
+        QVERIFY(!r);
+        QCOMPARE(r.error(), Vfs::AvailabilityError::NoSuchItem);
+    }
+
+    void testPinStateLocals()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        auto vfs = setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        fakeFolder.remoteModifier().mkdir("local");
+        fakeFolder.remoteModifier().mkdir("online");
+        fakeFolder.remoteModifier().mkdir("unspec");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::NoRecurse);
+        setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::NoRecurse);
+        setPinState(fakeFolder.localPath() + "unspec", PinState::Unspecified, cfapi::NoRecurse);
+
+        fakeFolder.localModifier().insert("file1");
+        fakeFolder.localModifier().insert("online/file1");
+        fakeFolder.localModifier().insert("online/file2");
+        fakeFolder.localModifier().insert("local/file1");
+        fakeFolder.localModifier().insert("unspec/file1");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        // root is unspecified
+        QCOMPARE(*vfs->pinState("file1"), PinState::Unspecified);
+        QCOMPARE(*vfs->pinState("local/file1"), PinState::AlwaysLocal);
+        QCOMPARE(*vfs->pinState("online/file1"), PinState::Unspecified);
+        QCOMPARE(*vfs->pinState("unspec/file1"), PinState::Unspecified);
+
+        // Sync again: bad pin states of new local files usually take effect on second sync
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        // When a file in an online-only folder is renamed, it retains its pin
+        fakeFolder.localModifier().rename("online/file1", "online/file1rename");
+        fakeFolder.remoteModifier().rename("online/file2", "online/file2rename");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(*vfs->pinState("online/file1rename"), PinState::Unspecified);
+        QCOMPARE(*vfs->pinState("online/file2rename"), PinState::Unspecified);
+
+        // When a folder is renamed, the pin states inside should be retained
+        fakeFolder.localModifier().rename("online", "onlinerenamed1");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(*vfs->pinState("onlinerenamed1"), PinState::OnlineOnly);
+        QCOMPARE(*vfs->pinState("onlinerenamed1/file1rename"), PinState::Unspecified);
+
+        fakeFolder.remoteModifier().rename("onlinerenamed1", "onlinerenamed2");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(*vfs->pinState("onlinerenamed2"), PinState::OnlineOnly);
+        QCOMPARE(*vfs->pinState("onlinerenamed2/file1rename"), PinState::Unspecified);
+
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        // When a file is deleted and later a new file has the same name, the old pin
+        // state isn't preserved.
+        QCOMPARE(*vfs->pinState("onlinerenamed2/file1rename"), PinState::Unspecified);
+        fakeFolder.remoteModifier().remove("onlinerenamed2/file1rename");
+        QVERIFY(fakeFolder.syncOnce());
+        QVERIFY(!vfs->pinState("onlinerenamed2/file1rename"));
+        fakeFolder.remoteModifier().insert("onlinerenamed2/file1rename");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(*vfs->pinState("onlinerenamed2/file1rename"), PinState::OnlineOnly);
+
+        // When a file is hydrated or dehydrated due to pin state it retains its pin state
+        vfs->setPinState("onlinerenamed2/file1rename", PinState::AlwaysLocal);
+        QVERIFY(fakeFolder.syncOnce());
+        QVERIFY(fakeFolder.currentLocalState().find("onlinerenamed2/file1rename"));
+        QCOMPARE(*vfs->pinState("onlinerenamed2/file1rename"), PinState::AlwaysLocal);
+
+        vfs->setPinState("onlinerenamed2", PinState::Unspecified);
+        vfs->setPinState("onlinerenamed2/file1rename", PinState::OnlineOnly);
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "onlinerenamed2/file1rename");
+
+        QCOMPARE(*vfs->pinState("onlinerenamed2/file1rename"), PinState::OnlineOnly);
+    }
+
+    void testIncompatiblePins()
+    {
+        FakeFolder fakeFolder{ FileInfo() };
+        auto vfs = setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        fakeFolder.remoteModifier().mkdir("local");
+        fakeFolder.remoteModifier().mkdir("online");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::NoRecurse);
+        setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::NoRecurse);
+
+        fakeFolder.localModifier().insert("local/file1");
+        fakeFolder.localModifier().insert("online/file1");
+        QVERIFY(fakeFolder.syncOnce());
+
+        markForDehydration(fakeFolder, "local/file1");
+        triggerDownload(fakeFolder, "online/file1");
+
+        // the sync sets the changed files pin states to unspecified
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_NONVIRTUAL(fakeFolder, "online/file1");
+        CFVERIFY_VIRTUAL(fakeFolder, "local/file1");
+
+        QCOMPARE(*vfs->pinState("online/file1"), PinState::Unspecified);
+        QCOMPARE(*vfs->pinState("local/file1"), PinState::Unspecified);
+
+        // no change on another sync
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_NONVIRTUAL(fakeFolder, "online/file1");
+        CFVERIFY_VIRTUAL(fakeFolder, "local/file1");
+    }
+
+    void testOpeningOnlineFileTriggersDownload_data()
+    {
+        QTest::addColumn<int>("errorKind");
+        QTest::newRow("no error") << static_cast<int>(NoError);
+        QTest::newRow("400") << 400;
+        QTest::newRow("401") << 401;
+        QTest::newRow("403") << 403;
+        QTest::newRow("404") << 404;
+        QTest::newRow("500") << 500;
+        QTest::newRow("503") << 503;
+        QTest::newRow("Timeout") << static_cast<int>(Timeout);
+    }
+
+    void testOpeningOnlineFileTriggersDownload()
+    {
+        QFETCH(int, errorKind);
+
+        FakeFolder fakeFolder{ FileInfo() };
+        setupVfs(fakeFolder);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        fakeFolder.remoteModifier().mkdir("online");
+        fakeFolder.remoteModifier().mkdir("online/sub");
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::Recurse);
+
+        fakeFolder.remoteModifier().insert("online/sub/file1", 10 * 1024 * 1024);
+        QVERIFY(fakeFolder.syncOnce());
+
+        CFVERIFY_VIRTUAL(fakeFolder, "online/sub/file1");
+
+        // Setup error case if needed
+        if (errorKind == Timeout) {
+            fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &req, QIODevice *) -> QNetworkReply * {
+                if (req.url().path().endsWith("online/sub/file1")) {
+                    return new FakeHangingReply(op, req, this);
+                }
+                return nullptr;
+            });
+        } else if (errorKind != NoError) {
+            fakeFolder.serverErrorPaths().append("online/sub/file1", errorKind);
+        }
+
+        // So the test that test timeout finishes fast
+        QScopedValueRollback<int> setHttpTimeout(AbstractNetworkJob::httpTimeout, errorKind == Timeout ? 1 : 10000);
+
+        // Simulate another process requesting the open
+        QEventLoop loop;
+        bool openResult = false;
+        bool readResult = false;
+        std::thread t([&] {
+            QFile file(fakeFolder.localPath() + "online/sub/file1");
+            openResult = file.open(QFile::ReadOnly);
+            readResult = !file.readAll().isEmpty();
+            file.close();
+            QMetaObject::invokeMethod(&loop, &QEventLoop::quit, Qt::QueuedConnection);
+        });
+        loop.exec();
+        t.join();
+
+        if (errorKind == NoError) {
+            CFVERIFY_NONVIRTUAL(fakeFolder, "online/sub/file1");
+        } else {
+            CFVERIFY_VIRTUAL(fakeFolder, "online/sub/file1");
+        }
+
+        // Nothing should change
+        ItemCompletedSpy completeSpy(fakeFolder);
+        QVERIFY(fakeFolder.syncOnce());
+        QVERIFY(completeSpy.isEmpty());
+
+        if (errorKind == NoError) {
+            CFVERIFY_NONVIRTUAL(fakeFolder, "online/sub/file1");
+        } else {
+            CFVERIFY_VIRTUAL(fakeFolder, "online/sub/file1");
+        }
+    }
+};
+
+QTEST_GUILESS_MAIN(TestSyncCfApi)
+#include "testsynccfapi.moc"