VFS + E2EE VFS implicit hydration of E2EE files.

Signed-off-by: allexzander <blackslayer4@gmail.com>

src/common/constants.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) by Oleksandr Zolotov <alex@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#pragma once
+
+namespace OCC {
+namespace CommonConstants {
+    const qint32 e2EeTagSize = 16;
+};
+};

src/libsync/clientsideencryption.cpp

@@ -36,7 +36,8 @@
 #include <QRandomGenerator>
 
 #include <qt5keychain/keychain.h>
-#include "common/utility.h"
+#include <common/utility.h>
+#include <common/constants.h>
 
 #include "wordlist.h"
 
@@ -66,6 +67,8 @@ namespace {
     const char e2e_private[] = "_e2e-private";
     const char e2e_mnemonic[] = "_e2e-mnemonic";
 
+    const int blockSize = 1024;
+
     QList<QByteArray> oldCipherFormatSplit(const QByteArray &cipher)
     {
         const auto separator = QByteArrayLiteral("fA=="); // BASE64 encoded '|'
@@ -295,7 +298,7 @@ namespace {
     };
 
     QByteArray BIO2ByteArray(Bio &b) {
-        int pending = BIO_ctrl_pending(b);
+        int pending = static_cast<int>(BIO_ctrl_pending(b));
         QByteArray res(pending, '\0');
         BIO_read(b, unsignedData(res), pending);
         return res;
@@ -424,8 +427,8 @@ QByteArray encryptPrivateKey(
     clen += len;
 
     /* Get the tag */
-    QByteArray tag(16, '\0');
-    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, unsignedData(tag))) {
+    QByteArray tag(OCC::CommonConstants::e2EeTagSize, '\0');
+    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, OCC::CommonConstants::e2EeTagSize, unsignedData(tag))) {
         qCInfo(lcCse()) << "Error getting the tag";
         handleErrors();
     }
@@ -463,8 +466,8 @@ QByteArray decryptPrivateKey(const QByteArray& key, const QByteArray& data) {
     QByteArray cipherTXT = QByteArray::fromBase64(cipherTXT64);
     QByteArray iv = QByteArray::fromBase64(ivB64);
 
-    QByteArray tag = cipherTXT.right(16);
-    cipherTXT.chop(16);
+    QByteArray tag = cipherTXT.right(OCC::CommonConstants::e2EeTagSize);
+    cipherTXT.chop(OCC::CommonConstants::e2EeTagSize);
 
     // Init
     CipherCtx ctx;
@@ -493,7 +496,7 @@ QByteArray decryptPrivateKey(const QByteArray& key, const QByteArray& data) {
         return QByteArray();
     }
 
-    QByteArray ptext(cipherTXT.size() + 16, '\0');
+    QByteArray ptext(cipherTXT.size() + OCC::CommonConstants::e2EeTagSize, '\0');
     int plen = 0;
 
     /* Provide the message to be decrypted, and obtain the plaintext output.
@@ -553,8 +556,8 @@ QByteArray decryptStringSymmetric(const QByteArray& key, const QByteArray& data)
     QByteArray cipherTXT = QByteArray::fromBase64(cipherTXT64);
     QByteArray iv = QByteArray::fromBase64(ivB64);
 
-    QByteArray tag = cipherTXT.right(16);
-    cipherTXT.chop(16);
+    QByteArray tag = cipherTXT.right(OCC::CommonConstants::e2EeTagSize);
+    cipherTXT.chop(OCC::CommonConstants::e2EeTagSize);
 
     // Init
     CipherCtx ctx;
@@ -583,7 +586,7 @@ QByteArray decryptStringSymmetric(const QByteArray& key, const QByteArray& data)
         return QByteArray();
     }
 
-    QByteArray ptext(cipherTXT.size() + 16, '\0');
+    QByteArray ptext(cipherTXT.size() + OCC::CommonConstants::e2EeTagSize, '\0');
     int plen = 0;
 
     /* Provide the message to be decrypted, and obtain the plaintext output.
@@ -687,8 +690,8 @@ QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data)
     clen += len;
 
     /* Get the tag */
-    QByteArray tag(16, '\0');
-    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, unsignedData(tag))) {
+    QByteArray tag(OCC::CommonConstants::e2EeTagSize, '\0');
+    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, OCC::CommonConstants::e2EeTagSize, unsignedData(tag))) {
         qCInfo(lcCse()) << "Error getting the tag";
         handleErrors();
         return {};
@@ -753,7 +756,7 @@ QByteArray decryptStringAsymmetric(EVP_PKEY *privateKey, const QByteArray& data)
         qCInfo(lcCseDecryption()) << "Size of data is: " << data.size();
     }
 
-    QByteArray out(outlen, '\0');
+    QByteArray out(static_cast<int>(outlen), '\0');
 
     if (EVP_PKEY_decrypt(ctx, unsignedData(out), &outlen, (unsigned char *)data.constData(), data.size()) <= 0) {
         const auto error = handleErrors();
@@ -804,7 +807,7 @@ QByteArray encryptStringAsymmetric(EVP_PKEY *publicKey, const QByteArray& data)
         qCInfo(lcCse()) << "Encryption Length:" << outLen;
     }
 
-    QByteArray out(outLen, '\0');
+    QByteArray out(static_cast<int>(outLen), '\0');
     if (EVP_PKEY_encrypt(ctx, unsignedData(out), &outLen, (unsigned char *)data.constData(), data.size()) != 1) {
         qCInfo(lcCse()) << "Could not encrypt key." << err;
         exit(1);
@@ -1650,13 +1653,13 @@ bool EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i
         return false;
     }
 
-    QByteArray out(1024 + 16 - 1, '\0');
+    QByteArray out(blockSize + OCC::CommonConstants::e2EeTagSize - 1, '\0');
     int len = 0;
     int total_len = 0;
 
     qCDebug(lcCse) << "Starting to encrypt the file" << input->fileName() << input->atEnd();
     while(!input->atEnd()) {
-        QByteArray data = input->read(1024);
+        QByteArray data = input->read(blockSize);
 
         if (data.size() == 0) {
             qCInfo(lcCse()) << "Could not read data from file";
@@ -1681,14 +1684,14 @@ bool EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i
     total_len += len;
 
     /* Get the tag */
-    QByteArray tag(16, '\0');
-    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, unsignedData(tag))) {
+    QByteArray tag(OCC::CommonConstants::e2EeTagSize, '\0');
+    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, OCC::CommonConstants::e2EeTagSize, unsignedData(tag))) {
         qCInfo(lcCse()) << "Could not get tag";
         return false;
     }
 
     returnTag = tag;
-    output->write(tag, 16);
+    output->write(tag, OCC::CommonConstants::e2EeTagSize);
 
     input->close();
     output->close();
@@ -1731,16 +1734,16 @@ bool EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i
         return false;
     }
 
-    qint64 size = input->size() - 16;
+    qint64 size = input->size() - OCC::CommonConstants::e2EeTagSize;
 
-    QByteArray out(1024 + 16 - 1, '\0');
+    QByteArray out(blockSize + OCC::CommonConstants::e2EeTagSize - 1, '\0');
     int len = 0;
 
     while(input->pos() < size) {
 
         auto toRead = size - input->pos();
-        if (toRead > 1024) {
-            toRead = 1024;
+        if (toRead > blockSize) {
+            toRead = blockSize;
         }
 
         QByteArray data = input->read(toRead);
@@ -1758,7 +1761,7 @@ bool EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i
         output->write(out, len);
     }
 
-    QByteArray tag = input->read(16);
+    QByteArray tag = input->read(OCC::CommonConstants::e2EeTagSize);
 
     /* Set expected tag value. Works in OpenSSL 1.0.1d and later */
     if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, tag.size(), (unsigned char *)tag.constData())) {
@@ -1777,4 +1780,172 @@ bool EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i
     return true;
 }
 
+EncryptionHelper::StreamingDecryptor::StreamingDecryptor(const QByteArray &key, const QByteArray &iv, quint64 totalSize) : _totalSize(totalSize)
+{
+    if (_ctx && !key.isEmpty() && !iv.isEmpty() && totalSize > 0) {
+        _isInitialized = true;
+
+        /* Initialize the decryption operation. */
+        if(!EVP_DecryptInit_ex(_ctx, EVP_aes_128_gcm(), nullptr, nullptr, nullptr)) {
+            qCritical(lcCse()) << "Could not init cipher";
+            _isInitialized = false;
+        }
+
+        EVP_CIPHER_CTX_set_padding(_ctx, 0);
+
+        /* Set IV length. */
+        if(!EVP_CIPHER_CTX_ctrl(_ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), nullptr)) {
+            qCritical(lcCse()) << "Could not set iv length";
+            _isInitialized = false;
+        }
+
+        /* Initialize key and IV */
+        if(!EVP_DecryptInit_ex(_ctx, nullptr, nullptr, reinterpret_cast<const unsigned char*>(key.constData()), reinterpret_cast<const unsigned char*>(iv.constData()))) {
+            qCritical(lcCse()) << "Could not set key and iv";
+            _isInitialized = false;
+        }
+    }
+}
+
+qint32 EncryptionHelper::StreamingDecryptor::chunkDecryption(const char *input, QIODevice *output, quint32 chunkSize)
+{
+    Q_ASSERT(isInitialized());
+    if (!isInitialized()) {
+        qCritical(lcCse()) << "Decryption failed. Decryptor is not initialized!";
+        return -1;
+    }
+
+    Q_ASSERT(output && output->isOpen() && output->isWritable());
+    if (!output || !output->isOpen() || !output->isWritable()) {
+        qCritical(lcCse()) << "Decryption failed. Incorrect output device!";
+        return -1;
+    }
+
+    Q_ASSERT(input);
+    if (!input) {
+        qCritical(lcCse()) << "Decryption failed. Incorrect input!";
+        return -1;
+    }
+
+    Q_ASSERT(chunkSize > 0);
+    if (chunkSize <= 0) {
+        qCritical(lcCse()) << "Decryption failed. Incorrect chunkSize!";
+        return -1;
+    }
+
+    if (_decryptedSoFar == 0) {
+        qCDebug(lcCse()) << "Decryption started";
+    }
+
+    Q_ASSERT(_decryptedSoFar + chunkSize <= _totalSize);
+    if (_decryptedSoFar + chunkSize > _totalSize) {
+        qCritical(lcCse()) << "Decryption failed. Chunk is out of range!";
+        return -1;
+    }
+
+    const bool isLastChunk = _decryptedSoFar + chunkSize == _totalSize;
+
+    // last OCC::CommonConstants::e2EeTagSize bytes is ALWAYS a tag!!!
+    const qint32 size = isLastChunk ? static_cast<qint32>(chunkSize) - OCC::CommonConstants::e2EeTagSize : static_cast<qint32>(chunkSize);
+
+    Q_ASSERT(size > 0);
+    if (size <= 0) {
+        qCritical(lcCse()) << "Decryption failed. Invalid input size: " << size << " !";
+        return -1;
+    }
+
+    int bytesWritten = 0;
+
+    QByteArray decryptedBlock(blockSize + OCC::CommonConstants::e2EeTagSize - 1, '\0');
+    int inputPos = 0;
+
+    while(inputPos < size) {
+        // read blockSize or less bytes
+        const QByteArray encryptedBlock = QByteArray(input + inputPos, qMin(size - inputPos, blockSize));
+
+        if (encryptedBlock.size() == 0) {
+            qCritical(lcCse()) << "Could not read data from the input buffer.";
+            return -1;
+        }
+
+        int outLen = 0;
+
+        if(!EVP_DecryptUpdate(_ctx, unsignedData(decryptedBlock), &outLen, reinterpret_cast<const unsigned char*>(encryptedBlock.data()), encryptedBlock.size())) {
+            qCritical(lcCse()) << "Could not decrypt";
+            return -1;
+        }
+
+        const auto writtenToOutput = output->write(decryptedBlock, outLen);
+
+        Q_ASSERT(writtenToOutput == outLen);
+        if (writtenToOutput != outLen) {
+            qCritical(lcCse()) << "Failed to write decrypted data to device.";
+            return -1;
+        }
+
+        bytesWritten += writtenToOutput;
+
+        // advance input position for further read
+        inputPos += encryptedBlock.size();
+
+        _decryptedSoFar += encryptedBlock.size();
+    }
+
+    if (isLastChunk) {
+        // if it's a last chunk, we'd need to read a tag at the end and finalize the decryption
+
+        Q_ASSERT(chunkSize - inputPos == OCC::CommonConstants::e2EeTagSize);
+        if (chunkSize - inputPos != OCC::CommonConstants::e2EeTagSize) {
+            qCritical(lcCse()) << "Decryption failed. E2EE tag is missing!";
+            return -1;
+        }
+
+        int outLen = 0;
+
+        QByteArray tag = QByteArray(input + inputPos, OCC::CommonConstants::e2EeTagSize);
+
+        /* Set expected tag value. Works in OpenSSL 1.0.1d and later */
+        if(!EVP_CIPHER_CTX_ctrl(_ctx, EVP_CTRL_GCM_SET_TAG, tag.size(), reinterpret_cast<unsigned char*>(tag.data()))) {
+            qCritical(lcCse()) << "Could not set expected tag";
+            return -1;
+        }
+
+        if(1 != EVP_DecryptFinal_ex(_ctx, unsignedData(decryptedBlock), &outLen)) {
+            qCritical(lcCse()) << "Could finalize decryption";
+            return -1;
+        }
+
+        const auto writtenToOutput = output->write(decryptedBlock, outLen);
+
+        Q_ASSERT(writtenToOutput == outLen);
+        if (writtenToOutput != outLen) {
+            qCritical(lcCse()) << "Failed to write decrypted data to device.";
+            return -1;
+        }
+
+        bytesWritten += writtenToOutput;
+
+        _decryptedSoFar += OCC::CommonConstants::e2EeTagSize;
+
+        _isFinished = true;
+    }
+
+    // qCDebug(lcCse()) <<"Decrypting:" << _decryptedSoFar << "/" << _totalSize;
+
+    if (isFinished()) {
+        qCDebug(lcCse()) << "Decryption complete";
+    }
+
+    return bytesWritten;
+}
+
+bool EncryptionHelper::StreamingDecryptor::isInitialized() const
+{
+    return _isInitialized;
+}
+
+bool EncryptionHelper::StreamingDecryptor::isFinished() const
+{
+    return _isFinished;
 }
+};

src/libsync/clientsideencryption.h

@@ -27,7 +27,7 @@ QString baseUrl();
 
 namespace EncryptionHelper {
     QByteArray generateRandomFilename();
-    QByteArray generateRandom(int size);
+    OWNCLOUDSYNC_EXPORT QByteArray generateRandom(int size);
     QByteArray generatePassword(const QString &wordlist, const QByteArray& salt);
     OWNCLOUDSYNC_EXPORT QByteArray encryptPrivateKey(
             const QByteArray& key,
@@ -60,12 +60,59 @@ namespace EncryptionHelper {
             const QByteArray& data
     );
 
-    bool fileEncryption(const QByteArray &key, const QByteArray &iv,
+    OWNCLOUDSYNC_EXPORT bool fileEncryption(const QByteArray &key, const QByteArray &iv,
                       QFile *input, QFile *output, QByteArray& returnTag);
 
-    bool fileDecryption(const QByteArray &key, const QByteArray& iv,
+    OWNCLOUDSYNC_EXPORT bool fileDecryption(const QByteArray &key, const QByteArray &iv,
                                QFile *input, QFile *output);
-}
+
+    //
+    // Simple classes for safe (RAII) handling of OpenSSL
+    // data structures
+    //
+
+class CipherCtx {
+public:
+    CipherCtx() : _ctx(EVP_CIPHER_CTX_new())
+    {
+    }
+
+    ~CipherCtx()
+    {
+        EVP_CIPHER_CTX_free(_ctx);
+    }
+
+    operator EVP_CIPHER_CTX*()
+    {
+        return _ctx;
+    }
+
+private:
+    Q_DISABLE_COPY(CipherCtx)
+    EVP_CIPHER_CTX *_ctx;
+};
+
+class OWNCLOUDSYNC_EXPORT StreamingDecryptor
+{
+public:
+    StreamingDecryptor(const QByteArray &key, const QByteArray &iv, quint64 totalSize);
+    ~StreamingDecryptor() = default;
+
+    qint32 chunkDecryption(const char *input, QIODevice *output, quint32 chunkSize);
+
+    bool isInitialized() const;
+    bool isFinished() const;
+
+private:
+    Q_DISABLE_COPY(StreamingDecryptor)
+
+    CipherCtx _ctx;
+    bool _isInitialized = false;
+    bool _isFinished = false;
+    quint64 _decryptedSoFar = 0;
+    quint64 _totalSize = 0;
+};
+};
 
 class OWNCLOUDSYNC_EXPORT ClientSideEncryption : public QObject {
     Q_OBJECT

src/libsync/discovery.cpp

@@ -24,7 +24,8 @@
 #include <QFileInfo>
 #include <QFile>
 #include <QThreadPool>
-#include "common/checksums.h"
+#include <common/checksums.h>
+#include <common/constants.h>
 #include "csync_exclude.h"
 #include "csync.h"
 
@@ -459,13 +460,22 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(
 
     // The file is known in the db already
     if (dbEntry.isValid()) {
+        qint64 size = serverEntry.size;
+
+        if (dbEntry.isVirtualFile() && (!item->_encryptedFileName.isEmpty()) && size > 0) {
+            // make sure we set correct size when file was downloaded previously and has now been changed on the server
+            // serverEntry always includes extra CommonConstants::e2EeTagSize bytes for e2e encrypted files
+            // we don't need those neither when creating a placeholder nor when storing hydrated file on disk
+            size = serverEntry.size - CommonConstants::e2EeTagSize;
+        }
+
         if (serverEntry.isDirectory != dbEntry.isDirectory()) {
             // If the type of the entity changed, it's like NEW, but
             // needs to delete the other entity first.
             item->_instruction = CSYNC_INSTRUCTION_TYPE_CHANGE;
             item->_direction = SyncFileItem::Down;
             item->_modtime = serverEntry.modtime;
-            item->_size = serverEntry.size;
+            item->_size = size;
         } else if ((dbEntry._type == ItemTypeVirtualFileDownload || localEntry.type == ItemTypeVirtualFileDownload)
             && (localEntry.isValid() || _queryLocal == ParentNotChanged)) {
             // The above check for the localEntry existing is important. Otherwise it breaks
@@ -476,7 +486,7 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(
         } else if (dbEntry._etag != serverEntry.etag) {
             item->_direction = SyncFileItem::Down;
             item->_modtime = serverEntry.modtime;
-            item->_size = serverEntry.size;
+            item->_size = size;
             if (serverEntry.isDirectory) {
                 ENFORCE(dbEntry.isDirectory());
                 item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
@@ -490,6 +500,9 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(
             item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
             item->_direction = SyncFileItem::Down;
         } else {
+            // if (is virtual mode enabled and folder is encrypted - check if the size is the same as on the server and then - trigger server query
+            // to update a placeholder with corrected size (-16 Bytes)
+            // or, maybe, add a flag to the database - vfsE2eeSizeCorrected? if it is not set - subtract it from the placeholder's size and re-create/update a placeholder?
             processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, ParentNotChanged);
             return;
         }
@@ -528,6 +541,13 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(
             && _pinState != PinState::AlwaysLocal
             && !FileSystem::isExcludeFile(item->_file)) {
             item->_type = ItemTypeVirtualFile;
+            if (!item->_encryptedFileName.isEmpty()) {
+                // We are syncing a file for the first time (local entry is invalid) and it is encrypted file that will be virtual once synced
+                // to avoid having error of "file has changed during sync" when trying to hydrate it excplicitly - we must remove CommonConstants::e2EeTagSize bytes from the end
+                // as explicit hydration does not care if these bytes are present in the placeholder or not, but, the size must not change in the middle of the sync
+                // this way it works for both implicit and explicit hydration by making a placeholder size that does not includes encryption tag CommonConstants::e2EeTagSize bytes
+                item->_size = serverEntry.size - CommonConstants::e2EeTagSize;
+            }
             if (isVfsWithSuffix())
                 addVirtualFileSuffix(tmp_path._original);
         }

src/libsync/propagatedownload.cpp

@@ -73,7 +73,6 @@ GETFileJob::GETFileJob(AccountPtr account, const QString &path, QIODevice *devic
     const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
     qint64 resumeStart, QObject *parent)
     : AbstractNetworkJob(account, path, parent)
-    , _device(device)
     , _headers(headers)
     , _expectedEtagForResume(expectedEtagForResume)
     , _expectedContentLength(-1)
@@ -86,6 +85,7 @@ GETFileJob::GETFileJob(AccountPtr account, const QString &path, QIODevice *devic
     , _bandwidthManager(nullptr)
     , _hasEmittedFinishedSignal(false)
     , _lastModified()
+    , _device(device)
 {
 }
 
@@ -93,7 +93,6 @@ GETFileJob::GETFileJob(AccountPtr account, const QUrl &url, QIODevice *device,
     const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
     qint64 resumeStart, QObject *parent)
     : AbstractNetworkJob(account, url.toEncoded(), parent)
-    , _device(device)
     , _headers(headers)
     , _expectedEtagForResume(expectedEtagForResume)
     , _expectedContentLength(-1)
@@ -107,6 +106,7 @@ GETFileJob::GETFileJob(AccountPtr account, const QUrl &url, QIODevice *device,
     , _bandwidthManager(nullptr)
     , _hasEmittedFinishedSignal(false)
     , _lastModified()
+    , _device(device)
 {
 }
 
@@ -250,6 +250,8 @@ void GETFileJob::slotMetaDataChanged()
     }
 
     _saveBodyToFile = true;
+
+    processMetaData();
 }
 
 void GETFileJob::setBandwidthManager(BandwidthManager *bwm)
@@ -284,6 +286,11 @@ qint64 GETFileJob::currentDownloadPosition()
     return _resumeStart;
 }
 
+qint64 GETFileJob::writeToDevice(const char *data, qint64 len)
+{
+    return _device->write(data, len);
+}
+
 void GETFileJob::slotReadyRead()
 {
     if (!reply())
@@ -306,7 +313,7 @@ void GETFileJob::slotReadyRead()
             _bandwidthQuota -= toRead;
         }
 
-        qint64 r = reply()->read(buffer.data(), toRead);
+        const qint64 r = reply()->read(buffer.data(), toRead);
         if (r < 0) {
             _errorString = networkReplyErrorString(*reply());
             _errorStatus = SyncFileItem::NormalError;
@@ -315,7 +322,7 @@ void GETFileJob::slotReadyRead()
             return;
         }
 
-        qint64 w = _device->write(buffer.constData(), r);
+        const qint64 w = writeToDevice(buffer.constData(), r);
         if (w != r) {
             _errorString = _device->errorString();
             _errorStatus = SyncFileItem::NormalError;
@@ -371,6 +378,47 @@ QString GETFileJob::errorString() const
     return AbstractNetworkJob::errorString();
 }
 
+GETEncryptedFileJob::GETEncryptedFileJob(AccountPtr account, const QString &path, QIODevice *device,
+    const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
+    qint64 resumeStart, EncryptedFile encryptedInfo, QObject *parent)
+    : GETFileJob(account, path, device, headers, expectedEtagForResume, resumeStart, parent)
+    , _encryptedFileInfo(encryptedInfo)
+{
+}
+
+GETEncryptedFileJob::GETEncryptedFileJob(AccountPtr account, const QUrl &url, QIODevice *device,
+    const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
+    qint64 resumeStart, EncryptedFile encryptedInfo, QObject *parent)
+    : GETFileJob(account, url, device, headers, expectedEtagForResume, resumeStart, parent)
+    , _encryptedFileInfo(encryptedInfo)
+{
+}
+
+qint64 GETEncryptedFileJob::writeToDevice(const char *data, qint64 len)
+{
+    if (!_decryptor->isInitialized()) {
+        return -1;
+    }
+
+    const auto bytesDecrypted = _decryptor->chunkDecryption(data, _device, len);
+
+    if (bytesDecrypted == -1) {
+        qCCritical(lcPropagateDownload) << "Decryption failed!";
+        return -1;
+    }
+
+    return len;
+}
+
+void GETEncryptedFileJob::processMetaData()
+{
+    if (!_decryptor) {
+        // only initialize the decryptor once, because, according to Qt documentation, metadata might get changed during the processing of the data sometimes
+        // https://doc.qt.io/qt-5/qnetworkreply.html#metaDataChanged
+        _decryptor.reset(new EncryptionHelper::StreamingDecryptor(_encryptedFileInfo.encryptionKey, _encryptedFileInfo.initializationVector, _contentLength));
+    }
+}
+
 void PropagateDownloadFile::start()
 {
     if (propagator()->_abortRequested)

src/libsync/propagatedownload.h

@@ -31,12 +31,10 @@ class PropagateDownloadEncrypted;
 class OWNCLOUDSYNC_EXPORT GETFileJob : public AbstractNetworkJob
 {
     Q_OBJECT
-    QIODevice *_device;
     QMap<QByteArray, QByteArray> _headers;
     QString _errorString;
     QByteArray _expectedEtagForResume;
     qint64 _expectedContentLength;
-    qint64 _contentLength;
     qint64 _resumeStart;
     SyncFileItem::Status _errorStatus;
     QUrl _directDownloadUrl;
@@ -51,6 +49,10 @@ class OWNCLOUDSYNC_EXPORT GETFileJob : public AbstractNetworkJob
     /// Will be set to true once we've seen a 2xx response header
     bool _saveBodyToFile = false;
 
+protected:
+    QIODevice *_device;
+    qint64 _contentLength;
+
 public:
     // DOES NOT take ownership of the device.
     explicit GETFileJob(AccountPtr account, const QString &path, QIODevice *device,
@@ -110,6 +112,10 @@ public:
     qint64 expectedContentLength() const { return _expectedContentLength; }
     void setExpectedContentLength(qint64 size) { _expectedContentLength = size; }
 
+protected:
+    virtual qint64 writeToDevice(const char *data, qint64 len);
+    virtual void processMetaData() {}
+
 signals:
     void finishedSignal();
     void downloadProgress(qint64, qint64);
@@ -118,6 +124,33 @@ private slots:
     void slotMetaDataChanged();
 };
 
+/**
+ * @brief The GETEncryptedFileJob class that provides file decryption on the fly while the download is running
+ * @ingroup libsync
+ */
+class OWNCLOUDSYNC_EXPORT GETEncryptedFileJob : public GETFileJob
+{
+    Q_OBJECT
+
+public:
+    // DOES NOT take ownership of the device.
+    explicit GETEncryptedFileJob(AccountPtr account, const QString &path, QIODevice *device,
+        const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
+        qint64 resumeStart, EncryptedFile encryptedInfo, QObject *parent = nullptr);
+    explicit GETEncryptedFileJob(AccountPtr account, const QUrl &url, QIODevice *device,
+        const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
+        qint64 resumeStart, EncryptedFile encryptedInfo, QObject *parent = nullptr);
+    virtual ~GETEncryptedFileJob() = default;
+
+protected:
+    virtual qint64 writeToDevice(const char *data, qint64 len) override;
+    virtual void processMetaData() override;
+
+private:
+    QSharedPointer<EncryptionHelper::StreamingDecryptor> _decryptor;
+    EncryptedFile _encryptedFileInfo = {};
+};
+
 /**
  * @brief The PropagateDownloadFile class
  * @ingroup libsync
@@ -219,6 +252,6 @@ private:
 
     QElapsedTimer _stopwatch;
 
-    PropagateDownloadEncrypted *_downloadEncryptedHelper;
+    PropagateDownloadEncrypted *_downloadEncryptedHelper = nullptr;
 };
 }

src/libsync/vfs/cfapi/hydrationjob.cpp

@@ -17,6 +17,9 @@
 #include "common/syncjournaldb.h"
 #include "propagatedownload.h"
 #include "vfs/cfapi/vfs_cfapi.h"
+#include <clientsideencryptionjobs.h>
+
+#include "filesystem.h"
 
 #include <QLocalServer>
 #include <QLocalSocket>
@@ -88,6 +91,26 @@ void OCC::HydrationJob::setFolderPath(const QString &folderPath)
     _folderPath = folderPath;
 }
 
+bool OCC::HydrationJob::isEncryptedFile() const
+{
+    return _isEncryptedFile;
+}
+
+void OCC::HydrationJob::setIsEncryptedFile(bool isEncrypted)
+{
+    _isEncryptedFile = isEncrypted;
+}
+
+QString OCC::HydrationJob::encryptedFileName() const
+{
+    return _encryptedFileName;
+}
+
+void OCC::HydrationJob::setEncryptedFileName(const QString &encryptedName)
+{
+    _encryptedFileName = encryptedName;
+}
+
 OCC::HydrationJob::Status OCC::HydrationJob::status() const
 {
     return _status;
@@ -137,6 +160,70 @@ void OCC::HydrationJob::start()
     connect(_transferDataServer, &QLocalServer::newConnection, this, &HydrationJob::onNewConnection);
 }
 
+void OCC::HydrationJob::slotFolderIdError()
+{
+    // TODO: the following code is borrowed from PropagateDownloadEncrypted (see HydrationJob::onNewConnection() for explanation of next steps)
+    qCCritical(lcHydration) << "Failed to get encrypted metadata of folder" << _requestId << _localPath << _folderPath;
+    emitFinished(Error);
+}
+
+void OCC::HydrationJob::slotCheckFolderId(const QStringList &list)
+{
+    // TODO: the following code is borrowed from PropagateDownloadEncrypted (see HydrationJob::onNewConnection() for explanation of next steps)
+    auto job = qobject_cast<LsColJob *>(sender());
+    const QString folderId = list.first();
+    qCDebug(lcHydration) << "Received id of folder" << folderId;
+
+    const ExtraFolderInfo &folderInfo = job->_folderInfos.value(folderId);
+
+    // Now that we have the folder-id we need it's JSON metadata
+    auto metadataJob = new GetMetadataApiJob(_account, folderInfo.fileId);
+    connect(metadataJob, &GetMetadataApiJob::jsonReceived,
+        this, &HydrationJob::slotCheckFolderEncryptedMetadata);
+    connect(metadataJob, &GetMetadataApiJob::error,
+        this, &HydrationJob::slotFolderEncryptedMetadataError);
+
+    metadataJob->start();
+}
+
+void OCC::HydrationJob::slotFolderEncryptedMetadataError(const QByteArray & /*fileId*/, int /*httpReturnCode*/)
+{
+    // TODO: the following code is borrowed from PropagateDownloadEncrypted (see HydrationJob::onNewConnection() for explanation of next steps)
+    qCCritical(lcHydration) << "Failed to find encrypted metadata information of remote file" << encryptedFileName();
+    emitFinished(Error);
+    return;
+}
+
+void OCC::HydrationJob::slotCheckFolderEncryptedMetadata(const QJsonDocument &json)
+{
+    // TODO: the following code is borrowed from PropagateDownloadEncrypted (see HydrationJob::onNewConnection() for explanation of next steps)
+    qCDebug(lcHydration) << "Metadata Received reading" << encryptedFileName();
+    const QString filename = encryptedFileName();
+    auto meta = new FolderMetadata(_account, json.toJson(QJsonDocument::Compact));
+    const QVector<EncryptedFile> files = meta->files();
+
+    EncryptedFile encryptedInfo = {};
+
+    const QString encryptedFileExactName = encryptedFileName().section(QLatin1Char('/'), -1);
+    for (const EncryptedFile &file : files) {
+        if (encryptedFileExactName == file.encryptedFilename) {
+            EncryptedFile encryptedInfo = file;
+            encryptedInfo = file;
+
+            qCDebug(lcHydration) << "Found matching encrypted metadata for file, starting download" << _requestId << _folderPath;
+            _transferDataSocket = _transferDataServer->nextPendingConnection();
+            _job = new GETEncryptedFileJob(_account, _remotePath + encryptedFileName(), _transferDataSocket, {}, {}, 0, encryptedInfo, this);
+
+            connect(qobject_cast<GETEncryptedFileJob *>(_job), &GETEncryptedFileJob::finishedSignal, this, &HydrationJob::onGetFinished);
+            _job->start();
+            return;
+        }
+    }
+
+    qCCritical(lcHydration) << "Failed to find encrypted metadata information of a remote file" << filename;
+    emitFinished(Error);
+}
+
 void OCC::HydrationJob::cancel()
 {
     Q_ASSERT(_signalSocket);
@@ -184,11 +271,36 @@ void OCC::HydrationJob::onNewConnection()
     Q_ASSERT(!_transferDataSocket);
     Q_ASSERT(!_job);
 
-    qCInfo(lcHydration) << "Got new connection starting GETFileJob" << _requestId << _folderPath;
-    _transferDataSocket = _transferDataServer->nextPendingConnection();
-    _job = new GETFileJob(_account, _remotePath + _folderPath, _transferDataSocket, {}, {}, 0, this);
-    connect(_job, &GETFileJob::finishedSignal, this, &HydrationJob::onGetFinished);
-    _job->start();
+    if (isEncryptedFile()) {
+        // TODO: the following code is borrowed from PropagateDownloadEncrypted (should we factor it out and reuse? YES! Should we do it now? Probably not, as, this would imply modifying PropagateDownloadEncrypted, so we need a separate PR)
+        qCInfo(lcHydration) << "Got new connection for encrypted file. Getting required info for decryption...";
+        const auto rootPath = [=]() {
+            const auto result = _remotePath;
+            if (result.startsWith('/')) {
+                return result.mid(1);
+            } else {
+                return result;
+            }
+        }();
+
+        const auto remoteFilename = encryptedFileName();
+        const auto remotePath = QString(rootPath + remoteFilename);
+        const auto remoteParentPath = remotePath.left(remotePath.lastIndexOf('/'));
+
+        auto job = new LsColJob(_account, remoteParentPath, this);
+        job->setProperties({ "resourcetype", "http://owncloud.org/ns:fileid" });
+        connect(job, &LsColJob::directoryListingSubfolders,
+            this, &HydrationJob::slotCheckFolderId);
+        connect(job, &LsColJob::finishedWithError,
+            this, &HydrationJob::slotFolderIdError);
+        job->start();
+    } else {
+        qCInfo(lcHydration) << "Got new connection starting GETFileJob" << _requestId << _folderPath;
+        _transferDataSocket = _transferDataServer->nextPendingConnection();
+        _job = new GETFileJob(_account, _remotePath + _folderPath, _transferDataSocket, {}, {}, 0, this);
+        connect(_job, &GETFileJob::finishedSignal, this, &HydrationJob::onGetFinished);
+        _job->start();
+    }
 }
 
 void OCC::HydrationJob::finalize(OCC::VfsCfApi *vfs)
@@ -214,6 +326,9 @@ void OCC::HydrationJob::finalize(OCC::VfsCfApi *vfs)
     }
 
     record._type = ItemTypeFile;
+    // store the actual size of a file that has been decrypted as we will need its actual size when dehydrating it if requested
+    record._fileSize = FileSystem::getSize(localPath() + folderPath());
+
     _journal->setFileRecord(record);
 }
 

src/libsync/vfs/cfapi/hydrationjob.h

@@ -25,6 +25,10 @@ class GETFileJob;
 class SyncJournalDb;
 class VfsCfApi;
 
+namespace EncryptionHelper {
+    class StreamingDecryptor;
+};
+
 class HydrationJob : public QObject
 {
     Q_OBJECT
@@ -56,12 +60,27 @@ public:
     QString folderPath() const;
     void setFolderPath(const QString &folderPath);
 
+    bool isEncryptedFile() const;
+    void setIsEncryptedFile(bool isEncrypted);
+
+    QString encryptedFileName() const;
+    void setEncryptedFileName(const QString &encryptedName);
+
+    qint64 fileTotalSize() const;
+    void setFileTotalSize(qint64 totalSize);
+
     Status status() const;
 
     void start();
     void cancel();
     void finalize(OCC::VfsCfApi *vfs);
 
+public slots:
+    void slotCheckFolderId(const QStringList &list);
+    void slotFolderIdError();
+    void slotCheckFolderEncryptedMetadata(const QJsonDocument &json);
+    void slotFolderEncryptedMetadataError(const QByteArray &fileId, int httpReturnCode);
+
 signals:
     void finished(HydrationJob *job);
 
@@ -72,6 +91,8 @@ private:
     void onCancellationServerNewConnection();
     void onGetFinished();
 
+    void startServerAndWaitForConnections();
+
     AccountPtr _account;
     QString _remotePath;
     QString _localPath;
@@ -81,6 +102,9 @@ private:
     QString _requestId;
     QString _folderPath;
 
+    bool _isEncryptedFile = false;
+    QString _encryptedFileName;
+
     QLocalServer *_transferDataServer = nullptr;
     QLocalServer *_signalServer = nullptr;
     QLocalSocket *_transferDataSocket = nullptr;

src/libsync/vfs/cfapi/vfs_cfapi.cpp

@@ -16,7 +16,6 @@
 
 #include <QDir>
 #include <QFile>
-#include <QMessageBox>
 
 #include "cfapiwrapper.h"
 #include "hydrationjob.h"
@@ -323,28 +322,8 @@ void VfsCfApi::requestHydration(const QString &requestId, const QString &path)
         return;
     }
 
-    // This is impossible to handle with CfAPI since the file size is generally different
-    // between the encrypted and the decrypted file which would make CfAPI reject the hydration
-    // of the placeholder with decrypted data
-    if (record._isE2eEncrypted || !record._e2eMangledName.isEmpty()) {
-        qCInfo(lcCfApi) << "Couldn't hydrate, the file is E2EE this is not supported";
-
-        QMessageBox e2eeFileDownloadRequestWarningMsgBox;
-        e2eeFileDownloadRequestWarningMsgBox.setText(tr("Download of end-to-end encrypted file failed"));
-        e2eeFileDownloadRequestWarningMsgBox.setInformativeText(tr("It seems that you are trying to download a virtual file that"
-                                                                   " is end-to-end encrypted. Implicitly downloading such files is not"
-                                                                   " supported at the moment. To workaround this issue, go to the"
-                                                                   " settings and mark the encrypted folder with \"Make always available"
-                                                                   " locally\"."));
-        e2eeFileDownloadRequestWarningMsgBox.setIcon(QMessageBox::Warning);
-        e2eeFileDownloadRequestWarningMsgBox.exec();
-
-        emit hydrationRequestFailed(requestId);
-        return;
-    }
-
     // All good, let's hydrate now
-    scheduleHydrationJob(requestId, relativePath);
+    scheduleHydrationJob(requestId, relativePath, record);
 }
 
 void VfsCfApi::fileStatusChanged(const QString &systemFileName, SyncFileStatus fileStatus)
@@ -353,7 +332,7 @@ void VfsCfApi::fileStatusChanged(const QString &systemFileName, SyncFileStatus f
     Q_UNUSED(fileStatus);
 }
 
-void VfsCfApi::scheduleHydrationJob(const QString &requestId, const QString &folderPath)
+void VfsCfApi::scheduleHydrationJob(const QString &requestId, const QString &folderPath, const SyncJournalFileRecord &record)
 {
     const auto jobAlreadyScheduled = std::any_of(std::cbegin(d->hydrationJobs), std::cend(d->hydrationJobs), [=](HydrationJob *job) {
         return job->requestId() == requestId || job->folderPath() == folderPath;
@@ -376,6 +355,8 @@ void VfsCfApi::scheduleHydrationJob(const QString &requestId, const QString &fol
     job->setJournal(params().journal);
     job->setRequestId(requestId);
     job->setFolderPath(folderPath);
+    job->setIsEncryptedFile(!record._e2eMangledName.isEmpty());
+    job->setEncryptedFileName(record._e2eMangledName);
     connect(job, &HydrationJob::finished, this, &VfsCfApi::onHydrationJobFinished);
     d->hydrationJobs << job;
     job->start();

src/libsync/vfs/cfapi/vfs_cfapi.h

@@ -22,6 +22,7 @@
 namespace OCC {
 class HydrationJob;
 class VfsCfApiPrivate;
+class SyncJournalFileRecord;
 
 class VfsCfApi : public Vfs
 {
@@ -71,7 +72,7 @@ protected:
     void startImpl(const VfsSetupParams &params) override;
 
 private:
-    void scheduleHydrationJob(const QString &requestId, const QString &folderPath);
+    void scheduleHydrationJob(const QString &requestId, const QString &folderPath, const SyncJournalFileRecord &record);
     void onHydrationJobFinished(HydrationJob *job);
     HydrationJob *findHydrationJob(const QString &requestId) const;
 

test/testclientsideencryption.cpp

@@ -6,6 +6,9 @@
 
 #include <QtTest>
 
+#include <QTemporaryFile>
+#include <QRandomGenerator>
+
 #include "clientsideencryption.h"
 
 using namespace OCC;
@@ -132,6 +135,79 @@ private slots:
         // THEN
         QCOMPARE(data, originalData);
     }
+
+    void testStreamingDecryptor()
+    {
+        QTemporaryFile dummyInputFile;
+
+        QVERIFY(dummyInputFile.open());
+
+        const auto dummyFileRandomContents = EncryptionHelper::generateRandom(272);
+
+        QCOMPARE(dummyInputFile.write(dummyFileRandomContents), dummyFileRandomContents.size());
+
+        const auto generateHash = [](const QByteArray &data) {
+            QCryptographicHash hash(QCryptographicHash::Sha1);
+            hash.addData(data);
+            return hash.result();
+        };
+
+        const QByteArray originalFileHash = generateHash(dummyFileRandomContents);
+
+        QVERIFY(!originalFileHash.isEmpty());
+
+        dummyInputFile.close();
+        QVERIFY(!dummyInputFile.isOpen());
+
+        const auto encryptionKey = EncryptionHelper::generateRandom(16);
+        const auto initializationVector = EncryptionHelper::generateRandom(16);
+
+        // test normal file encryption/decryption
+        QTemporaryFile dummyEncryptionOutputFile;
+
+        QByteArray tag;
+
+        QVERIFY(EncryptionHelper::fileEncryption(encryptionKey, initializationVector, &dummyInputFile, &dummyEncryptionOutputFile, tag));
+        dummyInputFile.close();
+        QVERIFY(!dummyInputFile.isOpen());
+
+        dummyEncryptionOutputFile.close();
+        QVERIFY(!dummyEncryptionOutputFile.isOpen());
+
+        QTemporaryFile dummyDecryptionOutputFile;
+
+        QVERIFY(EncryptionHelper::fileDecryption(encryptionKey, initializationVector, &dummyEncryptionOutputFile, &dummyDecryptionOutputFile));
+        QVERIFY(dummyDecryptionOutputFile.open());
+        const auto dummyDecryptionOutputFileHash = generateHash(dummyDecryptionOutputFile.readAll());
+        QCOMPARE(dummyDecryptionOutputFileHash, originalFileHash);
+
+        // test streaming decryptor
+        EncryptionHelper::StreamingDecryptor streamingDecryptor(encryptionKey, initializationVector, dummyEncryptionOutputFile.size());
+        QVERIFY(streamingDecryptor.isInitialized());
+
+        QBuffer chunkedOutputDecrypted;
+        QVERIFY(chunkedOutputDecrypted.open(QBuffer::WriteOnly));
+
+        QVERIFY(dummyEncryptionOutputFile.open());
+
+        const auto readBytesAvailable = dummyEncryptionOutputFile.bytesAvailable();
+        while (dummyEncryptionOutputFile.pos() < readBytesAvailable) {
+            // auto toRead = QRandomGenerator::global()->bounded(8, 128);
+            // decryption is going to fail if last chunk does not include or does not equal to 16-bytes tag (accumulation required? - but where? in the caller's code?)
+            auto toRead = 64;
+            if (dummyEncryptionOutputFile.pos() + toRead > readBytesAvailable) {
+                toRead = readBytesAvailable - dummyEncryptionOutputFile.pos();
+            }
+            const auto decryptedBytes = streamingDecryptor.chunkDecryption(dummyEncryptionOutputFile.read(toRead).constData(), &chunkedOutputDecrypted, toRead);
+            QVERIFY(decryptedBytes != -1);
+            QVERIFY(decryptedBytes == toRead || streamingDecryptor.isFinished());
+        }
+        chunkedOutputDecrypted.close();
+
+        QVERIFY(chunkedOutputDecrypted.open(QBuffer::ReadOnly));
+        QCOMPARE(generateHash(chunkedOutputDecrypted.readAll()), originalFileHash);
+        chunkedOutputDecrypted.close();
+    }
 };
 
 QTEST_APPLESS_MAIN(TestClientSideEncryption)