Start hooking up E2EE to the new discovery algorithm

Now we pull the encrypted metadata during the discovery which is a
better approach than before. This shall remove the need for some of the
deep propfinds we have been using so far. It already simplifies the code
a bit for the download scenario.

Signed-off-by: Kevin Ottens <kevin.ottens@nextcloud.com>

src/libsync/discovery.cpp

@@ -161,6 +161,11 @@ void ProcessDirectoryJob::process()
             }
         }
 
+        // On the server the path is mangled in case of E2EE
+        if (!e.serverEntry.e2eMangledName.isEmpty()) {
+            path._server = e.serverEntry.e2eMangledName;
+        }
+
         // If the filename starts with a . we consider it a hidden file
         // For windows, the hidden state is also discovered within the vio
         // local stat function.
@@ -306,7 +311,9 @@ void ProcessDirectoryJob::processFile(PathTuple path,
                               << " | perm: " << dbEntry._remotePerm << "//" << serverEntry.remotePerm
                               << " | fileid: " << dbEntry._fileId << "//" << serverEntry.fileId
                               << " | inode: " << dbEntry._inode << "/" << localEntry.inode << "/"
-                              << " | type: " << dbEntry._type << "/" << localEntry.type << "/" << (serverEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile);
+                              << " | type: " << dbEntry._type << "/" << localEntry.type << "/" << (serverEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile)
+                              << " | e2ee: " << dbEntry._isE2eEncrypted << "/" << serverEntry.isE2eEncrypted
+                              << " | e2eeMangledName: " << dbEntry._e2eMangledName << "/" << serverEntry.e2eMangledName;
 
     if (_discoveryData->isRenamed(path._original)) {
         qCDebug(lcDisco) << "Ignoring renamed";
@@ -391,6 +398,7 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(
     item->_etag = serverEntry.etag;
     item->_directDownloadUrl = serverEntry.directDownloadUrl;
     item->_directDownloadCookies = serverEntry.directDownloadCookies;
+    item->_encryptedFileName = serverEntry.e2eMangledName;
 
     // Check for missing server data
     {

src/libsync/discoveryphase.cpp

@@ -16,6 +16,8 @@
 #include "discovery.h"
 
 #include "account.h"
+#include "clientsideencryptionjobs.h"
+
 #include "common/asserts.h"
 #include "common/checksums.h"
 
@@ -332,6 +334,7 @@ DiscoverySingleDirectoryJob::DiscoverySingleDirectoryJob(const AccountPtr &accou
     , _ignoredFirst(false)
     , _isRootPath(false)
     , _isExternalStorage(false)
+    , _isE2eEncrypted(false)
 {
 }
 
@@ -356,6 +359,9 @@ void DiscoverySingleDirectoryJob::start()
         // Server older than 10.0 have performances issue if we ask for the share-types on every PROPFIND
         props << "http://owncloud.org/ns:share-types";
     }
+    if (_account->capabilities().clientSideEncryptionAvailable()) {
+        props << "http://nextcloud.org/ns:is-encrypted";
+    }
 
     lsColJob->setProperties(props);
 
@@ -416,6 +422,8 @@ static void propertyMapToRemoteInfo(const QMap<QString, QString> &map, RemoteInf
                 // Piggy back on the persmission field
                 result.remotePerm.setPermission(RemotePermissions::IsShared);
             }
+        } else if (property == "is-encrypted" && value == QStringLiteral("1")) {
+            result.isE2eEncrypted = true;
         }
     }
 }
@@ -437,6 +445,13 @@ void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(const QString &fi
                 _dataFingerprint = "[empty]";
             }
         }
+        if (map.contains("id")) {
+            _fileId = map.value("id").toUtf8();
+        }
+        if (map.contains("is-encrypted") && map.value("is-encrypted") == QStringLiteral("1")) {
+            _isE2eEncrypted = true;
+            Q_ASSERT(!_fileId.isEmpty());
+        }
     } else {
 
         RemoteInfo result;
@@ -483,6 +498,9 @@ void DiscoverySingleDirectoryJob::lsJobFinishedWithoutErrorSlot()
         emit finished(HttpError{ 0, _error });
         deleteLater();
         return;
+    } else if (_isE2eEncrypted) {
+        fetchE2eMetadata();
+        return;
     }
     emit etag(_firstEtag);
     emit finished(_results);
@@ -502,4 +520,57 @@ void DiscoverySingleDirectoryJob::lsJobFinishedWithErrorSlot(QNetworkReply *r)
     emit finished(HttpError{ httpCode, msg });
     deleteLater();
 }
+
+void DiscoverySingleDirectoryJob::fetchE2eMetadata()
+{
+    auto job = new GetMetadataApiJob(_account, _fileId);
+    connect(job, &GetMetadataApiJob::jsonReceived,
+            this, &DiscoverySingleDirectoryJob::metadataReceived);
+    connect(job, &GetMetadataApiJob::error,
+            this, &DiscoverySingleDirectoryJob::metadataError);
+    job->start();
+}
+
+void DiscoverySingleDirectoryJob::metadataReceived(const QJsonDocument &json, int statusCode)
+{
+    qCDebug(lcDiscovery) << "Metadata received, applying it to the result list";
+    Q_ASSERT(_subPath.startsWith('/'));
+
+    const auto metadata = FolderMetadata(_account, json.toJson(QJsonDocument::Compact), statusCode);
+    const auto encryptedFiles = metadata.files();
+
+    const auto findEncryptedFile = [=](const QString &name) {
+        const auto it = std::find_if(std::cbegin(encryptedFiles), std::cend(encryptedFiles), [=](const EncryptedFile &file) {
+            return file.encryptedFilename == name;
+        });
+        if (it == std::cend(encryptedFiles)) {
+            return Optional<EncryptedFile>();
+        } else {
+            return Optional<EncryptedFile>(*it);
+        }
+    };
+
+    std::transform(std::cbegin(_results), std::cend(_results), std::begin(_results), [=](const RemoteInfo &info) {
+        auto result = info;
+        const auto encryptedFileInfo = findEncryptedFile(result.name);
+        if (encryptedFileInfo) {
+            result.isE2eEncrypted = true;
+            result.e2eMangledName = _subPath.mid(1) + QLatin1Char('/') + result.name;
+            result.name = encryptedFileInfo->originalFilename;
+        }
+        return result;
+    });
+
+    emit etag(_firstEtag);
+    emit finished(_results);
+    deleteLater();
+}
+
+void DiscoverySingleDirectoryJob::metadataError(const QByteArray &fileId, int httpReturnCode)
+{
+    qCWarning(lcDiscovery) << "E2EE Metadata job error. Trying to proceed without it." << fileId << httpReturnCode;
+    emit etag(_firstEtag);
+    emit finished(_results);
+    deleteLater();
+}
 }

src/libsync/discoveryphase.h

@@ -56,6 +56,9 @@ struct RemoteInfo
     time_t modtime = 0;
     int64_t size = 0;
     bool isDirectory = false;
+    bool isE2eEncrypted = false;
+    QString e2eMangledName;
+
     bool isValid() const { return !name.isNull(); }
 
     QString directDownloadUrl;
@@ -130,11 +133,15 @@ private slots:
     void directoryListingIteratedSlot(const QString &, const QMap<QString, QString> &);
     void lsJobFinishedWithoutErrorSlot();
     void lsJobFinishedWithErrorSlot(QNetworkReply *);
+    void fetchE2eMetadata();
+    void metadataReceived(const QJsonDocument &json, int statusCode);
+    void metadataError(const QByteArray& fileId, int httpReturnCode);
 
 private:
     QVector<RemoteInfo> _results;
     QString _subPath;
     QString _firstEtag;
+    QByteArray _fileId;
     AccountPtr _account;
     // The first result is for the directory itself and need to be ignored.
     // This flag is true if it was already ignored.
@@ -143,6 +150,8 @@ private:
     bool _isRootPath;
     // If this directory is an external storage (The first item has 'M' in its permission)
     bool _isExternalStorage;
+    // If this directory is e2ee
+    bool _isE2eEncrypted;
     // If set, the discovery will finish with an error
     QString _error;
     QPointer<LsColJob> _lsColJob;

src/libsync/propagatedownloadencrypted.cpp

@@ -95,20 +95,10 @@ void PropagateDownloadEncrypted::checkFolderEncryptedMetadata(const QJsonDocumen
   auto meta = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact));
   const QVector<EncryptedFile> files = meta->files();
 
-  const QString encryptedFilename = _item->_instruction == CSYNC_INSTRUCTION_NEW ?
-              _item->_file.section(QLatin1Char('/'), -1) :
-              _item->_encryptedFileName.section(QLatin1Char('/'), -1);
+  const QString encryptedFilename = _item->_encryptedFileName.section(QLatin1Char('/'), -1);
   for (const EncryptedFile &file : files) {
     if (encryptedFilename == file.encryptedFilename) {
       _encryptedInfo = file;
-      if (_item->_encryptedFileName.isEmpty()) {
-        _item->_encryptedFileName = _item->_file;
-      }
-      if (!_localParentPath.isEmpty()) {
-          _item->_file = _localParentPath + QLatin1Char('/') + _encryptedInfo.originalFilename;
-      } else {
-          _item->_file = _encryptedInfo.originalFilename;
-      }
 
       qCDebug(lcPropagateDownloadEncrypted) << "Found matching encrypted metadata for file, starting download";
       emit folderStatusEncrypted();