Merge pull request #4100 from nextcloud/feature/ask-server-to-recalc-checksum

Ask server to recalculate checksum(hash)

src/common/checksums.cpp

@@ -337,7 +337,7 @@ ComputeChecksum *ValidateChecksumHeader::prepareStart(const QByteArray &checksum
 
     if (!parseChecksumHeader(checksumHeader, &_expectedChecksumType, &_expectedChecksum)) {
         qCWarning(lcChecksums) << "Checksum header malformed:" << checksumHeader;
-        emit validationFailed(tr("The checksum header is malformed."));
+        emit validationFailed(tr("The checksum header is malformed."), _calculatedChecksumType, _calculatedChecksum, ChecksumHeaderMalformed);
         return nullptr;
     }
 
@@ -360,15 +360,30 @@ void ValidateChecksumHeader::start(std::unique_ptr<QIODevice> device, const QByt
         calculator->start(std::move(device));
 }
 
+QByteArray ValidateChecksumHeader::calculatedChecksumType() const
+{
+    return _calculatedChecksumType;
+}
+
+QByteArray ValidateChecksumHeader::calculatedChecksum() const
+{
+    return _calculatedChecksum;
+}
+
 void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray &checksumType,
     const QByteArray &checksum)
 {
+    _calculatedChecksumType = checksumType;
+    _calculatedChecksum = checksum;
+
     if (checksumType != _expectedChecksumType) {
-        emit validationFailed(tr("The checksum header contained an unknown checksum type \"%1\"").arg(QString::fromLatin1(_expectedChecksumType)));
+        emit validationFailed(tr("The checksum header contained an unknown checksum type \"%1\"").arg(QString::fromLatin1(_expectedChecksumType)),
+            _calculatedChecksumType, _calculatedChecksum, ChecksumTypeUnknown);
         return;
     }
     if (checksum != _expectedChecksum) {
-        emit validationFailed(tr(R"(The downloaded file does not match the checksum, it will be resumed. "%1" != "%2")").arg(QString::fromUtf8(_expectedChecksum), QString::fromUtf8(checksum)));
+        emit validationFailed(tr(R"(The downloaded file does not match the checksum, it will be resumed. "%1" != "%2")").arg(QString::fromUtf8(_expectedChecksum), QString::fromUtf8(checksum)),
+            _calculatedChecksumType, _calculatedChecksum, ChecksumMismatch);
         return;
     }
     emit validated(checksumType, checksum);

src/common/checksums.h

@@ -140,6 +140,14 @@ class OCSYNC_EXPORT ValidateChecksumHeader : public QObject
 {
     Q_OBJECT
 public:
+    enum FailureReason {
+        Success,
+        ChecksumHeaderMalformed,
+        ChecksumTypeUnknown,
+        ChecksumMismatch,
+    };
+    Q_ENUM(FailureReason)
+
     explicit ValidateChecksumHeader(QObject *parent = nullptr);
 
     /**
@@ -161,9 +169,13 @@ public:
      */
     void start(std::unique_ptr<QIODevice> device, const QByteArray &checksumHeader);
 
+    QByteArray calculatedChecksumType() const;
+    QByteArray calculatedChecksum() const;
+
 signals:
     void validated(const QByteArray &checksumType, const QByteArray &checksum);
-    void validationFailed(const QString &errMsg);
+    void validationFailed(const QString &errMsg, const QByteArray &calculatedChecksumType,
+        const QByteArray &calculatedChecksum, const ValidateChecksumHeader::FailureReason reason);
 
 private slots:
     void slotChecksumCalculated(const QByteArray &checksumType, const QByteArray &checksum);
@@ -173,6 +185,9 @@ private:
 
     QByteArray _expectedChecksumType;
     QByteArray _expectedChecksum;
+
+    QByteArray _calculatedChecksumType;
+    QByteArray _calculatedChecksum;
 };
 
 /**

src/libsync/account.cpp

@@ -57,7 +57,8 @@ using namespace QKeychain;
 
 namespace {
 constexpr int pushNotificationsReconnectInterval = 1000 * 60 * 2;
-constexpr int usernamePrefillServerVersinMinSupportedMajor = 24;
+constexpr int usernamePrefillServerVersionMinSupportedMajor = 24;
+constexpr int checksumRecalculateRequestServerVersionMinSupportedMajor = 24;
 }
 
 namespace OCC {
@@ -632,7 +633,17 @@ bool Account::serverVersionUnsupported() const
 
 bool Account::isUsernamePrefillSupported() const
 {
-    return serverVersionInt() >= makeServerVersion(usernamePrefillServerVersinMinSupportedMajor, 0, 0);
+    return serverVersionInt() >= makeServerVersion(usernamePrefillServerVersionMinSupportedMajor, 0, 0);
+}
+
+bool Account::isChecksumRecalculateRequestSupported() const
+{
+    return serverVersionInt() >= makeServerVersion(checksumRecalculateRequestServerVersionMinSupportedMajor, 0, 0);
+}
+
+int Account::checksumRecalculateServerVersionMinSupportedMajor() const
+{
+    return checksumRecalculateRequestServerVersionMinSupportedMajor;
 }
 
 void Account::setServerVersion(const QString &version)

src/libsync/account.h

@@ -232,6 +232,10 @@ public:
 
     bool isUsernamePrefillSupported() const;
 
+    bool isChecksumRecalculateRequestSupported() const;
+
+    int checksumRecalculateServerVersionMinSupportedMajor() const;
+
     /** True when the server connection is using HTTP2  */
     bool isHttp2Supported() { return _http2Supported; }
     void setHttp2Supported(bool value) { _http2Supported = value; }

src/libsync/deletejob.cpp

@@ -21,12 +21,12 @@ namespace OCC {
 Q_LOGGING_CATEGORY(lcDeleteJob, "nextcloud.sync.networkjob.delete", QtInfoMsg)
 
 DeleteJob::DeleteJob(AccountPtr account, const QString &path, QObject *parent)
-    : AbstractNetworkJob(account, path, parent)
+    : SimpleFileJob(account, path, parent)
 {
 }
 
 DeleteJob::DeleteJob(AccountPtr account, const QUrl &url, QObject *parent)
-    : AbstractNetworkJob(account, QString(), parent)
+    : SimpleFileJob(account, QString(), parent)
     , _url(url)
 {
 }
@@ -39,24 +39,10 @@ void DeleteJob::start()
     }
 
     if (_url.isValid()) {
-        sendRequest("DELETE", _url, req);
+        startRequest("DELETE", _url, req);
     } else {
-        sendRequest("DELETE", makeDavUrl(path()), req);
+        startRequest("DELETE", req);
     }
-
-    if (reply()->error() != QNetworkReply::NoError) {
-        qCWarning(lcDeleteJob) << " Network error: " << reply()->errorString();
-    }
-    AbstractNetworkJob::start();
-}
-
-bool DeleteJob::finished()
-{
-    qCInfo(lcDeleteJob) << "DELETE of" << reply()->request().url() << "FINISHED WITH STATUS"
-                       << replyStatusString();
-
-    emit finishedSignal();
-    return true;
 }
 
 QByteArray DeleteJob::folderToken() const

src/libsync/deletejob.h

@@ -23,7 +23,7 @@ namespace OCC {
  * @brief The DeleteJob class
  * @ingroup libsync
  */
-class DeleteJob : public AbstractNetworkJob
+class DeleteJob : public SimpleFileJob
 {
     Q_OBJECT
 public:
@@ -31,14 +31,10 @@ public:
     explicit DeleteJob(AccountPtr account, const QUrl &url, QObject *parent = nullptr);
 
     void start() override;
-    bool finished() override;
 
     QByteArray folderToken() const;
     void setFolderToken(const QByteArray &folderToken);
 
-signals:
-    void finishedSignal();
-
 private:
     QUrl _url; // Only used if the constructor taking a url is taken.
     QByteArray _folderToken;

src/libsync/networkjobs.cpp

@@ -55,6 +55,7 @@ Q_LOGGING_CATEGORY(lcMkColJob, "nextcloud.sync.networkjob.mkcol", QtInfoMsg)
 Q_LOGGING_CATEGORY(lcProppatchJob, "nextcloud.sync.networkjob.proppatch", QtInfoMsg)
 Q_LOGGING_CATEGORY(lcJsonApiJob, "nextcloud.sync.networkjob.jsonapi", QtInfoMsg)
 Q_LOGGING_CATEGORY(lcDetermineAuthTypeJob, "nextcloud.sync.networkjob.determineauthtype", QtInfoMsg)
+Q_LOGGING_CATEGORY(lcSimpleFileJob, "nextcloud.sync.networkjob.simplefilejob", QtInfoMsg)
 const int notModifiedStatusCode = 304;
 
 QByteArray parseEtag(const char *header)
@@ -1084,9 +1085,39 @@ bool SimpleNetworkJob::finished()
     return true;
 }
 
+SimpleFileJob::SimpleFileJob(AccountPtr account, const QString &filePath, QObject *parent)
+    : AbstractNetworkJob(account, filePath, parent)
+{
+}
+
+QNetworkReply *SimpleFileJob::startRequest(
+    const QByteArray &verb, const QNetworkRequest req, QIODevice *requestBody)
+{
+    return startRequest(verb, makeDavUrl(path()), req, requestBody);
+}
+
+QNetworkReply *SimpleFileJob::startRequest(
+    const QByteArray &verb, const QUrl &url, const QNetworkRequest req, QIODevice *requestBody)
+{
+    _verb = verb;
+    const auto reply = sendRequest(verb, url, req, requestBody);
+
+    if (reply->error() != QNetworkReply::NoError) {
+        qCWarning(lcSimpleFileJob) << verb << " Network error: " << reply->errorString();
+    }
+    AbstractNetworkJob::start();
+    return reply;
+}
+
+bool SimpleFileJob::finished()
+{
+    qCInfo(lcSimpleFileJob) << _verb << "for" << reply()->request().url() << "FINISHED WITH STATUS" << replyStatusString();
+    emit finishedSignal(reply());
+    return true;
+}
 
 DeleteApiJob::DeleteApiJob(AccountPtr account, const QString &path, QObject *parent)
-    : AbstractNetworkJob(account, path, parent)
+    : SimpleFileJob(account, path, parent)
 {
 
 }
@@ -1095,14 +1126,13 @@ void DeleteApiJob::start()
 {
     QNetworkRequest req;
     req.setRawHeader("OCS-APIREQUEST", "true");
-    QUrl url = Utility::concatUrlPath(account()->url(), path());
-    sendRequest("DELETE", url, req);
-    AbstractNetworkJob::start();
+
+    startRequest("DELETE", req);
 }
 
 bool DeleteApiJob::finished()
 {
-    qCInfo(lcJsonApiJob) << "JsonApiJob of" << reply()->request().url() << "FINISHED WITH STATUS"
+    qCInfo(lcJsonApiJob) << "DeleteApiJob of" << reply()->request().url() << "FINISHED WITH STATUS"
                          << reply()->error()
                          << (reply()->error() == QNetworkReply::NoError ? QLatin1String("") : errorString());
 
@@ -1118,7 +1148,7 @@ bool DeleteApiJob::finished()
     const auto replyData = QString::fromUtf8(reply()->readAll());
     qCInfo(lcJsonApiJob()) << "TMX Delete Job" << replyData;
     emit result(httpStatus);
-    return true;
+    return SimpleFileJob::finished();
 }
 
 void fetchPrivateLinkUrl(AccountPtr account, const QString &remotePath,

src/libsync/networkjobs.h

@@ -60,6 +60,31 @@ private slots:
     bool finished() override;
 };
 
+/**
+ * @brief A basic file manipulation job
+ * @ingroup libsync
+ */
+class OWNCLOUDSYNC_EXPORT SimpleFileJob : public AbstractNetworkJob
+{
+    Q_OBJECT
+public:
+    explicit SimpleFileJob(AccountPtr account, const QString &filePath, QObject *parent = nullptr);
+
+    QNetworkReply *startRequest(
+        const QByteArray &verb, const QNetworkRequest req = QNetworkRequest(), QIODevice *requestBody = nullptr);
+
+    QNetworkReply *startRequest(const QByteArray &verb, const QUrl &url, const QNetworkRequest req = QNetworkRequest(),
+        QIODevice *requestBody = nullptr);
+
+signals:
+    void finishedSignal(QNetworkReply *reply);
+protected slots:
+    bool finished() override;
+
+private:
+    QByteArray _verb;
+};
+
 /**
  * @brief sends a DELETE http request to a url.
  *
@@ -67,7 +92,7 @@ private slots:
  *
  * This does *not* delete files, it does a http request.
  */
-class OWNCLOUDSYNC_EXPORT DeleteApiJob : public AbstractNetworkJob
+class OWNCLOUDSYNC_EXPORT DeleteApiJob : public SimpleFileJob
 {
     Q_OBJECT
 public:

src/libsync/propagatedownload.cpp

@@ -22,7 +22,6 @@
 #include "common/utility.h"
 #include "filesystem.h"
 #include "propagatorjobs.h"
-#include <common/checksums.h>
 #include <common/asserts.h>
 #include <common/constants.h>
 #include "clientsideencryptionjobs.h"
@@ -923,8 +922,53 @@ void PropagateDownloadFile::slotGetFinished()
     validator->start(_tmpFile.fileName(), checksumHeader);
 }
 
-void PropagateDownloadFile::slotChecksumFail(const QString &errMsg)
-{ 
+void PropagateDownloadFile::slotChecksumFail(const QString &errMsg,
+    const QByteArray &calculatedChecksumType, const QByteArray &calculatedChecksum, const ValidateChecksumHeader::FailureReason reason)
+{
+    if (reason == ValidateChecksumHeader::FailureReason::ChecksumMismatch && propagator()->account()->isChecksumRecalculateRequestSupported()) {
+            const QByteArray calculatedChecksumHeader(calculatedChecksumType + ':' + calculatedChecksum);
+            const QString fullRemotePathForFile(propagator()->fullRemotePath(_isEncrypted ? _item->_encryptedFileName : _item->_file));
+            auto *job = new SimpleFileJob(propagator()->account(), fullRemotePathForFile);
+            QObject::connect(job, &SimpleFileJob::finishedSignal, this,
+                [this, calculatedChecksumHeader, errMsg](const QNetworkReply *reply) { processChecksumRecalculate(reply, calculatedChecksumHeader, errMsg);
+            });
+
+            qCWarning(lcPropagateDownload) << "Checksum validation has failed for file:" << fullRemotePathForFile
+                                           << " Requesting checksum recalculation on the server...";
+            QNetworkRequest req;
+            req.setRawHeader(checksumRecalculateOnServerHeaderC, calculatedChecksumType);
+            job->startRequest(QByteArrayLiteral("PATCH"), req);
+            return;
+    }
+
+    checksumValidateFailedAbortDownload(errMsg);
+}
+
+void PropagateDownloadFile::processChecksumRecalculate(const QNetworkReply *reply, const QByteArray &originalChecksumHeader, const QString &errorMessage)
+{
+    if (reply->error() != QNetworkReply::NoError) {
+        qCCritical(lcPropagateDownload) << "Checksum recalculation has failed for file:" << reply->url()
+                                        << " Recalculation request has finished with error:" << reply->errorString();
+        checksumValidateFailedAbortDownload(errorMessage);
+        return;
+    }
+
+    const auto newChecksumHeaderFromServer = reply->rawHeader(checkSumHeaderC);
+    if (newChecksumHeaderFromServer == originalChecksumHeader) {
+        const auto newChecksumHeaderFromServerSplit = newChecksumHeaderFromServer.split(':');
+        if (newChecksumHeaderFromServerSplit.size() > 1) {
+            transmissionChecksumValidated(newChecksumHeaderFromServerSplit.first(), newChecksumHeaderFromServerSplit.last());
+            return;
+        }
+    }
+
+    qCCritical(lcPropagateDownload) << "Checksum recalculation has failed for file:" << reply->url() << " "
+                                    << checkSumHeaderC << " received is:" << newChecksumHeaderFromServer;
+    checksumValidateFailedAbortDownload(errorMessage);
+}
+
+void PropagateDownloadFile::checksumValidateFailedAbortDownload(const QString &errMsg)
+{
     FileSystem::remove(_tmpFile.fileName());
     propagator()->_anotherSyncNeeded = true;
     done(SyncFileItem::SoftError, errMsg); // tr("The file downloaded with a broken checksum, will be redownloaded."));

src/libsync/propagatedownload.h

@@ -17,6 +17,7 @@
 #include "owncloudpropagator.h"
 #include "networkjobs.h"
 #include "clientsideencryption.h"
+#include <common/checksums.h>
 
 #include <QBuffer>
 #include <QFile>
@@ -235,7 +236,10 @@ private slots:
 
     void abort(PropagatorJob::AbortType abortType) override;
     void slotDownloadProgress(qint64, qint64);
-    void slotChecksumFail(const QString &errMsg);
+    void slotChecksumFail(const QString &errMsg, const QByteArray &calculatedChecksumType,
+        const QByteArray &calculatedChecksum, const ValidateChecksumHeader::FailureReason reason);
+    void processChecksumRecalculate(const QNetworkReply *reply, const QByteArray &originalChecksumHeader, const QString &errorMessage);
+    void checksumValidateFailedAbortDownload(const QString &errMsg);
 
 private:
     void startAfterIsEncryptedIsChecked();

src/libsync/propagatorjobs.h

@@ -24,8 +24,9 @@ namespace OCC {
  * Tags for checksum header.
  * It's here for being shared between Upload- and Download Job
  */
-static const char checkSumHeaderC[] = "OC-Checksum";
-static const char contentMd5HeaderC[] = "Content-MD5";
+constexpr auto checkSumHeaderC = "OC-Checksum";
+constexpr auto contentMd5HeaderC = "Content-MD5";
+constexpr auto checksumRecalculateOnServerHeaderC = "X-Recalculate-Hash";
 
 /**
  * @brief Declaration of the other propagation jobs

test/testchecksumvalidator.cpp

@@ -25,6 +25,7 @@ using namespace OCC::Utility;
         QTemporaryDir _root;
         QString _testfile;
         QString _expectedError;
+        ValidateChecksumHeader::FailureReason _expectedFailureReason = ValidateChecksumHeader::FailureReason::Success;
         QByteArray     _expected;
         QByteArray     _expectedType;
         bool           _successDown;
@@ -42,10 +43,14 @@ using namespace OCC::Utility;
          _successDown = true;
     }
 
-    void slotDownError(const QString &errMsg)
+    void slotDownError(const QString &errMsg, const QByteArray &calculatedChecksumType,
+        const QByteArray &calculatedChecksum, const ValidateChecksumHeader::FailureReason reason)
     {
-         QCOMPARE(_expectedError, errMsg);
-         _errorSeen = true;
+        Q_UNUSED(calculatedChecksumType);
+        Q_UNUSED(calculatedChecksum);
+        QCOMPARE(_expectedError, errMsg);
+        QCOMPARE(_expectedFailureReason, reason);
+        _errorSeen = true;
     }
 
     static QByteArray shellSum( const QByteArray& cmd, const QString& file )
@@ -198,12 +203,14 @@ using namespace OCC::Utility;
         QTRY_VERIFY(_successDown);
 
         _expectedError = QStringLiteral("The downloaded file does not match the checksum, it will be resumed. \"543345\" != \"%1\"").arg(QString::fromUtf8(_expected));
+        _expectedFailureReason = ValidateChecksumHeader::FailureReason::ChecksumMismatch;
         _errorSeen = false;
         file->seek(0);
         vali->start(_testfile, "Adler32:543345");
         QTRY_VERIFY(_errorSeen);
 
         _expectedError = QLatin1String("The checksum header contained an unknown checksum type \"Klaas32\"");
+        _expectedFailureReason = ValidateChecksumHeader::FailureReason::ChecksumTypeUnknown;
         _errorSeen = false;
         file->seek(0);
         vali->start(_testfile, "Klaas32:543345");

test/testsyncengine.cpp

@@ -8,6 +8,7 @@
 #include <QtTest>
 #include "syncenginetestutils.h"
 #include <syncengine.h>
+#include <propagatorjobs.h>
 
 using namespace OCC;
 
@@ -551,16 +552,27 @@ private slots:
         QObject parent;
 
         QByteArray checksumValue;
+        QByteArray checksumValueRecalculated;
         QByteArray contentMd5Value;
+        bool isChecksumRecalculateSupported = false;
 
         fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *) -> QNetworkReply * {
             if (op == QNetworkAccessManager::GetOperation) {
                 auto reply = new FakeGetReply(fakeFolder.remoteModifier(), op, request, &parent);
                 if (!checksumValue.isNull())
-                    reply->setRawHeader("OC-Checksum", checksumValue);
+                    reply->setRawHeader(OCC::checkSumHeaderC, checksumValue);
                 if (!contentMd5Value.isNull())
-                    reply->setRawHeader("Content-MD5", contentMd5Value);
+                    reply->setRawHeader(OCC::contentMd5HeaderC, contentMd5Value);
                 return reply;
+            } else if (op == QNetworkAccessManager::CustomOperation) {
+                if (request.hasRawHeader(OCC::checksumRecalculateOnServerHeaderC)) {
+                    if (!isChecksumRecalculateSupported) {
+                        return new FakeErrorReply(op, request, &parent, 402);
+                    }
+                    auto reply = new FakeGetReply(fakeFolder.remoteModifier(), op, request, &parent);
+                    reply->setRawHeader(OCC::checkSumHeaderC, checksumValueRecalculated);
+                    return reply;
+                }
             }
             return nullptr;
         });
@@ -575,8 +587,11 @@ private slots:
         fakeFolder.remoteModifier().create("A/a4", 16, 'A');
         QVERIFY(!fakeFolder.syncOnce());
 
+        const QByteArray matchedSha1Checksum(QByteArrayLiteral("SHA1:19b1928d58a2030d08023f3d7054516dbc186f20"));
+        const QByteArray mismatchedSha1Checksum(matchedSha1Checksum.chopped(1));
+
         // Good OC-Checksum
-        checksumValue = "SHA1:19b1928d58a2030d08023f3d7054516dbc186f20"; // printf 'A%.0s' {1..16} | sha1sum -
+        checksumValue = matchedSha1Checksum; // printf 'A%.0s' {1..16} | sha1sum -
         QVERIFY(fakeFolder.syncOnce());
         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
         checksumValue = QByteArray();
@@ -610,6 +625,35 @@ private slots:
         checksumValue =  "Unsupported:XXXX SHA1:19b1928d58a2030d08023f3d7054516dbc186f20 Invalid:XxX";
         QVERIFY(fakeFolder.syncOnce()); // The supported SHA1 checksum is valid now, so the file are downloaded
         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        // Begin Test mismatch recalculation---------------------------------------------------------------------------------
+
+        const auto prevServerVersion = fakeFolder.account()->serverVersion();
+        fakeFolder.account()->setServerVersion(QString("%1.0.0").arg(fakeFolder.account()->checksumRecalculateServerVersionMinSupportedMajor()));
+
+        // Mismatched OC-Checksum and X-Recalculate-Hash is not supported -> sync must fail
+        isChecksumRecalculateSupported = false;
+        checksumValue = mismatchedSha1Checksum;
+        checksumValueRecalculated = matchedSha1Checksum;
+        fakeFolder.remoteModifier().create("A/a9", 16, 'A');
+        QVERIFY(!fakeFolder.syncOnce());
+
+        // Mismatched OC-Checksum and X-Recalculate-Hash is supported, but, recalculated checksum is again mismatched -> sync must fail
+        isChecksumRecalculateSupported = true;
+        checksumValue = mismatchedSha1Checksum;
+        checksumValueRecalculated = mismatchedSha1Checksum;
+        QVERIFY(!fakeFolder.syncOnce());
+
+        // Mismatched OC-Checksum and X-Recalculate-Hash is supported, and, recalculated checksum is a match -> sync must succeed
+        isChecksumRecalculateSupported = true;
+        checksumValue = mismatchedSha1Checksum;
+        checksumValueRecalculated = matchedSha1Checksum;
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+        checksumValue = QByteArray();
+
+        fakeFolder.account()->setServerVersion(prevServerVersion);
+        // End Test mismatch recalculation-----------------------------------------------------------------------------------
     }
 
     // Tests the behavior of invalid filename detection