Allow download with mismatched checksum if a config option is set.

Signed-off-by: allexzander <blackslayer4@gmail.com>

src/common/checksums.cpp

@@ -328,7 +328,7 @@ ComputeChecksum *ValidateChecksumHeader::prepareStart(const QByteArray &checksum
 
     if (!parseChecksumHeader(checksumHeader, &_expectedChecksumType, &_expectedChecksum)) {
         qCWarning(lcChecksums) << "Checksum header malformed:" << checksumHeader;
-        emit validationFailed(tr("The checksum header is malformed."));
+        emit validationFailed(tr("The checksum header is malformed."), {}, {}, _filePath);
         return nullptr;
     }
 
@@ -341,6 +341,7 @@ ComputeChecksum *ValidateChecksumHeader::prepareStart(const QByteArray &checksum
 
 void ValidateChecksumHeader::start(const QString &filePath, const QByteArray &checksumHeader)
 {
+    _filePath = filePath;
     if (auto calculator = prepareStart(checksumHeader))
         calculator->start(filePath);
 }
@@ -355,11 +356,11 @@ void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray &checksumTy
     const QByteArray &checksum)
 {
     if (checksumType != _expectedChecksumType) {
-        emit validationFailed(tr("The checksum header contained an unknown checksum type '%1'").arg(QString::fromLatin1(_expectedChecksumType)));
+        emit validationFailed(tr("The checksum header contained an unknown checksum type '%1'").arg(QString::fromLatin1(_expectedChecksumType)), {}, {}, _filePath);
         return;
     }
     if (checksum != _expectedChecksum) {
-        emit validationFailed(tr("The downloaded file does not match the checksum, it will be resumed. '%1' != '%2'").arg(QString::fromUtf8(_expectedChecksum), QString::fromUtf8(checksum)));
+        emit validationFailed(tr("The downloaded file does not match the checksum, it will be resumed. '%1' != '%2'").arg(QString::fromUtf8(_expectedChecksum), QString::fromUtf8(checksum)), checksumType, checksum, _filePath);
         return;
     }
     emit validated(checksumType, checksum);

src/common/checksums.h

@@ -163,7 +163,7 @@ public:
 
 signals:
     void validated(const QByteArray &checksumType, const QByteArray &checksum);
-    void validationFailed(const QString &errMsg);
+    void validationFailed(const QString &errMsg, const QByteArray &checksumType, const QByteArray &checksum, const QString &filePath);
 
 private slots:
     void slotChecksumCalculated(const QByteArray &checksumType, const QByteArray &checksum);
@@ -173,6 +173,8 @@ private:
 
     QByteArray _expectedChecksumType;
     QByteArray _expectedChecksum;
+
+    QString _filePath;
 };
 
 /**

src/common/syncjournaldb.cpp

@@ -976,7 +976,7 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record)
     }
 }
 
-void SyncJournalDb::keyValueStoreSet(const QString &key, qint64 value)
+void SyncJournalDb::keyValueStoreSet(const QString &key, QVariant value)
 {
     QMutexLocker locker(&_mutex);
     if (!checkConnect()) {
@@ -988,7 +988,7 @@ void SyncJournalDb::keyValueStoreSet(const QString &key, qint64 value)
     }
 
     _setKeyValueStoreQuery.bindValue(1, key);
-    _setKeyValueStoreQuery.bindValue(2, QString::number(value));
+    _setKeyValueStoreQuery.bindValue(2, value);
     _setKeyValueStoreQuery.exec();
 }
 
@@ -1013,6 +1013,40 @@ qint64 SyncJournalDb::keyValueStoreGetInt(const QString &key, qint64 defaultValu
     return _getKeyValueStoreQuery.int64Value(0);
 }
 
+QVariant SyncJournalDb::keyValueStoreGet(const QString &key, QVariant defaultValue)
+{
+    QMutexLocker locker(&_mutex);
+    if (!checkConnect()) {
+        return defaultValue;
+    }
+
+    if (!_getKeyValueStoreQuery.initOrReset(QByteArrayLiteral("SELECT value FROM key_value_store WHERE key = ?1;"), _db)) {
+        return defaultValue;
+    }
+
+    _getKeyValueStoreQuery.bindValue(1, key);
+    _getKeyValueStoreQuery.exec();
+
+    if (!_getKeyValueStoreQuery.next().hasData) {
+        return defaultValue;
+    }
+
+    return _getKeyValueStoreQuery.stringValue(0);
+}
+
+void SyncJournalDb::keyValueStoreDelete(const QString &key)
+{
+    if (!_deleteKeyValueStoreQuery.initOrReset("DELETE FROM key_value_store WHERE key=?1;", _db)) {
+        qCWarning(lcDb) << "Failed to initOrReset _deleteKeyValueStoreQuery";
+        Q_ASSERT(false);
+    }
+    _deleteKeyValueStoreQuery.bindValue(1, key);
+    if (!_deleteKeyValueStoreQuery.exec()) {
+        qCWarning(lcDb) << "Failed to exec _deleteKeyValueStoreQuery for key" << key;
+        Q_ASSERT(false);
+    }
+}
+
 // TODO: filename -> QBytearray?
 bool SyncJournalDb::deleteFileRecord(const QString &filename, bool recursively)
 {

src/common/syncjournaldb.h

@@ -23,6 +23,7 @@
 #include <QDateTime>
 #include <QHash>
 #include <QMutex>
+#include <QVariant>
 #include <functional>
 
 #include "common/utility.h"
@@ -66,8 +67,10 @@ public:
     bool listFilesInPath(const QByteArray &path, const std::function<void(const SyncJournalFileRecord&)> &rowCallback);
     bool setFileRecord(const SyncJournalFileRecord &record);
 
-    void keyValueStoreSet(const QString &key, qint64 value);
+    void keyValueStoreSet(const QString &key, QVariant value);
     qint64 keyValueStoreGetInt(const QString &key, qint64 defaultValue);
+    QVariant keyValueStoreGet(const QString &key, QVariant defaultValue = {});
+    void keyValueStoreDelete(const QString &key);
 
     bool deleteFileRecord(const QString &filename, bool recursively = false);
     bool updateFileRecordChecksum(const QString &filename,
@@ -423,6 +426,7 @@ private:
     SqlQuery _setDataFingerprintQuery2;
     SqlQuery _setKeyValueStoreQuery;
     SqlQuery _getKeyValueStoreQuery;
+    SqlQuery _deleteKeyValueStoreQuery;
     SqlQuery _getConflictRecordQuery;
     SqlQuery _setConflictRecordQuery;
     SqlQuery _deleteConflictRecordQuery;

src/libsync/configfile.cpp

@@ -49,6 +49,10 @@
 #define DEFAULT_REMOTE_POLL_INTERVAL 30000 // default remote poll time in milliseconds
 #define DEFAULT_MAX_LOG_LINES 20000
 
+namespace {
+    static constexpr char allowChecksumValidationFailC[] = "allowChecksumValidationFail";
+}
+
 namespace OCC {
 
 namespace chrono = std::chrono;
@@ -101,7 +105,6 @@ static const char useNewBigFolderSizeLimitC[] = "useNewBigFolderSizeLimit";
 static const char confirmExternalStorageC[] = "confirmExternalStorage";
 static const char moveToTrashC[] = "moveToTrash";
 
-
 const char certPath[] = "http_certificatePath";
 const char certPasswd[] = "http_certificatePasswd";
 QString ConfigFile::_confDir = QString();
@@ -113,7 +116,6 @@ static chrono::milliseconds millisecondsValue(const QSettings &setting, const ch
     return chrono::milliseconds(setting.value(QLatin1String(key), qlonglong(defaultValue.count())).toLongLong());
 }
 
-
 bool copy_dir_recursive(QString from_dir, QString to_dir)
 {
     QDir dir;
@@ -889,6 +891,11 @@ void ConfigFile::setMoveToTrash(bool isChecked)
     setValue(moveToTrashC, isChecked);
 }
 
+bool ConfigFile::allowChecksumValidationFail() const
+{
+    return getValue(allowChecksumValidationFailC, {}, false).toBool();
+}
+
 bool ConfigFile::promptDeleteFiles() const
 {
     QSettings settings(configFile(), QSettings::IniFormat);

src/libsync/configfile.h

@@ -145,6 +145,9 @@ public:
     bool moveToTrash() const;
     void setMoveToTrash(bool);
 
+    /** should we allow checksum validation to fail? set to true to workaround corrupted checksums **/
+    bool allowChecksumValidationFail() const;
+
     static bool setConfDir(const QString &value);
 
     bool optionalServerNotifications() const;

src/libsync/propagatedownload.cpp

@@ -28,6 +28,8 @@
 #include "propagatedownloadencrypted.h"
 #include "common/vfs.h"
 
+#include "configfile.h"
+
 #include <QLoggingCategory>
 #include <QNetworkAccessManager>
 #include <QFileInfo>
@@ -38,6 +40,11 @@
 #include <unistd.h>
 #endif
 
+namespace {
+    constexpr quint16 numChecksumFailuresAllowed = 1;
+    constexpr char *checksumFailureDbRecordPrefix = "ChecksumValidationFailed_";
+}
+
 namespace OCC {
 
 Q_LOGGING_CATEGORY(lcGetJob, "nextcloud.sync.networkjob.get", QtInfoMsg)
@@ -821,8 +828,38 @@ void PropagateDownloadFile::slotGetFinished()
     validator->start(_tmpFile.fileName(), checksumHeader);
 }
 
-void PropagateDownloadFile::slotChecksumFail(const QString &errMsg)
+void PropagateDownloadFile::slotChecksumFail(const QString &errMsg, const QByteArray &checksumType, const QByteArray &checksum, const QString &filePath)
 {
+    if (!checksumType.isEmpty() && !checksum.isEmpty() && !filePath.isEmpty()) {
+        ConfigFile cfgFile;
+
+        if (cfgFile.allowChecksumValidationFail()) {
+            const auto key = QString(checksumFailureDbRecordPrefix + _item->_fileId);
+            const QStringList mismatchEntryForFileSplitted = propagator()->_journal->keyValueStoreGet(key).toString().split(":", QString::SkipEmptyParts);
+            const QByteArray mismatchChecksumForFile = mismatchEntryForFileSplitted.size() > 0 ? mismatchEntryForFileSplitted[0].toUtf8() : QByteArray();
+            const auto numChecksumMismatchCases = mismatchEntryForFileSplitted.size() > 1 ? mismatchEntryForFileSplitted[1].toInt() : 0;
+
+            // format must be CHECKSUM:COUNT
+            Q_ASSERT(mismatchEntryForFileSplitted.size() != 1);
+            if (mismatchEntryForFileSplitted.size() == 1) {
+                qCCritical(lcPropagateDownload) << "mismatchEntryForFile has incorrect format. Should be CHECKSUM:COUNT";
+            }
+
+            if (numChecksumMismatchCases < numChecksumFailuresAllowed || mismatchChecksumForFile != checksum) {
+                // not enough failures or different checksum this time
+                qCInfo(lcPropagateDownload) << "Checksum validation has failed" << numChecksumMismatchCases << " times, with previous checksum<" << mismatchChecksumForFile << "> and, current checksum<" << checksum << ">, but, allowChecksumValidationFail is set.Let's give it another try...";
+                const auto numCasesToSet = mismatchChecksumForFile != checksum ? 1 : numChecksumMismatchCases + 1;
+                const QString value = QString::fromUtf8(checksum) + QStringLiteral(":") + QString::number(numCasesToSet);
+                propagator()->_journal->keyValueStoreSet(key, value);
+            } else {
+                propagator()->_journal->keyValueStoreDelete(key);
+                qCInfo(lcPropagateDownload) << "Checksum validation has failed" << numChecksumMismatchCases << " times, but, allowChecksumValidationFail is set, so, let's continue...";
+                startContentChecksumCompute(checksumType, filePath);
+                return;
+            }
+        }
+    }
+    
     FileSystem::remove(_tmpFile.fileName());
     propagator()->_anotherSyncNeeded = true;
     done(SyncFileItem::SoftError, errMsg); // tr("The file downloaded with a broken checksum, will be redownloaded."));
@@ -850,6 +887,18 @@ void PropagateDownloadFile::deleteExistingFolder()
     }
 }
 
+void PropagateDownloadFile::startContentChecksumCompute(const QByteArray &checksumType, const QString &path)
+{
+    qCInfo(lcPropagateDownload) << "Start checksum compute with checksumType:" << checksumType << " for path:" << path;
+    // Compute the content checksum.
+    const auto computeChecksum = new ComputeChecksum(this);
+    computeChecksum->setChecksumType(checksumType);
+
+    connect(computeChecksum, &ComputeChecksum::done,
+        this, &PropagateDownloadFile::contentChecksumComputed);
+    computeChecksum->start(path);
+}
+
 namespace { // Anonymous namespace for the recall feature
     static QString makeRecallFileName(const QString &fn)
     {
@@ -938,13 +987,9 @@ void PropagateDownloadFile::transmissionChecksumValidated(const QByteArray &chec
         return contentChecksumComputed(checksumType, checksum);
     }
 
-    // Compute the content checksum.
-    auto computeChecksum = new ComputeChecksum(this);
-    computeChecksum->setChecksumType(theContentChecksumType);
+    startContentChecksumCompute(theContentChecksumType, _tmpFile.fileName());
 
-    connect(computeChecksum, &ComputeChecksum::done,
-        this, &PropagateDownloadFile::contentChecksumComputed);
-    computeChecksum->start(_tmpFile.fileName());
+    propagator()->_journal->keyValueStoreDelete(QString(checksumFailureDbRecordPrefix + _item->_fileId));
 }
 
 void PropagateDownloadFile::contentChecksumComputed(const QByteArray &checksumType, const QByteArray &checksum)

src/libsync/propagatedownload.h

@@ -202,12 +202,14 @@ private slots:
 
     void abort(PropagatorJob::AbortType abortType) override;
     void slotDownloadProgress(qint64, qint64);
-    void slotChecksumFail(const QString &errMsg);
+    void slotChecksumFail(const QString &errMsg, const QByteArray &checksumType, const QByteArray &checksum, const QString &filePath);
 
 private:
     void startAfterIsEncryptedIsChecked();
     void deleteExistingFolder();
 
+    void startContentChecksumCompute(const QByteArray &checksumType, const QString &path);
+
     qint64 _resumeStart;
     qint64 _downloadProgress;
     QPointer<GETFileJob> _job;

test/testchecksumvalidator.cpp

@@ -42,7 +42,7 @@ using namespace OCC::Utility;
          _successDown = true;
     }
 
-    void slotDownError( const QString& errMsg ) {
+    void slotDownError(const QString &errMsg, const QByteArray&, const QByteArray&, const QString&) {
          QCOMPARE(_expectedError, errMsg);
          _errorSeen = true;
     }
@@ -179,8 +179,8 @@ using namespace OCC::Utility;
         QSKIP("ZLIB not found.", SkipSingle);
 #else
         auto *vali = new ValidateChecksumHeader(this);
-        connect(vali, SIGNAL(validated(QByteArray,QByteArray)), this, SLOT(slotDownValidated()));
-        connect(vali, SIGNAL(validationFailed(QString)), this, SLOT(slotDownError(QString)));
+        connect(vali, &ValidateChecksumHeader::validated, this, &TestChecksumValidator::slotDownValidated);
+        connect(vali, &ValidateChecksumHeader::validationFailed, this, &TestChecksumValidator::slotDownError);
 
         auto file = new QFile(_testfile, vali);
         file->open(QIODevice::ReadOnly);