Checksums: Don't reupload if size and checksum are unchanged #3235

* Compute the content checksum (in addition to the optional
  transmission checksum) during upload (.eml files only)

* Add hook to compute and compare the checksum in csync_update

* Add content checksum to database, remove transmission checksum

csync/src/csync.c

@@ -742,6 +742,7 @@ void csync_file_stat_free(csync_file_stat_t *st)
     SAFE_FREE(st->directDownloadCookies);
     SAFE_FREE(st->etag);
     SAFE_FREE(st->destpath);
+    SAFE_FREE(st->checksum);
     SAFE_FREE(st);
   }
 }

csync/src/csync.h

@@ -301,6 +301,12 @@ typedef void (*csync_vio_closedir_hook) (csync_vio_handle_t *dhhandle,
 typedef int (*csync_vio_stat_hook) (csync_vio_handle_t *dhhandle,
                                                               void *userdata);
 
+/* compute the checksum of the given \a checksumTypeId for \a path
+ * and return true if it's the same as \a checksum */
+typedef bool (*csync_checksum_hook) (const char *path,
+                                     uint32_t checksumTypeId, const char *checksum,
+                                     void *userdata);
+
 /**
  * @brief Allocate a csync context.
  *

csync/src/csync_private.h

@@ -96,6 +96,11 @@ struct csync_s {
       csync_vio_readdir_hook remote_readdir_hook;
       csync_vio_closedir_hook remote_closedir_hook;
       void *vio_userdata;
+
+      /* hook for comparing checksums of files during discovery */
+      csync_checksum_hook checksum_hook;
+      void *checksum_userdata;
+
   } callbacks;
   c_strlist_t *excludes;
 
@@ -192,6 +197,9 @@ struct csync_file_stat_s {
   char *directDownloadCookies;
   char remotePerm[REMOTE_PERM_BUF_SIZE+1];
 
+  char *checksum;
+  uint32_t checksumTypeId;
+
   CSYNC_STATUS error_status;
 
   enum csync_instructions_e instruction; /* u32 */

csync/src/csync_statedb.c

@@ -226,6 +226,8 @@ int csync_statedb_close(CSYNC *ctx) {
   return rc;
 }
 
+#define METADATA_COLUMNS "phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId"
+
 // This funciton parses a line from the metadata table into the given csync_file_stat
 // structure which it is also allocating.
 // Note that this function calls laso sqlite3_step to actually get the info from db and
@@ -286,6 +288,11 @@ static int _csync_file_stat_from_metadata_table( csync_file_stat_t **st, sqlite3
             if(column_count > 13) {
                 (*st)->has_ignored_files = sqlite3_column_int(stmt, 13);
             }
+            if(column_count > 15 && sqlite3_column_int(stmt, 15)) {
+                (*st)->checksum = c_strdup( (char*) sqlite3_column_text(stmt, 14));
+                (*st)->checksumTypeId = sqlite3_column_int(stmt, 15);
+            }
+
         }
     } else {
         if( rc != SQLITE_DONE ) {
@@ -307,7 +314,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx,
   }
 
   if( ctx->statedb.by_hash_stmt == NULL ) {
-      const char *hash_query = "SELECT * FROM metadata WHERE phash=?1";
+      const char *hash_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE phash=?1";
 
       SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, hash_query, strlen(hash_query), &ctx->statedb.by_hash_stmt, NULL));
       ctx->statedb.lastReturnValue = rc;
@@ -350,7 +357,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_file_id(CSYNC *ctx,
     }
 
     if( ctx->statedb.by_fileid_stmt == NULL ) {
-        const char *query = "SELECT * FROM metadata WHERE fileid=?1";
+        const char *query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE fileid=?1";
 
         SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, query, strlen(query), &ctx->statedb.by_fileid_stmt, NULL));
         ctx->statedb.lastReturnValue = rc;
@@ -390,7 +397,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx,
   }
 
   if( ctx->statedb.by_inode_stmt == NULL ) {
-      const char *inode_query = "SELECT * FROM metadata WHERE inode=?1";
+      const char *inode_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE inode=?1";
 
       SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, inode_query, strlen(inode_query), &ctx->statedb.by_inode_stmt, NULL));
       ctx->statedb.lastReturnValue = rc;
@@ -433,7 +440,7 @@ int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) {
      * In other words, anything that is between  path+'/' and path+'0',
      * (because '0' follows '/' in ascii)
      */
-    const char *below_path_query = "SELECT phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote FROM metadata WHERE path > (?||'/') AND path < (?||'0')";
+    const char *below_path_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE path > (?||'/') AND path < (?||'0')";
     SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, below_path_query, -1, &stmt, NULL));
     ctx->statedb.lastReturnValue = rc;
     if( rc != SQLITE_OK ) {

csync/src/csync_update.c

@@ -270,25 +270,43 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
                   ((int64_t) fs->mtime), ((int64_t) tmp->modtime),
                   fs->etag, tmp->etag, (uint64_t) fs->inode, (uint64_t) tmp->inode,
                   (uint64_t) fs->size, (uint64_t) tmp->size, fs->remotePerm, tmp->remotePerm, tmp->has_ignored_files );
-        if((ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag ))
-            || (ctx->current == LOCAL_REPLICA && (!_csync_mtime_equal(fs->mtime, tmp->modtime)
-                                                  // zero size in statedb can happen during migration
-                                                  || (tmp->size != 0 && fs->size != tmp->size)
+        if (ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag)) {
+            st->instruction = CSYNC_INSTRUCTION_EVAL;
+            goto out;
+        }
+        if (ctx->current == LOCAL_REPLICA &&
+                (!_csync_mtime_equal(fs->mtime, tmp->modtime)
+                 // zero size in statedb can happen during migration
+                 || (tmp->size != 0 && fs->size != tmp->size)
 #if 0
-                                                  || fs->inode != tmp->inode
+                 /* Comparison of the local inode is disabled because people reported problems
+                  * on windows with flacky inode values, see github bug #779
+                  *
+                  * The inode needs to be observed because:
+                  * $>  echo a > a.txt ; echo b > b.txt
+                  * both files have the same mtime
+                  * sync them.
+                  * $> rm a.txt && mv b.txt a.txt
+                  * makes b.txt appearing as a.txt yet a sync is not performed because
+                  * both have the same modtime as mv does not change that.
+                  */
+                 || fs->inode != tmp->inode
 #endif
-                                                  ))) {
-            /* Comparison of the local inode is disabled because people reported problems
-             * on windows with flacky inode values, see github bug #779
-             *
-             * The inode needs to be observed because:
-             * $>  echo a > a.txt ; echo b > b.txt
-             * both files have the same mtime
-             * sync them.
-             * $> rm a.txt && mv b.txt a.txt
-             * makes b.txt appearing as a.txt yet a sync is not performed because
-             * both have the same modtime as mv does not change that.
-             */
+                 )) {
+
+            if (fs->size == tmp->size && tmp->checksumTypeId) {
+                bool checksumIdentical = false;
+                if (ctx->callbacks.checksum_hook) {
+                    checksumIdentical = ctx->callbacks.checksum_hook(
+                                file, tmp->checksumTypeId, tmp->checksum,
+                                ctx->callbacks.checksum_userdata);
+                }
+                if (checksumIdentical) {
+                    st->instruction = CSYNC_INSTRUCTION_NONE;
+                    st->should_update_metadata = true;
+                    goto out;
+                }
+            }
             st->instruction = CSYNC_INSTRUCTION_EVAL;
             goto out;
         }

csync/tests/csync_tests/check_csync_update.c

@@ -41,6 +41,12 @@ static void statedb_create_metadata_table(sqlite3 *db)
                           "modtime INTEGER(8),"
                           "type INTEGER,"
                           "md5 VARCHAR(32),"
+                          "fileid VARCHAR(128),"
+                          "remotePerm VARCHAR(128),"
+                          "filesize BIGINT,"
+                          "ignoredChildrenRemote INT,"
+                          "contentChecksum TEXT,"
+                          "contentChecksumTypeId INTEGER,"
                           "PRIMARY KEY(phash));";
 
         rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

src/libsync/propagatedownload.cpp

@@ -351,10 +351,7 @@ void PropagateDownloadFileQNAM::start()
         if (_resumeStart == _item->_size) {
             qDebug() << "File is already complete, no need to download";
             _tmpFile.close();
-
-            // Unfortunately we lost the checksum header, if any...
-            QByteArray noChecksumData;
-            downloadFinished(noChecksumData, noChecksumData);
+            downloadFinished();
             return;
         }
     }
@@ -537,7 +534,7 @@ void PropagateDownloadFileQNAM::slotGetFinished()
     // as this is (still) also correct.
     ValidateChecksumHeader *validator = new ValidateChecksumHeader(this);
     connect(validator, SIGNAL(validated(QByteArray,QByteArray)),
-            SLOT(downloadFinished(QByteArray,QByteArray)));
+            SLOT(downloadFinished()));
     connect(validator, SIGNAL(validationFailed(QString)),
             SLOT(slotChecksumFail(QString)));
     auto checksumHeader = job->reply()->rawHeader(checkSumHeaderC);
@@ -621,13 +618,8 @@ static void handleRecallFile(const QString &fn)
 }
 } // end namespace
 
-void PropagateDownloadFileQNAM::downloadFinished(const QByteArray& checksumType, const QByteArray& checksum)
+void PropagateDownloadFileQNAM::downloadFinished()
 {
-    if (!checksumType.isEmpty()) {
-        _item->_transmissionChecksum = checksum;
-        _item->_transmissionChecksumType = checksumType;
-    }
-
     QString fn = _propagator->getFilePath(_item->_file);
 
     // In case of file name clash, report an error

src/libsync/propagatedownload.h

@@ -117,7 +117,7 @@ public:
 private slots:
     void slotGetFinished();
     void abort() Q_DECL_OVERRIDE;
-    void downloadFinished(const QByteArray& checksumType, const QByteArray& checksum);
+    void downloadFinished();
     void slotDownloadProgress(qint64,qint64);
     void slotChecksumFail( const QString& errMsg );
 

src/libsync/propagateremotemove.cpp

@@ -158,8 +158,8 @@ void PropagateRemoteMove::finalize()
 
     SyncJournalFileRecord record(*_item, _propagator->getFilePath(_item->_renameTarget));
     record._path = _item->_renameTarget;
-    record._transmissionChecksum = oldRecord._transmissionChecksum;
-    record._transmissionChecksumType = oldRecord._transmissionChecksumType;
+    record._contentChecksum = oldRecord._contentChecksum;
+    record._contentChecksumType = oldRecord._contentChecksumType;
 
     _propagator->_journal->setFileRecord(record);
     _propagator->_journal->commit("Remote Rename");

src/libsync/propagateupload.cpp

@@ -216,24 +216,38 @@ void PropagateUploadFileQNAM::start()
 
     _stopWatch.start();
 
-    auto supportedChecksumTypes = _propagator->account()->capabilities().supportedChecksumTypes();
-
-    // If we already have a checksum header and the checksum type is supported
-    // by the server, we keep that - otherwise recompute.
-    //
-    // Note: Currently we *always* recompute because we usually only upload
-    // files that have changed and thus have a new checksum. But if an earlier
-    // phase computed a checksum, this is where we would make use of it.
-    if (!_item->_transmissionChecksumType.isEmpty()) {
-        if (supportedChecksumTypes.contains(_item->_transmissionChecksumType)) {
-            // TODO: We could validate the old checksum and thereby determine whether
-            // an upload is necessary or not.
-            slotStartUpload(_item->_transmissionChecksumType, _item->_transmissionChecksum);
-            return;
-        }
+    // Compute the content checksum.
+    auto computeChecksum = new ComputeChecksum(this);
+    // We currently only do content checksums for the particular .eml case
+    // This should be done more generally in the future!
+    if (filePath.endsWith(QLatin1String(".eml"), Qt::CaseInsensitive)) {
+        computeChecksum->setChecksumType("MD5");
+    } else {
+        computeChecksum->setChecksumType(QByteArray());
+    }
+
+    connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)),
+            SLOT(slotComputeTransmissionChecksum(QByteArray,QByteArray)));
+    computeChecksum->start(filePath);
+}
+
+void PropagateUploadFileQNAM::slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum)
+{
+    _item->_contentChecksum = contentChecksum;
+    _item->_contentChecksumType = contentChecksumType;
+
+    _stopWatch.addLapTime(QLatin1String("ContentChecksum"));
+    _stopWatch.start();
+
+    // Reuse the content checksum as the transmission checksum if possible
+    const auto supportedTransmissionChecksums =
+            _propagator->account()->capabilities().supportedChecksumTypes();
+    if (supportedTransmissionChecksums.contains(contentChecksumType)) {
+        slotStartUpload(contentChecksumType, contentChecksum);
+        return;
     }
 
-    // Compute a new checksum.
+    // Compute the transmission checksum.
     auto computeChecksum = new ComputeChecksum(this);
     if (uploadChecksumEnabled()) {
         computeChecksum->setChecksumType(_propagator->account()->capabilities().preferredChecksumType());
@@ -243,19 +257,14 @@ void PropagateUploadFileQNAM::start()
 
     connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)),
             SLOT(slotStartUpload(QByteArray,QByteArray)));
+    const QString filePath = _propagator->getFilePath(_item->_file);
     computeChecksum->start(filePath);
 }
 
-void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& checksumType, const QByteArray& checksum)
+void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum)
 {
-    // Store the computed checksum in the database, if different
-    if (checksumType != _item->_transmissionChecksumType
-            || checksum != _item->_transmissionChecksum) {
-        _item->_transmissionChecksum = checksum;
-        _item->_transmissionChecksumType = checksumType;
-        _propagator->_journal->updateFileRecordChecksum(
-                _item->_file, checksum, checksumType);
-    }
+    _transmissionChecksum = transmissionChecksum;
+    _transmissionChecksumType = transmissionChecksumType;
 
     const QString fullFilePath = _propagator->getFilePath(_item->_file);
 
@@ -263,7 +272,7 @@ void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& checksumType, co
         done(SyncFileItem::SoftError, tr("File Removed"));
         return;
     }
-    _stopWatch.addLapTime(QLatin1String("Checksum"));
+    _stopWatch.addLapTime(QLatin1String("TransmissionChecksum"));
 
     time_t prevModtime = _item->_modtime; // the _item value was set in PropagateUploadFileQNAM::start()
     // but a potential checksum calculation could have taken some time during which the file could
@@ -511,9 +520,9 @@ void PropagateUploadFileQNAM::startNextChunk()
         isFinalChunk = true;
     }
 
-    if (isFinalChunk && !_item->_transmissionChecksumType.isEmpty()) {
+    if (isFinalChunk && !_transmissionChecksumType.isEmpty()) {
         headers[checkSumHeaderC] = makeChecksumHeader(
-                _item->_transmissionChecksumType, _item->_transmissionChecksum);
+                _transmissionChecksumType, _transmissionChecksum);
     }
 
     if (! device->prepareAndOpen(_propagator->getFilePath(_item->_file), chunkStart, currentChunkSize)) {
@@ -724,7 +733,10 @@ void PropagateUploadFileQNAM::slotPutFinished()
 
     // performance logging
     _item->_requestDuration = _stopWatch.stop();
-    qDebug() << "*==* duration UPLOAD" << _item->_size << _stopWatch.durationOfLap(QLatin1String("Checksum")) << _item->_requestDuration;
+    qDebug() << "*==* duration UPLOAD" << _item->_size
+             << _stopWatch.durationOfLap(QLatin1String("ContentChecksum"))
+             << _stopWatch.durationOfLap(QLatin1String("TransmissionChecksum"))
+             << _item->_requestDuration;
 
     finalize(*_item);
 }

src/libsync/propagateupload.h

@@ -183,6 +183,9 @@ private:
     // measure the performance of checksum calc and upload
     Utility::StopWatch _stopWatch;
 
+    QByteArray _transmissionChecksum;
+    QByteArray _transmissionChecksumType;
+
 public:
     PropagateUploadFileQNAM(OwncloudPropagator* propagator,const SyncFileItemPtr& item)
         : PropagateItemJob(propagator, item), _startChunk(0), _currentChunk(0), _chunkCount(0), _transferId(0), _finished(false) {}
@@ -195,7 +198,8 @@ private slots:
     void startNextChunk();
     void finalize(const SyncFileItem&);
     void slotJobDestroyed(QObject *job);
-    void slotStartUpload(const QByteArray& checksumType, const QByteArray& checksum);
+    void slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum);
+    void slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum);
 
 private:
     void startPollJob(const QString& path);

src/libsync/propagatorjobs.cpp

@@ -209,8 +209,8 @@ void PropagateLocalRename::start()
 
     SyncJournalFileRecord record(*_item, targetFile);
     record._path = _item->_renameTarget;
-    record._transmissionChecksum = oldRecord._transmissionChecksum;
-    record._transmissionChecksumType = oldRecord._transmissionChecksumType;
+    record._contentChecksum = oldRecord._contentChecksum;
+    record._contentChecksumType = oldRecord._contentChecksumType;
 
     if (!_item->_isDirectory) { // Directories are saved at the end
         _propagator->_journal->setFileRecord(record);

src/libsync/syncengine.cpp

@@ -71,6 +71,7 @@ SyncEngine::SyncEngine(AccountPtr account, CSYNC *ctx, const QString& localPath,
   , _uploadLimit(0)
   , _downloadLimit(0)
   , _newBigFolderSizeLimit(-1)
+  , _checksum_hook(journal)
   , _anotherSyncNeeded(false)
 {
     qRegisterMetaType<SyncFileItem>("SyncFileItem");
@@ -466,10 +467,11 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
 
     int re = 0;
     switch(file->instruction) {
-    case CSYNC_INSTRUCTION_NONE:
+    case CSYNC_INSTRUCTION_NONE: {
         if (remote && item->_should_update_metadata && !item->_isDirectory && item->_instruction == CSYNC_INSTRUCTION_NONE) {
-            // Update the database now already:  New fileid or Etag or RemotePerm
+            // Update the database now already:  New remote fileid or Etag or RemotePerm
             // Or for files that were detected as "resolved conflict".
+            // Or a local inode/mtime change (see localMetadataUpdate below)
 
             // In case of "resolved conflict": there should have been a conflict because they
             // both were new, or both had their local mtime or remote etag modified, but the
@@ -496,21 +498,28 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
 
             _journal->setFileRecordMetadata(SyncJournalFileRecord(*item, filePath));
             item->_should_update_metadata = false;
+
+            // Technically we're done with this item. See localMetadataUpdate hack below.
+            _syncItemMap.remove(key);
+        }
+        // Any files that are instruction NONE?
+        if (!item->_isDirectory && file->other.instruction == CSYNC_INSTRUCTION_NONE) {
+            _hasNoneFiles = true;
         }
-        if (item->_isDirectory && file->should_update_metadata) {
-            // Because we want to still update etags of directories
-            dir = SyncFileItem::None;
-        } else {
-            // No need to do anything.
-            if (file->other.instruction == CSYNC_INSTRUCTION_NONE
-                    // Directories with ignored files does not count as 'None'
-                    && (file->type != CSYNC_FTW_TYPE_DIR || !file->has_ignored_files)) {
-                _hasNoneFiles = true;
+        // We want to still update etags of directories, other NONE
+        // items can be ignored.
+        bool directoryEtagUpdate = item->_isDirectory && file->should_update_metadata;
+        bool localMetadataUpdate = !remote && file->should_update_metadata;
+        if (!directoryEtagUpdate) {
+            if (localMetadataUpdate) {
+                // Hack, we want a local metadata update to happen, but only if the
+                // remote tree doesn't ask us to do some kind of propagation.
+                _syncItemMap.insert(key, item);
             }
-
             return re;
         }
         break;
+    }
     case CSYNC_INSTRUCTION_RENAME:
         dir = !remote ? SyncFileItem::Down : SyncFileItem::Up;
         item->_renameTarget = renameTarget;
@@ -686,6 +695,10 @@ void SyncEngine::startSync()
 
     csync_set_userdata(_csync_ctx, this);
 
+    // Set up checksumming hook
+    _csync_ctx->callbacks.checksum_hook = &CSyncChecksumHook::hook;
+    _csync_ctx->callbacks.checksum_userdata = &_checksum_hook;
+
     _stopWatch.start();
 
     qDebug() << "#### Discovery start #################################################### >>";

src/libsync/syncengine.h

@@ -37,6 +37,7 @@
 #include "syncfilestatus.h"
 #include "accountfwd.h"
 #include "discoveryphase.h"
+#include "transmissionchecksumvalidator.h"
 
 class QProcess;
 
@@ -216,6 +217,9 @@ private:
     // hash containing the permissions on the remote directory
     QHash<QString, QByteArray> _remotePerms;
 
+    /// Hook for computing checksums from csync_update
+    CSyncChecksumHook _checksum_hook;
+
     bool _anotherSyncNeeded;
 };
 

src/libsync/syncfileitem.h

@@ -165,8 +165,8 @@ public:
     quint64              _inode;
     QByteArray           _fileId;
     QByteArray           _remotePerm;
-    QByteArray           _transmissionChecksum;
-    QByteArray           _transmissionChecksumType;
+    QByteArray           _contentChecksum;
+    QByteArray           _contentChecksumType;
     QString              _directDownloadUrl;
     QString              _directDownloadCookies;
 

src/libsync/syncjournaldb.cpp

@@ -204,10 +204,13 @@ bool SyncJournalDb::checkConnect()
                          "modtime INTEGER(8),"
                          "type INTEGER,"
                          "md5 VARCHAR(32)," /* This is the etag.  Called md5 for compatibility */
-                        // updateDatabaseStructure() will add a fileid column
-                        // updateDatabaseStructure() will add a remotePerm column
-                        // updateDatabaseStructure() will add a transmissionChecksum column
-                        // updateDatabaseStructure() will add a transmissionChecksumTypeId column
+                        // updateDatabaseStructure() will add
+                        // fileid
+                        // remotePerm
+                        // filesize
+                        // ignoredChildrenRemote
+                        // contentChecksum
+                        // contentChecksumTypeId
                          "PRIMARY KEY(phash)"
                          ");");
 
@@ -358,20 +361,20 @@ bool SyncJournalDb::checkConnect()
     _getFileRecordQuery.reset(new SqlQuery(_db));
     _getFileRecordQuery->prepare(
             "SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize,"
-            "  ignoredChildrenRemote, transmissionChecksum, checksumtype.name"
+            "  ignoredChildrenRemote, contentChecksum, contentchecksumtype.name"
             " FROM metadata"
-            "  LEFT JOIN checksumtype ON metadata.transmissionChecksumTypeId == checksumtype.id"
+            "  LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id"
             " WHERE phash=?1" );
 
     _setFileRecordQuery.reset(new SqlQuery(_db) );
     _setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata "
-                                 "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, transmissionChecksum, transmissionChecksumTypeId) "
+                                 "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId) "
                                  "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7,  ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16);" );
 
     _setFileRecordChecksumQuery.reset(new SqlQuery(_db) );
     _setFileRecordChecksumQuery->prepare(
             "UPDATE metadata"
-            " SET transmissionChecksum = ?2, transmissionChecksumTypeId = ?3"
+            " SET contentChecksum = ?2, contentChecksumTypeId = ?3"
             " WHERE phash == ?1;");
 
     _getDownloadInfoQuery.reset(new SqlQuery(_db) );
@@ -426,6 +429,9 @@ bool SyncJournalDb::checkConnect()
     _getChecksumTypeIdQuery.reset(new SqlQuery(_db));
     _getChecksumTypeIdQuery->prepare("SELECT id FROM checksumtype WHERE name=?1");
 
+    _getChecksumTypeQuery.reset(new SqlQuery(_db));
+    _getChecksumTypeQuery->prepare("SELECT name FROM checksumtype WHERE id=?1");
+
     _insertChecksumTypeQuery.reset(new SqlQuery(_db));
     _insertChecksumTypeQuery->prepare("INSERT OR IGNORE INTO checksumtype (name) VALUES (?1)");
 
@@ -463,6 +469,7 @@ void SyncJournalDb::close()
     _setErrorBlacklistQuery.reset(0);
     _getSelectiveSyncListQuery.reset(0);
     _getChecksumTypeIdQuery.reset(0);
+    _getChecksumTypeQuery.reset(0);
     _insertChecksumTypeQuery.reset(0);
 
     _db.close();
@@ -557,23 +564,23 @@ bool SyncJournalDb::updateMetadataTableStructure()
         commitInternal("update database structure: add ignoredChildrenRemote col");
     }
 
-    if( columns.indexOf(QLatin1String("transmissionChecksum")) == -1 ) {
+    if( columns.indexOf(QLatin1String("contentChecksum")) == -1 ) {
         SqlQuery query(_db);
-        query.prepare("ALTER TABLE metadata ADD COLUMN transmissionChecksum TEXT;");
+        query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksum TEXT;");
         if( !query.exec()) {
-            sqlFail("updateMetadataTableStructure: add transmissionChecksum column", query);
+            sqlFail("updateMetadataTableStructure: add contentChecksum column", query);
             re = false;
         }
-        commitInternal("update database structure: add transmissionChecksum col");
+        commitInternal("update database structure: add contentChecksum col");
     }
-    if( columns.indexOf(QLatin1String("transmissionChecksumTypeId")) == -1 ) {
+    if( columns.indexOf(QLatin1String("contentChecksumTypeId")) == -1 ) {
         SqlQuery query(_db);
-        query.prepare("ALTER TABLE metadata ADD COLUMN transmissionChecksumTypeId INTEGER;");
+        query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksumTypeId INTEGER;");
         if( !query.exec()) {
-            sqlFail("updateMetadataTableStructure: add transmissionChecksumTypeId column", query);
+            sqlFail("updateMetadataTableStructure: add contentChecksumTypeId column", query);
             re = false;
         }
-        commitInternal("update database structure: add transmissionChecksumTypeId col");
+        commitInternal("update database structure: add contentChecksumTypeId col");
     }
 
 
@@ -677,7 +684,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
         if( fileId.isEmpty() ) fileId = "";
         QString remotePerm (record._remotePerm);
         if (remotePerm.isEmpty()) remotePerm = QString(); // have NULL in DB (vs empty)
-        int checksumTypeId = mapChecksumType(record._transmissionChecksumType);
+        int contentChecksumTypeId = mapChecksumType(record._contentChecksumType);
         _setFileRecordQuery->reset();
         _setFileRecordQuery->bindValue(1, QString::number(phash));
         _setFileRecordQuery->bindValue(2, plen);
@@ -693,8 +700,8 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
         _setFileRecordQuery->bindValue(12, remotePerm );
         _setFileRecordQuery->bindValue(13, record._fileSize );
         _setFileRecordQuery->bindValue(14, record._serverHasIgnoredFiles ? 1:0);
-        _setFileRecordQuery->bindValue(15, record._transmissionChecksum );
-        _setFileRecordQuery->bindValue(16, checksumTypeId );
+        _setFileRecordQuery->bindValue(15, record._contentChecksum );
+        _setFileRecordQuery->bindValue(16, contentChecksumTypeId );
 
         if( !_setFileRecordQuery->exec() ) {
             qWarning() << "Error SQL statement setFileRecord: " << _setFileRecordQuery->lastQuery() <<  " :"
@@ -705,7 +712,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
         qDebug() <<  _setFileRecordQuery->lastQuery() << phash << plen << record._path << record._inode
                  << QString::number(Utility::qDateTimeToTime_t(record._modtime)) << QString::number(record._type)
                  << record._etag << record._fileId << record._remotePerm << record._fileSize << (record._serverHasIgnoredFiles ? 1:0)
-                 << record._transmissionChecksum << record._transmissionChecksumType << checksumTypeId;
+                 << record._contentChecksum << record._contentChecksumType << contentChecksumTypeId;
 
         _setFileRecordQuery->reset();
         return true;
@@ -785,9 +792,9 @@ SyncJournalFileRecord SyncJournalDb::getFileRecord( const QString& filename )
             rec._remotePerm = _getFileRecordQuery->baValue(9);
             rec._fileSize   = _getFileRecordQuery->int64Value(10);
             rec._serverHasIgnoredFiles = (_getFileRecordQuery->intValue(11) > 0);
-            rec._transmissionChecksum = _getFileRecordQuery->baValue(12);
+            rec._contentChecksum = _getFileRecordQuery->baValue(12);
             if( !_getFileRecordQuery->nullValue(13) ) {
-                rec._transmissionChecksumType = _getFileRecordQuery->baValue(13);
+                rec._contentChecksumType = _getFileRecordQuery->baValue(13);
             }
         } else {
             QString err = _getFileRecordQuery->error();
@@ -878,8 +885,8 @@ int SyncJournalDb::getFileRecordCount()
 }
 
 bool SyncJournalDb::updateFileRecordChecksum(const QString& filename,
-                                             const QByteArray& transmisisonChecksum,
-                                             const QByteArray& transmissionChecksumType)
+                                             const QByteArray& contentChecksum,
+                                             const QByteArray& contentChecksumType)
 {
     QMutexLocker locker(&_mutex);
 
@@ -889,12 +896,12 @@ bool SyncJournalDb::updateFileRecordChecksum(const QString& filename,
         return false;
     }
 
-    int checksumTypeId = mapChecksumType(transmissionChecksumType);
+    int checksumTypeId = mapChecksumType(contentChecksumType);
     auto & query = _setFileRecordChecksumQuery;
 
     query->reset();
     query->bindValue(1, QString::number(phash));
-    query->bindValue(2, transmisisonChecksum);
+    query->bindValue(2, contentChecksum);
     query->bindValue(3, checksumTypeId);
 
     if( !query->exec() ) {
@@ -904,8 +911,8 @@ bool SyncJournalDb::updateFileRecordChecksum(const QString& filename,
         return false;
     }
 
-    qDebug() << query->lastQuery() << phash << transmisisonChecksum
-             << transmissionChecksumType << checksumTypeId;
+    qDebug() << query->lastQuery() << phash << contentChecksum
+             << contentChecksumType << checksumTypeId;
 
     query->reset();
     return true;
@@ -1501,6 +1508,32 @@ void SyncJournalDb::forceRemoteDiscoveryNextSyncLocked()
     }
 }
 
+
+QByteArray SyncJournalDb::getChecksumType(int checksumTypeId)
+{
+    QMutexLocker locker(&_mutex);
+    if( !checkConnect() ) {
+        return QByteArray();
+    }
+
+    // Retrieve the id
+    auto & query = *_getChecksumTypeQuery;
+    query.reset();
+    query.bindValue(1, checksumTypeId);
+    if( !query.exec() ) {
+        qWarning() << "Error SQL statement getChecksumType: "
+                   << query.lastQuery() <<  " :"
+                   << query.error();
+        return 0;
+    }
+
+    if( !query.next() ) {
+        qDebug() << "No checksum type mapping found for" << checksumTypeId;
+        return 0;
+    }
+    return query.baValue(0);
+}
+
 int SyncJournalDb::mapChecksumType(const QByteArray& checksumType)
 {
     if (checksumType.isEmpty()) {

src/libsync/syncjournaldb.h

@@ -47,8 +47,8 @@ public:
     bool deleteFileRecord( const QString& filename, bool recursively = false );
     int getFileRecordCount();
     bool updateFileRecordChecksum(const QString& filename,
-                                  const QByteArray& transmisisonChecksum,
-                                  const QByteArray& transmissionChecksumType);
+                                  const QByteArray& contentChecksum,
+                                  const QByteArray& contentChecksumType);
     bool exists();
     void walCheckpoint();
 
@@ -146,6 +146,11 @@ public:
      */
     bool isConnected();
 
+    /**
+     * Returns the checksum type for an id.
+     */
+    QByteArray getChecksumType(int checksumTypeId);
+
 private:
     bool updateDatabaseStructure();
     bool updateMetadataTableStructure();
@@ -186,6 +191,7 @@ private:
     QScopedPointer<SqlQuery> _setErrorBlacklistQuery;
     QScopedPointer<SqlQuery> _getSelectiveSyncListQuery;
     QScopedPointer<SqlQuery> _getChecksumTypeIdQuery;
+    QScopedPointer<SqlQuery> _getChecksumTypeQuery;
     QScopedPointer<SqlQuery> _insertChecksumTypeQuery;
 
     /* This is the list of paths we called avoidReadFromDbOnNextSync on.

src/libsync/syncjournalfilerecord.cpp

@@ -36,8 +36,8 @@ SyncJournalFileRecord::SyncJournalFileRecord(const SyncFileItem &item, const QSt
     : _path(item._file), _modtime(Utility::qDateTimeFromTime_t(item._modtime)),
       _type(item._type), _etag(item._etag), _fileId(item._fileId), _fileSize(item._size),
       _remotePerm(item._remotePerm), _serverHasIgnoredFiles(item._serverHasIgnoredFiles),
-      _transmissionChecksum(item._transmissionChecksum),
-      _transmissionChecksumType(item._transmissionChecksumType)
+      _contentChecksum(item._contentChecksum),
+      _contentChecksumType(item._contentChecksumType)
 {
     // use the "old" inode coming with the item for the case where the
     // filesystem stat fails. That can happen if the the file was removed
@@ -97,8 +97,8 @@ SyncFileItem SyncJournalFileRecord::toSyncFileItem()
     item._size = _fileSize;
     item._remotePerm = _remotePerm;
     item._serverHasIgnoredFiles = _serverHasIgnoredFiles;
-    item._transmissionChecksum = _transmissionChecksum;
-    item._transmissionChecksumType = _transmissionChecksumType;
+    item._contentChecksum = _contentChecksum;
+    item._contentChecksumType = _contentChecksumType;
     return item;
 }
 
@@ -176,8 +176,8 @@ bool operator==(const SyncJournalFileRecord & lhs,
             && lhs._fileSize == rhs._fileSize
             && lhs._remotePerm == rhs._remotePerm
             && lhs._serverHasIgnoredFiles == rhs._serverHasIgnoredFiles
-            && lhs._transmissionChecksum == rhs._transmissionChecksum
-            && lhs._transmissionChecksumType == rhs._transmissionChecksumType;
+            && lhs._contentChecksum == rhs._contentChecksum
+            && lhs._contentChecksumType == rhs._contentChecksumType;
 }
 
 }

src/libsync/syncjournalfilerecord.h

@@ -55,8 +55,8 @@ public:
     qint64     _fileSize;
     QByteArray _remotePerm;
     bool       _serverHasIgnoredFiles;
-    QByteArray _transmissionChecksum;
-    QByteArray _transmissionChecksumType;
+    QByteArray _contentChecksum;
+    QByteArray _contentChecksumType;
 };
 
 bool OWNCLOUDSYNC_EXPORT

src/libsync/transmissionchecksumvalidator.cpp

@@ -77,36 +77,40 @@ QByteArray ComputeChecksum::checksumType() const
 
 void ComputeChecksum::start(const QString& filePath)
 {
-    const QString csType = checksumType();
-
     // Calculate the checksum in a different thread first.
     connect( &_watcher, SIGNAL(finished()),
              this, SLOT(slotCalculationDone()),
              Qt::UniqueConnection );
-    if( csType == checkSumMD5C ) {
-        _watcher.setFuture(QtConcurrent::run(FileSystem::calcMd5, filePath));
+    _watcher.setFuture(QtConcurrent::run(ComputeChecksum::computeNow, filePath, checksumType()));
+}
 
-    } else if( csType == checkSumSHA1C ) {
-        _watcher.setFuture(QtConcurrent::run( FileSystem::calcSha1, filePath));
+QByteArray ComputeChecksum::computeNow(const QString& filePath, const QByteArray& checksumType)
+{
+    if( checksumType == checkSumMD5C ) {
+        return FileSystem::calcMd5(filePath);
+    } else if( checksumType == checkSumSHA1C ) {
+        return FileSystem::calcSha1(filePath);
     }
 #ifdef ZLIB_FOUND
-    else if( csType == checkSumAdlerC) {
-        _watcher.setFuture(QtConcurrent::run(FileSystem::calcAdler32, filePath));
+    else if( checksumType == checkSumAdlerC) {
+        return FileSystem::calcAdler32(filePath);
     }
 #endif
-    else {
-        // for an unknown checksum or no checksum, we're done right now
-        if( !csType.isEmpty() ) {
-            qDebug() << "Unknown checksum type:" << csType;
-        }
-        emit done(QByteArray(), QByteArray());
+    // for an unknown checksum or no checksum, we're done right now
+    if( !checksumType.isEmpty() ) {
+        qDebug() << "Unknown checksum type:" << checksumType;
     }
+    return QByteArray();
 }
 
 void ComputeChecksum::slotCalculationDone()
 {
     QByteArray checksum = _watcher.future().result();
-    emit done(_checksumType, checksum);
+    if (!checksum.isNull()) {
+        emit done(_checksumType, checksum);
+    } else {
+        emit done(QByteArray(), QByteArray());
+    }
 }
 
 
@@ -151,4 +155,32 @@ void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray& checksumTy
     emit validated(checksumType, checksum);
 }
 
+CSyncChecksumHook::CSyncChecksumHook(SyncJournalDb *journal)
+    : _journal(journal)
+{
+}
+
+bool CSyncChecksumHook::hook(const char* path, uint32_t checksumTypeId, const char* checksum, void *this_obj)
+{
+    CSyncChecksumHook* checksumHook = static_cast<CSyncChecksumHook*>(this_obj);
+    return checksumHook->check(QString::fromUtf8(path), checksumTypeId, QByteArray(checksum));
+}
+
+bool CSyncChecksumHook::check(const QString& path, int checksumTypeId, const QByteArray& checksum)
+{
+    QByteArray checksumType = _journal->getChecksumType(checksumTypeId);
+    if (checksumType.isEmpty()) {
+        qDebug() << "Checksum type" << checksumTypeId << "not found";
+        return false;
+    }
+
+    QByteArray newChecksum = ComputeChecksum::computeNow(path, checksumType);
+    if (newChecksum.isNull()) {
+        qDebug() << "Failed to compute checksum" << checksumType << "for" << path;
+        return false;
+    }
+    return newChecksum == checksum;
+}
+
+
 }

src/libsync/transmissionchecksumvalidator.h

@@ -23,6 +23,8 @@
 
 namespace OCC {
 
+class SyncJournalDb;
+
 /// Creates a checksum header from type and value.
 QByteArray makeChecksumHeader(const QByteArray& checksumType, const QByteArray& checksum);
 
@@ -59,6 +61,11 @@ public:
      */
     void start(const QString& filePath);
 
+    /**
+     * Computes the checksum synchronously.
+     */
+    static QByteArray computeNow(const QString& filePath, const QByteArray& checksumType);
+
 signals:
     void done(const QByteArray& checksumType, const QByteArray& checksum);
 
@@ -103,4 +110,29 @@ private:
     QByteArray _expectedChecksum;
 };
 
+/**
+ * Hooks checksum computations into csync.
+ * @ingroup libsync
+ */
+class OWNCLOUDSYNC_EXPORT CSyncChecksumHook : public QObject
+{
+    Q_OBJECT
+public:
+    explicit CSyncChecksumHook(SyncJournalDb* journal);
+
+    /**
+     * Returns true if the checksum for \a path is the same as the one provided.
+     *
+     * Called from csync, where a instance of CSyncChecksumHook
+     * has to be set as userdata.
+     */
+    static bool hook(const char* path, uint32_t checksumTypeId, const char* checksum,
+                     void* this_obj);
+
+    bool check(const QString& path, int checksumTypeId, const QByteArray& checksum);
+
+private:
+    SyncJournalDb* _journal;
+};
+
 }

test/testsyncjournaldb.h

@@ -59,17 +59,17 @@ private slots:
         record._fileId = "abcd";
         record._remotePerm = "744";
         record._fileSize = 213089055;
-        record._transmissionChecksum = "mychecksum";
-        record._transmissionChecksumType = "MD5";
+        record._contentChecksum = "mychecksum";
+        record._contentChecksumType = "MD5";
         QVERIFY(_db.setFileRecord(record));
 
         SyncJournalFileRecord storedRecord = _db.getFileRecord("foo");
         QVERIFY(storedRecord == record);
 
         // Update checksum
-        record._transmissionChecksum = "newchecksum";
-        record._transmissionChecksumType = "Adler32";
-        _db.updateFileRecordChecksum("foo", record._transmissionChecksum, record._transmissionChecksumType);
+        record._contentChecksum = "newchecksum";
+        record._contentChecksumType = "Adler32";
+        _db.updateFileRecordChecksum("foo", record._contentChecksum, record._contentChecksumType);
         storedRecord = _db.getFileRecord("foo");
         QVERIFY(storedRecord == record);
 
@@ -97,8 +97,8 @@ private slots:
             SyncJournalFileRecord record;
             record._path = "foo-checksum";
             record._remotePerm = "744";
-            record._transmissionChecksum = "mychecksum";
-            record._transmissionChecksumType = "MD5";
+            record._contentChecksum = "mychecksum";
+            record._contentChecksumType = "MD5";
             QVERIFY(_db.setFileRecord(record));
 
             SyncJournalFileRecord storedRecord = _db.getFileRecord("foo-checksum");