From b33cffde7f80977af8d077f10a93d871d855d85c Mon Sep 17 00:00:00 2001 From: Mike Ryan Date: Thu, 4 Apr 2013 10:21:13 -0400 Subject: [PATCH] Issue #1835822 by mikeryan: Hash source rows to detect changes --- includes/map.inc | 4 ++- includes/migration.inc | 14 ++++++---- includes/source.inc | 64 +++++++++++++++++++++++++++++++++++++++++----- plugins/sources/sqlmap.inc | 21 ++++++++++++++- 5 files changed, 90 insertions(+), 14 deletions(-) diff --git a/includes/map.inc b/includes/map.inc index ed76db5..a2e2bfb 100644 --- a/includes/map.inc +++ b/includes/map.inc @@ -58,9 +58,11 @@ abstract class MigrateMap implements Iterator { * @param $dest_ids * @param $status * @param $rollback_action + * @param $hash */ abstract public function saveIDMapping(stdClass $source_row, array $dest_ids, - $status = MigrateMap::STATUS_IMPORTED, $rollback_action = MigrateMap::ROLLBACK_DELETE); + $status = MigrateMap::STATUS_IMPORTED, + $rollback_action = MigrateMap::ROLLBACK_DELETE, $hash = NULL); /** * Record a message related to a source record diff --git a/includes/migration.inc b/includes/migration.inc index a4f9562..1df68e9 100644 --- a/includes/migration.inc +++ b/includes/migration.inc @@ -563,14 +563,16 @@ abstract class Migration extends MigrationBase { $ids = $this->destination->import($this->destinationValues, $this->sourceValues); migrate_instrument_stop('destination import'); if ($ids) { - $this->map->saveIDMapping($this->sourceValues, $ids, $this->needsUpdate, - $this->rollbackAction); + $this->map->saveIDMapping($this->sourceValues, $ids, + $this->needsUpdate, $this->rollbackAction, + $data_row->migrate_map_hash); $this->successes_since_feedback++; $this->total_successes++; } else { $this->map->saveIDMapping($this->sourceValues, array(), - MigrateMap::STATUS_FAILED, $this->rollbackAction); + MigrateMap::STATUS_FAILED, $this->rollbackAction, + $data_row->migrate_map_hash); $message = t('New object was not saved, no error provided'); $this->saveMessage($message); self::displayMessage($message); @@ -578,13 +580,15 @@ abstract class Migration extends MigrationBase { } catch (MigrateException $e) { $this->map->saveIDMapping($this->sourceValues, array(), - MigrateMap::STATUS_FAILED, $this->rollbackAction); + MigrateMap::STATUS_FAILED, $this->rollbackAction, + $data_row->migrate_map_hash); $this->saveMessage($e->getMessage(), $e->getLevel()); self::displayMessage($e->getMessage()); } catch (Exception $e) { $this->map->saveIDMapping($this->sourceValues, array(), - MigrateMap::STATUS_FAILED, $this->rollbackAction); + MigrateMap::STATUS_FAILED, $this->rollbackAction, + $data_row->migrate_map_hash); $this->handleException($e); } $this->total_processed++; diff --git a/includes/source.inc b/includes/source.inc index 14fca25..b16f68f 100644 --- a/includes/source.inc +++ b/includes/source.inc @@ -117,6 +117,14 @@ abstract class MigrateSource implements Iterator { protected $skipCount = FALSE; /** + * If TRUE, we will maintain hashed source rows to determine whether incoming + * data has changed. + * + * @var bool + */ + protected $trackHashes = FALSE; + + /** * By default, next() will directly read the map row and add it to the data * row. A source plugin implementation may do this itself (in particular, the * SQL source can incorporate the map table into the query) - if so, it should @@ -186,6 +194,9 @@ abstract class MigrateSource implements Iterator { if (!empty($options['cache_key'])) { $this->cacheKey = $options['cache_key']; } + if (!empty($options['track_hashes'])) { + $this->trackHashes = $options['track_hashes']; + } } /** @@ -248,9 +259,15 @@ abstract class MigrateSource implements Iterator { public function next() { $this->currentKey = NULL; $this->currentRow = NULL; + $hash = NULL; migrate_instrument_start(get_class($this) . ' getNextRow'); while ($row = $this->getNextRow()) { migrate_instrument_stop(get_class($this) . ' getNextRow'); + + if ($this->trackHashes) { + $hash = $this->hash($row); + } + // Populate the source key for this row $this->currentKey = $this->activeMigration->prepareKey( $this->activeMap->getSourceKey(), $row); @@ -267,8 +284,8 @@ abstract class MigrateSource implements Iterator { } } - // First, determine if this row should be passed to prepareRow(), or skipped - // entirely. The rules are: + // First, determine if this row should be passed to prepareRow(), or + // skipped entirely. The rules are: // 1. If there's an explicit idlist, that's all we care about (ignore // highwaters and map rows). $prepared = FALSE; @@ -293,11 +310,22 @@ abstract class MigrateSource implements Iterator { } // 4. At this point, we have a row which has previously been imported and // not marked for update. If we're not using highwater marks, then we - // will not take this row. + // will not take this row, unless we're hashing the rows and the hash + // has changed. elseif (empty($this->highwaterField)) { - // No highwater, skip - $this->currentRow = NULL; - continue; + if ($this->trackHashes) { + if ($hash == $row->migrate_map_hash) { + // No change in the data, skip + $this->currentRow = NULL; + continue; + } + // The data has changed, so fall through to process this row. + } + else { + // No highwater, skip + $this->currentRow = NULL; + continue; + } } // 5. The initial highwater mark, before anything is migrated, is ''. We // want to make sure we don't mistakenly skip rows with a highwater @@ -326,6 +354,10 @@ abstract class MigrateSource implements Iterator { $prepared = TRUE; } + // Stash the hash where the migration can find it, and ultimately save it + // to the map. + $row->migrate_map_hash = $hash; + // Allow the Migration to prepare this row. prepareRow() can return boolean // FALSE to ignore this row. if (!$prepared) { @@ -359,7 +391,8 @@ abstract class MigrateSource implements Iterator { // We're explicitly skipping this row - keep track in the map table if ($return === FALSE) { $this->activeMigration->getMap()->saveIDMapping($row, array(), - MigrateMap::STATUS_IGNORED, $this->activeMigration->rollbackAction); + MigrateMap::STATUS_IGNORED, $this->activeMigration->rollbackAction, + $row->migrate_map_hash); $this->numIgnored++; $this->currentRow = NULL; $this->currentKey = NULL; @@ -370,4 +403,21 @@ abstract class MigrateSource implements Iterator { $this->numProcessed++; return $return; } + + /** + * + * @param $row + * + * @return string + */ + protected function hash($row) { + migrate_instrument_start('MigrateSource::hash'); + // We make sure that serialize always has the same effect for the same data + // by sorting it by property name. + $array = (array)$row; + ksort($array); + $hash = md5(serialize($array)); + migrate_instrument_stop('MigrateSource::hash'); + return $hash; + } } diff --git a/plugins/sources/sqlmap.inc b/plugins/sources/sqlmap.inc index 9d2b3bb..6448e72 100644 --- a/plugins/sources/sqlmap.inc +++ b/plugins/sources/sqlmap.inc @@ -147,6 +147,15 @@ class MigrateSQLMap extends MigrateMap { 'default' => 0, 'description' => 'UNIX timestamp of the last time this row was imported', ); + $fields['hash'] = array( + 'type' => 'varchar', + 'length' => '32', + 'not null' => FALSE, + 'description' => 'Hash of source row data, for detecting changes', + ); + $schema = array( + 'description' => t('Messages generated during a migration process'), + 'fields' => $fields, $schema = array( 'description' => t('Mappings from source key to destination key'), 'fields' => $fields, @@ -300,10 +309,12 @@ class MigrateSQLMap extends MigrateMap { * @param int $rollback_action * How to handle the destination object on rollback. Defaults to * ROLLBACK_DELETE. + * @param string $hash + * If hashing is enabled, the hash of the raw source row. */ public function saveIDMapping(stdClass $source_row, array $dest_ids, - $needs_update = MigrateMap::STATUS_IMPORTED, $rollback_action = MigrateMap::ROLLBACK_DELETE) { - migrate_instrument_start('saveIDMapping'); + $needs_update = MigrateMap::STATUS_IMPORTED, $rollback_action = MigrateMap::ROLLBACK_DELETE + , $hash = NULL) { migrate_instrument_start('saveIDMapping'); // Construct the source key $keys = array(); foreach ($this->sourceKeyMap as $field_name => $key_name) { @@ -320,6 +332,7 @@ class MigrateSQLMap extends MigrateMap { $fields = array( 'needs_update' => (int)$needs_update, 'rollback_action' => (int)$rollback_action, + 'hash' => $hash, ); $count = 1; if (!empty($dest_ids)) { -- 1.8.2.1