diff --git a/core/modules/migrate/config/migrate.migration.d6_system_site.yml b/core/modules/migrate/config/migrate.migration.d6_system_site.yml new file mode 100644 index 0000000..dd1428c --- /dev/null +++ b/core/modules/migrate/config/migrate.migration.d6_system_site.yml @@ -0,0 +1,40 @@ +id: d6_system_site +source: + plugin: drupal6_variable + variables: + - site_name + - site_mail + - site_slogan + - site_frontpage + - site_403 + - site_404 + - drupal_weight_select_max + - admin_compact_mode +process: + - + source: site_name + destination: name + - + source: site_mail + destination: mail + - + source: site_slogan + destination: slogan + - + source: site_frontpage + destination: page:front + - + source: site_403 + destination: page:403 + - + source: site_404 + destination: page:404 + - + source: drupal_weight_select_max + destination: weight_select_max + - + source: admin_compact_mode + destination: admin_compact_mode +destination: + plugin: d8_config + config_name: system.site diff --git a/core/modules/migrate/lib/Drupal/migrate/DrupalMessage.php b/core/modules/migrate/lib/Drupal/migrate/DrupalMessage.php new file mode 100644 index 0000000..4d2490c --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/DrupalMessage.php @@ -0,0 +1,15 @@ +sourcePlugin)) { + $this->sourcePlugin = \Drupal::service('plugin.manager.migrate.source')->createInstance($this->source['plugin'], $this->source, $this); + } + return $this->sourcePlugin; + } + + /** + * @return \Drupal\migrate\Plugin\MigrateProcessBag + */ + public function getProcess() { + if (!$this->migrateProcessBag) { + $this->migrateProcessBag = new MigrateProcessBag(\Drupal::service('plugin.manager.migrate.process'), $this->process, $this); + } + return $this->migrateProcessBag; + } + + /** + * @return \Drupal\migrate\Plugin\MigrateDestinationInterface + */ + public function getDestination() { + if (!isset($this->destinationPlugin)) { + $this->destinationPlugin = \Drupal::service('plugin.manager.migrate.destination')->createInstance($this->destination['plugin'], $this->destination, $this); + } + return $this->destinationPlugin; + } + + /** + * @return \Drupal\migrate\Plugin\MigrateIdMapInterface + */ + public function getIdMap() { + if (!isset($this->idMapPlugin)) { + $configuration = $this->idMap; + $plugin = isset($configuration['plugin']) ? $configuration['plugin'] : 'sql'; + $this->idMapPlugin = \Drupal::service('plugin.manager.migrate.id_map')->createInstance($plugin, $configuration, $this); + } + return $this->idMapPlugin; + } + + /** + * @return \Drupal\Core\KeyValueStore\KeyValueStoreInterface + */ + protected function getHigherWaterStorage() { + if (!isset($this->highwaterStorage)) { + $this->highwaterStorage = \Drupal::keyValue('migrate:highwater'); + } + return $this->highwaterStorage; + } + + public function getHighwater() { + return $this->getHigherWaterStorage()->get($this->id()); + } + + public function saveHighwater($highwater) { + $this->getHigherWaterStorage()->set($this->id(), $highwater); + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Entity/MigrationInterface.php b/core/modules/migrate/lib/Drupal/migrate/Entity/MigrationInterface.php new file mode 100644 index 0000000..916cfb5 --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Entity/MigrationInterface.php @@ -0,0 +1,77 @@ +level; + } + + /** + * The status to record in the map table for the current item (a + * MigrateMap::STATUS_* constant) + * + * @var int + */ + protected $status; + public function getStatus() { + return $this->status; + } + + public function __construct($message, $level = MigrationInterface::MESSAGE_ERROR, $status = MigrateIdMapInterface::STATUS_FAILED) { + $this->level = $level; + $this->status = $status; + parent::__construct($message); + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/MigrateExecutable.php b/core/modules/migrate/lib/Drupal/migrate/MigrateExecutable.php new file mode 100644 index 0000000..968725d --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/MigrateExecutable.php @@ -0,0 +1,433 @@ +migration = $migration; + $this->message = $message; + $this->migration->getIdMap()->setMessage($message); + // Record the memory limit in bytes + $limit = trim(ini_get('memory_limit')); + if ($limit == '-1') { + $this->memoryLimit = PHP_INT_MAX; + } + else { + if (!is_numeric($limit)) { + $last = strtolower(substr($limit, -1)); + switch ($last) { + case 'g': + $limit *= 1024; + case 'm': + $limit *= 1024; + case 'k': + $limit *= 1024; + break; + default: + throw new \Exception(t('Invalid PHP memory_limit !limit', + array('!limit' => $limit))); + } + } + $this->memoryLimit = $limit; + } + } + + /** + * @return \Drupal\migrate\Source + */ + public function getSource() { + if (!isset($this->source)) { + $this->source = new Source($this->migration); + } + return $this->source; + } + + /** + * The rollback action to be saved for the current row. + * + * @var int + */ + public $rollbackAction; + + /** + * An array of counts. Initially used for cache hit/miss tracking. + * + * @var array + */ + protected $counts = array(); + + /** + * When performing a bulkRollback(), the maximum number of items to pass in + * a single call. Can be overridden in derived class constructor. + * + * @var int + */ + protected $rollbackBatchSize = 50; + + /** + * The object currently being constructed + * @var \stdClass + */ + protected $destinationValues; + + /** + * The current data row retrieved from the source. + * @var \stdClass + */ + protected $sourceValues; + + /** + * Perform an import operation - migrate items from source to destination. + */ + public function import() { + $return = MigrationInterface::RESULT_COMPLETED; + $source = $this->getSource(); + $destination = $this->migration->getDestination(); + $id_map = $this->migration->getIdMap(); + + try { + $source->rewind(); + } + catch (\Exception $e) { + $this->message->display( + t('Migration failed with source plugin exception: !e', + array('!e' => $e->getMessage()))); + return MigrationInterface::RESULT_FAILED; + } + while ($this->getSource()->valid()) { + $row = $source->current(); + $this->sourceIdValues = $row->getSourceIdValues(); + + // Wipe old messages, and save any new messages. + $id_map->delete($row->getSourceIdValues(), TRUE); + $this->saveQueuedMessages(); + + $this->processRow($row); + + try { + $destination_id_values = $destination->import($row); + if ($destination_id_values) { + $id_map->saveIDMapping($row, $destination_id_values, $this->needsUpdate, $this->rollbackAction); + $this->successes_since_feedback++; + $this->total_successes++; + } + else { + $id_map->saveIDMapping($row, array(), MigrateIdMapInterface::STATUS_FAILED, $this->rollbackAction); + if ($id_map->messageCount() == 0) { + $message = t('New object was not saved, no error provided'); + $this->saveMessage($message); + $this->message->display($message); + } + } + } + catch (\MigrateException $e) { + $this->migration->getIdMap()->saveIDMapping($row, array(), $e->getStatus(), $this->rollbackAction); + $this->saveMessage($e->getMessage(), $e->getLevel()); + $this->message->display($e->getMessage()); + } + catch (\Exception $e) { + $this->migration->getIdMap()->saveIDMapping($row, array(), MigrateIdMapInterface::STATUS_FAILED, $this->rollbackAction); + $this->handleException($e); + } + $this->total_processed++; + $this->processed_since_feedback++; + if ($highwater_property = $this->migration->get('highwaterProperty')) { + $this->migration->saveHighwater($row->getSourceProperty($highwater_property['name'])); + } + + // Reset row properties. + unset($sourceValues, $destinationValues); + $this->needsUpdate = MigrateIdMapInterface::STATUS_IMPORTED; + + // TODO: Temporary. Remove when http://drupal.org/node/375494 is committed. + // TODO: Should be done in MigrateDestinationEntity + if (!empty($destination->entityType)) { + entity_get_controller($destination->entityType)->resetCache(); + } + + if ($this->timeOptionExceeded()) { + break; + } + if (($return = $this->checkStatus()) != MigrationInterface::RESULT_COMPLETED) { + break; + } + if ($this->timeOptionExceeded()) { + break; + } + try { + $source->next(); + } + catch (\Exception $e) { + $this->message->display( + t('Migration failed with source plugin exception: !e', + array('!e' => $e->getMessage()))); + return MigrationInterface::RESULT_FAILED; + } + } + + /** + * @TODO uncomment this + */ + #$this->progressMessage($return); + + return $return; + } + + /** + * Apply field mappings to a data row received from the source, returning + * a populated destination object. + */ + protected function processRow(Row $row) { + foreach ($this->migration->getProcess() as $process) { + $process->apply($row, $this); + } + } + + /** + * Fetch the key array for the current source record. + * + * @return array + */ + protected function currentSourceIds() { + return $this->getSource()->getCurrentIds(); + } + + /** + * Test whether we've exceeded the designated time limit. + * + * @return boolean + * TRUE if the threshold is exceeded, FALSE if not. + */ + protected function timeOptionExceeded() { + if (!$time_limit = $this->getTimeLimit()) { + return FALSE; + } + $time_elapsed = time() - REQUEST_TIME; + if ($time_elapsed >= $time_limit) { + return TRUE; + } + else { + return FALSE; + } + } + + public function getTimeLimit() { + if (isset($this->options['limit']) && + ($this->options['limit']['unit'] == 'seconds' || $this->options['limit']['unit'] == 'second')) { + return $this->options['limit']['value']; + } + else { + return NULL; + } + } + + /** + * Pass messages through to the map class. + * + * @param string $message + * The message to record. + * @param int $level + * Optional message severity (defaults to MESSAGE_ERROR). + */ + public function saveMessage($message, $level = MigrationInterface::MESSAGE_ERROR) { + $this->migration->getIdMap()->saveMessage($this->sourceIdValues, $message, $level); + } + + /** + * Queue messages to be later saved through the map class. + * + * @param string $message + * The message to record. + * @param int $level + * Optional message severity (defaults to MESSAGE_ERROR). + */ + public function queueMessage($message, $level = MigrationInterface::MESSAGE_ERROR) { + $this->queuedMessages[] = array('message' => $message, 'level' => $level); + } + + /** + * Save any messages we've queued up to the message table. + */ + public function saveQueuedMessages() { + foreach ($this->queuedMessages as $queued_message) { + $this->saveMessage($queued_message['message'], $queued_message['level']); + } + $this->queuedMessages = array(); + } + + /** + * Standard top-of-loop stuff, common between rollback and import - check + * for exceptional conditions, and display feedback. + */ + protected function checkStatus() { + if ($this->memoryExceeded()) { + return MigrationInterface::RESULT_INCOMPLETE; + } + if ($this->timeExceeded()) { + return MigrationInterface::RESULT_INCOMPLETE; + } + /* + * @TODO uncomment this + if ($this->getStatus() == MigrationInterface::STATUS_STOPPING) { + return MigrationBase::RESULT_STOPPED; + } + */ + // If feedback is requested, produce a progress message at the proper time + /* + * @TODO uncomment this + if (isset($this->feedback)) { + if (($this->feedback_unit == 'seconds' && time() - $this->lastfeedback >= $this->feedback) || + ($this->feedback_unit == 'items' && $this->processed_since_feedback >= $this->feedback)) { + $this->progressMessage(MigrationInterface::RESULT_INCOMPLETE); + } + } + */ + + return MigrationInterface::RESULT_COMPLETED; + } + + /** + * Test whether we've exceeded the desired memory threshold. If so, output a message. + * + * @return boolean + * TRUE if the threshold is exceeded, FALSE if not. + */ + protected function memoryExceeded() { + $usage = memory_get_usage(); + $pct_memory = $usage / $this->memoryLimit; + if ($pct_memory > $this->memoryThreshold) { + $this->message->display( + t('Memory usage is !usage (!pct% of limit !limit), resetting statics', + array('!pct' => round($pct_memory*100), + '!usage' => format_size($usage), + '!limit' => format_size($this->memoryLimit))), + 'warning'); + // First, try resetting Drupal's static storage - this frequently releases + // plenty of memory to continue + drupal_static_reset(); + $usage = memory_get_usage(); + $pct_memory = $usage/$this->memoryLimit; + // Use a lower threshold - we don't want to be in a situation where we keep + // coming back here and trimming a tiny amount + if ($pct_memory > (.90 * $this->memoryThreshold)) { + $this->message->display( + t('Memory usage is now !usage (!pct% of limit !limit), not enough reclaimed, starting new batch', + array('!pct' => round($pct_memory*100), + '!usage' => format_size($usage), + '!limit' => format_size($this->memoryLimit))), + 'warning'); + return TRUE; + } + else { + $this->message->display( + t('Memory usage is now !usage (!pct% of limit !limit), reclaimed enough, continuing', + array('!pct' => round($pct_memory*100), + '!usage' => format_size($usage), + '!limit' => format_size($this->memoryLimit))), + 'warning'); + return FALSE; + } + } + else { + return FALSE; + } + } + + /** + * Test whether we're approaching the PHP time limit. + * + * @return boolean + * TRUE if the threshold is exceeded, FALSE if not. + */ + protected function timeExceeded() { + if ($this->timeLimit == 0) { + return FALSE; + } + $time_elapsed = time() - REQUEST_TIME; + $pct_time = $time_elapsed / $this->timeLimit; + if ($pct_time > $this->timeThreshold) { + return TRUE; + } + else { + return FALSE; + } + } + + /** + * Takes an Exception object and both saves and displays it, pulling additional + * information on the location triggering the exception. + * + * @param \Exception $exception + * Object representing the exception. + * @param boolean $save + * Whether to save the message in the migration's mapping table. Set to FALSE + * in contexts where this doesn't make sense. + */ + public function handleException($exception, $save = TRUE) { + $result = _drupal_decode_exception($exception); + $message = $result['!message'] . ' (' . $result['%file'] . ':' . $result['%line'] . ')'; + if ($save) { + $this->saveMessage($message); + } + $this->message->display($message); + } + +} diff --git a/core/modules/migrate/lib/Drupal/migrate/MigrateMessageInterface.php b/core/modules/migrate/lib/Drupal/migrate/MigrateMessageInterface.php new file mode 100644 index 0000000..6553ea7 --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/MigrateMessageInterface.php @@ -0,0 +1,13 @@ +alterInfo($module_handler, 'migrate_' . $type . '_info'); + $this->setCacheBackend($cache_backend, $language_manager, 'migrate_plugins_' . $type); + } + + /** + * {@inheritdoc} + * + * A specific createInstance method is necessary to pass the migration on. + */ + public function createInstance($plugin_id, array $configuration, MigrationInterface $migration = NULL) { + $plugin_definition = $this->discovery->getDefinition($plugin_id); + $plugin_class = DefaultFactory::getPluginClass($plugin_id, $plugin_definition); + // If the plugin provides a factory method, pass the container to it. + if (is_subclass_of($plugin_class, 'Drupal\Core\Plugin\ContainerFactoryPluginInterface')) { + return $plugin_class::create(\Drupal::getContainer(), $configuration, $plugin_id, $plugin_definition, $migration); + } + return new $plugin_class($configuration, $plugin_id, $plugin_definition, $migration); + } + +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Plugin/MigrateProcessBag.php b/core/modules/migrate/lib/Drupal/migrate/Plugin/MigrateProcessBag.php new file mode 100644 index 0000000..d519920 --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Plugin/MigrateProcessBag.php @@ -0,0 +1,51 @@ +migration = $migration; + } + + /** + * {@inheritdoc} + * + * @return \Drupal\migrate\Plugin\ProcessInterface + */ + public function &get($instance_id) { + return parent::get($instance_id); + } + + /** + * {@inheritdoc} + */ + public function initializePlugin($instance_id) { + $this->configurations[$instance_id] += array('plugin' => 'property_map'); + $configuration = isset($this->configurations[$instance_id]) ? $this->configurations[$instance_id] : array(); + if (!isset($configuration[$this->pluginKey])) { + throw new UnknownPluginException($instance_id); + } + $this->pluginInstances[$instance_id] = $this->manager->createInstance($configuration[$this->pluginKey], $configuration, $this->migration); + $this->addInstanceID($instance_id); + } + +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Plugin/MigrateSourceInterface.php b/core/modules/migrate/lib/Drupal/migrate/Plugin/MigrateSourceInterface.php new file mode 100644 index 0000000..0818c45 --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Plugin/MigrateSourceInterface.php @@ -0,0 +1,44 @@ +config = $config; + } + + public static function create(ContainerInterface $container, array $configuration, $plugin_id, array $plugin_definition) { + return new static( + $configuration, + $plugin_id, + $plugin_definition, + $container->get('config.factory')->get($configuration['config_name']) + ); + } + + /** + * {@inheritdoc} + */ + public function import(Row $row) { + $this->config + ->setData($row->getDestination()) + ->save(); + } + + public function rollbackMultiple(array $destination_keys) { + throw new \MigrateException('Configuration can not be rolled back'); + } + + public function fields(Migration $migration = NULL) { + // @todo Dynamically fetch fields using Config Schema API. + } + + public function getIdsSchema() { + return array($this->config->getName() => array()); + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/destination/DestinationBase.php b/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/destination/DestinationBase.php new file mode 100644 index 0000000..7cdf11b --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/destination/DestinationBase.php @@ -0,0 +1,53 @@ +mapTable; + } + public function getMessageTable() { + return $this->messageTable; + } + + /** + * @var \Drupal\Core\Database\Connection + */ + protected $database; + + /** + * @var \Drupal\Core\Database\Query\SelectInterface + */ + protected $query; + + /** + * @var \Drupal\migrate\Entity\MigrationInterface + */ + protected $migration; + + + /** + * Qualifying the map table name with the database name makes cross-db joins + * possible. Note that, because prefixes are applied after we do this (i.e., + * it will prefix the string we return), we do not qualify the table if it has + * a prefix. This will work fine when the source data is in the default + * (prefixed) database (in particular, for simpletest), but not if the primary + * query is in an external database. + * + * @return string + */ + public function getQualifiedMapTable() { + $options = $this->getDatabase()->getConnectionOptions(); + $prefix = $this->getDatabase()->tablePrefix($this->mapTable); + if ($prefix) { + return $this->mapTable; + } + else { + return $options['database'] . '.' . $this->mapTable; + } + } + + /** + * We don't need to check the tables more than once per request. + * + * @var boolean + */ + protected $ensured; + + public function __construct($configuration, $plugin_id, $plugin_definition, MigrationInterface $migration) { + parent::__construct($configuration, $plugin_id, $plugin_definition); + $this->migration = $migration; + $machine_name = $migration->id(); + + // Default generated table names, limited to 63 characters + $prefixLength = strlen($this->getDatabase()->tablePrefix()) ; + $this->mapTable = 'migrate_map_' . drupal_strtolower($machine_name); + $this->mapTable = drupal_substr($this->mapTable, 0, 63 - $prefixLength); + $this->messageTable = 'migrate_message_' . drupal_strtolower($machine_name); + $this->messageTable = drupal_substr($this->messageTable, 0, 63 - $prefixLength); + $this->sourceIds = $migration->get('sourceIds'); + $this->destinationIds = $migration->get('destinationIds'); + + // Build the source and destination key maps + $this->sourceKeyMap = array(); + $count = 1; + foreach ($this->sourceIds as $field => $schema) { + $this->sourceKeyMap[$field] = 'sourceid' . $count++; + } + $this->destinationKeyMap = array(); + $count = 1; + foreach ($this->destinationIds as $field => $schema) { + $this->destinationKeyMap[$field] = 'destid' . $count++; + } + $this->ensureTables(); + } + + protected function getDatabase() { + return SqlBase::getDatabaseConnection($this->migration->id(), $this->configuration); + } + + public function setMessage(MigrateMessageInterface $message) { + $this->message = $message; + } + + /** + * Create the map and message tables if they don't already exist. + */ + protected function ensureTables() { + if (!$this->ensured) { + if (!$this->getDatabase()->schema()->tableExists($this->mapTable)) { + // Generate appropriate schema info for the map and message tables, + // and map from the source field names to the map/msg field names + $count = 1; + $source_id_schema = array(); + $pks = array(); + foreach ($this->sourceIds as $field_schema) { + $mapkey = 'sourceid' . $count++; + $source_id_schema[$mapkey] = $field_schema; + $pks[] = $mapkey; + } + + $fields = $source_id_schema; + + // Add destination keys to map table + // TODO: How do we discover the destination schema? + $count = 1; + foreach ($this->destinationIds as $field_schema) { + // Allow dest key fields to be NULL (for IGNORED/FAILED cases) + $field_schema['not null'] = FALSE; + $mapkey = 'destid' . $count++; + $fields[$mapkey] = $field_schema; + } + $fields['needs_update'] = array( + 'type' => 'int', + 'size' => 'tiny', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'default' => MigrateIdMapInterface::STATUS_IMPORTED, + 'description' => 'Indicates current status of the source row', + ); + $fields['rollback_action'] = array( + 'type' => 'int', + 'size' => 'tiny', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'default' => MigrateIdMapInterface::ROLLBACK_DELETE, + 'description' => 'Flag indicating what to do for this item on rollback', + ); + $fields['last_imported'] = array( + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'default' => 0, + 'description' => 'UNIX timestamp of the last time this row was imported', + ); + $fields['hash'] = array( + 'type' => 'varchar', + 'length' => '32', + 'not null' => FALSE, + 'description' => 'Hash of source row data, for detecting changes', + ); + $schema = array( + 'description' => t('Mappings from source key to destination key'), + 'fields' => $fields, + ); + if ($pks) { + $schema['primary key'] = $pks; + } + $this->getDatabase()->schema()->createTable($this->mapTable, $schema); + + // Now for the message table + $fields = array(); + $fields['msgid'] = array( + 'type' => 'serial', + 'unsigned' => TRUE, + 'not null' => TRUE, + ); + $fields += $source_id_schema; + + $fields['level'] = array( + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'default' => 1, + ); + $fields['message'] = array( + 'type' => 'text', + 'size' => 'medium', + 'not null' => TRUE, + ); + $schema = array( + 'description' => t('Messages generated during a migration process'), + 'fields' => $fields, + 'primary key' => array('msgid'), + ); + if ($pks) { + $schema['indexes']['sourcekey'] = $pks; + } + $this->getDatabase()->schema()->createTable($this->messageTable, $schema); + } + else { + // Add any missing columns to the map table + if (!$this->getDatabase()->schema()->fieldExists($this->mapTable, + 'rollback_action')) { + $this->getDatabase()->schema()->addField($this->mapTable, + 'rollback_action', array( + 'type' => 'int', + 'size' => 'tiny', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'default' => 0, + 'description' => 'Flag indicating what to do for this item on rollback', + )); + } + if (!$this->getDatabase()->schema()->fieldExists($this->mapTable, 'hash')) { + $this->getDatabase()->schema()->addField($this->mapTable, 'hash', array( + 'type' => 'varchar', + 'length' => '32', + 'not null' => FALSE, + 'description' => 'Hash of source row data, for detecting changes', + )); + } + } + $this->ensured = TRUE; + } + } + + /** + * Retrieve a row from the map table, given a source ID + * + * @param array $source_id + */ + public function getRowBySource(array $source_id) { + $query = $this->getDatabase()->select($this->mapTable, 'map') + ->fields('map'); + foreach ($this->sourceKeyMap as $key_name) { + $query = $query->condition("map.$key_name", array_shift($source_id), '='); + } + $result = $query->execute(); + return $result->fetchAssoc(); + } + + /** + * Retrieve a row from the map table, given a destination ID + * + * @param array $source_id + */ + public function getRowByDestination(array $destination_id) { + $query = $this->getDatabase()->select($this->mapTable, 'map') + ->fields('map'); + foreach ($this->destinationKeyMap as $key_name) { + $query = $query->condition("map.$key_name", array_shift($destination_id), '='); + } + $result = $query->execute(); + return $result->fetchAssoc(); + } + + /** + * Retrieve an array of map rows marked as needing update. + * + * @param int $count + * Maximum rows to return; defaults to 10,000 + * @return array + * Array of map row objects with needs_update==1. + */ + public function getRowsNeedingUpdate($count) { + $rows = array(); + $result = $this->getDatabase()->select($this->mapTable, 'map') + ->fields('map') + ->condition('needs_update', MigrateIdMapInterface::STATUS_NEEDS_UPDATE) + ->range(0, $count) + ->execute(); + foreach ($result as $row) { + $rows[] = $row; + } + return $rows; + } + + /** + * Given a (possibly multi-field) destination key, return the (possibly multi-field) + * source key mapped to it. + * + * @param array $destination_id + * Array of destination key values. + * @return array + * Array of source key values, or NULL on failure. + */ + public function lookupSourceID(array $destination_id) { + $query = $this->getDatabase()->select($this->mapTable, 'map') + ->fields('map', $this->sourceKeyMap); + foreach ($this->destinationKeyMap as $key_name) { + $query = $query->condition("map.$key_name", array_shift($destination_id), '='); + } + $result = $query->execute(); + $source_id = $result->fetchAssoc(); + return $source_id; + } + + /** + * Given a (possibly multi-field) source key, return the (possibly multi-field) + * destination key it is mapped to. + * + * @param array $source_id + * Array of source key values. + * @return array + * Array of destination key values, or NULL on failure. + */ + public function lookupDestinationID(array $source_id) { + $query = $this->getDatabase()->select($this->mapTable, 'map') + ->fields('map', $this->destinationKeyMap); + foreach ($this->sourceKeyMap as $key_name) { + $query = $query->condition("map.$key_name", array_shift($source_id), '='); + } + $result = $query->execute(); + $destination_id = $result->fetchAssoc(); + return $destination_id; + } + + /** + * Called upon import of one record, we record a mapping from the source key + * to the destination key. Also may be called, setting the third parameter to + * NEEDS_UPDATE, to signal an existing record should be remigrated. + * + * @param stdClass $row + * The raw source data. We use the key map derived from the source object + * to get the source key values. + * @param array $dest_ids + * The destination key values. + * @param int $needs_update + * Status of the source row in the map. Defaults to STATUS_IMPORTED. + * @param int $rollback_action + * How to handle the destination object on rollback. Defaults to + * ROLLBACK_DELETE. + * $param string $hash + * If hashing is enabled, the hash of the raw source row. + */ + public function saveIDMapping(Row $row, array $destination_id_values, $needs_update = MigrateIdMapInterface::STATUS_IMPORTED, $rollback_action = MigrateIdMapInterface::ROLLBACK_DELETE) { + // Construct the source key + $keys = array(); + $destination = $row->getDestination(); + foreach ($this->sourceKeyMap as $field_name => $key_name) { + // A NULL key value will fail. + if (!isset($destination[$field_name])) { + $this->message->display(t( + 'Could not save to map table due to NULL value for key field !field', + array('!field' => $field_name))); + return; + } + $keys[$key_name] = $destination[$field_name]; + } + + $fields = array( + 'needs_update' => (int) $needs_update, + 'rollback_action' => (int) $rollback_action, + 'hash' => $row->getHash(), + ); + $count = 1; + foreach ($destination_id_values as $dest_id) { + $fields['destid' . $count++] = $dest_id; + } + if ($this->migration->get('trackLastImported')) { + $fields['last_imported'] = time(); + } + if ($keys) { + $this->getDatabase()->merge($this->mapTable) + ->key($keys) + ->fields($fields) + ->execute(); + } + } + + /** + * Record a message in the migration's message table. + * + * @param array $source_id_values + * Source ID of the record in error + * @param string $message + * The message to record. + * @param int $level + * Optional message severity (defaults to MESSAGE_ERROR). + */ + public function saveMessage(array $source_id_values, $message, $level = MigrationInterface::MESSAGE_ERROR) { + // Source IDs as arguments + $count = 1; + foreach ($source_id_values as $id_value) { + $fields['sourceid' . $count++] = $id_value; + // If any key value is empty, we can't save - print out and abort + if (empty($id_value)) { + print($message); + return; + } + } + $fields['level'] = $level; + $fields['message'] = $message; + $this->getDatabase()->insert($this->messageTable) + ->fields($fields) + ->execute(); + } + + /** + * Prepares this migration to run as an update - that is, in addition to + * unmigrated content (source records not in the map table) being imported, + * previously-migrated content will also be updated in place. + */ + public function prepareUpdate() { + $this->getDatabase()->update($this->mapTable) + ->fields(array('needs_update' => MigrateIdMapInterface::STATUS_NEEDS_UPDATE)) + ->execute(); + } + + /** + * Returns a count of records in the map table (i.e., the number of + * source records which have been processed for this migration). + * + * @return int + */ + public function processedCount() { + $query = $this->getDatabase()->select($this->mapTable); + $query->addExpression('COUNT(*)', 'count'); + $count = $query->execute()->fetchField(); + return $count; + } + + /** + * Returns a count of imported records in the map table. + * + * @return int + */ + public function importedCount() { + $query = $this->getDatabase()->select($this->mapTable); + $query->addExpression('COUNT(*)', 'count'); + $query->condition('needs_update', array(MigrateIdMapInterface::STATUS_IMPORTED, MigrateIdMapInterface::STATUS_NEEDS_UPDATE), 'IN'); + $count = $query->execute()->fetchField(); + return $count; + } + + /** + * Returns a count of records which are marked as needing update. + * + * @return int + */ + public function updateCount() { + $query = $this->getDatabase()->select($this->mapTable); + $query->addExpression('COUNT(*)', 'count'); + $query->condition('needs_update', MigrateIdMapInterface::STATUS_NEEDS_UPDATE); + $count = $query->execute()->fetchField(); + return $count; + } + + /** + * Get the number of source records which failed to import. + * + * @return int + * Number of records errored out. + */ + public function errorCount() { + $query = $this->getDatabase()->select($this->mapTable); + $query->addExpression('COUNT(*)', 'count'); + $query->condition('needs_update', MigrateIdMapInterface::STATUS_FAILED); + $count = $query->execute()->fetchField(); + return $count; + } + + /** + * Get the number of messages saved. + * + * @return int + * Number of messages. + */ + public function messageCount() { + $query = $this->getDatabase()->select($this->messageTable); + $query->addExpression('COUNT(*)', 'count'); + $count = $query->execute()->fetchField(); + return $count; + } + + /** + * Delete the map entry and any message table entries for the specified source row. + * + * @param array $source_id + */ + public function delete(array $source_id, $messages_only = FALSE) { + if (!$messages_only) { + $map_query = $this->getDatabase()->delete($this->mapTable); + } + $message_query = $this->getDatabase()->delete($this->messageTable); + $count = 1; + foreach ($source_id as $key_value) { + if (!$messages_only) { + $map_query->condition('sourceid' . $count, $key_value); + } + $message_query->condition('sourceid' . $count, $key_value); + $count++; + } + + if (!$messages_only) { + $map_query->execute(); + } + $message_query->execute(); + } + + /** + * Delete the map entry and any message table entries for the specified destination row. + * + * @param array $destination_id + */ + public function deleteDestination(array $destination_id) { + $map_query = $this->getDatabase()->delete($this->mapTable); + $message_query = $this->getDatabase()->delete($this->messageTable); + $source_id = $this->lookupSourceID($destination_id); + if (!empty($source_id)) { + $count = 1; + foreach ($destination_id as $key_value) { + $map_query->condition('destid' . $count, $key_value); + $count++; + } + $map_query->execute(); + $count = 1; + foreach ($source_id as $key_value) { + $message_query->condition('sourceid' . $count, $key_value); + $count++; + } + $message_query->execute(); + } + } + + /** + * Set the specified row to be updated, if it exists. + */ + public function setUpdate(array $source_id) { + $query = $this->getDatabase()->update($this->mapTable) + ->fields(array('needs_update' => MigrateIdMapInterface::STATUS_NEEDS_UPDATE)); + $count = 1; + foreach ($source_id as $key_value) { + $query->condition('sourceid' . $count++, $key_value); + } + $query->execute(); + } + + /** + * Delete all map and message table entries specified. + * + * @param array $source_ids + * Each array member is an array of key fields for one source row. + */ + public function deleteBulk(array $source_ids) { + // If we have a single-column key, we can shortcut it + if (count($this->sourceIds) == 1) { + $sourceids = array(); + foreach ($source_ids as $source_id) { + $sourceids[] = $source_id; + } + $this->getDatabase()->delete($this->mapTable) + ->condition('sourceid1', $sourceids, 'IN') + ->execute(); + $this->getDatabase()->delete($this->messageTable) + ->condition('sourceid1', $sourceids, 'IN') + ->execute(); + } + else { + foreach ($source_ids as $source_id) { + $map_query = $this->getDatabase()->delete($this->mapTable); + $message_query = $this->getDatabase()->delete($this->messageTable); + $count = 1; + foreach ($source_id as $key_value) { + $map_query->condition('sourceid' . $count, $key_value); + $message_query->condition('sourceid' . $count++, $key_value); + } + $map_query->execute(); + $message_query->execute(); + } + } + } + + /** + * Clear all messages from the message table. + */ + public function clearMessages() { + $this->getDatabase()->truncate($this->messageTable) + ->execute(); + } + + /** + * Remove the associated map and message tables. + */ + public function destroy() { + $this->getDatabase()->schema()->dropTable($this->mapTable); + $this->getDatabase()->schema()->dropTable($this->messageTable); + } + + protected $result = NULL; + protected $currentRow = NULL; + protected $currentKey = array(); + public function getCurrentKey() { + return $this->currentKey; + } + + /** + * Implementation of Iterator::rewind() - called before beginning a foreach loop. + * TODO: Support idlist, itemlimit + */ + public function rewind() { + $this->currentRow = NULL; + $fields = array(); + foreach ($this->sourceKeyMap as $field) { + $fields[] = $field; + } + foreach ($this->destinationKeyMap as $field) { + $fields[] = $field; + } + + /* TODO + if (isset($this->options['itemlimit'])) { + $query = $query->range(0, $this->options['itemlimit']); + } + */ + $this->result = $this->getDatabase()->select($this->mapTable, 'map') + ->fields('map', $fields) + ->execute(); + $this->next(); + } + + /** + * Implementation of Iterator::current() - called when entering a loop + * iteration, returning the current row + */ + public function current() { + return $this->currentRow; + } + + /** + * Implementation of Iterator::key - called when entering a loop iteration, returning + * the key of the current row. It must be a scalar - we will serialize + * to fulfill the requirement, but using getCurrentKey() is preferable. + */ + public function key() { + return serialize($this->currentKey); + } + + /** + * Implementation of Iterator::next() - called at the bottom of the loop implicitly, + * as well as explicitly from rewind(). + */ + public function next() { + $this->currentRow = $this->result->fetchObject(); + $this->currentKey = array(); + if (!is_object($this->currentRow)) { + $this->currentRow = NULL; + } + else { + foreach ($this->sourceKeyMap as $map_field) { + $this->currentKey[$map_field] = $this->currentRow->$map_field; + // Leave only destination fields + unset($this->currentRow->$map_field); + } + } + } + + /** + * Implementation of Iterator::valid() - called at the top of the loop, returning + * TRUE to process the loop and FALSE to terminate it + */ + public function valid() { + // TODO: Check numProcessed against itemlimit + return !is_null($this->currentRow); + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/process/PropertyMap.php b/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/process/PropertyMap.php new file mode 100644 index 0000000..b3d6f4c --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/process/PropertyMap.php @@ -0,0 +1,180 @@ +configuration['source']) && $row->hasSourceProperty($this->configuration['source'])) { + $destination_values = $row->getSourceProperty($this->configuration['source']); + } + // Otherwise, apply the default value (if any) + elseif (isset($this->configuration['default'])) { + $destination_values = $this->configuration['default']; + } + + // If there's a separator specified for this destination, then it + // will be populated as an array exploded from the source value + if (!empty($this->configuration['separator']) && isset($destination_values)) { + $destination_values = explode($this->configuration['separator'], $destination_values); + } + + // If a source migration is supplied, use the current value for this property + // to look up a destination ID from the provided migration + if (!empty($this->configuration['source_migration']) && isset($destination_values)) { + $destination_values = $migrate_executable->handleSourceMigration($this->configuration['source_migration'], $destination_values, $this->configuration['default'], $this); + } + + // Call any designated callbacks + if (!empty($this->configuration['callbacks'])) { + foreach ($this->callbacks as $callback) { + if (isset($destination_values) && is_callable($callback)) { + $destination_values = call_user_func($callback, $destination_values); + } + } + } + + // If specified, assure a unique value for this property. + if (!empty($this->configuration['dedupe'])&& isset($destination_values)) { + $destination_values = $migrate_executable->handleDedupe($this->configuration['dedupe'], $destination_values); + } + + // Store the destination together with possible configuration. + if (isset($destination_values)) { + $keys = explode(':', $this->configuration['destination']); + $row->setDestinationPropertyDeep($keys, $destination_values); + } + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/source/D6Variable.php b/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/source/D6Variable.php new file mode 100644 index 0000000..4c96d71 --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/source/D6Variable.php @@ -0,0 +1,56 @@ +variables = $this->configuration['variables']; + } + + protected function runQuery() { + return new \ArrayIterator(array(array_map('unserialize', $this->query()->execute()->fetchAllKeyed()))); + } + + public function count() { + return intval($this->query()->countQuery()->execute()->fetchField() > 0); + } + + /** + * {@inheritdoc} + */ + public function fields() { + return drupal_map_assoc($this->variables); + } + + /** + * {@inheritdoc} + */ + function query() { + return $this->getDatabase() + ->select('variable', 'v') + ->fields('v', array('name', 'value')) + ->condition('name', $this->variables, 'IN'); + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/source/SqlBase.php b/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/source/SqlBase.php new file mode 100644 index 0000000..17aa9e2 --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/source/SqlBase.php @@ -0,0 +1,188 @@ +migration = $migration; + $this->mapJoinable = TRUE; + } + + /** + * @return \Drupal\Core\Database\Connection + */ + function __toString() { + return (string) $this->query; + } + + protected function getDatabase() { + if (!isset($this->database)) { + $this->database = static::getDatabaseConnection($this->migration->id(), $this->configuration); + } + return $this->database; + } + + public static function getDatabaseConnection($id, array $configuration) { + if (isset($configuration['database'])) { + $key = 'migrate_' . $id; + Database::addConnectionInfo($key, 'default', $configuration['database']); + } + else { + $key = 'default'; + } + return Database::getConnection('default', $key); + } + + /** + * Implementation of MigrateSource::performRewind(). + * + * We could simply execute the query and be functionally correct, but + * we will take advantage of the PDO-based API to optimize the query up-front. + */ + protected function runQuery() { + $this->query = clone $this->query(); + $highwaterProperty = $this->migration->get('highwaterProperty'); + + // Get the key values, for potential use in joining to the map table, or + // enforcing idlist. + $keys = array(); + foreach ($this->migration->get('sourceIds') as $field_name => $field_schema) { + if (isset($field_schema['alias'])) { + $field_name = $field_schema['alias'] . '.' . $field_name; + } + $keys[] = $field_name; + } + + // The rules for determining what conditions to add to the query are as + // follows (applying first applicable rule) + // 1. If idlist is provided, then only process items in that list (AND key + // IN (idlist)). Only applicable with single-value keys. + if ($id_list = $this->migration->get('idlist')) { + $this->query->condition($keys[0], $id_list, 'IN'); + } + else { + // 2. If the map is joinable, join it. We will want to accept all rows + // which are either not in the map, or marked in the map as NEEDS_UPDATE. + // Note that if highwater fields are in play, we want to accept all rows + // above the highwater mark in addition to those selected by the map + // conditions, so we need to OR them together (but AND with any existing + // conditions in the query). So, ultimately the SQL condition will look + // like (original conditions) AND (map IS NULL OR map needs update + // OR above highwater). + $conditions = $this->query->orConditionGroup(); + $condition_added = FALSE; + if ($this->mapJoinable) { + // Build the join to the map table. Because the source key could have + // multiple fields, we need to build things up. + $count = 1; + $map_join = ''; + $delimiter = ''; + foreach ($this->migration->get('sourceIds') as $field_name => $field_schema) { + if (isset($field_schema['alias'])) { + $field_name = $field_schema['alias'] . '.' . $field_name; + } + $map_join .= "$delimiter$field_name = map.sourceid" . $count++; + $delimiter = ' AND '; + } + + $alias = $this->query->leftJoin($this->migration->getIdMap()->getQualifiedMapTable(), 'map', $map_join); + $conditions->isNull($alias . '.sourceid1'); + $conditions->condition($alias . '.needs_update', MigrateIdMapInterface::STATUS_NEEDS_UPDATE); + $condition_added = TRUE; + + // And as long as we have the map table, add its data to the row. + $n = count($this->migration->get('sourceIds')); + for ($count = 1; $count <= $n; $count++) { + $map_key = 'sourceid' . $count; + $this->query->addField($alias, $map_key, "migrate_map_$map_key"); + } + $n = count($this->migration->get('destinationIds')); + for ($count = 1; $count <= $n; $count++) { + $map_key = 'destid' . $count++; + $this->query->addField($alias, $map_key, "migrate_map_$map_key"); + } + $this->query->addField($alias, 'needs_update', 'migrate_map_needs_update'); + } + // 3. If we are using highwater marks, also include rows above the mark. + // But, include all rows if the highwater mark is not set. + if (isset($highwaterProperty['name']) && ($highwater = $this->migration->getHighwater()) !== '') { + if (isset($highwaterProperty['alias'])) { + $highwater = $highwaterProperty['alias'] . '.' . $highwaterProperty['name']; + } + else { + $highwater = $highwaterProperty['name']; + } + $conditions->condition($highwater, $highwater, '>'); + $condition_added = TRUE; + } + if ($condition_added) { + $this->query->condition($conditions); + } + } + + return $this->query->execute(); + } + + /** + * @return \Drupal\Core\Database\Query\SelectInterface + */ + abstract function query(); + + /** + * {@inheritdoc} + */ + public function count() { + return $this->query()->countQuery()->execute()->fetchField(); + } + + /** + * Returns the iterator that will yield the row arrays to be processed. + * + * @return \Iterator + */ + public function getIterator() { + if (!isset($this->iterator)) { + $this->iterator = $this->runQuery(); + } + return $this->iterator; + } + + public function prepareRow(Row $row) { + return TRUE; + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Row.php b/core/modules/migrate/lib/Drupal/migrate/Row.php new file mode 100644 index 0000000..be45ebf --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Row.php @@ -0,0 +1,262 @@ + '', + 'hash' => '', + 'needs_update' => MigrateIdMapInterface::STATUS_NEEDS_UPDATE, + ); + + /** + * Whether the source has been frozen already. + * + * Once frozen the source can not be changed any more. + * + * @var bool + */ + protected $frozen = FALSE; + + /** + * Constructs a Migrate>Row object. + * + * @param array $source_ids + * An array containing the ids of the source using the keys as the field + * names. + * @param array $values + * An array of values to add as properties on the object. + * + * @throws \InvalidArgumentException + * Thrown when a source id property does not exist. + */ + public function __construct(array $source_ids, array $values) { + $this->source = $values; + foreach (array_keys($source_ids) as $id) { + if ($this->hasSourceProperty($id)) { + $this->sourceIdValues[$id] = $values[$id]; + } + else { + throw new \InvalidArgumentException("$id has no value"); + } + } + } + + /** + * Retrieves the values of the source identifiers. + * + * @return array + * An array containing the values of the source identifiers. + */ + public function getSourceIdValues() { + return $this->sourceIdValues; + } + + /** + * Determines whether a source has a property. + * + * @param string $property + * A property on the source. + * + * @return bool + * TRUE if the source has property; FALSE otherwise. + */ + public function hasSourceProperty($property) { + return isset($this->source[$property]) || array_key_exists($property, $this->source); + } + + /** + * Retrieves a source property. + * + * @param string $property + * A property on the source. + * + * @return mixed|null + * The found returned property or NULL if not found. + */ + public function getSourceProperty($property) { + if (isset($this->source[$property])) { + return $this->source[$property]; + } + } + + /** + * This returns the whole source array. + * + * @return array + * An array of source plugins. + */ + public function getSource() { + return $this->source; + } + + /** + * Sets a source property. This can only be called from the source plugin. + * + * @param string $property + * A property on the source. + * @param mixed $data + * The property value to set on the source. + * + * @throws \Exception + */ + public function setSourceProperty($property, $data) { + if ($this->frozen) { + throw new \Exception("The source is frozen and can't be changed any more"); + } + else { + $this->source[$property] = $data; + } + } + + /** + * Freezes the source. + */ + public function freezeSource() { + $this->frozen = TRUE; + } + + /** + * Determines whether a destination has a property. + * + * @param string $property + * A property on the destination. + * + * @return bool + * TRUE if the destination has property; FALSE otherwise. + */ + public function hasDestinationProperty($property) { + return isset($this->destination[$property]) || array_key_exists($property, $this->destination); + } + + /** + * Sets a destination property. + * + * @param string $property + * A property on the destination. + * @param mixed $value + * The property value to set on the destination. + */ + public function setDestinationProperty($property, $value) { + $this->destination[$property] = $value; + } + + /** + * Sets destination properties. + * + * @param array $property_keys + * An array of properties on the destination. + * @param mixed $value + * The property value to set on the destination. + */ + public function setDestinationPropertyDeep(array $property_keys, $value) { + NestedArray::setValue($this->destination, $property_keys, $value, TRUE); + } + + /** + * This returns the whole destination array. + * + * @return array + * An array of destination plugins. + */ + public function getDestination() { + return $this->destination; + } + + /** + * Sets the Migrate id mappings. + * + * @param array $id_map + * An array of mappings between source ID and destination ID. + */ + public function setIdMap(array $id_map) { + $this->idMap = $id_map; + } + + /** + * Retrieves the Migrate id mappings. + * + * @return array + * An array of mapping between source and destination identifiers. + */ + public function getIdMap() { + return $this->idMap; + } + + /** + * Recalculate the hash for the row. + */ + public function rehash() { + $this->idMap['original_hash'] = $this->idMap['hash']; + $this->idMap['hash'] = hash('sha256', serialize($this->source)); + } + + /** + * Checks whether the row has changed compared to the original id map. + * + * return bool + * TRUE if the row has changed, FALSE otherwise. If setIdMap() was not + * called, this always returns FALSE. + */ + public function changed() { + return $this->idMap['original_hash'] != $this->idMap['hash']; + } + + /** + * Returns if this row needs an update. + * + * @return bool + * TRUE if the row needs updating, FALSE otherwise. + */ + public function needsUpdate() { + return $this->idMap['needs_update'] == MigrateIdMapInterface::STATUS_NEEDS_UPDATE; + } + + /** + * Returns the hash for the source values.. + * + * @return mixed + * The hash of the source values. + */ + public function getHash() { + return $this->idMap['hash']; + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Source.php b/core/modules/migrate/lib/Drupal/migrate/Source.php new file mode 100644 index 0000000..839e716 --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Source.php @@ -0,0 +1,407 @@ +currentIds; + } + + public function getIgnored() { + return $this->numIgnored; + } + + public function getProcessed() { + return $this->numProcessed; + } + + /** + * Reset numIgnored back to 0. + */ + public function resetStats() { + $this->numIgnored = 0; + } + + /** + * Return a count of available source records, from the cache if appropriate. + * Returns -1 if the source is not countable. + * + * @param boolean $refresh + * @return int + */ + public function count($refresh = FALSE) { + if ($this->skipCount) { + return -1; + } + $source = $this->migration->getSource(); + + if (!isset($this->cacheKey)) { + $this->cacheKey = md5((string) $source); + } + + // If a refresh is requested, or we're not caching counts, ask the derived + // class to get the count from the source. + if ($refresh || !$this->cacheCounts) { + $count = $source->count(); + $this->cache->set($this->cacheKey, $count, 'cache'); + } + else { + // Caching is in play, first try to retrieve a cached count. + $cache_object = $this->cache->get($this->cacheKey, 'cache'); + if (is_object($cache_object)) { + // Success + $count = $cache_object->data; + } + else { + // No cached count, ask the derived class to count 'em up, and cache + // the result + $count = $source->count(); + $this->cache->set($this->cacheKey, $count, 'cache'); + } + } + return $count; + } + + /** + * Class constructor. + * + * @param \Drupal\migrate\Entity\MigrationInterface $migration + */ + function __construct(MigrationInterface $migration) { + $this->migration = $migration; + $configuration = $migration->get('source'); + if (!empty($configuration['cache_counts'])) { + $this->cacheCounts = TRUE; + } + if (!empty($configuration['skip_count'])) { + $this->skipCount = TRUE; + } + if (!empty($configuration['cache_key'])) { + $this->cacheKey = $configuration['cache_key']; + } + if (!empty($configuration['track_changes'])) { + $this->trackChanges = $configuration['track_changes']; + } + } + + /** + * @return \Drupal\Core\Cache\CacheBackendInterface + */ + protected function getCache() { + if (!isset($this->cache)) { + $this->cache = \Drupal::cache('migrate'); + } + return $this->cache; + } + + /** + * @return \Iterator + */ + protected function getIterator() { + if (!isset($this->iterator)) { + $this->iterator = $this->migration->getSource()->getIterator(); + } + return $this->iterator; + } + + /** + * {@inheritdoc} + */ + public function current() { + return $this->currentRow; + } + + /** + * Implementation of Iterator::key - called when entering a loop iteration, returning + * the key of the current row. It must be a scalar - we will serialize + * to fulfill the requirement, but using getCurrentIds() is preferable. + */ + public function key() { + return serialize($this->currentIds); + } + + /** + * Implementation of Iterator::valid() - called at the top of the loop, returning + * TRUE to process the loop and FALSE to terminate it + */ + public function valid() { + return isset($this->currentRow); + } + + /** + * Implementation of Iterator::rewind() - subclasses of MigrateSource should + * implement performRewind() to do any class-specific setup for iterating + * source records. + */ + public function rewind() { + $this->idMap = $this->migration->getIdMap(); + $this->numProcessed = 0; + $this->numIgnored = 0; + $this->originalHighwater = $this->migration->getHighwater(); + $this->highwaterProperty = $this->migration->get('highwaterProperty'); + if ($id_list = $this->migration->get('idlist')) { + $this->idList = $id_list; + } + $this->next(); + } + + /** + * {@inheritdoc} + */ + public function next() { + $this->currentIds = NULL; + $this->currentRow = NULL; + + while ($this->getIterator()->valid()) { + $row_data = $this->getIterator()->current(); + $this->getIterator()->next(); + $row = new Row($this->migration->get('sourceIds'), $row_data); + + // Populate the source key for this row + $this->currentIds = $row->getSourceIdValues(); + + // Pick up the existing map row, if any, unless getNextRow() did it. + if (!$this->mapRowAdded && ($id_map = $this->idMap->getRowBySource($this->currentIds))) { + $row->setIdMap($id_map); + } + + // First, determine if this row should be passed to prepareRow(), or + // skipped entirely. The rules are: + // 1. If there's an explicit idlist, that's all we care about (ignore + // highwaters and map rows). + $prepared = FALSE; + if (!empty($this->idList)) { + if (in_array(reset($this->currentIds), $this->idList)) { + // In the list, fall through. + } + else { + // Not in the list, skip it + continue; + } + } + // 2. If the row is not in the map (we have never tried to import it + // before), we always want to try it. + elseif (!$row->getIdMap()) { + // Fall through + } + // 3. If the row is marked as needing update, pass it. + elseif ($row->needsUpdate()) { + // Fall through + } + // 4. At this point, we have a row which has previously been imported and + // not marked for update. If we're not using highwater marks, then we + // will not take this row. Except, if we're looking for changes in the + // data, we need to go through prepareRow() before we can decide to + // skip it. + elseif (!empty($highwater['field'])) { + if ($this->trackChanges) { + if ($this->prepareRow($row) !== FALSE) { + if ($row->changed()) { + // This is a keeper + $this->currentRow = $row; + break; + } + else { + // No change, skip it. + continue; + } + } + else { + // prepareRow() told us to skip it. + continue; + } + } + else { + // No highwater and not tracking changes, skip. + continue; + } + } + // 5. The initial highwater mark, before anything is migrated, is ''. We + // want to make sure we don't mistakenly skip rows with a highwater + // field value of 0, so explicitly handle '' here. + elseif ($this->originalHighwater === '') { + // Fall through + } + // 6. So, we are using highwater marks. Take the row if its highwater + // field value is greater than the saved mark, otherwise skip it. + else { + // Call prepareRow() here, in case the highwaterField needs preparation + if ($this->prepareRow($row) !== FALSE) { + if ($row->getSourceProperty($this->highwaterProperty['name']) > $this->originalHighwater) { + $this->currentRow = $row; + break; + } + else { + // Skip + continue; + } + } + $prepared = TRUE; + } + + // Allow the Migration to prepare this row. prepareRow() can return boolean + // FALSE to ignore this row. + if (!$prepared) { + if ($this->prepareRow($row) !== FALSE) { + // Finally, we've got a keeper. + $this->currentRow = $row; + break; + } + else { + $this->currentRow = NULL; + } + } + } + if ($this->currentRow) { + $this->currentRow->freezeSource(); + } + else { + $this->currentIds = NULL; + } + } + + /** + * Source classes should override this as necessary and manipulate $keep. + * + * @param \Drupal\migrate\Row $row + */ + protected function prepareRow(Row $row) { + // We're explicitly skipping this row - keep track in the map table + if ($this->migration->getSource()->prepareRow($row) === FALSE) { + // Make sure we replace any previous messages for this item with any + // new ones. + $this->migration->getIdMap()->delete($this->currentIds, TRUE); + $this->migration->saveQueuedMessages(); + $this->migration->getIdMap()->saveIDMapping($row, array(), + \MigrateMap::STATUS_IGNORED, $this->migration->rollbackAction); + $this->numIgnored++; + $this->currentRow = NULL; + $this->currentIds = NULL; + } + else { + // When tracking changed data, We want to quietly skip (rather than + // "ignore") rows with changes. The caller needs to make that decision, + // so we need to provide them with the necessary information (before and + // after hashes). + if ($this->trackChanges) { + $row->rehash(); + } + } + $this->numProcessed++; + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Tests/Dump/Drupal6SystemSite.php b/core/modules/migrate/lib/Drupal/migrate/Tests/Dump/Drupal6SystemSite.php new file mode 100644 index 0000000..749652b --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Tests/Dump/Drupal6SystemSite.php @@ -0,0 +1,69 @@ +schema()->createTable('variable', array( + 'fields' => array( + 'name' => array( + 'type' => 'varchar', + 'length' => 128, + 'not null' => TRUE, + 'default' => '', + ), + 'value' => array( + 'type' => 'blob', + 'not null' => TRUE, + 'size' => 'big', + 'translatable' => TRUE, + ), + ), + 'primary key' => array( + 'name', + ), + 'module' => 'system', + 'name' => 'variable', + )); + $database->insert('variable')->fields(array( + 'name', + 'value', + )) + ->values(array( + 'name' => 'site_name', + 'value' => 's:6:"drupal";', + )) + ->values(array( + 'name' => 'site_mail', + 'value' => 's:17:"admin@example.com";', + )) + ->values(array( + 'name' => 'site_slogan', + 'value' => 's:13:"Migrate rocks";', + )) + ->values(array( + 'name' => 'site_frontpage', + 'value' => 's:12:"anonymous-hp";', + )) + ->values(array( + 'name' => 'site_403', + 'value' => 's:4:"user";', + )) + ->values(array( + 'name' => 'site_404', + 'value' => 's:14:"page-not-found";', + )) + ->values(array( + 'name' => 'drupal_weight_select_max', + 'value' => 'i:99;', + )) + ->values(array( + 'name' => 'admin_compact_mode', + 'value' => 'b:0;', + )) + ->execute(); + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Tests/MigrateSystemConfigsTest.php b/core/modules/migrate/lib/Drupal/migrate/Tests/MigrateSystemConfigsTest.php new file mode 100644 index 0000000..8d3ef6e --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Tests/MigrateSystemConfigsTest.php @@ -0,0 +1,41 @@ + 'Migrate variables to system.site.yml', + 'description' => 'Upgrade variables to system.site.yml', + 'group' => 'Migrate', + ); + } + + function testSystemSite() { + $migration = entity_load('migration', 'd6_system_site'); + $dumps = array( + drupal_get_path('module', 'system') . '/tests/upgrade/Drupal6SystemSite.php', + ); + $this->prepare($migration, $dumps); + $executable = new MigrateExecutable($migration, new DrupalMessage); + $executable->import(); + $config = \Drupal::config('system.site'); + $this->assertIdentical($config->get('name'), 'drupal'); + $this->assertIdentical($config->get('mail'), 'admin@example.com'); + $this->assertIdentical($config->get('slogan'), 'Migrate rocks'); + $this->assertIdentical($config->get('page.front'), 'anonymous-hp'); + $this->assertIdentical($config->get('page.403'), 'user'); + $this->assertIdentical($config->get('page.404'), 'page-not-found'); + $this->assertIdentical($config->get('weight_select_max'), 99); + $this->assertIdentical($config->get('admin_compact_mode'), FALSE); + } +} diff --git a/core/modules/migrate/lib/Drupal/migrate/Tests/MigrateTestBase.php b/core/modules/migrate/lib/Drupal/migrate/Tests/MigrateTestBase.php new file mode 100644 index 0000000..f45d520 --- /dev/null +++ b/core/modules/migrate/lib/Drupal/migrate/Tests/MigrateTestBase.php @@ -0,0 +1,49 @@ +id(), array('database' => $connection_info['default'])); + foreach (array('source', 'destination', 'id_map') as $key) { + $configuration = $migration->get($key); + $configuration['database'] = $database; + $migration->set($key, $configuration); + } + + // Load the database from the portable PHP dump. + // The files may be gzipped. + foreach ($files as $file) { + if (substr($file, -3) == '.gz') { + $file = "compress.zlib://$file"; + require $file; + } + $class = 'Drupal\migrate\Tests\Dump\\' . basename($file, '.php'); + $class::load($database); + } + return $database; + } +} diff --git a/core/modules/migrate/migrate.info.yml b/core/modules/migrate/migrate.info.yml new file mode 100644 index 0000000..cb46bac --- /dev/null +++ b/core/modules/migrate/migrate.info.yml @@ -0,0 +1,8 @@ +name: Migrate +type: module +description: 'Handles migrations' +package: Core +version: VERSION +core: 8.x +;required: true +;configure: admin/structure/migrate diff --git a/core/modules/migrate/migrate.module b/core/modules/migrate/migrate.module new file mode 100644 index 0000000..e69de29 diff --git a/core/modules/migrate/migrate.services.yml b/core/modules/migrate/migrate.services.yml new file mode 100644 index 0000000..1f6e0e9 --- /dev/null +++ b/core/modules/migrate/migrate.services.yml @@ -0,0 +1,20 @@ +services: + cache.migrate: + class: Drupal\Core\Cache\CacheBackendInterface + tags: + - { name: cache.bin } + factory_method: get + factory_service: cache_factory + arguments: [migrate] + plugin.manager.migrate.source: + class: Drupal\migrate\Plugin\MigratePluginManager + arguments: [source, '@container.namespaces', '@cache.cache', '@language_manager', '@module_handler'] + plugin.manager.migrate.process: + class: Drupal\migrate\Plugin\MigratePluginManager + arguments: [process, '@container.namespaces', '@cache.cache', '@language_manager', '@module_handler'] + plugin.manager.migrate.destination: + class: Drupal\migrate\Plugin\MigratePluginManager + arguments: [destination, '@container.namespaces', '@cache.cache', '@language_manager', '@module_handler'] + plugin.manager.migrate.id_map: + class: Drupal\migrate\Plugin\MigratePluginManager + arguments: [id_map, '@container.namespaces', '@cache.cache', '@language_manager', '@module_handler'] diff --git a/core/modules/migrate/tests/Drupal/migrate/Tests/D6VariableTest.php b/core/modules/migrate/tests/Drupal/migrate/Tests/D6VariableTest.php new file mode 100644 index 0000000..7c0471d --- /dev/null +++ b/core/modules/migrate/tests/Drupal/migrate/Tests/D6VariableTest.php @@ -0,0 +1,59 @@ + 'test', + 'highwaterProperty' => array('field' => 'test'), + 'idlist' => array(), + 'source' => array( + 'plugin' => 'drupal6_variable', + 'variables' => array( + 'foo', + 'bar', + ), + ), + 'sourceIds' => array(), + 'destinationIds' => array(), + ); + + protected $mapJoinable = FALSE; + + protected $results = array( + array( + 'foo' => 1, + 'bar' => FALSE, + ), + ); + + protected $databaseContents = array( + 'variable' => array( + array('name' => 'foo', 'value' => 'i:1;'), + array('name' => 'bar', 'value' => 'b:0;'), + ), + ); + + /** + * {@inheritdoc} + */ + public static function getInfo() { + return array( + 'name' => 'D6 variabe source functionality', + 'description' => 'Tests D6 variable source plugin.', + 'group' => 'Migrate', + ); + } + +} diff --git a/core/modules/migrate/tests/Drupal/migrate/Tests/FakeDatabaseSchema.php b/core/modules/migrate/tests/Drupal/migrate/Tests/FakeDatabaseSchema.php new file mode 100644 index 0000000..82ee22a --- /dev/null +++ b/core/modules/migrate/tests/Drupal/migrate/Tests/FakeDatabaseSchema.php @@ -0,0 +1,155 @@ +databaseContents = $database_contents; + } + + public function tableExists($table) { + return in_array($table, array_keys($this->databaseContents)); + } + + public function prefixNonTable($table) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + protected function buildTableNameCondition($table_name, $operator = '=', $add_prefix = TRUE) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + protected function getPrefixInfo($table = 'default', $add_prefix = TRUE) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function addField($table, $field, $spec, $keys_new = array()) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function addIndex($table, $name, $fields) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function addPrimaryKey($table, $fields) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function addUniqueKey($table, $name, $fields) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function changeField($table, $field, $field_new, $spec, $keys_new = array()) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function __clone() { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function copyTable($source, $destination) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function createTable($name, $table) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function dropField($table, $field) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function dropIndex($table, $name) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function dropPrimaryKey($table) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function dropTable($table) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function dropUniqueKey($table, $name) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function fieldExists($table, $column) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function fieldNames($fields) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function fieldSetDefault($table, $field, $default) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function fieldSetNoDefault($table, $field) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function findTables($table_expression) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function getFieldTypeMap() { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function indexExists($table, $name) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function nextPlaceholder() { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function prepareComment($comment, $length = NULL) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function renameTable($table, $new_name) { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + public function uniqueIdentifier() { + throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__)); + } + + /** + * Provide meta information about this battery of tests. + */ + public static function getInfo() { + return array( + 'name' => 'Fake database schema', + 'description' => 'Tests for fake database schema plugin.', + 'group' => 'Migrate', + ); + } + +} diff --git a/core/modules/migrate/tests/Drupal/migrate/Tests/FakeSelect.php b/core/modules/migrate/tests/Drupal/migrate/Tests/FakeSelect.php new file mode 100644 index 0000000..9ae6f30 --- /dev/null +++ b/core/modules/migrate/tests/Drupal/migrate/Tests/FakeSelect.php @@ -0,0 +1,540 @@ + array( + * array( + * 'uid' => 1, + * 'name' => 'admin', + * ), + * array( + * 'uid' => 2, + * 'name' => 'alice', + * ), + * ), + * 'node' => array( + * array( + * 'nid' => 1, + * ) + * ) + * ) + * @endcode + * + * @var array + */ + protected $databaseContents; + + protected $countQuery = FALSE; + protected $fieldsWithTable = array(); + + /** + * Constructs a new FakeSelect. + * + * @param string $table + * The base table name used within fake select. + * + * @param string $alias + * The base table alias used within fake select. + * + * @param array $database_contents + * An array of mocked database content. + * + * @param string $conjunction + * The operator to use to combine conditions: 'AND' or 'OR'. + */ + public function __construct($table, $alias, array $database_contents, $conjunction = 'AND') { + $this->addJoin(NULL, $table, $alias); + $this->where = new Condition($conjunction); + $this->having = new Condition($conjunction); + $this->databaseContents = $database_contents; + } + + /** + * {@inheritdoc} + */ + public function leftJoin($table, $alias = NULL, $condition = NULL, $arguments = array()) { + return $this->addJoin('LEFT', $table, $alias, $condition, $arguments); + } + + /** + * {@inheritdoc} + */ + public function addJoin($type, $table, $alias = NULL, $condition = NULL, $arguments = array()) { + if ($table instanceof SelectInterface) { + // @todo implement this. + throw new \Exception('Subqueries are not supported at this moment.'); + } + $alias = parent::addJoin($type, $table, $alias, $condition, $arguments); + if (isset($type)) { + if ($type != 'INNER' && $type != 'LEFT') { + throw new \Exception(sprintf('%s type not supported, only INNER and LEFT.', $type)); + } + if (!preg_match('/(\w+)\.(\w+)\s*=\s*(\w+)\.(\w+)/', $condition, $matches)) { + throw new \Exception('Only x.field1 = y.field2 conditions are supported.' . $condition); + } + if ($matches[1] == $alias) { + $this->tables[$alias] += array( + 'added_field' => $matches[2], + 'original_table_alias' => $matches[3], + 'original_field' => $matches[4], + ); + } + elseif ($matches[3] == $alias) { + $this->tables[$alias] += array( + 'added_field' => $matches[4], + 'original_table_alias' => $matches[1], + 'original_field' => $matches[2], + ); + } + else { + throw new \Exception('The JOIN condition does not contain the alias of the joined table.'); + } + } + return $alias; + } + + /** + * {@inheritdoc} + */ + public function execute() { + // @todo: Implement distinct() handling. + $all_rows = $this->executeJoins(); + $this->resolveConditions($this->where, $all_rows); + if (!empty($this->order)) { + usort($all_rows, array($this, 'sortCallback')); + } + // Now flatten the rows so that each row becomes a field alias => value + // array. + $results = array(); + foreach ($all_rows as $table_rows) { + $result_row = array(); + foreach ($table_rows as $row) { + $result_row += $row; + } + $results[] = $result_row; + } + if (!empty($this->range)) { + $results = array_slice($results, $this->range['start'], $this->range['length']); + } + if ($this->countQuery) { + $results = array(array(count($results))); + } + return new FakeStatement($results); + } + + /** + * Create an initial result set by executing the joins and picking fields. + * + * @return array + * A multidimensional array, the first key are table aliases, the second + * are field aliases, the values are the database contents or NULL in case + * of JOINs. + */ + protected function executeJoins() { + // @TODO add support for all_fields. + $fields = array(); + foreach ($this->fields as $field_info) { + $this->fieldsWithTable[$field_info['table'] . '.' . $field_info['field']] = $field_info; + $fields[$field_info['table']][$field_info['field']] = NULL; + } + + $results = array(); + foreach ($this->tables as $table_alias => $table_info) { + if (isset($table_info['join type'])) { + $new_rows = array(); + foreach ($results as $row) { + $joined = FALSE; + foreach ($this->databaseContents[$table_info['table']] as $candidate_row) { + if ($row[$table_info['original_table_alias']][$table_info['original_field']] == $candidate_row[$table_info['added_field']]) { + $joined = TRUE; + $new_rows[] = $this->getNewRow($table_alias, $fields, $candidate_row, $row); + } + } + if (!$joined && $table_info['join type'] == 'LEFT') { + $new_rows[] = array($table_alias => $fields[$table_alias]) + $row; + } + } + $results = $new_rows; + } + else { + foreach ($this->databaseContents[$table_info['table']] as $candidate_row) { + $results[] = $this->getNewRow($table_alias, $fields, $candidate_row); + } + } + } + return $results; + } + + /** + * Retrieves a new row. + * + * @param string $table_alias + * @param array $fields + * @param array $candidate_row + * @param array $row + * + * @return array + */ + protected function getNewRow($table_alias, $fields, $candidate_row, $row = array()) { + $new_row = array(); + foreach ($fields[$table_alias] as $field => $v) { + $new_row[$table_alias][$field] = $candidate_row[$field]; + } + return $new_row + $row; + } + + /** + * {@inheritdoc} + */ + public function countQuery() { + $query = clone $this; + return $query->setCountQuery(); + } + + /** + * Set this query to be a count query. + */ + protected function setCountQuery() { + $this->countQuery = TRUE; + return $this; + } + + /** + * usort callback to order the results. + */ + protected function sortCallback($a, $b) { + foreach ($this->order as $field => $direction) { + $field_info = $this->getFieldInfo($field); + $a_value = $a[$field_info['table']][$field_info['field']]; + $b_value = $b[$field_info['table']][$field_info['field']]; + if ($a_value != $b_value) { + return (($a_value < $b_value) == ($direction == 'ASC')) ? -1 : 1; + } + } + return 0; + } + + protected function getFieldInfo($field) { + return isset($this->fieldsWithTable[$field]) ? $this->fieldsWithTable[$field] : $this->fields[$field]; + } + + /** + * Resolves conditions by removing non-matching rows. + * + * @param array $rows + * An array of rows excluding non-matching rows. + */ + protected function resolveConditions(Condition $condition_group, array &$rows) { + foreach ($rows as $k => $row) { + if (!$this->matchGroup($row, $condition_group)) { + unset($rows[$k]); + } + } + } + + /** + * Match a row against a group of conditions. + * + * @param array $row + * + * @param \Drupal\Core\Database\Query\Condition $condition_group + * + * @return bool + */ + protected function matchGroup(array $row, Condition $condition_group) { + $conditions = $condition_group->conditions(); + $and = $conditions['#conjunction'] == 'AND'; + unset($conditions['#conjunction']); + $match = TRUE; + foreach ($conditions as $condition) { + $match = $condition['field'] instanceof Condition ? $this->matchGroup($row, $condition['field']) : $this->matchSingle($row, $condition); + // For AND, finish matching on the first fail. For OR, finish on first + // success. + if ($and != $match) { + break; + } + } + return $match; + } + + /** + * Match a single row and its condition. + * + * @param array $row + * The row to match. + * + * @param array $condition + * An array representing a single condition. + * + * @return bool + * TRUE if the condition matches. + */ + protected function matchSingle(array $row, array $condition) { + $field_info = $this->getFieldInfo($condition['field']); + $row_value = $row[$field_info['table']][$field_info['field']]; + switch ($condition['operator']) { + case '=': + return $row_value == $condition['value']; + + case '<=': + return $row_value <= $condition['value']; + + case '>=': + return $row_value >= $condition['value']; + + case '!=': + return $row_value != $condition['value']; + + case '<>': + return $row_value != $condition['value']; + + case '<': + return $row_value < $condition['value']; + + case '>': + return $row_value > $condition['value']; + + case 'IN': + return in_array($row_value, $condition['value']); + + case 'IS NULL': + return !isset($row_value); + + case 'IS NOT NULL': + return isset($row_value); + + default: + throw new \Exception(sprintf('operator %s is not supported', $condition['operator'])); + } + } + + /** + * {@inheritdoc} + */ + public function orderBy($field, $direction = 'ASC') { + $this->order[$field] = strtoupper($direction); + return $this; + } + + // ================== we could support these. + /** + * {@inheritdoc} + */ + public function groupBy($field) { + // @todo: Implement groupBy() method. + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function havingCondition($field, $value = NULL, $operator = NULL) { + // @todo: Implement havingCondition() method. + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function uniqueIdentifier() { + // TODO: Implement uniqueIdentifier() method. + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + // ================== the rest won't be supported, ever. + /** + * {@inheritdoc} + */ + public function nextPlaceholder() { + // TODO: Implement nextPlaceholder() method. + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function isPrepared() { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function preExecute(SelectInterface $query = NULL) { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function where($snippet, $args = array()) { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function extend($extender_name) { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function &getExpressions() { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function &getGroupBy() { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function &getUnion() { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function forUpdate($set = TRUE) { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function rightJoin($table, $alias = NULL, $condition = NULL, $arguments = array()) { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function &conditions() { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function orderRandom() { + // We could implement this but why bother. + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function union(SelectInterface $query, $type = '') { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function addExpression($expression, $alias = NULL, $arguments = array()) { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function &getTables() { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function getArguments(PlaceholderInterface $query_place_holder = NULL) { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function &getOrderBy() { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function &getFields() { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function exists(SelectInterface $select) { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function notExists(SelectInterface $select) { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function arguments() { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function compile(Connection $connection, PlaceholderInterface $query_place_holder) { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * {@inheritdoc} + */ + public function compiled() { + throw new \Exception(sprintf('Method "%s" is not supported', __METHOD__)); + } + + /** + * Provide meta information about this battery of tests. + */ + public static function getInfo() { + return array( + 'name' => 'Fake select test', + 'description' => 'Tests for fake select plugin.', + 'group' => 'Migrate', + ); + } + +} diff --git a/core/modules/migrate/tests/Drupal/migrate/Tests/FakeStatement.php b/core/modules/migrate/tests/Drupal/migrate/Tests/FakeStatement.php new file mode 100644 index 0000000..9088c50 --- /dev/null +++ b/core/modules/migrate/tests/Drupal/migrate/Tests/FakeStatement.php @@ -0,0 +1,103 @@ +count(); + } + + /** + * {@inheritdoc} + */ + public function fetchField($index = 0) { + $row = array_values($this->current()); + $return = $row[$index]; + $this->next(); + return $return; + } + + /** + * {@inheritdoc} + */ + public function fetchAssoc() { + $return = $this->current(); + $this->next(); + return $return; + } + + /** + * {@inheritdoc} + */ + public function fetchCol($index = 0) { + $return = array(); + foreach ($this as $row) { + $row = array_values($row); + $return[] = $row[$index]; + } + return $return; + } + + /** + * {@inheritdoc} + */ + public function fetchAllKeyed($key_index = 0, $value_index = 1) { + $return = array(); + foreach ($this as $row) { + $row = array_values($row); + $return[$row[$key_index]] = $row[$value_index]; + } + return $return; + } + + /** + * {@inheritdoc} + */ + public function fetchAllAssoc($key, $fetch = NULL) { + $return = array(); + foreach ($this as $row) { + $return[$row[$key]] = $row; + } + return $return; + } + + /** + * {@inheritdoc} + */ + public static function getInfo() { + return array( + 'name' => 'Fake statement test', + 'description' => 'Tests for fake statement plugin.', + 'group' => 'Migrate', + ); + } + +} diff --git a/core/modules/migrate/tests/Drupal/migrate/Tests/MigrateSqlSourceTestCase.php b/core/modules/migrate/tests/Drupal/migrate/Tests/MigrateSqlSourceTestCase.php new file mode 100644 index 0000000..bdee148 --- /dev/null +++ b/core/modules/migrate/tests/Drupal/migrate/Tests/MigrateSqlSourceTestCase.php @@ -0,0 +1,134 @@ +databaseContents + array('test_map' => array()); + $database = $this->getMockBuilder('Drupal\Core\Database\Connection') + ->disableOriginalConstructor() + ->getMock(); + $database->expects($this->any())->method('select')->will($this->returnCallback(function ($base_table, $base_alias) use ($database_contents) { + return new FakeSelect($base_table, $base_alias, $database_contents); + })); + $database->expects($this->any())->method('schema')->will($this->returnCallback(function () use ($database, $database_contents) { + return new FakeDatabaseSchema($database, $database_contents); + })); + + $migration = $this->getMigration(); + $migration->expects($this->any()) + ->method('getHighwater') + ->will($this->returnValue(static::ORIGINAL_HIGHWATER)); + + $plugin_class = static::PLUGIN_CLASS; + $plugin = new $plugin_class($this->migrationConfiguration['source'], $this->migrationConfiguration['source']['plugin'], array(), $migration); + $this->writeAttribute($plugin, 'database', $database); + $migration->expects($this->any()) + ->method('getSource') + ->will($this->returnValue($plugin)); + $this->source = new Source($migration); + + $cache = $this->getMock('Drupal\Core\Cache\CacheBackendInterface'); + $this->writeAttribute($this->source, 'cache', $cache); + } + + /** + * Sets attribute values for a source test. + * + * @param object $object + * The destination of the written attribute value. + * @param string $attribute_name + * The name of the attribute to write. + * @param mixed $value + * The value of the written attribute. + */ + protected function writeAttribute($object, $attribute_name, $value) { + $reflection = new \ReflectionClass($object); + $reflection_property = $reflection->getProperty($attribute_name); + $reflection_property->setAccessible(TRUE); + $reflection_property->setValue($object, $value); + } + + /** + * Tests retrieval. + */ + public function testRetrieval() { + $this->assertSame(count($this->results), count($this->source), 'Number of results match'); + $count = 0; + foreach ($this->source as $data_row) { + $expected_row = $this->results[$count]; + $count++; + foreach ($expected_row as $key => $expected_value) { + $this->retrievalAssertHelper($expected_value, $data_row->getSourceProperty($key), sprintf('Value matches for key "%s"', $key)); + } + } + $this->assertSame(count($this->results), $count); + } + + /** + * Asserts tested values during test retrieval. + * + * @param mixed $expected_value + * The incoming expected value to test. + * @param mixed $actual_value + * The incoming value itself. + * @param string $message + * The tested result as a formatted string. + */ + protected function retrievalAssertHelper($expected_value, $actual_value, $message) { + if (is_array($expected_value)) { + foreach ($expected_value as $k => $v) { + $this->retrievalAssertHelper($v, $actual_value[$k], $message); + } + } + else { + $this->assertSame((string) $expected_value, (string) $actual_value, $message); + } + } + + /** + * {@inheritdoc} + */ + public static function getInfo() { + return array( + 'name' => 'SQL source test', + 'description' => 'Tests for SQL source plugin.', + 'group' => 'Migrate', + ); + } + +} diff --git a/core/modules/migrate/tests/Drupal/migrate/Tests/MigrateTestCase.php b/core/modules/migrate/tests/Drupal/migrate/Tests/MigrateTestCase.php new file mode 100644 index 0000000..84204ce --- /dev/null +++ b/core/modules/migrate/tests/Drupal/migrate/Tests/MigrateTestCase.php @@ -0,0 +1,65 @@ +getMock('Drupal\migrate\Plugin\MigrateIdMapInterface'); + if ($this->mapJoinable) { + $idmap->expects($this->once()) + ->method('getQualifiedMapTable') + ->will($this->returnValue('test_map')); + } + + $migration = $this->getMock('Drupal\migrate\Entity\MigrationInterface'); + $migration->expects($this->any()) + ->method('getIdMap') + ->will($this->returnValue($idmap)); + $configuration = $this->migrationConfiguration; + $migration->expects($this->any())->method('get')->will($this->returnCallback(function ($argument) use ($configuration) { + return isset($configuration[$argument]) ? $configuration[$argument] : ''; + })); + $migration->expects($this->any()) + ->method('id') + ->will($this->returnValue($configuration['id'])); + return $migration; + } + + /** + * Provide meta information about this battery of tests. + */ + public static function getInfo() { + return array( + 'name' => 'Migrate test', + 'description' => 'Tests for migrate plugin.', + 'group' => 'Migrate', + ); + } + +}