From bb4b6627502c128991acea806ab6ea24c9360118 Mon Sep 17 00:00:00 2001 From: Rosenstrauch Date: Sat, 22 Nov 2014 06:22:25 +0100 Subject: [PATCH] Issue #2377151 add typo3_files import --- typo3_files/files.inc | 233 +++++++++++++++++++++++++++++++++++++++++ typo3_files/typo3_files.info | 13 +++ typo3_files/typo3_files.module | 32 ++++++ typo3_migrate.module | 17 ++- 4 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 typo3_files/files.inc create mode 100644 typo3_files/typo3_files.info create mode 100644 typo3_files/typo3_files.module diff --git a/typo3_files/files.inc b/typo3_files/files.inc new file mode 100644 index 0000000..d2174cd --- /dev/null +++ b/typo3_files/files.inc @@ -0,0 +1,233 @@ +description = t('Migrate Files Category Taxonomy'); + + $this->map = new MigrateSQLMap($this->machineName, + array( + 'uid' => array('type' => 'int', + 'length' => 11, + 'not null' => TRUE, + 'description' => 'Category', + ) + ), + MigrateDestinationTerm::getKeySchema() + ); + + // initiate connection to the typo3 database and select the table... + $query = db_select(TYPO3_DATABASE_NAME . '.tx_dam_cat', 'cat') + ->fields('cat', array('uid', 'title', 'parent_category')); + + $query->condition('cat.deleted', '0'); + if ($this->queryParams['fileCat']['hidden'] != '') { + $query->condition('cat.hidden', $this->queryParams['fileCat']['hidden'], $this->queryParams['fileCat']['hidden_operator']); + } + + // Create a MigrateSource object, which manages retrieving the input data. + $this->source = new MigrateSourceSQL($query); + + // Set up our destination - terms in the Files Category vocabulary + $this->destination = new MigrateDestinationTerm('Files Category'); + + $this->addFieldMapping('name', 'title'); + $this->addFieldMapping('description', 'description'); + + $this->addFieldMapping('parent', 'parent_category') + ->sourceMigration('Typo3fileCategory') + ->defaultValue(0); + + $this->addFieldMapping('format') + ->issueGroup(t('DNM')); + $this->addFieldMapping('weight') + ->issueGroup(t('DNM')); + } +} +/** + * Base class for all file migrations - handles commonalities across all + * supported source Typo3 versions. + * + * In addition to the arguments supported by Typo3Migration, the following + * must be passed in the $arguments array: + * source_dir: Path to folder containing files to be migrated. + * + * The following optional arguments may be passed: + * user_migration - Machine name of a user migration, used to establish + * dependencies and a sourceMigration for the uid mapping. + * default_uid - Drupal7 7 (destination) uid of the user account to use as + * the default. + * bundle - File bundle to use as the target - defaults to 'file'. + * file_class - Override for the default MigrateFileUri file class. + * destination_dir - Destination directory for the files (defaults to public://). + */ +class Typo3FileMigration extends BaseMigration { + + protected $baseDir; + + public function __construct(array $arguments) { + parent::__construct($arguments); + if (!$this->newOnly) { + $this->highwaterField = array( + 'name' => 'timestamp', + 'alias' => 'f', + 'type' => 'int', + ); + } + $this->description = t('Import files.'); + $this->base_dir = TYPO3_FILE_SRC; + +// the actual files are in tx_dam +// TODO: initiate databas connection in base.inc https://www.drupal.org/node/1006984 source object + $query = db_select(TYPO3_DATABASE_NAME . '.tx_dam', 'f') + ->fields('f', array('file_path', 'file_name','file_mime_type','media_type','title', 'description','caption','keywords','instructions' )); + //->orderBy('tstamp', 'ASC'); + + //$query = db_select(TYPO3_DATABASE_NAME . '.tx_dam_mm_ref', 'u') + // ->fields('u', array('uid_local', 'uid_foreign','tablenames','ident','sorting_foreign','sorting' )) + + $query->condition('f.deleted', '0'); + $allowed = array('jpg','jpeg','gif','png','txt'); + + $query->condition('file_type', $allowed, 'IN'); + $query->leftJoin( TYPO3_DATABASE_NAME . '.tx_dam_mm_cat', 'catmm', 'catmm.uid_local = f.uid'); + $query->leftJoin( TYPO3_DATABASE_NAME . '.tx_dam_cat', 'filecat', 'filecat.uid = catmm.uid_foreign'); + $query->groupBy('f.uid'); + + $query->addExpression('GROUP_CONCAT(filecat.title)', 'filetags'); + + $query->addExpression('GROUP_CONCAT(catmm.uid_foreign)', 'cat_list'); + if (!empty($arguments['user_migration'])) { + $user_migration = $arguments['user_migration']; + $this->dependencies[] = $user_migration; + } + if (empty($arguments['bundle'])) { + $arguments['bundle'] = 'file'; + } + if (empty($arguments['file_class'])) { + $arguments['file_class'] = 'MigrateFileUri'; + } + if (empty($arguments['destination_dir'])) { + $arguments['destination_dir'] = 'public://'; + } + + // Allow derived classes to override this definition by setting it before + // calling their parent constructor + if (!isset($this->map)) { + $this->map = new MigrateSQLMap($this->machineName, + array( + 'uid' => array( + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'description' => 'Source file ID', + 'alias' => 'f', + ), + ), + MigrateDestinationFile::getKeySchema() + ); + } + // todo: make sure we are mapping correctly https://www.drupal.org/node/1007004 + $this->source = new MigrateSourceSQL($query, array(), NULL, array('map_joinable' => FALSE)); + + $directories = array( + $this->baseDir, + ); + + + $this->destination = new MigrateDestinationFile($arguments['bundle'], + $arguments['file_class']); + + //Field mappings https://www.drupal.org/node/1133448 + // field mappings for metadata on the files + $this->addFieldMapping('title', 'field_title'); + $this->addFieldMapping('description', 'field_body'); + $this->addFieldMapping('field_tags', 'tags'); + + // In case of Taxonomy, no need to pass the Source Migration. + // Only pass title to the taxonmoy + $this->addFieldMapping('File Category', 'filetags') + ->separator(','); + + // metatags https://www.drupal.org/node/2049565 + $this->addFieldMapping('metatag_title', 'legacy_seo_title'); + $this->addFieldMapping('metatag_description', 'legacy_seo_description'); + + // Setup common mappings + // Save to the default file scheme. + + $this->addFieldMapping('destination_dir') + ->defaultValue($arguments['destination_dir']); + + + + // the path to the source file + $this->addFieldMapping('source_dir') + ->callbacks(array($this, 'fixUri')) + ->defaultValue('file_path');; + + + // Set the value to the file name + $this->addFieldMapping('value', 'file_name'); + + + $this->addFieldMapping('file_replace') + ->defaultValue(MigrateFile::FILE_EXISTS_REUSE); + $this->addFieldmapping('preserve_files') + ->defaultValue(TRUE); + + if (isset($arguments['default_uid'])) { + $default_uid = $arguments['default_uid']; + } + else { + $default_uid = 1; + } + if (isset($user_migration)) { + $this->addFieldMapping('uid', 'uid') + ->sourceMigration($user_migration) + ->defaultValue($default_uid); + } + else { + $this->addFieldMapping('uid') + ->defaultValue($default_uid); + } + $this->addUnmigratedSources(array('filename', 'filemime', 'filesize')); + } + public function prepareRow($row) { + + + + if (parent::prepareRow($row) === FALSE) { + return FALSE; + } + + + +// if the source file doesn exist in the directory then skip it +$file = TYPO3_FILE_SRC . '/' . $row->file_path . $row->file_name; +if (!file_exists($file)) { + + watchdog('typo3_migrate', 'file %file doesnt exist, skipping' , array('%file' => $file), WATCHDOG_NOTICE); + return FALSE; +} + + + // Remove the leading forward slash. + //$row->destination_file = substr($row->sourceid, 1); + } + protected function fixUri($uri) { + $result = TYPO3_FILE_SRC . '/' . $uri; + watchdog('typo3_migrate', 'source url %url fixed' , array('%url' => $result), WATCHDOG_NOTICE); + return $result; + } + +} diff --git a/typo3_files/typo3_files.info b/typo3_files/typo3_files.info new file mode 100644 index 0000000..0bc257c --- /dev/null +++ b/typo3_files/typo3_files.info @@ -0,0 +1,13 @@ +name = "Typo3 Files Migrate" +description = "Typo3 Files to drupal migration." +package = "Migration" +core = 7.x +project = typo3_files_migrate + +dependencies[] = taxonomy +dependencies[] = file +dependencies[] = media +dependencies[] = typo3_migrate +dependencies[] = migrate_extras + +files[] = files.inc diff --git a/typo3_files/typo3_files.module b/typo3_files/typo3_files.module new file mode 100644 index 0000000..da8b383 --- /dev/null +++ b/typo3_files/typo3_files.module @@ -0,0 +1,32 @@ + 2, + // Give the group a human-readable title + 'groups' => array( + 'typo3' => array( + 'title' => t('typo3'), + ), + ), + + // Migration classes. + 'migrations' => array( + 'typo3_files_cat' => array( + 'class_name' => 'Typo3FileCategoryMigration', + 'group_name' => 'typo3', + ), + 'typo3_files' => array( + 'class_name' => 'Typo3FileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILE_SRC, + ), + ), + ); + return $api; +} diff --git a/typo3_migrate.module b/typo3_migrate.module index 96bc1dc..f191982 100644 --- a/typo3_migrate.module +++ b/typo3_migrate.module @@ -1,6 +1,6 @@ 2, + // Groups. + 'groups' => array( + 'typo3' => array( + 'title' => t('typo3'), + ), + ), ); return $api; } @@ -57,9 +63,14 @@ function typo3_migrate_config($form_state) { '#type' => 'textfield', '#title' => t('Database Name'), '#description' => t('The Typo3 db must be accessible by the drupal db user and must reside on the same db server.'), - '#default_value' => variable_get('typo3_database', 'typo3_DB'), + '#default_value' => TYPO3_DATABASE_NAME, '#required' => TRUE ); + $form['source_directory'] = array( + '#type' => 'textfield', + '#size' => 60, + '#title' => t('File prefix'), + '#default_value' => TYPO3_FILES_SRC, + ); return system_settings_form($form); } - -- 1.9.1