From 7dd72136dff92f99ea5a9aea4b3157e5c0f6a69b Mon Sep 17 00:00:00 2001 From: discipolo Date: Sun, 23 Nov 2014 01:13:31 +0100 Subject: [PATCH] Issue #2377151 add file migrations --- base.inc | 59 ++++++++- typo3_files/audio.inc | 28 +++++ typo3_files/document.inc | 28 +++++ typo3_files/files.inc | 271 +++++++++++++++++++++++++++++++++++++++++ typo3_files/image.inc | 32 +++++ typo3_files/typo3_files.info | 17 +++ typo3_files/typo3_files.module | 51 ++++++++ typo3_files/video.inc | 28 +++++ typo3_migrate.info | 3 +- typo3_migrate.module | 11 +- 10 files changed, 522 insertions(+), 6 deletions(-) create mode 100644 typo3_files/audio.inc create mode 100644 typo3_files/document.inc create mode 100644 typo3_files/files.inc create mode 100644 typo3_files/image.inc create mode 100644 typo3_files/typo3_files.info create mode 100644 typo3_files/typo3_files.module create mode 100644 typo3_files/video.inc diff --git a/base.inc b/base.inc index 9edb965..6620e49 100644 --- a/base.inc +++ b/base.inc @@ -13,12 +13,50 @@ abstract class BaseMigration extends Migration { public $queryParams = array(); - public function __construct() { + /** + * Arguments for the containing migration. Primarily of interest for + * the source_connection. + * + * @var array + */ + protected $arguments; + /** + * @abstract + * The base source query for this migration. + * + * @return QueryConditionInterface + */ + abstract protected function query(); + /** + * Options to be passed to source constructors. + * + * @var array + */ + protected $sourceOptions = array(); + + /** + * An array of available source fields, beyond those in the base query. + * Derived classes should populate this before calling the parent + * constructor. + * + * @var array + */ + protected $sourceFields = array(); + + public function __construct($arguments) { + $this->arguments = $arguments; // Always call the parent constructor first for basic setup - parent::__construct(); + parent::__construct($arguments); $this->team = array( new MigrateTeamMember('Test', 'test@srijan.in', t('Product Owner')), ); + + $this->sourceOptions = array('map_joinable' => FALSE, 'cache_counts' => TRUE, + 'cache_key' => 'migrate_' . $this->machineName); + if (!empty($this->arguments['source_options'])) { + $this->sourceOptions = array_merge($this->sourceOptions, + $this->arguments['source_options']); + } $this->issuePattern = 'http://drupal.org/node/:id'; @@ -46,7 +84,13 @@ abstract class BaseMigration extends Migration { // News settings $this->queryParams['newsCat']['hidden'] = array(0); $this->queryParams['newsCat']['hidden_operator'] = 'IN'; + + + // File settings + $this->queryParams['fileCat']['hidden'] = array(0); + $this->queryParams['fileCat']['hidden_operator'] = 'IN'; // Check if news folder setings have been set. By default all the news content is migrated. + $pids = variable_get('typo3_pids', '0'); if ($pids != '0') { $this->queryParams['news']['pid'] = explode(",", $pids); @@ -118,6 +162,17 @@ abstract class BaseMigration extends Migration { /** * + * Returns the query object + * //@TODO: Make this configurable + * @param Array $words + */ + public function getQueryObject() { + return Database::getConnection('default', 'typo3'); + } + + + /** + * * This will remove unwanted data from the word after the link tag is found * @param Array $words * @param Integer $pos diff --git a/typo3_files/audio.inc b/typo3_files/audio.inc new file mode 100644 index 0000000..34d71bf --- /dev/null +++ b/typo3_files/audio.inc @@ -0,0 +1,28 @@ +condition('file_type', $allowed, 'IN'); + + return $query; + + } + +} diff --git a/typo3_files/document.inc b/typo3_files/document.inc new file mode 100644 index 0000000..ab34003 --- /dev/null +++ b/typo3_files/document.inc @@ -0,0 +1,28 @@ +condition('file_type', $allowed, 'IN'); + + return $query; + + } + +} diff --git a/typo3_files/files.inc b/typo3_files/files.inc new file mode 100644 index 0000000..9bc767c --- /dev/null +++ b/typo3_files/files.inc @@ -0,0 +1,271 @@ +description = t('Migrate Files Category Taxonomy'); + + $this->map = new MigrateSQLMap($this->machineName, + array( + 'uid' => array('type' => 'int', + 'length' => 11, + 'not null' => TRUE, + 'description' => 'Category', + ) + ), + MigrateDestinationTerm::getKeySchema() + ); + + + + // Create a MigrateSource object, which manages retrieving the input data. + $this->source = new MigrateSourceSQL($this->query()); + + // Set up our destination - terms in the Files Category vocabulary + $this->destination = new MigrateDestinationTerm('Files Category'); + + $this->addFieldMapping('name', 'title'); + $this->addFieldMapping('description', 'description'); + + $this->addFieldMapping('parent', 'parent_category') + ->sourceMigration('Typo3fileCategory') + ->defaultValue(0); + + $this->addFieldMapping('format') + ->issueGroup(t('DNM')); + $this->addFieldMapping('weight') + ->issueGroup(t('DNM')); + } + /** + * The base source query for this migration. + * + * @return QueryConditionInterface + */ + protected function query() { +// initiate connection to the typo3 database and select the table... + $query = $this->getQueryObject()->select('tx_dam_cat', 'cat') + ->fields('cat', array('uid', 'title', 'parent_category')); + + $query->condition('cat.deleted', '0'); + if ($this->queryParams['fileCat']['hidden'] != '') { + $query->condition('cat.hidden', $this->queryParams['fileCat']['hidden'], $this->queryParams['fileCat']['hidden_operator']); + } + + return $query; + } +} +/** + * Base class for all file migrations - handles commonalities across all + * supported source Typo3 versions. + * + * In addition to the arguments supported by Typo3Migration, the following + * must be passed in the $arguments array: + * source_dir: Path to folder containing files to be migrated. + * + * The following optional arguments may be passed: + * user_migration - Machine name of a user migration, used to establish + * dependencies and a sourceMigration for the uid mapping. + * default_uid - Drupal7 7 (destination) uid of the user account to use as + * the default. + * bundle - File bundle to use as the target - defaults to 'file'. + * file_class - Override for the default MigrateFileUri file class. + * destination_dir - Destination directory for the files (defaults to public://). + */ +class Typo3FileMigration extends BaseMigration { + protected $baseDir; + public function __construct(array $arguments) { + $this->arguments = $arguments; + + parent::__construct($arguments); + if (!$this->newOnly) { + $this->highwaterField = array( + 'name' => 'timestamp', + 'alias' => 'f', + 'type' => 'int', + ); + } + $this->description = t('Import files.'); + $this->base_dir = TYPO3_FILES_SRC; + + +// the actual files are in tx_dam +// TODO: initiate databas connection in base.inc https://www.drupal.org/node/1006984 source object + + if (!empty($arguments['user_migration'])) { + $user_migration = $arguments['user_migration']; + $this->dependencies[] = $user_migration; + } + + if (empty($arguments['bundle'])) { + $arguments['bundle'] = 'file'; + } + if (empty($arguments['file_class'])) { + $arguments['file_class'] = 'MigrateFileUri'; + } + if (empty($arguments['destination_dir'])) { + $arguments['destination_dir'] = 'public://'; + } + + + // Allow derived classes to override this definition by setting it before + // calling their parent constructor + if (!isset($this->map)) { + $this->map = new MigrateSQLMap($this->machineName, + array( + 'uid' => array( + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'description' => 'Source file ID', + 'alias' => 'f', + ), + ), + MigrateDestinationFile::getKeySchema() + ); + } + $this->sourceFields = array( + 'file_path' => t('file_path'), + 'file_name' => t('file_name'), + 'file_mime_type' => t('file_mime_type'), + 'media_type' => t('media_type'), + 'title' => t('title'), + 'description' => t('description'), + 'caption' => t('caption'), + 'keywords' => t('keywords'), + 'instructions' => t('instructions'), + 'uid' => t('uid'), + +); + + $this->source = new MigrateSourceSQL($this->query(), + $this->sourceFields, NULL, $this->sourceOptions); + + + // todo: make sure we are mapping correctly https://www.drupal.org/node/1007004 + + $directories = array( + $this->baseDir, + ); + + + $this->destination = new MigrateDestinationFile($arguments['bundle'],$arguments['file_class']); + + //Field mappings https://www.drupal.org/node/1133448 + +// field mappings for metadata on the files + + $this->addFieldMapping('field_body', 'description'); + $this->addFieldMapping('field_tags', 'tags'); + + + $this->addFieldMapping('field_metadata_title', 'title'); + $this->addFieldMapping('field_metadata_caption', 'caption'); + + + // use filename and path seperately + $this->addFieldMapping('value', 'file_name'); + // the path to the source file + $this->addFieldMapping('source_dir', 'file_path') + ->callbacks(array($this, 'fixUri')); + // In case of Taxonomy, no need to pass the Source Migration. + // Only pass title to the taxonmoy + $this->addFieldMapping('File Category', 'filetags') + ->separator(','); + + // metatags https://www.drupal.org/node/2049565 + $this->addFieldMapping('metatag_title', 'title'); + $this->addFieldMapping('metatag_description', 'description'); + $this->addFieldMapping('metatag_keywords', 'keywords'); + + // Setup common mappings + // Save to the default file scheme. + + $this->addFieldMapping('destination_dir') + ->defaultValue($arguments['destination_dir']); + + $this->addFieldMapping('file_replace') + ->defaultValue(MigrateFile::FILE_EXISTS_REUSE); + $this->addFieldmapping('preserve_files') + ->defaultValue(TRUE); + + if (isset($arguments['default_uid'])) { + $default_uid = $arguments['default_uid']; + } + else { + $default_uid = 1; + } + if (isset($user_migration)) { + $this->addFieldMapping('uid', 'uid') + ->sourceMigration($user_migration) + ->defaultValue($default_uid); + } + else { + $this->addFieldMapping('uid') + ->defaultValue($default_uid); + } + $this->addUnmigratedSources(array('media_type', 'filemime', 'filesize')); + } + + + + /** + * The base source query for this migration. + * + * @return QueryConditionInterface + */ + protected function query() { + $query = $this->getQueryObject()->select('tx_dam', 'f'); + $query->fields('f'); + //->orderBy('tstamp', 'ASC'); + + $query->condition('f.deleted', '0'); + + $count_query = $query->countQuery(); + $query->leftJoin( 'tx_dam_mm_cat', 'catmm', 'catmm.uid_local = f.uid'); + $query->leftJoin( 'tx_dam_cat', 'filecat', 'filecat.uid = catmm.uid_foreign'); + $query->groupBy('f.uid'); + + $query->addExpression('GROUP_CONCAT(filecat.title)', 'filetags'); + + $query->addExpression('GROUP_CONCAT(catmm.uid_foreign)', 'cat_list'); + + return $query; + } + public function prepareRow($row) { + + if (parent::prepareRow($row) === FALSE) { + return FALSE; + } + watchdog('typo3_migrate', 'migrating row: %file' , array('%file' => print_r($row,true)), WATCHDOG_NOTICE); + // if the source file doesn exist in the directory then skip it + // add source + $row->file_path = TYPO3_FILES_SRC . '/' . $row->file_path; + $file = $row->file_path . $row->file_name; + if (!file_exists($file)) { + + watchdog('typo3_migrate', 'file %file doesnt exist, skipping' , array('%file' => $file), WATCHDOG_NOTICE); + return FALSE; + } + + + + // Remove the leading forward slash. + //$row->destination_file = substr($row->sourceid, 1); + } + + protected function fixUri($uri) { + $result = $uri; + watchdog('typo3_migrate', 'source url %url fixed' , array('%url' => $result), WATCHDOG_NOTICE); + return $result; + } + +} diff --git a/typo3_files/image.inc b/typo3_files/image.inc new file mode 100644 index 0000000..5628c96 --- /dev/null +++ b/typo3_files/image.inc @@ -0,0 +1,32 @@ + $arguments['bundle']), WATCHDOG_NOTICE); + $this->addFieldMapping('field_file_image_title_text', 'title'); + $this->addFieldMapping('field_file_image_alt_text', 'title'); + + } + +// extending the protected query function in file.in + protected function query() { + $allowed = array('jpg','jpeg','gif','png'); + $query = parent::query() + ->condition('file_type', $allowed, 'IN'); + + return $query; + + } +} diff --git a/typo3_files/typo3_files.info b/typo3_files/typo3_files.info new file mode 100644 index 0000000..0666033 --- /dev/null +++ b/typo3_files/typo3_files.info @@ -0,0 +1,17 @@ +name = "Typo3 Files Migrate" +description = "Typo3 Files to drupal migration." +package = "Migration" +core = 7.x +project = typo3_files_migrate + +dependencies[] = taxonomy +dependencies[] = file +dependencies[] = media +dependencies[] = typo3_migrate +dependencies[] = migrate_extras + +files[] = files.inc +files[] = image.inc +files[] = video.inc +files[] = audio.inc +files[] = document.inc diff --git a/typo3_files/typo3_files.module b/typo3_files/typo3_files.module new file mode 100644 index 0000000..dabd8cb --- /dev/null +++ b/typo3_files/typo3_files.module @@ -0,0 +1,51 @@ + 2, + // Give the group a human-readable title + 'groups' => array( + 'typo3' => array( + 'title' => t('typo3'), + ), + ), + + // Migration classes. + 'migrations' => array( + 'Typo3FileCategoryMigration' => array( + 'class_name' => 'Typo3FileCategoryMigration', + 'group_name' => 'typo3', + ), + + 'Typo3ImageFileMigration' => array( + 'class_name' => 'Typo3ImageFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILES_SRC, + 'bundle' => 'image', + ), + 'Typo3DocumentFileMigration' => array( + 'class_name' => 'Typo3DocumentFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILES_SRC, + 'bundle' => 'document', + ), + 'Typo3VideoFileMigration' => array( + 'class_name' => 'Typo3VideoFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILES_SRC, + 'bundle' => 'video', + ), + 'Typo3AudioFileMigration' => array( + 'class_name' => 'Typo3AudioFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILES_SRC, + 'bundle' => 'audio', + ), + ), + ); + return $api; +} diff --git a/typo3_files/video.inc b/typo3_files/video.inc new file mode 100644 index 0000000..07bea09 --- /dev/null +++ b/typo3_files/video.inc @@ -0,0 +1,28 @@ +allowed = $arguments['allowed']; + } + +// extending the protected query function in file.in + protected function query() { + $allowed = array('avi','mpeg','mp4','flv'); + $query = parent::query() + ->condition('file_type', $allowed, 'IN'); + return $query; + + } + +} diff --git a/typo3_migrate.info b/typo3_migrate.info index f3e00db..b135c36 100644 --- a/typo3_migrate.info +++ b/typo3_migrate.info @@ -2,7 +2,8 @@ name = "Typo3 Migrate" description = "Typo3 to drupal migration." package = "Migration" core = 7.x - +php = 5.2 dependencies[] = migrate //dependencies[] = migrate_extras +dependencies[] = date files[] = base.inc diff --git a/typo3_migrate.module b/typo3_migrate.module index 96bc1dc..e817794 100644 --- a/typo3_migrate.module +++ b/typo3_migrate.module @@ -1,6 +1,6 @@ 'textfield', '#title' => t('Database Name'), '#description' => t('The Typo3 db must be accessible by the drupal db user and must reside on the same db server.'), - '#default_value' => variable_get('typo3_database', 'typo3_DB'), + '#default_value' => TYPO3_DATABASE_NAME, '#required' => TRUE ); + $form['typo3_file_source'] = array( + '#type' => 'textfield', + '#size' => 60, + '#title' => t('File prefix'), + '#default_value' => TYPO3_FILES_SRC, + ); return system_settings_form($form); } - -- 1.9.1