From e4135c44a0fcb34ae585d489061560bfdcb015cf Mon Sep 17 00:00:00 2001 From: discipolo Date: Sun, 23 Nov 2014 01:13:31 +0100 Subject: [PATCH] Issue #2377151 add file migrations --- base.inc | 43 ++++++- typo3_files/audio.inc | 33 +++++ typo3_files/documents.inc | 33 +++++ typo3_files/files.inc | 273 +++++++++++++++++++++++++++++++++++++++++ typo3_files/images.inc | 34 +++++ typo3_files/typo3_files.info | 17 +++ typo3_files/typo3_files.module | 48 ++++++++ typo3_files/videos.inc | 41 +++++++ typo3_migrate.info | 3 +- typo3_migrate.module | 11 +- 10 files changed, 530 insertions(+), 6 deletions(-) create mode 100644 typo3_files/audio.inc create mode 100644 typo3_files/documents.inc create mode 100644 typo3_files/files.inc create mode 100644 typo3_files/images.inc create mode 100644 typo3_files/typo3_files.info create mode 100644 typo3_files/typo3_files.module create mode 100644 typo3_files/videos.inc diff --git a/base.inc b/base.inc index 9edb965..63f4584 100644 --- a/base.inc +++ b/base.inc @@ -12,13 +12,46 @@ */ abstract class BaseMigration extends Migration { public $queryParams = array(); + protected $arguments; - public function __construct() { + + /** + * @abstract + * The base source query for this migration. + * + * @return QueryConditionInterface + */ + abstract protected function query(); + /** + * Options to be passed to source constructors. + * + * @var array + */ + protected $sourceOptions = array(); + + /** + * An array of available source fields, beyond those in the base query. + * Derived classes should populate this before calling the parent + * constructor. + * + * @var array + */ + protected $sourceFields = array(); + + public function __construct($arguments) { + $this->arguments = $arguments; // Always call the parent constructor first for basic setup - parent::__construct(); + parent::__construct($arguments); $this->team = array( new MigrateTeamMember('Test', 'test@srijan.in', t('Product Owner')), ); + + $this->sourceOptions = array('map_joinable' => FALSE, 'cache_counts' => TRUE, + 'cache_key' => 'migrate_' . $this->machineName); + if (!empty($this->arguments['source_options'])) { + $this->sourceOptions = array_merge($this->sourceOptions, + $this->arguments['source_options']); + } $this->issuePattern = 'http://drupal.org/node/:id'; @@ -46,7 +79,13 @@ abstract class BaseMigration extends Migration { // News settings $this->queryParams['newsCat']['hidden'] = array(0); $this->queryParams['newsCat']['hidden_operator'] = 'IN'; + + + // File settings + $this->queryParams['fileCat']['hidden'] = array(0); + $this->queryParams['fileCat']['hidden_operator'] = 'IN'; // Check if news folder setings have been set. By default all the news content is migrated. + $pids = variable_get('typo3_pids', '0'); if ($pids != '0') { $this->queryParams['news']['pid'] = explode(",", $pids); diff --git a/typo3_files/audio.inc b/typo3_files/audio.inc new file mode 100644 index 0000000..f465379 --- /dev/null +++ b/typo3_files/audio.inc @@ -0,0 +1,33 @@ +condition('file_type', $allowed, 'IN'); + + return $query; + + } + +} diff --git a/typo3_files/documents.inc b/typo3_files/documents.inc new file mode 100644 index 0000000..3dfba88 --- /dev/null +++ b/typo3_files/documents.inc @@ -0,0 +1,33 @@ +condition('file_type', $allowed, 'IN'); + + return $query; + + } + +} diff --git a/typo3_files/files.inc b/typo3_files/files.inc new file mode 100644 index 0000000..a7cba5e --- /dev/null +++ b/typo3_files/files.inc @@ -0,0 +1,273 @@ +description = t('Migrate Files Category Taxonomy'); + + $this->map = new MigrateSQLMap($this->machineName, + array( + 'uid' => array('type' => 'int', + 'length' => 11, + 'not null' => TRUE, + 'description' => 'Category', + ) + ), + MigrateDestinationTerm::getKeySchema() + ); + + + + // Create a MigrateSource object, which manages retrieving the input data. + $this->source = new MigrateSourceSQL($this->query()); + + // Set up our destination - terms in the Files Category vocabulary + $this->destination = new MigrateDestinationTerm('Files Category'); + + $this->addFieldMapping('name', 'title'); + $this->addFieldMapping('description', 'description'); + + $this->addFieldMapping('parent', 'parent_category') + ->sourceMigration('Typo3fileCategory') + ->defaultValue(0); + + $this->addFieldMapping('format') + ->issueGroup(t('DNM')); + $this->addFieldMapping('weight') + ->issueGroup(t('DNM')); + } + /** + * The base source query for this migration. + * + * @return QueryConditionInterface + */ + protected function query() { +// initiate connection to the typo3 database and select the table... + $query = db_select(TYPO3_DATABASE_NAME . '.tx_dam_cat', 'cat') + ->fields('cat', array('uid', 'title', 'parent_category')); + + $query->condition('cat.deleted', '0'); + if ($this->queryParams['fileCat']['hidden'] != '') { + $query->condition('cat.hidden', $this->queryParams['fileCat']['hidden'], $this->queryParams['fileCat']['hidden_operator']); + } + + return $query; + } +} +/** + * Base class for all file migrations - handles commonalities across all + * supported source Typo3 versions. + * + * In addition to the arguments supported by Typo3Migration, the following + * must be passed in the $arguments array: + * source_dir: Path to folder containing files to be migrated. + * + * The following optional arguments may be passed: + * user_migration - Machine name of a user migration, used to establish + * dependencies and a sourceMigration for the uid mapping. + * default_uid - Drupal7 7 (destination) uid of the user account to use as + * the default. + * bundle - File bundle to use as the target - defaults to 'file'. + * file_class - Override for the default MigrateFileUri file class. + * destination_dir - Destination directory for the files (defaults to public://). + */ +class Typo3FileMigration extends BaseMigration { + + protected $baseDir; + + public function __construct(array $arguments) { + + + parent::__construct($arguments); + if (!$this->newOnly) { + $this->highwaterField = array( + 'name' => 'timestamp', + 'alias' => 'f', + 'type' => 'int', + ); + } + $this->description = t('Import files.'); + $this->base_dir = TYPO3_FILE_SRC; + + +// the actual files are in tx_dam +// TODO: initiate databas connection in base.inc https://www.drupal.org/node/1006984 source object + + if (!empty($arguments['user_migration'])) { + $user_migration = $arguments['user_migration']; + $this->dependencies[] = $user_migration; + } + if (empty($arguments['bundle'])) { + $arguments['bundle'] = 'file'; + } + if (empty($arguments['file_class'])) { + $arguments['file_class'] = 'MigrateFileUri'; + } + if (empty($arguments['destination_dir'])) { + $arguments['destination_dir'] = 'public://'; + } + + + // Allow derived classes to override this definition by setting it before + // calling their parent constructor + if (!isset($this->map)) { + $this->map = new MigrateSQLMap($this->machineName, + array( + 'uid' => array( + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'description' => 'Source file ID', + 'alias' => 'f', + ), + ), + MigrateDestinationFile::getKeySchema() + ); + } + $this->sourceFields = array( + 'file_path' => t('file_path'), + 'file_name' => t('file_name'), + 'file_mime_type' => t('file_mime_type'), + 'media_type' => t('media_type'), + 'title' => t('title'), + 'description' => t('description'), + 'caption' => t('caption'), + 'keywords' => t('keywords'), + 'instructions' => t('instructions'), + 'uid' => t('uid'), + + +); + + + + $this->source = new MigrateSourceSQL($this->query(), + $this->sourceFields, NULL, $this->sourceOptions); + + + // todo: make sure we are mapping correctly https://www.drupal.org/node/1007004 + + $directories = array( + $this->baseDir, + ); + + + $this->destination = new MigrateDestinationFile($arguments['bundle'], + $arguments['file_class']); + + //Field mappings https://www.drupal.org/node/1133448 + // field mappings for metadata on the files + $this->addFieldMapping('field_file_image_title_text', 'title'); + + $this->addFieldMapping('field_body', 'description'); + $this->addFieldMapping('field_tags', 'tags'); + $this->addFieldMapping('metatag_keywords', 'keywords'); + + // use filename and path seperately + $this->addFieldMapping('value', 'file_name'); + // the path to the source file + $this->addFieldMapping('source_dir', 'file_path') + ->callbacks(array($this, 'fixUri')); + // In case of Taxonomy, no need to pass the Source Migration. + // Only pass title to the taxonmoy + $this->addFieldMapping('File Category', 'filetags') + ->separator(','); + + // metatags https://www.drupal.org/node/2049565 + $this->addFieldMapping('metatag_title', 'title'); + $this->addFieldMapping('metatag_description', 'description'); + + // Setup common mappings + // Save to the default file scheme. + + $this->addFieldMapping('destination_dir') + ->defaultValue($arguments['destination_dir']); + + $this->addFieldMapping('file_replace') + ->defaultValue(MigrateFile::FILE_EXISTS_REUSE); + $this->addFieldmapping('preserve_files') + ->defaultValue(TRUE); + + if (isset($arguments['default_uid'])) { + $default_uid = $arguments['default_uid']; + } + else { + $default_uid = 1; + } + if (isset($user_migration)) { + $this->addFieldMapping('uid', 'uid') + ->sourceMigration($user_migration) + ->defaultValue($default_uid); + } + else { + $this->addFieldMapping('uid') + ->defaultValue($default_uid); + } + $this->addUnmigratedSources(array('media_type', 'filemime', 'filesize')); + } + + + + /** + * The base source query for this migration. + * + * @return QueryConditionInterface + */ + protected function query() { + $query = db_select(TYPO3_DATABASE_NAME . '.tx_dam', 'f'); + //->fields('f', array('file_path','file_name','file_mime_type','media_type','title','description','caption','keywords','instructions','uid')); + //->orderBy('tstamp', 'ASC'); + + //$query = db_select(TYPO3_DATABASE_NAME . '.tx_dam_mm_ref', 'u') + // ->fields('u', array('uid_local', 'uid_foreign','tablenames','ident','sorting_foreign','sorting' )) + $query->condition('f.deleted', '0'); + + $count_query = $query->countQuery(); + $query->leftJoin( TYPO3_DATABASE_NAME . '.tx_dam_mm_cat', 'catmm', 'catmm.uid_local = f.uid'); + $query->leftJoin( TYPO3_DATABASE_NAME . '.tx_dam_cat', 'filecat', 'filecat.uid = catmm.uid_foreign'); + $query->groupBy('f.uid'); + + $query->addExpression('GROUP_CONCAT(filecat.title)', 'filetags'); + + $query->addExpression('GROUP_CONCAT(catmm.uid_foreign)', 'cat_list'); + + return $query; + } + public function prepareRow($row) { + + if (parent::prepareRow($row) === FALSE) { + return FALSE; + } + + // if the source file doesn exist in the directory then skip it + // add source + $row->file_path = TYPO3_FILE_SRC . '/' . $row->file_path; + $file = $row->file_path . $row->file_name; + if (!file_exists($file)) { + + watchdog('typo3_migrate', 'file %file doesnt exist, skipping' , array('%file' => $file), WATCHDOG_NOTICE); + return FALSE; + } + + + + // Remove the leading forward slash. + //$row->destination_file = substr($row->sourceid, 1); + } + + protected function fixUri($uri) { + $result = TYPO3_FILE_SRC . '/' . $uri; + watchdog('typo3_migrate', 'source url %url fixed' , array('%url' => $result), WATCHDOG_NOTICE); + return $result; + } + +} diff --git a/typo3_files/images.inc b/typo3_files/images.inc new file mode 100644 index 0000000..f99eaca --- /dev/null +++ b/typo3_files/images.inc @@ -0,0 +1,34 @@ +condition('file_type', $allowed, 'IN'); + + return $query; + + } +} diff --git a/typo3_files/typo3_files.info b/typo3_files/typo3_files.info new file mode 100644 index 0000000..3a5bed4 --- /dev/null +++ b/typo3_files/typo3_files.info @@ -0,0 +1,17 @@ +name = "Typo3 Files Migrate" +description = "Typo3 Files to drupal migration." +package = "Migration" +core = 7.x +project = typo3_files_migrate + +dependencies[] = taxonomy +dependencies[] = file +dependencies[] = media +dependencies[] = typo3_migrate +dependencies[] = migrate_extras + +files[] = files.inc +files[] = images.inc +files[] = videos.inc +files[] = audio.inc +files[] = documents.inc diff --git a/typo3_files/typo3_files.module b/typo3_files/typo3_files.module new file mode 100644 index 0000000..5bbbf18 --- /dev/null +++ b/typo3_files/typo3_files.module @@ -0,0 +1,48 @@ + 2, + // Give the group a human-readable title + 'groups' => array( + 'typo3' => array( + 'title' => t('typo3'), + ), + ), + + // Migration classes. + 'migrations' => array( + 'typo3_files_cat' => array( + 'class_name' => 'Typo3FileCategoryMigration', + 'group_name' => 'typo3', + ), + + 'typo3_files_image' => array( + 'class_name' => 'Typo3ImageFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILE_SRC, + ), + 'typo3_files_document' => array( + 'class_name' => 'Typo3DocumentFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILE_SRC, + ), + 'typo3_files_video' => array( + 'class_name' => 'Typo3VideoFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILE_SRC, + ), + 'typo3_files_audio' => array( + 'class_name' => 'Typo3AudioFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILE_SRC, + ), + ), + ); + return $api; +} diff --git a/typo3_files/videos.inc b/typo3_files/videos.inc new file mode 100644 index 0000000..05816b9 --- /dev/null +++ b/typo3_files/videos.inc @@ -0,0 +1,41 @@ +bundle = $arguments['source_type']; + $this->allowed = $arguments['allowed']; + } + +// extending the protected query function in file.in + protected function query() { + $allowed = array('avi','mpeg','mp4','flv'); + $query = parent::query() + ->condition('file_type', $allowed, 'IN'); + + + return $query; + + } + +} diff --git a/typo3_migrate.info b/typo3_migrate.info index f3e00db..b135c36 100644 --- a/typo3_migrate.info +++ b/typo3_migrate.info @@ -2,7 +2,8 @@ name = "Typo3 Migrate" description = "Typo3 to drupal migration." package = "Migration" core = 7.x - +php = 5.2 dependencies[] = migrate //dependencies[] = migrate_extras +dependencies[] = date files[] = base.inc diff --git a/typo3_migrate.module b/typo3_migrate.module index 96bc1dc..798b59c 100644 --- a/typo3_migrate.module +++ b/typo3_migrate.module @@ -1,6 +1,6 @@ 'textfield', '#title' => t('Database Name'), '#description' => t('The Typo3 db must be accessible by the drupal db user and must reside on the same db server.'), - '#default_value' => variable_get('typo3_database', 'typo3_DB'), + '#default_value' => TYPO3_DATABASE_NAME, '#required' => TRUE ); + $form['source_directory'] = array( + '#type' => 'textfield', + '#size' => 60, + '#title' => t('File prefix'), + '#default_value' => TYPO3_FILES_SRC, + ); return system_settings_form($form); } - -- 1.9.1