From b062b2575df458c0efeb4e2a0fe5ce70131c98d9 Mon Sep 17 00:00:00 2001 From: Rosenstrauch Date: Sat, 22 Nov 2014 06:22:25 +0100 Subject: [PATCH] Issue #2377151 add typo3_files import --- base.inc | 10 +- typo3_files/documents.inc | 28 +++++ typo3_files/files.inc | 243 +++++++++++++++++++++++++++++++++++++++++ typo3_files/images.inc | 29 +++++ typo3_files/typo3_files.info | 16 +++ typo3_files/typo3_files.module | 43 ++++++++ typo3_files/videos.inc | 28 +++++ typo3_migrate.info | 9 +- typo3_migrate.module | 17 ++- typo3_news/tt_news.inc | 8 +- typo3_news/typo3_news.module | 17 +++ typo3_pages/pages.inc | 4 +- typo3_pages/typo3_pages.module | 13 +++ typo3_users/typo3_users.module | 17 +++ typo3_users/users.inc | 17 +-- 15 files changed, 479 insertions(+), 20 deletions(-) create mode 100644 typo3_files/documents.inc create mode 100644 typo3_files/files.inc create mode 100644 typo3_files/images.inc create mode 100644 typo3_files/typo3_files.info create mode 100644 typo3_files/typo3_files.module create mode 100644 typo3_files/videos.inc diff --git a/base.inc b/base.inc index 9edb965..736d792 100644 --- a/base.inc +++ b/base.inc @@ -13,9 +13,9 @@ abstract class BaseMigration extends Migration { public $queryParams = array(); - public function __construct() { + public function __construct($arguments) { // Always call the parent constructor first for basic setup - parent::__construct(); + parent::__construct($arguments); $this->team = array( new MigrateTeamMember('Test', 'test@srijan.in', t('Product Owner')), ); @@ -46,7 +46,13 @@ abstract class BaseMigration extends Migration { // News settings $this->queryParams['newsCat']['hidden'] = array(0); $this->queryParams['newsCat']['hidden_operator'] = 'IN'; + + + // File settings + $this->queryParams['fileCat']['hidden'] = array(0); + $this->queryParams['fileCat']['hidden_operator'] = 'IN'; // Check if news folder setings have been set. By default all the news content is migrated. + $pids = variable_get('typo3_pids', '0'); if ($pids != '0') { $this->queryParams['news']['pid'] = explode(",", $pids); diff --git a/typo3_files/documents.inc b/typo3_files/documents.inc new file mode 100644 index 0000000..7c089a6 --- /dev/null +++ b/typo3_files/documents.inc @@ -0,0 +1,28 @@ +description = t('Migrate Files Category Taxonomy'); + + $this->map = new MigrateSQLMap($this->machineName, + array( + 'uid' => array('type' => 'int', + 'length' => 11, + 'not null' => TRUE, + 'description' => 'Category', + ) + ), + MigrateDestinationTerm::getKeySchema() + ); + + // initiate connection to the typo3 database and select the table... + $query = db_select(TYPO3_DATABASE_NAME . '.tx_dam_cat', 'cat') + ->fields('cat', array('uid', 'title', 'parent_category')); + + $query->condition('cat.deleted', '0'); + if ($this->queryParams['fileCat']['hidden'] != '') { + $query->condition('cat.hidden', $this->queryParams['fileCat']['hidden'], $this->queryParams['fileCat']['hidden_operator']); + } + + // Create a MigrateSource object, which manages retrieving the input data. + $this->source = new MigrateSourceSQL($query); + + // Set up our destination - terms in the Files Category vocabulary + $this->destination = new MigrateDestinationTerm('Files Category'); + + $this->addFieldMapping('name', 'title'); + $this->addFieldMapping('description', 'description'); + + $this->addFieldMapping('parent', 'parent_category') + ->sourceMigration('Typo3fileCategory') + ->defaultValue(0); + + $this->addFieldMapping('format') + ->issueGroup(t('DNM')); + $this->addFieldMapping('weight') + ->issueGroup(t('DNM')); + } +} +/** + * Base class for all file migrations - handles commonalities across all + * supported source Typo3 versions. + * + * In addition to the arguments supported by Typo3Migration, the following + * must be passed in the $arguments array: + * source_dir: Path to folder containing files to be migrated. + * + * The following optional arguments may be passed: + * user_migration - Machine name of a user migration, used to establish + * dependencies and a sourceMigration for the uid mapping. + * default_uid - Drupal7 7 (destination) uid of the user account to use as + * the default. + * bundle - File bundle to use as the target - defaults to 'file'. + * file_class - Override for the default MigrateFileUri file class. + * destination_dir - Destination directory for the files (defaults to public://). + */ +class Typo3FileMigration extends BaseMigration { + + protected $baseDir; + + public function __construct($arguments) { + parent::__construct($arguments); + if (!$this->newOnly) { + $this->highwaterField = array( + 'name' => 'timestamp', + 'alias' => 'f', + 'type' => 'int', + ); + } + $this->description = t('Import files.'); + $this->base_dir = TYPO3_FILE_SRC; + if (empty($arguments['allowed'])) { + $allowed = array('pdf','txt','odt'); + $arguments['allowed'] = $allowed; + } + +// the actual files are in tx_dam +// TODO: initiate databas connection in base.inc https://www.drupal.org/node/1006984 source object + $query = db_select(TYPO3_DATABASE_NAME . '.tx_dam', 'f') + ->fields('f', array('file_path', 'file_name','file_mime_type','media_type','title', 'description','caption','keywords','instructions', 'uid' )); + //->orderBy('tstamp', 'ASC'); + + //$query = db_select(TYPO3_DATABASE_NAME . '.tx_dam_mm_ref', 'u') + // ->fields('u', array('uid_local', 'uid_foreign','tablenames','ident','sorting_foreign','sorting' )) + + $query->condition('f.deleted', '0'); + + + + $query->condition('file_type', $arguments['allowed'], 'IN'); + //$query->leftJoin( TYPO3_DATABASE_NAME . '.tx_dam_mm_cat', 'catmm', 'catmm.uid_local = f.uid'); + // $query->leftJoin( TYPO3_DATABASE_NAME . '.tx_dam_cat', 'filecat', 'filecat.uid = catmm.uid_foreign'); + // $query->groupBy('f.uid'); + + // $query->addExpression('GROUP_CONCAT(filecat.title)', 'filetags'); + + // $query->addExpression('GROUP_CONCAT(catmm.uid_foreign)', 'cat_list'); + if (!empty($arguments['user_migration'])) { + $user_migration = $arguments['user_migration']; + $this->dependencies[] = $user_migration; + } + if (empty($arguments['bundle'])) { + $arguments['bundle'] = 'file'; + } + if (empty($arguments['file_class'])) { + $arguments['file_class'] = 'MigrateFileUri'; + } + if (empty($arguments['destination_dir'])) { + $arguments['destination_dir'] = 'public://'; + } + + // Allow derived classes to override this definition by setting it before + // calling their parent constructor + if (!isset($this->map)) { + $this->map = new MigrateSQLMap($this->machineName, + array( + 'uid' => array( + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'description' => 'Source file ID', + 'alias' => 'f', + ), + ), + MigrateDestinationFile::getKeySchema() + ); + } + // todo: make sure we are mapping correctly https://www.drupal.org/node/1007004 + $count_query = $query->countQuery(); + $this->source = new MigrateSourceSQL($query, array(), $count_query); + + $directories = array( + $this->baseDir, + ); + + + $this->destination = new MigrateDestinationFile($arguments['bundle'], + $arguments['file_class']); + + //Field mappings https://www.drupal.org/node/1133448 + // field mappings for metadata on the files + $this->addFieldMapping('field_file_image_title_text', 'title'); + + $this->addFieldMapping('field_body', 'description'); + $this->addFieldMapping('field_tags', 'tags'); + + // In case of Taxonomy, no need to pass the Source Migration. + // Only pass title to the taxonmoy + $this->addFieldMapping('File Category', 'filetags') + ->separator(','); + + // metatags https://www.drupal.org/node/2049565 + $this->addFieldMapping('metatag_title', 'title'); + $this->addFieldMapping('metatag_description', 'description'); + + // Setup common mappings + // Save to the default file scheme. + + $this->addFieldMapping('destination_dir') + ->defaultValue($arguments['destination_dir']); + + + + // the path to the source file + $this->addFieldMapping('source_dir') + ->callbacks(array($this, 'fixUri')) + ->defaultValue('file_path'); + + + // Set the value to the file name + $this->addFieldMapping('value', 'file_name'); + + + $this->addFieldMapping('file_replace') + ->defaultValue(MigrateFile::FILE_EXISTS_REUSE); + $this->addFieldmapping('preserve_files') + ->defaultValue(TRUE); + + if (isset($arguments['default_uid'])) { + $default_uid = $arguments['default_uid']; + } + else { + $default_uid = 1; + } + if (isset($user_migration)) { + $this->addFieldMapping('uid', 'uid') + ->sourceMigration($user_migration) + ->defaultValue($default_uid); + } + else { + $this->addFieldMapping('uid') + ->defaultValue($default_uid); + } + $this->addUnmigratedSources(array('file_name', 'filemime', 'filesize')); + } + public function prepareRow($row) { + + + + if (parent::prepareRow($row) === FALSE) { + return FALSE; + } + + + +// if the source file doesn exist in the directory then skip it +// add source +$row->file_path = TYPO3_FILE_SRC . '/' . $row->file_path; +$file = $row->file_path . $row->file_name; +if (!file_exists($file)) { + + watchdog('typo3_migrate', 'file %file doesnt exist, skipping' , array('%file' => $file), WATCHDOG_NOTICE); + return FALSE; +} +dpm($row); + + + // Remove the leading forward slash. + //$row->destination_file = substr($row->sourceid, 1); + } + protected function fixUri($uri) { + $result = TYPO3_FILE_SRC . '/' . $uri; + watchdog('typo3_migrate', 'source url %url fixed' , array('%url' => $result), WATCHDOG_NOTICE); + return $result; + } + +} diff --git a/typo3_files/images.inc b/typo3_files/images.inc new file mode 100644 index 0000000..562449a --- /dev/null +++ b/typo3_files/images.inc @@ -0,0 +1,29 @@ + 2, + // Give the group a human-readable title + 'groups' => array( + 'typo3' => array( + 'title' => t('typo3'), + ), + ), + + // Migration classes. + 'migrations' => array( + 'typo3_files_cat' => array( + 'class_name' => 'Typo3FileCategoryMigration', + 'group_name' => 'typo3', + ), + + 'typo3_files_image' => array( + 'class_name' => 'Typo3ImageFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILE_SRC, + ), + 'typo3_files_document' => array( + 'class_name' => 'Typo3DocumentFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILE_SRC, + ), + 'typo3_files_video' => array( + 'class_name' => 'Typo3VideoFileMigration', + 'group_name' => 'typo3', + 'source_dir' => TYPO3_FILE_SRC, + ), + ), + ); + return $api; +} diff --git a/typo3_files/videos.inc b/typo3_files/videos.inc new file mode 100644 index 0000000..35f06f9 --- /dev/null +++ b/typo3_files/videos.inc @@ -0,0 +1,28 @@ + 2, + // Groups. + 'groups' => array( + 'typo3' => array( + 'title' => t('typo3'), + ), + ), ); return $api; } @@ -57,9 +63,14 @@ function typo3_migrate_config($form_state) { '#type' => 'textfield', '#title' => t('Database Name'), '#description' => t('The Typo3 db must be accessible by the drupal db user and must reside on the same db server.'), - '#default_value' => variable_get('typo3_database', 'typo3_DB'), + '#default_value' => TYPO3_DATABASE_NAME, '#required' => TRUE ); + $form['source_directory'] = array( + '#type' => 'textfield', + '#size' => 60, + '#title' => t('File prefix'), + '#default_value' => TYPO3_FILES_SRC, + ); return system_settings_form($form); } - diff --git a/typo3_news/tt_news.inc b/typo3_news/tt_news.inc index 3a9c3fb..1ecc395 100644 --- a/typo3_news/tt_news.inc +++ b/typo3_news/tt_news.inc @@ -8,8 +8,8 @@ * Migrate all the tt news category in a drupal taxonomy(NewsCatgeory). */ class Typo3NewsCategoryMigration extends BaseMigration { - public function __construct() { - parent::__construct(); + public function __construct($arguments) { + parent::__construct($arguments); $this->description = t('Migrate News Category Taxonomy'); $this->map = new MigrateSQLMap($this->machineName, @@ -58,8 +58,8 @@ class Typo3NewsCategoryMigration extends BaseMigration { */ class Typo3NewsMigration extends BaseMigration { - public function __construct() { - parent::__construct(); + public function __construct($arguments) { + parent::__construct($arguments); $this->description = t('tt_news migration.'); // NewsCategory migration needs to run first. $this->dependencies = array('Typo3NewsCategory'); diff --git a/typo3_news/typo3_news.module b/typo3_news/typo3_news.module index 92c93cd..c818dd9 100644 --- a/typo3_news/typo3_news.module +++ b/typo3_news/typo3_news.module @@ -7,6 +7,23 @@ function typo3_news_migrate_api() { $api = array( 'api' => 2, + // Groups. + 'groups' => array( + 'typo3' => array( + 'title' => t('typo3'), + ), + ), + // Migration classes. + 'migrations' => array( + 'typo3_newss' => array( + 'class_name' => 'Typo3NewsMigration', + 'group_name' => 'typo3' + ), + 'typo3_news_category' => array( + 'class_name' => 'Typo3NewsCategoryMigration', + 'group_name' => 'typo3' + ), + ), ); return $api; } diff --git a/typo3_pages/pages.inc b/typo3_pages/pages.inc index 5e427ac..f6c2c1d 100644 --- a/typo3_pages/pages.inc +++ b/typo3_pages/pages.inc @@ -11,8 +11,8 @@ * */ class Typo3PagesMigration extends BaseMigration { - public function __construct() { - parent::__construct(); + public function __construct($arguments) { + parent::__construct($arguments); $this->description = t('Pages and tt_content migration.'); $this->map = new MigrateSQLMap($this->machineName, diff --git a/typo3_pages/typo3_pages.module b/typo3_pages/typo3_pages.module index a27fa05..e85a288 100644 --- a/typo3_pages/typo3_pages.module +++ b/typo3_pages/typo3_pages.module @@ -7,6 +7,19 @@ function typo3_pages_migrate_api() { $api = array( 'api' => 2, + // Groups. + 'groups' => array( + 'typo3' => array( + 'title' => t('typo3'), + ), + ), + // Migration classes. + 'migrations' => array( + 'typo3_pages' => array( + 'class_name' => 'Typo3PagesMigration', + 'group_name' => 'typo3' + ), + ), ); return $api; } \ No newline at end of file diff --git a/typo3_users/typo3_users.module b/typo3_users/typo3_users.module index 7770a33..93dd65b 100644 --- a/typo3_users/typo3_users.module +++ b/typo3_users/typo3_users.module @@ -7,6 +7,23 @@ function typo3_users_migrate_api() { $api = array( 'api' => 2, + // Groups. + 'groups' => array( + 'typo3' => array( + 'title' => t('typo3'), + ), + ), + // Migration classes. + 'migrations' => array( + 'typo3_backend_users' => array( + 'class_name' => 'Typo3BeUserMigration', + 'group_name' => 'typo3' + ), + 'typo3_frontend_users' => array( + 'class_name' => 'Typo3FeUserMigration', + 'group_name' => 'typo3' + ), + ), ); return $api; } \ No newline at end of file diff --git a/typo3_users/users.inc b/typo3_users/users.inc index d1b3621..53306dd 100644 --- a/typo3_users/users.inc +++ b/typo3_users/users.inc @@ -8,9 +8,11 @@ * * Migrate Backend Users */ + class Typo3BeUserMigration extends BaseMigration { - public function __construct() { - parent::__construct(); + public function __construct($arguments) { + parent::__construct($arguments); + $this->description = t('Backend users of Typo3 site.'); $this->map = new MigrateSQLMap($this->machineName, array('uid' => array( @@ -27,7 +29,7 @@ class Typo3BeUserMigration extends BaseMigration { ->orderBy($this->queryParams['beuser']['order_field'], $this->queryParams['beuser']['order_field_operator']); $query->condition('u.deleted', '0'); $query->condition('u.disable', $this->queryParams['beuser']['disable'], $this->queryParams['beuser']['disable_operator']); - + $this->source = new MigrateSourceSQL($query); $this->destination = new MigrateDestinationUser(array('md5_passwords' => TRUE)); @@ -83,8 +85,9 @@ class Typo3BeUserMigration extends BaseMigration { * Front end User Migration */ class Typo3FeUserMigration extends BaseMigration { - public function __construct() { - parent::__construct(); + public function __construct($arguments) { + parent::__construct($arguments); + $this->description = t('FrondEnd users of the Typo3 site.'); $this->map = new MigrateSQLMap($this->machineName, array('uid' => array( @@ -102,8 +105,8 @@ class Typo3FeUserMigration extends BaseMigration { $query->condition('u.deleted', '0'); $query->condition('u.disable', $this->queryParams['feuser']['disable'], $this->queryParams['feuser']['disable_operator']); - - $this->source = new MigrateSourceSQL($query); + $count_query = $query->countQuery(); + $this->source = new MigrateSourceSQL($query, array(), $count_query, array('map_joinable' => FALSE)); $this->destination = new MigrateDestinationUser(); // Mapped fields -- 1.9.1