diff --git a/core/modules/locale/lib/Drupal/locale/Form/LocaleSettingsForm.php b/core/modules/locale/lib/Drupal/locale/Form/LocaleSettingsForm.php index 4236bf2..5669d8f 100644 --- a/core/modules/locale/lib/Drupal/locale/Form/LocaleSettingsForm.php +++ b/core/modules/locale/lib/Drupal/locale/Form/LocaleSettingsForm.php @@ -32,8 +32,8 @@ public function buildForm(array $form, array &$form_state) { '#default_value' => $config->get('translation.update_interval_days'), '#options' => array( '0' => t('Never (manually)'), - '1' => t('Daily'), '7' => t('Weekly'), + '30' => t('Monthly'), ), '#description' => t('Select how frequently you want to check for new interface translations for your currently installed modules and themes. Check updates now.', array('@url' => url('admin/reports/translations/check'))), ); diff --git a/core/modules/locale/lib/Drupal/locale/Tests/LocaleUpdateInterfaceTest.php b/core/modules/locale/lib/Drupal/locale/Tests/LocaleUpdateInterfaceTest.php index 836ee97..d5f9ed2 100644 --- a/core/modules/locale/lib/Drupal/locale/Tests/LocaleUpdateInterfaceTest.php +++ b/core/modules/locale/lib/Drupal/locale/Tests/LocaleUpdateInterfaceTest.php @@ -36,6 +36,19 @@ function setUp() { } /** + * Adds a language. + * + * @param $langcode + * The language code of the language to add. + */ + function addLanguage($langcode) { + $edit = array('predefined_langcode' => $langcode); + $this->drupalPost('admin/config/regional/language/add', $edit, t('Add language')); + drupal_static_reset('language_list'); + $this->assertTrue(language_load($langcode), t('Language %langcode added.', array('%langcode' => $langcode))); + } + + /** * Tests the user interfaces of the interface translation update system. * * Testing the Available updates summary on the side wide status page and the @@ -51,16 +64,14 @@ function testInterface() { $this->assertRaw(t('No translatable languages available. Add a language first.', array('@add_language' => url('admin/config/regional/language'))), 'Language message'); // Add German language. - $edit = array( - 'predefined_langcode' => 'de', - ); - $this->drupalPost('admin/config/regional/language/add', $edit, t('Add language')); + $this->addLanguage('de'); // Drupal core is probably in 8.x, but tests may also be executed with // stable releases. As this is an uncontrolled factor in the test, we will - // ignore Drupal core here and continue with the prepared modules. - $status = state()->get('locale.translation_status'); - unset($status['drupal']); + // mark Drupal core as being translated. + $status = locale_translation_get_status(); + $status['drupal']['de']->type = LOCALE_TRANSLATION_CURRENT; + $status['drupal']['de']->timestamp = REQUEST_TIME; state()->set('locale.translation_status', $status); // One language added, all translations up to date. @@ -71,7 +82,7 @@ function testInterface() { $this->assertText(t('All translations up to date.'), 'Translations up to date'); // Set locale_test_translate module to have a local translation available. - $status = state()->get('locale.translation_status'); + $status = locale_translation_get_status(); $status['locale_test_translate']['de']->type = 'local'; state()->set('locale.translation_status', $status); @@ -84,9 +95,9 @@ function testInterface() { // Set locale_test_translate module to have a dev release and no // translation found. - $status = state()->get('locale.translation_status'); + $status = locale_translation_get_status(); $status['locale_test_translate']['de']->version = '1.3-dev'; - unset($status['locale_test_translate']['de']->type); + $status['locale_test_translate']['de']->type = ''; state()->set('locale.translation_status', $status); // Check if no updates were found. diff --git a/core/modules/locale/lib/Drupal/locale/Tests/LocaleUpdateTest.php b/core/modules/locale/lib/Drupal/locale/Tests/LocaleUpdateTest.php index e5dccbd..4a5fcd5 100644 --- a/core/modules/locale/lib/Drupal/locale/Tests/LocaleUpdateTest.php +++ b/core/modules/locale/lib/Drupal/locale/Tests/LocaleUpdateTest.php @@ -66,7 +66,7 @@ function setUp() { // We use German as test language. This language must match the translation // file that come with the locale_test module (test.de.po) and can therefore // not be chosen randomly. - $this->drupalPost('admin/config/regional/language/add', array('predefined_langcode' => 'de'), t('Add language')); + $this->addLanguage('de'); // Setup timestamps to identify old and new translation sources. $this->timestamp_old = REQUEST_TIME - 300; @@ -378,7 +378,7 @@ function testUpdateCheckStatus() { // Get status of translation sources at local file system. $this->drupalGet('admin/reports/translations/check'); - $result = state()->get('locale.translation_status'); + $result = locale_translation_get_status(); $this->assertEqual($result['contrib_module_one']['de']->type, LOCALE_TRANSLATION_LOCAL, 'Translation of contrib_module_one found'); $this->assertEqual($result['contrib_module_one']['de']->timestamp, $this->timestamp_old, 'Translation timestamp found'); $this->assertEqual($result['contrib_module_two']['de']->type, LOCALE_TRANSLATION_LOCAL, 'Translation of contrib_module_two found'); @@ -394,8 +394,8 @@ function testUpdateCheckStatus() { // Get status of translation sources at both local and remote locations. $this->drupalGet('admin/reports/translations/check'); - $result = state()->get('locale.translation_status'); - $this->assertEqual($result['contrib_module_one']['de']->type, 'remote', 'Translation of contrib_module_one found'); + $result = locale_translation_get_status(); + $this->assertEqual($result['contrib_module_one']['de']->type, LOCALE_TRANSLATION_REMOTE, 'Translation of contrib_module_one found'); $this->assertEqual($result['contrib_module_one']['de']->timestamp, $this->timestamp_new, 'Translation timestamp found'); $this->assertEqual($result['contrib_module_two']['de']->type, LOCALE_TRANSLATION_LOCAL, 'Translation of contrib_module_two found'); $this->assertEqual($result['contrib_module_two']['de']->timestamp, $this->timestamp_new, 'Translation timestamp found'); @@ -411,7 +411,6 @@ function testUpdateCheckStatus() { * Test conditions: * - Source: remote and local files * - Import overwrite: all existing translations - * - Translation directory: available */ function testUpdateImportSourceRemote() { $config = config('locale.settings'); @@ -434,7 +433,6 @@ function testUpdateImportSourceRemote() { // Check the status on the Available translation status page. $this->assertRaw('', 'German language found'); $this->assertText('Updates for: Contributed module one, Contributed module two, Custom module one, Locale test', 'Updates found'); - $this->assertText('Updates for: Contributed module one, Contributed module two, Custom module one, Locale test', 'Updates found'); $this->assertText('Contributed module one (' . format_date($this->timestamp_now, 'html_date') . ')', 'Updates for Contrib module one'); $this->assertText('Contributed module two (' . format_date($this->timestamp_new, 'html_date') . ')', 'Updates for Contrib module two'); @@ -442,7 +440,7 @@ function testUpdateImportSourceRemote() { $this->drupalPost('admin/reports/translations', array(), t('Update translations')); // Check if the translation has been updated, using the status cache. - $status = state()->get('locale.translation_status'); + $status = locale_translation_get_status(); $this->assertEqual($status['contrib_module_one']['de']->type, LOCALE_TRANSLATION_CURRENT, 'Translation of contrib_module_one found'); $this->assertEqual($status['contrib_module_two']['de']->type, LOCALE_TRANSLATION_CURRENT, 'Translation of contrib_module_two found'); $this->assertEqual($status['contrib_module_three']['de']->type, LOCALE_TRANSLATION_CURRENT, 'Translation of contrib_module_three found'); @@ -475,7 +473,6 @@ function testUpdateImportSourceRemote() { * Test conditions: * - Source: local files only * - Import overwrite: all existing translations - * - Translation directory: available */ function testUpdateImportSourceLocal() { $config = config('locale.settings'); @@ -497,7 +494,7 @@ function testUpdateImportSourceLocal() { $this->drupalPost('admin/reports/translations', array(), t('Update translations')); // Check if the translation has been updated, using the status cache. - $status = state()->get('locale.translation_status'); + $status = locale_translation_get_status(); $this->assertEqual($status['contrib_module_one']['de']->type, LOCALE_TRANSLATION_CURRENT, 'Translation of contrib_module_one found'); $this->assertEqual($status['contrib_module_two']['de']->type, LOCALE_TRANSLATION_CURRENT, 'Translation of contrib_module_two found'); $this->assertEqual($status['contrib_module_three']['de']->type, LOCALE_TRANSLATION_CURRENT, 'Translation of contrib_module_three found'); @@ -525,68 +522,12 @@ function testUpdateImportSourceLocal() { } /** - * Tests translation import without a translations directory. - * - * Test conditions: - * - Source: remote and local files - * - Import overwrite: all existing translations - * - Translation directory: not available - */ - function testUpdateImportWithoutDirectory() { - $config = config('locale.settings'); - - // Build the test environment. - $this->setTranslationFiles(); - $this-> setCurrentTranslations(); - $config->set('translation.default_filename', '%project-%version.%language._po'); - - // Set the update conditions for this test. - $this->setTranslationsDirectory(''); - $edit = array( - 'use_source' => LOCALE_TRANSLATION_USE_SOURCE_REMOTE_AND_LOCAL, - 'overwrite' => LOCALE_TRANSLATION_OVERWRITE_ALL, - ); - $this->drupalPost('admin/config/regional/translate/settings', $edit, t('Save configuration')); - - // Execute the translation update. - $this->drupalGet('admin/reports/translations/check'); - $this->drupalPost('admin/reports/translations', array(), t('Update translations')); - - // Check if the translation has been updated, using the status cache. - $status = state()->get('locale.translation_status'); - $this->assertEqual($status['contrib_module_one']['de']->type, LOCALE_TRANSLATION_CURRENT, 'Translation of contrib_module_one found'); - $this->assertEqual($status['contrib_module_two']['de']->type, LOCALE_TRANSLATION_CURRENT, 'Translation of contrib_module_two found'); - $this->assertEqual($status['contrib_module_three']['de']->type, LOCALE_TRANSLATION_CURRENT, 'Translation of contrib_module_three found'); - - // Check the new translation status. - // The static cache needs to be flushed first to get the most recent data - // from the database. The function was called earlier during this test. - drupal_static_reset('locale_translation_get_file_history'); - $history = locale_translation_get_file_history(); - $this->assertTrue($history['contrib_module_one']['de']->timestamp >= $this->timestamp_now, 'Translation of contrib_module_one is imported'); - $this->assertTrue($history['contrib_module_one']['de']->last_checked >= $this->timestamp_now, 'Translation of contrib_module_one is updated'); - $this->assertEqual($history['contrib_module_two']['de']->timestamp, $this->timestamp_medium, 'Translation of contrib_module_two is imported'); - $this->assertEqual($history['contrib_module_two']['de']->last_checked, $this->timestamp_medium, 'Translation of contrib_module_two is updated'); - $this->assertEqual($history['contrib_module_three']['de']->timestamp, $this->timestamp_medium, 'Translation of contrib_module_three is not imported'); - $this->assertEqual($history['contrib_module_three']['de']->last_checked, $this->timestamp_medium, 'Translation of contrib_module_three is not updated'); - - // Check whether existing translations have (not) been overwritten. - $this->assertEqual(t('January', array(), array('langcode' => 'de')), 'Januar_1', 'Translation of January'); - $this->assertEqual(t('February', array(), array('langcode' => 'de')), 'Februar_1', 'Translation of February'); - $this->assertEqual(t('March', array(), array('langcode' => 'de')), 'Marz_1', 'Translation of March'); - $this->assertEqual(t('May', array(), array('langcode' => 'de')), 'Mai_customized', 'Translation of May'); - $this->assertEqual(t('June', array(), array('langcode' => 'de')), 'Juni', 'Translation of June'); - $this->assertEqual(t('Monday', array(), array('langcode' => 'de')), 'Montag', 'Translation of Monday'); - } - - /** * Tests translation import with a translations directory and only overwrite * non-customized translations. * * Test conditions: * - Source: remote and local files * - Import overwrite: only overwrite non-customized translations - * - Translation directory: available */ function testUpdateImportModeNonCustomized() { $config = config('locale.settings'); @@ -624,7 +565,6 @@ function testUpdateImportModeNonCustomized() { * Test conditions: * - Source: remote and local files * - Import overwrite: don't overwrite any existing translation - * - Translation directory: available */ function testUpdateImportModeNone() { $config = config('locale.settings'); diff --git a/core/modules/locale/locale.batch.inc b/core/modules/locale/locale.batch.inc index 3709406..4c17bec 100644 --- a/core/modules/locale/locale.batch.inc +++ b/core/modules/locale/locale.batch.inc @@ -29,10 +29,12 @@ * parameter of the context. * * @see locale_translation_batch_status_fetch_local() - * @see locale_translation_batch_status_compare() */ -function locale_translation_batch_status_fetch_remote($source, &$context) { +function locale_translation_batch_status_fetch_remote($project, $langcode, &$context) { $t = get_t(); + $source = locale_translation_get_status(array($project), array($langcode)); + $source = $source[$project][$langcode]; + // Check the translation file at the remote server and update the source // data with the remote status. if (isset($source->files[LOCALE_TRANSLATION_REMOTE])) { @@ -46,8 +48,9 @@ function locale_translation_batch_status_fetch_remote($source, &$context) { if (isset($result['last_modified'])) { $remote_file->uri = isset($result['location']) ? $result['location'] : $remote_file->uri; $remote_file->timestamp = $result['last_modified']; - $source->files[LOCALE_TRANSLATION_REMOTE] = $remote_file; + locale_translation_status_save($source->name, $source->langcode, LOCALE_TRANSLATION_REMOTE, $remote_file); } + // @todo What to do with 404s ($result == TRUE)? Do we need to record the action to prevent re-checking within the TTL (1day, 1week)? // Record success. $context['results']['files'][$source->name] = $source->name; } @@ -56,7 +59,6 @@ function locale_translation_batch_status_fetch_remote($source, &$context) { // reporting at the end of the batch. $context['results']['failed_files'][] = $source->name; } - $context['results']['sources'][$source->name][$source->langcode] = $source; $context['message'] = $t('Checked translation for %project.', array('%project' => $source->project)); } } @@ -67,101 +69,31 @@ function locale_translation_batch_status_fetch_remote($source, &$context) { * Checks the presence and creation time of po files in the local file system. * The file path and the timestamp are stored. * - * @param array $sources - * Array of translation source objects of projects for which to check the - * state of local po files. + * @param array $source + * A translation source object of the project for which to check the state of + * a remote po file. * @param array $context * The batch context array. The collected state is stored in the 'results' * parameter of the context. * * @see locale_translation_batch_status_fetch_remote() - * @see locale_translation_batch_status_compare() */ -function locale_translation_batch_status_fetch_local($sources, &$context) { +function locale_translation_batch_status_fetch_local($project, $langcode, &$context) { $t = get_t(); + $source = locale_translation_get_status(array($project), array($langcode)); + $source = $source[$project][$langcode]; + // Get the status of local translation files and store the result data in the // batch results for later processing. - foreach ($sources as $source) { - if (isset($source->files[LOCALE_TRANSLATION_LOCAL])) { - locale_translation_source_check_file($source); - - // If remote data was collected before, we merge it into the newly - // collected result. - if (isset($context['results']['sources'][$source->name][$source->langcode])) { - $source->files[LOCALE_TRANSLATION_REMOTE] = $context['results']['sources'][$source->name][$source->langcode]->files[LOCALE_TRANSLATION_REMOTE]; - } + if (isset($source->files[LOCALE_TRANSLATION_LOCAL])) { + if ($file = locale_translation_source_check_file($source)) { + locale_translation_status_save($source->name, $source->langcode, LOCALE_TRANSLATION_LOCAL, $file); // Record success and store the updated source data. $context['results']['files'][$source->name] = $source->name; - $context['results']['sources'][$source->name][$source->langcode] = $source; - } - } - $context['message'] = $t('Checked all translations.'); -} - -/** - * Batch operation callback: Compare states and store the result. - * - * In the preceding batch processes data of remote and local translation sources - * is collected. Here we compare the collected results and update the source - * object with the data of the most recent translation file. The end result is - * stored in the 'locale.translation_status' state variable. Other - * processes can collect this data after the batch process is completed. - * - * @param array $context - * The batch context array. The 'results' element contains a structured array - * of project data with languages, local and remote source data. - * - * @see locale_translation_batch_status_fetch_remote() - * @see locale_translation_batch_status_fetch_local() - */ -function locale_translation_batch_status_compare(&$context) { - $t = get_t(); - $history = locale_translation_get_file_history(); - $results = array(); - - if (isset($context['results']['sources'])) { - foreach ($context['results']['sources'] as $project => $langcodes) { - foreach ($langcodes as $langcode => $source) { - $local = isset($source->files[LOCALE_TRANSLATION_LOCAL]) ? $source->files[LOCALE_TRANSLATION_LOCAL] : NULL; - $remote = isset($source->files[LOCALE_TRANSLATION_REMOTE]) ? $source->files[LOCALE_TRANSLATION_REMOTE] : NULL; - - // The available translation files are compared and data of the most - // recent file is used to update the source object. - $file = _locale_translation_source_compare($local, $remote) == LOCALE_TRANSLATION_SOURCE_COMPARE_LT ? $remote : $local; - if (isset($file->timestamp)) { - $source->type = $file->type; - $source->timestamp = $file->timestamp; - } - - // Compare the available translation with the current translations - // status. If the project/language was translated before and it is more - // recent than the most recent translation, the translation is up to - // date. Which is marked in the source object with type "current". - if (isset($history[$source->project][$source->langcode])) { - $current = $history[$source->project][$source->langcode]; - // Add the current translation to the source object to save it in - // the status cache. - $source->files[LOCALE_TRANSLATION_CURRENT] = $current; - - if (isset($source->type)) { - $available = $source->files[$source->type]; - $result = _locale_translation_source_compare($current, $available) == LOCALE_TRANSLATION_SOURCE_COMPARE_LT ? $available : $current; - $source->type = $result->type; - $source->timestamp = $result->timestamp; - } - else { - $source->type = $current->type; - $source->timestamp = $current->timestamp; - } - } - - $results[$project][$langcode] = $source; - } } - $context['message'] = $t('Updated translation status.'); } - locale_translation_status_save($results); + $context['message'] = $t('Checked translation for %project.', array('%project' => $source->project)); } /** @@ -186,7 +118,7 @@ function locale_translation_batch_status_finished($success, $results) { } if (isset($results['files'])) { drupal_set_message(format_plural( - count($results['sources']), + count($results['files']), 'Checked available interface translation updates for one project.', 'Checked available interface translation updates for @count projects.' )); @@ -194,6 +126,7 @@ function locale_translation_batch_status_finished($success, $results) { if (!isset($results['failed_files']) && !isset($results['files'])) { drupal_set_message(t('Nothing to check.')); } + state()->set('locale.translation_last_checked', REQUEST_TIME); } else { drupal_set_message($t('An error occurred trying to check available interface translation updates.'), 'error'); @@ -212,12 +145,8 @@ function locale_translation_batch_status_finished($success, $results) { * * @see locale_translation_batch_fetch_download() * @see locale_translation_batch_fetch_import() - * @see locale_translation_batch_fetch_update_status() - * @see locale_translation_batch_status_compare() */ function locale_translation_batch_fetch_sources($projects, $langcodes, &$context) { - $context['results']['input'] = locale_translation_load_sources($projects, $langcodes); - // If this batch operation is preceded by the status check operations, the // results of those operation are stored in the context. We remove them here // to keep the result records clean. @@ -242,23 +171,20 @@ function locale_translation_batch_fetch_sources($projects, $langcodes, &$context * * @see locale_translation_batch_fetch_sources() * @see locale_translation_batch_fetch_import() - * @see locale_translation_batch_fetch_update_status() - * @see locale_translation_batch_status_compare() */ function locale_translation_batch_fetch_download($project, $langcode, &$context) { - $sources = $context['results']['input']; - if (isset($sources[$project . ':' . $langcode])) { - $source = $sources[$project . ':' . $langcode]; + $sources = locale_translation_get_status(array($project), array($langcode)); + if (isset($sources[$project][$langcode])) { + $source = $sources[$project][$langcode]; if (isset($source->type) && $source->type == LOCALE_TRANSLATION_REMOTE) { $t = get_t(); - if ($file = locale_translation_download_source($source->files[LOCALE_TRANSLATION_REMOTE])) { + if ($file = locale_translation_download_source($source->files[LOCALE_TRANSLATION_REMOTE], 'translations://')) { $context['message'] = $t('Downloaded translation for %project.', array('%project' => $source->project)); - $source->files[LOCALE_TRANSLATION_DOWNLOADED] = $file; + locale_translation_status_save($source->name, $source->langcode, LOCALE_TRANSLATION_LOCAL, $file); } else { $context['results']['failed_files'][] = $source->files[LOCALE_TRANSLATION_REMOTE]; } - $context['results']['sources'][$project][$langcode] = $source; } } } @@ -285,31 +211,15 @@ function locale_translation_batch_fetch_download($project, $langcode, &$context) * @see locale_translate_batch_import_files() * @see locale_translation_batch_fetch_sources() * @see locale_translation_batch_fetch_download() - * @see locale_translation_batch_fetch_update_status() - * @see locale_translation_batch_status_compare() */ function locale_translation_batch_fetch_import($project, $langcode, $options, &$context) { - $sources = $context['results']['input']; - if (isset($sources[$project . ':' . $langcode])) { - $source = $sources[$project . ':' . $langcode]; + $sources = locale_translation_get_status(array($project), array($langcode)); + if (isset($sources[$project][$langcode])) { + $source = $sources[$project][$langcode]; if (isset($source->type)) { if ($source->type == LOCALE_TRANSLATION_REMOTE || $source->type == LOCALE_TRANSLATION_LOCAL) { - $t = get_t(); - // If we are working on a remote file we will import the downloaded - // file. If the file was local just mark the result as such. - if ($source->type == LOCALE_TRANSLATION_REMOTE) { - if (isset($context['results']['sources'][$source->project][$source->langcode]->files[LOCALE_TRANSLATION_DOWNLOADED])) { - $import_type = LOCALE_TRANSLATION_DOWNLOADED; - $source_result = $context['results']['sources'][$source->project][$source->langcode]; - } - } - else { - $import_type = LOCALE_TRANSLATION_LOCAL; - $source_result = $source; - } - - $file = $source_result->files[$import_type]; + $file = $source->files[LOCALE_TRANSLATION_LOCAL]; module_load_include('bulk.inc', 'locale'); $options += array( 'message' => $t('Importing translation for %project.', array('%project' => $source->project)), @@ -324,101 +234,17 @@ function locale_translation_batch_fetch_import($project, $langcode, $options, &$ if (isset($context['results']['files'][$file->uri])) { $context['message'] = $t('Imported translation for %project.', array('%project' => $source->project)); - // Keep the data of imported source. In the following batch - // operation it will be saved in the {locale_file} table. - $source_result->files[LOCALE_TRANSLATION_IMPORTED] = $source_result->files[$source->type]; - - // Downloaded files are stored in the temporary files directory. If - // files should be kept locally, they will be moved to the local - // translations after successfull import. Otherwise the temporary - // file is deleted after being imported. - if ($import_type == LOCALE_TRANSLATION_DOWNLOADED && config('locale.settings')->get('translation.path') && isset($source_result->files[LOCALE_TRANSLATION_LOCAL])) { - if (file_unmanaged_move($file->uri, $source_result->files[LOCALE_TRANSLATION_LOCAL]->uri, FILE_EXISTS_REPLACE)) { - // The downloaded file is now moved to the local file location. - // From this point forward we can treat it as if we imported a - // local file. - $import_type = LOCALE_TRANSLATION_LOCAL; - } - } - // The downloaded file is imported but will not be stored locally. - // Store the timestamp and delete the file. - if ($import_type == LOCALE_TRANSLATION_DOWNLOADED) { - $timestamp = filemtime($source_result->files[$import_type]->uri); - $source_result->files[LOCALE_TRANSLATION_IMPORTED]->timestamp = $timestamp; - $source_result->files[LOCALE_TRANSLATION_IMPORTED]->last_checked = REQUEST_TIME; - file_unmanaged_delete($file->uri); - } - // If the translation file is stored in the local directory. The - // timestamp of the file is stored. - if ($import_type == LOCALE_TRANSLATION_LOCAL) { - $timestamp = filemtime($source_result->files[$import_type]->uri); - $source_result->files[LOCALE_TRANSLATION_LOCAL]->timestamp = $timestamp; - $source_result->files[LOCALE_TRANSLATION_IMPORTED]->timestamp = $timestamp; - $source_result->files[LOCALE_TRANSLATION_IMPORTED]->last_checked = REQUEST_TIME; - - } - } - else { - // File import failed. We can delete the temporary file. - if ($import_type == LOCALE_TRANSLATION_DOWNLOADED) { - file_unmanaged_delete($file->uri); - } + // Save the data of imported source into the {locale_file} table and + // update the current translation status. + locale_translation_status_save($project, $langcode, LOCALE_TRANSLATION_CURRENT, $source->files[LOCALE_TRANSLATION_LOCAL]); } } - $context['results']['sources'][$source->project][$source->langcode] = $source_result; } } } } /** - * Batch process: Update the download history table. - * - * This batch process updates the {local_file} table with the data of imported - * gettext files. Import data is taken from $context['results']['sources']. - * - * @param $context - * Batch context array. - * - * @see locale_translation_batch_fetch_sources() - * @see locale_translation_batch_fetch_download() - * @see locale_translation_batch_fetch_import() - * @see locale_translation_batch_status_compare() - */ -function locale_translation_batch_fetch_update_status(&$context) { - $t = get_t(); - $results = array(); - - if (isset($context['results']['sources'])) { - foreach ($context['results']['sources'] as $project => $langcodes) { - foreach ($langcodes as $langcode => $source) { - - // Store the state of the imported translations in {locale_file} table. - // During the batch execution the data of the imported files is - // temporary stored in $context['results']['sources']. Now it will be - // stored in the database. Afterwards the temporary import and download - // data can be deleted. - if (isset($source->files[LOCALE_TRANSLATION_IMPORTED])) { - $file = $source->files[LOCALE_TRANSLATION_IMPORTED]; - locale_translation_update_file_history($file); - unset($source->files[LOCALE_TRANSLATION_IMPORTED]); - } - unset($source->files[LOCALE_TRANSLATION_DOWNLOADED]); - - // The source data is now up to date. Data of local and/or remote source - // file is up to date including an updated time stamp. In a next batch - // operation this can be used to update the translation status. - $context['results']['sources'][$project][$langcode] = $source; - } - } - $context['message'] = $t('Updated translations.'); - - // The file history has changed, flush the static cache now. - drupal_static_reset('locale_translation_get_file_history'); - } -} - -/** * Batch finished callback: Set result message. * * @param boolean $success @@ -428,6 +254,9 @@ function locale_translation_batch_fetch_update_status(&$context) { */ function locale_translation_batch_fetch_finished($success, $results) { module_load_include('bulk.inc', 'locale'); + if ($success) { + state()->set('locale.translation_last_checked', REQUEST_TIME); + } return locale_translate_batch_finished($success, $results); } @@ -496,21 +325,22 @@ function locale_translation_http_check($uri) { * - "langcode": Translation language. * - "version": Project version. * - "filename": File name. + * @param string $directory + * Directory where the downloaded file will be saved. Defaults to the + * temporary file path. * * @return object * File object if download was successful. FALSE on failure. */ -function locale_translation_download_source($source_file) { - if ($uri = system_retrieve_file($source_file->uri, 'temporary://')) { - $file = new stdClass(); - $file->project = $source_file->project; - $file->langcode = $source_file->langcode; - $file->version = $source_file->version; - $file->type = LOCALE_TRANSLATION_DOWNLOADED; +function locale_translation_download_source($source_file, $directory = 'temporary://') { + if ($uri = system_retrieve_file($source_file->uri, $directory)) { + $file = clone($source_file); + $file->type = LOCALE_TRANSLATION_LOCAL; $file->uri = $uri; - $file->filename = $source_file->filename; + $file->directory = $directory; + $file->timestamp = filemtime($uri); return $file; } - watchdog('locale', 'Unable to download translation file @uri.', array('@uri' => $source->files[LOCALE_TRANSLATION_REMOTE]->uri), WATCHDOG_ERROR); + watchdog('locale', 'Unable to download translation file @uri.', array('@uri' => $source_file->uri), WATCHDOG_ERROR); return FALSE; } diff --git a/core/modules/locale/locale.bulk.inc b/core/modules/locale/locale.bulk.inc index 5173132..a34570a 100644 --- a/core/modules/locale/locale.bulk.inc +++ b/core/modules/locale/locale.bulk.inc @@ -442,78 +442,137 @@ function locale_translate_batch_build($files, $options) { * Contains a list of files imported. */ function locale_translate_batch_import($file, $options, &$context) { - // Merge the default values in the $options array. - $options += array( - 'overwrite_options' => array(), - 'customized' => LOCALE_NOT_CUSTOMIZED, - ); + if ($report = locale_translate_file_import($file, $options)) { + // If not yet finished with reading, mark progress based on size and + // position. + if ($report['finished'] < 1) { + + // Maximize the progress bar at 95% before completion, the batch API + // could trigger the end of the operation before file reading is done, + // because of floating point inaccuracies. See + // http://drupal.org/node/1089472 + $context['finished'] = min(0.95, $report['finished']); + if (isset($options['message'])) { + $context['message'] = t('!message (@percent%).', array('!message' => $options['message'], '@percent' => (int) ($context['finished'] * 100))); + } + else { + $context['message'] = t('Importing translation file: %filename (@percent%).', array('%filename' => $file->filename, '@percent' => (int) ($context['finished'] * 100))); + } + } + else { + // We are finished here. + $context['finished'] = 1; + + // Store the file data for processing by the next batch operation. + $file->timestamp = filemtime($file->uri); + $context['results']['files'][$file->uri] = $file; + $context['results']['languages'][$file->uri] = $file->langcode; + } + // Add the reported values to the statistics for this file. + // Each import iteration reports statistics in an array. The results of + // each iteration are added and merged here and stored per file. + if (!isset($context['results']['stats']) || !isset($context['results']['stats'][$file->uri])) { + $context['results']['stats'][$file->uri] = array(); + } + foreach ($report as $key => $value) { + if (is_numeric($report[$key])) { + if (!isset($context['results']['stats'][$file->uri][$key])) { + $context['results']['stats'][$file->uri][$key] = 0; + } + $context['results']['stats'][$file->uri][$key] += $report[$key]; + } + elseif (is_array($value)) { + $context['results']['stats'][$file->uri] += array($key => array()); + $context['results']['stats'][$file->uri][$key] = array_merge($context['results']['stats'][$file->uri][$key], $value); + } + } + } + else { + // Import failed. Store the data of the failing file. + $context['results']['failed_files'][] = $file; + watchdog('locale', 'Unable to import translations file: @file', array('@file' => $file->uri)); + } +} + +/** + * Import translation file into the database. + * + * Translation import can handle large translation files. It does this with + * incremental imports. Each time this function is called fixed number of + * translations are imported. Make sure to call this function as many times + * until "finished" is equal to 1. + * + * @param object $file + * A file object of the gettext file to be imported. The file object must + * contain a language parameter (other than LANGUAGE_NOT_SPECIFIED). This + * is used as the language of the import. + * + * @param array $options + * An array with options that can have the following elements: + * - 'langcode': The language code. + * - 'overwrite_options': Overwrite options array as defined in + * Drupal\locale\PoDatabaseWriter. Optional, defaults to an empty array. + * - 'customized': Flag indicating whether the strings imported from $file + * are customized translations or come from a community source. Use + * LOCALE_CUSTOMIZED or LOCALE_NOT_CUSTOMIZED. Optional, defaults to + * LOCALE_NOT_CUSTOMIZED. + * - 'message': Alternative message to display during import. Note, this must + * be sanitized text. + * + * @return array or bool + * On successfull import: Array of import data containing: + * - "filesize": Size of the translation file. + * - "seek": Position until where the import progressed. + * - "finished": Fraction of import completion. '1' is 100% complete. + * FALSE when a fault occurred during import. + */ +function locale_translate_file_import($file, $options) { if (isset($file->langcode) && $file->langcode != LANGUAGE_NOT_SPECIFIED) { + // Get stored import status data. + $import_state = state()->get('locale_translate_file_import_' . $file->filename, array()); + + // Merge the default values in the $options array. + $options += array( + 'overwrite_options' => array(), + 'customized' => LOCALE_NOT_CUSTOMIZED, + ); try { - if (empty($context['sandbox'])) { - $context['sandbox']['parse_state'] = array( - 'filesize' => filesize(drupal_realpath($file->uri)), - 'chunk_size' => 200, + // Initialize the import state. + if (empty($import_state)) { + $import_state = array( + 'filesize' => filesize($file->uri), + 'items' => 200, 'seek' => 0, ); } - // Update the seek and the number of items in the $options array(). - $options['seek'] = $context['sandbox']['parse_state']['seek']; - $options['items'] = $context['sandbox']['parse_state']['chunk_size']; + + // Import translations from file into the database. The number of items to + // import ("items") and the position at which to start ("seek") are merged + // into the $options array. + $options += $import_state; $report = GetText::fileToDatabase($file, $options); - // If not yet finished with reading, mark progress based on size and - // position. - if ($report['seek'] < filesize($file->uri)) { - - $context['sandbox']['parse_state']['seek'] = $report['seek']; - // Maximize the progress bar at 95% before completion, the batch API - // could trigger the end of the operation before file reading is done, - // because of floating point inaccuracies. See - // http://drupal.org/node/1089472 - $context['finished'] = min(0.95, $report['seek'] / filesize($file->uri)); - if (isset($options['message'])) { - $context['message'] = t('!message (@percent%).', array('!message' => $options['message'], '@percent' => (int) ($context['finished'] * 100))); - } - else { - $context['message'] = t('Importing translation file: %filename (@percent%).', array('%filename' => $file->filename, '@percent' => (int) ($context['finished'] * 100))); - } - } - else { - // We are finished here. - $context['finished'] = 1; - // Store the file data for processing by the next batch operation. - $file->timestamp = filemtime($file->uri); - $context['results']['files'][$file->uri] = $file; - $context['results']['languages'][$file->uri] = $file->langcode; - } + // Calculate and store state for next import run. When import is completed + // the import state is dropped. + $import_state['seek'] = $report['seek']; + $report['finished'] = min($report['seek'] / $import_state['filesize'], 1); - // Add the reported values to the statistics for this file. - // Each import iteration reports statistics in an array. The results of - // each iteration are added and merged here and stored per file. - if (!isset($context['results']['stats']) || !isset($context['results']['stats'][$file->uri])) { - $context['results']['stats'][$file->uri] = array(); + if ($report['finished'] == 1) { + state()->delete('locale_translate_file_import_' . $file->filename); } - foreach ($report as $key => $value) { - if (is_numeric($report[$key])) { - if (!isset($context['results']['stats'][$file->uri][$key])) { - $context['results']['stats'][$file->uri][$key] = 0; - } - $context['results']['stats'][$file->uri][$key] += $report[$key]; - } - elseif (is_array($value)) { - $context['results']['stats'][$file->uri] += array($key => array()); - $context['results']['stats'][$file->uri][$key] = array_merge($context['results']['stats'][$file->uri][$key], $value); - } + else { + state()->set('locale_translate_file_import_' . $file->filename, $import_state); } + return $report; } catch (Exception $exception) { - // Import failed. Store the data of the failing file. - $context['results']['failed_files'][] = $file; watchdog('locale', 'Unable to import translations file: @file', array('@file' => $file->uri)); } } + + return FALSE; } /** diff --git a/core/modules/locale/locale.compare.inc b/core/modules/locale/locale.compare.inc index 5ec2752..1cb4d67 100644 --- a/core/modules/locale/locale.compare.inc +++ b/core/modules/locale/locale.compare.inc @@ -80,6 +80,7 @@ function locale_translation_build_projects() { } } } + // @todo Remove this elseif. See http://drupal.org/node/1883154 // If project is not a dev version, but is core, pick latest release. elseif ($name == "drupal") { // Pick latest available release. @@ -212,31 +213,6 @@ function locale_translation_default_translation_server() { } /** - * Build path to translation source, out of a server path replacement pattern. - * - * @param stdClass $project - * Project object containing data to be inserted in the template. - * @param string $template - * String containing placeholders. Available placeholders: - * - "%project": Project name. - * - "%version": Project version. - * - "%core": Project core version. - * - "%language": Language code. - * - * @return string - * String with replaced placeholders. - */ -function locale_translation_build_server_pattern($project, $template) { - $variables = array( - '%project' => $project->name, - '%version' => $project->version, - '%core' => $project->core, - '%language' => isset($project->langcode) ? $project->langcode : '%language', - ); - return strtr($template, $variables); -} - -/** * Check for the latest release of project translations. * * @param array $projects @@ -247,6 +223,7 @@ function locale_translation_build_server_pattern($project, $template) { * @return array * Available sources indexed by project and language. */ +// @todo Return batch or NULL function locale_translation_check_projects($projects = array(), $langcodes = array()) { if (locale_translation_use_remote_source()) { // Retrieve the status of both remote and local translation sources by @@ -256,6 +233,7 @@ function locale_translation_check_projects($projects = array(), $langcodes = arr else { // Retrieve and save the status of local translations only. locale_translation_check_projects_local($projects, $langcodes); + state()->set('locale.translation_last_checked', REQUEST_TIME); } } @@ -272,6 +250,7 @@ function locale_translation_check_projects($projects = array(), $langcodes = arr * @param string $langcodes * Array of language codes. Defaults to all translatable languages. */ +// @todo Return batch, don't set it here. function locale_translation_check_projects_batch($projects = array(), $langcodes = array()) { // Build and set the batch process. $batch = locale_translation_batch_status_build($projects, $langcodes); @@ -328,20 +307,20 @@ function locale_translation_batch_status_build($projects = array(), $langcodes = */ function _locale_translation_batch_status_operations($projects, $langcodes) { $operations = array(); + $use_remote = locale_translation_use_remote_source(); - // Set the batch processes for remote sources. - $sources = locale_translation_build_sources($projects, $langcodes); - if (locale_translation_use_remote_source()) { - foreach ($sources as $source) { - $operations[] = array('locale_translation_batch_status_fetch_remote', array($source)); + foreach ($projects as $project) { + foreach ($langcodes as $langcode) { + // Check for local sources and save the result. + $operations[] = array('locale_translation_batch_status_fetch_local', array($project, $langcode)); + + // Set the batch processes for remote sources. + if ($use_remote) { + $operations[] = array('locale_translation_batch_status_fetch_remote', array($project, $langcode)); + } } } - // Check for local sources, compare the results of local and remote and store - // the most recent. - $operations[] = array('locale_translation_batch_status_fetch_local', array($sources)); - $operations[] = array('locale_translation_batch_status_compare', array()); - return $operations; } @@ -377,35 +356,79 @@ function locale_translation_check_projects_local($projects = array(), $langcodes foreach ($projects as $name => $project) { foreach ($langcodes as $langcode) { $source = locale_translation_source_build($project, $langcode); - if (locale_translation_source_check_file($source)) { - $source->type = 'local'; - $source->timestamp = $source->files['local']->timestamp; - } + $file = locale_translation_source_check_file($source); + locale_translation_status_save($name, $langcode, LOCALE_TRANSLATION_LOCAL, $file); + } + } +} + +/** + * Check updates for active projects and languages. + * + * @param $count + * Number of package translations to check. + * @param $last + * Unix timestamp, check only updates that haven't been checked for this time. + * @param $limit + * Maximum number of updates to do. We check $count translations + * but we stop after we do $limit updates. + * @return array + */ +function locale_translation_update_cron($count, $last) { + $projects_to_update = $languages_to_update = array(); + + $languages = locale_translatable_language_list(); + if (empty($languages)) { + return; + } - // Compare the available translation with the current translations status. - // If the project/language was translated before and it is more recent - // than the most recent translation, the translation is up to date. Which - // is marked in the source object with type "current". - if (isset($history[$source->project][$source->langcode])) { - $current = $history[$source->project][$source->langcode]; - // Add the current translation to the source object to save it in - // the status cache. - $source->files[LOCALE_TRANSLATION_CURRENT] = $current; - - if (isset($source->type)) { - $available = $source->files[$source->type]; - $result = _locale_translation_source_compare($current, $available) == LOCALE_TRANSLATION_SOURCE_COMPARE_LT ? $available : $current; - $source->type = $result->type; - $source->timestamp = $result->timestamp; + // Select active projects x languages ordered by last checked time + $query = db_select('locale_project', 'p'); + $query->leftJoin('locale_file', 'f', 'p.name = f.project'); + $query->condition('p.status', 1); + $query->condition('f.last_checked', $last, '<'); + $query->condition('f.langcode', array_keys($languages)); + $query->fields('f', array('project')); + $query->range(0, $count); + $query->orderBy('last_checked'); + $projects = $query->execute()->fetchAllKeyed(0, 0); +watchdog('debug', 'Projects to update: ' . print_r($projects, 1)); + + if ($projects) { + module_load_include('fetch.inc', 'locale'); + + // Get status of translations. + // @todo This kicks off a batch, but it does not work in Cron. + // Need to convert the batch exection to a queue process. +// locale_translation_check_projects($projects); + $status = locale_translation_get_status(); + $status = array_intersect_key($status, $projects); + + // Mark 'last_checked' for up to date translations. + foreach ($status as $project => $languages) { + foreach ($languages as $langcode => $source) { + if (!isset($source->type) || $source->type == LOCALE_TRANSLATION_CURRENT) { + // The translation is up-to-date update the last_updated timestamp in {locale_file}. +watchdog('debug', "Mark up-to-date: $project : $langcode"); } else { - $source->type = $current->type; - $source->timestamp = $current->timestamp; + $projects_to_update[$project] = $project; + $languages_to_update[$langcode] = $langcode; } } - - $results[$name][$langcode] = $source; + } + if ($projects_to_update && $languages_to_update) { +watchdog('debug', "Update translations for " . print_r($projects_to_update, 1) . ' in ' . print_r($languages_to_update, 1) . "."); + // Update the remaining projects/languages. + $options = _locale_translation_default_update_options(); + // @todo This kicks off a batch, but it does not work in Cron. + // Need to convert the batch exection to a queue process. +// if ($batch = locale_translation_batch_fetch_build($projects_to_update, $languages_to_update, $options)) { +// batch_set($batch); +// batch_process(); +// } + return count($projects_to_update) * count($languages_to_update); } } - locale_translation_status_save($results); + return FALSE; } diff --git a/core/modules/locale/locale.fetch.inc b/core/modules/locale/locale.fetch.inc index c17ff4a..62a488f 100644 --- a/core/modules/locale/locale.fetch.inc +++ b/core/modules/locale/locale.fetch.inc @@ -94,6 +94,7 @@ function _locale_translation_fetch_operations($projects, $langcodes, $options) { $operations = array(); $config = config('locale.settings'); + //@todo Rework this batch operation to only reset the result counter. $operations[] = array('locale_translation_batch_fetch_sources', array($projects, $langcodes)); foreach ($projects as $project) { foreach ($langcodes as $langcode) { @@ -104,12 +105,5 @@ function _locale_translation_fetch_operations($projects, $langcodes, $options) { } } - // Update and save the translation status. - $operations[] = array('locale_translation_batch_fetch_update_status', array()); - - // Update and save the source status. New translation files have been - // downloaded, so other sources will be newer. We update the status now. - $operations[] = array('locale_translation_batch_status_compare', array()); - return $operations; } diff --git a/core/modules/locale/locale.install b/core/modules/locale/locale.install index 0bcf093..1921355 100644 --- a/core/modules/locale/locale.install +++ b/core/modules/locale/locale.install @@ -228,6 +228,12 @@ function locale_schema() { 'default' => 0, 'description' => 'Unix timestamp of the last time this translation was confirmed to be the most recent release available.', ), + 'queued' => array( + 'type' => 'int', + 'not null' => FALSE, + 'default' => 0, + 'description' => 'Unix timestamp when the file was put in to the queue for update check.', + ), ), 'primary key' => array('project', 'langcode'), ); @@ -290,17 +296,17 @@ function locale_requirements($phase) { $requirements = array(); if ($phase == 'runtime') { $available_updates = array(); - $updates_not_found = array(); + $untranslated = array(); $languages = locale_translatable_language_list(); if ($languages) { // Determine the status of the translation updates per lanuage. - $status = state()->get('locale.translation_status'); + $status = locale_translation_get_status(); if ($status) { foreach ($status as $project_id => $project) { foreach ($project as $langcode => $project_info) { - if (!isset($project_info->type)) { - $updates_not_found[$langcode] = $languages[$langcode]->name; + if (empty($project_info->type)) { + $untranslated[$langcode] = $languages[$langcode]->name; } elseif ($project_info->type == LOCALE_TRANSLATION_LOCAL || $project_info->type == LOCALE_TRANSLATION_REMOTE) { $available_updates[$langcode] = $languages[$langcode]->name; @@ -308,7 +314,7 @@ function locale_requirements($phase) { } } - if ($available_updates || $updates_not_found) { + if ($available_updates || $untranslated) { if ($available_updates) { $requirements['locale_translation'] = array( 'title' => 'Translation update status', @@ -322,7 +328,7 @@ function locale_requirements($phase) { 'title' => 'Translation update status', 'value' => t('Missing translations'), 'severity' => REQUIREMENT_INFO, - 'description' => t('Missing translations for: @languages. See the Available translation updates page for more information.', array('@languages' => implode(', ', $updates_not_found), '@updates' => url('admin/reports/translations'))), + 'description' => t('Missing translations for: @languages. See the Available translation updates page for more information.', array('@languages' => implode(', ', $untranslated), '@updates' => url('admin/reports/translations'))), ); } } @@ -972,6 +978,15 @@ function locale_update_8017() { )); } +function locale_update_8018() { + db_add_field('locale_file', 'queued', array( + 'type' => 'int', + 'not null' => FALSE, + 'default' => 0, + 'description' => 'Unix timestamp when the file was put in to the queue for update check.', + )); +} + /** * @} End of "addtogroup updates-7.x-to-8.x". * The next series of updates should start at 9000. diff --git a/core/modules/locale/locale.module b/core/modules/locale/locale.module index 31fd0fc..b4bb01b 100644 --- a/core/modules/locale/locale.module +++ b/core/modules/locale/locale.module @@ -132,16 +132,6 @@ const LOCALE_TRANSLATION_CURRENT = 'current'; /** - * Translation source is a downloaded file. - */ -const LOCALE_TRANSLATION_DOWNLOADED = 'download'; - -/** - * Translation source is an imported file. - */ -const LOCALE_TRANSLATION_IMPORTED = 'import'; - -/** * Implements hook_help(). */ function locale_help($path, $arg) { @@ -509,6 +499,83 @@ function locale_themes_disabled($themes) { } /** + * Implements hook_cron(). + */ +function locale_cron() { + //if ($frequency = config('locale.settings')->get('translation.update_interval_days')) { + if (TRUE) { + module_load_include('inc', 'locale', 'locale.translation'); + $projects = array_keys(locale_translation_get_projects()); + $langcodes = array_keys(locale_translatable_language_list()); + + module_load_include('inc', 'locale', 'locale.compare'); + $operations = _locale_translation_batch_status_operations($projects, $langcodes); + + $queue = Drupal::queue('locale_translation_update'); + foreach($operations as $operation) { + $callback = $operation[0]; + list($projects, $langcodes) = $operation[1]; + //TODO: why are these arrays for local and not remote $callback? + if (!is_array($projects)) $projects = array($projects); + if (!is_array($langcodes)) $langcodes = array($langcodes); + + foreach($projects as $project) { + foreach($langcodes as $langcode) { + // transform $task into $item if necessary + $item = array( + 'callback' => $callback, + 'project' => $project, + 'langcode' => $langcode, + ); + watchdog('locale', 'Queued for !callback: !project (!langcode)', array('callback'=>$callback,'!project'=>$project, '!langcode'=>$langcode)); + $queue->createItem($item); + } + } + } + } +} + +/** + * Implements hook_queue_info(). + */ +function locale_queue_info() { + $queues['locale_translation_update'] = array( + 'title' => t('Updates the translations'), + 'worker callback' => 'locale_translation_update_worker', + 'cron' => array( + 'time' => 60, + ), + ); + return $queues; +} + +/** + * Process 1 file download + * + * TODO: define type for item: object or custom class or array. + * @param type $item + * + * @see locale_cron(). + */ +function locale_translation_update_worker($item) { + module_load_include('inc', 'locale', 'locale.batch'); + + $callback = $item['callback']; + $project = $item['project']; + $langcode = $item['langcode']; + $context = array(); + if (function_exists($callback)) { + if ($callback == 'locale_translation_batch_status_fetch_local') { + locale_translation_batch_status_fetch_local($project, $langcode, $context); + } + else { + locale_translation_batch_status_fetch_remote($project, $langcode, $context); + } + } + watchdog('locale', 'Translation downloaded !callback: !project (!langcode)', array('callback'=>$callback,'!project'=>$project, '!langcode'=>$langcode)); +} + +/** * Imports translations when new modules or themes are installed. * * This function will start a batch to import translations for the added @@ -519,7 +586,6 @@ function locale_themes_disabled($themes) { * translations for, indexed by type. */ function locale_system_update(array $components) { - $components += array('module' => array(), 'theme' => array()); $list = array_merge($components['module'], $components['theme']); @@ -843,6 +909,7 @@ function locale_form_system_file_system_settings_alter(&$form, $form_state) { '#default_value' => config('locale.settings')->get('translation.path'), '#maxlength' => 255, '#description' => t('A local file system path where interface translation files will be stored.'), + '#required' => TRUE, '#after_build' => array('system_check_directory'), '#weight' => 10, ); @@ -903,7 +970,7 @@ function locale_translation_get_file_history() { if (empty($history)) { // Get file history from the database. - $result = db_query('SELECT project, langcode, filename, version, uri, timestamp, last_checked FROM {locale_file}'); + $result = db_query('SELECT project, langcode, filename, version, uri, timestamp, last_checked, queued FROM {locale_file}'); foreach ($result as $file) { $file->type = LOCALE_TRANSLATION_CURRENT; $history[$file->project][$file->langcode] = $file; @@ -931,7 +998,11 @@ function locale_translation_update_file_history($file) { else { $update = array(); } - return drupal_write_record('locale_file', $file, $update); + $result = drupal_write_record('locale_file', $file, $update); + // The file history has changed, flush the static cache now. + // @todo Can we make this more fine grained? + drupal_static_reset('locale_translation_get_file_history'); + return $result; } /** @@ -955,24 +1026,92 @@ function locale_translation_file_history_delete($projects = array(), $langcodes } /** + * Gets the current translation status. + * + * @todo What is 'translation status'? + */ +function locale_translation_get_status($projects = NULL, $langcodes = NULL) { + $result = array(); + $status = state()->get('locale.translation_status'); + module_load_include('translation.inc', 'locale'); + $projects = $projects ? $projects : array_keys(locale_translation_get_projects()); + $langcodes = $langcodes ? $langcodes : array_keys(locale_translatable_language_list()); + + // Get the translation status of each project-language combination. If no + // status was stored, a new translation source is created. + foreach ($projects as $project) { + foreach ($langcodes as $langcode) { + if (isset($status[$project][$langcode])) { + $result[$project][$langcode] = $status[$project][$langcode]; + } + else { + $sources = locale_translation_build_sources(array($project), array($langcode)); + if (isset($sources[$project][$langcode])) { + $result[$project][$langcode] = $sources[$project][$langcode]; + } + } + } + } + return $result; +} + +/** * Saves the status of translation sources in static cache. * + * @param string $project + * Machine readable project name. + * @param string $langcode + * Language code. + * @param string $type + * Type of data to be stored. * @param array $data - * Array of translation source data, structured by project name and langcode. + * File object also containing timestamp when the translation is last updated. */ -function locale_translation_status_save($data) { +function locale_translation_status_save($project, $langcode, $type, $data) { // Followup issue: http://drupal.org/node/1842362 // Split status storage per module/language and expire individually. This will // improve performance for large sites. - $status = state()->get('locale.translation_status'); + module_load_include('translation.inc', 'locale'); + $status = locale_translation_get_status(); $status = empty($status) ? array() : $status; - // Merge the new data into the existing structured status array. - foreach ($data as $project => $languages) { - foreach ($languages as $langcode => $source) { - $status[$project][$langcode] = $source; + // @todo Can this be done better? Use ..._build_sources instead? + if (!isset($status[$project])) { + $projects = locale_translation_get_projects(array($project)); + if (isset($projects[$project])) { + $status[$project][$langcode] = locale_translation_source_build($projects[$project], $langcode); } } + elseif (!isset($status[$project][$langcode])) { + $projects = locale_translation_get_projects(array($project)); + $status[$project][$langcode] = locale_translation_source_build($projects[$project], $langcode); + } + + // Merge the new data into the existing structured status array. + switch ($type) { + case LOCALE_TRANSLATION_REMOTE: + case LOCALE_TRANSLATION_LOCAL: + // Add the source data to the status array. + $status[$project][$langcode]->files[$type] = $data; + + // Check if this translation is the most recent one. Set timestamp and + // data type of the most recent translation source. + if (isset($data->timestamp) && $data->timestamp) { + if ($data->timestamp > $status[$project][$langcode]->timestamp) { + $status[$project][$langcode]->timestamp = $data->timestamp; + $status[$project][$langcode]->last_checked = REQUEST_TIME; + $status[$project][$langcode]->type = $type; + } + } + break; + case LOCALE_TRANSLATION_CURRENT: + $data->last_checked = REQUEST_TIME; + $status[$project][$langcode]->timestamp = $data->timestamp; + $status[$project][$langcode]->last_checked = $data->last_checked; + $status[$project][$langcode]->type = $type; + locale_translation_update_file_history($data); + break; + } state()->set('locale.translation_status', $status); state()->set('locale.translation_last_checked', REQUEST_TIME); @@ -985,7 +1124,7 @@ function locale_translation_status_save($data) { * Language code(s) to be deleted from the cache. */ function locale_translation_status_delete_languages($langcodes) { - if ($status = state()->get('locale.translation_status')) { + if ($status = locale_translation_get_status()) { foreach ($status as $project => $languages) { foreach ($languages as $langcode => $source) { if (in_array($langcode, $langcodes)) { @@ -1004,7 +1143,7 @@ function locale_translation_status_delete_languages($langcodes) { * Project name(s) to be deleted from the cache. */ function locale_translation_status_delete_projects($projects) { - $status = state()->get('locale.translation_status'); + $status = locale_translation_get_status(); foreach ($status as $project => $languages) { if (in_array($project, $projects)) { diff --git a/core/modules/locale/locale.pages.inc b/core/modules/locale/locale.pages.inc index a109f5f..c29b719 100644 --- a/core/modules/locale/locale.pages.inc +++ b/core/modules/locale/locale.pages.inc @@ -493,14 +493,14 @@ function locale_translation_status_form($form, &$form_state) { module_load_include('compare.inc', 'locale'); $updates = $options = array(); $languages_update = $languages_not_found = array(); + $projects_update = array(); // @todo Calling locale_translation_build_projects() is an expensive way to // get a module name. In follow-up issue http://drupal.org/node/1842362 // the project name will be stored to display use, like here. $project_data = locale_translation_build_projects(); $languages = locale_translatable_language_list(); - $projects = locale_translation_get_projects(); - $status = state()->get('locale.translation_status'); + $status = locale_translation_get_status(); // Prepare information about projects which have available translation // updates. @@ -508,7 +508,7 @@ function locale_translation_status_form($form, &$form_state) { foreach ($status as $project_id => $project) { foreach ($project as $langcode => $project_info) { // No translation file found for this project-language combination. - if (!isset($project_info->type)) { + if (empty($project_info->type)) { $updates[$langcode]['not_found'][] = array( 'name' => $project_info->name == 'drupal' ? t('Drupal core') : $project_data[$project_info->name]->info['name'], 'version' => $project_info->version, @@ -527,6 +527,7 @@ function locale_translation_status_form($form, &$form_state) { 'timestamp' => $recent->timestamp, ); $languages_update[$langcode] = $langcode; + $projects_update[$project_info->name] = $project_info->name; } } } @@ -569,6 +570,12 @@ function locale_translation_status_form($form, &$form_state) { $empty = t('No translation status available. Check manually.', array('@check' => url('admin/reports/translations/check'))); } + // The projects which require an update. Used by the _submit callback. + $form['projects_update'] = array( + '#type' => 'value', + '#value' => $projects_update, + ); + $form['langcodes'] = array( '#type' => 'tableselect', '#header' => $header, @@ -612,6 +619,7 @@ function locale_translation_status_form_validate($form, &$form_state) { function locale_translation_status_form_submit($form, &$form_state) { module_load_include('fetch.inc', 'locale'); $langcodes = array_filter($form_state['values']['langcodes']); + $projects = array_filter($form_state['values']['projects_update']); // Set the translation import options. This determines if existing // translations will be overwritten by imported strings. @@ -627,7 +635,7 @@ function locale_translation_status_form_submit($form, &$form_state) { batch_set($batch); } else { - $batch = locale_translation_batch_fetch_build(array(), $langcodes, $options); + $batch = locale_translation_batch_fetch_build($projects, $langcodes, $options); batch_set($batch); } } diff --git a/core/modules/locale/locale.queue.inc b/core/modules/locale/locale.queue.inc new file mode 100644 index 0000000..4fb0093 --- /dev/null +++ b/core/modules/locale/locale.queue.inc @@ -0,0 +1,178 @@ +get('translation.update_interval_days')) { + locale_cron_fill_queue(); + locale_cron_process_queue(); + } +} + +/** + * Populate a queue with project to check for translation updates. + */ +function locale_cron_fill_queue() { + $languages = locale_translatable_language_list(); + + // If no languages are enabled, there is nothing to do here. + if (empty($languages)) { + return; + } + + // Select active projects x languages ordered by last checked time + $last = REQUEST_TIME - config('locale.settings')->get('translation.update_interval_days') * 3600 * 24; + // @todo For testing only. vv + $last = REQUEST_TIME; + $query = db_select('locale_project', 'p'); + $query->join('locale_file', 'f', 'p.name = f.project'); + $query->condition('p.status', 1); + $query->condition('f.last_checked', $last, '<'); + $query->condition('f.langcode', array_keys($languages)); + $query->condition('f.queued', 0); + $query->fields('f', array('project', 'langcode')); + $query->orderBy('last_checked'); + if ($updates = $query->execute()->fetchAll()) { + // @todo $updates will only contain existing translations. Project/language without translations needs to be added. + // @todo Instead: Query for all up to date translations and exclude these from a full list based on all projects and all languages. + // Add update tasks to the queue. + // @todo Consider using _locale_translation_fetch_operations() as source of queue items. + $queue = Drupal::queue('locale_translation', TRUE); + foreach ($updates as $update) { + $queue->createItem(array( + 'callback' => 'locale_translation_batch_status_fetch_local', + 'arguments' => array( + 'project' => $update->project, + 'langcode' => $update->langcode, + ), + 'return_value' => FALSE, + )); + $queue->createItem(array( + 'callback' => 'locale_translation_batch_status_fetch_remote', + 'arguments' => array( + 'project' => $update->project, + 'langcode' => $update->langcode, + ), + 'return_value' => FALSE, + )); + $queue->createItem(array( + 'callback' => 'locale_translation_batch_fetch_download', + 'arguments' => array( + 'project' => $update->project, + 'langcode' => $update->langcode, + ), + 'return_value' => FALSE, + )); + $queue->createItem(array( + 'callback' => 'locale_translation_queue_fetch_import', + 'arguments' => array( + 'project' => $update->project, + 'langcode' => $update->langcode, + ), + 'return_value' => TRUE, + )); + } + + // Mark all projects as being added to the queue to prevent them being added + // multiple times. + // @todo Flagging in locale_file does not work for project/language without translation file. + $query = db_update('locale_file'); + $query->fields(array('queued' => REQUEST_TIME)); + $or = db_or(); + foreach($updates as $update) { + $and = db_and(); + $and->condition('project', $update->project); + $and->condition('langcode', $update->langcode); + $or->condition($and); + } + $query->condition($or); + $query->execute(); + } +} + +/** + * Processes project translation items in the queue. + * + * Checks for project translation update. Download and imports if update is + * available. + */ +function locale_cron_process_queue() { + $end = time() + 15; + $queue = Drupal::queue('locale_translation', TRUE); + while (time() < $end && ($item = $queue->claimItem(300))) { + if (locale_translation_update_queue_worker($item->data)) { + $queue->deleteItem($item); + + // Unset queued flag. + $query = db_update('locale_file'); + $query->fields(array('queued' => 0)); + $query->condition('project', $item->data['arguments']['project']); + $query->condition('langcode', $item->data['arguments']['langcode']); + $query->execute(); + } + else { + $queue->releaseItem($item); + } + } +} + +/** +function locale_cron_unset_queue($item) { + +} + +/** + * Performs check, download and import of project translations. + * May be called multiple times if the import is not completed. + */ +function locale_translation_update_queue_worker($data) { + $function = $data['callback']; + + module_load_include('batch.inc', 'locale'); + if (function_exists($function)) { + $context = array(); + if (!$data['return_value']) { + $function($data['arguments']['project'], $data['arguments']['langcode'], $context); + return TRUE; + } + return $function($data['arguments']['project'], $data['arguments']['langcode']); + } +} + +/** + * Import a local translation file. + * + * @param $project + * @param $langcode + */ +// @todo Consider using the batch function locale_translation_batch_fetch_import() instead. +// @todo If this function is not required, function locale_translate_file_import() can be droped and the change in http://drupal.org/node/1998056 #11 be reverted. +function locale_translation_queue_fetch_import($project, $langcode) { + $sources = locale_translation_get_status(array($project), array($langcode)); + if (isset($sources[$project][$langcode])) { + $source = $sources[$project][$langcode]; + if (isset($source->type) && ($source->type == LOCALE_TRANSLATION_REMOTE || $source->type == LOCALE_TRANSLATION_LOCAL)) { + module_load_include('translation.inc', 'locale'); + $options = _locale_translation_default_update_options(); + $file = $source->files[LOCALE_TRANSLATION_LOCAL]; + module_load_include('bulk.inc', 'locale'); + // Import the translation file. For large files the batch operations is + // progressive and will be called repeatedly until finished. + if ($result = locale_translate_file_import($file, $options)) { + if ($result['finished'] == 1) { + // Save the data of imported source into the {locale_file} table and + // update the current translation status. + locale_translation_status_save($project, $langcode, LOCALE_TRANSLATION_CURRENT, $source->files[LOCALE_TRANSLATION_LOCAL]); + return TRUE; + } + // Import not completed. + return FALSE; + } + // An import error occurred. Return TRUE to terminate the queue job. + } + } + // Nothing to do. Return TRUE to terminate the queue job. + return TRUE; +} diff --git a/core/modules/locale/locale.translation.inc b/core/modules/locale/locale.translation.inc index 7edf847..c6f8505 100644 --- a/core/modules/locale/locale.translation.inc +++ b/core/modules/locale/locale.translation.inc @@ -104,12 +104,12 @@ function locale_translation_load_sources($projects = NULL, $langcodes = NULL) { $langcodes = $langcodes ? $langcodes : array_keys(locale_translatable_language_list()); // Load source data from locale_translation_status cache. - $status = state()->get('locale.translation_status'); + $status = locale_translation_get_status(); // Use only the selected projects and languages for update. foreach($projects as $project) { foreach ($langcodes as $langcode) { - $sources[$project . ':' . $langcode] = isset($status[$project][$langcode]) ? $status[$project][$langcode] : NULL; + $sources[$project][$langcode] = isset($status[$project][$langcode]) ? $status[$project][$langcode] : NULL; } } return $sources; @@ -124,10 +124,12 @@ function locale_translation_load_sources($projects = NULL, $langcodes = NULL) { * Array of language codes. Defaults to all translatable languages. * * @return array - * Array of source objects. Keyed with :. + * Array of source objects. Keyed by project name and language code. * * @see locale_translation_source_build() */ +// @todo What is the most efficient and flexible way to organize the source data? [$project][$langcode] or ["$project:$langcode"] or other? +// @todo Change the name of this function? Change the parameters too (it is currently only called for single project/langcode)?. function locale_translation_build_sources($projects = array(), $langcodes = array()) { $sources = array(); $projects = locale_translation_get_projects($projects); @@ -136,7 +138,7 @@ function locale_translation_build_sources($projects = array(), $langcodes = arra foreach ($projects as $project) { foreach ($langcodes as $langcode) { $source = locale_translation_source_build($project, $langcode); - $sources[$source->name . ':' . $source->langcode] = $source; + $sources[$source->name][$source->langcode] = $source; } } return $sources; @@ -159,32 +161,32 @@ function locale_translation_build_sources($projects = array(), $langcodes = arra * Translation source object. * * @return stdClass - * File object (filename, basename, name) updated with data of the po file. - * On success the files property of the source object is updated. - * files[LOCALE_TRANSLATION_LOCAL]: + * Source file object of the po file, updated with: * - "uri": File name and path. * - "timestamp": Last updated time of the po file. * FALSE if the file is not found. * * @see locale_translation_source_build() */ -function locale_translation_source_check_file(&$source) { +function locale_translation_source_check_file($source) { if (isset($source->files[LOCALE_TRANSLATION_LOCAL])) { - $directory = $source->files[LOCALE_TRANSLATION_LOCAL]->directory; - $filename = '/' . preg_quote($source->files[LOCALE_TRANSLATION_LOCAL]->filename) . '$/'; + $source_file = $source->files[LOCALE_TRANSLATION_LOCAL]; + $directory = $source_file->directory; + $filename = '/' . preg_quote($source_file->filename) . '$/'; // If the directory contains a stream wrapper, it is converted to a real // path. This is required for file_scan_directory() which can not handle // stream wrappers. + // @todo file_scan_directory() has changed does it handle stream wrappers now? if ($scheme = file_uri_scheme($directory)) { $directory = str_replace($scheme . '://', drupal_realpath($scheme . '://'), $directory); } if ($files = file_scan_directory($directory, $filename, array('key' => 'name', 'recurse' => FALSE))) { $file = current($files); - $source->files[LOCALE_TRANSLATION_LOCAL]->uri = $file->uri; - $source->files[LOCALE_TRANSLATION_LOCAL]->timestamp = filemtime($file->uri); - return $file; + $source_file->uri = $file->uri; + $source_file->timestamp = filemtime($file->uri); + return $source_file; } } return FALSE; @@ -211,13 +213,14 @@ function locale_translation_source_check_file(&$source) { * - "files": Array of file objects containing properties of local and remote * translation files. * Other processes can add the following properties: - * - "type": Most recent file type LOCALE_TRANSLATION_REMOTE or - * LOCALE_TRANSLATION_LOCAL. Corresponding with a key of the - * "files" array. - * - "timestamp": Timestamp of the most recent translation file. + * - "type": Most recent translation source found. LOCALE_TRANSLATION_REMOTE and + * LOCALE_TRANSLATION_LOCAL indicate available new translations, + * LOCALE_TRANSLATION_CURRENT indicate that the current translation is them + * most recent. "type" sorresponds with a key of the "files" array. + * - "timestamp": The creation time of the "type" translation (file). + * - "last_checked": The time when the "type" translation was last checked. * The "files" array can hold file objects of type: - * LOCALE_TRANSLATION_LOCAL, LOCALE_TRANSLATION_REMOTE, - * LOCALE_TRANSLATION_DOWNLOADED, LOCALE_TRANSLATION_IMPORTED and + * LOCALE_TRANSLATION_LOCAL, LOCALE_TRANSLATION_REMOTE and * LOCALE_TRANSLATION_CURRENT. Each contains following properties: * - "type": The object type (LOCALE_TRANSLATION_LOCAL, * LOCALE_TRANSLATION_REMOTE, etc. see above). @@ -239,6 +242,9 @@ function locale_translation_source_build($project, $langcode, $filename = NULL) $source = clone $project; $source->project = $project->name; $source->langcode = $langcode; + $source->type = ''; + $source->timestamp = 0; + $source->last_checked = 0; $filename = $filename ? $filename : config('locale.settings')->get('translation.default_filename'); @@ -256,17 +262,15 @@ function locale_translation_source_build($project, $langcode, $filename = NULL) 'filename' => locale_translation_build_server_pattern($source, basename($source->server_pattern)), 'uri' => locale_translation_build_server_pattern($source, $source->server_pattern), ); - if (config('locale.settings')->get('translation.path')) { - $files[LOCALE_TRANSLATION_LOCAL] = (object) array( - 'project' => $project->name, - 'langcode' => $langcode, - 'version' => $project->version, - 'type' => LOCALE_TRANSLATION_LOCAL, - 'filename' => locale_translation_build_server_pattern($source, $filename), - 'directory' => 'translations://', - ); - $files[LOCALE_TRANSLATION_LOCAL]->uri = $files[LOCALE_TRANSLATION_LOCAL]->directory . $files[LOCALE_TRANSLATION_LOCAL]->filename; - } + $files[LOCALE_TRANSLATION_LOCAL] = (object) array( + 'project' => $project->name, + 'langcode' => $langcode, + 'version' => $project->version, + 'type' => LOCALE_TRANSLATION_LOCAL, + 'filename' => locale_translation_build_server_pattern($source, $filename), + 'directory' => 'translations://', + ); + $files[LOCALE_TRANSLATION_LOCAL]->uri = $files[LOCALE_TRANSLATION_LOCAL]->directory . $files[LOCALE_TRANSLATION_LOCAL]->filename; } else { $files[LOCALE_TRANSLATION_LOCAL] = (object) array( @@ -281,10 +285,46 @@ function locale_translation_source_build($project, $langcode, $filename = NULL) } $source->files = $files; + // If this project/language combination is already translated, we add its + // translation status and update the current translation timestamp and + // last_updated time. + $history = locale_translation_get_file_history(); + if (isset($history[$project->name][$langcode])) { + $source->files[LOCALE_TRANSLATION_CURRENT] = $history[$project->name][$langcode]; + $source->type = LOCALE_TRANSLATION_CURRENT; + $source->timestamp = $history[$project->name][$langcode]->timestamp; + $source->last_checked = $history[$project->name][$langcode]->last_checked; + } + return $source; } /** + * Build path to translation source, out of a server path replacement pattern. + * + * @param stdClass $project + * Project object containing data to be inserted in the template. + * @param string $template + * String containing placeholders. Available placeholders: + * - "%project": Project name. + * - "%version": Project version. + * - "%core": Project core version. + * - "%language": Language code. + * + * @return string + * String with replaced placeholders. + */ +function locale_translation_build_server_pattern($project, $template) { + $variables = array( + '%project' => $project->name, + '%version' => $project->version, + '%core' => $project->core, + '%language' => isset($project->langcode) ? $project->langcode : '%language', + ); + return strtr($template, $variables); +} + +/** * Determine if a file is a remote file. * * @param string $uri