diff --git a/includes/gettext.inc b/includes/gettext.inc index 155d0d4..3c7595d 100644 --- a/includes/gettext.inc +++ b/includes/gettext.inc @@ -3,8 +3,6 @@ /** * @file * Gettext parsing and generating API. - * - * @todo Decouple these functions from Locale API and put to gettext_ namespace. */ /** @@ -17,78 +15,311 @@ */ /** - * Parses Gettext Portable Object file information and inserts into database - * - * @param $file - * Drupal file object corresponding to the PO file to import. - * @param $langcode - * Language code. - * @param $mode - * Should existing translations be replaced LOCALE_IMPORT_KEEP or - * LOCALE_IMPORT_OVERWRITE. + * Default read callback that reads in the .po file all at once. */ -function _locale_import_po($file, $langcode, $mode) { - // Try to allocate enough time to parse and import the data. - drupal_set_time_limit(240); +function gettext_import_read_callback(&$parse_state, $error_callback) { + + if (empty($parse_state['handle'])) { + // File not yet open. Open for reading. + $parse_state['handle'] = fopen($parse_state['file'], 'rb'); + if (empty($parse_state['handle'])) { + // Record error and mark parsing finished. + $error_callback('file could not be read', $parse_state); + $parse_state['finished'] = TRUE; + return FALSE; + } + else { + // File was just opened. Start chunk line counting. Initialize other + // values if we did not get those initialized from the outside. + // chunk_size can be set to use chunked reading, in which case these + // state values will be remembered in batch information. We only + // initialize the values if not yet provided. + $parse_state['chunk_lineno'] = 0; + $parse_state += array('lineno' => 0, 'last_write_lineno' => -1, 'chunk_size' => 0, 'finished' => FALSE); + if (!empty($parse_state['fseek'])) { + // Seek into the file where we left off reading. + fseek($parse_state['handle'], $parse_state['fseek']); + } + } + } - // Check if we have the language already in the database. - if (!db_query("SELECT COUNT(language) FROM {languages} WHERE language = :language", array(':language' => $langcode))->fetchField()) { - drupal_set_message(t('The language selected for import is not supported.'), 'error'); + if (!feof($parse_state['handle'])) { + // The file is already open for reading either from above or from previously. + + if (!empty($parse_state['chunk_size']) && + ($parse_state['last_write_lineno'] == $parse_state['lineno']) && + ($parse_state['chunk_lineno'] > $parse_state['chunk_size'])) { + // If we are doing chunk reads and the previously read line resulted in + // cached data to be written out, we can store the position at the start + // of the previous line, close the file and continue reading later. + fclose($parse_state['handle']); + unset($parse_state['handle']); + // Set the context to COMMENT (which is valid inbetween saved strings + // although not always true due to condensed file structures). This will + // let parsing to finish peacefully. + $parse_state['context'] = 'COMMENT'; + // Pretend the file ended. + return FALSE; + } + + // Remember current position in case we need it for chunked reading. + $parse_state['fseek'] = ftell($parse_state['handle']); + + // A line should not be longer than 10 * 1024. + $line = fgets($parse_state['handle'], 10 * 1024); + + if ($parse_state['lineno'] == 0) { + // The first line might come with a UTF-8 BOM, which should be removed. + $line = str_replace("\xEF\xBB\xBF", '', $line); + } + + // Update line number and chunk line number counting. + $parse_state['lineno']++; + $parse_state['chunk_lineno']++; + + return $line; + } + else { + // File reached its end. Mark read state finished. + $parse_state['finished'] = TRUE; return FALSE; } +} - // Get strings from file (returns on failure after a partial import, or on success) - $status = _locale_import_read_po('db-store', $file, $mode, $langcode); - if ($status === FALSE) { - // Error messages are set in _locale_import_read_po(). - return FALSE; +/** + * Default write callback for gettextapi. + */ +function gettext_import_write_callback($string, &$parse_state, $write_options, $error_callback) { + // Remember last write line number. Required for chunk reading support. + $parse_state['last_write_lineno'] = $parse_state['lineno']; + + // Initialize database storage. + $write_options += array('storage' => 'db'); + + // Initialize stats we collect while storing strings. + if (!isset($parse_state['write_stats'])) { + $parse_state['write_stats'] = array( + 'added' => 0, + 'updated' => 0, + 'deleted' => 0, + 'skipped' => 0, + 'header' => FALSE, + 'strings' => array(), + ); } - // Get status information on import process. - list($header_done, $additions, $updates, $deletes, $skips) = _locale_import_one_string('db-report'); + if ($write_options['storage'] == 'memory') { + $parse_state['write_state']['strings'][isset($string['msgctxt']) ? $string['msgctxt'] : ''][$string['msgid']] = $string['msgstr']; + } + else { - if (!$header_done) { - drupal_set_message(t('The translation file %filename appears to have a missing or malformed header.', array('%filename' => $file->filename)), 'error'); + if ($string['msgid'] == '') { + // Gettext PO file headers are keyes with the empty source string. + $languages = language_list(); + if (($write_options['mode'] != LOCALE_IMPORT_KEEP) || empty($languages[$write_options['langcode']]->plurals)) { + // Since we only need to parse the header if we ought to update the + // plural formula, only run this if we don't need to keep existing + // data untouched or if we don't have an existing plural formula. + $header = gettext_import_parse_header($string['msgstr']); + + // Get the plural formula and update in the database if successful. + if (!empty($header["Plural-Forms"]) && $p = gettext_import_parse_plural_forms($header["Plural-Forms"], $parse_state, $error_callback)) { + list($nplurals, $plural) = $p; + db_update('languages') + ->fields(array( + 'plurals' => $nplurals, + 'formula' => $plural, + )) + ->condition('language', $write_options['langcode']) + ->execute(); + } + } + $parse_state['write_stats']['header'] = TRUE; + } + + else { + // Initialize array used to save our data. + $save = array( + 'context' => isset($string['msgctxt']) ? $string['msgctxt'] : '', + 'location' => gettext_import_shorten_comments(empty($string['#']) ? array() : $string['#']), + 'plid' => 0, + 'plural' => 0, + ); + + if (strpos($string['msgid'], "\0")) { + // This string has plural versions. + $english = explode("\0", $string['msgid'], 2); + $entries = array_keys($string['msgstr']); + for ($i = 3; $i <= count($entries); $i++) { + $english[] = $english[1]; + } + $translation = array_map('gettext_import_append_plural', $string['msgstr'], $entries); + $english = array_map('gettext_import_append_plural', $english, $entries); + foreach ($translation as $key => $trans) { + if ($key == 0) { + $save['plid'] = 0; + } + $save['source'] = $english[$key]; + $save['translation'] = $trans; + $save['plural'] = $key; + $save['plid'] = gettext_import_save_string($save, $parse_state, $write_options, $error_callback); + } + } + else { + // A simple string to import. + $save['source'] = $string['msgid']; + $save['translation'] = $string['msgstr']; + gettext_import_save_string($save, $parse_state, $write_options, $error_callback); + } + } } +} + +/** + * Import one string into the database. + * + * @param $save + * String data to save. + * @param $parse_state + * Current parser state. + * @param $write_options + * Options for writing the string to the database. + * @param $error_callback + * To invoke if we find an error. + * + * @return + * The string ID of the existing string modified or the new string added. + */ +function gettext_import_save_string($save, &$parse_state, $write_options, $error_callback) { + $lid = db_query("SELECT lid FROM {locales_source} WHERE source = :source AND context = :context", array(':source' => $save['source'], ':context' => $save['context']))->fetchField(); - // Clear cache and force refresh of JavaScript translations. - _locale_invalidate_js($langcode); - cache()->deletePrefix('locale:'); + if (!empty($save['translation'])) { + // Skip this string unless it passes a check for dangerous code. + if (!locale_string_is_safe($save['translation'])) { + $parse_state['write_stats']['skipped']++; + $lid = 0; + $error_callback('translation contains disallowed HTML tags, skipped for security reasons', $parse_state); + } + elseif (!empty($lid)) { + // We have this source string saved already. Update location information. + db_update('locales_source') + ->fields(array( + 'location' => $save['location'], + )) + ->condition('lid', $lid) + ->execute(); - // Rebuild the menu, strings may have changed. - menu_rebuild(); + $exists = db_query("SELECT COUNT(lid) FROM {locales_target} WHERE lid = :lid AND language = :language", array(':lid' => $lid, ':language' => $write_options['langcode']))->fetchField(); - drupal_set_message(t('The translation was successfully imported. There are %number newly created translated strings, %update strings were updated and %delete strings were removed.', array('%number' => $additions, '%update' => $updates, '%delete' => $deletes))); - watchdog('locale', 'Imported %file into %locale: %number new strings added, %update updated and %delete removed.', array('%file' => $file->filename, '%locale' => $langcode, '%number' => $additions, '%update' => $updates, '%delete' => $deletes)); - if ($skips) { - $skip_message = format_plural($skips, 'One translation string was skipped because it contains disallowed HTML.', '@count translation strings were skipped because they contain disallowed HTML.'); - drupal_set_message($skip_message); - watchdog('locale', '@count disallowed HTML string(s) in %file', array('@count' => $skips, '%file' => $file->uri), WATCHDOG_WARNING); + if (!$exists) { + // No translation in this language. + db_insert('locales_target') + ->fields(array( + 'lid' => $lid, + 'language' => $write_options['langcode'], + 'translation' => $save['translation'], + 'plid' => $save['plid'], + 'plural' => $save['plural'], + )) + ->execute(); + + $parse_state['write_stats']['added']++; + } + elseif ($write_options['mode'] == LOCALE_IMPORT_OVERWRITE) { + // Translation exists, only overwrite if instructed. + db_update('locales_target') + ->fields(array( + 'translation' => $save['translation'], + 'plid' => $save['plid'], + 'plural' => $save['plural'], + )) + ->condition('language', $write_options['langcode']) + ->condition('lid', $lid) + ->execute(); + + $parse_state['write_stats']['updated']++; + } + } + else { + // No such source string in the database yet. + $lid = db_insert('locales_source') + ->fields(array( + 'location' => $save['location'], + 'source' => $save['source'], + 'context' => (string) $save['context'], + )) + ->execute(); + + db_insert('locales_target') + ->fields(array( + 'lid' => $lid, + 'language' => $write_options['langcode'], + 'translation' => $save['translation'], + 'plid' => $save['plid'], + 'plural' => $save['plural'], + )) + ->execute(); + + $parse_state['write_stats']['added']++; + } } - return TRUE; + elseif ($write_options['mode'] == LOCALE_IMPORT_OVERWRITE) { + // Empty translation, remove existing if instructed. + db_delete('locales_target') + ->condition('language', $write_options['langcode']) + ->condition('lid', $lid) + ->condition('plid', $save['plid']) + ->condition('plural', $save['plural']) + ->execute(); + + $parse_state['write_stats']['deleted']++; + } + + return $lid; } /** - * Parses Gettext Portable Object file into an array + * Sets an error message occurred during Gettext .po parsing. * - * @param $op - * Storage operation type: db-store or mem-store. - * @param $file - * Drupal file object corresponding to the PO file to import. - * @param $mode - * Should existing translations be replaced LOCALE_IMPORT_KEEP or - * LOCALE_IMPORT_OVERWRITE. - * @param $lang - * Language code. + * @param $message + * Explanation of the error. + * @param $parse_state + * State of the read operation such as file and line number. */ -function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { +function gettext_import_error_callback($message, $parse_state) { + $t = get_t(); + drupal_set_message( + $t( + 'Error: @error in @filename on line @line.', + array( + '@filename' => $parse_state['file'], + // @todo: this still keeps $message untranslated. + '@error' => $message, + '@line' => $parse_state['lineno'], + ) + ), + 'error' + ); +} - // The file will get closed by PHP on returning from this function. - $fd = fopen($file->uri, 'rb'); - if (!$fd) { - _locale_import_message('The translation import failed because the file %filename could not be read.', $file); - return FALSE; - } +// == Parser state machine ===================================================== + +/** + * Parse (part of) Gettext Portable Object data and use a writer to store strings. + * + * @param $read_callback + * Callback that provides lines of a Gettext PO structure for parsing. + * @param $parse_state + * State and options for reading; updated with current state. + * @param $write_callback + * Callback to invoke when a saveable part of the .po stream is identified. + * @param $write_options + * Options to send to the write callback that are results of code outside of + * this parser, such as writing mode. + * @param $error_callback + * Callback invoked when an error happens. + */ +function gettext_import_read_po($read_callback, &$parse_state, $write_callback, $write_options, $error_callback) { /* * The parser context. Can be: @@ -99,7 +330,8 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { * - 'MSGSTR' (msgstr or msgstr[]) * - 'MSGSTR_ARR' (msgstr_arg) */ - $context = 'COMMENT'; + $parse_state += array('context' => 'COMMENT'); + $context = &$parse_state['context']; // Current entry being read. $current = array(); @@ -107,19 +339,7 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { // Current plurality for 'msgstr[]'. $plural = 0; - // Current line. - $lineno = 0; - - while (!feof($fd)) { - // A line should not be longer than 10 * 1024. - $line = fgets($fd, 10 * 1024); - - if ($lineno == 0) { - // The first line might come with a UTF-8 BOM, which should be removed. - $line = str_replace("\xEF\xBB\xBF", '', $line); - } - - $lineno++; + while ($line = $read_callback($parse_state, $error_callback)) { // Trim away the linefeed. $line = trim(strtr($line, array("\\\n" => ""))); @@ -133,7 +353,7 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { } elseif (($context == 'MSGSTR') || ($context == 'MSGSTR_ARR')) { // We are currently in string token, close it out. - _locale_import_one_string($op, $current, $mode, $lang, $file); + $write_callback($current, $parse_state, $write_options, $error_callback); // Start a new entry for the comment. $current = array(); @@ -143,7 +363,7 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { } else { // A comment following any other token is a syntax error. - _locale_import_message('The translation file %filename contains an error: "msgstr" was expected but not found on line %line.', $file, $lineno); + $error_callback('"msgstr" expected but not found', $parse_state); return FALSE; } } @@ -152,7 +372,7 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { if ($context != 'MSGID') { // A plural form cannot be added to anything else but the id directly. - _locale_import_message('The translation file %filename contains an error: "msgid_plural" was expected but not found on line %line.', $file, $lineno); + $error_callback('"msgid_plural" expected but not found', $parse_state); return FALSE; } @@ -160,10 +380,10 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { $line = trim(substr($line, 12)); // At this point, $line should now contain only the plural form. - $quoted = _locale_import_parse_quoted($line); + $quoted = gettext_import_parse_quoted($line); if ($quoted === FALSE) { // The plural form must be wrapped in quotes. - _locale_import_message('The translation file %filename contains a syntax error on line %line.', $file, $lineno); + $error_callback('"msgid_plural" not properly wrapped in quotes', $parse_state); return FALSE; } @@ -177,14 +397,14 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { if (($context == 'MSGSTR') || ($context == 'MSGSTR_ARR')) { // We are currently in a message string, close it out. - _locale_import_one_string($op, $current, $mode, $lang, $file); + $write_callback($current, $parse_state, $write_options, $error_callback); // Start a new context for the id. $current = array(); } elseif ($context == 'MSGID') { // We are currently already in the context, meaning we passed an id with no data. - _locale_import_message('The translation file %filename contains an error: "msgid" is unexpected on line %line.', $file, $lineno); + $error_callback('unexpected "msgid"', $parse_state); return FALSE; } @@ -192,10 +412,10 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { $line = trim(substr($line, 5)); // At this point, $line should now contain only the message id. - $quoted = _locale_import_parse_quoted($line); + $quoted = gettext_import_parse_quoted($line); if ($quoted === FALSE) { // The message id must be wrapped in quotes. - _locale_import_message('The translation file %filename contains a syntax error on line %line.', $file, $lineno); + $error_callback('"msgid" not properly wrapped in quotes', $parse_state); return FALSE; } @@ -207,12 +427,12 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { if (($context == 'MSGSTR') || ($context == 'MSGSTR_ARR')) { // We are currently in a message, start a new one. - _locale_import_one_string($op, $current, $mode, $lang, $file); + $write_callback($current, $parse_state, $write_options, $error_callback); $current = array(); } elseif (!empty($current['msgctxt'])) { // A context cannot apply to another context. - _locale_import_message('The translation file %filename contains an error: "msgctxt" is unexpected on line %line.', $file, $lineno); + $error_callback('unexpected "msgctxt"', $parse_state); return FALSE; } @@ -220,10 +440,10 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { $line = trim(substr($line, 7)); // At this point, $line should now contain the context. - $quoted = _locale_import_parse_quoted($line); + $quoted = gettext_import_parse_quoted($line); if ($quoted === FALSE) { // The context string must be quoted. - _locale_import_message('The translation file %filename contains a syntax error on line %line.', $file, $lineno); + $error_callback('"msgctxt" not properly wrapped in quotes', $parse_state); return FALSE; } @@ -236,13 +456,13 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { if (($context != 'MSGID') && ($context != 'MSGCTXT') && ($context != 'MSGID_PLURAL') && ($context != 'MSGSTR_ARR')) { // Message strings must come after msgid, msgxtxt, msgid_plural, or other msgstr[] entries. - _locale_import_message('The translation file %filename contains an error: "msgstr[]" is unexpected on line %line.', $file, $lineno); + $error_callback('unexpected "msgstr[]"', $parse_state); return FALSE; } // Ensure the plurality is terminated. if (strpos($line, ']') === FALSE) { - _locale_import_message('The translation file %filename contains a syntax error on line %line.', $file, $lineno); + $error_callback('plural variant not properly specified', $parse_state); return FALSE; } @@ -253,10 +473,10 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { // Skip to the next whitespace and trim away any further whitespace, bringing $line to the message data. $line = trim(strstr($line, " ")); - $quoted = _locale_import_parse_quoted($line); + $quoted = gettext_import_parse_quoted($line); if ($quoted === FALSE) { // The string must be quoted. - _locale_import_message('The translation file %filename contains a syntax error on line %line.', $file, $lineno); + $error_callback('plural variant not properly wrapped in quotes', $parse_state); return FALSE; } @@ -269,7 +489,7 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { if (($context != 'MSGID') && ($context != 'MSGCTXT')) { // Strings are only valid within an id or context scope. - _locale_import_message('The translation file %filename contains an error: "msgstr" is unexpected on line %line.', $file, $lineno); + $error_callback('unexpected "msgstr"', $parse_state); return FALSE; } @@ -277,10 +497,10 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { $line = trim(substr($line, 6)); // At this point, $line should now contain the message. - $quoted = _locale_import_parse_quoted($line); + $quoted = gettext_import_parse_quoted($line); if ($quoted === FALSE) { // The string must be quoted. - _locale_import_message('The translation file %filename contains a syntax error on line %line.', $file, $lineno); + $error_callback('"msgstr" not properly wrapped in quotes', $parse_state); return FALSE; } @@ -291,10 +511,10 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { elseif ($line != '') { // Anything that is not a token may be a continuation of a previous token. - $quoted = _locale_import_parse_quoted($line); + $quoted = gettext_import_parse_quoted($line); if ($quoted === FALSE) { // The string must be quoted. - _locale_import_message('The translation file %filename contains a syntax error on line %line.', $file, $lineno); + $error_callback('unexpected data', $parse_state); return FALSE; } @@ -313,7 +533,7 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { } else { // No valid context to append to. - _locale_import_message('The translation file %filename contains an error: there is an unexpected string on line %line.', $file, $lineno); + $error_callback('unexpected string', $parse_state); return FALSE; } } @@ -321,249 +541,49 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL) { // End of PO file, closed out the last entry. if (($context == 'MSGSTR') || ($context == 'MSGSTR_ARR')) { - _locale_import_one_string($op, $current, $mode, $lang, $file); + $write_callback($current, $parse_state, $write_options, $error_callback); } elseif ($context != 'COMMENT') { - _locale_import_message('The translation file %filename ended unexpectedly at line %line.', $file, $lineno); + $error_callback('data ended unexpectedly', $parse_state); return FALSE; } -} -/** - * Sets an error message occurred during locale file parsing. - * - * @param $message - * The message to be translated. - * @param $file - * Drupal file object corresponding to the PO file to import. - * @param $lineno - * An optional line number argument. - */ -function _locale_import_message($message, $file, $lineno = NULL) { - $vars = array('%filename' => $file->filename); - if (isset($lineno)) { - $vars['%line'] = $lineno; - } - $t = get_t(); - drupal_set_message($t($message, $vars), 'error'); + return TRUE; } -/** - * Imports a string into the database - * - * @param $op - * Operation to perform: 'db-store', 'db-report', 'mem-store' or 'mem-report'. - * @param $value - * Details of the string stored. - * @param $mode - * Should existing translations be replaced LOCALE_IMPORT_KEEP or - * LOCALE_IMPORT_OVERWRITE. - * @param $lang - * Language to store the string in. - * @param $file - * Object representation of file being imported, only required when op is - * 'db-store'. - */ -function _locale_import_one_string($op, $value = NULL, $mode = NULL, $lang = NULL, $file = NULL) { - $report = &drupal_static(__FUNCTION__, array('additions' => 0, 'updates' => 0, 'deletes' => 0, 'skips' => 0)); - $header_done = &drupal_static(__FUNCTION__ . ':header_done', FALSE); - $strings = &drupal_static(__FUNCTION__ . ':strings', array()); - - switch ($op) { - // Return stored strings - case 'mem-report': - return $strings; - - // Store string in memory (only supports single strings) - case 'mem-store': - $strings[isset($value['msgctxt']) ? $value['msgctxt'] : ''][$value['msgid']] = $value['msgstr']; - return; - - // Called at end of import to inform the user - case 'db-report': - return array($header_done, $report['additions'], $report['updates'], $report['deletes'], $report['skips']); - - // Store the string we got in the database. - case 'db-store': - // We got header information. - if ($value['msgid'] == '') { - $languages = language_list(); - if (($mode != LOCALE_IMPORT_KEEP) || empty($languages[$lang]->plurals)) { - // Since we only need to parse the header if we ought to update the - // plural formula, only run this if we don't need to keep existing - // data untouched or if we don't have an existing plural formula. - $header = _locale_import_parse_header($value['msgstr']); - - // Get the plural formula and update in database. - if (isset($header["Plural-Forms"]) && $p = _locale_import_parse_plural_forms($header["Plural-Forms"], $file->uri)) { - list($nplurals, $plural) = $p; - db_update('languages') - ->fields(array( - 'plurals' => $nplurals, - 'formula' => $plural, - )) - ->condition('language', $lang) - ->execute(); - } - else { - db_update('languages') - ->fields(array( - 'plurals' => 0, - 'formula' => '', - )) - ->condition('language', $lang) - ->execute(); - } - } - $header_done = TRUE; - } - - else { - // Some real string to import. - $comments = _locale_import_shorten_comments(empty($value['#']) ? array() : $value['#']); - - if (strpos($value['msgid'], "\0")) { - // This string has plural versions. - $english = explode("\0", $value['msgid'], 2); - $entries = array_keys($value['msgstr']); - for ($i = 3; $i <= count($entries); $i++) { - $english[] = $english[1]; - } - $translation = array_map('_locale_import_append_plural', $value['msgstr'], $entries); - $english = array_map('_locale_import_append_plural', $english, $entries); - foreach ($translation as $key => $trans) { - if ($key == 0) { - $plid = 0; - } - $plid = _locale_import_one_string_db($report, $lang, isset($value['msgctxt']) ? $value['msgctxt'] : '', $english[$key], $trans, $comments, $mode, $plid, $key); - } - } - - else { - // A simple string to import. - $english = $value['msgid']; - $translation = $value['msgstr']; - _locale_import_one_string_db($report, $lang, isset($value['msgctxt']) ? $value['msgctxt'] : '', $english, $translation, $comments, $mode); - } - } - } // end of db-store operation -} +// == Utility functions for components of .po parsing ========================== /** - * Import one string into the database. - * - * @param $report - * Report array summarizing the number of changes done in the form: - * array(inserts, updates, deletes). - * @param $langcode - * Language code to import string into. - * @param $context - * The context of this string. - * @param $source - * Source string. - * @param $translation - * Translation to language specified in $langcode. - * @param $location - * Location value to save with source string. - * @param $mode - * Import mode to use, LOCALE_IMPORT_KEEP or LOCALE_IMPORT_OVERWRITE. - * @param $plid - * Optional plural ID to use. - * @param $plural - * Optional plural value to use. + * Parses a string in quotes. * + * @param $string + * A string specified with enclosing quotes. * @return - * The string ID of the existing string modified or the new string added. + * The string parsed from inside the quotes. */ -function _locale_import_one_string_db(&$report, $langcode, $context, $source, $translation, $location, $mode, $plid = 0, $plural = 0) { - $lid = db_query("SELECT lid FROM {locales_source} WHERE source = :source AND context = :context", array(':source' => $source, ':context' => $context))->fetchField(); - - if (!empty($translation)) { - // Skip this string unless it passes a check for dangerous code. - if (!locale_string_is_safe($translation)) { - $report['skips']++; - $lid = 0; - } - elseif ($lid) { - // We have this source string saved already. - db_update('locales_source') - ->fields(array( - 'location' => $location, - )) - ->condition('lid', $lid) - ->execute(); - - $exists = db_query("SELECT COUNT(lid) FROM {locales_target} WHERE lid = :lid AND language = :language", array(':lid' => $lid, ':language' => $langcode))->fetchField(); - - if (!$exists) { - // No translation in this language. - db_insert('locales_target') - ->fields(array( - 'lid' => $lid, - 'language' => $langcode, - 'translation' => $translation, - 'plid' => $plid, - 'plural' => $plural, - )) - ->execute(); - - $report['additions']++; - } - elseif ($mode == LOCALE_IMPORT_OVERWRITE) { - // Translation exists, only overwrite if instructed. - db_update('locales_target') - ->fields(array( - 'translation' => $translation, - 'plid' => $plid, - 'plural' => $plural, - )) - ->condition('language', $langcode) - ->condition('lid', $lid) - ->execute(); - - $report['updates']++; - } - } - else { - // No such source string in the database yet. - $lid = db_insert('locales_source') - ->fields(array( - 'location' => $location, - 'source' => $source, - 'context' => (string) $context, - )) - ->execute(); - - db_insert('locales_target') - ->fields(array( - 'lid' => $lid, - 'language' => $langcode, - 'translation' => $translation, - 'plid' => $plid, - 'plural' => $plural - )) - ->execute(); - - $report['additions']++; - } +function gettext_import_parse_quoted($string) { + if (substr($string, 0, 1) != substr($string, -1, 1)) { + // Start and end quotes must be the same. + return FALSE; } - elseif ($mode == LOCALE_IMPORT_OVERWRITE) { - // Empty translation, remove existing if instructed. - db_delete('locales_target') - ->condition('language', $langcode) - ->condition('lid', $lid) - ->condition('plid', $plid) - ->condition('plural', $plural) - ->execute(); - - $report['deletes']++; + $quote = substr($string, 0, 1); + $string = substr($string, 1, -1); + if ($quote == '"') { + // Double quotes: strip slashes. + return stripcslashes($string); + } + elseif ($quote == "'") { + // Simple quote: return as-is. + return $string; + } + else { + // Unrecognized quote. + return FALSE; } - - return $lid; } /** - * Parses a Gettext Portable Object file header + * Parses a Gettext Portable Object file header. * * @param $header * A string containing the complete header. @@ -571,7 +591,7 @@ function _locale_import_one_string_db(&$report, $langcode, $context, $source, $t * @return * An associative array of key-value pairs. */ -function _locale_import_parse_header($header) { +function gettext_import_parse_header($header) { $header_parsed = array(); $lines = array_map('trim', explode("\n", $header)); foreach ($lines as $line) { @@ -584,18 +604,20 @@ function _locale_import_parse_header($header) { } /** - * Parses a Plural-Forms entry from a Gettext Portable Object file header + * Parses a Plural-Forms entry from a Gettext Portable Object header. * * @param $pluralforms * A string containing the Plural-Forms entry. - * @param $filepath - * A string containing the filepath. + * @param $parse_state + * The current parser state. + * @param $error_callback + * Callback to invoke if we find an error parsing the plural formula. * * @return * An array containing the number of plurals and a * formula in PHP for computing the plural form. */ -function _locale_import_parse_plural_forms($pluralforms, $filepath) { +function gettext_import_parse_plural_forms($pluralforms, &$parse_state, $error_callback) { // First, delete all whitespace $pluralforms = strtr($pluralforms, array(" " => "", "\t" => "")); @@ -616,19 +638,19 @@ function _locale_import_parse_plural_forms($pluralforms, $filepath) { } // Get PHP version of the plural formula - $plural = _locale_import_parse_arithmetic($plural); + $plural = gettext_import_parse_arithmetic($plural); if ($plural !== FALSE) { return array($nplurals, $plural); } else { - drupal_set_message(t('The translation file %filepath contains an error: the plural formula could not be parsed.', array('%filepath' => $filepath)), 'error'); + $error_callback('plural formula could not be parsed', $parse_state); return FALSE; } } /** - * Parses and sanitizes an arithmetic formula into a PHP expression + * Parses and sanitizes an arithmetic formula into a PHP expression. * * While parsing, we ensure, that the operators have the right * precedence and associativity. @@ -639,13 +661,13 @@ function _locale_import_parse_plural_forms($pluralforms, $filepath) { * @return * The PHP version of the formula. */ -function _locale_import_parse_arithmetic($string) { +function gettext_import_parse_arithmetic($string) { // Operator precedence table $precedence = array("(" => -1, ")" => -1, "?" => 1, ":" => 1, "||" => 3, "&&" => 4, "==" => 5, "!=" => 5, "<" => 6, ">" => 6, "<=" => 6, ">=" => 6, "+" => 7, "-" => 7, "*" => 8, "/" => 8, "%" => 8); // Right associativity $right_associativity = array("?" => 1, ":" => 1); - $tokens = _locale_import_tokenize_formula($string); + $tokens = gettext_import_tokenize_formula($string); // Parse by converting into infix notation then back into postfix // Operator stack - holds math operators and symbols @@ -731,7 +753,7 @@ function _locale_import_parse_arithmetic($string) { } /** - * Backward compatible implementation of token_get_all() for formula parsing + * Backward compatible implementation of token_get_all() for formula parsing. * * @param $string * A string containing the arithmetic formula. @@ -739,7 +761,7 @@ function _locale_import_parse_arithmetic($string) { * @return * The PHP version of the formula. */ -function _locale_import_tokenize_formula($formula) { +function gettext_import_tokenize_formula($formula) { $formula = str_replace(" ", "", $formula); $tokens = array(); for ($i = 0; $i < strlen($formula); $i++) { @@ -796,16 +818,16 @@ function _locale_import_tokenize_formula($formula) { } /** - * Modify a string to contain proper count indices + * Modify a string to contain proper count indices. * - * This is a callback function used via array_map() + * This is a callback function used via array_map(). * * @param $entry * An array element. * @param $key * Index of the array element. */ -function _locale_import_append_plural($entry, $key) { +function gettext_import_append_plural($entry, $key) { // No modifications for 0, 1 if ($key == 0 || $key == 1) { return $entry; @@ -817,7 +839,7 @@ function _locale_import_append_plural($entry, $key) { } /** - * Generate a short, one string version of the passed comment array + * Generate a short, one string version of the passed comment array. * * @param $comment * An array of strings containing a comment. @@ -825,7 +847,7 @@ function _locale_import_append_plural($entry, $key) { * @return * Short one string version of the comment. */ -function _locale_import_shorten_comments($comment) { +function gettext_import_shorten_comments($comment) { $comm = ''; while (count($comment)) { $test = $comm . substr(array_shift($comment), 1) . ', '; @@ -840,32 +862,6 @@ function _locale_import_shorten_comments($comment) { } /** - * Parses a string in quotes - * - * @param $string - * A string specified with enclosing quotes. - * - * @return - * The string parsed from inside the quotes. - */ -function _locale_import_parse_quoted($string) { - if (substr($string, 0, 1) != substr($string, -1, 1)) { - return FALSE; // Start and end quotes must be the same - } - $quote = substr($string, 0, 1); - $string = substr($string, 1, -1); - if ($quote == '"') { // Double quotes: strip slashes - return stripcslashes($string); - } - elseif ($quote == "'") { // Simple quote: return as-is - return $string; - } - else { - return FALSE; // Unrecognized quote - } -} - -/** * Generates a structured array of all strings with translations in * $language, if given. This array can be used to generate an export * of the string in the database. diff --git a/includes/install.inc b/includes/install.inc index e52c0ad..8081e25 100644 --- a/includes/install.inc +++ b/includes/install.inc @@ -1109,10 +1109,25 @@ function st($string, array $args = array(), array $options = array()) { $po_files = file_scan_directory('./profiles/' . $install_state['parameters']['profile'] . '/translations', '/'. $install_state['parameters']['locale'] .'\.po$/', array('recurse' => FALSE)); if (count($po_files)) { require_once DRUPAL_ROOT . '/includes/gettext.inc'; - foreach ($po_files as $po_file) { - _locale_import_read_po('mem-store', $po_file); + $locale_strings = array(); + foreach ($po_files as $file) { + $parse_state = array( + 'file' => $file->uri, + 'filesize' => filesize($file->uri), + ); + // Pass on environment specific write options. + $write_options = array( + 'langcode' => $install_state['parameters']['locale'], + 'mode' => LOCALE_IMPORT_KEEP, + 'storage' => 'memory', + ); + $success = gettext_import_read_po( + 'gettext_import_read_callback', $parse_state, + 'gettext_import_write_callback', $write_options, + 'gettext_import_error_callback' + ); + $locale_strings = array_merge_recursive($locale_strings, $parse_state['write_state']['strings']); } - $locale_strings = _locale_import_one_string('mem-report'); } } } diff --git a/modules/locale/locale.bulk.inc b/modules/locale/locale.bulk.inc index 51f80a6..d55dc1d 100644 --- a/modules/locale/locale.bulk.inc +++ b/modules/locale/locale.bulk.inc @@ -77,21 +77,24 @@ function locale_translate_import_form_submit($form, &$form_state) { drupal_set_message(t('The language %language has been created.', array('%language' => t($predefined[$langcode][0])))); } - // Now import strings into the language - if ($return = _locale_import_po($file, $langcode, $form_state['values']['mode']) == FALSE) { - $variables = array('%filename' => $file->filename); - drupal_set_message(t('The translation import of %filename failed.', $variables), 'error'); - watchdog('locale', 'The translation import of %filename failed.', $variables, WATCHDOG_ERROR); - } + // Set up a batch to import this file in smaller chunks. + $batch = array( + 'title' => t('Importing translations'), + 'operations' => array( + array('locale_import_batch_op', array($file, $langcode, $form_state['values']['mode'])), + ), + 'finished' => 'locale_import_batch_finished', + 'file' => drupal_get_path('module', 'locale') . '/locale.bulk.inc', + ); + batch_set($batch); + + // Batch API will return here when the batch processing is done. + $form_state['redirect'] = 'admin/config/regional/translate'; } else { drupal_set_message(t('File to import not found.'), 'error'); $form_state['redirect'] = 'admin/config/regional/translate/import'; - return; } - - $form_state['redirect'] = 'admin/config/regional/translate'; - return; } /** @@ -168,8 +171,6 @@ function locale_translate_export_po_form_submit($form, &$form_state) { * * @param $langcode * Language code to import translations for. - * @param $finished - * Optional finished callback for the batch. * @param $skip * Array of component names to skip. Used in the installer for the * second pass import, when most components are already imported. @@ -177,7 +178,7 @@ function locale_translate_export_po_form_submit($form, &$form_state) { * @return * A batch structure or FALSE if no files found. */ -function locale_batch_by_language($langcode, $finished = NULL, $skip = array()) { +function locale_import_batch_by_language($langcode, $skip = array()) { // Collect all files to import for all enabled modules and themes. $files = array(); $components = array(); @@ -197,7 +198,7 @@ function locale_batch_by_language($langcode, $finished = NULL, $skip = array()) $components[] = $component->name; } - return _locale_batch_build($files, $finished, $components); + return locale_import_batch_build($files, $components); } /** @@ -209,10 +210,8 @@ function locale_batch_by_language($langcode, $finished = NULL, $skip = array()) * @param $components * An array of component (theme and/or module) names to import * translations for. - * @param $finished - * Optional finished callback for the batch. */ -function locale_batch_by_component($components, $finished = '_locale_batch_system_finished') { +function locale_import_batch_by_component($components) { $files = array(); $languages = language_list('enabled'); unset($languages[1]['en']); @@ -229,7 +228,7 @@ function locale_batch_by_component($components, $finished = '_locale_batch_syste $files = array_merge($files, file_scan_directory(dirname($component->filename) . '/translations', '/(^|\.)(' . $language_list . ')\.po$/', array('recurse' => FALSE))); } } - return _locale_batch_build($files, $finished); + return locale_import_batch_build($files); } return FALSE; } @@ -239,21 +238,21 @@ function locale_batch_by_component($components, $finished = '_locale_batch_syste * * @param $files * Array of files to import. - * @param $finished - * Optional finished callback for the batch. * @param $components * Optional list of component names the batch covers. Used in the installer. * * @return * A batch structure. */ -function _locale_batch_build($files, $finished = NULL, $components = array()) { +function locale_import_batch_build($files, $components = array()) { $t = get_t(); if (count($files)) { $operations = array(); foreach ($files as $file) { - // We call _locale_batch_import for every batch operation. - $operations[] = array('_locale_batch_import', array($file->uri)); + if (preg_match('!(/|\.)([^\./]+)\.po$!', $file->uri, $file_match)) { + // We call locale_import_batch_op for every batch operation. + $operations[] = array('locale_import_batch_op', array($file, $file_match[2], LOCALE_IMPORT_KEEP)); + } } $batch = array( 'operations' => $operations, @@ -261,13 +260,11 @@ function _locale_batch_build($files, $finished = NULL, $components = array()) { 'init_message' => $t('Starting import'), 'error_message' => $t('Error importing interface translations'), 'file' => drupal_get_path('module', 'locale') . '/locale.bulk.inc', + 'finished' => 'locale_import_batch_finished', // This is not a batch API construct, but data passed along to the // installer, so we know what did we import already. '#components' => $components, ); - if (isset($finished)) { - $batch['finished'] = $finished; - } return $batch; } return FALSE; @@ -276,37 +273,101 @@ function _locale_batch_build($files, $finished = NULL, $components = array()) { /** * Perform interface translation import as a batch step. * - * @param $filepath - * Path to a file to import. - * @param $results - * Contains a list of files imported. + * @param $file + * File object for .po file to import. + * @param $langcode + * Language code to import file into. + * @param $mode + * One of the LOCALE_IMPORT_* constants for import mode. + * @param $context + * Batch context. */ -function _locale_batch_import($filepath, &$context) { - // The filename is either {langcode}.po or {prefix}.{langcode}.po, so - // we can extract the language code to use for the import from the end. - if (preg_match('!(/|\.)([^\./]+)\.po$!', $filepath, $langcode)) { - $file = (object) array('filename' => basename($filepath), 'uri' => $filepath); - _locale_import_read_po('db-store', $file, LOCALE_IMPORT_KEEP, $langcode[2]); - $context['results'][] = $filepath; +function locale_import_batch_op($file, $langcode, $mode, &$context) { + if (empty($context['sandbox'])) { + // Initialize sandbox with file URI, size and chunk size for batch reading. + $context['sandbox']['parse_state'] = array( + 'file' => $file->uri, + 'filesize' => filesize($file->uri), + 'chunk_size' => 1000, + ); } -} -/** - * Finished callback of system page locale import batch. - * Inform the user of translation files imported. - */ -function _locale_batch_system_finished($success, $results) { - if ($success) { - drupal_set_message(format_plural(count($results), 'One translation file imported for the newly installed modules.', '@count translation files imported for the newly installed modules.')); + // Pass on environment specific write options. + $write_options = array( + 'langcode' => $langcode, + 'mode' => $mode, + ); + + // Execute one reading operation, which will process a bit more than 1000 + // lines at once with the above configuration. + $success = gettext_import_read_po( + 'gettext_import_read_callback', $context['sandbox']['parse_state'], + 'gettext_import_write_callback', $write_options, + 'gettext_import_error_callback' + ); + + if (!$context['sandbox']['parse_state']['finished']) { + // Not yet finished with reading, mark progress based on size and position. + // Maximize the progress bar at 95% before completion because batch API + // could trigger end of operation before file reading is done for floating + // point inaccuracies. See http://drupal.org/node/1089472 + $context['finished'] = min(0.95, ($context['sandbox']['parse_state']['fseek'] / $context['sandbox']['parse_state']['filesize'])); + } + else { + // Finished in this case. Mark this for the batch to go to the next operation. + $context['finished'] = 1; + + if ($success) { + // If the file reading ended successfully, inform the user here. We do + // not let this to the batch end function because we might handle any + // number of .po files in one batch set, and therefore we should do + // messaging right when we end each file. + $stats = $context['sandbox']['parse_state']['write_stats']; + drupal_set_message(t('%file successfully imported. There are %number newly created translated strings, %update strings were updated and %delete strings were removed.', array('%file' => $file->uri, '%number' => $stats['added'], '%update' => $stats['updated'], '%delete' => $stats['deleted']))); + watchdog('locale', 'Imported %file: %number new strings added, %update updated and %delete removed.', array('%file' => $file->uri, '%number' => $stats['added'], '%update' => $stats['updated'], '%delete' => $stats['deleted'])); + } + + if (empty($context['sandbox']['parse_state']['header'])) { + // Inform the user about the missing header. + drupal_set_message(t('%file header missing.', array('%file' => $file->uri)), 'warning'); + } } + + // No need to do anything special if the file was not parsed successfully, the + // error message was already communicated to the user, we don't need yet + // another one. + + // The finish callback only needs the affected languages for cache clearing. + $context['results']['languages_affected'][$write_options['langcode']] = TRUE; + + // Inform the user about our progress in the file textually. + $context['message'] = t('Processing %file from line @line.', array('%file' => $file->uri, '@line' => $context['sandbox']['parse_state']['lineno'])); } /** - * Finished callback of language addition locale import batch. - * Inform the user of translation files imported. + * Batch finish callback. Inform the user about the state of the import. */ -function _locale_batch_language_finished($success, $results) { +function locale_import_batch_finished($success, $results, $operations) { + if ($success) { - drupal_set_message(format_plural(count($results), 'One translation file imported for the enabled modules.', '@count translation files imported for the enabled modules.')); + // Clear cache and force refresh of JavaScript translations. + foreach ($results['languages_affected'] as $langcode => $true) { + _locale_invalidate_js($langcode); + } + // Clear all locale translation caches. + cache_clear_all('locale:', 'cache', TRUE); + // Rebuild the menu, strings may have changed. + menu_rebuild(); + } + + else { + // Pick the operation with the error from the list of remaining operations. + // Report the error to the user and the system log. + $error_operation = reset($operations); + $operation = array_shift($error_operation); + $arguments = array_shift($error_operation); + $arguments_as_string = implode(', ', $arguments); + watchdog('gettextapi', "Error when running operation %operation(%arguments)", array('%operation' => $operation, '%arguments' => $arguments_as_string)); + drupal_set_message(t('An error occured while importing translations. It was recorded in the system log.'), 'error'); } }