diff --git a/core/includes/gettext.inc b/core/includes/gettext.inc index a8498dc..edb61a4 100644 --- a/core/includes/gettext.inc +++ b/core/includes/gettext.inc @@ -6,7 +6,28 @@ * * @todo Decouple these functions from Locale API and put to gettext_ namespace. */ - +// NOTE: this code is based on Sutharsan version from +// http://drupal.org/node/1189184#comment-5776970 +// It's current purpose is to make the testbot happy and clemens lazy + +use Drupal\Core\Gettext\PoDbReader; +use Drupal\Core\Gettext\PoDbWriter; +use Drupal\Core\Gettext\PoFileReader; +use Drupal\Core\Gettext\PoFileWriter; +use Drupal\Core\Gettext\POMemoryWriter; + +function _locale_files_to_memory($langcode, $files) { + $writer = new POMemoryWriter(); + $writer->setLangcode($langcode); + foreach ($files as $file) { + $reader = new PoFileReader(); + $reader->setURI($file->uri); + $reader->setLangcode($langcode); + $reader->open(); + $writer->writeItems($reader, -1); + } + return $writer->getData(); +} /** * @defgroup locale-api-import-export Translation import/export API. * @{ @@ -32,48 +53,92 @@ * Use LOCALE_CUSTOMIZED or LOCALE_NOT_CUSTOMIZED. All strings in the file * will be saved with this customization flag. */ -function _locale_import_po($file, $langcode, $overwrite_options, $customized = LOCALE_NOT_CUSTOMIZED) { +function _locale_import_po($file, $langcode, $overwrite_options, $customized = LOCALE_NOT_CUSTOMIZED, $is_batch = FALSE) { // Try to allocate enough time to parse and import the data. drupal_set_time_limit(240); + //TODO: why not check when having a batch + //@see locale_translate_batch_import which calls _locale_import_read_po direct + // // Check if we have the language already in the database. - if (!language_load($langcode)) { - drupal_set_message(t('The language selected for import is not supported.'), 'error'); - return FALSE; + if (!$is_batch) { + if (!language_load($langcode)) { + drupal_set_message(t('The language selected for import is not supported.'), 'error'); + return FALSE; + } } - // Get strings from file (returns on failure after a partial import, or on success) - $status = _locale_import_read_po('db-store', $file, $overwrite_options, $langcode, $customized); - if ($status === FALSE) { - // Error messages are set in _locale_import_read_po(). - return FALSE; - } + $reader = new PoFileReader(); + $reader->setLangcode($langcode); + $reader->setURI($file->uri); - // Get status information on import process. - list($header_done, $additions, $updates, $deletes, $skips) = _locale_import_one_string('db-report'); + // ADD try|catch + // When opening header is parsed immediately + $reader->open(); + + // Add exception when file is not readably: @see _locale_import_read_po() + // + // Better testing for malformed header. + // @see + // - _locale_import_one_string() + // - _locale_import_parse_header(); + // - _locale_import_parse_plural_forms(); + $header = $reader->getHeader(); + if (!$is_batch) { + if (!$header) { + drupal_set_message(t('The translation file %filename appears to have a missing or malformed header.', array('%filename' => $file->filename)), 'error'); + } + } - if (!$header_done) { - drupal_set_message(t('The translation file %filename appears to have a missing or malformed header.', array('%filename' => $file->filename)), 'error'); + $writer = new PODbWriter(); + $writer->setLangcode($langcode); + $writer->setHeader($header); + $options = array( + 'overwrite_options' => $overwrite_options, + 'customized' => $customized, + ); + $writer->setOptions($options); + + try { + // TODO: make sure exception occurs + // Get strings from file (returns on failure after a partial import, or on success) + // $status = _locale_import_read_po('db-store', $file, $overwrite_options, $langcode, $customized); + // if ($status === FALSE) { + // // Error messages are set in _locale_import_read_po(). + // return FALSE; + // } + $writer->writeItems($reader, -1); + } + catch (Exception $exc) { + return FALSE; } - // Clear cache and force refresh of JavaScript translations. - _locale_invalidate_js($langcode); - cache()->deletePrefix('locale:'); + $report = $writer->getReport(); + $additions = $report['additions']; + $updates = $report['updates']; + $deletes = $report['deletes']; + $skips = $report['skips']; - // Rebuild the menu, strings may have changed. - menu_rebuild(); + if (!$is_batch) { + // Clear cache and force refresh of JavaScript translations. + _locale_invalidate_js($langcode); + cache()->deletePrefix('locale:'); - drupal_set_message(t('The translation was successfully imported. There are %number newly created translated strings, %update strings were updated and %delete strings were removed.', array('%number' => $additions, '%update' => $updates, '%delete' => $deletes))); - watchdog('locale', 'Imported %file into %locale: %number new strings added, %update updated and %delete removed.', array('%file' => $file->filename, '%locale' => $langcode, '%number' => $additions, '%update' => $updates, '%delete' => $deletes)); - if ($skips) { - if (module_exists('dblog')) { - $skip_message = format_plural($skips, 'A translation string was skipped because of disallowed or malformed HTML. See the log for details.', '@count translation strings were skipped because of disallowed or malformed HTML. See the log for details.', array('@url' => url('admin/reports/dblog'))); - } - else { - $skip_message = format_plural($skips, 'A translation string was skipped because of disallowed or malformed HTML. See the log for details.', '@count translation strings were skipped because of disallowed or malformed HTML. See the log for details.'); + // Rebuild the menu, strings may have changed. + menu_rebuild(); + + drupal_set_message(t('The translation was successfully imported. There are %number newly created translated strings, %update strings were updated and %delete strings were removed.', array('%number' => $additions, '%update' => $updates, '%delete' => $deletes))); + watchdog('locale', 'Imported %file into %locale: %number new strings added, %update updated and %delete removed.', array('%file' => $file->filename, '%locale' => $langcode, '%number' => $additions, '%update' => $updates, '%delete' => $deletes)); + if ($skips) { + if (module_exists('dblog')) { + $skip_message = format_plural($skips, 'A translation string was skipped because of disallowed or malformed HTML. See the log for details.', '@count translation strings were skipped because of disallowed or malformed HTML. See the log for details.', array('@url' => url('admin/reports/dblog'))); + } + else { + $skip_message = format_plural($skips, 'A translation string was skipped because of disallowed or malformed HTML. See the log for details.', '@count translation strings were skipped because of disallowed or malformed HTML. See the log for details.'); + } + drupal_set_message($skip_message, 'error'); + watchdog('locale', '@count disallowed HTML string(s) in %file', array('@count' => $skips, '%file' => $file->uri), WATCHDOG_WARNING); } - drupal_set_message($skip_message, 'error'); - watchdog('locale', '@count disallowed HTML string(s) in %file', array('@count' => $skips, '%file' => $file->uri), WATCHDOG_WARNING); } return TRUE; } @@ -150,8 +215,8 @@ function _locale_import_read_po($op, $file, $overwrite_options = NULL, $lang = N _locale_import_one_string($op, $current, $overwrite_options, $lang, $file, $customized); // Start a new entry for the comment. - $current = array(); - $current['#'][] = substr($line, 1); + $current = array(); + $current['#'][] = substr($line, 1); $context = 'COMMENT'; } @@ -426,7 +491,6 @@ function _locale_import_one_string($op, $value = NULL, $overwrite_options = NULL } $header_done = TRUE; } - else { // Found a string to store, clean up and prepare the data. $comments = _locale_import_shorten_comments(empty($value['#']) ? array() : $value['#']); @@ -439,14 +503,7 @@ function _locale_import_one_string($op, $value = NULL, $overwrite_options = NULL } _locale_import_one_string_db( - $report, - $lang, - isset($value['msgctxt']) ? $value['msgctxt'] : '', - $value['msgid'], - $value['msgstr'], - $comments, - $overwrite_options, - $customized + $report, $lang, isset($value['msgctxt']) ? $value['msgctxt'] : '', $value['msgid'], $value['msgstr'], $comments, $overwrite_options, $customized ); } } // end of db-store operation @@ -492,8 +549,8 @@ function _locale_import_one_string_db(&$report, $langcode, $context, $source, $t ':source' => $source, ':context' => $context, ':language' => $langcode, - )) - ->fetchObject(); + )) + ->fetchObject(); if (!empty($translation)) { // Skip this string unless it passes a check for dangerous code. @@ -505,35 +562,35 @@ function _locale_import_one_string_db(&$report, $langcode, $context, $source, $t elseif (isset($string->lid)) { // We have this source string saved already. db_update('locales_source') - ->fields(array( - 'location' => $location, - )) - ->condition('lid', $string->lid) - ->execute(); + ->fields(array( + 'location' => $location, + )) + ->condition('lid', $string->lid) + ->execute(); if (!isset($string->customized)) { // No translation in this language. db_insert('locales_target') - ->fields(array( - 'lid' => $string->lid, - 'language' => $langcode, - 'translation' => $translation, - 'customized' => $customized, - )) - ->execute(); + ->fields(array( + 'lid' => $string->lid, + 'language' => $langcode, + 'translation' => $translation, + 'customized' => $customized, + )) + ->execute(); $report['additions']++; } elseif ($overwrite_options[$string->customized ? 'customized' : 'not_customized']) { // Translation exists, only overwrite if instructed. db_update('locales_target') - ->fields(array( - 'translation' => $translation, - 'customized' => $customized, - )) - ->condition('language', $langcode) - ->condition('lid', $string->lid) - ->execute(); + ->fields(array( + 'translation' => $translation, + 'customized' => $customized, + )) + ->condition('language', $langcode) + ->condition('lid', $string->lid) + ->execute(); $report['updates']++; } @@ -542,21 +599,21 @@ function _locale_import_one_string_db(&$report, $langcode, $context, $source, $t else { // No such source string in the database yet. $lid = db_insert('locales_source') - ->fields(array( - 'location' => $location, - 'source' => $source, - 'context' => (string) $context, - )) - ->execute(); + ->fields(array( + 'location' => $location, + 'source' => $source, + 'context' => (string) $context, + )) + ->execute(); db_insert('locales_target') - ->fields(array( - 'lid' => $lid, - 'language' => $langcode, - 'translation' => $translation, - 'customized' => $customized, - )) - ->execute(); + ->fields(array( + 'lid' => $lid, + 'language' => $langcode, + 'translation' => $translation, + 'customized' => $customized, + )) + ->execute(); $report['additions']++; return $lid; @@ -565,9 +622,9 @@ function _locale_import_one_string_db(&$report, $langcode, $context, $source, $t elseif (isset($string->lid) && isset($string->customized) && $overwrite_options[$string->customized ? 'customized' : 'not_customized']) { // Empty translation, remove existing if instructed. db_delete('locales_target') - ->condition('language', $langcode) - ->condition('lid', $string->lid) - ->execute(); + ->condition('language', $langcode) + ->condition('lid', $string->lid) + ->execute(); $report['deletes']++; return $string->lid; @@ -575,239 +632,6 @@ function _locale_import_one_string_db(&$report, $langcode, $context, $source, $t } /** - * Parses a Gettext Portable Object file header. - * - * @param $header - * A string containing the complete header. - * - * @return - * An associative array of key-value pairs. - */ -function _locale_import_parse_header($header) { - $header_parsed = array(); - $lines = array_map('trim', explode("\n", $header)); - foreach ($lines as $line) { - if ($line) { - list($tag, $contents) = explode(":", $line, 2); - $header_parsed[trim($tag)] = trim($contents); - } - } - return $header_parsed; -} - -/** - * Parses a Plural-Forms entry from a Gettext Portable Object file header. - * - * @param $pluralforms - * A string containing the Plural-Forms entry. - * @param $filepath - * A string containing the filepath. - * - * @return - * An array containing the number of plurals and a - * formula in PHP for computing the plural form. - */ -function _locale_import_parse_plural_forms($pluralforms, $filepath) { - // First, delete all whitespace - $pluralforms = strtr($pluralforms, array(" " => "", "\t" => "")); - - // Select the parts that define nplurals and plural - $nplurals = strstr($pluralforms, "nplurals="); - if (strpos($nplurals, ";")) { - $nplurals = substr($nplurals, 9, strpos($nplurals, ";") - 9); - } - else { - return FALSE; - } - $plural = strstr($pluralforms, "plural="); - if (strpos($plural, ";")) { - $plural = substr($plural, 7, strpos($plural, ";") - 7); - } - else { - return FALSE; - } - - // Get PHP version of the plural formula - $plural = _locale_import_parse_arithmetic($plural); - - if ($plural !== FALSE) { - return array($nplurals, $plural); - } - else { - drupal_set_message(t('The translation file %filepath contains an error: the plural formula could not be parsed.', array('%filepath' => $filepath)), 'error'); - return FALSE; - } -} - -/** - * Parses and sanitizes an arithmetic formula into a PHP expression. - * - * While parsing, we ensure, that the operators have the right - * precedence and associativity. - * - * @param $string - * A string containing the arithmetic formula. - * - * @return - * The PHP version of the formula. - */ -function _locale_import_parse_arithmetic($string) { - // Operator precedence table - $precedence = array("(" => -1, ")" => -1, "?" => 1, ":" => 1, "||" => 3, "&&" => 4, "==" => 5, "!=" => 5, "<" => 6, ">" => 6, "<=" => 6, ">=" => 6, "+" => 7, "-" => 7, "*" => 8, "/" => 8, "%" => 8); - // Right associativity - $right_associativity = array("?" => 1, ":" => 1); - - $tokens = _locale_import_tokenize_formula($string); - - // Parse by converting into infix notation then back into postfix - // Operator stack - holds math operators and symbols - $operator_stack = array(); - // Element Stack - holds data to be operated on - $element_stack = array(); - - foreach ($tokens as $token) { - $current_token = $token; - - // Numbers and the $n variable are simply pushed into $element_stack - if (is_numeric($token)) { - $element_stack[] = $current_token; - } - elseif ($current_token == "n") { - $element_stack[] = '$n'; - } - elseif ($current_token == "(") { - $operator_stack[] = $current_token; - } - elseif ($current_token == ")") { - $topop = array_pop($operator_stack); - while (isset($topop) && ($topop != "(")) { - $element_stack[] = $topop; - $topop = array_pop($operator_stack); - } - } - elseif (!empty($precedence[$current_token])) { - // If it's an operator, then pop from $operator_stack into $element_stack until the - // precedence in $operator_stack is less than current, then push into $operator_stack - $topop = array_pop($operator_stack); - while (isset($topop) && ($precedence[$topop] >= $precedence[$current_token]) && !(($precedence[$topop] == $precedence[$current_token]) && !empty($right_associativity[$topop]) && !empty($right_associativity[$current_token]))) { - $element_stack[] = $topop; - $topop = array_pop($operator_stack); - } - if ($topop) { - $operator_stack[] = $topop; // Return element to top - } - $operator_stack[] = $current_token; // Parentheses are not needed - } - else { - return FALSE; - } - } - - // Flush operator stack - $topop = array_pop($operator_stack); - while ($topop != NULL) { - $element_stack[] = $topop; - $topop = array_pop($operator_stack); - } - - // Now extract formula from stack - $previous_size = count($element_stack) + 1; - while (count($element_stack) < $previous_size) { - $previous_size = count($element_stack); - for ($i = 2; $i < count($element_stack); $i++) { - $op = $element_stack[$i]; - if (!empty($precedence[$op])) { - $f = ""; - if ($op == ":") { - $f = $element_stack[$i - 2] . "):" . $element_stack[$i - 1] . ")"; - } - elseif ($op == "?") { - $f = "(" . $element_stack[$i - 2] . "?(" . $element_stack[$i - 1]; - } - else { - $f = "(" . $element_stack[$i - 2] . $op . $element_stack[$i - 1] . ")"; - } - array_splice($element_stack, $i - 2, 3, $f); - break; - } - } - } - - // If only one element is left, the number of operators is appropriate - if (count($element_stack) == 1) { - return $element_stack[0]; - } - else { - return FALSE; - } -} - -/** - * Provides backward-compatible formula parsing for token_get_all(). - * - * @param $string - * A string containing the arithmetic formula. - * - * @return - * The PHP version of the formula. - */ -function _locale_import_tokenize_formula($formula) { - $formula = str_replace(" ", "", $formula); - $tokens = array(); - for ($i = 0; $i < strlen($formula); $i++) { - if (is_numeric($formula[$i])) { - $num = $formula[$i]; - $j = $i + 1; - while ($j < strlen($formula) && is_numeric($formula[$j])) { - $num .= $formula[$j]; - $j++; - } - $i = $j - 1; - $tokens[] = $num; - } - elseif ($pos = strpos(" =<>!&|", $formula[$i])) { // We won't have a space - $next = $formula[$i + 1]; - switch ($pos) { - case 1: - case 2: - case 3: - case 4: - if ($next == '=') { - $tokens[] = $formula[$i] . '='; - $i++; - } - else { - $tokens[] = $formula[$i]; - } - break; - case 5: - if ($next == '&') { - $tokens[] = '&&'; - $i++; - } - else { - $tokens[] = $formula[$i]; - } - break; - case 6: - if ($next == '|') { - $tokens[] = '||'; - $i++; - } - else { - $tokens[] = $formula[$i]; - } - break; - } - } - else { - $tokens[] = $formula[$i]; - } - } - return $tokens; -} - -/** * Generates a short, one-string version of the passed comment array. * * @param $comment @@ -926,9 +750,9 @@ function _locale_export_get_strings($language = NULL, $options = array()) { $strings = array(); foreach ($result as $child) { $strings[$child->lid] = array( - 'comment' => $child->location, - 'source' => $child->source, - 'context' => $child->context, + 'comment' => $child->location, + 'source' => $child->source, + 'context' => $child->context, 'translation' => isset($child->translation) ? $child->translation : '', ); } diff --git a/core/includes/gettext.sketch.inc b/core/includes/gettext.sketch.inc new file mode 100644 index 0000000..07ca70f --- /dev/null +++ b/core/includes/gettext.sketch.inc @@ -0,0 +1,195 @@ +supportBatch() && $destination->supportBatch() && $source->size() > $threshold) { + // Transfer data in batches. + // Built and execute batch. + $batch = gettext_transfer_batch_setup($source, $destination); + batch_set($batch); + } + else { + // Transfer all translations without interruption. + // Process the translations one by one to keep the memory footprint low. + while ($translation = $source->read()) { + $destination->write($translation); + } + } +} + +/** + * Set up a batch process to transfer Gettext data. + */ +function gettext_transfer_batch_setup($source, $destination) { + $batch = array( + 'operations' => array( + array('gettext_transfer_batch_op', array($source, $destination)), + ), + 'finished' => 'gettext_transfer_batch_finished', + ); + return $batch; +} + +/** + * Batch operation transferring gettext data. + * + * @todo Add source filter conditions (context, language(?)) as parameter. + */ +function gettext_transfer_batch_op($source, $destination) { + // Initialize sandbox for batch reading. + if (empty($context['sandbox'])) { + $context['sandbox']['transfer'] = array( + 'chunk' => min($source->chunkSize(), $destination->chunkSize()), + ); + } + + // Execute one transfer cycle, which will process bite size 'chunks'. + $success = gettext_transfer_execute($source, $destination, $context['sandbox']['transfer']); + + // After a number of cycles executing gettext_transfer_execute() the transfer + // is completed. + // See gettextapi_import_batch_op() for details. + if ($context['sandbox']['transfer']['finished']) { + $context['finished'] = 1; + + if ($success) { + // Collect statistics from $context['sandbox']['transfer'] + // and tell the world about what great work we did. + } + } +} + +/** + * Batch wrap-up for gettext data transfer. + */ +function gettext_transfer_batch_finished($success, $results, $operations) { + if ($success) { + // Call post processing handler defined by the destination object. + // Can we get the callback and arguments via $results? + $post_process_callback = $results['post_process_callback']; + $arguments = $results['post_process_arguments']; + $post_process_callback($arguments); + + // We did it, Magoo! :) + } + else { + // Sadly reporting where it went wrong. + } +} + +/** + * Post processing for gettext language import into the database. + * + * To be called after successfull (batch) import. + */ +function gettext_post_process_import($langcode) { + // After succesfull import we refresh all affected parts of the system. + _locale_invalidate_js($langcode); + cache_clear_all('locale:', 'cache', TRUE); + menu_rebuild(); +} + +/** + * Transfer gettext data from source to destination in bite size chunks. + */ +function gettext_transfer_execute($source, $destination, &$transfer) { + // If transfer is not yet started we set the header data. + // The header (if supported) is written before the first translation is written. + if (!$destination->inProgress()) { + // Transfer translation header data. + $destination->setMetaData($source->getMetaData(), $langcode); + } + + $chunk = $transfer['chunk']; + // Transfer translations as long as valid data is available and + // the bite size chunk is not yet swallowed. + while ($source->valid() && $chunk > 0) { + // Get one translation from source, write it to destination. + $translation = $source->read(); + $destination->write($translation); + $chunk--; + } + + // Report the percentage of completion for progress reporting. + // Ai, user interface. Why do we bother ;) + $transfer['state']['percentage_of_completion'] = $source->poc(); + + // Close connections when we are done and report the results. + if ($source->finished()) { + $destination->finish(); + $transfer['finished'] = TRUE; + } + + // Report statistics including errors and error log. + $transfer['result'] = $destination->statistics(); +} diff --git a/core/includes/install.inc b/core/includes/install.inc index 72adf1c..62b8fa6 100644 --- a/core/includes/install.inc +++ b/core/includes/install.inc @@ -732,10 +732,7 @@ function st($string, array $args = array(), array $options = array()) { $files = install_find_translation_files($install_state['parameters']['langcode']); if (!empty($files)) { require_once DRUPAL_ROOT . '/core/includes/gettext.inc'; - foreach ($files as $file) { - _locale_import_read_po('mem-store', $file); - } - $strings = _locale_import_one_string('mem-report'); + $strings = _locale_files_to_memory($install_state['parameters']['langcode'], $files); } } } diff --git a/core/lib/Drupal/Core/Gettext/BatchStateInterface.php b/core/lib/Drupal/Core/Gettext/BatchStateInterface.php new file mode 100644 index 0000000..09313ed --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/BatchStateInterface.php @@ -0,0 +1,35 @@ +stream_open($uri, 'r', $options, $opened_url); + $this->_stream = $s; + $state = array( + 'uri' => $uri, + 'mode' => $mode, + 'options' => $options, + 'opened_url' => $opened_url, + ); + $state += $this->_addition_state; + $this->_addition_state = $state; + } + } + + public function getStream() { + return $this->_stream; + } + + public function getBatchState() { + $state = $this->_addition_state; + if ($this->getStream()) { + $state['position'] = $this->getStream()->stream_tell(); + $state['uri'] = $this->getStream()->getURI(); + } + return $state; + } + + public function setBatchState(array $state = array()) { + $current_state = $this->getBatchState(); + $state += $current_state; + $this->_setBatchState($state); + } + + private function _setBatchState($state) { + if ($this->getStream()) { + $this->getStream()->stream_close(); + $this->open($state['uri'], $state['mode'], $state['options'], $opened_url); + $this->getStream()->stream_seek($state['position'], SEEK_SET); + } + return $this->getBatchState(); + } + +} diff --git a/core/lib/Drupal/Core/Gettext/PODbReader.php b/core/lib/Drupal/Core/Gettext/PODbReader.php new file mode 100644 index 0000000..2bcdfa2 --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/PODbReader.php @@ -0,0 +1,184 @@ +setOptions(array()); + } + + public function getLangcode() { + return $this->_langcode; + } + + public function setLangcode($langcode) { + $this->_langcode = $langcode; + } + + function getOptions() { + return $this->_options; + } + + function setOptions(array $options) { + if (!isset($options['override_options'])) { + $options['override_options'] = array(); + } + if (!isset($options['customized'])) { + $options['customized'] = LOCALE_NOT_CUSTOMIZED; + } + $this->_options = array( + 'override_options' => $options['override_options'], + 'customized' => $options['customized'], + ); + } + + function setState(array $state) { + $this->_lid = $state['lid']; + $this->setOptions($state['options']); + $this->buildQuery(); + } + + function getState() { + return array( + '__CLASS__' => __CLASS__, + 'lid' => $this->_lid, + 'options' => $this->_options, + ); + } + + function getHeader() { + return new POHeader($this->getLangcode()); + } + + public function setHeader(POHeader $header) { + // empty on purpose + } + + /** + * Generates a structured array of all translated strings for the language. + * + * @param $language + * Language object to generate the output for, or NULL if generating + * translation template. + * @param $options + * (optional) An associative array specifying what to include in the output: + * - customized: include customized strings (if TRUE) + * - uncustomized: include non-customized string (if TRUE) + * - untranslated: include untranslated source strings (if TRUE) + * Ignored if $language is NULL. + * + * @return + * An array of translated strings that can be used to generate an export. + */ + private function buildQuery() { + $langcode = $this->_langcode; + $options = $this->_options; + + // Assume FALSE for all options if not provided by the API. + $options += array( + 'customized' => FALSE, + 'not_customized' => FALSE, + 'not_translated' => FALSE, + ); + if (array_sum($options) == 0) { + // If user asked to not include anything in the translation files, + // that would not make sense, so just fall back on providing a template. + $language = NULL; + } + + // Build and execute query to collect source strings and translations. + $query = db_select('locales_source', 's'); + if (!empty($language)) { + if ($options['not_translated']) { + // Left join to keep untranslated strings in. + $query->leftJoin('locales_target', 't', 's.lid = t.lid AND t.language = :language', array(':language' => $langcode)); + } + else { + // Inner join to filter for only translations. + $query->innerJoin('locales_target', 't', 's.lid = t.lid AND t.language = :language', array(':language' => $langcode)); + } + if ($options['customized']) { + if (!$options['not_customized']) { + // Filter for customized strings only. + $query->condition('t.customized', LOCALE_CUSTOMIZED); + } + // Else no filtering needed in this case. + } + else { + if ($options['not_customized']) { + // Filter for non-customized strings only. + $query->condition('t.customized', LOCALE_NOT_CUSTOMIZED); + } + else { + // Filter for strings without translation. + $query->isNull('t.translation'); + } + } + $query->fields('t', array('translation')); + } + else { + $query->leftJoin('locales_target', 't', 's.lid = t.lid'); + } + $query->fields('s', array('lid', 'source', 'context', 'location')); + + // TODO: we need to order by lid + // This does not seem to work + $query->orderBy('s.lid'); + $query->condition('s.lid', $this->_lid, '>'); + + $this->_result = $query->execute(); + //echo "Executing: (lid = $this->_lid) : \n" . $this->_result->getQueryString() . "\n"; + } + + private function getResult() { + if (!isset($this->_result)) { + $this->buildQuery(); + } + return $this->_result; + } + + function readItem() { + $result = $this->getResult(); + $values = $result->fetchAssoc(); + + if ($values) { + $poItem = new POItem(); + $poItem->fromArray($values); + // Manage state + $this->_lid = $values['lid']; + return $poItem; + } + } + +} \ No newline at end of file diff --git a/core/lib/Drupal/Core/Gettext/PODbWriter.php b/core/lib/Drupal/Core/Gettext/PODbWriter.php new file mode 100644 index 0000000..519eaac --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/PODbWriter.php @@ -0,0 +1,281 @@ + NULL, + 'report' => array( + 'additions' => 0, + 'updates' => 0, + 'deletes' => 0, + 'skips' => 0, + 'ignored' => 0, + ), + 'options' => array( + 'overwrite_options' => array( + 'not_customized' => FALSE, + 'customized' => FALSE, + ), + 'customized' => LOCALE_NOT_CUSTOMIZED, + ), + ); + } + + /** + * Report array summarizing the number of changes done in the form: + * array(inserts, updates, deletes). + * + * @var array + */ + private $_report; + + /** + * @see BatchStateInterface + */ + function __construct() { + $this->setState(array()); + } + + public function getLangcode() { + return $this->_langcode; + } + + public function setLangcode($langcode) { + $this->_langcode = $langcode; + } + + public function getReport() { + return $this->_report; + } + + function setReport($report) { + $report += array( + 'additions' => 0, + 'updates' => 0, + 'deletes' => 0, + 'skips' => 0, + 'ignored' => 0, + ); + $this->_report = $report; + } + + function getOptions() { + return $this->_options; + } + + function setOptions(array $options) { + if (!isset($options['overwrite_options'])) { + $options['overwrite_options'] = array( + 'not_customized' => FALSE, + 'customized' => FALSE, + ); + } + if (!isset($options['customized'])) { + $options['customized'] = LOCALE_NOT_CUSTOMIZED; + } + $this->_options = $options; + } + + /** + * Implementation of BatchInterface::setState + * + * @param array $state + */ + public function setState(array $state) { + $state += self::getDefaultState(); + $this->_report = $state['report']; + $this->setLangcode($state['langcode']); + $this->setOptions($state['options']); + } + + public function getState() { + return array( + 'class' => __CLASS__, + 'report' => $this->getReport(), + 'langcode' => $this->getLangcode(), + 'options' => $this->getOptions(), + ); + } + + function getHeader() { + return $this->_header; + } + + function setHeader(POHeader $header) { + $this->_header = $header; + $locale_plurals = variable_get('locale_translation_plurals', array()); + $options = $this->getOptions(); + $overwrite_options = $options['overwrite_options']; + $lang = $this->_langcode; + if (array_sum($overwrite_options) || empty($locale_plurals[$lang]['plurals'])) { + // Get and store the plural formula if available. + $plural = $header->getPlural(); + // TODO: this is a sloppy way to create a file name + // but _locale_import_parse_plural_forms is also weird to me still + $filepath = __CLASS__ . "::" . __METHOD__; + if (isset($plural) && $p = $header->_locale_import_parse_plural_forms($plural, $filepath)) { + list($nplurals, $formula) = $p; + $locale_plurals[$lang] = array( + 'plurals' => $nplurals, + 'formula' => $formula, + ); + variable_set('locale_translation_plurals', $locale_plurals); + } + } + } + + function writeItem(POItem $item) { + if ($item->plural) { + $item->source = join(LOCALE_PLURAL_DELIMITER, $item->source); + $item->translation = join(LOCALE_PLURAL_DELIMITER, $item->translation); + } + $this->_locale_import_one_string_db($this->_langcode, $item->context, $item->source, $item->translation, 'location', $this->_options['overwrite_options'], $this->_options['customized']); + } + + public function writeItems(PoReaderInterface $reader, $count = 10) { + $forever = $count == -1; + while (($count-- > 0 || $forever) && ($item = $reader->readItem())) { + $this->writeItem($item); + } + } + + /** + * Imports one string into the database. + * + * @param $langcode + * Language code to import string into. + * @param $context + * The context of this string. + * @param $source + * Source string. + * @param $translation + * Translation to language specified in $langcode. + * @param $location + * Location value to save with source string. + * @param $overwrite_options + * An associative array indicating what data should be overwritten, if any. + * - not_customized: not customized strings should be overwritten. + * - customized: customized strings should be overwritten. + * @param $customized + * (optional) Whether the strings being imported should be saved as customized. + * Use LOCALE_CUSTOMIZED or LOCALE_NOT_CUSTOMIZED. + * + * @return + * The string ID of the existing string modified or the new string added. + */ + function _locale_import_one_string_db($langcode, $context, $source, $translation, $location, $overwrite_options, $customized = LOCALE_NOT_CUSTOMIZED) { + + // Initialize overwrite options if not set. + $overwrite_options += array( + 'not_customized' => FALSE, + 'customized' => FALSE, + ); + + // Look up the source string and any existing translation. + $string = db_query("SELECT s.lid, t.customized FROM {locales_source} s LEFT JOIN {locales_target} t ON s.lid = t.lid AND t.language = :language WHERE s.source = :source AND s.context = :context", array( + ':source' => $source, + ':context' => $context, + ':language' => $langcode, + )) + ->fetchObject(); + + if (!empty($translation)) { + // Skip this string unless it passes a check for dangerous code. + if (!locale_string_is_safe($translation)) { + watchdog('locale', 'Import of string "%string" was skipped because of disallowed or malformed HTML.', array('%string' => $translation), WATCHDOG_ERROR); + $this->_report['skips']++; + return 0; + } + elseif (isset($string->lid)) { + // We have this source string saved already. + db_update('locales_source') + ->fields(array( + 'location' => $location, + )) + ->condition('lid', $string->lid) + ->execute(); + + if (!isset($string->customized)) { + // No translation in this language. + db_insert('locales_target') + ->fields(array( + 'lid' => $string->lid, + 'language' => $langcode, + 'translation' => $translation, + 'customized' => $customized, + )) + ->execute(); + + $this->_report['additions']++; + } + elseif ($overwrite_options[$string->customized ? 'customized' : 'not_customized']) { + // Translation exists, only overwrite if instructed. + db_update('locales_target') + ->fields(array( + 'translation' => $translation, + 'customized' => $customized, + )) + ->condition('language', $langcode) + ->condition('lid', $string->lid) + ->execute(); + + $this->_report['updates']++; + } + return $string->lid; + } + else { + // No such source string in the database yet. + $lid = db_insert('locales_source') + ->fields(array( + 'location' => $location, + 'source' => $source, + 'context' => (string) $context, + )) + ->execute(); + + db_insert('locales_target') + ->fields(array( + 'lid' => $lid, + 'language' => $langcode, + 'translation' => $translation, + 'customized' => $customized, + )) + ->execute(); + + $this->_report['additions']++; + return $lid; + } + } + elseif (isset($string->lid) && isset($string->customized) && $overwrite_options[$string->customized ? 'customized' : 'not_customized']) { + // Empty translation, remove existing if instructed. + db_delete('locales_target') + ->condition('language', $langcode) + ->condition('lid', $string->lid) + ->execute(); + + $this->_report['deletes']++; + return $string->lid; + } + } + +} \ No newline at end of file diff --git a/core/lib/Drupal/Core/Gettext/POHeader.php b/core/lib/Drupal/Core/Gettext/POHeader.php new file mode 100644 index 0000000..f4ab991 --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/POHeader.php @@ -0,0 +1,381 @@ +1);\n" + + * @author clemens + */ +class POHeader { + + private $_langcode; + private $_projectIdVersion; + private $_potCreationDate; + private $_poRevisionDate; + private $_languageTeam; + private $_mimeVersion; + private $_contentType; + private $_contentTransferEncoding; + private $_pluralForms; + private $_authors; + private $_po_date; + + public function __construct($langcode = NULL) { + $this->_langcode = $langcode; + } + + static public function mapping() { + return array( + 'Project-Id-Version' => '_projectIdVersion', + // * Report-Msgid-Bugs-To + 'POT-Creation-Date' => '_potCreationDate', + 'PO-Revision-Date' => '_poRevisionDate', + // * Last-Translator + 'Language-Team' => '_languageTeam', + 'MIME-Version' => '_mimeVersion', + // * Language + 'Content-Type' => '_contentType', + 'Content-Transfer-Encoding' => '_contentTransferEncoding', + 'Plural-Forms' => '_pluralForms', + ); + } + + function getPlural() { + return $this->_pluralForms; + } + + /** + * Compile the PO header. + */ + private function compileHeader() { + $output = ''; + + // Add language description and author as comment. + $languages = language_list(); + $language_name = isset($languages[$this->_langcode]) ? $languages[$this->_langcode]->name : ''; + $output .= '# ' . $language_name . ' translation of ' . variable_get('site_name', 'Drupal') . "\n"; + if (!empty($this->_authors)) { + $output .= '# Generated by ' . implode("\n# ", $this->_authors) . "\n"; + } + $output .= "#\n"; + + // Add the actual header information. + $output .= "msgid \"\"\n"; + $output .= "msgstr \"\"\n"; + $output .= "\"Project-Id-Version: PROJECT VERSION\\n\"\n"; + $output .= "\"POT-Creation-Date: " . $this->_po_date . "\\n\"\n"; + $output .= "\"PO-Revision-Date: " . $this->_po_date . "\\n\"\n"; + $output .= "\"Last-Translator: NAME \\n\"\n"; + $output .= "\"Language-Team: LANGUAGE \\n\"\n"; + $output .= "\"MIME-Version: 1.0\\n\"\n"; + $output .= "\"Content-Type: text/plain; charset=utf-8\\n\"\n"; + $output .= "\"Content-Transfer-Encoding: 8bit\\n\"\n"; + $output .= "\"Plural-Forms: " . $this->_pluralForms . "\\n\"\n"; + $output .= "\n"; + + return $output; + } + + /** + * Stores a given PO Header string + * + * TODO: the header string is cleaned by the parser :( + * we need to accept unclean version too + * + * @param type $header + */ + public function setFromString($header) { + $header_parsed = $this->_locale_import_parse_header($header); + + $mapping = self::mapping(); + foreach ($mapping as $key => $var) { + if (isset($header_parsed[$key])) { + $this->{$var} = $header_parsed[$key]; + } + } + } + + public function setDefaults($values) { + $defaults = array( + 'POT-Creation-Date' => date("Y-m-d H:iO"), + 'Plural-Forms' => 'nplurals=2; plural=(n > 1);', + ); + $values += $defaults; + $mapping = self::mapping(); + foreach ($mapping as $key => $var) { + if (isset($values[$key])) { + $this->{$var} = $values[$key]; + } + } + } + + public function __toString() { + $result = $this->compileHeader() . "\n"; + return $result; + } + + /** + * Parses a Plural-Forms entry from a Gettext Portable Object file header. + * + * @param $pluralforms + * A string containing the Plural-Forms entry. + * @param $filepath + * A string containing the filepath. + * + * @return + * An array containing the number of plurals and a + * formula in PHP for computing the plural form. + */ + function _locale_import_parse_plural_forms($pluralforms, $filepath) { + // First, delete all whitespace + $pluralforms = strtr($pluralforms, array(" " => "", "\t" => "")); + + // Select the parts that define nplurals and plural + $nplurals = strstr($pluralforms, "nplurals="); + if (strpos($nplurals, ";")) { + $nplurals = substr($nplurals, 9, strpos($nplurals, ";") - 9); + } + else { + return FALSE; + } + $plural = strstr($pluralforms, "plural="); + if (strpos($plural, ";")) { + $plural = substr($plural, 7, strpos($plural, ";") - 7); + } + else { + return FALSE; + } + + // Get PHP version of the plural formula + $plural = $this->_locale_import_parse_arithmetic($plural); + + if ($plural !== FALSE) { + return array($nplurals, $plural); + } + else { + drupal_set_message(t('The translation file %filepath contains an error: the plural formula could not be parsed.', array('%filepath' => $filepath)), 'error'); + return FALSE; + } + } + + /** + * Parses a Gettext Portable Object file header. + * + * @param $header + * A string containing the complete header. + * + * @return + * An associative array of key-value pairs. + */ + function _locale_import_parse_header($header) { + $header_parsed = array(); + $lines = array_map('trim', explode("\n", $header)); + foreach ($lines as $line) { + if ($line) { + list($tag, $contents) = explode(":", $line, 2); + $header_parsed[trim($tag)] = trim($contents); + } + } + return $header_parsed; + } + + /** + * Parses and sanitizes an arithmetic formula into a PHP expression. + * + * While parsing, we ensure, that the operators have the right + * precedence and associativity. + * + * @param $string + * A string containing the arithmetic formula. + * + * @return + * The PHP version of the formula. + */ + function _locale_import_parse_arithmetic($string) { + // Operator precedence table + $precedence = array("(" => -1, ")" => -1, "?" => 1, ":" => 1, "||" => 3, "&&" => 4, "==" => 5, "!=" => 5, "<" => 6, ">" => 6, "<=" => 6, ">=" => 6, "+" => 7, "-" => 7, "*" => 8, "/" => 8, "%" => 8); + // Right associativity + $right_associativity = array("?" => 1, ":" => 1); + + $tokens = $this->_locale_import_tokenize_formula($string); + + // Parse by converting into infix notation then back into postfix + // Operator stack - holds math operators and symbols + $operator_stack = array(); + // Element Stack - holds data to be operated on + $element_stack = array(); + + foreach ($tokens as $token) { + $current_token = $token; + + // Numbers and the $n variable are simply pushed into $element_stack + if (is_numeric($token)) { + $element_stack[] = $current_token; + } + elseif ($current_token == "n") { + $element_stack[] = '$n'; + } + elseif ($current_token == "(") { + $operator_stack[] = $current_token; + } + elseif ($current_token == ")") { + $topop = array_pop($operator_stack); + while (isset($topop) && ($topop != "(")) { + $element_stack[] = $topop; + $topop = array_pop($operator_stack); + } + } + elseif (!empty($precedence[$current_token])) { + // If it's an operator, then pop from $operator_stack into $element_stack until the + // precedence in $operator_stack is less than current, then push into $operator_stack + $topop = array_pop($operator_stack); + while (isset($topop) && ($precedence[$topop] >= $precedence[$current_token]) && !(($precedence[$topop] == $precedence[$current_token]) && !empty($right_associativity[$topop]) && !empty($right_associativity[$current_token]))) { + $element_stack[] = $topop; + $topop = array_pop($operator_stack); + } + if ($topop) { + $operator_stack[] = $topop; // Return element to top + } + $operator_stack[] = $current_token; // Parentheses are not needed + } + else { + return FALSE; + } + } + + // Flush operator stack + $topop = array_pop($operator_stack); + while ($topop != NULL) { + $element_stack[] = $topop; + $topop = array_pop($operator_stack); + } + + // Now extract formula from stack + $previous_size = count($element_stack) + 1; + while (count($element_stack) < $previous_size) { + $previous_size = count($element_stack); + for ($i = 2; $i < count($element_stack); $i++) { + $op = $element_stack[$i]; + if (!empty($precedence[$op])) { + $f = ""; + if ($op == ":") { + $f = $element_stack[$i - 2] . "):" . $element_stack[$i - 1] . ")"; + } + elseif ($op == "?") { + $f = "(" . $element_stack[$i - 2] . "?(" . $element_stack[$i - 1]; + } + else { + $f = "(" . $element_stack[$i - 2] . $op . $element_stack[$i - 1] . ")"; + } + array_splice($element_stack, $i - 2, 3, $f); + break; + } + } + } + + // If only one element is left, the number of operators is appropriate + if (count($element_stack) == 1) { + return $element_stack[0]; + } + else { + return FALSE; + } + } + + /** + * Provides backward-compatible formula parsing for token_get_all(). + * + * @param $string + * A string containing the arithmetic formula. + * + * @return + * The PHP version of the formula. + */ + function _locale_import_tokenize_formula($formula) { + $formula = str_replace(" ", "", $formula); + $tokens = array(); + for ($i = 0; $i < strlen($formula); $i++) { + if (is_numeric($formula[$i])) { + $num = $formula[$i]; + $j = $i + 1; + while ($j < strlen($formula) && is_numeric($formula[$j])) { + $num .= $formula[$j]; + $j++; + } + $i = $j - 1; + $tokens[] = $num; + } + elseif ($pos = strpos(" =<>!&|", $formula[$i])) { // We won't have a space + $next = $formula[$i + 1]; + switch ($pos) { + case 1: + case 2: + case 3: + case 4: + if ($next == '=') { + $tokens[] = $formula[$i] . '='; + $i++; + } + else { + $tokens[] = $formula[$i]; + } + break; + case 5: + if ($next == '&') { + $tokens[] = '&&'; + $i++; + } + else { + $tokens[] = $formula[$i]; + } + break; + case 6: + if ($next == '|') { + $tokens[] = '||'; + $i++; + } + else { + $tokens[] = $formula[$i]; + } + break; + } + } + else { + $tokens[] = $formula[$i]; + } + } + return $tokens; + } + +} + +?> diff --git a/core/lib/Drupal/Core/Gettext/POItem.php b/core/lib/Drupal/Core/Gettext/POItem.php new file mode 100644 index 0000000..ea7ced1 --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/POItem.php @@ -0,0 +1,139 @@ +1);\n" + + * @author clemens + */ +class POItem { + + public $context; + public $source; + public $this; + public $plural; + public $comment; + + static public function mapping() { + return array( + 'msgctxt' => 'context', + 'msgid' => 'source', + 'msgstr' => 'translation', + '#' => 'comment', + ); + } + + public function fromArray(array $values = array()) { + foreach ($values as $key => $value) { + $this->{$key} = $value; + } + } + + public function __toString() { + return $this->compileTranslation(); + } + + /** + * Compile PO translations strings from a translation object. + * + * Translation object consists of: + * source string (singular) or array of strings (plural) + * translation string (singular) or array of strings (plural) + * plural TRUE: source and translation are plurals + * context source context string + */ + private function compileTranslation() { + $output = ''; + + // Format string context. + if (!empty($this->context)) { + $output .= 'msgctxt ' . $this->formatString($this->context); + } + + // Format translation + if ($this->plural) { + $output .= $this->formatPlural(); + } + else { + $output .= $this->formatSingular(); + } + + // Add one empty line to separate the translations. + $output .= "\n"; + + return $output; + } + + /** + * Formats a plural translation. + */ + private function formatPlural() { + $output = ''; + + // Format source strings. + $output .= 'msgid ' . $this->formatString($this->source[0]); + $output .= 'msgid_plural ' . $this->formatString($this->source[1]); + + // Format translation strings. + $plurals = variable_get('locale_translation_plurals', array()); + // @todo What to when $plurals[$this->langcode] is not set? + // This (currently) happens if a language is created manually or importing a malformed po. + $nplurals = $plurals[$this->langcode]['plurals']; + for ($i = 0; $i < $nplurals; $i++) { + if (isset($this->translation[$i])) { + $output .= 'msgstr[' . $i . '] ' . $this->formatString($this->translation[$i]); + } + else { + $output .= 'msgstr[' . $i . '] ""' . "\n"; + } + } + + return $output; + } + + /** + * Formats a singular translation. + */ + private function formatSingular() { + $output = ''; + $output .= 'msgid ' . $this->formatString($this->source); + $output .= 'msgstr ' . $this->formatString($this->translation); + return $output; + } + + /** + * Formats a string for output on multiple lines. + */ + private function formatString($string) { + // Escape characters for processing. + $string = addcslashes($string, "\0..\37\\\""); + + // Always include a line break after the explicit \n line breaks from + // the source string. Otherwise wrap at 70 chars to accommodate the extra + // format overhead too. + $parts = explode("\n", wordwrap(str_replace('\n', "\\n\n", $string), 70, " \n")); + + // Multiline string should be exported starting with a "" and newline to + // have all lines aligned on the same column. + if (count($parts) > 1) { + return "\"\"\n\"" . implode("\"\n\"", $parts) . "\"\n"; + } + // Single line strings are output on the same line. + else { + return "\"$parts[0]\"\n"; + } + } + +} + +?> diff --git a/core/lib/Drupal/Core/Gettext/PoFileReader.php b/core/lib/Drupal/Core/Gettext/PoFileReader.php new file mode 100644 index 0000000..694497e --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/PoFileReader.php @@ -0,0 +1,540 @@ +_uri; + } + + public function setURI($uri) { + $this->_uri = $uri; + } + + public function getLangcode() { + return $this->_langcode; + } + + public function setLangcode($langcode) { + $this->_langcode = $langcode; + } + + public function open() { + if (!empty($this->_uri)) { + $this->_fd = fopen($this->_uri, 'rb'); + $this->_size = ftell($this->_fd); + // We immediately read the header as we are at BOF + $this->readHeader(); + } + else { + throw new \Exception("Cannot open without URI set"); + } + } + + public function close() { + if ($this->_fd) { + fclose($this->_fd); + } + } + + + public function setState(array $state) { + $this->setURI($state['uri']); + $this->setLangcode($state['langcode']); + // Make sure to (re)read the POHeader + $this->open(); + // Move to last read position. + if (isset($state['seekpos'])) { + fseek($this->_fd, $state['seekpos']); + } + if (isset($state['lineno'])) { + $this->lineno = $state['lineno']; + } + } + + public function getState() { + return array( + 'class' => __CLASS__, + 'uri' => $this->_uri, + 'langcode' => $this->_langcode, + 'seekpos' => ftell($this->_fd), + 'lineno' => $this->lineno, + ); + } + + /** + * Return a translation object (singular or plural) + * + * @todo Define a translation object for this purpose? + * Or use a standard class for better performance? + */ + public function readItem() { + $this->readTranslation(); + return $this->translation; + } + + private function readTranslation() { + $this->translation = NULL; + while (!$this->finished && is_null($this->translation)) { + $this->readLine(); + } + return $this->translation; + } + + public function getHeader() { + return $this->_header; + } + + public function setHeader(POHeader $header) { + // TODO : throw exception? + } + + /** + * Reads the header from the given input stream. + * + * We need to read the optional first COMMENT + * Next read a MSGID and a MSGSTR + * + * TODO: is a header required? + */ + private function readHeader() { + $translation = $this->readTranslation(); + $header = new POHeader; + $header->setFromString(trim($translation->translation)); + $this->_header = $header; + } + + /** + * Reads a line from a PO file. + * + * While reading a line it's content is processed according to current + * context. + * + * The parser context. Can be: + * - 'COMMENT' (#) + * - 'MSGID' (msgid) + * - 'MSGID_PLURAL' (msgid_plural) + * - 'MSGCTXT' (msgctxt) + * - 'MSGSTR' (msgstr or msgstr[]) + * - 'MSGSTR_ARR' (msgstr_arg) + * + * @return boolean FALSE or NULL + */ + private function readLine() { + // a string or boolean FALSE + $line = fgets($this->_fd); + $this->finished = ($line === FALSE); + if (!$this->finished) { + + if ($this->lineno == 0) { + // The first line might come with a UTF-8 BOM, which should be removed. + $line = str_replace("\xEF\xBB\xBF", '', $line); + // Current plurality for 'msgstr[]'. + $this->plural = 0; + } + + $this->lineno++; + + // Trim away the linefeed. + $line = trim(strtr($line, array("\\\n" => ""))); + + if (!strncmp('#', $line, 1)) { + // Lines starting with '#' are comments. + + if ($this->context == 'COMMENT') { + // Already in comment token, insert the comment. + $this->current['#'][] = substr($line, 1); + } + elseif (($this->context == 'MSGSTR') || ($this->context == 'MSGSTR_ARR')) { + // We are currently in string token, close it out. + $this->saveOneString(); + + // Start a new entry for the comment. + $this->current = array(); + $this->current['#'][] = substr($line, 1); + + $this->context = 'COMMENT'; + return TRUE; + } + else { + // A comment following any other token is a syntax error. + $this->log('The translation file %filename contains an error: "msgstr" was expected but not found on line %line.', $this->lineno); + return FALSE; + } + return; + } + elseif (!strncmp('msgid_plural', $line, 12)) { + // A plural form for the current message. + + if ($this->context != 'MSGID') { + // A plural form cannot be added to anything else but the id directly. + $this->log('The translation file %filename contains an error: "msgid_plural" was expected but not found on line %line.', $this->lineno); + return FALSE; + } + + // Remove 'msgid_plural' and trim away whitespace. + $line = trim(substr($line, 12)); + // At this point, $line should now contain only the plural form. + + $quoted = $this->parseQuoted($line); + if ($quoted === FALSE) { + // The plural form must be wrapped in quotes. + $this->log('The translation file %filename contains a syntax error on line %line.', $this->lineno); + return FALSE; + } + + // Append the plural form to the current entry. + if (is_string($this->current['msgid'])) { + // The first value was stored as string. Now we know the context is + // plural, it is converted to array. + $this->current['msgid'] = array($this->current['msgid']); + } + $this->current['msgid'][] = $quoted; + + $this->context = 'MSGID_PLURAL'; + return; + } + elseif (!strncmp('msgid', $line, 5)) { + // Starting a new message. + + if (($this->context == 'MSGSTR') || ($this->context == 'MSGSTR_ARR')) { + // We are currently in a message string, close it out. + $this->saveOneString(); + + // Start a new context for the id. + $this->current = array(); + } + elseif ($this->context == 'MSGID') { + // We are currently already in the context, meaning we passed an id with no data. + $this->log('The translation file %filename contains an error: "msgid" is unexpected on line %line.', $this->lineno); + return FALSE; + } + + // Remove 'msgid' and trim away whitespace. + $line = trim(substr($line, 5)); + // At this point, $line should now contain only the message id. + + $quoted = $this->parseQuoted($line); + if ($quoted === FALSE) { + // The message id must be wrapped in quotes. + $this->log('The translation file %filename contains a syntax error on line %line.', $this->lineno); + return FALSE; + } + + $this->current['msgid'] = $quoted; + $this->context = 'MSGID'; + return; + } + elseif (!strncmp('msgctxt', $line, 7)) { + // Starting a new context. + + if (($this->context == 'MSGSTR') || ($this->context == 'MSGSTR_ARR')) { + // We are currently in a message, start a new one. + $this->saveOneString($this->current); + $this->current = array(); + } + elseif (!empty($this->current['msgctxt'])) { + // A context cannot apply to another context. + $this->log('The translation file %filename contains an error: "msgctxt" is unexpected on line %line.', $this->lineno); + return FALSE; + } + + // Remove 'msgctxt' and trim away whitespaces. + $line = trim(substr($line, 7)); + // At this point, $line should now contain the context. + + $quoted = $this->parseQuoted($line); + if ($quoted === FALSE) { + // The context string must be quoted. + $this->log('The translation file %filename contains a syntax error on line %line.', $this->lineno); + return FALSE; + } + + $this->current['msgctxt'] = $quoted; + + $this->context = 'MSGCTXT'; + return; + } + elseif (!strncmp('msgstr[', $line, 7)) { + // A message string for a specific plurality. + + if (($this->context != 'MSGID') && ($this->context != 'MSGCTXT') && ($this->context != 'MSGID_PLURAL') && ($this->context != 'MSGSTR_ARR')) { + // Message strings must come after msgid, msgxtxt, msgid_plural, or other msgstr[] entries. + $this->log('The translation file %filename contains an error: "msgstr[]" is unexpected on line %line.', $this->lineno); + return FALSE; + } + + // Ensure the plurality is terminated. + if (strpos($line, ']') === FALSE) { + $this->log('The translation file %filename contains a syntax error on line %line.', $this->lineno); + return FALSE; + } + + // Extract the plurality. + $frombracket = strstr($line, '['); + $this->plural = substr($frombracket, 1, strpos($frombracket, ']') - 1); + + // Skip to the next whitespace and trim away any further whitespace, bringing $line to the message data. + $line = trim(strstr($line, " ")); + + $quoted = $this->parseQuoted($line); + if ($quoted === FALSE) { + // The string must be quoted. + $this->log('The translation file %filename contains a syntax error on line %line.', $this->lineno); + return FALSE; + } + if (!isset($this->current['msgstr']) || !is_array($this->current['msgstr'])) { + $this->current['msgstr'] = array(); + } + + $this->current['msgstr'][$this->plural] = $quoted; + + $this->context = 'MSGSTR_ARR'; + return; + } + elseif (!strncmp("msgstr", $line, 6)) { + // A string for the an id or context. + + if (($this->context != 'MSGID') && ($this->context != 'MSGCTXT')) { + // Strings are only valid within an id or context scope. + $this->log('The translation file %filename contains an error: "msgstr" is unexpected on line %line.', $this->lineno); + return FALSE; + } + + // Remove 'msgstr' and trim away away whitespaces. + $line = trim(substr($line, 6)); + // At this point, $line should now contain the message. + + $quoted = $this->parseQuoted($line); + if ($quoted === FALSE) { + // The string must be quoted. + $this->log('The translation file %filename contains a syntax error on line %line.', $this->lineno); + return FALSE; + } + + $this->current['msgstr'] = $quoted; + + $this->context = 'MSGSTR'; + return; + } + elseif ($line != '') { + // Anything that is not a token may be a continuation of a previous token. + + $quoted = $this->parseQuoted($line); + if ($quoted === FALSE) { + // The string must be quoted. + $this->log('The translation file %filename contains a syntax error on line %line.', $this->lineno); + return FALSE; + } + + // Append the string to the current context. + if (($this->context == 'MSGID') || ($this->context == 'MSGID_PLURAL')) { + if (is_array($this->current['msgid'])) { + // Add string to last array element. + $last_index = count($this->current['msgid']) - 1; + $this->current['msgid'][$last_index] .= $quoted; + } + else { + $this->current['msgid'] .= $quoted; + } + } + elseif ($this->context == 'MSGCTXT') { + $this->current['msgctxt'] .= $quoted; + } + elseif ($this->context == 'MSGSTR') { + $this->current['msgstr'] .= $quoted; + } + elseif ($this->context == 'MSGSTR_ARR') { + $this->current['msgstr'][$this->plural] .= $quoted; + } + else { + // No valid context to append to. + $this->log('The translation file %filename contains an error: there is an unexpected string on line %line.', $this->lineno); + return FALSE; + } + return; + } + } + + // Empty line read or EOF of PO file, closed out the last entry. + if (($this->context == 'MSGSTR') || ($this->context == 'MSGSTR_ARR')) { + $this->saveOneString($this->current); + $this->current = array(); + } + elseif ($this->context != 'COMMENT') { + $this->log('The translation file %filename ended unexpectedly at line %line.', $this->lineno); + return FALSE; + } + } + + /** + * Sets an error message if an error occurred during locale file parsing. + * + * @param $message + * The message to be translated. + * @param $lineno + * An optional line number argument. + */ + protected function log($message, $lineno = NULL) { + if (isset($lineno)) { + $vars['%line'] = $lineno; + } + $t = get_t(); + $this->errorLog[] = $t($message, $vars); + } + + /** + * Store the parsed values as translation object. + */ + public function saveOneString() { + $value = $this->current; + $plural = FALSE; + + $comments = ''; + if (isset($value['#'])) { + $comments = $this->shortenComments($value['#']); + } + + if (is_array($value['msgstr'])) { + // Sort plural variants by their form index. + ksort($value['msgstr']); + $plural = TRUE; + } + + $translation = new POItem; + $translation->context = isset($value['msgctxt']) ? $value['msgctxt'] : ''; + $translation->source = $value['msgid']; + $translation->translation = $value['msgstr']; + $translation->plural = $plural; + $translation->comment = $comments; + + $this->translation = $translation; + + $this->context = 'COMMENT'; + } + + /** + * Parses a string in quotes. + * + * @param $string + * A string specified with enclosing quotes. + * + * @return + * The string parsed from inside the quotes. + */ + function parseQuoted($string) { + if (substr($string, 0, 1) != substr($string, -1, 1)) { + return FALSE; // Start and end quotes must be the same + } + $quote = substr($string, 0, 1); + $string = substr($string, 1, -1); + if ($quote == '"') { // Double quotes: strip slashes + return stripcslashes($string); + } + elseif ($quote == "'") { // Simple quote: return as-is + return $string; + } + else { + return FALSE; // Unrecognized quote + } + } + + /** + * Generates a short, one-string version of the passed comment array. + * + * @param $comment + * An array of strings containing a comment. + * + * @return + * Short one-string version of the comment. + */ + private function shortenComments($comment) { + $comm = ''; + while (count($comment)) { + $test = $comm . substr(array_shift($comment), 1) . ', '; + if (strlen($comm) < 130) { + $comm = $test; + } + else { + break; + } + } + return trim(substr($comm, 0, -2)); + } + +} diff --git a/core/lib/Drupal/Core/Gettext/PoFileWriter.php b/core/lib/Drupal/Core/Gettext/PoFileWriter.php new file mode 100644 index 0000000..2f239b9 --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/PoFileWriter.php @@ -0,0 +1,112 @@ +_header; + } + + public function setHeader(POHeader $header) { + $this->_header = $header; + } + + public function getLangcode() { + return $this->_langcode; + } + + public function setLangcode($langcode) { + $this->_langcode = $langcode; + } + + public function open() { + // Open in append mode + $this->_fd = fopen($this->getURI(), 'a'); + $this->_seekpos = ftell($this->_fd); + if ($this->_seekpos == 0) { + // If file is new position == 0 + $this->writeHeader(); + } + else { + $reader = new PoFileReader($this->uri); + $this->_header = $reader->getHeader(); + } + } + + public function close() { + fclose($this->_fd); + } + + public function setState(array $state) { + $this->_uri = $state['uri']; + $this->open(); + } + + public function getState() { + return array( + 'uri' => $this->_uri, + 'seekpos' => ftell($this->_fd), + ); + } + + private function write($data) { + $result = fputs($this->_fd, $data); + if ($result === FALSE) { + // TODO: better context for message + throw new \Exception("Unable to write data : " . substr($data, 0, 20)); + } + $this->_seekpos = ftell($this->_fd); + } + + private function writeHeader() { + $this->write($this->_header); + } + + public function writeItem(POItem $item) { + $this->write($item); + } + + public function writeItems(PoReaderInterface $reader, $count = 10) { + $forever = $count == -1; + while (($count-- > 0 || $forever) && ($item = $reader->readItem())) { + $this->writeItem($item); + } + } + + public function getURI() { + if (empty($this->_uri)) { + throw new \Exception("Empty URI"); + } + return $this->_uri; + } + + public function setURI($uri) { + $this->_uri = $uri; + } + +} diff --git a/core/lib/Drupal/Core/Gettext/PoInterface.php b/core/lib/Drupal/Core/Gettext/PoInterface.php new file mode 100644 index 0000000..850f79f --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/PoInterface.php @@ -0,0 +1,31 @@ +_items = array(); + } + + public function setState(array $state) { + // nothing to do? + } + + public function getState() { + return array(); + } + + public function writeItem(POItem $item) { + if (is_array($item->source)) { + $item->source = implode(LOCALE_PLURAL_DELIMITER, $item->source); + $item->translation = implode(LOCALE_PLURAL_DELIMITER, $item->translation); + } + $this->_items[isset($item->context) ? $item->context : ''][$item->source] = $item->translation; + } + + public function writeItems(PoReaderInterface $reader, $count = 10) { + $forever = $count == -1; + while (($count-- > 0 || $forever) && ($item = $reader->readItem())) { + $this->writeItem($item); + } + } + + public function getHeader() { + // TODO: what + } + + public function getLangcode() { + // TODO: what + } + + public function setHeader(POHeader $header) { + // TODO: what + } + + public function setLangcode($langcode) { + // TODO: what + } + + public function getData() { + return $this->_items; + } +} diff --git a/core/lib/Drupal/Core/Gettext/PoReaderInterface.php b/core/lib/Drupal/Core/Gettext/PoReaderInterface.php new file mode 100644 index 0000000..4f76dad --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/PoReaderInterface.php @@ -0,0 +1,22 @@ +gettextInterface = $interface; + } + + /** + * Implements magic function __destruct(). + */ + public function __destruct() { + $this->gettextInterface->close(); + } + + /** + * Return a translation object (singular or plural) + * + * @todo Define a translation object for this purpose? + * Or use a standard class for better performance? + */ + public function read() { + } + + /** + * Return header/meta data (date, plural formula, etc.) + */ + public function getMetaData() { + return $this->metaData; + } + + /** + * Return TRUE if the file is opened or the transfer has started. + */ + public function inProgress() { + return $this->inProgress; + } + + /** + * Return the name of an error callback function + */ + public function errorCallback() { + return ''; + } + + /** + * Return arguments for an error callback function + */ + public function errorArguments() { + return array(); + } + + /** + * Return the name of a post processing callback function + */ + public function postProcessCallback() { + return ''; + } + + /** + * Return arguments for a post processing callback function + */ + public function postProcessArguments() { + return array(); + } + + /** + * Return the calculated or estimated size in number of translations. Zero for unknown. To be generated without opening the connection. e.g. use file size not number of lines. + */ + public function size() { + return $this->sourceSize; + } + + /** + * Return a bite size. Based on experience and size to be transfered within a reasonable time. Size in number of source/translations pairs. e.g. database records in locales_source or locales_target tables. A set of plural's are counted as one. + */ + public function biteSize() { + return $this->bite_size; + } + + /** + * Accept a set of data filter arguments: Language, Context + */ + public function setFilter($arguments) { + $this->filter = $arguments; + } + + /** + * Return Is valid. Valid data is available, not EOF, no errors, etc. + */ + public function valid() { + return $this->valid; + } + + /** + * Get percentage of completion (read) + */ + public function poc() { + if (!$this->$inProgress) { + return 0; + } + if ($this->finished) { + return 1; + } + // If reading is not finished, we limit the percentage to max. 95% + // Percentages above 100% are a result of low estimate of source size and + // will be suppressed. + return min(0.95, $this->index/$this->sourceSize); + } + + /** + * Return syntax errors + */ + public function getLog($category = NULL) { + return $this->errorLog; + } + + /** + * Internal: log syntax errors + */ + protected function log($line, $message) { + $this->errorLog[$line] = $message; + } + +} diff --git a/core/lib/Drupal/Core/Gettext/Writer.php b/core/lib/Drupal/Core/Gettext/Writer.php new file mode 100644 index 0000000..cafe618 --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/Writer.php @@ -0,0 +1,175 @@ +gettextInterface = $interface; + $this->langcode = $langcode; + + $languages = language_list(); + if (isset($languages[$langcode])) { + $this->language = $languages[$langcode]; + } + else { + // @todo throw error: Unknown language code. + } + + // Set default meta data. + $this->metaData = array( + 'authors' => array(), + 'po_date' => date("Y-m-d H:iO"), + 'plurals' => 'nplurals=2; plural=(n > 1);', + ); + } + + /** + * Implements magic function __destruct(). + */ + public function __destruct() { + $this->gettextInterface->close(); + } + + /** + * Write translation string (singular or plural) + * + * @todo Define a translation object for this purpose? + * Or use a standard class for better performance? + */ + public function write($translation) { + + } + + /** + * Set header/meta data (date, plural formula, etc.) + */ + public function setMetaData(array $data) { + $this->metaData = array_merge($this->metaData, $data); + } + + /** + * Return TRUE if the file is opened or the transfer has started. + */ + public function inProgress() { + return $this->inProgress; + } + + /** + * Return the name of an error callback function + */ + public function errorCallback() { + return ''; + } + + /** + * Return arguments for an error callback function + */ + public function errorArguments() { + return array(); + } + + /** + * Return the name of a post processing callback function + */ + public function postProcessCallback() { + return ''; + } + + /** + * Return arguments for a post processing callback function + */ + public function postProcessArguments() { + return array(); + } + + /** + * Return a bite size. Based on experience and size to be transfered within a reasonable time. Size in number of source/translations pairs. e.g. database records in locales_source or locales_target tables. A set of plural's are counted as one. + */ + public function biteSize() { + return $this->bite_size; + } + + /** + * Return the language code as defined by the data (e.g. po header). Use language filter (see below) as fallback. + */ + public function langcode() { + return $this->langcode; + } + + /** + * Accept write mode argument (replace, keep changes, skip existing) + * + * @todo Move this to __constructor()? + */ + public function setWriteMode($mode) { + $this->writeMode = $mode; + } + + /** + * Return Is valid. Valid data is available, not EOF, no errors, etc. + * + * @todo Needed for Writer? + */ + public function valid() { + return $this->valid; + } + + /** + * Get statistics (added, replaced, ignored, error, error log) + */ + public function statistics() { + return array( + 'added' => $this->resultsAdded, + 'replaced' => $this->resultsReplaced, + 'ignored' => $this->resultsIgnored, + 'error' => $this->resultsError, + ); + } + + /** + * Return syntax errors + */ + public function getLog($category = NULL) { + return $this->errorLog; + } + + /** + * Internal: log syntax errors + */ + protected function log($line, $message) { + $this->errorLog[$line] = $message; + } + +} diff --git a/core/lib/Drupal/Core/Gettext/testGettext.php b/core/lib/Drupal/Core/Gettext/testGettext.php new file mode 100644 index 0000000..824a320 --- /dev/null +++ b/core/lib/Drupal/Core/Gettext/testGettext.php @@ -0,0 +1,612 @@ + 'home', + 'translation' => 'thuis', + 'plural' => 0, + 'context' => '', + ), + array( + 'source' => 'delete', + 'translation' => 'verwijderen', + 'plural' => 0, + 'context' => '', + ), + array( + 'source' => array('1 day', '@count days'), + 'translation' => array('1 dag', '@count dagen'), + 'plural' => 1, + 'context' => '', + ), + ); + $items = array(); + foreach ($src as $values) { + $item = new POItem(); + $item->fromArray($values); + $items[] = $item; + } + $result = array( + 'langcode' => $langcode, + 'items' => $items, + ); + + return $result; +} + +function testWriter() { + logLine(__FUNCTION__); + $po = gettext_struct(); + $langcode = $po['langcode']; + $items = $po['items']; + + $writer = new PoFileWriter(); + + $writer->setLangcode($langcode); + $writer->setHeader(new POHeader($langcode)); + $writer->setURI(getPublicUri(__FUNCTION__, $langcode)); + $writer->open(); + foreach($items as $item) { + $writer->writeItem($item); + } + $writer->close(); +} + +function testBatchStreamManager() { + logLine(__FUNCTION__, '='); + $uri = 'public://infinite-text-file.txt.po'; + echo file_get_contents($uri); + + echo "Creating new BatchStreamManager()\n"; + $bsm = new BatchStreamManager(); + echo " state:\n"; + print_r($bsm->getBatchState()); + + echo "Opening stream $uri\n"; +// $uri, $mode, $options, &$opened_url + $bsm->open($uri, 'r', 0, $full_path); + echo " state:\n"; + print_r($bsm->getBatchState()); + + $my_s = $bsm->getStream(); + echo "Reading 10 bytes\n"; + $bytes = $my_s->stream_read(10); + echo " Bytes read: '$bytes'\n"; + + echo " state:\n"; + print_r($bsm->getBatchState()); + + echo "Moving to position 50\n"; + $state = $bsm->getBatchState(); + $state['position'] = 50; + $bsm->setBatchState($state); + $my_s = $bsm->getStream(); + + echo " state:\n"; + print_r($bsm->getBatchState()); + + echo "Reading 15 bytes\n"; + $bytes = $my_s->stream_read(15); + echo " Bytes read: '$bytes'\n"; + + echo " state:\n"; + print_r($bsm->getBatchState()); +} + +function getReadStream($uri) { + logLine(__FUNCTION__, '='); + $s = new GettextFileInterface($uri); + return $s; +} + +function testPoReader() { + logLine(__FUNCTION__, '='); + + $uri = 'public://test.po.txt'; + + logLine("Reading : $uri", '='); + logLine("File contents first 500 bytes"); + $contents = file_get_contents($uri); + echo substr($contents, 0, 500) . "\n"; + + logLine("Using PoFileReader"); + $reader = new PoFileReader($uri); + echo $reader->getHeader(); + + $i = 0; + while (($item = $reader->readItem()) && $i++ < 4) { + printItem($item, $i); + } +} + +function testHeader() { + logLine(__FUNCTION__, '='); + $h = new POHeader(); + + echo "----------------\n"; + $h->setFromString(''); + echo "empty header\n"; + echo $h; + + echo "----------------\n"; + $h->setFromString('"Project-Id-Version: ' . __FILE__ . '\n"'); + echo "-- one item -- \n"; + echo $h; +} + +function testFileToDb() { + logLine(__FUNCTION__, '='); + $uri = 'public://test.po.txt'; + logLine("Reading : $uri"); + logLine("File contents first 500 bytes"); + $contents = file_get_contents($uri); + echo substr($contents, 0, 500) . "\n"; + + logLine("POFileReader"); + $reader = new PoFileReader(); + $reader->setURI($uri); + printItem($reader->getHeader()); + + $langcode = 'ca'; + logLine("PODbWriter"); + $writer = new PODbWriter(); + $writer->setLangcode($langcode); + + $i = 0; + while (($item = $reader->readItem()) && $i < 4) { + printItem($item, $i++); + $writer->writeItem($item); + } +} + +function testDbDump() { + logLine(__FUNCTION__, '='); + $reader = new PODbReader('en'); + echo $reader->getHeader() . "\n"; + + $i = 0; + while (($item = $reader->readItem()) && $i < 4) { + printItem($item, $i++); + } + + echo "Saving state to simulate a batch\n"; + $state = $reader->getState(); + + echo "Create a new PODbReader so simulate a batch\n"; + $reader = new PODbReader('en'); + + // Set the state + $reader->setState($state); + $i = 0; + while (($item = $reader->readItem()) && $i < 4) { + printItem($item, $i++); + } +} + +function printItem($item, $context = 0) { + logLine(__FUNCTION__, '='); + if ($item) { + logLine("$context : $item->lid"); + print_r($item); + } +} + +function readItem($reader, $context = 0) { + $item = $reader->readItem(); + if ($item) { + printItem($item); + } +} + +function dumpState($state) { + logLine(__FUNCTION__, '='); + print_r($state); +} + +function testPOFileReader() { + logLine(__FUNCTION__, '='); + $uri = 'public://nl-nl.po'; + + $reader = new PoFileReader($uri); + dumpState($reader->getState()); + echo $reader->getHeader() . "\n"; + dumpState($reader->getState()); + $i = 0; + while (($item = readItem($reader)) && $i < 4) { + printItem($item, $i++); + } + dumpState($reader->getState()); +} + +function getLanguages($langcode = NULL) { + logLine(__FUNCTION__, '='); + if (!is_null($langcode)) { + return array($langcode); + } + return array( + 'nl', + 'ar', + 'ca', + 'en', // does not exists on d.o (should it be?) + 'NOP', // does really not exists on d.o + ); +} + +function getRemoteUris($langcode = NULL) { + logLine(__FUNCTION__, '='); + $langcodes = getLanguages($langcode); + $uris = array(); + foreach ($langcodes as $langcode) { + $uri = "http://ftp.drupal.org/files/translations/7.x/drupal/drupal-7.11.$langcode.po"; + $uris[$langcode] = $uri; + } + return $uris; +} + +function getPublicUri($name, $langcode) { + $result = "public://$name-$langcode.po"; + logLine($result); + return $result; +} + +function getRemoteUri($langcode) { + $uris = getRemoteUris($langcode); + return $uris[$langcode]; +} + +function testRemotePOPumper() { + logLine(__FUNCTION__, '='); + $uris = getRemoteUris(); + foreach ($uris as $langcode => $uri) { + logLine("langcode: $langcode"); + $reader = new PoFileReader($uri); + $writer = new PODbWriter($langcode); + $writer->writeItems($reader, 10); + } +} + +function testPOFileWriter() { + logLine(__FUNCTION__, '='); + $src = "http://ftp.drupal.org/files/translations/7.x/drupal/drupal-7.11.ar.po"; + + $reader = new PoFileReader(); + $reader->setURI($src); + $reader->open(); + $header = $reader->getHeader(); + + $dst = 'public://drupal-7.11.ar.po'; + zapUri($dst); + $writer = new PoFileWriter(); + $writer->setURI($dst); + $writer->setHeader($header); + $writer->open(); + + $i = 0; + while (($item = $reader->readItem()) && $i < 4) { + printItem($item, $i); + $i++; + $writer->writeItem($item); + dumpState($writer->getState()); + } + + $writer->close(); +} + +function testDbToFile() { + logLine(__FUNCTION__, '='); + $langcode = 'ca'; + $reader = new PODbReader(); + $reader->setLangcode($langcode); + + $dst = 'public://drupal-7.11.dummy.po'; + $header = $reader->getHeader(); + + $writer = new PoFileWriter(); + $writer->setURI($dst); + $writer->setHeader($header); + $writer->open(); + + $i = 0; + while (($item = $reader->readItem()) && $i < 4) { + printItem($item, $i); + $i++; + $writer->writeItem($item); + dumpState($writer->getState()); + } + + $writer->writeItems($reader, 10); + + $writer->close(); +} + +function testBatchSimulation() { + logLine(__FUNCTION__, '='); + + // Grab first langcode + $uris = getRemoteUris(); + $langcode = key($uris); + $src = current($uris); + + logLine("Opening $langcode : $src"); + $reader = new PoFileReader(); + $reader->setURI($src); + $reader->open(); + + $header = $reader->getHeader(); + logLine($header); + + $dst = getPublicUri(__FUNCTION__, $langcode); + + zapUri($dst); + logLine("Writing $langcode : $dst"); + $writer = new PoFileWriter(); + $writer->setURI($dst); + $writer->setHeader($header); + $writer->open(); + + logLine('Written header only', '='); + echo file_get_contents($dst); + + processN($writer, $reader, 2); + + dumpFileContents($dst); + + $state = $reader->getState(); + dumpState($state); + + logLine('Replacing reader', '='); + $reader = new PoFileReader($src); + + logLine('setting state back'); + $reader->setState($state); + dumpState($state); + + processN($writer, $reader, 3); + $reader->setState($state); + dumpState($state); + + dumpFileContents($dst); +} + +function testDBReaderState() { + logLine(__FUNCTION__); + $langcode = 'nl'; + $reader = new PODbReader(); + $reader->setLangcode($langcode); + + logLine("Init PODbReader", '='); + $state = $reader->getState(); + dumpState($state); + + $header = $reader->getHeader(); + + $uri = getPublicUri(__FUNCTION__, $langcode); + zapUri($uri); + $writer = new PoFileWriter($uri, $header); + $writer->setHeader($header); + $writer->setURI($uri); + $writer->open(); + + processN($writer, $reader, 4); + + logLine("Read some", '='); + $state = $reader->getState(); + dumpState($state); + + $reader = new PODbReader($langcode); + $reader->setState($state); + processN($writer, $reader, 4); + $state = $reader->getState(); + dumpState($state); + + logLine("File contents from $uri", '='); + echo file_get_contents($uri); +} + +function zapUri($uri) { + logLine("Truncate $uri", '='); + ftruncate(fopen($uri, 'w')); +} + +function processN($writer, $reader, $count = 10) { + if ($count == -1) { + logLine("processing items: __ALL__"); + } + else { + logLine("processing items: $count"); + } + $writer->writeItems($reader, $count); +} + +function dumpFileContents($uri) { + logLine("Written: $uri", '='); + echo file_get_contents($uri); +} + +function newPOFileReader($uri, $langcode = NULL) { + logLine("Reading from $uri using langcode: '$langcode'"); + $reader = new PoFileReader(); + $reader->setURI($uri); + $reader->setLangcode($langcode); + $reader->open(); + return $reader; +} + +function remoteToPublic($langcode) { + logLine(__FUNCTION__, '='); + $uri = getRemoteUri($langcode); + + logLine("Reading from $uri"); + $reader = newPoFileReader($uri, $langcode); + $header = $reader->getHeader(); + + $uri = getPublicUri($langcode, $langcode); + zapUri($uri); + logLine("Writing to $uri"); + $writer = new PoFileWriter(); + $writer->setURI($uri); + $writer->setHeader($header); + $writer->open(); + + processN($writer, $reader, -1); + + return $uri; +} + +function publicToDb($langcode) { + $uri = getPublicUri($langcode, $langcode); + + logLine("Reading from $uri using langcode: '$langcode'"); + $reader = new PoFileReader(); + $reader->setURI($uri); + $reader->setLangcode($langcode); + $reader->open(); + + logLine("Writing to DB"); + $writer = new PODbWriter(); + $writer->setLangcode($langcode); + $writer->setHeader($reader->getHeader()); + + $locale_plurals = variable_get('locale_translation_plurals', array()); + print_r(array("Should have $langcode" => $locale_plurals)); + + $options = $writer->getOptions(); + print_r($writer->getOptions()); + //$options['overwrite_options']['not_customized'] = TRUE; + $writer->setOptions($options); + print_r($writer->getOptions()); + print_r($options); + processN($writer, $reader, -1); + + print_r($writer->getReport()); +} + +function publicToMemory($langcode) { + $uri = getPublicUri($langcode, $langcode); + + logLine("Reading from $uri using langcode: '$langcode'"); + $reader = new PoFileReader(); + $reader->setURI($uri); + $reader->open(); + + logLine("Writing to Memory"); + $writer = new POMemoryWriter(); + $writer->setLangcode($langcode); + $writer->setHeader($reader->getHeader()); + + $locale_plurals = variable_get('locale_translation_plurals', array()); + print_r(array("Should have $langcode" => $locale_plurals)); + + processN($writer, $reader, 5); + + var_dump($writer); +} + +function pumpAround($langcode) { + $uri = remoteToPublic($langcode); + + $reader = newPoFileReader($uri, $langcode); + + logLine("Writing to DB"); + $writer = new PODbWriter(); + $writer->setLangcode($langcode); + + processN($writer, $reader, -1); + + + logLine("Reading from DB"); + $reader = new PODbReader(); + $reader->setLangcode($langcode); + $reader->setOptions(array()); + + var_dump($reader->getOptions()); + + $header = $reader->getHeader(); + + $uri = getPublicUri(__FUNCTION__ . '-db', $langcode); + zapUri($uri); + logLine("Writing to $uri"); + $writer = new PoFileWriter(); + $writer->setURI($uri); + $writer->setHeader($header); + $writer->open(); + processN($writer, $reader, -1); +} + +function runAll($langcode = 'nl') { + testWriter(); + testDBReaderState(); + testBatchStreamManager(); + testPoReader(); + testHeader(); + testFileToDb(); + testDbDump(); + testPOFileReader(); + testPOFileWriter(); + testDbToFile(); + testRemotePOPumper(); + testBatchSimulation(); + publicToMemory($langcode); + remoteToPublic($langcode); + publicToDb($langcode); + + pumpAround('ar'); + pumpAround('ca'); + pumpAround('nl'); +} + +$langcode = 'nl'; +//runAll(); + +function testT() { + $result = t('May', array(), array('langcode' => 'hr')); +} + +//testT(); + +remoteToPublic('af'); diff --git a/core/modules/locale/locale.bulk.inc b/core/modules/locale/locale.bulk.inc index ed3d3a6..f64749b 100644 --- a/core/modules/locale/locale.bulk.inc +++ b/core/modules/locale/locale.bulk.inc @@ -298,7 +298,7 @@ function locale_translate_batch_import($filepath, &$context) { // we can extract the language code to use for the import from the end. if (preg_match('!(/|\.)([^\./]+)\.po$!', $filepath, $langcode)) { $file = (object) array('filename' => drupal_basename($filepath), 'uri' => $filepath); - _locale_import_read_po('db-store', $file, array(), $langcode[2]); + _locale_import_po($file, $langcode, array(), LOCALE_NOT_CUSTOMIZED, TRUE); $context['results'][] = $filepath; } }