--- includes/locale.inc 2009-08-22 15:34:17.000000000 +0100 +++ includes/locale.inc 2009-09-05 17:56:54.000000000 +0100 @@ -1208,7 +1208,7 @@ function _locale_import_po($file, $langc } /** - * Parses Gettext Portable Object file into an array + * Open and passes on Gettext Portable Object file for parsing * * @param $op * Storage operation type: db-store or mem-store @@ -1223,18 +1223,66 @@ function _locale_import_po($file, $langc */ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL, $group = 'default') { - $fd = fopen($file->uri, "rb"); // File will get closed by PHP on return - if (!$fd) { - _locale_import_message('The translation import failed, because the file %filename could not be read.', $file); + if ($size = filesize($file->uri)) { + // File will get closed by PHP on return. + $fd = fopen(drupal_realpath($file->uri), "rb"); + if (!$fd) { + _locale_import_message('The translation import failed, because the file %filename could not be read.', $file); + return FALSE; + } + + // If the file size is above our seek limit, we try and seek into the file + // in small chunks. + if ($size > variable_get('locale_import_po_seek_limit', 50000)) { + $chunk_size = variable_get('locale_import_po_chunk_size', 100); + $seek_position = 0; + while(is_int($seek_position = _locale_import_parse_po($op, $file, $fd, $mode, $lang, $group, $seek_position, $chunk_size))); + } + else { + _locale_import_parse_po($op, $file, $fd, $mode, $lang, $group); + } + } + else { + _locale_import_message('The translation import failed, because the file %filename could not be found or was empty.', $file); return FALSE; } + +} + +/** + * Parses Gettext Portable Object file into an array + * + * @param $op + * Storage operation type: db-store or mem-store + * @param $file + * Drupal file object corresponding to the PO file being imported + * @param $fd + * File system pointer for the same PO file, opened by _locale_import_read_po + * @param $mode + * Should existing translations be replaced LOCALE_IMPORT_KEEP or LOCALE_IMPORT_OVERWRITE + * @param $lang + * Language code + * @param $group + * Text group to import PO file into (eg. 'default' for interface translations) + * @param $seek_position + * Position from which to start/continue parsing (that position returned by this function in a previous + * iteration) + * @param $chunk_size + * Approximate number of lines to parse in one iteration - but continue to end of current entry. If not + * set, or set to zero, continue to end of file. + */ +function _locale_import_parse_po($op, $file, $fd, $mode = NULL, $lang = NULL, $group = 'default', $seek_position = NULL, $chunk_size = NULL) { $context = "COMMENT"; // Parser context: COMMENT, MSGID, MSGID_PLURAL, MSGSTR and MSGSTR_ARR $current = array(); // Current entry being read $plural = 0; // Current plural form $lineno = 0; // Current line - + // Set file position indicator to where previous iteration left off. + if ($seek_position !== NULL) { + fseek($fd, $seek_position); + } while (!feof($fd)) { + $seek_position_before_line = ftell($fd); $line = fgets($fd, 10*1024); // A line should not be this long if ($lineno == 0) { // The first line might come with a UTF-8 BOM, which should be removed. @@ -1247,11 +1295,17 @@ function _locale_import_read_po($op, $fi if ($context == "COMMENT") { // Already in comment context: add $current["#"][] = substr($line, 1); } - elseif (($context == "MSGSTR") || ($context == "MSGSTR_ARR")) { // End current entry, start a new one + elseif (($context == "MSGSTR") || ($context == "MSGSTR_ARR")) { + // End current entry, start a new one, or end iteration if chunk too long. _locale_import_one_string($op, $current, $mode, $lang, $file, $group); $current = array(); - $current["#"][] = substr($line, 1); - $context = "COMMENT"; + if ($chunk_size && ($lineno > $chunk_size)) { + return ($seek_position_before_line); + } + else { + $current["#"][] = substr($line, 1); + $context = "COMMENT"; + } } else { // Parse error _locale_import_message('The translation file %filename contains an error: "msgstr" was expected but not found on line %line.', $file, $lineno); @@ -1273,9 +1327,12 @@ function _locale_import_read_po($op, $fi $context = "MSGID_PLURAL"; } elseif (!strncmp("msgid", $line, 5)) { - if ($context == "MSGSTR") { // End current entry, start a new one + if ($context == "MSGSTR") { // End current entry, start a new one, or end iteration if chunk too long _locale_import_one_string($op, $current, $mode, $lang, $file, $group); $current = array(); + if ($chunk_size && ($lineno > $chunk_size)) { + return ($seek_position_before_line); + } } elseif ($context == "MSGID") { // Already in this context? Parse error _locale_import_message('The translation file %filename contains an error: "msgid" is unexpected on line %line.', $file, $lineno); @@ -1370,12 +1427,12 @@ function _locale_import_read_po($op, $fi // End of PO file, flush last entry if (!empty($current) && !empty($current['msgstr'])) { _locale_import_one_string($op, $current, $mode, $lang, $file, $group); + return TRUE; } elseif ($context != "COMMENT") { _locale_import_message('The translation file %filename ended unexpectedly at line %line.', $file, $lineno); return FALSE; } - } /**