diff --git a/libraries/ParserCSV.inc b/libraries/ParserCSV.inc index 4ddc77a..a5262c9 100644 --- a/libraries/ParserCSV.inc +++ b/libraries/ParserCSV.inc @@ -76,6 +76,8 @@ class ParserCSV { public function __construct() { $this->delimiter = ','; + $this->from_encoding = 'UTF-8'; + $this->to_encoding = 'UTF-8'; $this->skipFirstLine = FALSE; $this->columnNames = FALSE; $this->timeout = FALSE; @@ -95,6 +97,14 @@ class ParserCSV { } /** + * Set the source file encoding. + * By default, UTF-8. + */ + public function setEncoding($encoding) { + $this->from_encoding = $encoding; + } + + /** * Set this to TRUE if the parser should skip the first line of the CSV text, * which might be desired if the first line contains the column names. * By default, this is set to FALSE and the first line is not skipped. @@ -197,7 +207,7 @@ class ParserCSV { for ($lineIterator->rewind($this->startByte); $lineIterator->valid(); $lineIterator->next()) { // Make really sure we've got lines without trailing newlines. - $line = trim($lineIterator->current(), "\r\n"); + $line = trim($this->fixEncoding($lineIterator->current()), "\r\n"); // Skip empty lines. if (empty($line)) { @@ -237,7 +247,7 @@ class ParserCSV { } // Ok, so, on with fetching the next line, as mentioned above. $currentField .= "\n"; - $line = trim($lineIterator->current(), "\r\n"); + $line = trim($this->fixEncoding($lineIterator->current()), "\r\n"); $currentIndex = 0; continue; } @@ -325,4 +335,28 @@ class ParserCSV { } return $rows; } + + /** + * Converts encoding of input data + * + * @param $data + * A chunk of data + * @return + * Data in correct encoding or throws exceptions if + * detected encoding doesn't match. + */ + private function fixEncoding($data) { + + if (extension_loaded('mbstring')) { + if (mb_check_encoding($data, $this->from_encoding)) { + // Always convert encoding to UTF-8 protect from SQL errors. + $data = mb_convert_encoding($data, $this->to_encoding, $this->from_encoding); + } + else { + throw new Exception(t('Source file is not in %encoding encoding.', array('%encoding' => $this->from_encoding))); + } + } + + return $data; + } } diff --git a/plugins/FeedsCSVParser.inc b/plugins/FeedsCSVParser.inc index 04d79b6..eccbc52 100644 --- a/plugins/FeedsCSVParser.inc +++ b/plugins/FeedsCSVParser.inc @@ -22,6 +22,7 @@ class FeedsCSVParser extends FeedsParser { $parser = new ParserCSV(); $delimiter = $source_config['delimiter'] == 'TAB' ? "\t" : $source_config['delimiter']; $parser->setDelimiter($delimiter); + $parser->setEncoding($source_config['encoding']); $iterator = new ParserCSVIterator($fetcher_result->getFilePath()); if (empty($source_config['no_headers'])) { @@ -107,6 +108,7 @@ class FeedsCSVParser extends FeedsParser { public function sourceDefaults() { return array( 'delimiter' => $this->config['delimiter'], + 'encoding' => $this->config['encoding'], 'no_headers' => $this->config['no_headers'], ); } @@ -166,6 +168,8 @@ class FeedsCSVParser extends FeedsParser { '#description' => t('Check if the imported CSV file does not start with a header row. If checked, mapping sources must be named \'0\', \'1\', \'2\' etc.'), '#default_value' => isset($source_config['no_headers']) ? $source_config['no_headers'] : 0, ); + $form['encoding'] = $this->configEncodingForm(); + $form['encoding']['#default_value'] = isset($source_config['encoding']) ? $source_config['encoding'] : $form['encoding']['#default_value']; return $form; } @@ -175,6 +179,7 @@ class FeedsCSVParser extends FeedsParser { public function configDefaults() { return array( 'delimiter' => ',', + 'encoding' => 'UTF-8', 'no_headers' => 0, ); } @@ -203,6 +208,31 @@ class FeedsCSVParser extends FeedsParser { '#description' => t('Check if the imported CSV file does not start with a header row. If checked, mapping sources must be named \'0\', \'1\', \'2\' etc.'), '#default_value' => $this->config['no_headers'], ); + $form['encoding'] = $this->configEncodingForm(); + return $form; + } + + public function configEncodingForm() { + if (extension_loaded('mbstring')) { + // Get the system's list of available encodings. + $options = mb_list_encodings(); + // Make the key/values the same in the array. + $options = array_combine($options, $options); + // Sort alphabetically not-case sensitive. + natcasesort($options); + return array( + '#type' => 'select', + '#title' => t('File encoding'), + '#description' => t('Performs character encoding conversion from selected option to UTF-8.'), + '#options' => $options, + '#default_value' => $this->config['encoding'], + ); + } + else { + return array( + '#markup' => '' . t('PHP mbstring extension must be available for character encoding conversion.') . '', + ); + } return $form; } diff --git a/tests/feeds/encoding.csv.php b/tests/feeds/encoding.csv.php new file mode 100644 index 0000000..05f300c --- /dev/null +++ b/tests/feeds/encoding.csv.php @@ -0,0 +1,33 @@ +_testSimple(); $this->_testBatching(); + $this->_testEncodingConversion(); + $this->_testEncodingConversionFailure(); } /** @@ -54,6 +56,51 @@ class ParserCSVTest extends DrupalWebTestCase { } /** + * Simple test of encoding conversion prior to parsing. + */ + protected function _testEncodingConversion() { + // Pull in the $control_result array. + include $this->absolutePath() . '/tests/feeds/encoding.csv.php'; + + $encodings = $this->getEncodings(); + foreach ($encodings as $encoding) { + $file = $this->absolutePath() . "/tests/feeds/encoding_{$encoding}.csv"; + $iterator = new ParserCSVIterator($file); + $parser = new ParserCSV(); + $parser->setDelimiter(','); + $parser->setEncoding($encoding); + $rows = $parser->parse($iterator); + $this->assertFalse($parser->lastLinePos(), t('CSV reports all lines parsed, with encoding: %encoding', array('%encoding' => $encoding))); + $this->assertEqual(md5(serialize($rows)), md5(serialize($control_result)), t('Converted and parsed result matches control result.')); + } + } + + /** + * Simple test of failed encoding conversion prior to parsing. + */ + protected function _testEncodingConversionFailure() { + // Pull in the $control_result array. + include $this->absolutePath() . '/tests/feeds/encoding.csv.php'; + + $encodings = $this->getEncodings(); + foreach ($encodings as $encoding) { + $file = $this->absolutePath() . "/tests/feeds/encoding_{$encoding}.csv"; + $iterator = new ParserCSVIterator($file); + $parser = new ParserCSV(); + $parser->setDelimiter(','); + // Attempt to read file as UTF-8. + $parser->setEncoding('UTF-8'); + try { + $rows = $parser->parse($iterator); + $this->assertTrue(FALSE, t('Incorrect conversion attempt did not throw exception.')); + } + catch (Exception $e) { + $this->assertNotNull($e->getMessage(), t('Incorrect conversion attempt throws exception.')); + } + } + } + + /** * Test batching. */ protected function _testBatching() { @@ -100,4 +147,11 @@ class ParserCSVTest extends DrupalWebTestCase { 'tab' => "\t", ); } + + static function getEncodings() { + return array( + 'SJIS-win', + 'SJIS', + ); + } }