From 20f3e88f127159b499000dc306c0de13a597d915 Mon Sep 17 00:00:00 2001 From: Colan Schwartz Date: Thu, 15 Dec 2011 01:00:00 -0500 Subject: [PATCH] Issue #1369874 by JeromeHollon, colan: Use fgetcsv() instead of own parser. --- plugins/FeedsCSVParser.inc | 141 ++++++++++++++++++++++++++++++++------------ 1 files changed, 103 insertions(+), 38 deletions(-) diff --git a/plugins/FeedsCSVParser.inc b/plugins/FeedsCSVParser.inc index 8f4f67f..a073f13 100644 --- a/plugins/FeedsCSVParser.inc +++ b/plugins/FeedsCSVParser.inc @@ -9,34 +9,50 @@ class FeedsCSVParser extends FeedsParser { * Implements FeedsParser::parse(). */ public function parse(FeedsSource $source, FeedsFetcherResult $fetcher_result) { + // Get the configuration information. $source_config = $source->getConfigFor($this); $state = $source->state(FEEDS_PARSE); + $headers_provided = empty($source_config['no_headers']); + $delimiter = $source_config['delimiter'] == 'TAB' ? + "\t" : $source_config['delimiter']; - // Load and configure parser. - feeds_include_library('ParserCSV.inc', 'ParserCSV'); - $parser = new ParserCSV(); - $delimiter = $source_config['delimiter'] == 'TAB' ? "\t" : $source_config['delimiter']; - $parser->setDelimiter($delimiter); - - $iterator = new ParserCSVIterator($fetcher_result->getFilePath()); - if (empty($source_config['no_headers'])) { - // Get first line and use it for column names, convert them to lower case. - $header = $this->parseHeader($parser, $iterator); - if (!$header) { + // Get the file path & position. + $filePath = realpath($fetcher_result->getFilePath()); + + // Open the file. + $handle = fopen($filePath, "r"); + + // If we're configured to get the header rows of column names, do so now. + if ($headers_provided) { + $headers = $this->parseHeader($handle, $delimiter); + if (!$headers) { return; } - $parser->setColumnNames($header); + } else /* no headers; set empty list */ { + $headers = array(); + } + + // If we were previously deeper into the file, continue from there. + // Otherwise, we'll be starting at the beginning of the data again. + if (isset($state->pointer) && ($state->pointer > ftell($handle))) { + fseek($handle, $state->pointer); } - // Determine section to parse, parse. - $start = $state->pointer ? $state->pointer : $parser->lastLinePos(); - $limit = $source->importer->getLimit(); - $rows = $this->parseItems($parser, $iterator, $start, $limit); + // Parse all of the row items and stick them in an array. + $rows = $this->parseItems( + $handle, + $headers, + $source->importer->getLimit(), + $delimiter + ); + + // Get the final position before closing the file. + $state->pointer = ftell($handle); + fclose($handle); // Report progress. $state->total = filesize($fetcher_result->getFilePath()); - $state->pointer = $parser->lastLinePos(); - $progress = $parser->lastLinePos() ? $parser->lastLinePos() : $state->total; + $progress = $state->pointer ? $state->pointer : $state->total; $state->progress($state->total, $progress); // Create a result object and return it. @@ -45,39 +61,88 @@ class FeedsCSVParser extends FeedsParser { /** * Get first line and use it for column names, convert them to lower case. - * Be aware that the $parser and iterator objects can be modified in this - * function since they are passed in by reference * - * @param ParserCSV $parser - * @param ParserCSVIterator $iterator + * @param $handle + * The file handle. + * @param $delimiter + * The field delimiter * @return * An array of lower-cased column names to use as keys for the parsed items. */ - protected function parseHeader(ParserCSV $parser, ParserCSVIterator $iterator) { - $parser->setLineLimit(1); - $rows = $parser->parse($iterator); - if (!count($rows)) { + protected function parseHeader($handle, $delimiter) { + + // Get the row of column names. + $headers = fgetcsv($handle, 0, $delimiter); + + // Back out if there was a problem; don't continue processing. + if (!$headers) { return FALSE; } - $header = array_shift($rows); - foreach ($header as $i => $title) { - $header[$i] = trim(drupal_strtolower($title)); + + // Convert to lowercase. + foreach ($headers as $index => $title) { + $headers[$index] = trim(drupal_strtolower($title)); } - return $header; + + // Return the list of column names. + return $headers; } /** - * Parse all of the items from the CSV. + * Parse all of the row items from the file. * - * @param ParserCSV $parser - * @param ParserCSVIterator $iterator + * @param $handle + * The file handle. + * @param $headers + * The field/column names, if any were provided. + * @param $limit + * The maximum number of lines to parse. + * @param $delimiter + * The field delimiter * @return - * An array of rows of the CSV keyed by the column names previously set + * An array of CSV rows possibly keyed by the column names previously set */ - protected function parseItems(ParserCSV $parser, ParserCSVIterator $iterator, $start = 0, $limit = 0) { - $parser->setLineLimit($limit); - $parser->setStartByte($start); - $rows = $parser->parse($iterator); + protected function parseItems($handle, $headers, $limit, $delimiter) { + + // Act on each line of the file until we hit the limit or run out of lines. + $rows = array(); + for ($i = 0; $i < $limit; $i++) { + + // Break each line into an array of fields. + $fields = fgetcsv($handle, 0, $delimiter); + + /* + * "If PHP is not properly recognizing the line endings when reading + * files either on or created by a Macintosh computer, enabling the + * auto_detect_line_endings run-time configuration option may help + * resolve the problem." (Source: the fgetcsv() man page) + */ + + // Skip blank lines. + if (($fields === FALSE) || ($fields === NULL)) { + continue; + } + + // Before we assign proper field names to each field, we need to make + // sure that we actually have them. + if (!empty($headers)) { + + // Rename the rows so that we can match them properly. + $row = array(); + foreach ($headers as $column_name) { + $field = array_shift($fields); + $row[$column_name] = isset($field) ? $field : ''; + } + } else /* headers not provided */ { + + // We weren't provided with any field names, so just keep the current + // indexing as it is. + $row = $fields; + } + + // Add the completed row to the list of total rows. + $rows[] = $row; + } return $rows; } -- 1.7.0.4