diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index 579add5601cb9353e52c81e37cf5b2163d179de4..bd7343a9c028327d871e4cb74f2d6ac9adb77ebf 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -3,6 +3,9 @@
Feeds 6.x 1.X XXXX
------------------
+- #850298 alex_b: ParserCSV: Support batching (only affects library, full parser
+ level batch support to be added later with #744660).
+- Minor cleanup of admin UI language and CSS.
- #647222 cglusky, jeffschuler: Specify input format for feed items.
Feeds 6.x 1.0 Beta 2, 2010-07-10
diff --git a/libraries/ParserCSV.inc b/libraries/ParserCSV.inc
index 5752f760932e452ef6856cf054d4aa0742533c65..28bd7a1bd1cfd83aaa0cb6dce64677994bbb7968 100644
--- a/libraries/ParserCSV.inc
+++ b/libraries/ParserCSV.inc
@@ -14,10 +14,12 @@
class ParserCSVIterator implements Iterator {
private $handle;
private $currentLine;
+ private $currentPos;
public function __construct($filepath) {
$this->handle = fopen($filepath, 'r');
$this->currentLine = NULL;
+ $this->currentPos = NULL;
}
function __destruct() {
@@ -26,9 +28,9 @@ class ParserCSVIterator implements Iterator {
}
}
- public function rewind() {
+ public function rewind($pos = 0) {
if ($this->handle) {
- fseek($this->handle, 0);
+ fseek($this->handle, $pos);
$this->next();
}
}
@@ -36,6 +38,7 @@ class ParserCSVIterator implements Iterator {
public function next() {
if ($this->handle) {
$this->currentLine = feof($this->handle) ? NULL : fgets($this->handle);
+ $this->currentPos = ftell($this->handle);
return $this->currentLine;
}
}
@@ -48,6 +51,10 @@ class ParserCSVIterator implements Iterator {
return $this->currentLine;
}
+ public function currentPos() {
+ return $this->currentPos;
+ }
+
public function key() {
return 'line';
}
@@ -62,6 +69,9 @@ class ParserCSV {
private $columnNames;
private $timeout;
private $timeoutReached;
+ private $startByte;
+ private $lineLimit;
+ private $lastLinePos;
public function __construct() {
$this->delimiter = ',';
@@ -69,6 +79,9 @@ class ParserCSV {
$this->columnNames = FALSE;
$this->timeout = FALSE;
$this->timeoutReached = FALSE;
+ $this->startByte = 0;
+ $this->lineLimit = 0;
+ $this->lastLinePos = 0;
}
/**
@@ -116,16 +129,57 @@ class ParserCSV {
/**
* After calling the parse() method, determine if the timeout (set by the
* setTimeout() method) has been reached.
+ *
+ * @deprecated Use lastLinePos() instead to determine whether a file has
+ * finished parsing.
*/
public function timeoutReached() {
return $this->timeoutReached;
}
+ /**
+ * Define the number of lines to parse in one parsing operation.
+ *
+ * By default, all lines of a file are being parsed.
+ */
+ public function setLineLimit($lines) {
+ $this->lineLimit = $lines;
+ }
+
+ /**
+ * Get the byte number where the parser left off after last parse() call.
+ *
+ * @return
+ * 0 if all lines or no line has been parsed, the byte position of where a
+ * timeout or the line limit has been reached otherwise. This position can be
+ * used to set the start byte for the next iteration after parse() has
+ * reached the timeout set with setTimeout() or the line limit set with
+ * setLineLimit().
+ *
+ * @see ParserCSV::setStartByte($start);
+ */
+ public function lastLinePos() {
+ return $this->lastLinePos;
+ }
+
+ /**
+ * Set the byte where file should be started to read.
+ *
+ * Useful when parsing a file in batches.
+ */
+ public function setStartByte($start) {
+ return $this->startByte = $start;
+ }
+
/**
* Parse CSV files into a two dimensional array.
*
* @param Iterator $lineIterator
* An Iterator object that yields line strings, e.g. ParserCSVIterator.
+ * @param $start
+ * The byte number from where to start parsing the file.
+ * @param $lines
+ * The number of lines to parse, 0 for all lines.
* @return
* Two dimensional array that contains the data in the CSV file.
*/
@@ -134,14 +188,11 @@ class ParserCSV {
$rows = array();
$this->timeoutReached = FALSE;
+ $this->lastLinePos = 0;
$maxTime = empty($this->timeout) ? FALSE : (microtime() + $this->timeout);
+ $linesParsed = 0;
- for ($lineIterator->rewind(); $lineIterator->valid(); $lineIterator->next()) {
- // If the timeout has been reached, quit parsing even if we're not yet done.
- if (!empty($maxTime) && microtime() > $maxTime) {
- $this->timeoutReached = TRUE;
- break;
- }
+ for ($lineIterator->rewind($this->startByte); $lineIterator->valid(); $lineIterator->next()) {
// Make really sure we've got lines without trailing newlines.
$line = trim($lineIterator->current(), "\r\n");
@@ -254,6 +305,19 @@ class ParserCSV {
}
}
$rows[] = $row;
+
+ // Quit parsing if timeout has been reached or requested lines have been
+ // reached.
+ if (!empty($maxTime) && microtime() > $maxTime) {
+ $this->timeoutReached = TRUE;
+ $this->lastLinePos = $lineIterator->currentPos();
+ break;
+ }
+ $linesParsed++;
+ if ($this->lineLimit && $linesParsed >= $this->lineLimit) {
+ $this->lastLinePos = $lineIterator->currentPos();
+ break;
+ }
}
return $rows;
}
diff --git a/tests/feeds/nodes.csv.php b/tests/feeds/nodes.csv.php
new file mode 100644
index 0000000000000000000000000000000000000000..fa8fe1cd81349eed8ace3b4d17754271a11c6324
--- /dev/null
+++ b/tests/feeds/nodes.csv.php
@@ -0,0 +1,79 @@
+<?php
+// $Id$
+/**
+ * @file
+ * Result of nodes.csv file parsed by ParserCSV.inc
+ */
+
+$control_result = array (
+ 0 =>
+ array (
+ 0 => 'Title',
+ 1 => 'Body',
+ 2 => 'published',
+ 3 => 'GUID',
+ ),
+ 1 =>
+ array (
+ 0 => 'Ut wisi enim ad minim veniam',
+ 1 => ' Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat.',
+ 2 => '205200720',
+ 3 => '2',
+ ),
+ 2 =>
+ array (
+ 0 => 'Duis autem vel eum iriure dolor',
+ 1 => ' Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi.',
+ 2 => '428112720',
+ 3 => '3',
+ ),
+ 3 =>
+ array (
+ 0 => 'Nam liber tempor',
+ 1 => ' Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum.',
+ 2 => '1151766000',
+ 3 => '1',
+ ),
+ 4 =>
+ array (
+ 0 => 'Typi non habent',
+ 1 => ' Typi non habent claritatem insitam; est usus legentis in iis qui facit eorum claritatem.',
+ 2 => '1256326995',
+ 3 => '4',
+ ),
+ 5 =>
+ array (
+ 0 => 'Lorem ipsum',
+ 1 => 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.',
+ 2 => '1251936720',
+ 3 => '1',
+ ),
+ 6 =>
+ array (
+ 0 => 'Investigationes demonstraverunt',
+ 1 => ' Investigationes demonstraverunt lectores legere me lius quod ii legunt saepius.',
+ 2 => '946702800',
+ 3 => '5',
+ ),
+ 7 =>
+ array (
+ 0 => 'Claritas est etiam',
+ 1 => ' Claritas est etiam processus dynamicus, qui sequitur mutationem consuetudium lectorum.',
+ 2 => '438112720',
+ 3 => '6',
+ ),
+ 8 =>
+ array (
+ 0 => 'Mirum est notare',
+ 1 => ' Mirum est notare quam littera gothica, quam nunc putamus parum claram, anteposuerit litterarum formas humanitatis per seacula quarta decima et quinta decima.',
+ 2 => '1151066000',
+ 3 => '7',
+ ),
+ 9 =>
+ array (
+ 0 => 'Eodem modo typi',
+ 1 => ' Eodem modo typi, qui nunc nobis videntur parum clari, fiant sollemnes in futurum.',
+ 2 => '1201936720',
+ 3 => '8',
+ ),
+);
diff --git a/tests/feeds_parser_csv.test b/tests/feeds_parser_csv.test
new file mode 100644
index 0000000000000000000000000000000000000000..a62f471028724c0471ccf162d5f9ed5e3af2dcdc
--- /dev/null
+++ b/tests/feeds_parser_csv.test
@@ -0,0 +1,85 @@
+<?php
+// $Id$
+/**
+ * @file
+ * Tests for ParserCSV library.
+ */
+
+/**
+ * Test aggregating a feed as node items.
+ *
+ * Using DrupalWebTestCase as DrupalUnitTestCase is broken in SimpleTest 2.8.
+ * Not inheriting from Feeds base class as ParserCSV should be moved out of
+ * Feeds at some time.
+ */
+class FeedsParserCSVTest extends DrupalWebTestCase {
+ /**
+ * Describe this test.
+ */
+ public function getInfo() {
+ return array(
+ 'name' => t('CSV Parser unit tests'),
+ 'description' => t('Base level test for Feeds\' built in CSV parser.'),
+ 'group' => t('Feeds'),
+ );
+ }
+
+ /**
+ * Test method.
+ */
+ public function test() {
+ feeds_include_library('ParserCSV.inc', 'ParserCSV');
+
+ $this->_testSimple();
+ $this->_testBatching();
+ }
+
+ /**
+ * Simple test of parsing functionality.
+ */
+ protected function _testSimple() {
+ $file = $this->absolutePath() .'/tests/feeds/nodes.csv';
+ include $this->absolutePath() .'/tests/feeds/nodes.csv.php';
+
+ $iterator = new ParserCSVIterator($file);
+ $parser = new ParserCSV();
+ $parser->setDelimiter(',');
+ $rows = $parser->parse($iterator);
+ $this->assertFalse($parser->lastLinePos(), t('Parser reports all lines parsed'));
+ $this->assertEqual(md5(serialize($rows)), md5(serialize($control_result)), t('Parsed result matches control result.'));
+ }
+
+ /**
+ * Test batching.
+ */
+ protected function _testBatching() {
+ $file = $this->absolutePath() .'/tests/feeds/nodes.csv';
+ include $this->absolutePath() .'/tests/feeds/nodes.csv.php';
+
+ // Set up parser with 2 lines to parse per call.
+ $iterator = new ParserCSVIterator($file);
+ $parser = new ParserCSV();
+ $parser->setDelimiter(',');
+ $parser->setLineLimit(2);
+ $rows = array();
+ $pos = 0;
+
+ // Call parser until all lines are parsed, then compare to control result.
+ do {
+ $parser->setStartByte($pos);
+ $rows = array_merge($rows, $parser->parse($iterator));
+ $pos = $parser->lastLinePos();
+ $this->assertTrue($parser->lastLinePos() || count($rows) == 10, t('Parser reports line limit correctly'));
+ }
+ while ($pos = $parser->lastLinePos());
+
+ $this->assertEqual(md5(serialize($rows)), md5(serialize($control_result)), t('Parsed result matches control result.'));
+ }
+
+ /**
+ * Absolute path to feeds.
+ */
+ public function absolutePath() {
+ return realpath(getcwd()) .'/'. drupal_get_path('module', 'feeds');
+ }
+}