Skip to content

Commit

Permalink
MDL-34075 - lib - Alteration to the csv import lib to include rfc-418…
Browse files Browse the repository at this point in the history 
…0 compliance
  • Loading branch information
@abgreeve
abgreeve committed Aug 28, 2012
1 parent d71c486 commit 96729f6
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 40 deletions.
84 changes: 44 additions & 40 deletions lib/csvlib.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,10 @@ function csv_import_reader($iid, $type) {
* @param string $encoding content encoding
* @param string $delimiter_name separator (comma, semicolon, colon, cfg)
* @param string $column_validation name of function for columns validation, must have one param $columns
* @param string $enclosure field wrapper. One character only.
* @return bool false if error, count of data lines if ok; use get_error() to get error string
*/
function load_csv_content(&$content, $encoding, $delimiter_name, $column_validation=null) {
function load_csv_content(&$content, $encoding, $delimiter_name, $column_validation=null, $enclosure='"') {
global $USER, $CFG;

$this->close();
Expand All @@ -89,62 +90,65 @@ function load_csv_content(&$content, $encoding, $delimiter_name, $column_validat
$content = textlib::trim_utf8_bom($content);
// Fix mac/dos newlines
$content = preg_replace('!\r\n?!', "\n", $content);
// is there anyting in file?
$columns = strtok($content, "\n");
if ($columns === false) {
$this->_error = get_string('csvemptyfile', 'error');
return false;
}

$csv_delimiter = csv_import_reader::get_delimiter($delimiter_name);
$csv_encode = csv_import_reader::get_encoded_delimiter($delimiter_name);
// $csv_encode = csv_import_reader::get_encoded_delimiter($delimiter_name);

// create a temporary file and store the csv file there.
$fp = tmpfile();
fwrite($fp, $content);
fseek($fp, 0);
// Create an array to store the imported data for error checking.
$columns = array();
// str_getcsv doesn't iterate through the csv data properly. It has
// problems with line returns.
while ($fgetdata = fgetcsv($fp, 0, $csv_delimiter, $enclosure)) {
$columns[] = $fgetdata;
}
$col_count = 0;

// process header - list of columns
$columns = explode($csv_delimiter, $columns);
$col_count = count($columns);
if ($col_count === 0) {
if (!isset($columns[0])) {
$this->_error = get_string('csvemptyfile', 'error');
fclose($fp);
return false;
} else {
$col_count = count($columns[0]);
}

foreach ($columns as $key=>$value) {
$columns[$key] = str_replace($csv_encode, $csv_delimiter, trim($value));
}
// Column validation.
if ($column_validation) {
$result = $column_validation($columns);
$result = $column_validation($columns[0]);
if ($result !== true) {
$this->_error = $result;
fclose($fp);
return false;
}
}
$this->_columns = $columns; // cached columns

// open file for writing
$filename = $CFG->tempdir.'/csvimport/'.$this->_type.'/'.$USER->id.'/'.$this->_iid;
$fp = fopen($filename, "w");
fwrite($fp, serialize($columns)."\n");

// again - do we have any data for processing?
$line = strtok("\n");
$data_count = 0;
while ($line !== false) {
$line = explode($csv_delimiter, $line);
foreach ($line as $key=>$value) {
$line[$key] = str_replace($csv_encode, $csv_delimiter, trim($value));
}
if (count($line) !== $col_count) {
// this is critical!!
$this->_columns = $columns[0]; // cached columns
// check to make sure that the data columns match up with the headers.
foreach ($columns as $rowdata) {
if (count($rowdata) !== $col_count) {
$this->_error = get_string('csvweirdcolumns', 'error');
fclose($fp);
$this->cleanup();
return false;
}
fwrite($fp, serialize($line)."\n");
$data_count++;
$line = strtok("\n");
}

$filename = $CFG->tempdir.'/csvimport/'.$this->_type.'/'.$USER->id.'/'.$this->_iid;
$filepointer = fopen($filename, "w");
// The information has been stored in csv format, as serialized data has issues
// with special characters and line returns.
$storedata = csv_export_writer::print_array($columns, ',', '"', true);
fwrite($filepointer, $storedata);

fclose($fp);
return $data_count;
fclose($filepointer);

$datacount = count($columns);
return $datacount;
}

/**
Expand All @@ -164,12 +168,12 @@ function get_columns() {
return false;
}
$fp = fopen($filename, "r");
$line = fgets($fp);
$line = fgetcsv($fp);
fclose($fp);
if ($line === false) {
return false;
}
$this->_columns = unserialize($line);
$this->_columns = $line;
return $this->_columns;
}

Expand All @@ -194,7 +198,7 @@ function init() {
return false;
}
//skip header
return (fgets($this->_fp) !== false);
return (fgetcsv($this->_fp) !== false);
}

/**
Expand All @@ -206,8 +210,8 @@ function next() {
if (empty($this->_fp) or feof($this->_fp)) {
return false;
}
if ($ser = fgets($this->_fp)) {
return unserialize($ser);
if ($ser = fgetcsv($this->_fp)) {
return $ser;
} else {
return false;
}
Expand Down
42 changes: 42 additions & 0 deletions lib/tests/csvclass_test.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ class csvclass_testcase extends advanced_testcase {

var $testdata = array();
var $teststring = '';
var $teststring2 = '';
var $teststring3 = '';

protected function setUp(){

Expand All @@ -57,6 +59,11 @@ protected function setUp(){
"Phillip Jenkins","<p>This field has </p>
<p>Multiple lines</p>
<p>and also contains ""double quotes""</p>",Yebisu
';

$this->teststring2 = 'fullname,"description of things",beer
"Fred Flint","<p>Find the stone inside the box</p>",Asahi,"A fourth column"
"Sarah Smith","<p>How are the people next door?</p>,Yebisu,"Forget the next"
';
}

Expand All @@ -71,5 +78,40 @@ public function test_csv_functions() {

$test_data = csv_export_writer::print_array($this->testdata, 'comma', '"', true);
$this->assertEquals($test_data, $this->teststring);

// Testing that the content is imported correctly.
$iid = csv_import_reader::get_new_iid('lib');
$csvimport = new csv_import_reader($iid, 'lib');
$contentcount = $csvimport->load_csv_content($this->teststring, 'utf-8', 'comma');
$csvimport->init();
$dataset = array();
$dataset[] = $csvimport->get_columns();
while ($record = $csvimport->next()) {
$dataset[] = $record;
}
$csvimport->cleanup();
$csvimport->close();
$this->assertEquals($dataset, $this->testdata);

// Testing for the wrong count of columns.
$errortext = get_string('csvweirdcolumns', 'error');
$iid = csv_import_reader::get_new_iid('lib');
$csvimport = new csv_import_reader($iid, 'lib');
$contentcount = $csvimport->load_csv_content($this->teststring2, 'utf-8', 'comma');
$importerror = $csvimport->get_error();
$csvimport->cleanup();
$csvimport->close();
$this->assertEquals($importerror, $errortext);

// Testing for empty content
$errortext = get_string('csvemptyfile', 'error');

$iid = csv_import_reader::get_new_iid('lib');
$csvimport = new csv_import_reader($iid, 'lib');
$contentcount = $csvimport->load_csv_content($this->teststring3, 'utf-8', 'comma');
$importerror = $csvimport->get_error();
$csvimport->cleanup();
$csvimport->close();
$this->assertEquals($importerror, $errortext);
}
}

0 comments on commit 96729f6

Please sign in to comment.