(manually merged from branches/roa)

git-svn-id: svn://svn.silverstripe.com/silverstripe/open/modules/sapphire/trunk@60225 467b73ca-7a2a-4603-9d3b-597d59a354a9
This commit is contained in:
Ingo Schommer 2008-08-09 05:45:43 +00:00
parent d6463f0b1f
commit 1162a186db
3 changed files with 111 additions and 31 deletions

View File

@ -1,6 +1,8 @@
<?php <?php
/** /**
* @package tests * @package tests
*
* @todo Test with columnn headers and custom mappings
*/ */
class CsvBulkLoaderTest extends SapphireTest { class CsvBulkLoaderTest extends SapphireTest {
static $fixture_file = 'sapphire/tests/CsvBulkLoaderTest.yml'; static $fixture_file = 'sapphire/tests/CsvBulkLoaderTest.yml';

View File

@ -5,9 +5,9 @@
* *
* You can configure column-handling, * You can configure column-handling,
* *
* @todo Allow updating of existing records based on user-specified unique criteria/callbacks (e.g. database ID)
* @todo Add support for adding/editing has_many relations. * @todo Add support for adding/editing has_many relations.
* @todo Add support for deep chaining of relation properties (e.g. Player.Team.Stats.GoalCount) * @todo Add support for deep chaining of relation properties (e.g. Player.Team.Stats.GoalCount)
* @todo Character conversion
* *
* @see http://rfc.net/rfc4180.html * @see http://rfc.net/rfc4180.html
* @package cms * @package cms
@ -40,6 +40,9 @@ abstract class BulkLoader extends ViewableData {
* *
* The column count should match the count of array elements, * The column count should match the count of array elements,
* fill with NULL values if you want to skip certain columns. * fill with NULL values if you want to skip certain columns.
*
* You can also combine {@link $hasHeaderRow} = true and {@link $columnMap}
* and omit the NULL values in your map.
* *
* Supports one-level chaining of has_one relations and properties with dot notation * Supports one-level chaining of has_one relations and properties with dot notation
* (e.g. Team.Title). The first part has to match a has_one relation name * (e.g. Team.Title). The first part has to match a has_one relation name
@ -129,11 +132,13 @@ abstract class BulkLoader extends ViewableData {
* Useful to analyze the input and give the users a chance to influence * Useful to analyze the input and give the users a chance to influence
* it through a UI. * it through a UI.
* *
* @todo Implement preview()
*
* @param string $filepath Absolute path to the file we're importing * @param string $filepath Absolute path to the file we're importing
* @return array See {@link self::processAll()} * @return array See {@link self::processAll()}
*/ */
public function preview($filepath) { public function preview($filepath) {
return $this->processAll($filepath, true); user_error("BulkLoader::preview(): Not implemented", E_USER_ERROR);
} }
/** /**
@ -154,10 +159,11 @@ abstract class BulkLoader extends ViewableData {
* Process a single record from the file. * Process a single record from the file.
* *
* @param array $record An map of the data, keyed by the header field defined in {@link self::$columnMap} * @param array $record An map of the data, keyed by the header field defined in {@link self::$columnMap}
* @param array $columnMap
* @param boolean $preview * @param boolean $preview
* @return ArrayData @see self::processAll() * @return ArrayData @see self::processAll()
*/ */
abstract protected function processRecord($record, $preview = false); abstract protected function processRecord($record, $columnMap, $preview = false);
/** /**
* Return a FieldSet containing all the options for this form; this * Return a FieldSet containing all the options for this form; this
@ -175,6 +181,38 @@ abstract class BulkLoader extends ViewableData {
return ($title = $this->stat('title')) ? $title : $this->class; return ($title = $this->stat('title')) ? $title : $this->class;
} }
/**
* Get a specification of all available columns and relations on the used model.
* Useful for generation of spec documents for technical end users.
*
* Return Format:
* <example>
* array(
* 'fields' => array('myFieldName'=>'myDescription'),
* 'relations' => array('myRelationName'=>'myDescription'),
* )
* </example>
*
* @todo Mix in custom column mappings
* @usedby {@link ModelAdmin}
*
* @return array
**/
public function getImportSpec() {
$spec = array();
// get database columns (fieldlabels include fieldname as a key)
$spec['fields'] = (array)singleton($this->objectClass)->fieldLabels();
$has_ones = singleton($this->objectClass)->has_one();
$has_manys = singleton($this->objectClass)->has_many();
$many_manys = singleton($this->objectClass)->many_many();
$spec['relations'] = (array)$has_ones + (array)$has_manys + (array)$many_manys;
return $spec;
}
/** /**
* Determines if a specific field is null. * Determines if a specific field is null.
* Can be useful for unusual "empty" flags in the file, * Can be useful for unusual "empty" flags in the file,

View File

@ -38,21 +38,51 @@ class CsvBulkLoader extends BulkLoader {
if(!$file) return false; if(!$file) return false;
$return = new DataObjectSet(); $return = new DataObjectSet();
// assuming that first row is column naming if no columnmap is passed
if($this->hasHeaderRow && $this->columnMap) { if($this->hasHeaderRow && $this->columnMap) {
$columnRow = fgetcsv($file, 0, $this->delimiter, $this->enclosure); $columnRow = fgetcsv($file, 0, $this->delimiter, $this->enclosure);
$columnMap = $this->columnMap; $columnMap = array();
foreach($columnRow as $k => $origColumnName) {
$origColumnName = trim($origColumnName);
if(isset($this->columnMap[$origColumnName])) {
$columnMap[$origColumnName] = $this->columnMap[$origColumnName];
} else {
$columnMap[$origColumnName] = null;
}
}
} elseif($this->columnMap) { } elseif($this->columnMap) {
$columnMap = $this->columnMap; $columnMap = $this->columnMap;
} else { } else {
// assuming that first row is column naming if no columnmap is passed
$columnRow = fgetcsv($file, 0, $this->delimiter, $this->enclosure); $columnRow = fgetcsv($file, 0, $this->delimiter, $this->enclosure);
$columnMap = array_combine($columnRow, $columnRow); $columnMap = array_combine($columnRow, $columnRow);
} }
$rowIndex = 0;
while (($row = fgetcsv($file, 0, $this->delimiter, $this->enclosure)) !== FALSE) { while (($row = fgetcsv($file, 0, $this->delimiter, $this->enclosure)) !== FALSE) {
$indexedRow = array_combine(array_values($columnMap), array_values($row)); $rowIndex++;
$return->push($this->processRecord($indexedRow));
/*
// the columnMap should have the same amount of columns as each record row
if(count(array_keys($columnMap)) == count(array_values($row))) {
user_error("CsvBulkLoader::processAll(): Columns in row {$rowIndex} don't match the \$columnMap", E_USER_WARNING);
}
*/
$indexedRow = array();
foreach($columnMap as $origColumnName => $fieldName) {
// in case the row has less fields than the columnmap,
// ignore the "leftover" mappings
if(!isset($row[count($indexedRow)])) {
user_error("CsvBulkLoader::processAll(): Columns in row {$rowIndex} don't match the \$columnMap", E_USER_NOTICE);
continue;
}
$indexedRow[$origColumnName] = $row[count($indexedRow)];
}
$return->push($this->processRecord($indexedRow, $columnMap));
} }
fclose($file); fclose($file);
@ -60,26 +90,28 @@ class CsvBulkLoader extends BulkLoader {
return $return; return $return;
} }
protected function processRecord($record, $columnMap, $preview = false) {
protected function processRecord($record, $preview = false) {
$class = $this->objectClass; $class = $this->objectClass;
// find existing object, or create new one // find existing object, or create new one
$existingObj = $this->findExistingObject($record); $existingObj = $this->findExistingObject($record, $columnMap);
$obj = ($existingObj) ? $existingObj : new $class(); $obj = ($existingObj) ? $existingObj : new $class();
// first run: find/create any relations and store them on the object // first run: find/create any relations and store them on the object
// we can't combine runs, as other columns might rely on the relation being present // we can't combine runs, as other columns might rely on the relation being present
$relations = array(); $relations = array();
foreach($record as $origColumnName => $val) {
foreach($record as $key => $val) { $fieldName = $columnMap[$origColumnName];
//if($this->isNullValue($val)) continue;
// don't bother querying of value is not set
if($this->isNullValue($val)) continue;
// checking for existing relations // checking for existing relations
if(isset($this->relationCallbacks[$key])) { if(isset($this->relationCallbacks[$fieldName])) {
// trigger custom search method for finding a relation based on the given value // trigger custom search method for finding a relation based on the given value
// and write it back to the relation (or create a new object) // and write it back to the relation (or create a new object)
$relationName = $this->relationCallbacks[$key]['relationname']; $relationName = $this->relationCallbacks[$fieldName]['relationname'];
$relationObj = $obj->{$this->relationCallbacks[$key]['callback']}($val, $record); $relationObj = $obj->{$this->relationCallbacks[$fieldName]['callback']}($val, $record);
if(!$relationObj || !$relationObj->exists()) { if(!$relationObj || !$relationObj->exists()) {
$relationClass = $obj->has_one($relationName); $relationClass = $obj->has_one($relationName);
$relationObj = new $relationClass(); $relationObj = new $relationClass();
@ -88,9 +120,9 @@ class CsvBulkLoader extends BulkLoader {
$obj->setComponent($relationName, $relationObj); $obj->setComponent($relationName, $relationObj);
$obj->{"{$relationName}ID"} = $relationObj->ID; $obj->{"{$relationName}ID"} = $relationObj->ID;
$obj->write(); $obj->write();
} elseif(strpos($key, '.') !== false) { } elseif(strpos($fieldName, '.') !== false) {
// we have a relation column with dot notation // we have a relation column with dot notation
list($relationName,$columnName) = split('\.', $key); list($relationName,$columnName) = split('\.', $fieldName);
$relationObj = $obj->getComponent($relationName); // always gives us an component (either empty or existing) $relationObj = $obj->getComponent($relationName); // always gives us an component (either empty or existing)
$obj->setComponent($relationName, $relationObj); $obj->setComponent($relationName, $relationObj);
$relationObj->write(); $relationObj->write();
@ -102,21 +134,25 @@ class CsvBulkLoader extends BulkLoader {
} }
$id = ($preview) ? 0 : $obj->write(); $id = ($preview) ? 0 : $obj->write();
// second run: save data // second run: save data
foreach($record as $key => $val) { foreach($record as $origColumnName => $val) {
if($obj->hasMethod("import{$key}")) { $fieldName = $columnMap[$origColumnName];
$obj->{"import{$key}"}($val, $record);
} elseif(strpos($key, '.') !== false) { if($this->isNullValue($val, $fieldName)) continue;
if($obj->hasMethod("import{$fieldName}")) {
$obj->{"import{$fieldName}"}($val, $record);
} elseif(strpos($fieldName, '.') !== false) {
// we have a relation column // we have a relation column
list($relationName,$columnName) = split('\.', $key); list($relationName,$columnName) = split('\.', $fieldName);
$relationObj = $obj->getComponent($relationName); $relationObj = $obj->getComponent($relationName);
$relationObj->{$columnName} = $val; $relationObj->{$columnName} = $val;
$relationObj->write(); $relationObj->write();
$obj->flushCache(); // avoid relation caching confusion $obj->flushCache(); // avoid relation caching confusion
} elseif($obj->hasField($key) || $obj->hasMethod($key)) { //} elseif($obj->hasField($fieldName) || $obj->hasMethod($fieldName)) {
} else {
// plain old value setter // plain old value setter
if(!$this->isNullValue($val, $key)) $obj->{$key} = $val; $obj->{$fieldName} = $val;
} }
} }
$id = ($preview) ? 0 : $obj->write(); $id = ($preview) ? 0 : $obj->write();
@ -139,18 +175,22 @@ class CsvBulkLoader extends BulkLoader {
* columns specified via {@link self::$duplicateChecks} * columns specified via {@link self::$duplicateChecks}
* *
* @param array $record CSV data column * @param array $record CSV data column
* @param array $columnMap
* @return unknown * @return unknown
*/ */
public function findExistingObject($record) { public function findExistingObject($record, $columnMap) {
// checking for existing records (only if not already found) // checking for existing records (only if not already found)
foreach($this->duplicateChecks as $fieldName => $duplicateCheck) { foreach($this->duplicateChecks as $fieldName => $duplicateCheck) {
if(is_string($duplicateCheck)) { if(is_string($duplicateCheck)) {
$SQL_fieldName = Convert::raw2sql($duplicateCheck); $SQL_fieldName = Convert::raw2sql($duplicateCheck);
$SQL_fieldValue = $record[$this->columnMap[$fieldName]]; if(!isset($record[$fieldName])) {
user_error("CsvBulkLoader:processRecord: Couldn't find duplicate identifier '{$fieldName}' in columns", E_USER_ERROR);
}
$SQL_fieldValue = $record[$fieldName];
$existingRecord = DataObject::get_one($this->objectClass, "`$SQL_fieldName` = '{$SQL_fieldValue}'"); $existingRecord = DataObject::get_one($this->objectClass, "`$SQL_fieldName` = '{$SQL_fieldValue}'");
if($existingRecord) return $existingRecord; if($existingRecord) return $existingRecord;
} elseif(is_array($duplicateCheck) && isset($duplicateCheck['callback'])) { } elseif(is_array($duplicateCheck) && isset($duplicateCheck['callback'])) {
$existingRecord = singleton($this->objectClass)->{$duplicateCheck['callback']}($val, $record); $existingRecord = singleton($this->objectClass)->{$duplicateCheck['callback']}($record[$fieldName], $record);
if($existingRecord) return $existingRecord; if($existingRecord) return $existingRecord;
} else { } else {
user_error('CsvBulkLoader:processRecord: Wrong format for $duplicateChecks', E_USER_ERROR); user_error('CsvBulkLoader:processRecord: Wrong format for $duplicateChecks', E_USER_ERROR);