diff --git a/tests/CsvBulkLoaderTest.php b/tests/CsvBulkLoaderTest.php index 4acdcc251..189781543 100644 --- a/tests/CsvBulkLoaderTest.php +++ b/tests/CsvBulkLoaderTest.php @@ -1,6 +1,8 @@ processAll($filepath, true); + user_error("BulkLoader::preview(): Not implemented", E_USER_ERROR); } /** @@ -154,10 +159,11 @@ abstract class BulkLoader extends ViewableData { * Process a single record from the file. * * @param array $record An map of the data, keyed by the header field defined in {@link self::$columnMap} + * @param array $columnMap * @param boolean $preview * @return ArrayData @see self::processAll() */ - abstract protected function processRecord($record, $preview = false); + abstract protected function processRecord($record, $columnMap, $preview = false); /** * Return a FieldSet containing all the options for this form; this @@ -175,6 +181,38 @@ abstract class BulkLoader extends ViewableData { return ($title = $this->stat('title')) ? $title : $this->class; } + /** + * Get a specification of all available columns and relations on the used model. + * Useful for generation of spec documents for technical end users. + * + * Return Format: + * + * array( + * 'fields' => array('myFieldName'=>'myDescription'), + * 'relations' => array('myRelationName'=>'myDescription'), + * ) + * + * + * @todo Mix in custom column mappings + * @usedby {@link ModelAdmin} + * + * @return array + **/ + public function getImportSpec() { + $spec = array(); + + // get database columns (fieldlabels include fieldname as a key) + $spec['fields'] = (array)singleton($this->objectClass)->fieldLabels(); + + $has_ones = singleton($this->objectClass)->has_one(); + $has_manys = singleton($this->objectClass)->has_many(); + $many_manys = singleton($this->objectClass)->many_many(); + + $spec['relations'] = (array)$has_ones + (array)$has_manys + (array)$many_manys; + + return $spec; + } + /** * Determines if a specific field is null. * Can be useful for unusual "empty" flags in the file, diff --git a/tools/CsvBulkLoader.php b/tools/CsvBulkLoader.php index 4151c5f4d..5d4c17d64 100644 --- a/tools/CsvBulkLoader.php +++ b/tools/CsvBulkLoader.php @@ -38,21 +38,51 @@ class CsvBulkLoader extends BulkLoader { if(!$file) return false; $return = new DataObjectSet(); - - // assuming that first row is column naming if no columnmap is passed + if($this->hasHeaderRow && $this->columnMap) { $columnRow = fgetcsv($file, 0, $this->delimiter, $this->enclosure); - $columnMap = $this->columnMap; + $columnMap = array(); + foreach($columnRow as $k => $origColumnName) { + $origColumnName = trim($origColumnName); + if(isset($this->columnMap[$origColumnName])) { + $columnMap[$origColumnName] = $this->columnMap[$origColumnName]; + } else { + $columnMap[$origColumnName] = null; + } + + } } elseif($this->columnMap) { $columnMap = $this->columnMap; } else { + // assuming that first row is column naming if no columnmap is passed $columnRow = fgetcsv($file, 0, $this->delimiter, $this->enclosure); $columnMap = array_combine($columnRow, $columnRow); } - + + $rowIndex = 0; while (($row = fgetcsv($file, 0, $this->delimiter, $this->enclosure)) !== FALSE) { - $indexedRow = array_combine(array_values($columnMap), array_values($row)); - $return->push($this->processRecord($indexedRow)); + $rowIndex++; + + /* + // the columnMap should have the same amount of columns as each record row + if(count(array_keys($columnMap)) == count(array_values($row))) { + user_error("CsvBulkLoader::processAll(): Columns in row {$rowIndex} don't match the \$columnMap", E_USER_WARNING); + } + */ + + $indexedRow = array(); + foreach($columnMap as $origColumnName => $fieldName) { + // in case the row has less fields than the columnmap, + // ignore the "leftover" mappings + if(!isset($row[count($indexedRow)])) { + user_error("CsvBulkLoader::processAll(): Columns in row {$rowIndex} don't match the \$columnMap", E_USER_NOTICE); + continue; + } + + $indexedRow[$origColumnName] = $row[count($indexedRow)]; + } + + $return->push($this->processRecord($indexedRow, $columnMap)); } fclose($file); @@ -60,26 +90,28 @@ class CsvBulkLoader extends BulkLoader { return $return; } - - protected function processRecord($record, $preview = false) { + protected function processRecord($record, $columnMap, $preview = false) { $class = $this->objectClass; // find existing object, or create new one - $existingObj = $this->findExistingObject($record); + $existingObj = $this->findExistingObject($record, $columnMap); $obj = ($existingObj) ? $existingObj : new $class(); // first run: find/create any relations and store them on the object // we can't combine runs, as other columns might rely on the relation being present $relations = array(); - - foreach($record as $key => $val) { - //if($this->isNullValue($val)) continue; + foreach($record as $origColumnName => $val) { + $fieldName = $columnMap[$origColumnName]; + + // don't bother querying of value is not set + if($this->isNullValue($val)) continue; + // checking for existing relations - if(isset($this->relationCallbacks[$key])) { + if(isset($this->relationCallbacks[$fieldName])) { // trigger custom search method for finding a relation based on the given value // and write it back to the relation (or create a new object) - $relationName = $this->relationCallbacks[$key]['relationname']; - $relationObj = $obj->{$this->relationCallbacks[$key]['callback']}($val, $record); + $relationName = $this->relationCallbacks[$fieldName]['relationname']; + $relationObj = $obj->{$this->relationCallbacks[$fieldName]['callback']}($val, $record); if(!$relationObj || !$relationObj->exists()) { $relationClass = $obj->has_one($relationName); $relationObj = new $relationClass(); @@ -88,9 +120,9 @@ class CsvBulkLoader extends BulkLoader { $obj->setComponent($relationName, $relationObj); $obj->{"{$relationName}ID"} = $relationObj->ID; $obj->write(); - } elseif(strpos($key, '.') !== false) { + } elseif(strpos($fieldName, '.') !== false) { // we have a relation column with dot notation - list($relationName,$columnName) = split('\.', $key); + list($relationName,$columnName) = split('\.', $fieldName); $relationObj = $obj->getComponent($relationName); // always gives us an component (either empty or existing) $obj->setComponent($relationName, $relationObj); $relationObj->write(); @@ -102,21 +134,25 @@ class CsvBulkLoader extends BulkLoader { } $id = ($preview) ? 0 : $obj->write(); - // second run: save data - foreach($record as $key => $val) { - if($obj->hasMethod("import{$key}")) { - $obj->{"import{$key}"}($val, $record); - } elseif(strpos($key, '.') !== false) { + foreach($record as $origColumnName => $val) { + $fieldName = $columnMap[$origColumnName]; + + if($this->isNullValue($val, $fieldName)) continue; + + if($obj->hasMethod("import{$fieldName}")) { + $obj->{"import{$fieldName}"}($val, $record); + } elseif(strpos($fieldName, '.') !== false) { // we have a relation column - list($relationName,$columnName) = split('\.', $key); + list($relationName,$columnName) = split('\.', $fieldName); $relationObj = $obj->getComponent($relationName); $relationObj->{$columnName} = $val; $relationObj->write(); $obj->flushCache(); // avoid relation caching confusion - } elseif($obj->hasField($key) || $obj->hasMethod($key)) { + //} elseif($obj->hasField($fieldName) || $obj->hasMethod($fieldName)) { + } else { // plain old value setter - if(!$this->isNullValue($val, $key)) $obj->{$key} = $val; + $obj->{$fieldName} = $val; } } $id = ($preview) ? 0 : $obj->write(); @@ -139,18 +175,22 @@ class CsvBulkLoader extends BulkLoader { * columns specified via {@link self::$duplicateChecks} * * @param array $record CSV data column + * @param array $columnMap * @return unknown */ - public function findExistingObject($record) { + public function findExistingObject($record, $columnMap) { // checking for existing records (only if not already found) foreach($this->duplicateChecks as $fieldName => $duplicateCheck) { if(is_string($duplicateCheck)) { $SQL_fieldName = Convert::raw2sql($duplicateCheck); - $SQL_fieldValue = $record[$this->columnMap[$fieldName]]; + if(!isset($record[$fieldName])) { + user_error("CsvBulkLoader:processRecord: Couldn't find duplicate identifier '{$fieldName}' in columns", E_USER_ERROR); + } + $SQL_fieldValue = $record[$fieldName]; $existingRecord = DataObject::get_one($this->objectClass, "`$SQL_fieldName` = '{$SQL_fieldValue}'"); if($existingRecord) return $existingRecord; } elseif(is_array($duplicateCheck) && isset($duplicateCheck['callback'])) { - $existingRecord = singleton($this->objectClass)->{$duplicateCheck['callback']}($val, $record); + $existingRecord = singleton($this->objectClass)->{$duplicateCheck['callback']}($record[$fieldName], $record); if($existingRecord) return $existingRecord; } else { user_error('CsvBulkLoader:processRecord: Wrong format for $duplicateChecks', E_USER_ERROR);