(manually merged from branches/roa)

git-svn-id: svn://svn.silverstripe.com/silverstripe/open/modules/sapphire/trunk@60225 467b73ca-7a2a-4603-9d3b-597d59a354a9

tests/CsvBulkLoaderTest.php

@@ -1,6 +1,8 @@
 <?php
 /**
  * @package tests
+ *
+ * @todo Test with columnn headers and custom mappings
  */
 class CsvBulkLoaderTest extends SapphireTest {
 	static $fixture_file = 'sapphire/tests/CsvBulkLoaderTest.yml';

tools/BulkLoader.php

@@ -5,9 +5,9 @@
  * 
  * You can configure column-handling, 
  * 
- * @todo Allow updating of existing records based on user-specified unique criteria/callbacks (e.g. database ID)
  * @todo Add support for adding/editing has_many relations.
  * @todo Add support for deep chaining of relation properties (e.g. Player.Team.Stats.GoalCount)
+ * @todo Character conversion
  * 
  * @see http://rfc.net/rfc4180.html
  * @package cms
@@ -41,6 +41,9 @@ abstract class BulkLoader extends ViewableData {
 	 * The column count should match the count of array elements,
 	 * fill with NULL values if you want to skip certain columns.
 	 *
+	 * You can also combine {@link $hasHeaderRow} = true and {@link $columnMap}
+	 * and omit the NULL values in your map.
+	 * 
 	 * Supports one-level chaining of has_one relations and properties with dot notation
 	 * (e.g. Team.Title). The first part has to match a has_one relation name
 	 * (not necessarily the classname of the used relation).
@@ -129,11 +132,13 @@ abstract class BulkLoader extends ViewableData {
 	 * Useful to analyze the input and give the users a chance to influence
 	 * it through a UI.
 	 *
+	 * @todo Implement preview()
+	 *
 	 * @param string $filepath Absolute path to the file we're importing
 	 * @return array See {@link self::processAll()}
 	 */
 	public function preview($filepath) {
-		return $this->processAll($filepath, true);
+		user_error("BulkLoader::preview(): Not implemented", E_USER_ERROR);
 	}
 	
 	/**
@@ -154,10 +159,11 @@ abstract class BulkLoader extends ViewableData {
 	 * Process a single record from the file.
 	 * 
 	 * @param array $record An map of the data, keyed by the header field defined in {@link self::$columnMap}
+	 * @param array $columnMap
 	 * @param boolean $preview
 	 * @return ArrayData @see self::processAll()
 	 */
-	abstract protected function processRecord($record, $preview = false);
+	abstract protected function processRecord($record, $columnMap, $preview = false);
 	
 	/**
 	 * Return a FieldSet containing all the options for this form; this
@@ -175,6 +181,38 @@ abstract class BulkLoader extends ViewableData {
 		return ($title = $this->stat('title')) ? $title : $this->class;
 	}
 	
+	/**
+	 * Get a specification of all available columns and relations on the used model.
+	 * Useful for generation of spec documents for technical end users.
+	 * 
+	 * Return Format:
+	 * <example>
+	 * array(
+	 *   'fields' => array('myFieldName'=>'myDescription'), 
+	 *   'relations' => array('myRelationName'=>'myDescription'), 
+	 * )
+	 * </example>
+	 *
+	 * @todo Mix in custom column mappings
+	 * @usedby {@link ModelAdmin}
+	 *
+	 * @return array
+	 **/
+	public function getImportSpec() {
+		$spec = array();
+
+		// get database columns (fieldlabels include fieldname as a key)
+		$spec['fields'] = (array)singleton($this->objectClass)->fieldLabels();
+		
+		$has_ones = singleton($this->objectClass)->has_one();
+		$has_manys = singleton($this->objectClass)->has_many();
+		$many_manys = singleton($this->objectClass)->many_many();
+		
+		$spec['relations'] = (array)$has_ones + (array)$has_manys + (array)$many_manys;
+		
+		return $spec;
+	}
+	
 	/**
 	 * Determines if a specific field is null.
 	 * Can be useful for unusual "empty" flags in the file,

tools/CsvBulkLoader.php

@@ -39,20 +39,50 @@ class CsvBulkLoader extends BulkLoader {
 		
 		$return = new DataObjectSet();
 
-		// assuming that first row is column naming if no columnmap is passed
 		if($this->hasHeaderRow && $this->columnMap) {
 			$columnRow = fgetcsv($file, 0, $this->delimiter, $this->enclosure);
-			$columnMap = $this->columnMap;
+			$columnMap = array();
+			foreach($columnRow as $k => $origColumnName) {
+				$origColumnName = trim($origColumnName);
+				if(isset($this->columnMap[$origColumnName])) {
+					$columnMap[$origColumnName] = $this->columnMap[$origColumnName];
+				} else {
+					$columnMap[$origColumnName] = null;
+				}
+				
+			}
 		} elseif($this->columnMap) {
 			$columnMap = $this->columnMap;
 		} else {
+			// assuming that first row is column naming if no columnmap is passed
 			$columnRow = fgetcsv($file, 0, $this->delimiter, $this->enclosure);
 			$columnMap = array_combine($columnRow, $columnRow);
 		}
 
+		$rowIndex = 0;
 		while (($row = fgetcsv($file, 0, $this->delimiter, $this->enclosure)) !== FALSE) {
-			$indexedRow = array_combine(array_values($columnMap), array_values($row));
+			$rowIndex++;
-			$return->push($this->processRecord($indexedRow));
+			
+			/*
+			// the columnMap should have the same amount of columns as each record row
+			if(count(array_keys($columnMap)) == count(array_values($row))) {
+				user_error("CsvBulkLoader::processAll(): Columns in row {$rowIndex} don't match the \$columnMap", E_USER_WARNING);
+			}
+			*/
+			
+			$indexedRow = array();
+			foreach($columnMap as $origColumnName => $fieldName) {
+				// in case the row has less fields than the columnmap,
+				// ignore the "leftover" mappings
+				if(!isset($row[count($indexedRow)])) {
+						user_error("CsvBulkLoader::processAll(): Columns in row {$rowIndex} don't match the \$columnMap", E_USER_NOTICE);
+					continue;
+				}
+	
+				$indexedRow[$origColumnName] = $row[count($indexedRow)];
+			}
+
+			$return->push($this->processRecord($indexedRow, $columnMap));
 		}
 		
 		fclose($file);
@@ -60,26 +90,28 @@ class CsvBulkLoader extends BulkLoader {
 		return $return;
 	}
 	
-
+	protected function processRecord($record, $columnMap, $preview = false) {
-	protected function processRecord($record, $preview = false) {
 		$class = $this->objectClass;
 		
 		// find existing object, or create new one
-		$existingObj = $this->findExistingObject($record);
+		$existingObj = $this->findExistingObject($record, $columnMap);
 		$obj = ($existingObj) ? $existingObj : new $class(); 
 		
 		// first run: find/create any relations and store them on the object
 		// we can't combine runs, as other columns might rely on the relation being present
 		$relations = array();
+		foreach($record as $origColumnName => $val) {
+			$fieldName = $columnMap[$origColumnName];
+			
+			// don't bother querying of value is not set
+			if($this->isNullValue($val)) continue;
 			
-		foreach($record as $key => $val) {
-			//if($this->isNullValue($val)) continue;
 			// checking for existing relations
-			if(isset($this->relationCallbacks[$key])) {
+			if(isset($this->relationCallbacks[$fieldName])) {
 				// trigger custom search method for finding a relation based on the given value
 				// and write it back to the relation (or create a new object)
-				$relationName = $this->relationCallbacks[$key]['relationname'];
+				$relationName = $this->relationCallbacks[$fieldName]['relationname'];
-				$relationObj = $obj->{$this->relationCallbacks[$key]['callback']}($val, $record);
+				$relationObj = $obj->{$this->relationCallbacks[$fieldName]['callback']}($val, $record);
 				if(!$relationObj || !$relationObj->exists()) {
 					$relationClass = $obj->has_one($relationName);
 					$relationObj = new $relationClass();
@@ -88,9 +120,9 @@ class CsvBulkLoader extends BulkLoader {
 				$obj->setComponent($relationName, $relationObj);
 				$obj->{"{$relationName}ID"} = $relationObj->ID;
 				$obj->write();
-			} elseif(strpos($key, '.') !== false) {
+			} elseif(strpos($fieldName, '.') !== false) {
 				// we have a relation column with dot notation
-				list($relationName,$columnName) = split('\.', $key);
+				list($relationName,$columnName) = split('\.', $fieldName);
 				$relationObj = $obj->getComponent($relationName); // always gives us an component (either empty or existing)
 				$obj->setComponent($relationName, $relationObj);
 				$relationObj->write();
@@ -102,21 +134,25 @@ class CsvBulkLoader extends BulkLoader {
 		}
 		$id = ($preview) ? 0 : $obj->write();
 
-		
 		// second run: save data
-		foreach($record as $key => $val) {
+		foreach($record as $origColumnName => $val) {
-			if($obj->hasMethod("import{$key}")) {
+			$fieldName = $columnMap[$origColumnName];
-				$obj->{"import{$key}"}($val, $record);
+
-			} elseif(strpos($key, '.') !== false) {
+			if($this->isNullValue($val, $fieldName)) continue;
+
+			if($obj->hasMethod("import{$fieldName}")) {
+				$obj->{"import{$fieldName}"}($val, $record);
+			} elseif(strpos($fieldName, '.') !== false) {
 				// we have a relation column
-				list($relationName,$columnName) = split('\.', $key);
+				list($relationName,$columnName) = split('\.', $fieldName);
 				$relationObj = $obj->getComponent($relationName);
 				$relationObj->{$columnName} = $val;
 				$relationObj->write();
 				$obj->flushCache(); // avoid relation caching confusion
-			} elseif($obj->hasField($key) || $obj->hasMethod($key)) {
+			//} elseif($obj->hasField($fieldName) || $obj->hasMethod($fieldName)) {
+			} else {
 				// plain old value setter
-				if(!$this->isNullValue($val, $key)) $obj->{$key} = $val;
+				$obj->{$fieldName} = $val;
 			}
 		}
 		$id = ($preview) ? 0 : $obj->write();
@@ -139,18 +175,22 @@ class CsvBulkLoader extends BulkLoader {
 	 * columns specified via {@link self::$duplicateChecks}
 	 *
 	 * @param array $record CSV data column
+	 * @param array $columnMap
 	 * @return unknown
 	 */
-	public function findExistingObject($record) {
+	public function findExistingObject($record, $columnMap) {
 		// checking for existing records (only if not already found)
 		foreach($this->duplicateChecks as $fieldName => $duplicateCheck) {
 			if(is_string($duplicateCheck)) {
 				$SQL_fieldName = Convert::raw2sql($duplicateCheck); 
-				$SQL_fieldValue = $record[$this->columnMap[$fieldName]];
+				if(!isset($record[$fieldName])) {
+					user_error("CsvBulkLoader:processRecord: Couldn't find duplicate identifier '{$fieldName}' in columns", E_USER_ERROR);
+				}
+				$SQL_fieldValue = $record[$fieldName];
 				$existingRecord = DataObject::get_one($this->objectClass, "`$SQL_fieldName` = '{$SQL_fieldValue}'");
 				if($existingRecord) return $existingRecord;
 			} elseif(is_array($duplicateCheck) && isset($duplicateCheck['callback'])) {
-				$existingRecord = singleton($this->objectClass)->{$duplicateCheck['callback']}($val, $record);
+				$existingRecord = singleton($this->objectClass)->{$duplicateCheck['callback']}($record[$fieldName], $record);
 				if($existingRecord) return $existingRecord;
 			} else {
 				user_error('CsvBulkLoader:processRecord: Wrong format for $duplicateChecks', E_USER_ERROR);