(merged from branches/roa. use "svn log -c <changeset> -g <module-svn-path>" for detailed commit message)

git-svn-id: svn://svn.silverstripe.com/silverstripe/open/modules/sapphire/trunk@60212 467b73ca-7a2a-4603-9d3b-597d59a354a9

api/RestfulServer.php

@@ -122,6 +122,8 @@ class RestfulServer extends Controller {
 	 * @return String The serialized representation of the requested object(s) - usually XML or JSON.
 	 */
 	protected function getHandler($className, $id, $relation, $formatter) {
+		$limit = (int)$this->request->getVar('limit');
+		
 		if($id) {
 			$obj = DataObject::get_by_id($className, $id);
 			if(!$obj) {
@@ -133,7 +135,7 @@ class RestfulServer extends Controller {
 			}
 			
 			if($relation) {
-				if($obj->hasMethod($relation)) $obj = $obj->$relation();
+				if($obj->hasMethod($relation)) $obj = $obj->$relation('', '', '', $limit);
 				else return $this->notFound();
 			} 
 			

core/model/DataObject.php

@@ -1207,8 +1207,6 @@ class DataObject extends ViewableData implements DataObjectInterface {
 		$fields->push(new HeaderField($this->singular_name()));
 		foreach($this->db() as $fieldName => $fieldType) {
 			// @todo Pass localized title
-			// commented out, to be less of a pain in the ass
-			//$fields->addFieldToTab('Root.Main', $this->dbObject($fieldName)->scaffoldFormField());
 			$fields->push($this->dbObject($fieldName)->scaffoldFormField());
 		}
 		foreach($this->has_one() as $relationship => $component) {
@@ -1227,13 +1225,6 @@ class DataObject extends ViewableData implements DataObjectInterface {
 	protected function addScaffoldRelationFields($fieldSet) {
 		
 		if($this->has_many()) {
-			// Refactor the fields that we have been given into a tab, "Main", in a tabset
-			$oldFields = $fieldSet;
-			$fieldSet = new FieldSet(
-				new TabSet("Root", new Tab("Main"))
-			);
-			foreach($oldFields as $field) $fieldSet->addFieldToTab("Root.Main", $field);
-			
 			// Add each relation as a separate tab
 			foreach($this->has_many() as $relationship => $component) {
 				$relationshipFields = singleton($component)->summary_fields();
@@ -1267,12 +1258,17 @@ class DataObject extends ViewableData implements DataObjectInterface {
 	 * @return FieldSet
 	 */
 	public function getCMSFields() {
-		$fields = $this->scaffoldFormFields();
+		$fieldSet = new FieldSet(new TabSet("Root", new Tab("Main")));
+		
+		$baseFields = $this->scaffoldFormFields();
+		foreach($baseFields as $field) $fieldSet->addFieldToTab("Root.Main", $field);
+		
 		// If we don't have an ID, then relation fields don't work
 		if($this->ID) {
-			$fields = $this->addScaffoldRelationFields($fields);
+			$fieldSet = $this->addScaffoldRelationFields($fieldSet);
 		}
-		return $fields;
+		
+		return $fieldSet;
 	}
 
 	/**
@@ -1385,7 +1381,7 @@ class DataObject extends ViewableData implements DataObjectInterface {
 					// Only existing fields
 					$this->fieldExists($fieldName)
 					// Catches "0"==NULL
-					&& (isset($this->record[$fieldName]) && (intval($val) != intval($this->record[$fieldName])))
+					&& (isset($this->record[$fieldName]) && is_numeric($val) && (intval($val) != intval($this->record[$fieldName])))
 					// Main non type-based check
 					&& (isset($this->record[$fieldName]) && $this->record[$fieldName] != $val)
 				) {

search/SearchContext.php

@@ -123,6 +123,8 @@ class SearchContext extends Object {
 		$baseTable = $this->applyBaseTable();
 		$query->from($baseTable);
 		
+		if($limit) $query->limit = (!empty($start)) ? "{$start},{$limit}" : $limit; 
+		
 		// SRM: This stuff is copied from DataObject, 
 		if($this->modelClass != $baseTable) {
 			$classNames = ClassInfo::subclassesFor($this->modelClass);

tests/CsvBulkLoaderTest.php

@@ -111,6 +111,34 @@ class CsvBulkLoaderTest extends SapphireTest {
 		fclose($file);
 	}
 	
+	/**
+	 * Test import with custom identifiers by importing the data.
+	 * 
+	 * @todo Test duplicateCheck callbacks
+	 */
+	function testLoadWithIdentifiers() {
+		// first load
+		$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
+		$filepath = Director::baseFolder() . '/sapphire/tests/CsvBulkLoaderTest_PlayersWithId.csv';
+		$loader->duplicateChecks = array(
+			'ExternalIdentifier' => 'ExternalIdentifier' 
+		);
+		$results = $loader->load($filepath);
+		
+		$player = DataObject::get_by_id('CsvBulkLoaderTest_Player', 1);
+		$this->assertEquals($player->FirstName, 'John');
+		$this->assertEquals($player->Biography, 'He\'s a good guy', 'test updating of duplicate imports within the same import works');
+
+		// load with updated data
+		$filepath = Director::baseFolder() . '/sapphire/tests/CsvBulkLoaderTest_PlayersWithIdUpdated.csv';
+		$results = $loader->load($filepath);
+		
+		$player = DataObject::get_by_id('CsvBulkLoaderTest_Player', 1);
+		$this->assertEquals($player->FirstName, 'JohnUpdated', 'Test updating of existing records works');
+		$this->assertEquals($player->Biography, 'He\'s a good guy', 'Test retaining of previous information on duplicate when overwriting with blank field');
+		
+	}
+	
 	protected function getLineCount(&$file) {
 		$i = 0;
 		while(fgets($file) !== false) $i++;
@@ -139,6 +167,7 @@ class CsvBulkLoaderTest_Player extends DataObject implements TestOnly {
 		'FirstName' => 'Varchar(255)',
 		'Biography' => 'HTMLText',
 		'Birthday' => 'Date',
+		'ExternalIdentifier' => 'Varchar(255)', // used for uniqueness checks on passed property
 	);
 	
 	static $has_one = array(

tests/CsvBulkLoaderTest_PlayersWithId.csv

@@ -0,0 +1,4 @@
+"ExternalIdentifier","FirstName","Biography","Birthday"
+222b,"John","","31/01/1988"
+222b,"John","He's a good guy",""
+9000a,"Jamie","Pretty old\, with an escaped comma","31/01/1882"

tests/CsvBulkLoaderTest_PlayersWithIdUpdated.csv

@@ -0,0 +1,3 @@
+"ExternalIdentifier","FirstName","Biography","Birthday"
+222b,"JohnUpdated","","31/01/1988"
+9000a,"JamieUpdated","Pretty old\, with an escaped comma","31/01/1882"

tests/DataObjectDecoratorTest.php

@@ -0,0 +1,64 @@
+<?php
+
+class DataObjectDecoratorTest extends SapphireTest {
+	static $fixture_file = 'sapphire/tests/DataObjectTest.yml';
+	
+	function testOneToManyAssociationWithDecorator() {
+		$contact = new DataObjectDecoratorTest_Member();
+		$contact->Website = "http://www.example.com";
+		$object = new DataObjectDecoratorTest_RelatedObject();
+		$object->FieldOne = "Lorem ipsum dolor";
+		$object->FieldTwo = "Random notes";
+		$contact->RelatedObjects()->add($object);
+		$contact->write();
+		unset($contact);
+		
+		$contact = DataObject::get_one("DataObjectDecoratorTest_Member", "Website='http://www.example.com'");
+		$this->assertType('DataObjectDecoratorTest_RelatedObject', $contact->RelatedObjects()->First());
+		$this->assertEquals("Lorem ipsum dolor", $contact->RelatedObjects()->First()->FieldOne);
+		$this->assertEquals("Random notes", $contact->RelatedObjects()->First()->FieldTwo);
+		$contact->delete();
+	}
+	
+}
+
+class DataObjectDecoratorTest_Member extends DataObject implements TestOnly {
+	
+	static $db = array(
+		"Name" => "Text",
+		"Email" => "Text"
+	);
+	
+}
+
+class DataObjectDecoratorTest_ContactRole extends DataObjectDecorator implements TestOnly {
+	
+	function extraDBFields() {
+		return array(
+			'db' => array(
+				'Website' => 'Text',
+				'Phone' => 'Varchar(255)',
+			),
+			'has_many' => array(
+				'RelatedObjects' => 'DataObjectDecoratorTest_RelatedObject'
+			)
+		);
+	}
+}
+
+class DataObjectDecoratorTest_RelatedObject extends DataObject implements TestOnly {
+	
+	static $db = array(
+		"FieldOne" => "Text",
+		"FieldOne" => "Text"
+	);
+	
+	static $has_one = array(
+		"Contact" => "Member"
+	);
+	
+}
+
+DataObject::add_extension('DataObjectDecoratorTest_Member', 'DataObjectDecoratorTest_ContactRole');
+
+?>

tests/DataObjectDecoratorTest.yml

@@ -0,0 +1,25 @@
+PageComment:
+   comment1:
+      Name: Joe
+      Comment: This is a test comment
+   comment2:
+      Name: Jane
+      Comment: This is another test comment
+   comment3:
+      Name: Bob
+      Comment: Another comment
+   comment4:
+      Name: Bob
+      Comment: Second comment by Bob
+
+Page:
+   home:
+      Title: Home
+      Comments: =>PageComment.comment1,=>PageComment.comment2
+   page1:
+      Title: First Page
+      Content: <p>Some test content</p>
+      Comments: =>PageComment.comment3,=>PageComment.comment4 
+   page2:
+      Title: Second Page
+

tools/BulkLoader.php

@@ -66,7 +66,7 @@ abstract class BulkLoader extends ViewableData {
 	 *
 	 * @var array
 	 */
-	public $columnMap;
+	public $columnMap = array();
 	
 	/**
 	 * Find a has_one relation based on a specific column value.
@@ -84,10 +84,35 @@ abstract class BulkLoader extends ViewableData {
 	 *
 	 * @var array
 	 */
-	public $relationCallbacks;
+	public $relationCallbacks = array();
+	
+	/**
+	 * Specifies how to determine duplicates based on one or more provided fields
+	 * in the imported data, matching to properties on the used {@link DataObject} class.
+	 * Alternatively the array values can contain a callback method (see example for
+	 * implementation details).
+	 * If multiple checks are specified, the first one "wins".
+	 * 
+	 *  <code>
+	 * <?php
+	 * array(
+	 * 		'customernumber' => 'ID',
+	 * 		'phonenumber' => array(
+	 * 			'callback' => 'getByImportedPhoneNumber'
+	 * 		)
+	 * );
+	 * ?>
+	 * </code>
+	 *
+	 * @var array
+	 */
+	public $duplicateChecks = array();
 	
 	function __construct($objectClass) {
 		$this->objectClass = $objectClass;
+		
+		ini_set('max_execution_time', 3600);
+		ini_set('memory_limit', '512M');
 	}
 	
 	/*
@@ -150,5 +175,20 @@ abstract class BulkLoader extends ViewableData {
 		return ($title = $this->stat('title')) ? $title : $this->class;
 	}
 	
+	/**
+	 * Determines if a specific field is null.
+	 * Can be useful for unusual "empty" flags in the file,
+	 * e.g. a "(not set)" value.
+	 * The usual {@link DBField::isNull()} checks apply when writing the {@link DataObject},
+	 * so this is mainly a customization method.
+	 *
+	 * @param mixed $val
+	 * @param string $field Name of the field as specified in the array-values for {@link self::$columnMap}.
+	 * @return boolean
+	 */
+	protected function isNullValue($val, $fieldName = null) {
+		return (empty($val));
+	}
+	
 }
 ?>

tools/CsvBulkLoader.php

@@ -63,12 +63,18 @@ class CsvBulkLoader extends BulkLoader {
 
 	protected function processRecord($record, $preview = false) {
 		$class = $this->objectClass;
-		$obj = new $class();
+		
+		// find existing object, or create new one
+		$existingObj = $this->findExistingObject($record);
+		$obj = ($existingObj) ? $existingObj : new $class(); 
 		
 		// first run: find/create any relations and store them on the object
 		// we can't combine runs, as other columns might rely on the relation being present
 		$relations = array();
+
 		foreach($record as $key => $val) {
+			//if($this->isNullValue($val)) continue;
+			// checking for existing relations
 			if(isset($this->relationCallbacks[$key])) {
 				// trigger custom search method for finding a relation based on the given value
 				// and write it back to the relation (or create a new object)
@@ -81,6 +87,7 @@ class CsvBulkLoader extends BulkLoader {
 				}
 				$obj->setComponent($relationName, $relationObj);
 				$obj->{"{$relationName}ID"} = $relationObj->ID;
+				$obj->write();
 			} elseif(strpos($key, '.') !== false) {
 				// we have a relation column with dot notation
 				list($relationName,$columnName) = split('\.', $key);
@@ -88,7 +95,9 @@ class CsvBulkLoader extends BulkLoader {
 				$obj->setComponent($relationName, $relationObj);
 				$relationObj->write();
 				$obj->{"{$relationName}ID"} = $relationObj->ID;
+				$obj->write();
 			}
+			
 			$obj->flushCache(); // avoid relation caching confusion
 		}
 		$id = ($preview) ? 0 : $obj->write();
@@ -105,9 +114,9 @@ class CsvBulkLoader extends BulkLoader {
 				$relationObj->{$columnName} = $val;
 				$relationObj->write();
 				$obj->flushCache(); // avoid relation caching confusion
-			} elseif($obj->hasField($key)) {
+			} elseif($obj->hasField($key) || $obj->hasMethod($key)) {
 				// plain old value setter
-				$obj->{$key} = $val;
+				if(!$this->isNullValue($val, $key)) $obj->{$key} = $val;
 			}
 		}
 		$id = ($preview) ? 0 : $obj->write();
@@ -115,6 +124,7 @@ class CsvBulkLoader extends BulkLoader {
 		$message = '';
 		
 		// memory usage
+		unset($existingObj);
 		unset($obj);
 		
 		return new ArrayData(array(
@@ -124,6 +134,32 @@ class CsvBulkLoader extends BulkLoader {
 		));
 	}
 	
+	/**
+	 * Find an existing objects based on one or more uniqueness
+	 * columns specified via {@link self::$duplicateChecks}
+	 *
+	 * @param array $record CSV data column
+	 * @return unknown
+	 */
+	public function findExistingObject($record) {
+		// checking for existing records (only if not already found)
+		foreach($this->duplicateChecks as $fieldName => $duplicateCheck) {
+			if(is_string($duplicateCheck)) {
+				$SQL_fieldName = Convert::raw2sql($duplicateCheck); 
+				$SQL_fieldValue = $record[$this->columnMap[$fieldName]];
+				$existingRecord = DataObject::get_one($this->objectClass, "`$SQL_fieldName` = '{$SQL_fieldValue}'");
+				if($existingRecord) return $existingRecord;
+			} elseif(is_array($duplicateCheck) && isset($duplicateCheck['callback'])) {
+				$existingRecord = singleton($this->objectClass)->{$duplicateCheck['callback']}($val, $record);
+				if($existingRecord) return $existingRecord;
+			} else {
+				user_error('CsvBulkLoader:processRecord: Wrong format for $duplicateChecks', E_USER_ERROR);
+			}
+		}
+		
+		return false;
+	}
+	
 	
 	/**
 	 * Determine wether any loaded files should be parsed