From 9f751829a6448d1a4a578ea209ca07ed61d831ea Mon Sep 17 00:00:00 2001 From: Ingo Schommer Date: Sat, 9 Aug 2008 04:53:34 +0000 Subject: [PATCH] (merged from branches/roa. use "svn log -c -g " for detailed commit message) git-svn-id: svn://svn.silverstripe.com/silverstripe/open/modules/sapphire/trunk@60211 467b73ca-7a2a-4603-9d3b-597d59a354a9 --- core/model/ComponentSet.php | 3 + core/model/DataObject.php | 93 ++++++++-- core/model/SQLQuery.php | 7 + core/model/fieldtypes/DBField.php | 2 +- search/SearchContext.php | 78 ++++++-- search/filters/ExactMatchFilter.php | 3 +- search/filters/PartialMatchFilter.php | 3 +- search/filters/SearchFilter.php | 73 +++++++- tests/CsvBulkLoaderTest.php | 172 ++++++++++++++++++ tests/CsvBulkLoaderTest.yml | 3 + tests/CsvBulkLoaderTest_Players.csv | 4 + ...est_PlayersWithCustomHeaderAndRelation.csv | 2 + tests/CsvBulkLoaderTest_PlayersWithHeader.csv | 5 + tests/SQLQueryTest.php | 2 +- tests/SearchContextTest.php | 22 ++- tests/SearchContextTest.yml | 34 ++-- tools/BulkLoader.php | 154 ++++++++++++++++ tools/CsvBulkLoader.php | 139 ++++++++++++++ 18 files changed, 746 insertions(+), 53 deletions(-) create mode 100644 tests/CsvBulkLoaderTest.php create mode 100644 tests/CsvBulkLoaderTest.yml create mode 100644 tests/CsvBulkLoaderTest_Players.csv create mode 100644 tests/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv create mode 100644 tests/CsvBulkLoaderTest_PlayersWithHeader.csv create mode 100644 tools/BulkLoader.php create mode 100644 tools/CsvBulkLoader.php diff --git a/core/model/ComponentSet.php b/core/model/ComponentSet.php index c91397261..4f44c515d 100755 --- a/core/model/ComponentSet.php +++ b/core/model/ComponentSet.php @@ -281,6 +281,9 @@ class ComponentSet extends DataObjectSet { /** * Returns information about this set in HTML format for debugging. + * + * @deprecated + * * @return string */ function debug() { diff --git a/core/model/DataObject.php b/core/model/DataObject.php index a58c8bf0d..74473faf0 100644 --- a/core/model/DataObject.php +++ b/core/model/DataObject.php @@ -1172,20 +1172,29 @@ class DataObject extends ViewableData implements DataObjectInterface { public function scaffoldSearchFields() { $fields = new FieldSet(); foreach($this->searchable_fields() as $fieldName => $fieldType) { - // @todo Pass localized title - $fields->push($this->dbObject($fieldName)->scaffoldSearchField()); - } - /*$extras = $this->invokeWithExtensions('extraSearchFields'); - if ($extras) { - foreach($extras as $result) { - foreach($result as $fieldName => $fieldType) { - $fields->push(new $fieldType($fieldName)); - } + $field = $this->relObject($fieldName)->scaffoldSearchField(); + if (strstr($fieldName, '.')) { + $field->setName(str_replace('.', '__', $fieldName)); + $parts = explode('.', $fieldName); + //$label = $parts[count($parts)-2] . $parts[count($parts)-1]; + $field->setTitle($this->toLabel($parts[count($parts)-2])); + } else { + $field->setTitle($this->toLabel($fieldName)); } - }*/ + $fields->push($field); + } return $fields; } - + + /** + * CamelCase to sentence case label transform + * + * @todo move into utility class (String::toLabel) + */ + function toLabel($string) { + return preg_replace("/([a-z]+)([A-Z])/","$1 $2", $string); + } + /** * Scaffold a simple edit form for all properties on this dataobject, * based on default {@link FormField} mapping in {@link DBField::scaffoldFormField()} @@ -1573,7 +1582,41 @@ class DataObject extends ViewableData implements DataObjectInterface { return $obj; */ } - + + /** + * Traverses to a DBField referenced by a relationship. + * + * @return DBField + */ + public function relObject($fieldPath) { + $parts = explode('.', $fieldPath); + $fieldName = array_pop($parts); + $component = $this; + foreach($parts as $relation) { + if ($rel = $component->has_one($relation)) { + $component = singleton($rel); + } elseif ($rel = $component->has_many($relation)) { + $component = singleton($rel); + } + } + return $component->dbObject($fieldName); + } + + /** + * Temporary hack to return an association name, based on class, toget around the mangle + * of having to deal with reverse lookup of relationships to determine autogenerated foreign keys. + */ + public function getReverseAssociation($className) { + if (is_array($this->has_many())) { + $has_many = array_flip($this->has_many()); + if (array_key_exists($className, $has_many)) return $has_many[$className]; + } + if (is_array($this->has_one())) { + $has_one = array_flip($this->has_one()); + if (array_key_exists($className, $has_one)) return $has_one[$className]; + } + } + /** * Build a {@link SQLQuery} object to perform the given query. * @@ -1921,6 +1964,14 @@ class DataObject extends ViewableData implements DataObjectInterface { user_error("DataObject::get_by_id passed a non-numeric ID #$id", E_USER_WARNING); } } + + /** + * Get the name of the base table for this object + */ + public function baseTable() { + $tableClasses = ClassInfo::dataClassesFor($this->class); + return array_shift($tableClasses); + } //-------------------------------------------------------------------------------------------// @@ -2144,7 +2195,7 @@ class DataObject extends ViewableData implements DataObjectInterface { $filters = array(); foreach($this->searchable_fields() as $name => $type) { if (is_int($name)) { - $filters[$type] = $this->dbObject($type)->defaultSearchFilter(); + $filters[$type] = $this->relObject($type)->defaultSearchFilter(); } else { if (is_array($type)) { $filter = current($type); @@ -2153,7 +2204,7 @@ class DataObject extends ViewableData implements DataObjectInterface { if (is_subclass_of($type, 'SearchFilter')) { $filters[$name] = new $type($name); } else { - $filters[$name] = $this->dbObject($name)->defaultSearchFilter(); + $filters[$name] = $this->relObject($name)->defaultSearchFilter(); } } } @@ -2161,6 +2212,20 @@ class DataObject extends ViewableData implements DataObjectInterface { return $filters; } + /** + * Replace named fields in a relationship with actual path to data objects. + * + * @param string $path + * @return SearchFilter + */ + protected function mapRelationshipObjects($path) { + $path = explode('.', $path); + $fieldName = array_pop($path); + foreach($path as $relation) { + // + } + } + /** * @return boolean True if the object is in the database */ diff --git a/core/model/SQLQuery.php b/core/model/SQLQuery.php index caf05a8d3..53f891685 100755 --- a/core/model/SQLQuery.php +++ b/core/model/SQLQuery.php @@ -132,6 +132,13 @@ class SQLQuery extends Object { return $this; } + /** + * Add a LEFT JOIN criteria to the FROM clause. + */ + public function leftJoin($table, $onPredicate) { + $this->from[] = "LEFT JOIN $table ON $onPredicate"; + } + /** * Apply a predicate filter to the where clause. * diff --git a/core/model/fieldtypes/DBField.php b/core/model/fieldtypes/DBField.php index 847aef616..f5be348b0 100644 --- a/core/model/fieldtypes/DBField.php +++ b/core/model/fieldtypes/DBField.php @@ -222,7 +222,7 @@ abstract class DBField extends ViewableData { * @return SearchFilter */ public function defaultSearchFilter() { - return new ExactMatchFilter($this->name); + return new PartialMatchFilter($this->name); } /** diff --git a/search/SearchContext.php b/search/SearchContext.php index 3aed17924..3dfbac093 100644 --- a/search/SearchContext.php +++ b/search/SearchContext.php @@ -68,7 +68,6 @@ class SearchContext extends Object { /** * Returns scaffolded search fields for UI. * - * @todo is this necessary in the SearchContext? - ModelAdmin could unwrap this and just use DataObject::scaffoldSearchFields * @return FieldSet */ public function getSearchFields() { @@ -77,6 +76,36 @@ class SearchContext extends Object { return singleton($this->modelClass)->scaffoldSearchFields(); } + /** + * @todo fix hack + */ + protected function applyBaseTableFields() { + $classes = ClassInfo::dataClassesFor($this->modelClass); + //Debug::dump($classes); + //die(); + $fields = array($classes[0].'.*', $this->modelClass.'.*'); + //$fields = array_keys($model->db()); + $fields[] = $classes[0].'.ClassName AS RecordClassName'; + return $fields; + } + + /** + * @todo fix hack + */ + protected function applyBaseTable() { + $classes = ClassInfo::dataClassesFor($this->modelClass); + return $classes[0]; + } + + /** + * @todo only works for one level deep of inheritance + * @todo fix hack + */ + protected function applyBaseTableJoin($query) { + $classes = ClassInfo::dataClassesFor($this->modelClass); + if (count($classes) > 1) $query->leftJoin($classes[1], "{$classes[1]}.ID = {$classes[0]}.ID"); + } + /** * Returns a SQL object representing the search context for the given * list of query parameters. @@ -86,13 +115,32 @@ class SearchContext extends Object { */ public function getQuery($searchParams, $start = false, $limit = false) { $model = singleton($this->modelClass); - $fields = array_keys($model->db()); - $query = new SQLQuery($fields, $this->modelClass); + + $fields = $this->applyBaseTableFields($model); + + $query = new SQLQuery($fields); + + $baseTable = $this->applyBaseTable(); + $query->from($baseTable); + + // SRM: This stuff is copied from DataObject, + if($this->modelClass != $baseTable) { + $classNames = ClassInfo::subclassesFor($this->modelClass); + $query->where[] = "`$baseTable`.ClassName IN ('" . implode("','", $classNames) . "')"; + } + + + $this->applyBaseTableJoin($query); + foreach($searchParams as $key => $value) { - $filter = $this->getFilter($key); - if ($filter) { - $filter->setValue($value); - $filter->apply($query); + if ($value != '0') { + $key = str_replace('__', '.', $key); + $filter = $this->getFilter($key); + if ($filter) { + $filter->setModel($this->modelClass); + $filter->setValue($value); + $filter->apply($query); + } } } return $query; @@ -111,16 +159,16 @@ class SearchContext extends Object { public function getResults($searchParams, $start = false, $limit = false) { $searchParams = array_filter($searchParams, array($this,'clearEmptySearchFields')); $query = $this->getQuery($searchParams, $start, $limit); - // + // use if a raw SQL query is needed - //$results = new DataObjectSet(); - //foreach($query->execute() as $row) { - // $className = $row['ClassName']; - // $results->push(new $className($row)); - //} - //return $results; + $results = new DataObjectSet(); + foreach($query->execute() as $row) { + $className = $row['RecordClassName']; + $results->push(new $className($row)); + } + return $results; // - return DataObject::get($this->modelClass, $query->getFilter(), "", "", $limit); + //return DataObject::get($this->modelClass, $query->getFilter(), "", "", $limit); } /** diff --git a/search/filters/ExactMatchFilter.php b/search/filters/ExactMatchFilter.php index e02b12a2b..4b461993e 100644 --- a/search/filters/ExactMatchFilter.php +++ b/search/filters/ExactMatchFilter.php @@ -16,7 +16,8 @@ class ExactMatchFilter extends SearchFilter { * @return unknown */ public function apply(SQLQuery $query) { - return $query->where("{$this->name} = '{$this->value}'"); + $query = $this->applyRelation($query); + return $query->where("{$this->getName()} = '{$this->value}'"); } } diff --git a/search/filters/PartialMatchFilter.php b/search/filters/PartialMatchFilter.php index b87e145c6..e67f08c02 100644 --- a/search/filters/PartialMatchFilter.php +++ b/search/filters/PartialMatchFilter.php @@ -8,7 +8,8 @@ class PartialMatchFilter extends SearchFilter { public function apply(SQLQuery $query) { - return $query->where("{$this->name} LIKE '%{$this->value}%'"); + $query = $this->applyRelation($query); + return $query->where("{$this->getName()} LIKE '%{$this->value}%'"); } } diff --git a/search/filters/SearchFilter.php b/search/filters/SearchFilter.php index 5c377a046..154c6bde6 100644 --- a/search/filters/SearchFilter.php +++ b/search/filters/SearchFilter.php @@ -7,11 +7,13 @@ */ abstract class SearchFilter extends Object { + protected $model; protected $name; protected $value; + protected $relation; function __construct($name, $value = false) { - $this->name = $name; + $this->addRelation($name); $this->value = $value; } @@ -19,6 +21,75 @@ abstract class SearchFilter extends Object { $this->value = $value; } + public function setModel($className) { + $this->model = $className; + } + + /** + * Normalizes the field name to table mapping. + * + */ + protected function getName() { + // SRM: This code finds the table where the field named $this->name lives + // Todo: move to somewhere more appropriate, such as DataMapper, the magical class-to-be? + $candidateClass = $this->model; + while($candidateClass != 'DataObject') { + if(singleton($candidateClass)->fieldExists($this->name)) break; + $candidateClass = get_parent_class($candidateClass); + } + if($candidateClass == 'DataObject') user_error("Couldn't find field $this->name in any of $this->model's tables.", E_USER_ERROR); + + return $candidateClass . "." . $this->name; + } + + protected function addRelation($name) { + if (strstr($name, '.')) { + $parts = explode('.', $name); + $this->name = array_pop($parts); + $this->relation = $parts; + } else { + $this->name = $name; + } + } + + /** + * Applies multiple-table inheritance to straight joins on the data objects + * + * @todo Should this be applied in SQLQuery->from instead? !!! + * + * @return void + */ + protected function applyJoin($query, $model, $component) { + $query->leftJoin($component, "{$this->model}.ID = $component.{$model->getReverseAssociation($this->model)}ID"); + } + + /** + * Traverse the relationship fields, and add the table + * mappings to the query object state. + * + * @todo move join specific crap into SQLQuery + * + * @param unknown_type $query + * @return unknown + */ + protected function applyRelation($query) { + if (is_array($this->relation)) { + $model = singleton($this->model); + foreach($this->relation as $rel) { + if ($component = $model->has_one($rel)) { + $model = singleton($component); + $this->applyJoin($query, $model, $component); + $this->model = $component; + } elseif ($component = $model->has_many($rel)) { + $model = singleton($component); + $this->applyJoin($query, $model, $component); + $this->model = $component; + } + } + } + return $query; + } + /** * Apply filter criteria to a SQL query. * diff --git a/tests/CsvBulkLoaderTest.php b/tests/CsvBulkLoaderTest.php new file mode 100644 index 000000000..830f54e1c --- /dev/null +++ b/tests/CsvBulkLoaderTest.php @@ -0,0 +1,172 @@ +getLineCount($file); + fgetcsv($file); // pop header row + $compareRow = fgetcsv($file); + $results = $loader->load($filepath); + + // Test that right amount of columns was imported + $this->assertEquals($results->Count(), $compareCount-1, 'Test correct count of imported data'); + + // Test that columns were correctly imported + $obj = DataObject::get_by_id('CsvBulkLoaderTest_Player', $results->First()->id); + $this->assertEquals($compareRow[0], $obj->FirstName); + $this->assertEquals($compareRow[1], $obj->Biography); + $date = DBField::create('Date', $compareRow[2])->RAW(); + $this->assertEquals($date, $obj->Birthday); + + fclose($file); + } + + /** + * Test import with manual column mapping + */ + function testLoadWithColumnMap() { + $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player'); + $filepath = Director::baseFolder() . '/sapphire/tests/CsvBulkLoaderTest_Players.csv'; + $file = fopen($filepath, 'r'); + $compareCount = $this->getLineCount($file); + $compareRow = fgetcsv($file); + $loader->columnMap = array( + 'FirstName', + 'Biography', + null, // ignored column + 'Birthday' + ); + $results = $loader->load($filepath); + + // Test that right amount of columns was imported + $this->assertEquals($results->Count(), $compareCount, 'Test correct count of imported data'); + + // Test that columns were correctly imported + $obj = DataObject::get_by_id('CsvBulkLoaderTest_Player', $results->First()->id); + $this->assertEquals($compareRow[0], $obj->FirstName); + $this->assertEquals($compareRow[1], $obj->Biography); + $date = DBField::create('Date', $compareRow[3])->RAW(); + $this->assertEquals($date, $obj->Birthday); + + fclose($file); + } + + /** + * Test import with manual column mapping and custom column names + */ + function testLoadWithCustomHeaderAndRelation() { + $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player'); + $filepath = Director::baseFolder() . '/sapphire/tests/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv'; + $file = fopen($filepath, 'r'); + $compareCount = $this->getLineCount($file); + fgetcsv($file); // pop header row + $compareRow = fgetcsv($file); + $loader->columnMap = array( + 'first name' => 'FirstName', + 'bio' => 'Biography', + 'bday' => 'Birthday', + 'teamtitle' => 'Team.Title', // test existing relation + 'teamsize' => 'Team.TeamSize', // test existing relation + 'salary' => 'Contract.Amount' // test relation creation + ); + $loader->hasHeaderRow = true; + $loader->relationCallbacks = array( + 'Team.Title' => array( + 'relationname' => 'Team', + 'callback' => 'getTeamByTitle' + ), + // contract should be automatically discovered + ); + $results = $loader->load($filepath); + + // Test that right amount of columns was imported + $this->assertEquals($results->Count(), $compareCount-1, 'Test correct count of imported data'); + + // Test of augumenting existing relation (created by fixture) + $testTeam = DataObject::get_one('CsvBulkLoaderTest_Team', null, null, 'Created DESC'); + $this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works'); + + // Test of creating relation + $testContract = DataObject::get_one('CsvBulkLoaderTest_PlayerContract'); + $testPlayer = DataObject::get_by_id('CsvBulkLoaderTest_Player', $results->First()->id); + $this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works'); + + // Test nested setting of relation properties + $contractAmount = DBField::create('Currency', $compareRow[5])->RAW(); + $testPlayer = DataObject::get_by_id('CsvBulkLoaderTest_Player', $results->First()->id); + $this->assertEquals($testPlayer->Contract()->Amount, $contractAmount, 'Setting nested values in a relation works'); + + // Test that columns were correctly imported + $obj = DataObject::get_by_id('CsvBulkLoaderTest_Player', $results->First()->id); + + fclose($file); + } + + protected function getLineCount(&$file) { + $i = 0; + while(fgets($file) !== false) $i++; + rewind($file); + return $i; + } + +} + +class CsvBulkLoaderTest_Team extends DataObject implements TestOnly { + + static $db = array( + 'Title' => 'Varchar(255)', + 'TeamSize' => 'Int', + ); + + static $has_many = array( + 'Players' => 'CsvBulkLoaderTest_Player', + ); + +} + +class CsvBulkLoaderTest_Player extends DataObject implements TestOnly { + + static $db = array( + 'FirstName' => 'Varchar(255)', + 'Biography' => 'HTMLText', + 'Birthday' => 'Date', + ); + + static $has_one = array( + 'Team' => 'CsvBulkLoaderTest_Team', + 'Contract' => 'CsvBulkLoaderTest_PlayerContract' + ); + + public function getTeamByTitle($title) { + $SQL_title = Convert::raw2sql($title); + return DataObject::get_one('CsvBulkLoaderTest_Team', "Title = '{$SQL_title}'"); + } + + /** + * Custom setter for "Birthday" property when passed/imported + * in different format. + * + * @param string $val + * @param array $record + */ + public function setUSBirthday($val, $record) { + $this->Birthday = preg_replace('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-90-9]{2,4})/', '\\3-\\1-\\2', $val); + } +} + +class CsvBulkLoaderTest_PlayerContract extends DataObject implements TestOnly { + static $db = array( + 'Amount' => 'Currency', + ); +} + +?> \ No newline at end of file diff --git a/tests/CsvBulkLoaderTest.yml b/tests/CsvBulkLoaderTest.yml new file mode 100644 index 000000000..51bdda398 --- /dev/null +++ b/tests/CsvBulkLoaderTest.yml @@ -0,0 +1,3 @@ +CsvBulkLoaderTest_Team: + team1: + Title: My Team \ No newline at end of file diff --git a/tests/CsvBulkLoaderTest_Players.csv b/tests/CsvBulkLoaderTest_Players.csv new file mode 100644 index 000000000..2f7f3d67c --- /dev/null +++ b/tests/CsvBulkLoaderTest_Players.csv @@ -0,0 +1,4 @@ +"John","He's a good guy","ignored","31/01/1988" +"Jane","She is awesome.\nSo awesome that she gets multiple rows and \"escaped\" strings in her biography","ignored","31/01/1982" +"Jamie","Pretty old\, with an escaped comma","ignored","31/01/1882" +"Järg","Unicode FTW","ignored","31/06/1982" \ No newline at end of file diff --git a/tests/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv b/tests/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv new file mode 100644 index 000000000..2c1302e09 --- /dev/null +++ b/tests/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv @@ -0,0 +1,2 @@ +"first name","bio","bday","teamtitle","teamsize","salary" +"John","He's a good guy","1988-01-31","My Team","20","20000" \ No newline at end of file diff --git a/tests/CsvBulkLoaderTest_PlayersWithHeader.csv b/tests/CsvBulkLoaderTest_PlayersWithHeader.csv new file mode 100644 index 000000000..e77222b2c --- /dev/null +++ b/tests/CsvBulkLoaderTest_PlayersWithHeader.csv @@ -0,0 +1,5 @@ +"FirstName","Biography","Birthday" +"John","He's a good guy","31/01/1988" +"Jane","She is awesome.\nSo awesome that she gets multiple rows and \"escaped\" strings in her biography","31/01/1982" +"Jamie","Pretty old\, with an escaped comma","31/01/1882" +"Järg","Unicode FTW","31/06/1982" \ No newline at end of file diff --git a/tests/SQLQueryTest.php b/tests/SQLQueryTest.php index 5d78d534a..098b714b8 100644 --- a/tests/SQLQueryTest.php +++ b/tests/SQLQueryTest.php @@ -14,7 +14,7 @@ class SQLQueryTest extends SapphireTest { $query->from[] = "MyTable"; $this->assertEquals("SELECT * FROM MyTable", $query->sql()); $query->from[] = "MyJoin"; - $this->assertEquals("SELECT * FROM MyTable, MyJoin", $query->sql()); + $this->assertEquals("SELECT * FROM MyTable MyJoin", $query->sql()); } function testSelectFromUserSpecifiedFields() { diff --git a/tests/SearchContextTest.php b/tests/SearchContextTest.php index bf065cb33..6ad595316 100644 --- a/tests/SearchContextTest.php +++ b/tests/SearchContextTest.php @@ -74,9 +74,22 @@ class SearchContextTest extends SapphireTest { $project = singleton('SearchContextTest_Project'); $context = $project->getDefaultSearchContext(); - $query = array("Name"=>"Blog Website"); + $params = array("Name"=>"Blog Website", "Actions__SolutionArea"=>"technical"); - $results = $context->getQuery($query); + $results = $context->getResults($params); + + $this->assertEquals(1, $results->Count()); + + Debug::dump(DB::query("select * from SearchContextTest_Deadline")->next()); + + $project = $results->First(); + + $this->assertType('SearchContextTest_Project', $project); + $this->assertEquals("Blog Website", $project->Name); + $this->assertEquals(2, $project->Actions()->Count()); + $this->assertEquals("Get RSS feeds working", $project->Actions()->First()->Description); + Debug::dump($project->Deadline()->CompletionDate); + //$this->assertEquals() } function testCanGenerateQueryUsingAllFilterTypes() { @@ -157,7 +170,8 @@ class SearchContextTest_Project extends DataObject implements TestOnly { static $searchable_fields = array( "Name" => "PartialMatchFilter", - "Actions.SolutionArea" => "ExactMatchFilter" + "Actions.SolutionArea" => "ExactMatchFilter", + "Actions.Description" => "PartialMatchFilter" ); } @@ -182,7 +196,7 @@ class SearchContextTest_Action extends DataObject implements TestOnly { ); static $has_one = array( - "Project" => "SearchContextTest_Project" + "Project" => "SearchContextTest_Project" ); } diff --git a/tests/SearchContextTest.yml b/tests/SearchContextTest.yml index 1e1c699c9..e889a9783 100644 --- a/tests/SearchContextTest.yml +++ b/tests/SearchContextTest.yml @@ -25,32 +25,36 @@ SearchContextTest_Person: HairColor: black EyeColor: green +SearchContextTest_Action: + action1: + Description: Get search context working + SolutionArea: technical + action2: + Description: Get relationship editor working + SolutionArea: design + action3: + Description: Get RSS feeds working + SolutionArea: technical + action4: + Description: New logotype + SolutionArea: design + + SearchContextTest_Deadline: deadline1: CompletionDate: 2008-05-29 09:00:00 deadline2: - CompletionDate: 2008-05-29 09:00:00 - -SearchContextTest_Action: - action1: - Description: Get search context working - SolutionArea: backend - action2: - Description: Get relationship editor working - SolutionArea: frontend - action3: - Description: Get RSS feeds working - SolutionArea: technical + CompletionDate: 2008-05-20 09:00:00 SearchContextTest_Project: project1: Name: CRM Application - Deadline: =>SearchContextTest_Deadline.deadline1 + DeadlineID: =>SearchContextTest_Deadline.deadline1 Actions: =>SearchContextTest_Action.action1,=>SearchContextTest_Action.action2 project2: Name: Blog Website - Deadline: =>SearchContextTest_Deadline.deadline2 - Actions: =>SearchContextTest_Action.action3 + DeadlineID: =>SearchContextTest_Deadline.deadline2 + Actions: =>SearchContextTest_Action.action3,=>SearchContextTest_Action.action4 SearchContextTest_AllFilterTypes: filter1: diff --git a/tools/BulkLoader.php b/tools/BulkLoader.php new file mode 100644 index 000000000..19f6777fa --- /dev/null +++ b/tools/BulkLoader.php @@ -0,0 +1,154 @@ +@silverstripe.com) + */ +abstract class BulkLoader extends ViewableData { + + /** + * Each row in the imported dataset should map to one instance + * of this class (with optional property translation + * through {@self::$columnMaps}. + * + * @var string + */ + public $objectClass; + + /** + * Override this on subclasses to give the specific functions names. + * + * @var string + */ + public static $title; + + /** + * Map columns to DataObject-properties. + * If not specified, we assume the first row + * in the file contains the column headers. + * The order of your array should match the column order. + * + * The column count should match the count of array elements, + * fill with NULL values if you want to skip certain columns. + * + * Supports one-level chaining of has_one relations and properties with dot notation + * (e.g. Team.Title). The first part has to match a has_one relation name + * (not necessarily the classname of the used relation). + * + * + * 'FirstName', // custom column name + * null, // ignored column + * 'RegionID', // direct has_one/has_many ID setting + * 'OrganisationTitle', // create has_one relation to existing record using $relationCallbacks + * 'street' => 'Organisation.StreetName', // match an existing has_one or create one and write property. + * ); + * ?> + * + * + * @var array + */ + public $columnMap; + + /** + * Find a has_one relation based on a specific column value. + * + * + * array( + * 'relationname' => 'Organisation', // relation accessor name + * 'callback' => 'getOrganisationByTitle', + * ); + * ); + * ?> + * + * + * @var array + */ + public $relationCallbacks; + + function __construct($objectClass) { + $this->objectClass = $objectClass; + } + + /* + * Load the given file via {@link self::processAll()} and {@link self::processRecord()}. + * + * @return array See {@link self::processAll()} + */ + public function load($filepath) { + return $this->processAll($filepath); + } + + /** + * Preview a file import (don't write anything to the database). + * Useful to analyze the input and give the users a chance to influence + * it through a UI. + * + * @param string $filepath Absolute path to the file we're importing + * @return array See {@link self::processAll()} + */ + public function preview($filepath) { + return $this->processAll($filepath, true); + } + + /** + * Process every record in the file + * + * @param string $filepath Absolute path to the file we're importing (with UTF8 content) + * @param boolean $preview If true, we'll just output a summary of changes but not actually do anything + * @return array Information about the import process, with each row matching a created or updated DataObject. + * Array structure: + * - 'id': Database id of the created or updated record + * - 'action': Performed action ('create', 'update') + * - 'message': free-text string that can optionally provide some more information about what changes have + */ + abstract protected function processAll($filepath, $preview = false); + + + /** + * Process a single record from the file. + * + * @param array $record An map of the data, keyed by the header field defined in {@link self::$columnMap} + * @param boolean $preview + * @return ArrayData @see self::processAll() + */ + abstract protected function processRecord($record, $preview = false); + + /** + * Return a FieldSet containing all the options for this form; this + * doesn't include the actual upload field itself + */ + public function getOptionFields() {} + + /** + * Return a human-readable name for this object. + * It defaults to the class name can be overridden by setting the static variable $title + * + * @return string + */ + public function Title() { + return ($title = $this->stat('title')) ? $title : $this->class; + } + +} +?> \ No newline at end of file diff --git a/tools/CsvBulkLoader.php b/tools/CsvBulkLoader.php new file mode 100644 index 000000000..26fa069f8 --- /dev/null +++ b/tools/CsvBulkLoader.php @@ -0,0 +1,139 @@ +@silverstripe.com) + */ +class CsvBulkLoader extends BulkLoader { + + /** + * Delimiter character (Default: comma). + * + * @var string + */ + public $delimiter = ','; + + /** + * Enclosure character (Default: doublequote) + * + * @var string + */ + public $enclosure = '"'; + + /** + * Identifies if the loaded file has a header row. + * If a {@link self::$columnMap} is passed, we assume + * the file has no headerrow, unless explicitly noted. + * + * @var boolean + */ + public $hasHeaderRow = false; + + protected function processAll($filepath, $preview = false) { + $file = fopen($filepath, 'r'); + if(!$file) return false; + + $return = new DataObjectSet(); + + // assuming that first row is column naming if no columnmap is passed + if($this->hasHeaderRow && $this->columnMap) { + $columnRow = fgetcsv($file, 0, $this->delimiter, $this->enclosure); + $columnMap = $this->columnMap; + } elseif($this->columnMap) { + $columnMap = $this->columnMap; + } else { + $columnRow = fgetcsv($file, 0, $this->delimiter, $this->enclosure); + $columnMap = array_combine($columnRow, $columnRow); + } + + while (($row = fgetcsv($file, 0, $this->delimiter, $this->enclosure)) !== FALSE) { + $indexedRow = array_combine(array_values($columnMap), array_values($row)); + $return->push($this->processRecord($indexedRow)); + } + + fclose($file); + + return $return; + } + + + protected function processRecord($record, $preview = false) { + $class = $this->objectClass; + $obj = new $class(); + + // first run: find/create any relations and store them on the object + // we can't combine runs, as other columns might rely on the relation being present + $relations = array(); + foreach($record as $key => $val) { + if(isset($this->relationCallbacks[$key])) { + // trigger custom search method for finding a relation based on the given value + // and write it back to the relation (or create a new object) + $relationName = $this->relationCallbacks[$key]['relationname']; + $relationObj = $obj->{$this->relationCallbacks[$key]['callback']}($val, $record); + if(!$relationObj || !$relationObj->exists()) { + $relationClass = $obj->has_one($relationName); + $relationObj = new $relationClass(); + $relationObj->write(); + } + $obj->setComponent($relationName, $relationObj); + $obj->{"{$relationName}ID"} = $relationObj->ID; + } elseif(strpos($key, '.') !== false) { + // we have a relation column with dot notation + list($relationName,$columnName) = split('\.', $key); + $relationObj = $obj->getComponent($relationName); // always gives us an component (either empty or existing) + $obj->setComponent($relationName, $relationObj); + $relationObj->write(); + $obj->{"{$relationName}ID"} = $relationObj->ID; + } + $obj->flushCache(); // avoid relation caching confusion + } + $id = ($preview) ? 0 : $obj->write(); + + + // second run: save data + foreach($record as $key => $val) { + if($obj->hasMethod("import{$key}")) { + $obj->{"import{$key}"}($val, $record); + } elseif(strpos($key, '.') !== false) { + // we have a relation column + list($relationName,$columnName) = split('\.', $key); + $relationObj = $obj->getComponent($relationName); + $relationObj->{$columnName} = $val; + $relationObj->write(); + $obj->flushCache(); // avoid relation caching confusion + } elseif($obj->hasField($key)) { + // plain old value setter + $obj->{$key} = $val; + } + } + $id = ($preview) ? 0 : $obj->write(); + $action = 'create'; + $message = ''; + + // memory usage + unset($obj); + + return new ArrayData(array( + 'id' => $id, + 'action' => $action, + 'message' => $message + )); + } + + + /** + * Determine wether any loaded files should be parsed + * with a header-row (otherwise we rely on {@link self::$columnMap}. + * + * @return boolean + */ + public function hasHeaderRow() { + return ($this->hasHeaderRow || isset($this->columnMap)); + } + +} +?> \ No newline at end of file