mirror of
https://github.com/silverstripe/silverstripe-framework
synced 2024-10-22 14:05:37 +02:00
c8af0fd7d1
Fixes http://open.silverstripe.org/ticket/6473 When using CSVParser::$columnMapping to map columns to a callback action, it previously used the action name as the key value. This prevented users from defining multiple entries to the same callback. This patch retains those key values and simply runs the callback field name filter later on.
289 lines
9.5 KiB
PHP
289 lines
9.5 KiB
PHP
<?php
|
|
|
|
/**
|
|
* @package framework
|
|
* @subpackage tests
|
|
*/
|
|
class CsvBulkLoaderTest extends SapphireTest {
|
|
|
|
protected static $fixture_file = 'CsvBulkLoaderTest.yml';
|
|
|
|
protected $extraDataObjects = array(
|
|
'CsvBulkLoaderTest_Team',
|
|
'CsvBulkLoaderTest_Player',
|
|
'CsvBulkLoaderTest_PlayerContract',
|
|
);
|
|
|
|
/**
|
|
* Test plain import with column auto-detection
|
|
*/
|
|
public function testLoad() {
|
|
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
|
$file = fopen($filepath, 'r');
|
|
$compareCount = $this->getLineCount($file);
|
|
fgetcsv($file); // pop header row
|
|
$compareRow = fgetcsv($file);
|
|
$results = $loader->load($filepath);
|
|
|
|
// Test that right amount of columns was imported
|
|
$this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
|
|
|
|
// Test that columns were correctly imported
|
|
$obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
|
$this->assertNotNull($obj);
|
|
$this->assertEquals("He's a good guy", $obj->Biography);
|
|
$this->assertEquals("1988-01-31", $obj->Birthday);
|
|
$this->assertEquals("1", $obj->IsRegistered);
|
|
|
|
fclose($file);
|
|
}
|
|
|
|
/**
|
|
* Test plain import with clear_table_before_import
|
|
*/
|
|
public function testDeleteExistingRecords() {
|
|
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
|
$loader->deleteExistingRecords = true;
|
|
$results1 = $loader->load($filepath);
|
|
$this->assertEquals(4, $results1->Count(), 'Test correct count of imported data on first load');
|
|
|
|
//delete existing data before doing second CSV import
|
|
$results2 = $loader->load($filepath, '512MB', true);
|
|
//get all instances of the loaded DataObject from the database and count them
|
|
$resultDataObject = DataObject::get('CsvBulkLoaderTest_Player');
|
|
|
|
$this->assertEquals(4, $resultDataObject->Count(),
|
|
'Test if existing data is deleted before new data is added');
|
|
}
|
|
|
|
/**
|
|
* Test import with manual column mapping
|
|
*/
|
|
public function testLoadWithColumnMap() {
|
|
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_Players.csv';
|
|
$file = fopen($filepath, 'r');
|
|
$compareCount = $this->getLineCount($file);
|
|
$compareRow = fgetcsv($file);
|
|
$loader->columnMap = array(
|
|
'FirstName',
|
|
'Biography',
|
|
null, // ignored column
|
|
'Birthday',
|
|
'IsRegistered'
|
|
);
|
|
$loader->hasHeaderRow = false;
|
|
$results = $loader->load($filepath);
|
|
|
|
// Test that right amount of columns was imported
|
|
$this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
|
|
|
|
// Test that columns were correctly imported
|
|
$obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
|
$this->assertNotNull($obj);
|
|
$this->assertEquals("He's a good guy", $obj->Biography);
|
|
$this->assertEquals("1988-01-31", $obj->Birthday);
|
|
$this->assertEquals("1", $obj->IsRegistered);
|
|
|
|
$obj2 = DataObject::get_one('CsvBulkLoaderTest_Player', "\"FirstName\" = 'Jane'");
|
|
$this->assertNotNull($obj2);
|
|
$this->assertEquals('0', $obj2->IsRegistered);
|
|
|
|
fclose($file);
|
|
}
|
|
|
|
/**
|
|
* Test import with manual column mapping and custom column names
|
|
*/
|
|
public function testLoadWithCustomHeaderAndRelation() {
|
|
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv';
|
|
$file = fopen($filepath, 'r');
|
|
$compareCount = $this->getLineCount($file);
|
|
fgetcsv($file); // pop header row
|
|
$compareRow = fgetcsv($file);
|
|
$loader->columnMap = array(
|
|
'first name' => 'FirstName',
|
|
'bio' => 'Biography',
|
|
'bday' => 'Birthday',
|
|
'teamtitle' => 'Team.Title', // test existing relation
|
|
'teamsize' => 'Team.TeamSize', // test existing relation
|
|
'salary' => 'Contract.Amount' // test relation creation
|
|
);
|
|
$loader->hasHeaderRow = true;
|
|
$loader->relationCallbacks = array(
|
|
'Team.Title' => array(
|
|
'relationname' => 'Team',
|
|
'callback' => 'getTeamByTitle'
|
|
),
|
|
// contract should be automatically discovered
|
|
);
|
|
$results = $loader->load($filepath);
|
|
|
|
// Test that right amount of columns was imported
|
|
$this->assertEquals(1, $results->Count(), 'Test correct count of imported data');
|
|
|
|
// Test of augumenting existing relation (created by fixture)
|
|
$testTeam = DataObject::get_one('CsvBulkLoaderTest_Team', null, null, '"Created" DESC');
|
|
$this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works');
|
|
|
|
// Test of creating relation
|
|
$testContract = DataObject::get_one('CsvBulkLoaderTest_PlayerContract');
|
|
$testPlayer = Dataobject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
|
$this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works');
|
|
|
|
// Test nested setting of relation properties
|
|
$contractAmount = DBField::create_field('Currency', $compareRow[5])->RAW();
|
|
$this->assertEquals($testPlayer->Contract()->Amount, $contractAmount,
|
|
'Setting nested values in a relation works');
|
|
|
|
fclose($file);
|
|
}
|
|
|
|
/**
|
|
* Test import with custom identifiers by importing the data.
|
|
*
|
|
* @todo Test duplicateCheck callbacks
|
|
*/
|
|
public function testLoadWithIdentifiers() {
|
|
// first load
|
|
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithId.csv';
|
|
$loader->duplicateChecks = array(
|
|
'ExternalIdentifier' => 'ExternalIdentifier',
|
|
'NonExistantIdentifier' => 'ExternalIdentifier',
|
|
'ExternalIdentifier' => 'ExternalIdentifier',
|
|
'AdditionalIdentifier' => 'ExternalIdentifier'
|
|
);
|
|
$results = $loader->load($filepath);
|
|
$createdPlayers = $results->Created();
|
|
|
|
$player = $createdPlayers->First();
|
|
$this->assertEquals($player->FirstName, 'John');
|
|
$this->assertEquals($player->Biography, 'He\'s a good guy',
|
|
'test updating of duplicate imports within the same import works');
|
|
|
|
// load with updated data
|
|
$filepath = FRAMEWORK_PATH . '/tests/dev/CsvBulkLoaderTest_PlayersWithIdUpdated.csv';
|
|
$results = $loader->load($filepath);
|
|
|
|
// HACK need to update the loaded record from the database
|
|
$player = DataObject::get_by_id('CsvBulkLoaderTest_Player', $player->ID);
|
|
$this->assertEquals($player->FirstName, 'JohnUpdated', 'Test updating of existing records works');
|
|
|
|
// null values are valid imported
|
|
// $this->assertEquals($player->Biography, 'He\'s a good guy',
|
|
// 'Test retaining of previous information on duplicate when overwriting with blank field');
|
|
}
|
|
|
|
public function testLoadWithCustomImportMethods() {
|
|
$loader = new CsvBulkLoaderTest_CustomLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
|
$loader->columnMap = array(
|
|
'FirstName' => '->importFirstName',
|
|
'Biography' => 'Biography',
|
|
'Birthday' => 'Birthday',
|
|
'IsRegistered' => 'IsRegistered'
|
|
);
|
|
$results = $loader->load($filepath);
|
|
$createdPlayers = $results->Created();
|
|
$player = $createdPlayers->First();
|
|
$this->assertEquals($player->FirstName, 'Customized John');
|
|
$this->assertEquals($player->Biography, "He's a good guy");
|
|
$this->assertEquals($player->IsRegistered, "1");
|
|
}
|
|
|
|
public function testLoadWithCustomImportMethodDuplicateMap() {
|
|
$loader = new CsvBulkLoaderTest_CustomLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
|
$loader->columnMap = array(
|
|
'FirstName' => '->updatePlayer',
|
|
'Biography' => '->updatePlayer',
|
|
'Birthday' => 'Birthday',
|
|
'IsRegistered' => 'IsRegistered'
|
|
);
|
|
|
|
$results = $loader->load($filepath);
|
|
|
|
$createdPlayers = $results->Created();
|
|
$player = $createdPlayers->First();
|
|
|
|
$this->assertEquals($player->FirstName, "John. He's a good guy. ");
|
|
}
|
|
|
|
|
|
protected function getLineCount(&$file) {
|
|
$i = 0;
|
|
while(fgets($file) !== false) $i++;
|
|
rewind($file);
|
|
return $i;
|
|
}
|
|
|
|
}
|
|
|
|
class CsvBulkLoaderTest_CustomLoader extends CsvBulkLoader implements TestOnly {
|
|
public function importFirstName(&$obj, $val, $record) {
|
|
$obj->FirstName = "Customized {$val}";
|
|
}
|
|
|
|
public function updatePlayer(&$obj, $val, $record) {
|
|
$obj->FirstName .= $val . '. ';
|
|
}
|
|
}
|
|
|
|
class CsvBulkLoaderTest_Team extends DataObject implements TestOnly {
|
|
|
|
private static $db = array(
|
|
'Title' => 'Varchar(255)',
|
|
'TeamSize' => 'Int',
|
|
);
|
|
|
|
private static $has_many = array(
|
|
'Players' => 'CsvBulkLoaderTest_Player',
|
|
);
|
|
|
|
}
|
|
|
|
class CsvBulkLoaderTest_Player extends DataObject implements TestOnly {
|
|
|
|
private static $db = array(
|
|
'FirstName' => 'Varchar(255)',
|
|
'Biography' => 'HTMLText',
|
|
'Birthday' => 'Date',
|
|
'ExternalIdentifier' => 'Varchar(255)', // used for uniqueness checks on passed property
|
|
'IsRegistered' => 'Boolean'
|
|
);
|
|
|
|
private static $has_one = array(
|
|
'Team' => 'CsvBulkLoaderTest_Team',
|
|
'Contract' => 'CsvBulkLoaderTest_PlayerContract'
|
|
);
|
|
|
|
public function getTeamByTitle($title) {
|
|
$SQL_title = Convert::raw2sql($title);
|
|
return DataObject::get_one('CsvBulkLoaderTest_Team', "\"Title\" = '{$SQL_title}'");
|
|
}
|
|
|
|
/**
|
|
* Custom setter for "Birthday" property when passed/imported
|
|
* in different format.
|
|
*
|
|
* @param string $val
|
|
* @param array $record
|
|
*/
|
|
public function setUSBirthday($val, $record = null) {
|
|
$this->Birthday = preg_replace('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-90-9]{2,4})/', '\\3-\\1-\\2', $val);
|
|
}
|
|
}
|
|
|
|
|
|
class CsvBulkLoaderTest_PlayerContract extends DataObject implements TestOnly {
|
|
private static $db = array(
|
|
'Amount' => 'Currency',
|
|
);
|
|
}
|
|
|
|
|