mirror of
https://github.com/silverstripe/silverstripe-framework
synced 2024-10-22 14:05:37 +02:00
d47b202697
FIX: Instead of CsvBulkLoader->findExistingRecord out right failing (i.e. no duplicate found) when the duplicate check field is empty, it will now continue on to check other duplicateCheck fields. Added extra testing data to CSVBulkLoaderTest so that it fails.
262 lines
8.8 KiB
PHP
262 lines
8.8 KiB
PHP
<?php
|
|
/**
|
|
* @package tests
|
|
*
|
|
* @todo Test with columnn headers and custom mappings
|
|
*/
|
|
class CsvBulkLoaderTest extends SapphireTest {
|
|
protected static $fixture_file = 'CsvBulkLoaderTest.yml';
|
|
|
|
protected $extraDataObjects = array(
|
|
'CsvBulkLoaderTest_Team',
|
|
'CsvBulkLoaderTest_Player',
|
|
'CsvBulkLoaderTest_PlayerContract',
|
|
);
|
|
|
|
/**
|
|
* Test plain import with column auto-detection
|
|
*/
|
|
public function testLoad() {
|
|
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
|
$file = fopen($filepath, 'r');
|
|
$compareCount = $this->getLineCount($file);
|
|
fgetcsv($file); // pop header row
|
|
$compareRow = fgetcsv($file);
|
|
$results = $loader->load($filepath);
|
|
|
|
// Test that right amount of columns was imported
|
|
$this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
|
|
|
|
// Test that columns were correctly imported
|
|
$obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
|
$this->assertNotNull($obj);
|
|
$this->assertEquals("He's a good guy", $obj->Biography);
|
|
$this->assertEquals("1988-01-31", $obj->Birthday);
|
|
$this->assertEquals("1", $obj->IsRegistered);
|
|
|
|
fclose($file);
|
|
}
|
|
|
|
/**
|
|
* Test plain import with clear_table_before_import
|
|
*/
|
|
public function testDeleteExistingRecords() {
|
|
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
|
$loader->deleteExistingRecords = true;
|
|
$results1 = $loader->load($filepath);
|
|
$this->assertEquals(4, $results1->Count(), 'Test correct count of imported data on first load');
|
|
|
|
//delete existing data before doing second CSV import
|
|
$results2 = $loader->load($filepath, '512MB', true);
|
|
//get all instances of the loaded DataObject from the database and count them
|
|
$resultDataObject = DataObject::get('CsvBulkLoaderTest_Player');
|
|
|
|
$this->assertEquals(4, $resultDataObject->Count(),
|
|
'Test if existing data is deleted before new data is added');
|
|
}
|
|
|
|
/**
|
|
* Test import with manual column mapping
|
|
*/
|
|
public function testLoadWithColumnMap() {
|
|
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_Players.csv';
|
|
$file = fopen($filepath, 'r');
|
|
$compareCount = $this->getLineCount($file);
|
|
$compareRow = fgetcsv($file);
|
|
$loader->columnMap = array(
|
|
'FirstName',
|
|
'Biography',
|
|
null, // ignored column
|
|
'Birthday',
|
|
'IsRegistered'
|
|
);
|
|
$loader->hasHeaderRow = false;
|
|
$results = $loader->load($filepath);
|
|
|
|
// Test that right amount of columns was imported
|
|
$this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
|
|
|
|
// Test that columns were correctly imported
|
|
$obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
|
$this->assertNotNull($obj);
|
|
$this->assertEquals("He's a good guy", $obj->Biography);
|
|
$this->assertEquals("1988-01-31", $obj->Birthday);
|
|
$this->assertEquals("1", $obj->IsRegistered);
|
|
|
|
$obj2 = DataObject::get_one('CsvBulkLoaderTest_Player', "\"FirstName\" = 'Jane'");
|
|
$this->assertNotNull($obj2);
|
|
$this->assertEquals('0', $obj2->IsRegistered);
|
|
|
|
fclose($file);
|
|
}
|
|
|
|
/**
|
|
* Test import with manual column mapping and custom column names
|
|
*/
|
|
public function testLoadWithCustomHeaderAndRelation() {
|
|
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv';
|
|
$file = fopen($filepath, 'r');
|
|
$compareCount = $this->getLineCount($file);
|
|
fgetcsv($file); // pop header row
|
|
$compareRow = fgetcsv($file);
|
|
$loader->columnMap = array(
|
|
'first name' => 'FirstName',
|
|
'bio' => 'Biography',
|
|
'bday' => 'Birthday',
|
|
'teamtitle' => 'Team.Title', // test existing relation
|
|
'teamsize' => 'Team.TeamSize', // test existing relation
|
|
'salary' => 'Contract.Amount' // test relation creation
|
|
);
|
|
$loader->hasHeaderRow = true;
|
|
$loader->relationCallbacks = array(
|
|
'Team.Title' => array(
|
|
'relationname' => 'Team',
|
|
'callback' => 'getTeamByTitle'
|
|
),
|
|
// contract should be automatically discovered
|
|
);
|
|
$results = $loader->load($filepath);
|
|
|
|
// Test that right amount of columns was imported
|
|
$this->assertEquals(1, $results->Count(), 'Test correct count of imported data');
|
|
|
|
// Test of augumenting existing relation (created by fixture)
|
|
$testTeam = DataObject::get_one('CsvBulkLoaderTest_Team', null, null, '"Created" DESC');
|
|
$this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works');
|
|
|
|
// Test of creating relation
|
|
$testContract = DataObject::get_one('CsvBulkLoaderTest_PlayerContract');
|
|
$testPlayer = Dataobject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
|
$this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works');
|
|
|
|
// Test nested setting of relation properties
|
|
$contractAmount = DBField::create_field('Currency', $compareRow[5])->RAW();
|
|
$this->assertEquals($testPlayer->Contract()->Amount, $contractAmount,
|
|
'Setting nested values in a relation works');
|
|
|
|
fclose($file);
|
|
}
|
|
|
|
/**
|
|
* Test import with custom identifiers by importing the data.
|
|
*
|
|
* @todo Test duplicateCheck callbacks
|
|
*/
|
|
public function testLoadWithIdentifiers() {
|
|
// first load
|
|
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithId.csv';
|
|
$loader->duplicateChecks = array(
|
|
'ExternalIdentifier' => 'ExternalIdentifier',
|
|
'NonExistantIdentifier' => 'ExternalIdentifier',
|
|
'ExternalIdentifier' => 'ExternalIdentifier',
|
|
'AdditionalIdentifier' => 'ExternalIdentifier'
|
|
);
|
|
$results = $loader->load($filepath);
|
|
$createdPlayers = $results->Created();
|
|
|
|
$player = $createdPlayers->First();
|
|
$this->assertEquals($player->FirstName, 'John');
|
|
$this->assertEquals($player->Biography, 'He\'s a good guy',
|
|
'test updating of duplicate imports within the same import works');
|
|
|
|
// load with updated data
|
|
$filepath = FRAMEWORK_PATH . '/tests/dev/CsvBulkLoaderTest_PlayersWithIdUpdated.csv';
|
|
$results = $loader->load($filepath);
|
|
|
|
// HACK need to update the loaded record from the database
|
|
$player = DataObject::get_by_id('CsvBulkLoaderTest_Player', $player->ID);
|
|
$this->assertEquals($player->FirstName, 'JohnUpdated', 'Test updating of existing records works');
|
|
$this->assertEquals($player->Biography, 'He\'s a good guy',
|
|
'Test retaining of previous information on duplicate when overwriting with blank field');
|
|
}
|
|
|
|
public function testLoadWithCustomImportMethods() {
|
|
$loader = new CsvBulkLoaderTest_CustomLoader('CsvBulkLoaderTest_Player');
|
|
$filepath = $this->getCurrentAbsolutePath() . '/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
|
$loader->columnMap = array(
|
|
'FirstName' => '->importFirstName',
|
|
'Biography' => 'Biography',
|
|
'Birthday' => 'Birthday',
|
|
'IsRegistered' => 'IsRegistered'
|
|
);
|
|
$results = $loader->load($filepath);
|
|
$createdPlayers = $results->Created();
|
|
$player = $createdPlayers->First();
|
|
$this->assertEquals($player->FirstName, 'Customized John');
|
|
$this->assertEquals($player->Biography, "He's a good guy");
|
|
$this->assertEquals($player->IsRegistered, "1");
|
|
}
|
|
|
|
protected function getLineCount(&$file) {
|
|
$i = 0;
|
|
while(fgets($file) !== false) $i++;
|
|
rewind($file);
|
|
return $i;
|
|
}
|
|
|
|
}
|
|
|
|
class CsvBulkLoaderTest_CustomLoader extends CsvBulkLoader implements TestOnly {
|
|
public function importFirstName(&$obj, $val, $record) {
|
|
$obj->FirstName = "Customized {$val}";
|
|
}
|
|
}
|
|
|
|
class CsvBulkLoaderTest_Team extends DataObject implements TestOnly {
|
|
|
|
private static $db = array(
|
|
'Title' => 'Varchar(255)',
|
|
'TeamSize' => 'Int',
|
|
);
|
|
|
|
private static $has_many = array(
|
|
'Players' => 'CsvBulkLoaderTest_Player',
|
|
);
|
|
|
|
}
|
|
|
|
class CsvBulkLoaderTest_Player extends DataObject implements TestOnly {
|
|
|
|
private static $db = array(
|
|
'FirstName' => 'Varchar(255)',
|
|
'Biography' => 'HTMLText',
|
|
'Birthday' => 'Date',
|
|
'ExternalIdentifier' => 'Varchar(255)', // used for uniqueness checks on passed property
|
|
'IsRegistered' => 'Boolean'
|
|
);
|
|
|
|
private static $has_one = array(
|
|
'Team' => 'CsvBulkLoaderTest_Team',
|
|
'Contract' => 'CsvBulkLoaderTest_PlayerContract'
|
|
);
|
|
|
|
public function getTeamByTitle($title) {
|
|
$SQL_title = Convert::raw2sql($title);
|
|
return DataObject::get_one('CsvBulkLoaderTest_Team', "\"Title\" = '{$SQL_title}'");
|
|
}
|
|
|
|
/**
|
|
* Custom setter for "Birthday" property when passed/imported
|
|
* in different format.
|
|
*
|
|
* @param string $val
|
|
* @param array $record
|
|
*/
|
|
public function setUSBirthday($val, $record = null) {
|
|
$this->Birthday = preg_replace('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-90-9]{2,4})/', '\\3-\\1-\\2', $val);
|
|
}
|
|
}
|
|
|
|
class CsvBulkLoaderTest_PlayerContract extends DataObject implements TestOnly {
|
|
private static $db = array(
|
|
'Amount' => 'Currency',
|
|
);
|
|
}
|
|
|
|
|