mirror of
https://github.com/silverstripe/silverstripe-framework
synced 2024-10-22 14:05:37 +02:00
ENHANCEMENT Added BulkLoader->deleteExistingRecords(), removed unnecessary parameters from BulkLoader->load() (merged from r94250)
ENHANCEMENT Decreased memory usage in BulkLoader->load() when deleting all records before importing (merged from r94250) MINOR Re-enabled CsvBulkloaderTest cases, were disabled by accident (merged from r94250) git-svn-id: svn://svn.silverstripe.com/silverstripe/open/modules/sapphire/branches/2.4@98713 467b73ca-7a2a-4603-9d3b-597d59a354a9
This commit is contained in:
parent
e0f9137afc
commit
f9f2ed9c8f
@ -118,27 +118,36 @@ abstract class BulkLoader extends ViewableData {
|
||||
*/
|
||||
public $duplicateChecks = array();
|
||||
|
||||
/**
|
||||
* @var Boolean $clearBeforeImport Delete ALL records before importing.
|
||||
*/
|
||||
public $deleteExistingRecords = false;
|
||||
|
||||
function __construct($objectClass) {
|
||||
$this->objectClass = $objectClass;
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/*
|
||||
* Load the given file via {@link self::processAll()} and {@link self::processRecord()}. Optionally truncates (clear) the table before it imports.
|
||||
* Load the given file via {@link self::processAll()} and {@link self::processRecord()}.
|
||||
* Optionally truncates (clear) the table before it imports.
|
||||
*
|
||||
* @return BulkLoader_Result See {@link self::processAll()}
|
||||
*/
|
||||
public function load($filepath, $memory_limit='512M', $clear_table_before_import=false) {
|
||||
public function load($filepath) {
|
||||
ini_set('max_execution_time', 3600);
|
||||
increase_memory_limit_to('512M');
|
||||
|
||||
increase_memory_limit_to($memory_limit);
|
||||
|
||||
if ($clear_table_before_import) {
|
||||
$objectSet = DataObject::get($this->objectClass); //get all instances of the to be imported data object
|
||||
if (!empty($objectSet)) {
|
||||
foreach($objectSet as $obj) {
|
||||
$obj->delete(); //deleting objects ensures that versions are also deleted (truncating would just delete the main table); performance is slower, however
|
||||
}
|
||||
//get all instances of the to be imported data object
|
||||
if($this->deleteExistingRecords) {
|
||||
$q = singleton($this->objectClass)->buildSQL();
|
||||
$q->select = array('"ID"');
|
||||
$ids = $q->execute()->column('ID');
|
||||
foreach($ids as $id) {
|
||||
$obj = DataObject::get_by_id($this->objectClass, $id);
|
||||
$obj->delete();
|
||||
$obj->destroy();
|
||||
unset($obj);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -13,128 +13,129 @@ class CsvBulkLoaderTest extends SapphireTest {
|
||||
'CsvBulkLoaderTest_PlayerContract',
|
||||
);
|
||||
|
||||
// /**
|
||||
// * Test plain import with column auto-detection
|
||||
// */
|
||||
// function testLoad() {
|
||||
// $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
||||
// $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
||||
// $file = fopen($filepath, 'r');
|
||||
// $compareCount = $this->getLineCount($file);
|
||||
// fgetcsv($file); // pop header row
|
||||
// $compareRow = fgetcsv($file);
|
||||
// $results = $loader->load($filepath);
|
||||
//
|
||||
// // Test that right amount of columns was imported
|
||||
// $this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
|
||||
//
|
||||
// // Test that columns were correctly imported
|
||||
// $obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
||||
// $this->assertNotNull($obj);
|
||||
// $this->assertEquals("He's a good guy", $obj->Biography);
|
||||
// $this->assertEquals("1988-01-31", $obj->Birthday);
|
||||
// $this->assertEquals("1", $obj->IsRegistered);
|
||||
//
|
||||
// fclose($file);
|
||||
// }
|
||||
//
|
||||
// /**
|
||||
// * Test plain import with clear_table_before_import
|
||||
// */
|
||||
// function testClearTableBeforeImport() {
|
||||
// $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
||||
// $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
||||
// $results1 = $loader->load($filepath, '512MB', false); //leave existing data there on first CSV import
|
||||
// $this->assertEquals(4, $results1->Count(), 'Test correct count of imported data on first load');
|
||||
//
|
||||
// $results2 = $loader->load($filepath, '512MB', true); //delete existing data before doing second CSV import
|
||||
// $resultDataObject = DataObject::get('CsvBulkLoaderTest_Player'); //get all instances of the loaded DataObject from the database and count them
|
||||
//
|
||||
// $this->assertEquals(4, $resultDataObject->Count(), 'Test if existing data is deleted before new data is added');
|
||||
// }
|
||||
//
|
||||
// /**
|
||||
// * Test import with manual column mapping
|
||||
// */
|
||||
// function testLoadWithColumnMap() {
|
||||
// $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
||||
// $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_Players.csv';
|
||||
// $file = fopen($filepath, 'r');
|
||||
// $compareCount = $this->getLineCount($file);
|
||||
// $compareRow = fgetcsv($file);
|
||||
// $loader->columnMap = array(
|
||||
// 'FirstName',
|
||||
// 'Biography',
|
||||
// null, // ignored column
|
||||
// 'Birthday',
|
||||
// 'IsRegistered'
|
||||
// );
|
||||
// $loader->hasHeaderRow = false;
|
||||
// $results = $loader->load($filepath);
|
||||
//
|
||||
// // Test that right amount of columns was imported
|
||||
// $this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
|
||||
//
|
||||
// // Test that columns were correctly imported
|
||||
// $obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
||||
// $this->assertNotNull($obj);
|
||||
// $this->assertEquals("He's a good guy", $obj->Biography);
|
||||
// $this->assertEquals("1988-01-31", $obj->Birthday);
|
||||
// $this->assertEquals("1", $obj->IsRegistered);
|
||||
//
|
||||
// $obj2 = DataObject::get_one('CsvBulkLoaderTest_Player', "\"FirstName\" = 'Jane'");
|
||||
// $this->assertNotNull($obj2);
|
||||
// $this->assertEquals('0', $obj2->IsRegistered);
|
||||
//
|
||||
// fclose($file);
|
||||
// }
|
||||
//
|
||||
// /**
|
||||
// * Test import with manual column mapping and custom column names
|
||||
// */
|
||||
// function testLoadWithCustomHeaderAndRelation() {
|
||||
// $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
||||
// $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv';
|
||||
// $file = fopen($filepath, 'r');
|
||||
// $compareCount = $this->getLineCount($file);
|
||||
// fgetcsv($file); // pop header row
|
||||
// $compareRow = fgetcsv($file);
|
||||
// $loader->columnMap = array(
|
||||
// 'first name' => 'FirstName',
|
||||
// 'bio' => 'Biography',
|
||||
// 'bday' => 'Birthday',
|
||||
// 'teamtitle' => 'Team.Title', // test existing relation
|
||||
// 'teamsize' => 'Team.TeamSize', // test existing relation
|
||||
// 'salary' => 'Contract.Amount' // test relation creation
|
||||
// );
|
||||
// $loader->hasHeaderRow = true;
|
||||
// $loader->relationCallbacks = array(
|
||||
// 'Team.Title' => array(
|
||||
// 'relationname' => 'Team',
|
||||
// 'callback' => 'getTeamByTitle'
|
||||
// ),
|
||||
// // contract should be automatically discovered
|
||||
// );
|
||||
// $results = $loader->load($filepath);
|
||||
//
|
||||
// // Test that right amount of columns was imported
|
||||
// $this->assertEquals(1, $results->Count(), 'Test correct count of imported data');
|
||||
//
|
||||
// // Test of augumenting existing relation (created by fixture)
|
||||
// $testTeam = DataObject::get_one('CsvBulkLoaderTest_Team', null, null, '"Created" DESC');
|
||||
// $this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works');
|
||||
//
|
||||
// // Test of creating relation
|
||||
// $testContract = DataObject::get_one('CsvBulkLoaderTest_PlayerContract');
|
||||
// $testPlayer = Dataobject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
||||
// $this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works');
|
||||
//
|
||||
// // Test nested setting of relation properties
|
||||
// $contractAmount = DBField::create('Currency', $compareRow[5])->RAW();
|
||||
// $this->assertEquals($testPlayer->Contract()->Amount, $contractAmount, 'Setting nested values in a relation works');
|
||||
//
|
||||
// fclose($file);
|
||||
// }
|
||||
/**
|
||||
* Test plain import with column auto-detection
|
||||
*/
|
||||
function testLoad() {
|
||||
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
||||
$filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
||||
$file = fopen($filepath, 'r');
|
||||
$compareCount = $this->getLineCount($file);
|
||||
fgetcsv($file); // pop header row
|
||||
$compareRow = fgetcsv($file);
|
||||
$results = $loader->load($filepath);
|
||||
|
||||
// Test that right amount of columns was imported
|
||||
$this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
|
||||
|
||||
// Test that columns were correctly imported
|
||||
$obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
||||
$this->assertNotNull($obj);
|
||||
$this->assertEquals("He's a good guy", $obj->Biography);
|
||||
$this->assertEquals("1988-01-31", $obj->Birthday);
|
||||
$this->assertEquals("1", $obj->IsRegistered);
|
||||
|
||||
fclose($file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test plain import with clear_table_before_import
|
||||
*/
|
||||
function testDeleteExistingRecords() {
|
||||
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
||||
$filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv';
|
||||
$loader->deleteExistingRecords = true;
|
||||
$results1 = $loader->load($filepath);
|
||||
$this->assertEquals(4, $results1->Count(), 'Test correct count of imported data on first load');
|
||||
|
||||
$results2 = $loader->load($filepath, '512MB', true); //delete existing data before doing second CSV import
|
||||
$resultDataObject = DataObject::get('CsvBulkLoaderTest_Player'); //get all instances of the loaded DataObject from the database and count them
|
||||
|
||||
$this->assertEquals(4, $resultDataObject->Count(), 'Test if existing data is deleted before new data is added');
|
||||
}
|
||||
|
||||
/**
|
||||
* Test import with manual column mapping
|
||||
*/
|
||||
function testLoadWithColumnMap() {
|
||||
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
||||
$filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_Players.csv';
|
||||
$file = fopen($filepath, 'r');
|
||||
$compareCount = $this->getLineCount($file);
|
||||
$compareRow = fgetcsv($file);
|
||||
$loader->columnMap = array(
|
||||
'FirstName',
|
||||
'Biography',
|
||||
null, // ignored column
|
||||
'Birthday',
|
||||
'IsRegistered'
|
||||
);
|
||||
$loader->hasHeaderRow = false;
|
||||
$results = $loader->load($filepath);
|
||||
|
||||
// Test that right amount of columns was imported
|
||||
$this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
|
||||
|
||||
// Test that columns were correctly imported
|
||||
$obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
||||
$this->assertNotNull($obj);
|
||||
$this->assertEquals("He's a good guy", $obj->Biography);
|
||||
$this->assertEquals("1988-01-31", $obj->Birthday);
|
||||
$this->assertEquals("1", $obj->IsRegistered);
|
||||
|
||||
$obj2 = DataObject::get_one('CsvBulkLoaderTest_Player', "\"FirstName\" = 'Jane'");
|
||||
$this->assertNotNull($obj2);
|
||||
$this->assertEquals('0', $obj2->IsRegistered);
|
||||
|
||||
fclose($file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test import with manual column mapping and custom column names
|
||||
*/
|
||||
function testLoadWithCustomHeaderAndRelation() {
|
||||
$loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
|
||||
$filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv';
|
||||
$file = fopen($filepath, 'r');
|
||||
$compareCount = $this->getLineCount($file);
|
||||
fgetcsv($file); // pop header row
|
||||
$compareRow = fgetcsv($file);
|
||||
$loader->columnMap = array(
|
||||
'first name' => 'FirstName',
|
||||
'bio' => 'Biography',
|
||||
'bday' => 'Birthday',
|
||||
'teamtitle' => 'Team.Title', // test existing relation
|
||||
'teamsize' => 'Team.TeamSize', // test existing relation
|
||||
'salary' => 'Contract.Amount' // test relation creation
|
||||
);
|
||||
$loader->hasHeaderRow = true;
|
||||
$loader->relationCallbacks = array(
|
||||
'Team.Title' => array(
|
||||
'relationname' => 'Team',
|
||||
'callback' => 'getTeamByTitle'
|
||||
),
|
||||
// contract should be automatically discovered
|
||||
);
|
||||
$results = $loader->load($filepath);
|
||||
|
||||
// Test that right amount of columns was imported
|
||||
$this->assertEquals(1, $results->Count(), 'Test correct count of imported data');
|
||||
|
||||
// Test of augumenting existing relation (created by fixture)
|
||||
$testTeam = DataObject::get_one('CsvBulkLoaderTest_Team', null, null, '"Created" DESC');
|
||||
$this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works');
|
||||
|
||||
// Test of creating relation
|
||||
$testContract = DataObject::get_one('CsvBulkLoaderTest_PlayerContract');
|
||||
$testPlayer = Dataobject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
|
||||
$this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works');
|
||||
|
||||
// Test nested setting of relation properties
|
||||
$contractAmount = DBField::create('Currency', $compareRow[5])->RAW();
|
||||
$this->assertEquals($testPlayer->Contract()->Amount, $contractAmount, 'Setting nested values in a relation works');
|
||||
|
||||
fclose($file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test import with custom identifiers by importing the data.
|
||||
|
Loading…
Reference in New Issue
Block a user