tonyair/silverstripe-framework
1
0
Fork 0
mirror of https://github.com/silverstripe/silverstripe-framework synced 2024-09-30 05:09:06 +02:00
Code Issues Projects Releases Wiki Activity

ENHANCEMENT Added BulkLoader->deleteExistingRecords(), removed unnecessary parameters from BulkLoader->load()

Browse Source 
ENHANCEMENT Decreased memory usage in BulkLoader->load() when deleting all records before importing
MINOR Re-enabled CsvBulkloaderTest cases, were disabled by accident

git-svn-id: svn://svn.silverstripe.com/silverstripe/open/modules/sapphire/trunk@94250 467b73ca-7a2a-4603-9d3b-597d59a354a9
This commit is contained in:
Ingo Schommer 2009-12-02 09:40:28 +00:00
parent 261f8f897a
commit ef28a27b25
2 changed files with 142 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
dev/BulkLoader.php
View File

@ -118,27 +118,36 @@ abstract class BulkLoader extends ViewableData {
*/ */
public $duplicateChecks = array(); public $duplicateChecks = array();
/**
* @var Boolean $clearBeforeImport Delete ALL records before importing.
*/
public $deleteExistingRecords = false;
function __construct($objectClass) { function __construct($objectClass) {
$this->objectClass = $objectClass; $this->objectClass = $objectClass;
parent::__construct(); parent::__construct();
} }
/* /*
* Load the given file via {@link self::processAll()} and {@link self::processRecord()}. Optionally truncates (clear) the table before it imports. * Load the given file via {@link self::processAll()} and {@link self::processRecord()}.
* Optionally truncates (clear) the table before it imports.
* *
* @return BulkLoader_Result See {@link self::processAll()} * @return BulkLoader_Result See {@link self::processAll()}
*/ */
public function load($filepath, $memory_limit='512M', $clear_table_before_import=false) { public function load($filepath) {
ini_set('max_execution_time', 3600); ini_set('max_execution_time', 3600);
increase_memory_limit_to('512M');
increase_memory_limit_to($memory_limit); //get all instances of the to be imported data object
if($this->deleteExistingRecords) {
if ($clear_table_before_import) { $q = singleton($this->objectClass)->buildSQL();
$objectSet = DataObject::get($this->objectClass); //get all instances of the to be imported data object $q->select = array('"ID"');
if (!empty($objectSet)) { $ids = $q->execute()->column('ID');
foreach($objectSet as $obj) { foreach($ids as $id) {
$obj->delete(); //deleting objects ensures that versions are also deleted (truncating would just delete the main table); performance is slower, however $obj = DataObject::get_by_id($this->objectClass, $id);
} $obj->delete();
$obj->destroy();
unset($obj);
} }
} }

245
tests/dev/CsvBulkLoaderTest.php
View File

@ -7,128 +7,129 @@
class CsvBulkLoaderTest extends SapphireTest { class CsvBulkLoaderTest extends SapphireTest {
static $fixture_file = 'sapphire/tests/dev/CsvBulkLoaderTest.yml'; static $fixture_file = 'sapphire/tests/dev/CsvBulkLoaderTest.yml';
// /** /**
// * Test plain import with column auto-detection * Test plain import with column auto-detection
// */ */
// function testLoad() { function testLoad() {
// $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player'); $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
// $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv'; $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv';
// $file = fopen($filepath, 'r'); $file = fopen($filepath, 'r');
// $compareCount = $this->getLineCount($file); $compareCount = $this->getLineCount($file);
// fgetcsv($file); // pop header row fgetcsv($file); // pop header row
// $compareRow = fgetcsv($file); $compareRow = fgetcsv($file);
// $results = $loader->load($filepath); $results = $loader->load($filepath);
//
// // Test that right amount of columns was imported // Test that right amount of columns was imported
// $this->assertEquals(4, $results->Count(), 'Test correct count of imported data'); $this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
//
// // Test that columns were correctly imported // Test that columns were correctly imported
// $obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'"); $obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
// $this->assertNotNull($obj); $this->assertNotNull($obj);
// $this->assertEquals("He's a good guy", $obj->Biography); $this->assertEquals("He's a good guy", $obj->Biography);
// $this->assertEquals("1988-01-31", $obj->Birthday); $this->assertEquals("1988-01-31", $obj->Birthday);
// $this->assertEquals("1", $obj->IsRegistered); $this->assertEquals("1", $obj->IsRegistered);
//
// fclose($file); fclose($file);
// } }
//
// /** /**
// * Test plain import with clear_table_before_import * Test plain import with clear_table_before_import
// */ */
// function testClearTableBeforeImport() { function testDeleteExistingRecords() {
// $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player'); $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
// $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv'; $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv';
// $results1 = $loader->load($filepath, '512MB', false); //leave existing data there on first CSV import $loader->deleteExistingRecords = true;
// $this->assertEquals(4, $results1->Count(), 'Test correct count of imported data on first load'); $results1 = $loader->load($filepath);
// $this->assertEquals(4, $results1->Count(), 'Test correct count of imported data on first load');
// $results2 = $loader->load($filepath, '512MB', true); //delete existing data before doing second CSV import
// $resultDataObject = DataObject::get('CsvBulkLoaderTest_Player'); //get all instances of the loaded DataObject from the database and count them $results2 = $loader->load($filepath, '512MB', true); //delete existing data before doing second CSV import
// $resultDataObject = DataObject::get('CsvBulkLoaderTest_Player'); //get all instances of the loaded DataObject from the database and count them
// $this->assertEquals(4, $resultDataObject->Count(), 'Test if existing data is deleted before new data is added');
// } $this->assertEquals(4, $resultDataObject->Count(), 'Test if existing data is deleted before new data is added');
// }
// /**
// * Test import with manual column mapping /**
// */ * Test import with manual column mapping
// function testLoadWithColumnMap() { */
// $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player'); function testLoadWithColumnMap() {
// $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_Players.csv'; $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
// $file = fopen($filepath, 'r'); $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_Players.csv';
// $compareCount = $this->getLineCount($file); $file = fopen($filepath, 'r');
// $compareRow = fgetcsv($file); $compareCount = $this->getLineCount($file);
// $loader->columnMap = array( $compareRow = fgetcsv($file);
// 'FirstName', $loader->columnMap = array(
// 'Biography', 'FirstName',
// null, // ignored column 'Biography',
// 'Birthday', null, // ignored column
// 'IsRegistered' 'Birthday',
// ); 'IsRegistered'
// $loader->hasHeaderRow = false; );
// $results = $loader->load($filepath); $loader->hasHeaderRow = false;
// $results = $loader->load($filepath);
// // Test that right amount of columns was imported
// $this->assertEquals(4, $results->Count(), 'Test correct count of imported data'); // Test that right amount of columns was imported
// $this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
// // Test that columns were correctly imported
// $obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'"); // Test that columns were correctly imported
// $this->assertNotNull($obj); $obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
// $this->assertEquals("He's a good guy", $obj->Biography); $this->assertNotNull($obj);
// $this->assertEquals("1988-01-31", $obj->Birthday); $this->assertEquals("He's a good guy", $obj->Biography);
// $this->assertEquals("1", $obj->IsRegistered); $this->assertEquals("1988-01-31", $obj->Birthday);
// $this->assertEquals("1", $obj->IsRegistered);
// $obj2 = DataObject::get_one('CsvBulkLoaderTest_Player', "\"FirstName\" = 'Jane'");
// $this->assertNotNull($obj2); $obj2 = DataObject::get_one('CsvBulkLoaderTest_Player', "\"FirstName\" = 'Jane'");
// $this->assertEquals('0', $obj2->IsRegistered); $this->assertNotNull($obj2);
// $this->assertEquals('0', $obj2->IsRegistered);
// fclose($file);
// } fclose($file);
// }
// /**
// * Test import with manual column mapping and custom column names /**
// */ * Test import with manual column mapping and custom column names
// function testLoadWithCustomHeaderAndRelation() { */
// $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player'); function testLoadWithCustomHeaderAndRelation() {
// $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv'; $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
// $file = fopen($filepath, 'r'); $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv';
// $compareCount = $this->getLineCount($file); $file = fopen($filepath, 'r');
// fgetcsv($file); // pop header row $compareCount = $this->getLineCount($file);
// $compareRow = fgetcsv($file); fgetcsv($file); // pop header row
// $loader->columnMap = array( $compareRow = fgetcsv($file);
// 'first name' => 'FirstName', $loader->columnMap = array(
// 'bio' => 'Biography', 'first name' => 'FirstName',
// 'bday' => 'Birthday', 'bio' => 'Biography',
// 'teamtitle' => 'Team.Title', // test existing relation 'bday' => 'Birthday',
// 'teamsize' => 'Team.TeamSize', // test existing relation 'teamtitle' => 'Team.Title', // test existing relation
// 'salary' => 'Contract.Amount' // test relation creation 'teamsize' => 'Team.TeamSize', // test existing relation
// ); 'salary' => 'Contract.Amount' // test relation creation
// $loader->hasHeaderRow = true; );
// $loader->relationCallbacks = array( $loader->hasHeaderRow = true;
// 'Team.Title' => array( $loader->relationCallbacks = array(
// 'relationname' => 'Team', 'Team.Title' => array(
// 'callback' => 'getTeamByTitle' 'relationname' => 'Team',
// ), 'callback' => 'getTeamByTitle'
// // contract should be automatically discovered ),
// ); // contract should be automatically discovered
// $results = $loader->load($filepath); );
// $results = $loader->load($filepath);
// // Test that right amount of columns was imported
// $this->assertEquals(1, $results->Count(), 'Test correct count of imported data'); // Test that right amount of columns was imported
// $this->assertEquals(1, $results->Count(), 'Test correct count of imported data');
// // Test of augumenting existing relation (created by fixture)
// $testTeam = DataObject::get_one('CsvBulkLoaderTest_Team', null, null, '"Created" DESC'); // Test of augumenting existing relation (created by fixture)
// $this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works'); $testTeam = DataObject::get_one('CsvBulkLoaderTest_Team', null, null, '"Created" DESC');
// $this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works');
// // Test of creating relation
// $testContract = DataObject::get_one('CsvBulkLoaderTest_PlayerContract'); // Test of creating relation
// $testPlayer = Dataobject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'"); $testContract = DataObject::get_one('CsvBulkLoaderTest_PlayerContract');
// $this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works'); $testPlayer = Dataobject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
// $this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works');
// // Test nested setting of relation properties
// $contractAmount = DBField::create('Currency', $compareRow[5])->RAW(); // Test nested setting of relation properties
// $this->assertEquals($testPlayer->Contract()->Amount, $contractAmount, 'Setting nested values in a relation works'); $contractAmount = DBField::create('Currency', $compareRow[5])->RAW();
// $this->assertEquals($testPlayer->Contract()->Amount, $contractAmount, 'Setting nested values in a relation works');
// fclose($file);
// } fclose($file);
}
/** /**
* Test import with custom identifiers by importing the data. * Test import with custom identifiers by importing the data.