silverstripe-framework/tests/php/Dev/CsvBulkLoaderTest.php

341 lines
11 KiB
PHP
Raw Normal View History

<?php
2016-10-14 03:30:05 +02:00
namespace SilverStripe\Dev\Tests;
use League\Csv\Writer;
2016-10-14 03:30:05 +02:00
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CustomLoader;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\Player;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\PlayerContract;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\Team;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\FieldType\DBField;
use SilverStripe\Core\Config\Config;
use SilverStripe\Dev\CsvBulkLoader;
use SilverStripe\Dev\SapphireTest;
class CsvBulkLoaderTest extends SapphireTest
{
protected static $fixture_file = 'CsvBulkLoaderTest.yml';
protected static $extra_dataobjects = [
Team::class,
Player::class,
PlayerContract::class,
];
/**
* Name of csv test dir
*
* @var string
*/
protected $csvPath = null;
protected function setUp(): void
{
parent::setUp();
$this->csvPath = __DIR__ . '/CsvBulkLoaderTest/csv/';
}
/**
* Test plain import with column auto-detection
*/
public function testLoad()
{
$loader = new CsvBulkLoader(Player::class);
$filepath = $this->csvPath . 'PlayersWithHeader.csv';
2022-04-14 03:12:59 +02:00
$file = fopen($filepath ?? '', 'r');
$compareCount = $this->getLineCount($file);
fgetcsv($file); // pop header row
$compareRow = fgetcsv($file);
$results = $loader->load($filepath);
// Test that right amount of columns was imported
$this->assertCount(5, $results, 'Test correct count of imported data');
// Test that columns were correctly imported
$obj = DataObject::get_one(
Player::class,
[
'"CsvBulkLoaderTest_Player"."FirstName"' => 'John'
]
);
$this->assertNotNull($obj);
$this->assertEquals("He's a good guy", $obj->Biography);
$this->assertEquals("1988-01-31", $obj->Birthday);
$this->assertEquals("1", $obj->IsRegistered);
fclose($file);
}
/**
* Test plain import with clear_table_before_import
*/
public function testDeleteExistingRecords()
{
$loader = new CsvBulkLoader(Player::class);
$filepath = $this->csvPath . 'PlayersWithHeader.csv';
$loader->deleteExistingRecords = true;
$results1 = $loader->load($filepath);
$this->assertCount(5, $results1, 'Test correct count of imported data on first load');
//delete existing data before doing second CSV import
$results2 = $loader->load($filepath);
//get all instances of the loaded DataObject from the database and count them
$resultDataObject = DataObject::get(Player::class);
$this->assertCount(
5,
$resultDataObject,
'Test if existing data is deleted before new data is added'
);
}
public function testLeadingTabs()
{
$loader = new CsvBulkLoader(Player::class);
$loader->hasHeaderRow = false;
$loader->columnMap = [
'FirstName',
'Biography',
null, // ignored column
'Birthday',
'IsRegistered'
];
$filepath = $this->csvPath . 'PlayersWithTabs.csv';
$results = $loader->load($filepath);
$this->assertCount(5, $results);
$expectedBios = [
"\tHe's a good guy",
"=She is awesome.\nSo awesome that she gets multiple rows and \"escaped\" strings in her biography",
"-Pretty old\, with an escaped comma",
"@Unicode FTW",
"+Unicode FTW",
];
foreach (Player::get()->column('Biography') as $bio) {
$this->assertContains($bio, $expectedBios);
}
2022-04-14 03:12:59 +02:00
$this->assertEquals(Player::get()->count(), count($expectedBios ?? []));
}
/**
* Test import with manual column mapping
*/
public function testLoadWithColumnMap()
{
$loader = new CsvBulkLoader(Player::class);
$filepath = $this->csvPath . 'Players.csv';
2022-04-14 03:12:59 +02:00
$file = fopen($filepath ?? '', 'r');
$compareCount = $this->getLineCount($file);
$compareRow = fgetcsv($file);
$loader->columnMap = [
'FirstName',
'Biography',
null, // ignored column
'Birthday',
'IsRegistered'
];
$loader->hasHeaderRow = false;
$results = $loader->load($filepath);
// Test that right amount of columns was imported
$this->assertCount(4, $results, 'Test correct count of imported data');
// Test that columns were correctly imported
$obj = DataObject::get_one(
Player::class,
[
'"CsvBulkLoaderTest_Player"."FirstName"' => 'John'
]
);
$this->assertNotNull($obj);
$this->assertEquals("He's a good guy", $obj->Biography);
$this->assertEquals("1988-01-31", $obj->Birthday);
$this->assertEquals("1", $obj->IsRegistered);
$obj2 = DataObject::get_one(
Player::class,
[
'"CsvBulkLoaderTest_Player"."FirstName"' => 'Jane'
]
);
$this->assertNotNull($obj2);
$this->assertEquals('0', $obj2->IsRegistered);
fclose($file);
}
/**
* Test import with manual column mapping and custom column names
*/
public function testLoadWithCustomHeaderAndRelation()
{
$loader = new CsvBulkLoader(Player::class);
$filepath = $this->csvPath . 'PlayersWithCustomHeaderAndRelation.csv';
2022-04-14 03:12:59 +02:00
$file = fopen($filepath ?? '', 'r');
$compareCount = $this->getLineCount($file);
fgetcsv($file); // pop header row
$compareRow = fgetcsv($file);
$loader->columnMap = [
'first name' => 'FirstName',
'bio' => 'Biography',
'bday' => 'Birthday',
'teamtitle' => 'Team.Title', // test existing relation
'teamsize' => 'Team.TeamSize', // test existing relation
'salary' => 'Contract.Amount' // test relation creation
];
$loader->hasHeaderRow = true;
$loader->relationCallbacks = [
'Team.Title' => [
'relationname' => 'Team',
'callback' => 'getTeamByTitle'
],
// contract should be automatically discovered
];
$results = $loader->load($filepath);
// Test that right amount of columns was imported
$this->assertCount(1, $results, 'Test correct count of imported data');
// Test of augumenting existing relation (created by fixture)
$testTeam = DataObject::get_one(Team::class, null, null, '"Created" DESC');
$this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works');
// Test of creating relation
$testContract = DataObject::get_one(PlayerContract::class);
$testPlayer = DataObject::get_one(
Player::class,
[
'"CsvBulkLoaderTest_Player"."FirstName"' => 'John'
]
);
$this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works');
// Test nested setting of relation properties
$contractAmount = DBField::create_field('Currency', $compareRow[5])->RAW();
$this->assertEquals(
$testPlayer->Contract()->Amount,
$contractAmount,
'Setting nested values in a relation works'
);
fclose($file);
}
/**
* Test import with custom identifiers by importing the data.
*
* @todo Test duplicateCheck callbacks
*/
public function testLoadWithIdentifiers()
{
// first load
$loader = new CsvBulkLoader(Player::class);
$filepath = $this->csvPath . 'PlayersWithId.csv';
$loader->duplicateChecks = [
'ExternalIdentifier' => 'ExternalIdentifier',
'NonExistantIdentifier' => 'ExternalIdentifier',
'AdditionalIdentifier' => 'ExternalIdentifier'
];
$results = $loader->load($filepath);
$createdPlayers = $results->Created();
$player = $createdPlayers->first();
$this->assertEquals($player->FirstName, 'John');
$this->assertEquals(
$player->Biography,
'He\'s a good guy',
'test updating of duplicate imports within the same import works'
);
// load with updated data
$filepath = $this->csvPath . 'PlayersWithIdUpdated.csv';
$results = $loader->load($filepath);
// HACK need to update the loaded record from the database
$player = DataObject::get_by_id(Player::class, $player->ID);
$this->assertEquals($player->FirstName, 'JohnUpdated', 'Test updating of existing records works');
// null values are valid imported
// $this->assertEquals($player->Biography, 'He\'s a good guy',
2017-12-14 01:50:52 +01:00
// 'Test retaining of previous information on duplicate when overwriting with blank field');
}
public function testLoadWithCustomImportMethods()
{
$loader = new CustomLoader(Player::class);
$filepath = $this->csvPath . 'PlayersWithHeader.csv';
$loader->columnMap = [
'FirstName' => '->importFirstName',
'Biography' => 'Biography',
'Birthday' => 'Birthday',
'IsRegistered' => 'IsRegistered'
];
$results = $loader->load($filepath);
$createdPlayers = $results->Created();
$player = $createdPlayers->first();
$this->assertEquals('Customized John', $player->FirstName);
$this->assertEquals("He's a good guy", $player->Biography);
$this->assertEquals("1", $player->IsRegistered);
}
public function testLoadWithCustomImportMethodDuplicateMap()
{
$loader = new CustomLoader(Player::class);
$filepath = $this->csvPath . 'PlayersWithHeader.csv';
$loader->columnMap = [
'FirstName' => '->updatePlayer',
'Biography' => '->updatePlayer',
'Birthday' => 'Birthday',
'IsRegistered' => 'IsRegistered'
];
$results = $loader->load($filepath);
$createdPlayers = $results->Created();
$player = $createdPlayers->first();
$this->assertEquals($player->FirstName, "John. He's a good guy. ");
}
public function testLoadWithByteOrderMark()
{
$loader = new CsvBulkLoader(Player::class);
$loader->load($this->csvPath . 'PlayersWithHeaderAndBOM.csv');
$players = Player::get();
$this->assertCount(3, $players);
$this->assertListContains([
['FirstName' => 'Jamie', 'Birthday' => '1882-01-31'],
['FirstName' => 'Järg', 'Birthday' => '1982-06-30'],
['FirstName' => 'Jacob', 'Birthday' => '2000-04-30'],
], $players);
}
protected function getLineCount(&$file)
{
$i = 0;
while (fgets($file) !== false) {
$i++;
}
rewind($file);
return $i;
}
public function testLargeFileSplitIntoSmallerFiles()
{
Config::modify()->set(CsvBulkLoader::class, 'lines', 3);
$loader = new CsvBulkLoader(Player::class);
$path = $this->csvPath . 'LargeListOfPlayers.csv';
$results = $loader->load($path);
$this->assertCount(10, $results);
}
}