411 lines
14 KiB
PHP
411 lines
14 KiB
PHP
<?php
|
|
|
|
namespace SilverStripe\Dev\Tests;
|
|
|
|
use League\Csv\Writer;
|
|
use SilverStripe\Control\HTTPResponse_Exception;
|
|
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CustomLoader;
|
|
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\Player;
|
|
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\PlayerContract;
|
|
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\Team;
|
|
use SilverStripe\ORM\DataObject;
|
|
use SilverStripe\ORM\FieldType\DBField;
|
|
use SilverStripe\Core\Config\Config;
|
|
use SilverStripe\Dev\CsvBulkLoader;
|
|
use SilverStripe\Dev\SapphireTest;
|
|
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CanModifyModel;
|
|
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CantCreateModel;
|
|
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CantDeleteModel;
|
|
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CantEditModel;
|
|
|
|
class CsvBulkLoaderTest extends SapphireTest
|
|
{
|
|
|
|
protected static $fixture_file = 'CsvBulkLoaderTest.yml';
|
|
|
|
protected static $extra_dataobjects = [
|
|
Team::class,
|
|
Player::class,
|
|
PlayerContract::class,
|
|
CanModifyModel::class,
|
|
CantCreateModel::class,
|
|
CantEditModel::class,
|
|
CantDeleteModel::class,
|
|
];
|
|
|
|
/**
|
|
* Name of csv test dir
|
|
*
|
|
* @var string
|
|
*/
|
|
protected $csvPath = null;
|
|
|
|
protected function setUp(): void
|
|
{
|
|
parent::setUp();
|
|
$this->csvPath = __DIR__ . '/CsvBulkLoaderTest/csv/';
|
|
}
|
|
|
|
/**
|
|
* Test plain import with column auto-detection
|
|
*/
|
|
public function testLoad()
|
|
{
|
|
$loader = new CsvBulkLoader(Player::class);
|
|
$filepath = $this->csvPath . 'PlayersWithHeader.csv';
|
|
$file = fopen($filepath ?? '', 'r');
|
|
$compareCount = $this->getLineCount($file);
|
|
fgetcsv($file); // pop header row
|
|
$compareRow = fgetcsv($file);
|
|
$results = $loader->load($filepath);
|
|
|
|
// Test that right amount of columns was imported
|
|
$this->assertCount(5, $results, 'Test correct count of imported data');
|
|
|
|
// Test that columns were correctly imported
|
|
$obj = DataObject::get_one(
|
|
Player::class,
|
|
[
|
|
'"CsvBulkLoaderTest_Player"."FirstName"' => 'John'
|
|
]
|
|
);
|
|
$this->assertNotNull($obj);
|
|
$this->assertEquals("He's a good guy", $obj->Biography);
|
|
$this->assertEquals("1988-01-31", $obj->Birthday);
|
|
$this->assertEquals("1", $obj->IsRegistered);
|
|
|
|
fclose($file);
|
|
}
|
|
|
|
/**
|
|
* Test plain import with clear_table_before_import
|
|
*/
|
|
public function testDeleteExistingRecords()
|
|
{
|
|
$loader = new CsvBulkLoader(Player::class);
|
|
$filepath = $this->csvPath . 'PlayersWithHeader.csv';
|
|
$loader->deleteExistingRecords = true;
|
|
$results1 = $loader->load($filepath);
|
|
$this->assertCount(5, $results1, 'Test correct count of imported data on first load');
|
|
|
|
//delete existing data before doing second CSV import
|
|
$results2 = $loader->load($filepath);
|
|
//get all instances of the loaded DataObject from the database and count them
|
|
$resultDataObject = DataObject::get(Player::class);
|
|
|
|
$this->assertCount(
|
|
5,
|
|
$resultDataObject,
|
|
'Test if existing data is deleted before new data is added'
|
|
);
|
|
}
|
|
|
|
public function testLeadingTabs()
|
|
{
|
|
$loader = new CsvBulkLoader(Player::class);
|
|
$loader->hasHeaderRow = false;
|
|
$loader->columnMap = [
|
|
'FirstName',
|
|
'Biography',
|
|
null, // ignored column
|
|
'Birthday',
|
|
'IsRegistered'
|
|
];
|
|
$filepath = $this->csvPath . 'PlayersWithTabs.csv';
|
|
$results = $loader->load($filepath);
|
|
$this->assertCount(5, $results);
|
|
|
|
$expectedBios = [
|
|
"\tHe's a good guy",
|
|
"=She is awesome.\nSo awesome that she gets multiple rows and \"escaped\" strings in her biography",
|
|
"-Pretty old\, with an escaped comma",
|
|
"@Unicode FTW",
|
|
"+Unicode FTW",
|
|
];
|
|
|
|
foreach (Player::get()->column('Biography') as $bio) {
|
|
$this->assertContains($bio, $expectedBios);
|
|
}
|
|
|
|
$this->assertEquals(Player::get()->count(), count($expectedBios ?? []));
|
|
}
|
|
|
|
/**
|
|
* Test import with manual column mapping
|
|
*/
|
|
public function testLoadWithColumnMap()
|
|
{
|
|
$loader = new CsvBulkLoader(Player::class);
|
|
$filepath = $this->csvPath . 'Players.csv';
|
|
$file = fopen($filepath ?? '', 'r');
|
|
$compareCount = $this->getLineCount($file);
|
|
$compareRow = fgetcsv($file);
|
|
$loader->columnMap = [
|
|
'FirstName',
|
|
'Biography',
|
|
null, // ignored column
|
|
'Birthday',
|
|
'IsRegistered'
|
|
];
|
|
$loader->hasHeaderRow = false;
|
|
$results = $loader->load($filepath);
|
|
|
|
// Test that right amount of columns was imported
|
|
$this->assertCount(4, $results, 'Test correct count of imported data');
|
|
|
|
// Test that columns were correctly imported
|
|
$obj = DataObject::get_one(
|
|
Player::class,
|
|
[
|
|
'"CsvBulkLoaderTest_Player"."FirstName"' => 'John'
|
|
]
|
|
);
|
|
$this->assertNotNull($obj);
|
|
$this->assertEquals("He's a good guy", $obj->Biography);
|
|
$this->assertEquals("1988-01-31", $obj->Birthday);
|
|
$this->assertEquals("1", $obj->IsRegistered);
|
|
|
|
$obj2 = DataObject::get_one(
|
|
Player::class,
|
|
[
|
|
'"CsvBulkLoaderTest_Player"."FirstName"' => 'Jane'
|
|
]
|
|
);
|
|
$this->assertNotNull($obj2);
|
|
$this->assertEquals('0', $obj2->IsRegistered);
|
|
|
|
fclose($file);
|
|
}
|
|
|
|
/**
|
|
* Test import with manual column mapping and custom column names
|
|
*/
|
|
public function testLoadWithCustomHeaderAndRelation()
|
|
{
|
|
$loader = new CsvBulkLoader(Player::class);
|
|
$filepath = $this->csvPath . 'PlayersWithCustomHeaderAndRelation.csv';
|
|
$file = fopen($filepath ?? '', 'r');
|
|
$compareCount = $this->getLineCount($file);
|
|
fgetcsv($file); // pop header row
|
|
$compareRow = fgetcsv($file);
|
|
$loader->columnMap = [
|
|
'first name' => 'FirstName',
|
|
'bio' => 'Biography',
|
|
'bday' => 'Birthday',
|
|
'teamtitle' => 'Team.Title', // test existing relation
|
|
'teamsize' => 'Team.TeamSize', // test existing relation
|
|
'salary' => 'Contract.Amount' // test relation creation
|
|
];
|
|
$loader->hasHeaderRow = true;
|
|
$loader->relationCallbacks = [
|
|
'Team.Title' => [
|
|
'relationname' => 'Team',
|
|
'callback' => 'getTeamByTitle'
|
|
],
|
|
// contract should be automatically discovered
|
|
];
|
|
$results = $loader->load($filepath);
|
|
|
|
// Test that right amount of columns was imported
|
|
$this->assertCount(1, $results, 'Test correct count of imported data');
|
|
|
|
// Test of augumenting existing relation (created by fixture)
|
|
$testTeam = DataObject::get_one(Team::class, null, null, '"Created" DESC');
|
|
$this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works');
|
|
|
|
// Test of creating relation
|
|
$testContract = DataObject::get_one(PlayerContract::class);
|
|
$testPlayer = DataObject::get_one(
|
|
Player::class,
|
|
[
|
|
'"CsvBulkLoaderTest_Player"."FirstName"' => 'John'
|
|
]
|
|
);
|
|
$this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works');
|
|
|
|
// Test nested setting of relation properties
|
|
$contractAmount = DBField::create_field('Currency', $compareRow[5])->RAW();
|
|
$this->assertEquals(
|
|
$testPlayer->Contract()->Amount,
|
|
$contractAmount,
|
|
'Setting nested values in a relation works'
|
|
);
|
|
|
|
fclose($file);
|
|
}
|
|
|
|
/**
|
|
* Test import with custom identifiers by importing the data.
|
|
*
|
|
* @todo Test duplicateCheck callbacks
|
|
*/
|
|
public function testLoadWithIdentifiers()
|
|
{
|
|
// first load
|
|
$loader = new CsvBulkLoader(Player::class);
|
|
$filepath = $this->csvPath . 'PlayersWithId.csv';
|
|
$loader->duplicateChecks = [
|
|
'ExternalIdentifier' => 'ExternalIdentifier',
|
|
'NonExistantIdentifier' => 'ExternalIdentifier',
|
|
'AdditionalIdentifier' => 'ExternalIdentifier'
|
|
];
|
|
$results = $loader->load($filepath);
|
|
$createdPlayers = $results->Created();
|
|
|
|
$player = $createdPlayers->first();
|
|
$this->assertEquals($player->FirstName, 'John');
|
|
$this->assertEquals(
|
|
$player->Biography,
|
|
'He\'s a good guy',
|
|
'test updating of duplicate imports within the same import works'
|
|
);
|
|
|
|
// load with updated data
|
|
$filepath = $this->csvPath . 'PlayersWithIdUpdated.csv';
|
|
$results = $loader->load($filepath);
|
|
|
|
// HACK need to update the loaded record from the database
|
|
$player = DataObject::get_by_id(Player::class, $player->ID);
|
|
$this->assertEquals($player->FirstName, 'JohnUpdated', 'Test updating of existing records works');
|
|
|
|
// null values are valid imported
|
|
// $this->assertEquals($player->Biography, 'He\'s a good guy',
|
|
// 'Test retaining of previous information on duplicate when overwriting with blank field');
|
|
}
|
|
|
|
public function testLoadWithCustomImportMethods()
|
|
{
|
|
$loader = new CustomLoader(Player::class);
|
|
$filepath = $this->csvPath . 'PlayersWithHeader.csv';
|
|
$loader->columnMap = [
|
|
'FirstName' => '->importFirstName',
|
|
'Biography' => 'Biography',
|
|
'Birthday' => 'Birthday',
|
|
'IsRegistered' => 'IsRegistered'
|
|
];
|
|
$results = $loader->load($filepath);
|
|
$createdPlayers = $results->Created();
|
|
$player = $createdPlayers->first();
|
|
$this->assertEquals('Customized John', $player->FirstName);
|
|
$this->assertEquals("He's a good guy", $player->Biography);
|
|
$this->assertEquals("1", $player->IsRegistered);
|
|
}
|
|
|
|
public function testLoadWithCustomImportMethodDuplicateMap()
|
|
{
|
|
$loader = new CustomLoader(Player::class);
|
|
$filepath = $this->csvPath . 'PlayersWithHeader.csv';
|
|
$loader->columnMap = [
|
|
'FirstName' => '->updatePlayer',
|
|
'Biography' => '->updatePlayer',
|
|
'Birthday' => 'Birthday',
|
|
'IsRegistered' => 'IsRegistered'
|
|
];
|
|
|
|
$results = $loader->load($filepath);
|
|
|
|
$createdPlayers = $results->Created();
|
|
$player = $createdPlayers->first();
|
|
|
|
$this->assertEquals($player->FirstName, "John. He's a good guy. ");
|
|
}
|
|
|
|
public function testLoadWithByteOrderMark()
|
|
{
|
|
$loader = new CsvBulkLoader(Player::class);
|
|
$loader->load($this->csvPath . 'PlayersWithHeaderAndBOM.csv');
|
|
|
|
$players = Player::get();
|
|
|
|
$this->assertCount(3, $players);
|
|
$this->assertListContains([
|
|
['FirstName' => 'Jamie', 'Birthday' => '1882-01-31'],
|
|
['FirstName' => 'Järg', 'Birthday' => '1982-06-30'],
|
|
['FirstName' => 'Jacob', 'Birthday' => '2000-04-30'],
|
|
], $players);
|
|
}
|
|
|
|
protected function getLineCount(&$file)
|
|
{
|
|
$i = 0;
|
|
while (fgets($file) !== false) {
|
|
$i++;
|
|
}
|
|
rewind($file);
|
|
return $i;
|
|
}
|
|
|
|
public function testLargeFileSplitIntoSmallerFiles()
|
|
{
|
|
Config::modify()->set(CsvBulkLoader::class, 'lines', 3);
|
|
|
|
$loader = new CsvBulkLoader(Player::class);
|
|
$path = $this->csvPath . 'LargeListOfPlayers.csv';
|
|
|
|
$results = $loader->load($path);
|
|
|
|
$this->assertCount(10, $results);
|
|
}
|
|
|
|
/**
|
|
* @dataProvider provideCheckPermissions
|
|
*/
|
|
public function testCheckPermissions(string $class, string $file, bool $respectPerms, string $exceptionMessage)
|
|
{
|
|
$loader = new CsvBulkLoader($class);
|
|
$loader->setCheckPermissions($respectPerms);
|
|
// Don't delete CantEditModel records, 'cause we need to explicitly edit them
|
|
$loader->deleteExistingRecords = $class !== CantEditModel::class;
|
|
// We can't rely on IDs in unit tests, so use Title as the unique field
|
|
$loader->duplicateChecks['Title'] = 'Title';
|
|
|
|
if ($exceptionMessage) {
|
|
$this->expectException(HTTPResponse_Exception::class);
|
|
$this->expectExceptionMessage($exceptionMessage);
|
|
}
|
|
|
|
$results = $loader->load($this->csvPath . $file);
|
|
|
|
// If there's no permission exception, we should get some valid results.
|
|
if (!$exceptionMessage) {
|
|
$this->assertCount(3, $results);
|
|
}
|
|
}
|
|
|
|
public function provideCheckPermissions()
|
|
{
|
|
$scenarios = [
|
|
'Has all permissions' => [
|
|
'class' => CanModifyModel::class,
|
|
'file' => 'PermissionCheck.csv',
|
|
'respectPerms' => true,
|
|
'exceptionMessage' => '',
|
|
],
|
|
'No create permissions' => [
|
|
'class' => CantCreateModel::class,
|
|
'file' => 'PermissionCheck.csv',
|
|
'respectPerms' => true,
|
|
'exceptionMessage' => "Not allowed to create 'Cant Create Model' records",
|
|
],
|
|
'No edit permissions' => [
|
|
'class' => CantEditModel::class,
|
|
'file' => 'PermissionCheck.csv',
|
|
'respectPerms' => true,
|
|
'exceptionMessage' => "Not allowed to edit 'Cant Edit Model' records",
|
|
],
|
|
'No delete permissions' => [
|
|
'class' => CantDeleteModel::class,
|
|
'file' => 'PermissionCheck.csv',
|
|
'respectPerms' => true,
|
|
'exceptionMessage' => "Not allowed to delete 'Cant Delete Model' records",
|
|
],
|
|
];
|
|
foreach ($scenarios as $name => $scenario) {
|
|
$scenario['respectPerms'] = false;
|
|
$scenario['exceptionMessage'] = '';
|
|
$scenarios[$name . ' but no perm checks'] = $scenario;
|
|
}
|
|
return $scenarios;
|
|
}
|
|
}
|