silverstripe-framework/tests/php/Dev/CsvBulkLoaderTest.php

<?php

namespace SilverStripe\Dev\Tests;

use League\Csv\Writer;
use SilverStripe\Control\HTTPResponse_Exception;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CustomLoader;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\Player;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\PlayerContract;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\Team;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\FieldType\DBField;
use SilverStripe\Core\Config\Config;
use SilverStripe\Dev\CsvBulkLoader;
use SilverStripe\Dev\SapphireTest;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CanModifyModel;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CantCreateModel;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CantDeleteModel;
use SilverStripe\Dev\Tests\CsvBulkLoaderTest\CantEditModel;

class CsvBulkLoaderTest extends SapphireTest
{

    protected static $fixture_file = 'CsvBulkLoaderTest.yml';

    protected static $extra_dataobjects = [
        Team::class,
        Player::class,
        PlayerContract::class,
        CanModifyModel::class,
        CantCreateModel::class,
        CantEditModel::class,
        CantDeleteModel::class,
    ];

    /**
     * Name of csv test dir
     *
     * @var string
     */
    protected $csvPath = null;

    protected function setUp(): void
    {
        parent::setUp();
        $this->csvPath = __DIR__ . '/CsvBulkLoaderTest/csv/';
    }

    /**
     * Test plain import with column auto-detection
     */
    public function testLoad()
    {
        $loader = new CsvBulkLoader(Player::class);
        $filepath = $this->csvPath . 'PlayersWithHeader.csv';
        $file = fopen($filepath ?? '', 'r');
        $compareCount = $this->getLineCount($file);
        fgetcsv($file); // pop header row
        $compareRow = fgetcsv($file);
        $results = $loader->load($filepath);

        // Test that right amount of columns was imported
        $this->assertCount(5, $results, 'Test correct count of imported data');

        // Test that columns were correctly imported
        $obj = DataObject::get_one(
            Player::class,
            [
            '"CsvBulkLoaderTest_Player"."FirstName"' => 'John'
            ]
        );
        $this->assertNotNull($obj);
        $this->assertEquals("He's a good guy", $obj->Biography);
        $this->assertEquals("1988-01-31", $obj->Birthday);
        $this->assertEquals("1", $obj->IsRegistered);

        fclose($file);
    }

    /**
     * Test plain import with clear_table_before_import
         */
    public function testDeleteExistingRecords()
    {
        $loader = new CsvBulkLoader(Player::class);
        $filepath = $this->csvPath . 'PlayersWithHeader.csv';
        $loader->deleteExistingRecords = true;
        $results1 = $loader->load($filepath);
        $this->assertCount(5, $results1, 'Test correct count of imported data on first load');

        //delete existing data before doing second CSV import
        $results2 = $loader->load($filepath);
        //get all instances of the loaded DataObject from the database and count them
        $resultDataObject = DataObject::get(Player::class);

        $this->assertCount(
            5,
            $resultDataObject,
            'Test if existing data is deleted before new data is added'
        );
    }

    public function testLeadingTabs()
    {
        $loader = new CsvBulkLoader(Player::class);
        $loader->hasHeaderRow = false;
        $loader->columnMap = [
            'FirstName',
            'Biography',
            null, // ignored column
            'Birthday',
            'IsRegistered'
        ];
        $filepath = $this->csvPath . 'PlayersWithTabs.csv';
        $results = $loader->load($filepath);
        $this->assertCount(5, $results);

        $expectedBios = [
            "\tHe's a good guy",
            "=She is awesome.\nSo awesome that she gets multiple rows and \"escaped\" strings in her biography",
            "-Pretty old\, with an escaped comma",
            "@Unicode FTW",
            "+Unicode FTW",
        ];

        foreach (Player::get()->column('Biography') as $bio) {
            $this->assertContains($bio, $expectedBios);
        }

        $this->assertEquals(Player::get()->count(), count($expectedBios ?? []));
    }

    /**
     * Test import with manual column mapping
     */
    public function testLoadWithColumnMap()
    {
        $loader = new CsvBulkLoader(Player::class);
        $filepath = $this->csvPath . 'Players.csv';
        $file = fopen($filepath ?? '', 'r');
        $compareCount = $this->getLineCount($file);
        $compareRow = fgetcsv($file);
        $loader->columnMap = [
            'FirstName',
            'Biography',
            null, // ignored column
            'Birthday',
            'IsRegistered'
        ];
        $loader->hasHeaderRow = false;
        $results = $loader->load($filepath);

        // Test that right amount of columns was imported
        $this->assertCount(4, $results, 'Test correct count of imported data');

        // Test that columns were correctly imported
        $obj = DataObject::get_one(
            Player::class,
            [
            '"CsvBulkLoaderTest_Player"."FirstName"' => 'John'
            ]
        );
        $this->assertNotNull($obj);
        $this->assertEquals("He's a good guy", $obj->Biography);
        $this->assertEquals("1988-01-31", $obj->Birthday);
        $this->assertEquals("1", $obj->IsRegistered);

        $obj2 = DataObject::get_one(
            Player::class,
            [
            '"CsvBulkLoaderTest_Player"."FirstName"' => 'Jane'
            ]
        );
        $this->assertNotNull($obj2);
        $this->assertEquals('0', $obj2->IsRegistered);

        fclose($file);
    }

    /**
     * Test import with manual column mapping and custom column names
     */
    public function testLoadWithCustomHeaderAndRelation()
    {
        $loader = new CsvBulkLoader(Player::class);
        $filepath = $this->csvPath . 'PlayersWithCustomHeaderAndRelation.csv';
        $file = fopen($filepath ?? '', 'r');
        $compareCount = $this->getLineCount($file);
        fgetcsv($file); // pop header row
        $compareRow = fgetcsv($file);
        $loader->columnMap = [
            'first name' => 'FirstName',
            'bio' => 'Biography',
            'bday' => 'Birthday',
            'teamtitle' => 'Team.Title', // test existing relation
            'teamsize' => 'Team.TeamSize', // test existing relation
            'salary' => 'Contract.Amount' // test relation creation
        ];
        $loader->hasHeaderRow = true;
        $loader->relationCallbacks = [
            'Team.Title' => [
                'relationname' => 'Team',
                'callback' => 'getTeamByTitle'
            ],
            // contract should be automatically discovered
        ];
        $results = $loader->load($filepath);

        // Test that right amount of columns was imported
        $this->assertCount(1, $results, 'Test correct count of imported data');

        // Test of augumenting existing relation (created by fixture)
        $testTeam = DataObject::get_one(Team::class, null, null, '"Created" DESC');
        $this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works');

        // Test of creating relation
        $testContract = DataObject::get_one(PlayerContract::class);
        $testPlayer = DataObject::get_one(
            Player::class,
            [
            '"CsvBulkLoaderTest_Player"."FirstName"' => 'John'
            ]
        );
        $this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works');

        // Test nested setting of relation properties
        $contractAmount = DBField::create_field('Currency', $compareRow[5])->RAW();
        $this->assertEquals(
            $testPlayer->Contract()->Amount,
            $contractAmount,
            'Setting nested values in a relation works'
        );

        fclose($file);
    }

    /**
     * Test import with custom identifiers by importing the data.
     */
    public function testLoadWithIdentifiers()
    {
        // first load
        $loader = new CsvBulkLoader(Player::class);
        $filepath = $this->csvPath . 'PlayersWithId.csv';
        $loader->duplicateChecks = [
            'ExternalIdentifier' => 'ExternalIdentifier',
            'NonExistantIdentifier' => 'ExternalIdentifier',
            'AdditionalIdentifier' => 'ExternalIdentifier'
        ];
        $results = $loader->load($filepath);
        $createdPlayers = $results->Created();

        $player = $createdPlayers->first();
        $this->assertEquals($player->FirstName, 'John');
        $this->assertEquals(
            $player->Biography,
            'He\'s a good guy',
            'test updating of duplicate imports within the same import works'
        );

        // load with updated data
        $filepath = $this->csvPath . 'PlayersWithIdUpdated.csv';
        $results = $loader->load($filepath);

        // HACK need to update the loaded record from the database
        $player = DataObject::get_by_id(Player::class, $player->ID);
        $this->assertEquals($player->FirstName, 'JohnUpdated', 'Test updating of existing records works');

        // null values are valid imported
        // $this->assertEquals($player->Biography, 'He\'s a good guy',
        //  'Test retaining of previous information on duplicate when overwriting with blank field');
    }

    public function testLoadWithCustomImportMethods()
    {
        $loader = new CustomLoader(Player::class);
        $filepath = $this->csvPath . 'PlayersWithHeader.csv';
        $loader->columnMap = [
            'FirstName' => '->importFirstName',
            'Biography' => 'Biography',
            'Birthday' => 'Birthday',
            'IsRegistered' => 'IsRegistered'
        ];
        $results = $loader->load($filepath);
        $createdPlayers = $results->Created();
        $player = $createdPlayers->first();
        $this->assertEquals('Customized John', $player->FirstName);
        $this->assertEquals("He's a good guy", $player->Biography);
        $this->assertEquals("1", $player->IsRegistered);
    }

    public function testLoadWithCustomImportMethodDuplicateMap()
    {
        $loader = new CustomLoader(Player::class);
        $filepath = $this->csvPath . 'PlayersWithHeader.csv';
        $loader->columnMap = [
            'FirstName' => '->updatePlayer',
            'Biography' => '->updatePlayer',
            'Birthday' => 'Birthday',
            'IsRegistered' => 'IsRegistered'
        ];

        $results = $loader->load($filepath);

        $createdPlayers = $results->Created();
        $player = $createdPlayers->first();

        $this->assertEquals($player->FirstName, "John. He's a good guy. ");
    }

    public function testLoadWithByteOrderMark()
    {
        $loader = new CsvBulkLoader(Player::class);
        $loader->load($this->csvPath . 'PlayersWithHeaderAndBOM.csv');

        $players = Player::get();

        $this->assertCount(3, $players);
        $this->assertListContains([
            ['FirstName' => 'Jamie', 'Birthday' => '1882-01-31'],
            ['FirstName' => 'Järg', 'Birthday' => '1982-06-30'],
            ['FirstName' => 'Jacob', 'Birthday' => '2000-04-30'],
        ], $players);
    }

    protected function getLineCount(&$file)
    {
        $i = 0;
        while (fgets($file) !== false) {
            $i++;
        }
        rewind($file);
        return $i;
    }

    public function testLargeFileSplitIntoSmallerFiles()
    {
        Config::modify()->set(CsvBulkLoader::class, 'lines', 3);

        $loader = new CsvBulkLoader(Player::class);
        $path = $this->csvPath . 'LargeListOfPlayers.csv';

        $results = $loader->load($path);

        $this->assertCount(10, $results);
    }

    /**
     * @dataProvider provideCheckPermissions
     */
    public function testCheckPermissions(string $class, string $file, bool $respectPerms, string $exceptionMessage)
    {
        $loader = new CsvBulkLoader($class);
        $loader->setCheckPermissions($respectPerms);
        // Don't delete CantEditModel records, 'cause we need to explicitly edit them
        $loader->deleteExistingRecords = $class !== CantEditModel::class;
        // We can't rely on IDs in unit tests, so use Title as the unique field
        $loader->duplicateChecks['Title'] = 'Title';

        if ($exceptionMessage) {
            $this->expectException(HTTPResponse_Exception::class);
            $this->expectExceptionMessage($exceptionMessage);
        }

        $results = $loader->load($this->csvPath . $file);

        // If there's no permission exception, we should get some valid results.
        if (!$exceptionMessage) {
            $this->assertCount(3, $results);
        }
    }

    public function provideCheckPermissions()
    {
        $scenarios = [
            'Has all permissions' => [
                'class' => CanModifyModel::class,
                'file' => 'PermissionCheck.csv',
                'respectPerms' => true,
                'exceptionMessage' => '',
            ],
            'No create permissions' => [
                'class' => CantCreateModel::class,
                'file' => 'PermissionCheck.csv',
                'respectPerms' => true,
                'exceptionMessage' => "Not allowed to create 'Cant Create Model' records",
            ],
            'No edit permissions' => [
                'class' => CantEditModel::class,
                'file' => 'PermissionCheck.csv',
                'respectPerms' => true,
                'exceptionMessage' => "Not allowed to edit 'Cant Edit Model' records",
            ],
            'No delete permissions' => [
                'class' => CantDeleteModel::class,
                'file' => 'PermissionCheck.csv',
                'respectPerms' => true,
                'exceptionMessage' => "Not allowed to delete 'Cant Delete Model' records",
            ],
        ];
        foreach ($scenarios as $name => $scenario) {
            $scenario['respectPerms'] = false;
            $scenario['exceptionMessage'] = '';
            $scenarios[$name . ' but no perm checks'] = $scenario;
        }
        return $scenarios;
    }
}