silverstripe-framework/src/Dev/BulkLoader.php

268 lines
8.4 KiB
PHP
Raw Normal View History

<?php
namespace SilverStripe\Dev;
API Refactor bootstrap, request handling See https://github.com/silverstripe/silverstripe-framework/pull/7037 and https://github.com/silverstripe/silverstripe-framework/issues/6681 Squashed commit of the following: commit 8f65e5653211240650eaa4fa65bb83b45aae6d58 Author: Ingo Schommer <me@chillu.com> Date: Thu Jun 22 22:25:50 2017 +1200 Fixed upgrade guide spelling commit 76f95944fa89b0b540704b8d744329f690f9698c Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 16:38:34 2017 +1200 BUG Fix non-test class manifest including sapphiretest / functionaltest commit 9379834cb4b2e5177a2600049feec05bf111c16b Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 15:50:47 2017 +1200 BUG Fix nesting bug in Kernel commit 188ce35d82599360c40f0f2de29579c56fb90761 Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 15:14:51 2017 +1200 BUG fix db bootstrapping issues commit 7ed4660e7a63915e8e974deeaba9807bc4d38b0d Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 14:49:07 2017 +1200 BUG Fix issue in DetailedErrorFormatter commit 738f50c497166f81ccbe3f40fbcff895ce71f82f Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 11:49:19 2017 +1200 Upgrading notes on mysite/_config.php commit 6279d28e5e455916f902a2f963c014d8899f7fc7 Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 11:43:28 2017 +1200 Update developer documentation commit 5c90d53a84ef0139c729396949a7857fae60436f Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 10:48:44 2017 +1200 Update installer to not use global databaseConfig commit f9b2ba4755371f08bd95f6908ac612fcbb7ca205 Author: Damian Mooyman <damian@silverstripe.com> Date: Wed Jun 21 21:04:39 2017 +1200 Fix behat issues commit 5b59a912b60282b4dad4ef10ed3b97c5d0a761ac Author: Damian Mooyman <damian@silverstripe.com> Date: Wed Jun 21 17:07:11 2017 +1200 Move HTTPApplication to SilverStripe\Control namespace commit e2c4a18f637bdd3d276619554de60ee8b4d95ced Author: Damian Mooyman <damian@silverstripe.com> Date: Wed Jun 21 16:29:03 2017 +1200 More documentation Fix up remaining tests Refactor temp DB into TempDatabase class so it’s available outside of unit tests. commit 5d235e64f341d6251bfe9f4833f15cc8593c5034 Author: Damian Mooyman <damian@silverstripe.com> Date: Wed Jun 21 12:13:15 2017 +1200 API HTTPRequestBuilder::createFromEnvironment() now cleans up live globals BUG Fix issue with SSViewer Fix Security / View tests commit d88d4ed4e48291cb65407f222f190064b1f1deeb Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 20 16:39:43 2017 +1200 API Refactor AppKernel into CoreKernel commit f7946aec3391139ae1b4029c353c327a36552b36 Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 20 16:00:40 2017 +1200 Docs and minor cleanup commit 12bd31f9366327650b5c0c0f96cd0327d44faf0a Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 20 15:34:34 2017 +1200 API Remove OutputMiddleware API Move environment / global / ini management into Environment class API Move getTempFolder into TempFolder class API Implement HTTPRequestBuilder / CLIRequestBuilder BUG Restore SS_ALLOWED_HOSTS check in original location API CoreKernel now requires $basePath to be passed in API Refactor installer.php to use application to bootstrap API move memstring conversion globals to Convert BUG Fix error in CoreKernel nesting not un-nesting itself properly. commit bba979114624247cf463cf2a8c9e4be9a7c3a772 Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 19 18:07:53 2017 +1200 API Create HTTPMiddleware and standardise middleware for request handling commit 2a10c2397bdc53001013f607b5d38087ce6c0730 Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 19 17:42:42 2017 +1200 Fixed ORM tests commit d75a8d1d93398af4bd0432df9e4bc6295c15a3fe Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 19 17:15:07 2017 +1200 FIx i18n tests commit 06364af3c379c931889c4cc34dd920fee3db204a Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 19 16:59:34 2017 +1200 Fix controller namespace Move states to sub namespace commit 2a278e2953d2dbb19f78d91c919048e1fc935436 Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 19 12:49:45 2017 +1200 Fix forms namespace commit b65c21241bee019730027071d815dbf7571197a4 Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 15 18:56:48 2017 +1200 Update API usages commit d1d4375c95a264a6b63cbaefc2c1d12f808bfd82 Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 15 18:41:44 2017 +1200 API Refactor $flush into HTPPApplication API Enforce health check in Controller::pushCurrent() API Better global backup / restore Updated Director::test() to use new API commit b220534f06732db4fa940d8724c2a85c0ba2495a Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 22:05:57 2017 +1200 Move app nesting to a test state helper commit 603704165c08d0c1c81fd5e6bb9506326eeee17b Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 21:46:04 2017 +1200 Restore kernel stack to fix multi-level nesting commit 2f6336a15bf79dc8c2edd44cec1931da2dd51c28 Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 17:23:21 2017 +1200 API Implement kernel nesting commit fc7188da7d6ad6785354bab61f08700454c81d91 Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 15:43:13 2017 +1200 Fix core tests commit a0ae7235148fffd71f2f02d1fe7fe45bf3aa39eb Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 15:23:52 2017 +1200 Fix manifest tests commit ca033952513633e182040d3d13e1caa9000ca184 Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 15:00:00 2017 +1200 API Move extension management into test state commit c66d4339777663a8a04661fea32a0cf35b95d20f Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 14:10:59 2017 +1200 API Refactor SapphireTest state management into SapphireTestState API Remove Injector::unregisterAllObjects() API Remove FakeController commit f26ae75c6ecaafa0dec1093264e0187191e6764d Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 12 18:04:34 2017 +1200 Implement basic CLI application object commit 001d5596621404892de0a5413392379eff990641 Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 12 17:39:38 2017 +1200 Remove references to SapphireTest::is_running_test() Upgrade various code commit de079c041dacd96bc4f4b66421fa2b2cc4c320f8 Author: Damian Mooyman <damian@silverstripe.com> Date: Wed Jun 7 18:07:33 2017 +1200 API Implement APP object API Refactor of Session
2017-06-22 12:50:45 +02:00
use SilverStripe\Core\Environment;
use SilverStripe\ORM\DataObject;
use SilverStripe\View\ViewableData;
/**
* A base for bulk loaders of content into the SilverStripe database.
* Bulk loaders give SilverStripe authors the ability to do large-scale uploads into their SilverStripe databases.
2014-08-15 08:53:05 +02:00
*
* You can configure column-handling,
*
* @todo Add support for adding/editing has_many relations.
* @todo Add support for deep chaining of relation properties (e.g. Player.Team.Stats.GoalCount)
* @todo Character conversion
2014-08-15 08:53:05 +02:00
*
* @see http://tools.ietf.org/html/rfc4180
* @author Ingo Schommer, Silverstripe Ltd. (<firstname>@silverstripe.com)
*/
2016-11-29 00:31:16 +01:00
abstract class BulkLoader extends ViewableData
{
/**
* Each row in the imported dataset should map to one instance
* of this class (with optional property translation
* through {@self::$columnMaps}.
*
* @var string
*/
public $objectClass;
/**
* Override this on subclasses to give the specific functions names.
*
* @var string
*/
public static $title;
/**
* Map columns to DataObject-properties.
* If not specified, we assume the first row
* in the file contains the column headers.
* The order of your array should match the column order.
*
* The column count should match the count of array elements,
* fill with NULL values if you want to skip certain columns.
*
* You can also combine {@link $hasHeaderRow} = true and {@link $columnMap}
* and omit the NULL values in your map.
*
* Supports one-level chaining of has_one relations and properties with dot notation
* (e.g. Team.Title). The first part has to match a has_one relation name
* (not necessarily the classname of the used relation).
*
* <code>
* <?php
* // simple example
* array(
* 'Title',
* 'Birthday'
* )
*
* // complex example
* array(
* 'first name' => 'FirstName', // custom column name
* null, // ignored column
* 'RegionID', // direct has_one/has_many ID setting
* 'OrganisationTitle', // create has_one relation to existing record using $relationCallbacks
* 'street' => 'Organisation.StreetName', // match an existing has_one or create one and write property.
* );
* ?>
* </code>
*
* @var array
*/
public $columnMap = array();
/**
* Find a has_one relation based on a specific column value.
*
* <code>
* <?php
* array(
* 'OrganisationTitle' => array(
* 'relationname' => 'Organisation', // relation accessor name
* 'callback' => 'getOrganisationByTitle',
* );
* );
* ?>
* </code>
*
* @var array
*/
public $relationCallbacks = array();
/**
* Specifies how to determine duplicates based on one or more provided fields
* in the imported data, matching to properties on the used {@link DataObject} class.
* Alternatively the array values can contain a callback method (see example for
* implementation details). The callback method should be defined on the source class.
*
* NOTE: If you're trying to get a unique Member record by a particular field that
* isn't Email, you need to ensure that Member is correctly set to the unique field
* you want, as it will merge any duplicates during {@link Member::onBeforeWrite()}.
*
* {@see Member::$unique_identifier_field}.
*
* If multiple checks are specified, the first non-empty field "wins".
*
* <code>
* <?php
* array(
* 'customernumber' => 'ID',
* 'phonenumber' => array(
* 'callback' => 'getByImportedPhoneNumber'
* )
* );
* ?>
* </code>
*
* @var array
*/
public $duplicateChecks = array();
/**
* @var Boolean $clearBeforeImport Delete ALL records before importing.
*/
public $deleteExistingRecords = false;
public function __construct($objectClass)
{
$this->objectClass = $objectClass;
parent::__construct();
}
/*
2017-12-14 02:18:41 +01:00
* Load the given file via {@link self::processAll()} and {@link self::processRecord()}.
* Optionally truncates (clear) the table before it imports.
*
* @return BulkLoader_Result See {@link self::processAll()}
*/
2016-11-29 00:31:16 +01:00
public function load($filepath)
{
API Refactor bootstrap, request handling See https://github.com/silverstripe/silverstripe-framework/pull/7037 and https://github.com/silverstripe/silverstripe-framework/issues/6681 Squashed commit of the following: commit 8f65e5653211240650eaa4fa65bb83b45aae6d58 Author: Ingo Schommer <me@chillu.com> Date: Thu Jun 22 22:25:50 2017 +1200 Fixed upgrade guide spelling commit 76f95944fa89b0b540704b8d744329f690f9698c Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 16:38:34 2017 +1200 BUG Fix non-test class manifest including sapphiretest / functionaltest commit 9379834cb4b2e5177a2600049feec05bf111c16b Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 15:50:47 2017 +1200 BUG Fix nesting bug in Kernel commit 188ce35d82599360c40f0f2de29579c56fb90761 Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 15:14:51 2017 +1200 BUG fix db bootstrapping issues commit 7ed4660e7a63915e8e974deeaba9807bc4d38b0d Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 14:49:07 2017 +1200 BUG Fix issue in DetailedErrorFormatter commit 738f50c497166f81ccbe3f40fbcff895ce71f82f Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 11:49:19 2017 +1200 Upgrading notes on mysite/_config.php commit 6279d28e5e455916f902a2f963c014d8899f7fc7 Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 11:43:28 2017 +1200 Update developer documentation commit 5c90d53a84ef0139c729396949a7857fae60436f Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 22 10:48:44 2017 +1200 Update installer to not use global databaseConfig commit f9b2ba4755371f08bd95f6908ac612fcbb7ca205 Author: Damian Mooyman <damian@silverstripe.com> Date: Wed Jun 21 21:04:39 2017 +1200 Fix behat issues commit 5b59a912b60282b4dad4ef10ed3b97c5d0a761ac Author: Damian Mooyman <damian@silverstripe.com> Date: Wed Jun 21 17:07:11 2017 +1200 Move HTTPApplication to SilverStripe\Control namespace commit e2c4a18f637bdd3d276619554de60ee8b4d95ced Author: Damian Mooyman <damian@silverstripe.com> Date: Wed Jun 21 16:29:03 2017 +1200 More documentation Fix up remaining tests Refactor temp DB into TempDatabase class so it’s available outside of unit tests. commit 5d235e64f341d6251bfe9f4833f15cc8593c5034 Author: Damian Mooyman <damian@silverstripe.com> Date: Wed Jun 21 12:13:15 2017 +1200 API HTTPRequestBuilder::createFromEnvironment() now cleans up live globals BUG Fix issue with SSViewer Fix Security / View tests commit d88d4ed4e48291cb65407f222f190064b1f1deeb Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 20 16:39:43 2017 +1200 API Refactor AppKernel into CoreKernel commit f7946aec3391139ae1b4029c353c327a36552b36 Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 20 16:00:40 2017 +1200 Docs and minor cleanup commit 12bd31f9366327650b5c0c0f96cd0327d44faf0a Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 20 15:34:34 2017 +1200 API Remove OutputMiddleware API Move environment / global / ini management into Environment class API Move getTempFolder into TempFolder class API Implement HTTPRequestBuilder / CLIRequestBuilder BUG Restore SS_ALLOWED_HOSTS check in original location API CoreKernel now requires $basePath to be passed in API Refactor installer.php to use application to bootstrap API move memstring conversion globals to Convert BUG Fix error in CoreKernel nesting not un-nesting itself properly. commit bba979114624247cf463cf2a8c9e4be9a7c3a772 Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 19 18:07:53 2017 +1200 API Create HTTPMiddleware and standardise middleware for request handling commit 2a10c2397bdc53001013f607b5d38087ce6c0730 Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 19 17:42:42 2017 +1200 Fixed ORM tests commit d75a8d1d93398af4bd0432df9e4bc6295c15a3fe Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 19 17:15:07 2017 +1200 FIx i18n tests commit 06364af3c379c931889c4cc34dd920fee3db204a Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 19 16:59:34 2017 +1200 Fix controller namespace Move states to sub namespace commit 2a278e2953d2dbb19f78d91c919048e1fc935436 Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 19 12:49:45 2017 +1200 Fix forms namespace commit b65c21241bee019730027071d815dbf7571197a4 Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 15 18:56:48 2017 +1200 Update API usages commit d1d4375c95a264a6b63cbaefc2c1d12f808bfd82 Author: Damian Mooyman <damian@silverstripe.com> Date: Thu Jun 15 18:41:44 2017 +1200 API Refactor $flush into HTPPApplication API Enforce health check in Controller::pushCurrent() API Better global backup / restore Updated Director::test() to use new API commit b220534f06732db4fa940d8724c2a85c0ba2495a Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 22:05:57 2017 +1200 Move app nesting to a test state helper commit 603704165c08d0c1c81fd5e6bb9506326eeee17b Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 21:46:04 2017 +1200 Restore kernel stack to fix multi-level nesting commit 2f6336a15bf79dc8c2edd44cec1931da2dd51c28 Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 17:23:21 2017 +1200 API Implement kernel nesting commit fc7188da7d6ad6785354bab61f08700454c81d91 Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 15:43:13 2017 +1200 Fix core tests commit a0ae7235148fffd71f2f02d1fe7fe45bf3aa39eb Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 15:23:52 2017 +1200 Fix manifest tests commit ca033952513633e182040d3d13e1caa9000ca184 Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 15:00:00 2017 +1200 API Move extension management into test state commit c66d4339777663a8a04661fea32a0cf35b95d20f Author: Damian Mooyman <damian@silverstripe.com> Date: Tue Jun 13 14:10:59 2017 +1200 API Refactor SapphireTest state management into SapphireTestState API Remove Injector::unregisterAllObjects() API Remove FakeController commit f26ae75c6ecaafa0dec1093264e0187191e6764d Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 12 18:04:34 2017 +1200 Implement basic CLI application object commit 001d5596621404892de0a5413392379eff990641 Author: Damian Mooyman <damian@silverstripe.com> Date: Mon Jun 12 17:39:38 2017 +1200 Remove references to SapphireTest::is_running_test() Upgrade various code commit de079c041dacd96bc4f4b66421fa2b2cc4c320f8 Author: Damian Mooyman <damian@silverstripe.com> Date: Wed Jun 7 18:07:33 2017 +1200 API Implement APP object API Refactor of Session
2017-06-22 12:50:45 +02:00
Environment::increaseTimeLimitTo(3600);
Environment::increaseMemoryLimitTo('512M');
2016-11-29 00:31:16 +01:00
//get all instances of the to be imported data object
if ($this->deleteExistingRecords) {
DataObject::get($this->objectClass)->removeAll();
}
return $this->processAll($filepath);
}
/**
* Preview a file import (don't write anything to the database).
* Useful to analyze the input and give the users a chance to influence
* it through a UI.
*
* @param string $filepath Absolute path to the file we're importing
* @return array See {@link self::processAll()}
*/
abstract public function preview($filepath);
/**
* Process every record in the file
*
* @param string $filepath Absolute path to the file we're importing (with UTF8 content)
* @param boolean $preview If true, we'll just output a summary of changes but not actually do anything
* @return BulkLoader_Result A collection of objects which are either created, updated or deleted.
* 'message': free-text string that can optionally provide some more information about what changes have
*/
abstract protected function processAll($filepath, $preview = false);
/**
* Process a single record from the file.
*
* @param array $record An map of the data, keyed by the header field defined in {@link self::$columnMap}
* @param array $columnMap
* @param $result BulkLoader_Result (passed as reference)
* @param boolean $preview
*/
abstract protected function processRecord($record, $columnMap, &$result, $preview = false);
/**
* Return a FieldList containing all the options for this form; this
* doesn't include the actual upload field itself
*/
public function getOptionFields()
{
}
/**
* Return a human-readable name for this object.
* It defaults to the class name can be overridden by setting the static variable $title
*
* @return string
*/
public function Title()
{
$title = $this->config()->get('title');
return $title ?: static::class;
2016-11-29 00:31:16 +01:00
}
/**
* Get a specification of all available columns and relations on the used model.
* Useful for generation of spec documents for technical end users.
*
* Return Format:
* <code>
* array(
* 'fields' => array('myFieldName'=>'myDescription'),
* 'relations' => array('myRelationName'=>'myDescription'),
* )
* </code>
*
* @todo Mix in custom column mappings
*
* @return array
**/
public function getImportSpec()
{
$singleton = DataObject::singleton($this->objectClass);
2016-11-29 00:31:16 +01:00
// get database columns (fieldlabels include fieldname as a key)
// using $$includerelations flag as false, so that it only contain $db fields
$fields = (array)$singleton->fieldLabels(false);
2016-11-29 00:31:16 +01:00
// Merge relations
$relations = array_merge(
$singleton->hasOne(),
$singleton->hasMany(),
$singleton->manyMany()
);
2016-11-29 00:31:16 +01:00
// Ensure description is string (e.g. many_many through)
foreach ($relations as $name => $desc) {
if (!is_string($desc)) {
$relations[$name] = $name;
}
}
2016-11-29 00:31:16 +01:00
return [
'fields' => $fields,
'relations' => $relations,
];
2016-11-29 00:31:16 +01:00
}
/**
* Determines if a specific field is null.
* Can be useful for unusual "empty" flags in the file,
* e.g. a "(not set)" value.
* The usual {@link DBField::isNull()} checks apply when writing the {@link DataObject},
* so this is mainly a customization method.
*
* @param mixed $val
* @param string $fieldName Name of the field as specified in the array-values for {@link self::$columnMap}.
* @return boolean
*/
protected function isNullValue($val, $fieldName = null)
{
return (empty($val) && $val !== '0');
}
}