diff --git a/composer.json b/composer.json index 70628bb75..4e76d8bf9 100644 --- a/composer.json +++ b/composer.json @@ -22,6 +22,7 @@ "sake" ], "require": { + "bramus/monolog-colored-line-formatter": "~2.0", "composer/installers": "~1.0", "embed/embed": "^3.0", "league/csv": "^8", diff --git a/docs/en/02_Developer_Guides/14_Files/05_File_Migration.md b/docs/en/02_Developer_Guides/14_Files/05_File_Migration.md index 59fe7b3c4..c2168f8f1 100644 --- a/docs/en/02_Developer_Guides/14_Files/05_File_Migration.md +++ b/docs/en/02_Developer_Guides/14_Files/05_File_Migration.md @@ -37,7 +37,26 @@ This task will perform a number of subtasks: One or more subtasks can be run individually through the `only` argument. Example: `only=move-files,move-thumbnails` +The output is quite verbose by default. Look for `WARNING` and `ERROR` in the log files. +When executing the task on CLI, you'll get colour coded error messages. + +## Background migration through the Queuedjobs module + You can also run this task without CLI access through the [queuedjobs](https://github.com/symbiote/silverstripe-queuedjobs) module. +Open up `admin/queuedjobs`, then create a job of type `RunBuildTaskJob`. +The only constructor parameter allowed is the full name of the task: `SilverStripe\Dev\Tasks\MigrateFileTask`. +The task output will be progressively written to the job record, and can be inspected via the "Messages" tab within the job in the CMS. +It attempts to continue running to "complete" status even if it encounters errors, so you'll need to review the logs +to ensure if everything went smoothly. Note that it's currently not possible to run specific subtasks via a queuedjob. + +While you can run the job directly through the CMS, it'll usually be more constrained by PHP `max_execution_time` settings. +Many platforms such as the New Zealand Government Common Web Platform or SilverStripe Platform +are configured to run jobs automatically without time limits +([1](https://docs.platform.silverstripe.com/development/platform-yml-file/#cron-tasks), +[2](https://www.cwp.govt.nz/developer-docs/en/2/working_with_projects/infrastructural_considerations/)). +It is not recommended to run +[multiple processes](https://github.com/symbiote/silverstripe-queuedjobs/blob/master/docs/en/configuring-multi-process-execution.md) +when executing the file migration job. ## Migration of existing thumbnails diff --git a/src/Dev/Tasks/MigrateFileTask.php b/src/Dev/Tasks/MigrateFileTask.php index 7bdaca5e7..eda5cf33f 100644 --- a/src/Dev/Tasks/MigrateFileTask.php +++ b/src/Dev/Tasks/MigrateFileTask.php @@ -2,6 +2,7 @@ namespace SilverStripe\Dev\Tasks; +use Monolog\Handler\FilterHandler; use Monolog\Handler\StreamHandler; use Monolog\Logger; use Psr\Log\LoggerInterface; @@ -11,10 +12,12 @@ use SilverStripe\Assets\Dev\Tasks\FileMigrationHelper; use SilverStripe\Assets\Storage\AssetStore; use SilverStripe\Assets\Storage\FileHashingService; use SilverStripe\Control\Director; +use SilverStripe\Core\Environment; use SilverStripe\Core\Injector\Injector; use SilverStripe\Logging\PreformattedEchoHandler; use SilverStripe\Dev\BuildTask; use SilverStripe\Assets\Dev\Tasks\SecureAssetsMigrationHelper; +use \Bramus\Monolog\Formatter\ColoredLineFormatter; /** * Migrates all 3.x file dataobjects to use the new DBFile field. @@ -49,8 +52,18 @@ class MigrateFileTask extends BuildTask Injector::inst()->get(FileHashingService::class)->enableCache(); + // Set max time and memory limit + Environment::increaseTimeLimitTo(); + Environment::setMemoryLimitMax(-1); + Environment::increaseMemoryLimitTo(-1); + $this->extend('preFileMigration'); + $this->logger->warn( + 'Please read https://docs.silverstripe.org/en/4/developer_guides/files/file_migration/ ' . + 'before running this task.' + ); + $subtasks = !empty($args['only']) ? explode(',', $args['only']) : $this->defaultSubtasks; $subtask = 'move-files'; @@ -59,15 +72,23 @@ class MigrateFileTask extends BuildTask $this->logger->error("No file migration helper detected"); } else { $this->extend('preFileMigrationSubtask', $subtask); - $this->logger->info("### Migrating filesystem and database records ({$subtask})"); - $this->logger->info('If the task fails or times out, run it again and it will start where it left off.'); - $migrated = FileMigrationHelper::singleton()->run(); - if ($migrated) { - $this->logger->info("{$migrated} File DataObjects upgraded"); - } else { - $this->logger->info("No File DataObjects need upgrading"); - } + $this->logger->notice("######################################################"); + $this->logger->notice("Migrating filesystem and database records ({$subtask})"); + $this->logger->notice("######################################################"); + + FileMigrationHelper::singleton() + ->setLogger($this->logger) + ->run(); + + // TODO Split file migration helper into two tasks, + // and report back on their process counts consistently here + // if ($count) { + // $this->logger->info("{$count} File objects upgraded"); + // } else { + // $this->logger->info("No File objects needed upgrading"); + // } + $this->extend('postFileMigrationSubtask', $subtask); } } @@ -78,16 +99,19 @@ class MigrateFileTask extends BuildTask $this->logger->error("LegacyThumbnailMigrationHelper not found"); } else { $this->extend('preFileMigrationSubtask', $subtask); - $this->logger->info("### Migrating existing thumbnails ({$subtask})"); - $moved = LegacyThumbnailMigrationHelper::singleton() + $this->logger->notice("#############################################################"); + $this->logger->notice("Migrating existing thumbnails to new file format ({$subtask})"); + $this->logger->notice("#############################################################"); + + $paths = LegacyThumbnailMigrationHelper::singleton() ->setLogger($this->logger) ->run($this->getStore()); - if ($moved) { - $this->logger->info(sprintf("%d thumbnails moved", count($moved))); + if ($paths) { + $this->logger->info(sprintf("%d thumbnails moved", count($paths))); } else { - $this->logger->info("No thumbnails moved"); + $this->logger->info("No thumbnails needed to be moved"); } $this->extend('postFileMigrationSubtask', $subtask); @@ -100,8 +124,21 @@ class MigrateFileTask extends BuildTask $this->logger->error("ImageThumbnailHelper not found"); } else { $this->extend('preFileMigrationSubtask', $subtask); - $this->logger->info("### Generating new CMS UI thumbnails ({$subtask})"); - ImageThumbnailHelper::singleton()->run(); + + $this->logger->notice("#############################################"); + $this->logger->notice("Generating new CMS UI thumbnails ({$subtask})"); + $this->logger->notice("#############################################"); + + $count = ImageThumbnailHelper::singleton() + ->setLogger($this->logger) + ->run(); + + if ($count > 0) { + $this->logger->info("Created {$count} CMS UI thumbnails"); + } else { + $this->logger->info("No CMS UI thumbnails needed to be created"); + } + $this->extend('postFileMigrationSubtask', $subtask); } } @@ -113,11 +150,17 @@ class MigrateFileTask extends BuildTask } else { $this->extend('preFileMigrationSubtask', $subtask); - $this->logger->info("### Fixing folder permissions ({$subtask})"); - $updated = FixFolderPermissionsHelper::singleton()->run(); + $this->logger->notice("####################################################"); + $this->logger->notice("Fixing secure-assets folder permissions ({$subtask})"); + $this->logger->notice("####################################################"); + $this->logger->debug('Only required if the 3.x project included silverstripe/secure-assets'); - if ($updated > 0) { - $this->logger->info("Repaired {$updated} folders with broken CanViewType settings"); + $count = FixFolderPermissionsHelper::singleton() + ->setLogger($this->logger) + ->run(); + + if ($count > 0) { + $this->logger->info("Repaired {$count} folders with broken CanViewType settings"); } else { $this->logger->info("No folders required fixes"); } @@ -133,11 +176,21 @@ class MigrateFileTask extends BuildTask } else { $this->extend('preFileMigrationSubtask', $subtask); - $this->logger->info("### Fixing secure-assets ({$subtask})"); - $moved = SecureAssetsMigrationHelper::singleton() + $this->logger->notice("#####################################################"); + $this->logger->notice("Fixing secure-assets folder restrictions ({$subtask})"); + $this->logger->notice("#####################################################"); + $this->logger->debug('Only required if the 3.x project included silverstripe/secure-assets'); + + $paths = SecureAssetsMigrationHelper::singleton() ->setLogger($this->logger) ->run($this->getStore()); + if (count($paths) > 0) { + $this->logger->info(sprintf("Repaired %d folders broken folder restrictions", count($paths))); + } else { + $this->logger->info("No folders required fixes"); + } + $this->extend('postFileMigrationSubtask', $subtask); } } @@ -151,9 +204,8 @@ class MigrateFileTask extends BuildTask { return <<get(LoggerInterface::class)) { - if (Director::is_cli()) { - $logger->pushHandler(new StreamHandler('php://stdout')); - $logger->pushHandler(new StreamHandler('php://stderr', Logger::WARNING)); - } else { - $logger->pushHandler(new PreformattedEchoHandler()); - } - } + // Using a global service here so other systems can control and redirect log output, + // for example when this task is run as part of a queuedjob + $logger = Injector::inst()->get(LoggerInterface::class)->withName('log'); + + $formatter = new ColoredLineFormatter(); + $formatter->ignoreEmptyContextAndExtra(); + + $errorHandler = new StreamHandler('php://stderr', Logger::ERROR); + $errorHandler->setFormatter($formatter); + + $standardHandler = new StreamHandler('php://stdout'); + $standardHandler->setFormatter($formatter); + + // Avoid double logging of errors + $standardFilterHandler = new FilterHandler( + $standardHandler, + Logger::DEBUG, + Logger::WARNING + ); + + $logger->pushHandler($standardFilterHandler); + $logger->pushHandler($errorHandler); + + $this->logger = $logger; } }