Merge pull request #9022 from open-sausages/pulls/4.4/migrate-task-colours

NEW Clearer file migration output with colours

composer.json

@@ -22,6 +22,7 @@
         "sake"
     ],
     "require": {
+        "bramus/monolog-colored-line-formatter": "~2.0",
         "composer/installers": "~1.0",
         "embed/embed": "^3.0",
         "league/csv": "^8",

docs/en/02_Developer_Guides/14_Files/05_File_Migration.md

@@ -37,7 +37,26 @@ This task will perform a number of subtasks:
 One or more subtasks can be run individually through the `only` argument.
 Example: `only=move-files,move-thumbnails`
 
+The output is quite verbose by default. Look for `WARNING` and `ERROR` in the log files.
+When executing the task on CLI, you'll get colour coded error messages.
+
+## Background migration through the Queuedjobs module
+
 You can also run this task without CLI access through the [queuedjobs](https://github.com/symbiote/silverstripe-queuedjobs) module.
+Open up `admin/queuedjobs`, then create a job of type `RunBuildTaskJob`.
+The only constructor parameter allowed is the full name of the task: `SilverStripe\Dev\Tasks\MigrateFileTask`.
+The task output will be progressively written to the job record, and can be inspected via the "Messages" tab within the job in the CMS.
+It attempts to continue running to "complete" status even if it encounters errors, so you'll need to review the logs
+to ensure if everything went smoothly. Note that it's currently not possible to run specific subtasks via a queuedjob.
+
+While you can run the job directly through the CMS, it'll usually be more constrained by PHP `max_execution_time` settings.
+Many platforms such as the New Zealand Government Common Web Platform or SilverStripe Platform
+are configured to run jobs automatically without time limits
+([1](https://docs.platform.silverstripe.com/development/platform-yml-file/#cron-tasks),
+[2](https://www.cwp.govt.nz/developer-docs/en/2/working_with_projects/infrastructural_considerations/)).
+It is not recommended to run
+[multiple processes](https://github.com/symbiote/silverstripe-queuedjobs/blob/master/docs/en/configuring-multi-process-execution.md)
+when executing the file migration job. 
 
 ## Migration of existing thumbnails
 

src/Dev/Tasks/MigrateFileTask.php

@@ -2,6 +2,7 @@
 
 namespace SilverStripe\Dev\Tasks;
 
+use Monolog\Handler\FilterHandler;
 use Monolog\Handler\StreamHandler;
 use Monolog\Logger;
 use Psr\Log\LoggerInterface;
@@ -11,10 +12,12 @@ use SilverStripe\Assets\Dev\Tasks\FileMigrationHelper;
 use SilverStripe\Assets\Storage\AssetStore;
 use SilverStripe\Assets\Storage\FileHashingService;
 use SilverStripe\Control\Director;
+use SilverStripe\Core\Environment;
 use SilverStripe\Core\Injector\Injector;
 use SilverStripe\Logging\PreformattedEchoHandler;
 use SilverStripe\Dev\BuildTask;
 use SilverStripe\Assets\Dev\Tasks\SecureAssetsMigrationHelper;
+use \Bramus\Monolog\Formatter\ColoredLineFormatter;
 
 /**
  * Migrates all 3.x file dataobjects to use the new DBFile field.
@@ -49,8 +52,18 @@ class MigrateFileTask extends BuildTask
 
         Injector::inst()->get(FileHashingService::class)->enableCache();
 
+        // Set max time and memory limit
+        Environment::increaseTimeLimitTo();
+        Environment::setMemoryLimitMax(-1);
+        Environment::increaseMemoryLimitTo(-1);
+
         $this->extend('preFileMigration');
 
+        $this->logger->warn(
+            'Please read https://docs.silverstripe.org/en/4/developer_guides/files/file_migration/ ' .
+            'before running this task.'
+        );
+
         $subtasks = !empty($args['only']) ? explode(',', $args['only']) : $this->defaultSubtasks;
 
         $subtask = 'move-files';
@@ -59,15 +72,23 @@ class MigrateFileTask extends BuildTask
                 $this->logger->error("No file migration helper detected");
             } else {
                 $this->extend('preFileMigrationSubtask', $subtask);
-                $this->logger->info("### Migrating filesystem and database records ({$subtask})");
-                $this->logger->info('If the task fails or times out, run it again and it will start where it left off.');
 
-                $migrated = FileMigrationHelper::singleton()->run();
+                $this->logger->notice("######################################################");
-                if ($migrated) {
+                $this->logger->notice("Migrating filesystem and database records ({$subtask})");
-                    $this->logger->info("{$migrated} File DataObjects upgraded");
+                $this->logger->notice("######################################################");
-                } else {
+
-                    $this->logger->info("No File DataObjects need upgrading");
+                FileMigrationHelper::singleton()
-                }
+                    ->setLogger($this->logger)
+                    ->run();
+
+                // TODO Split file migration helper into two tasks,
+                // and report back on their process counts consistently here
+                // if ($count) {
+                //     $this->logger->info("{$count} File objects upgraded");
+                // } else {
+                //     $this->logger->info("No File objects needed upgrading");
+                // }
+
                 $this->extend('postFileMigrationSubtask', $subtask);
             }
         }
@@ -78,16 +99,19 @@ class MigrateFileTask extends BuildTask
                 $this->logger->error("LegacyThumbnailMigrationHelper not found");
             } else {
                 $this->extend('preFileMigrationSubtask', $subtask);
-                $this->logger->info("### Migrating existing thumbnails ({$subtask})");
 
-                $moved = LegacyThumbnailMigrationHelper::singleton()
+                $this->logger->notice("#############################################################");
+                $this->logger->notice("Migrating existing thumbnails to new file format ({$subtask})");
+                $this->logger->notice("#############################################################");
+
+                $paths = LegacyThumbnailMigrationHelper::singleton()
                     ->setLogger($this->logger)
                     ->run($this->getStore());
 
-                if ($moved) {
+                if ($paths) {
-                    $this->logger->info(sprintf("%d thumbnails moved", count($moved)));
+                    $this->logger->info(sprintf("%d thumbnails moved", count($paths)));
                 } else {
-                    $this->logger->info("No thumbnails moved");
+                    $this->logger->info("No thumbnails needed to be moved");
                 }
 
                 $this->extend('postFileMigrationSubtask', $subtask);
@@ -100,8 +124,21 @@ class MigrateFileTask extends BuildTask
                 $this->logger->error("ImageThumbnailHelper not found");
             } else {
                 $this->extend('preFileMigrationSubtask', $subtask);
-                $this->logger->info("### Generating new CMS UI thumbnails ({$subtask})");
+
-                ImageThumbnailHelper::singleton()->run();
+                $this->logger->notice("#############################################");
+                $this->logger->notice("Generating new CMS UI thumbnails ({$subtask})");
+                $this->logger->notice("#############################################");
+
+                $count = ImageThumbnailHelper::singleton()
+                    ->setLogger($this->logger)
+                    ->run();
+
+                if ($count > 0) {
+                    $this->logger->info("Created {$count} CMS UI thumbnails");
+                } else {
+                    $this->logger->info("No CMS UI thumbnails needed to be created");
+                }
+
                 $this->extend('postFileMigrationSubtask', $subtask);
             }
         }
@@ -113,11 +150,17 @@ class MigrateFileTask extends BuildTask
             } else {
                 $this->extend('preFileMigrationSubtask', $subtask);
 
-                $this->logger->info("### Fixing folder permissions ({$subtask})");
+                $this->logger->notice("####################################################");
-                $updated = FixFolderPermissionsHelper::singleton()->run();
+                $this->logger->notice("Fixing secure-assets folder permissions ({$subtask})");
+                $this->logger->notice("####################################################");
+                $this->logger->debug('Only required if the 3.x project included silverstripe/secure-assets');
 
-                if ($updated > 0) {
+                $count = FixFolderPermissionsHelper::singleton()
-                    $this->logger->info("Repaired {$updated} folders with broken CanViewType settings");
+                    ->setLogger($this->logger)
+                    ->run();
+
+                if ($count > 0) {
+                    $this->logger->info("Repaired {$count} folders with broken CanViewType settings");
                 } else {
                     $this->logger->info("No folders required fixes");
                 }
@@ -133,11 +176,21 @@ class MigrateFileTask extends BuildTask
             } else {
                 $this->extend('preFileMigrationSubtask', $subtask);
 
-                $this->logger->info("### Fixing secure-assets ({$subtask})");
+                $this->logger->notice("#####################################################");
-                $moved = SecureAssetsMigrationHelper::singleton()
+                $this->logger->notice("Fixing secure-assets folder restrictions ({$subtask})");
+                $this->logger->notice("#####################################################");
+                $this->logger->debug('Only required if the 3.x project included silverstripe/secure-assets');
+
+                $paths = SecureAssetsMigrationHelper::singleton()
                     ->setLogger($this->logger)
                     ->run($this->getStore());
 
+                if (count($paths) > 0) {
+                    $this->logger->info(sprintf("Repaired %d folders broken folder restrictions", count($paths)));
+                } else {
+                    $this->logger->info("No folders required fixes");
+                }
+
                 $this->extend('postFileMigrationSubtask', $subtask);
             }
         }
@@ -151,9 +204,8 @@ class MigrateFileTask extends BuildTask
     {
         return <<<TXT
 Imports all files referenced by File dataobjects into the new Asset Persistence Layer introduced in 4.0.
-Moves existing thumbnails, and generates new thumbnail sizes for the CMS UI.
+Moves existing thumbnails, and generates new thumbnail sizes for the CMS UI. Fixes file permissions.
-Fixes file permissions.
+If the task fails or times out, run it again and if possible the tasks will start where they left off.
-If the task fails or times out, run it again and it will start where it left off.
 You need to flush your cache after running this task via CLI.
 See https://docs.silverstripe.org/en/4/developer_guides/files/file_migration/.
 TXT;
@@ -198,13 +250,29 @@ TXT;
      */
     protected function addLogHandlers()
     {
-        if ($logger = Injector::inst()->get(LoggerInterface::class)) {
+        // Using a global service here so other systems can control and redirect log output,
-            if (Director::is_cli()) {
+        // for example when this task is run as part of a queuedjob
-                $logger->pushHandler(new StreamHandler('php://stdout'));
+        $logger = Injector::inst()->get(LoggerInterface::class)->withName('log');
-                $logger->pushHandler(new StreamHandler('php://stderr', Logger::WARNING));
+
-            } else {
+        $formatter = new ColoredLineFormatter();
-                $logger->pushHandler(new PreformattedEchoHandler());
+        $formatter->ignoreEmptyContextAndExtra();
-            }
+
-        }
+        $errorHandler = new StreamHandler('php://stderr', Logger::ERROR);
+        $errorHandler->setFormatter($formatter);
+
+        $standardHandler = new StreamHandler('php://stdout');
+        $standardHandler->setFormatter($formatter);
+
+        // Avoid double logging of errors
+        $standardFilterHandler = new FilterHandler(
+            $standardHandler,
+            Logger::DEBUG,
+            Logger::WARNING
+        );
+
+        $logger->pushHandler($standardFilterHandler);
+        $logger->pushHandler($errorHandler);
+
+        $this->logger = $logger;
     }
 }