diff --git a/code/tasks/FTFileMakerTask.php b/code/tasks/FTFileMakerTask.php index a9e8a39..5fd586a 100644 --- a/code/tasks/FTFileMakerTask.php +++ b/code/tasks/FTFileMakerTask.php @@ -7,6 +7,8 @@ use SilverStripe\Dev\BuildTask; use SilverStripe\ORM\DB; use GuzzleHttp\Client; use GuzzleHttp\Promise; +use SilverStripe\Security\Member; +use SilverStripe\Security\Security; /** * Creates sample folder and file structure, useful to test performance, @@ -24,12 +26,72 @@ use GuzzleHttp\Promise; * - reset=1: Optionally truncate ALL files and folders in the database, plus delete * the entire `assets/` directory. * + * + * Configuration + * + * app/_config/frameworktest.yml + * + * The following yml config make 1040 files / 520mb: + * + * To make 100K records, increase the base folderCountByDepth from 1 to 100 and run the task overnight + +FTFileMakerTask: + documentsOnly: true + doSetFolderPermissions: true + doSetOldCreationDate: false + doRandomlyPublish: false + depth: 3 + folderCountByDepth: + - 1 + - 9 + - 7 + - 0 + - 0 + fileCountByDepth: + - 50 + - 25 + - 20 + - 0 + - 0 + * + * Flush and run: + * /dev/tasks/FTFileMakerTask?flush&reset=1 + * * @todo Automatically retrieve file listing from S3 * @todo Handle HTTP errors from S3 */ class FTFileMakerTask extends BuildTask { + protected $documentsOnly = false; + + protected $doSetFolderPermissions = false; + + // simulate scenario where protected files were uploaded into the wrong asset store + protected $doPutProtectedFilesInPublicStore = false; + + protected $doSetOldCreationDate = true; + + protected $doRandomlyPublish = true; + + protected $depth = 2; + + protected $folderCountByDepth = [ + 0 => 2, + 1 => 2, + 2 => 2, + 3 => 2, + 4 => 2, + ]; + + protected $fileCountByDepth = [ + 0 => 10, + 1 => 8, + 2 => 5, + 3 => 5, + 4 => 5, + ]; + protected $fixtureFileBaseUrl = "https://s3-ap-southeast-2.amazonaws.com/silverstripe-frameworktest-assets/"; protected $defaultImageFileName = 'image-huge-tall.jpg'; @@ -70,46 +132,55 @@ class FTFileMakerTask extends BuildTask 'video.m4v' => 'SilverStripe\Assets\File', ]; - protected $folderCountByDepth = [ - 0 => 2, - 1 => 2, - 2 => 2, - 3 => 2, - 4 => 2, - ]; + protected $lineBreak = "\n
"; - protected $fileCountByDepth = [ - 0 => 100, - 1 => 30, - 2 => 5, - 3 => 5, - 4 => 5, - ]; - - /** - * @var int Constrained by elements in $folderCountByDepth and $fileCountByDepth - */ - protected $depth = 2; + /** @var Member */ + protected $anonymousMember = null; public function run($request) { + set_time_limit(0); + + // used to test canView() permissions + $this->anonymousMember = Member::get()->find('Email', 'frameworktestuser'); + if (!$this->anonymousMember) { + $this->anonymousMember = Member::create(); + $this->anonymousMember->Email = 'frameworktestuser'; + $this->anonymousMember->write(); + } + if (!$this->anonymousMember->inGroup('content-authors')) { + $this->anonymousMember->addToGroupByCode('content-authors'); + } + Security::setCurrentUser($this->anonymousMember); + + if (php_sapi_name() == "cli") { + $this->lineBreak = "\n"; + } + if ($request->getVar('reset')) { $this->reset(); } - echo "Downloading fixtures\n"; + echo "Downloading fixtures" . $this->lineBreak; $fixtureFilePaths = $this->downloadFixtureFiles(); - echo "Generate thumbnails\n"; - $this->generateThumbnails($fixtureFilePaths); + if (!self::config()->get('documentsOnly')) { + echo "Generate thumbnails" . $this->lineBreak; + $this->generateThumbnails($fixtureFilePaths); + } - echo "Generate files\n"; + echo "Generate files" . $this->lineBreak; $this->generateFiles($fixtureFilePaths); + + if (!self::config()->get('doPutProtectedFilesInPublicStore')) { + echo "Incorrectly putting protected files into public asset store on purpose" . $this->lineBreak; + $this->putProtectedFilesInPublicAssetStore(); + } } protected function reset() { - echo "Resetting assets\n"; + echo "Resetting assets" . $this->lineBreak; DB::query('TRUNCATE "File"'); DB::query('TRUNCATE "File_Live"'); @@ -124,10 +195,17 @@ class FTFileMakerTask extends BuildTask { $client = new Client(['base_uri' => $this->fixtureFileBaseUrl]); + $fixtureFileNames = $this->fixtureFileNames; + if (self::config()->get('documentsOnly')) { + $fixtureFileNames = array_filter($fixtureFileNames, function($v) { + return (bool) preg_match('%\.(docx|xlsx|pdf)$%', $v); + }); + } + // Initiate each request but do not block $promises = []; $paths = []; - foreach ($this->fixtureFileNames as $filename) { + foreach ($fixtureFileNames as $filename) { $path = TEMP_FOLDER . '/' . $filename; $paths[$filename] = $path; $url = "{$this->fixtureFileBaseUrl}/{$filename}"; @@ -135,7 +213,7 @@ class FTFileMakerTask extends BuildTask $promises[$filename] = $client->getAsync($filename, [ 'sink' => $path ]); - echo "Downloading $url\n"; + echo "Downloading $url" . $this->lineBreak; } } @@ -169,27 +247,44 @@ class FTFileMakerTask extends BuildTask $file->publishFile(); } - $file->Pad(60,60)->CropHeight(30); + $file->Pad(60, 60)->CropHeight(30); } } protected function generateFiles($fixtureFilePaths, $depth = 0, $prefix = "0", $parentID = 0) { - $folderCount = $this->folderCountByDepth[$depth]; - $fileCount = $this->fileCountByDepth[$depth]; - for ($i=1; $i<=$folderCount; $i++) { + $folderCount = self::config()->get('folderCountByDepth')[$depth]; + $fileCount = self::config()->get('fileCountByDepth')[$depth]; + + $doSetFolderPermissions = (bool) self::config()->get('doSetFolderPermissions'); + + $doSetOldCreationDate = (bool) self::config()->get('doSetOldCreationDate'); + $doRandomlyPublish = (bool) self::config()->get('doRandomlyPublish'); + + for ($i = 1; $i <= $folderCount; $i++) { $folder = new Folder([ 'ParentID' => $parentID, 'Title' => "testfolder-{$prefix}{$i}", 'Name' => "testfolder-{$prefix}{$i}", ]); - $folder->write(); - echo "\n"; - echo "Created Folder: '$folder->Title'\n"; + if ($doSetFolderPermissions) { + if ($i == 1) { + // the first folder should always be public to ensure there's some public folders + $folder->CanViewType = 'Inherit'; + } elseif ($i == $folderCount) { + // the last folder should always be protected to ensure there's some protected folders + $folder->CanViewType = 'OnlyTheseUsers'; + } else { + // all the other folder have a 33% chance of being a protected folder + $folder->CanViewType = rand(0, 2) == 0 ? 'OnlyTheseUsers' : 'Inherit'; + } + } - for ($j=1; $j<=$fileCount; $j++) { - $randomFileName = array_keys($fixtureFilePaths)[rand(0, count($fixtureFilePaths)-1)]; + $folder->write(); + + for ($j = 1; $j <= $fileCount; $j++) { + $randomFileName = array_keys($fixtureFilePaths)[rand(0, count($fixtureFilePaths) - 1)]; $randomFilePath = $fixtureFilePaths[$randomFileName]; $fileName = pathinfo($randomFilePath, PATHINFO_FILENAME) @@ -208,28 +303,53 @@ class FTFileMakerTask extends BuildTask 'Name' => $fileName, ]); $file->File->setFromLocalFile($randomFilePath, $folder->getFilename() . $fileName); - $file->write(); - - // Randomly publish - if (rand(0, 1) == 0) { - $file->publishFile(); - } // Randomly set old created date (for testing) - if (rand(0, 10) == 0) { - $file->Created = '2010-01-01 00:00:00'; - $file->Title = '[old] ' . $file->Title; - $file->write(); + if ($doSetOldCreationDate) { + if (rand(0, 10) == 0) { + $file->Created = '2010-01-01 00:00:00'; + $file->Title = '[old] ' . $file->Title; + } } + $file->write(); - echo " Created File: '$file->Title'\n"; + if ($doRandomlyPublish) { + if (rand(0, 1) == 0) { + $file->publishFile(); + } + } else { + // publish files that should be viewable + if ($file->canView($this->anonymousMember)) { + $url = $file->getAbsoluteURL(); + $file->publishFile(); + } + } } - if ($depth < $this->depth) { - $this->generateFiles($fixtureFilePaths, $depth+1, "{$prefix}-{$i}", $folder->ID); + if ($depth < self::config()->get('depth') - 1) { + $this->generateFiles($fixtureFilePaths, $depth + 1, "{$prefix}-{$i}", $folder->ID); } } } + protected function putProtectedFilesInPublicAssetStore() + { + /** @var File $file */ + foreach (File::get()->exclude(['ClassName' => Folder::class]) as $file) { + // file is already in public asset store + if ($file->canView($this->anonymousMember)) { + continue; + } + // randomly move 50% of the files that should be in the protected store to the public store + if (rand(0, 1) == 0) { + continue; + } + // this will move the file into the public asset store, even it it should be protected + // i.e. the parent folder CanViewType = OnlyTheseUsers + $file->publishFile(); + $url = $file->getAbsoluteURL(); + } + } + }