Merge pull request #66 from creative-commoners/pulls/master/increase-limits
WIP: Increase limits
This commit is contained in:
Serge Latyntsev
GitHub
commit
fc7d41930d
|
|
@ -7,6 +7,8 @@ use SilverStripe\Dev\BuildTask;
|
|||
use SilverStripe\ORM\DB;
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Promise;
|
||||
use SilverStripe\Security\Member;
|
||||
use SilverStripe\Security\Security;
|
||||
|
||||
/**
|
||||
* Creates sample folder and file structure, useful to test performance,
|
||||
|
|
@ -24,12 +26,72 @@ use GuzzleHttp\Promise;
|
|||
* - reset=1: Optionally truncate ALL files and folders in the database, plus delete
|
||||
* the entire `assets/` directory.
|
||||
*
|
||||
*
|
||||
* Configuration
|
||||
*
|
||||
* app/_config/frameworktest.yml
|
||||
*
|
||||
* The following yml config make 1040 files / 520mb:
|
||||
*
|
||||
* To make 100K records, increase the base folderCountByDepth from 1 to 100 and run the task overnight
|
||||
|
||||
FTFileMakerTask:
|
||||
documentsOnly: true
|
||||
doSetFolderPermissions: true
|
||||
doSetOldCreationDate: false
|
||||
doRandomlyPublish: false
|
||||
depth: 3
|
||||
folderCountByDepth:
|
||||
- 1
|
||||
- 9
|
||||
- 7
|
||||
- 0
|
||||
- 0
|
||||
fileCountByDepth:
|
||||
- 50
|
||||
- 25
|
||||
- 20
|
||||
- 0
|
||||
- 0
|
||||
*
|
||||
* Flush and run:
|
||||
* /dev/tasks/FTFileMakerTask?flush&reset=1
|
||||
*
|
||||
* @todo Automatically retrieve file listing from S3
|
||||
* @todo Handle HTTP errors from S3
|
||||
*/
|
||||
class FTFileMakerTask extends BuildTask
|
||||
{
|
||||
|
||||
protected $documentsOnly = false;
|
||||
|
||||
protected $doSetFolderPermissions = false;
|
||||
|
||||
// simulate scenario where protected files were uploaded into the wrong asset store
|
||||
protected $doPutProtectedFilesInPublicStore = false;
|
||||
|
||||
protected $doSetOldCreationDate = true;
|
||||
|
||||
protected $doRandomlyPublish = true;
|
||||
|
||||
protected $depth = 2;
|
||||
|
||||
protected $folderCountByDepth = [
|
||||
0 => 2,
|
||||
1 => 2,
|
||||
2 => 2,
|
||||
3 => 2,
|
||||
4 => 2,
|
||||
];
|
||||
|
||||
protected $fileCountByDepth = [
|
||||
0 => 10,
|
||||
1 => 8,
|
||||
2 => 5,
|
||||
3 => 5,
|
||||
4 => 5,
|
||||
];
|
||||
|
||||
protected $fixtureFileBaseUrl = "https://s3-ap-southeast-2.amazonaws.com/silverstripe-frameworktest-assets/";
|
||||
|
||||
protected $defaultImageFileName = 'image-huge-tall.jpg';
|
||||
|
|
@ -70,46 +132,55 @@ class FTFileMakerTask extends BuildTask
|
|||
'video.m4v' => 'SilverStripe\Assets\File',
|
||||
];
|
||||
|
||||
protected $folderCountByDepth = [
|
||||
0 => 2,
|
||||
1 => 2,
|
||||
2 => 2,
|
||||
3 => 2,
|
||||
4 => 2,
|
||||
];
|
||||
protected $lineBreak = "\n<br>";
|
||||
|
||||
protected $fileCountByDepth = [
|
||||
0 => 100,
|
||||
1 => 30,
|
||||
2 => 5,
|
||||
3 => 5,
|
||||
4 => 5,
|
||||
];
|
||||
|
||||
/**
|
||||
* @var int Constrained by elements in $folderCountByDepth and $fileCountByDepth
|
||||
*/
|
||||
protected $depth = 2;
|
||||
/** @var Member */
|
||||
protected $anonymousMember = null;
|
||||
|
||||
public function run($request)
|
||||
{
|
||||
set_time_limit(0);
|
||||
|
||||
// used to test canView() permissions
|
||||
$this->anonymousMember = Member::get()->find('Email', 'frameworktestuser');
|
||||
if (!$this->anonymousMember) {
|
||||
$this->anonymousMember = Member::create();
|
||||
$this->anonymousMember->Email = 'frameworktestuser';
|
||||
$this->anonymousMember->write();
|
||||
}
|
||||
if (!$this->anonymousMember->inGroup('content-authors')) {
|
||||
$this->anonymousMember->addToGroupByCode('content-authors');
|
||||
}
|
||||
Security::setCurrentUser($this->anonymousMember);
|
||||
|
||||
if (php_sapi_name() == "cli") {
|
||||
$this->lineBreak = "\n";
|
||||
}
|
||||
|
||||
if ($request->getVar('reset')) {
|
||||
$this->reset();
|
||||
}
|
||||
|
||||
echo "Downloading fixtures\n";
|
||||
echo "Downloading fixtures" . $this->lineBreak;
|
||||
$fixtureFilePaths = $this->downloadFixtureFiles();
|
||||
|
||||
echo "Generate thumbnails\n";
|
||||
$this->generateThumbnails($fixtureFilePaths);
|
||||
if (!self::config()->get('documentsOnly')) {
|
||||
echo "Generate thumbnails" . $this->lineBreak;
|
||||
$this->generateThumbnails($fixtureFilePaths);
|
||||
}
|
||||
|
||||
echo "Generate files\n";
|
||||
echo "Generate files" . $this->lineBreak;
|
||||
$this->generateFiles($fixtureFilePaths);
|
||||
|
||||
if (!self::config()->get('doPutProtectedFilesInPublicStore')) {
|
||||
echo "Incorrectly putting protected files into public asset store on purpose" . $this->lineBreak;
|
||||
$this->putProtectedFilesInPublicAssetStore();
|
||||
}
|
||||
}
|
||||
|
||||
protected function reset()
|
||||
{
|
||||
echo "Resetting assets\n";
|
||||
echo "Resetting assets" . $this->lineBreak;
|
||||
|
||||
DB::query('TRUNCATE "File"');
|
||||
DB::query('TRUNCATE "File_Live"');
|
||||
|
|
@ -124,10 +195,17 @@ class FTFileMakerTask extends BuildTask
|
|||
{
|
||||
$client = new Client(['base_uri' => $this->fixtureFileBaseUrl]);
|
||||
|
||||
$fixtureFileNames = $this->fixtureFileNames;
|
||||
if (self::config()->get('documentsOnly')) {
|
||||
$fixtureFileNames = array_filter($fixtureFileNames, function($v) {
|
||||
return (bool) preg_match('%\.(docx|xlsx|pdf)$%', $v);
|
||||
});
|
||||
}
|
||||
|
||||
// Initiate each request but do not block
|
||||
$promises = [];
|
||||
$paths = [];
|
||||
foreach ($this->fixtureFileNames as $filename) {
|
||||
foreach ($fixtureFileNames as $filename) {
|
||||
$path = TEMP_FOLDER . '/' . $filename;
|
||||
$paths[$filename] = $path;
|
||||
$url = "{$this->fixtureFileBaseUrl}/{$filename}";
|
||||
|
|
@ -135,7 +213,7 @@ class FTFileMakerTask extends BuildTask
|
|||
$promises[$filename] = $client->getAsync($filename, [
|
||||
'sink' => $path
|
||||
]);
|
||||
echo "Downloading $url\n";
|
||||
echo "Downloading $url" . $this->lineBreak;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -169,27 +247,44 @@ class FTFileMakerTask extends BuildTask
|
|||
$file->publishFile();
|
||||
}
|
||||
|
||||
$file->Pad(60,60)->CropHeight(30);
|
||||
$file->Pad(60, 60)->CropHeight(30);
|
||||
}
|
||||
}
|
||||
|
||||
protected function generateFiles($fixtureFilePaths, $depth = 0, $prefix = "0", $parentID = 0)
|
||||
{
|
||||
$folderCount = $this->folderCountByDepth[$depth];
|
||||
$fileCount = $this->fileCountByDepth[$depth];
|
||||
|
||||
for ($i=1; $i<=$folderCount; $i++) {
|
||||
$folderCount = self::config()->get('folderCountByDepth')[$depth];
|
||||
$fileCount = self::config()->get('fileCountByDepth')[$depth];
|
||||
|
||||
$doSetFolderPermissions = (bool) self::config()->get('doSetFolderPermissions');
|
||||
|
||||
$doSetOldCreationDate = (bool) self::config()->get('doSetOldCreationDate');
|
||||
$doRandomlyPublish = (bool) self::config()->get('doRandomlyPublish');
|
||||
|
||||
for ($i = 1; $i <= $folderCount; $i++) {
|
||||
$folder = new Folder([
|
||||
'ParentID' => $parentID,
|
||||
'Title' => "testfolder-{$prefix}{$i}",
|
||||
'Name' => "testfolder-{$prefix}{$i}",
|
||||
]);
|
||||
$folder->write();
|
||||
echo "\n";
|
||||
echo "Created Folder: '$folder->Title'\n";
|
||||
if ($doSetFolderPermissions) {
|
||||
if ($i == 1) {
|
||||
// the first folder should always be public to ensure there's some public folders
|
||||
$folder->CanViewType = 'Inherit';
|
||||
} elseif ($i == $folderCount) {
|
||||
// the last folder should always be protected to ensure there's some protected folders
|
||||
$folder->CanViewType = 'OnlyTheseUsers';
|
||||
} else {
|
||||
// all the other folder have a 33% chance of being a protected folder
|
||||
$folder->CanViewType = rand(0, 2) == 0 ? 'OnlyTheseUsers' : 'Inherit';
|
||||
}
|
||||
}
|
||||
|
||||
for ($j=1; $j<=$fileCount; $j++) {
|
||||
$randomFileName = array_keys($fixtureFilePaths)[rand(0, count($fixtureFilePaths)-1)];
|
||||
$folder->write();
|
||||
|
||||
for ($j = 1; $j <= $fileCount; $j++) {
|
||||
$randomFileName = array_keys($fixtureFilePaths)[rand(0, count($fixtureFilePaths) - 1)];
|
||||
$randomFilePath = $fixtureFilePaths[$randomFileName];
|
||||
|
||||
$fileName = pathinfo($randomFilePath, PATHINFO_FILENAME)
|
||||
|
|
@ -208,28 +303,53 @@ class FTFileMakerTask extends BuildTask
|
|||
'Name' => $fileName,
|
||||
]);
|
||||
$file->File->setFromLocalFile($randomFilePath, $folder->getFilename() . $fileName);
|
||||
$file->write();
|
||||
|
||||
// Randomly publish
|
||||
if (rand(0, 1) == 0) {
|
||||
$file->publishFile();
|
||||
}
|
||||
|
||||
// Randomly set old created date (for testing)
|
||||
if (rand(0, 10) == 0) {
|
||||
$file->Created = '2010-01-01 00:00:00';
|
||||
$file->Title = '[old] ' . $file->Title;
|
||||
$file->write();
|
||||
if ($doSetOldCreationDate) {
|
||||
if (rand(0, 10) == 0) {
|
||||
$file->Created = '2010-01-01 00:00:00';
|
||||
$file->Title = '[old] ' . $file->Title;
|
||||
}
|
||||
}
|
||||
|
||||
$file->write();
|
||||
|
||||
echo " Created File: '$file->Title'\n";
|
||||
if ($doRandomlyPublish) {
|
||||
if (rand(0, 1) == 0) {
|
||||
$file->publishFile();
|
||||
}
|
||||
} else {
|
||||
// publish files that should be viewable
|
||||
if ($file->canView($this->anonymousMember)) {
|
||||
$url = $file->getAbsoluteURL();
|
||||
$file->publishFile();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($depth < $this->depth) {
|
||||
$this->generateFiles($fixtureFilePaths, $depth+1, "{$prefix}-{$i}", $folder->ID);
|
||||
if ($depth < self::config()->get('depth') - 1) {
|
||||
$this->generateFiles($fixtureFilePaths, $depth + 1, "{$prefix}-{$i}", $folder->ID);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected function putProtectedFilesInPublicAssetStore()
|
||||
{
|
||||
/** @var File $file */
|
||||
foreach (File::get()->exclude(['ClassName' => Folder::class]) as $file) {
|
||||
// file is already in public asset store
|
||||
if ($file->canView($this->anonymousMember)) {
|
||||
continue;
|
||||
}
|
||||
// randomly move 50% of the files that should be in the protected store to the public store
|
||||
if (rand(0, 1) == 0) {
|
||||
continue;
|
||||
}
|
||||
// this will move the file into the public asset store, even it it should be protected
|
||||
// i.e. the parent folder CanViewType = OnlyTheseUsers
|
||||
$file->publishFile();
|
||||
$url = $file->getAbsoluteURL();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue