diff --git a/_config/asset.yml b/_config/asset.yml new file mode 100644 index 000000000..e51c01b69 --- /dev/null +++ b/_config/asset.yml @@ -0,0 +1,25 @@ +--- +Name: assetstore +--- +Injector: + # Public url plugin + FlysystemUrlPlugin: + class: 'SilverStripe\Filesystem\Flysystem\FlysystemUrlPlugin' + # Define the default adapter for this filesystem + FlysystemDefaultAdapter: + class: 'SilverStripe\Filesystem\Flysystem\AssetAdapter' + # Define the default filesystem + FlysystemBackend: + class: 'League\Flysystem\Filesystem' + constructor: + Adapter: '%$FlysystemDefaultAdapter' + calls: + PublicURLPlugin: [ addPlugin, [ %$FlysystemUrlPlugin ] ] + # Define our SS asset backend + AssetStore: + class: 'SilverStripe\Filesystem\Flysystem\FlysystemAssetStore' + properties: + Filesystem: '%$FlysystemBackend' + AssetNameGenerator: + class: SilverStripe\Filesystem\Storage\DefaultAssetNameGenerator + type: prototype diff --git a/composer.json b/composer.json index 55c4a1e07..6d4acb8e5 100644 --- a/composer.json +++ b/composer.json @@ -18,7 +18,8 @@ "require": { "php": ">=5.4.0", "composer/installers": "~1.0", - "monolog/monolog": "~1.11" + "monolog/monolog": "~1.11", + "league/flysystem": "~1.0.12" }, "require-dev": { "phpunit/PHPUnit": "~3.7" diff --git a/docs/en/02_Developer_Guides/14_Files/01_File_Management.md b/docs/en/02_Developer_Guides/14_Files/01_File_Management.md index f1d957a29..6b2cad676 100644 --- a/docs/en/02_Developer_Guides/14_Files/01_File_Management.md +++ b/docs/en/02_Developer_Guides/14_Files/01_File_Management.md @@ -2,9 +2,125 @@ summary: Learn how to work with File and Image records # File Management -## Files, Images and Folders as database records +## Storage via database columns -All files, images and folders in the 'assets' directory are stored in the database. Each record has the following database fields: +Asset storage is provided out of the box via a [Flysystem](http://flysystem.thephpleague.com/) backed store. +However, any class that implements the `AssetStore` interface could be substituted to provide storage backends +via other mechanisms. + +Internally, files are stored as `[api:DBFile]` records on the rows of parent objects. These records are composite fields +which contain sufficient information useful to the configured asset backend in order to store, manage, and +publish files. By default this composite field behind this field stores the following details: + + +| Field name | Description | +| ---------- | ----------- +| `Hash` | The sha1 of the file content, useful for versioning (if supported by the backend) | +| `Filename` | The internal identifier for this file, which may contain a directory path (not including assets). Multiple versions of the same file will have the same filename. | +| `Variant` | The variant for this file. If a file has multiple derived versions (such as resized files or reformatted documents) then you can point to one of the variants here. | + + +Note that the `Hash` and `Filename` always point to the original file, if a `Variant` is specified. It is up to the +storage backend to determine how variants are managed. + +Note that the storage backend used will not be automatically synchronised with the database. Only files which +are loaded into the backend through the asset API will be available for use within a site. + +## Compatibility with 3.x filename paths + +If upgrading to 4.0 from earlier versions when using the default asset store, it's important to consider +how existing files will be migrated. + +Because the filesystem now uses the sha1 of file contents in order to version multiple versions under the same +filename, the default storage paths in 4.0 will not be the same as in 3. + +In order to retain existing file paths in line with framework version 3 you should set the +`\SilverStripe\Filesystem\Flysystem\FlysystemAssetStore.legacy_paths` config to true. +Note that this will not allow you to utilise certain file versioning features in 4.0. + + :::yaml + \SilverStripe\Filesystem\Flysystem\FlysystemAssetStore: + legacy_paths: true + +## Loading content into `DBFile` + +A file can be written to the backend from a file which exists on the local filesystem (but not necessarily +within the assets folder). + +For example, to load a temporary file into a DataObject you could use the below: + + + :::php + 'DBFile' + ); + } + + // Image could be assigned in other parts of the code using the below + $banner = new Banner(); + $banner->Image->setFromLocalFile($tempfile['path'], 'uploads/banner-file.jpg'); + + +When uploading a file it's normally necessary to give the file a useful name and directory, otherwise the +asset storage backend will choose one for you. + +Alternatively, content can be loaded using one of the below methods: + + +| Method | Description | +| -------------------------- | --------------------------------------- | +| `DBFile::setFromLocalFile` | Load a local file into the asset store | +| `DBFile::setFromStream` | Will store content from a stream | +| `DBFile::setFromString` | Will store content from a binary string | + + +The result of these methods will be an array of data (tuple) which contains the values of the above fields +(Filename, Hash, and Variant). + +## Conflict resolution + +When storing files, it's possible to determine the mechanism the backend should use when it encounters +an existing file pattern. The conflict resolution to use can be passed into the third parameter of the +above methods (after content and filename). The available constants are: + +| Constant | If an existing file is found then: | +| ----------------------------------- | ----------------------------------- | +| `AssetStore::CONFLICT_EXCEPTION` | An exception will be thrown | +| `AssetStore::CONFLICT_OVERWRITE` | The existing file will be replaced | +| `AssetStore::CONFLICT_RENAME` | The backend will choose a new name. | +| `AssetStore::CONFLICT_USE_EXISTING` | The existing file will be used | + + +If no conflict resolution scheme is chosen, or an unsupported one is requested, then the backend will choose one. +The default asset store supports each of these. + +## Getting content from a `DBFile` + +When placed into a template (e.g. `$MyFileField`) then `[api:DBFile]` will automatically generate the appropriate +template for the file mime type. For images, this will embed an image tag. For documents a download link will be presented. + +As with storage, there are also different ways of loading the content (or properties) of the file: + +| Method | Description | +| ------------------------ | ---------------------------------------------------------- | +| `DBFile::getStream` | Will get an output stream of the file content | +| `DBFile::getString` | Gets the binary content | +| `DBFile::getURL` | Gets the url for this resource. May or may not be absolute | +| `DBFile::getAbsoluteURL` | Gets the absolute URL to this resource | +| `DBFile::getMimeType` | Get the mime type of this file | +| `DBFile::getMetaData` | Gets other metadata from the file as an array | + + +As with other db field types, `DBField` can also be subclassed or extended to add additional properties +(such as SEO related fields). + +## Storage via `File` DataObject + +Other than files stored exclusively via DBFile, files can also exist as subclasses of the `File` DataObject. + +Each record has the following database fields: | Field name | Description | | ---------- | ----------- | @@ -17,6 +133,7 @@ All files, images and folders in the 'assets' directory are stored in the databa | `ParentID` | The ID of the parent Folder that this File/Folder is in. A ParentID of '0' indicates that the File/Folder is in the 'assets' directory. | | `OwnerID` | The ID of the Member that 'owns' the File/Folder (not related to filesystem permissions). | + ## Management through the "Files" section of the CMS If you have the CMS module installed, you can manage files, folders and images in the "Files" section of the CMS. Inside this section, you will see a list of files and folders like below: @@ -37,4 +154,4 @@ You may also notice the 'Sync files' button (highlighted below). This button all ## Upload -Files can be managed through a `FileField` or an `UploadField`. The `[api:FileField]` class provides a simple HTML input with a type of "file", whereas an `[api:UploadField]` provides a much more feature-rich field (including AJAX-based uploads, previews, relationship management and file data management). See [`Reference - UploadField`](/developer_guides/forms/field_types/uploadfield) for more information about how to use the `UploadField` class. \ No newline at end of file +Files can be managed through a `FileField` or an `UploadField`. The `[api:FileField]` class provides a simple HTML input with a type of "file", whereas an `[api:UploadField]` provides a much more feature-rich field (including AJAX-based uploads, previews, relationship management and file data management). See [`Reference - UploadField`](/developer_guides/forms/field_types/uploadfield) for more information about how to use the `UploadField` class. diff --git a/docs/en/04_Changelogs/4.0.0.md b/docs/en/04_Changelogs/4.0.0.md index 781be09e3..9b6baf23b 100644 --- a/docs/en/04_Changelogs/4.0.0.md +++ b/docs/en/04_Changelogs/4.0.0.md @@ -12,6 +12,27 @@ ## Upgrading +### New asset storage mechanism + +File system has been abstracted into an abstract interface. By default, the out of the box filesystem +uses [Flysystem](http://flysystem.thephpleague.com/) with a local storage mechanism (under the assets directory). + +Because the filesystem now uses the sha1 of file contents in order to version multiple versions under the same +filename, the default storage paths in 4.0 will not be the same as in 3. + +In order to retain existing file paths in line with framework version 3 you should set the +`\SilverStripe\Filesystem\Flysystem\FlysystemAssetStore.legacy_paths` config to true. +Note that this will not allow you to utilise certain file versioning features in 4.0. + + + :::yaml + \SilverStripe\Filesystem\Flysystem\FlysystemAssetStore: + legacy_paths: true + + +See [/developer_guides/files/file_management] for more information on how the new system works. + + ### Upgrading code that uses composite db fields. `CompositeDBField` is now an abstract class, not an interface. In many cases, custom code that handled diff --git a/filesystem/flysystem/AssetAdapter.php b/filesystem/flysystem/AssetAdapter.php new file mode 100644 index 000000000..962380d16 --- /dev/null +++ b/filesystem/flysystem/AssetAdapter.php @@ -0,0 +1,39 @@ +pathPrefix); + + if(stripos($filesPath, $rootPath) === 0) { + $dir = substr($filesPath, strlen($rootPath)); + return Controller::join_links(Director::baseURL(), $dir, $path); + } + + // File outside of webroot can't be used + return null; + } +} diff --git a/filesystem/flysystem/FlysystemAssetStore.php b/filesystem/flysystem/FlysystemAssetStore.php new file mode 100644 index 000000000..dd9239837 --- /dev/null +++ b/filesystem/flysystem/FlysystemAssetStore.php @@ -0,0 +1,371 @@ +filesystem = $filesystem; + return $this; + } + + /** + * Get the currently assigned flysystem backend + * + * @return Filesystem + */ + public function getFilesystem() { + return $this->filesystem; + } + + public function getAsStream($hash, $filename, $variant = null) { + $fileID = $this->getFileID($hash, $filename, $variant); + return $this->getFilesystem()->readStream($fileID); + } + + public function getAsString($hash, $filename, $variant = null) { + $fileID = $this->getFileID($hash, $filename, $variant); + return $this->getFilesystem()->read($fileID); + } + + public function getAsURL($hash, $filename, $variant = null) { + $fileID = $this->getFileID($hash, $filename, $variant); + return $this->getFilesystem()->getPublicUrl($fileID); + } + + public function setFromLocalFile($path, $filename = null, $conflictResolution = null) { + // Validate this file exists + if(!file_exists($path)) { + throw new InvalidArgumentException("$path does not exist"); + } + + // Get filename to save to + if(empty($filename)) { + $filename = basename($path); + } + + // Callback for saving content + $filesystem = $this->getFilesystem(); + $callback = function($fileID) use ($filesystem, $path) { + // Read contents as string into flysystem + $handle = fopen($path, 'r'); + if($handle === false) { + throw new InvalidArgumentException("$path could not be opened for reading"); + } + $result = $filesystem->putStream($fileID, $handle); + fclose($handle); + return $result; + }; + + // Submit to conflict check + $hash = sha1_file($path); + return $this->writeWithCallback($callback, $hash, $filename, $conflictResolution); + } + + public function setFromString($data, $filename, $conflictResolution = null) { + // Callback for saving content + $filesystem = $this->getFilesystem(); + $callback = function($fileID) use ($filesystem, $data) { + return $filesystem->put($fileID, $data); + }; + + // Submit to conflict check + $hash = sha1($data); + return $this->writeWithCallback($callback, $hash, $filename, $conflictResolution); + } + + public function setFromStream($stream, $filename, $conflictResolution = null) { + // If the stream isn't rewindable, write to a temporary filename + if(!$this->isSeekableStream($stream)) { + $path = $this->getStreamAsFile($stream); + $result = $this->setFromLocalFile($path, $filename, $conflictResolution); + unlink($path); + return $result; + } + + // Callback for saving content + $filesystem = $this->getFilesystem(); + $callback = function($fileID) use ($filesystem, $stream) { + return $filesystem->putStream($fileID, $stream); + }; + + // Submit to conflict check + $hash = $this->getStreamSHA1($stream); + return $this->writeWithCallback($callback, $hash, $filename, $conflictResolution); + } + + /** + * get sha1 hash from stream + * + * @param resource $stream + * @return string str1 hash + */ + protected function getStreamSHA1($stream) { + Util::rewindStream($stream); + $context = hash_init('sha1'); + hash_update_stream($context, $stream); + return hash_final($context); + } + + /** + * Get stream as a file + * + * @param resource $stream + * @return string Filename of resulting stream content + */ + protected function getStreamAsFile($stream) { + // Get temporary file and name + $file = tempnam(sys_get_temp_dir(), 'ssflysystem'); + $buffer = fopen($file, 'w'); + if (!$buffer) { + throw new Exception("Could not create temporary file"); + } + + // Transfer from given stream + Util::rewindStream($stream); + stream_copy_to_stream($stream, $buffer); + if (! fclose($buffer)) { + throw new Exception("Could not write stream to temporary file"); + } + + return $file; + } + + /** + * Determine if this stream is seekable + * + * @param resource $stream + * @return bool True if this stream is seekable + */ + protected function isSeekableStream($stream) { + return Util::isSeekableStream($stream); + } + + /** + * Invokes the conflict resolution scheme on the given content, and invokes a callback if + * the storage request is approved. + * + * @param callable $callback Will be invoked and passed a fileID if the file should be stored + * @param string $hash SHA1 of the file content + * @param string $filename Name for the resulting file + * @param string $conflictResolution {@see AssetStore}. Will default to one chosen by the backend + * @return array Tuple associative array (Filename, Hash, Variant) + * @throws Exception + */ + protected function writeWithCallback($callback, $hash, $filename, $conflictResolution = null) { + $filename = $this->cleanFilename($filename); + $fileID = $this->getFileID($hash, $filename); + + // Check conflict resolution scheme + $resolvedID = $this->resolveConflicts($conflictResolution, $fileID); + if($resolvedID === false) { + // If defering to the existing file, return the sha of the existing file + $stream = $this + ->getFilesystem() + ->readStream($fileID); + $hash = $this->getStreamSHA1($stream); + } else { + // Submit and validate result + $result = $callback($resolvedID); + if(!$result) { + throw new Exception("Could not save {$filename}"); + } + + // in case conflict resolution renamed the file, return the renamed + $filename = $this->getOriginalFilename($resolvedID); + } + + return array( + 'Hash' => $hash, + 'Filename' => $filename, + 'Variant' => '' + ); + } + + public function getMetadata($hash, $filename, $variant = null) { + $fileID = $this->getFileID($hash, $filename, $variant); + return $this->getFilesystem()->getMetadata($fileID); + } + + public function getMimeType($hash, $filename, $variant = null) { + $fileID = $this->getFileID($hash, $filename, $variant); + return $this->getFilesystem()->getMimetype($fileID); + } + + /** + * Determine the path that should be written to, given the conflict resolution scheme + * + * @param string $conflictResolution + * @param string $fileID + * @return string|false Safe filename to write to. If false, then don't write. + * @throws Exception + */ + protected function resolveConflicts($conflictResolution, $fileID) { + // If overwrite is requested, simply put + if($conflictResolution === AssetStore::CONFLICT_OVERWRITE) { + return $fileID; + } + + // Otherwise, check if this exists + $exists = $this->getFilesystem()->has($fileID); + if(!$exists) { + return $fileID; + } + + // Flysystem defaults to use_existing + switch($conflictResolution) { + // Throw tantrum + case AssetStore::CONFLICT_EXCEPTION: { + throw new \InvalidArgumentException("File already exists at path {$fileID}"); + } + + // Rename + case AssetStore::CONFLICT_RENAME: { + foreach($this->fileGeneratorFor($fileID) as $candidate) { + // @todo better infinite loop breaking + if(!$this->getFilesystem()->has($candidate)) { + return $candidate; + } + } + + throw new \InvalidArgumentException("File could not be renamed with path {$fileID}"); + } + + // Default to use existing file + case AssetStore::CONFLICT_USE_EXISTING: + default: { + return false; + } + } + } + + /** + * Get an asset renamer for the given filename. + * + * @param string $fileID Adaptor specific identifier for this file/version + * @return AssetNameGenerator + */ + protected function fileGeneratorFor($fileID){ + return Injector::inst()->createWithArgs('AssetNameGenerator', array($fileID)); + } + + /** + * Performs filename cleanup before sending it back. + * + * This name should not contain hash or variants. + * + * @param string $filename + * @return string + */ + protected function cleanFilename($filename) { + // Since we use double underscore to delimit variants, eradicate them from filename + return preg_replace('/_{2,}/', '_', $filename); + } + + /** + * Given a FileID, map this back to the original filename, trimming variant + * + * @param string $fileID Adaptor specific identifier for this file/version + * @param string $variant Out parameter for any found variant + * @return string + */ + protected function getOriginalFilename($fileID, &$variant = '') { + // Remove variant + $original = $fileID; + $variant = ''; + if(preg_match('/^(?((?[^\\.]+)(?.*)$/', $fileID, $matches)) { + $original = $matches['before'].$matches['after']; + $variant = $matches['variant']; + } + + // Remove hash + return preg_replace( + '/(?[a-zA-Z0-9]{10}\\/)(?[^\\/]+)$/', + '$2', + $original + ); + } + + /** + * Map file tuple (hash, name, variant) to a filename to be used by flysystem + * + * The resulting file will look something like my/directory/EA775CB4D4/filename__variant.jpg + * + * @param string $hash + * @param string $filename Name of file + * @param string $variant (if given) + * @return string Adaptor specific identifier for this file/version + */ + protected function getFileID($hash, $filename, $variant = null) { + // Since we use double underscore to delimit variants, eradicate them from filename + $filename = $this->cleanFilename($filename); + $name = basename($filename); + + // Split extension + $extension = null; + if(($pos = strpos($name, '.')) !== false) { + $extension = substr($name, $pos); + $name = substr($name, 0, $pos); + } + + // Unless in legacy mode, inject hash just prior to the filename + if(Config::inst()->get(__CLASS__, 'legacy_filenames')) { + $fileID = $name; + } else { + $fileID = substr($hash, 0, 10) . '/' . $name; + } + + // Add directory + $dirname = ltrim(dirname($filename), '.'); + if($dirname) { + $fileID = $dirname . '/' . $fileID; + } + + // Add variant + if($variant) { + $fileID .= '__' . $variant; + } + + // Add extension + if($extension) { + $fileID .= $extension; + } + + return $fileID; + } + +} diff --git a/filesystem/flysystem/FlysystemUrlPlugin.php b/filesystem/flysystem/FlysystemUrlPlugin.php new file mode 100644 index 000000000..ddd6fd23e --- /dev/null +++ b/filesystem/flysystem/FlysystemUrlPlugin.php @@ -0,0 +1,73 @@ +adapter = $filesystem->getAdapter(); + } + + public function getMethod() { + return 'getPublicUrl'; + } + + /** + * Generate public url + * + * @param string $path + * @return string The full url to the file + */ + public function handle($path) { + // Default adaptor + if($this->adapter instanceof AssetAdapter) { + return $this->adapter->getPublicUrl($path); + } + + // Check S3 adaptor + if (class_exists('League\Flysystem\AwsS3v2\AwsS3Adapter') + && $this->adapter instanceof AwsS3Adapter + ) { + return sprintf( + 'https://s3.amazonaws.com/%s/%s', + $this->adapter->getBucket(), + $path + ); + } + + // Local with host + if (class_exists('Oneup\FlysystemBundle\Adapter\LocalWithHost') + && $this->adapter instanceof LocalWithHost + ) { + return sprintf( + '%s/%s/%s', + $this->adapter->getBasePath(), + $this->adapter->getWebpath(), + $path + ); + } + + // no url available + return null; + } +} \ No newline at end of file diff --git a/filesystem/storage/AssetContainer.php b/filesystem/storage/AssetContainer.php new file mode 100644 index 000000000..fd385b8c5 --- /dev/null +++ b/filesystem/storage/AssetContainer.php @@ -0,0 +1,79 @@ +filename = $filename; + $this->directory = ltrim(dirname($filename), '.'); + $name = basename($this->filename); + if(($pos = strpos($name, '.')) !== false) { + $this->extension = substr($name, $pos); + $name = substr($name, 0, $pos); + } else { + $this->extension = null; + } + + // Extract version prefix if already applied to this file + $pattern = '/^(?.+)' . preg_quote($this->getPrefix()) . '(?[0-9]+)$/'; + if(preg_match($pattern, $name, $matches)) { + $this->first = $matches['version'] + 1; + $this->name = $matches['name']; + } else { + $this->first = 1; + $this->name = $name; + } + + $this->rewind(); + } + + /** + * Get numeric prefix + * + * @return string + */ + protected function getPrefix() { + return Config::inst()->get(__CLASS__, 'version_prefix'); + } + + public function current() { + $version = $this->version; + + // Initially suggest original name + if($version === 1) { + return $this->filename; + } + + // If there are more than $this->max files we need a new scheme + if($version >= $this->max) { + $version = substr(md5(time()), 0, 10); + } + + // Build next name + $filename = $this->name . $this->getPrefix() . $version . $this->extension; + if($this->directory) { + $filename = $this->directory . DIRECTORY_SEPARATOR . $filename; + } + return $filename; + } + + public function key() { + return $this->version; + } + + public function next() { + $this->version++; + } + + public function rewind() { + $this->version = $this->first; + } + + public function valid() { + return $this->version <= $this->max; + } + +} diff --git a/model/fieldtypes/CompositeDBField.php b/model/fieldtypes/CompositeDBField.php index f26a908d5..ac74aaa0e 100644 --- a/model/fieldtypes/CompositeDBField.php +++ b/model/fieldtypes/CompositeDBField.php @@ -263,6 +263,7 @@ abstract class CompositeDBField extends DBField { return $fields[$field]; } + parent::castingHelper($field); } diff --git a/model/fieldtypes/DBFile.php b/model/fieldtypes/DBFile.php new file mode 100644 index 000000000..828206fdc --- /dev/null +++ b/model/fieldtypes/DBFile.php @@ -0,0 +1,202 @@ +get('AssetStore'); + } + + /** + * Mapping of mime patterns to templates to use + */ + private static $templates = array( + '/image\\/.+/' => 'DBFile_image', + '/.+/' => 'DBFile_download' + ); + + private static $composite_db = array( + "Hash" => "Varchar(255)", // SHA of the base content + "Filename" => "Varchar(255)", // Path identifier of the base content + "Variant" => "Varchar(255)", // Identifier of the variant to the base, if given + ); + + private static $casting = array( + 'URL' => 'Varchar', + 'AbsoluteURL' => 'Varchar', + 'Basename' => 'Varchar', + 'Title' => 'Varchar', + 'MimeType' => 'Varchar', + 'String' => 'Text', + 'Tag' => 'HTMLText' + ); + + public function scaffoldFormField($title = null, $params = null) { + // @todo - This doesn't actually work with DBFile yet + return new UploadField($this->getName(), $title); + } + + /** + * Return a html5 tag of the appropriate for this file (normally img or a) + * + * @return string + */ + public function forTemplate() { + return $this->getTag() ?: ''; + } + + /** + * Return a html5 tag of the appropriate for this file (normally img or a) + * + * @return string + */ + public function getTag() { + // Check mime type + $mime = $this->getMimeType(); + if(empty($mime)) { + return ''; + } + + // Check that path is available + $url = $this->getURL(); + if(empty($url)) { + return ''; + } + + $template = $this->getTemplateForMime($mime); + if(empty($template)) { + return ''; + } + + // Render + return (string)$this->renderWith($template); + } + + /** + * Given a mime type, determine the template to render as on the frontend + * + * @param string $mimetype + * @return string Name of template + */ + protected function getTemplateForMime($mimetype) { + foreach($this->config()->templates as $pattern => $template) { + if($pattern === $mimetype || preg_match($pattern, $mimetype)) { + return $template; + } + } + return null; + } + + /** + * Get trailing part of filename + * + * @return string + */ + public function getBasename() { + // @todo - add variant onto this ? + if($this->Filename) { + return basename($this->Filename); + } + } + + /** + * Alt title for this + * + * @return string + */ + public function getTitle() { + // @todo - better solution? + return $this->getBasename(); + } + + public function setFromLocalFile($path, $filename = null, $conflictResolution = null) { + $result = $this + ->getStore() + ->setFromLocalFile($path, $filename, $conflictResolution); + // Update from result + if($result) { + $this->setValue($result); + } + return $result; + } + + public function setFromStream($stream, $filename, $conflictResolution = null) { + $result = $this + ->getStore() + ->setFromStream($stream, $filename, $conflictResolution); + // Update from result + if($result) { + $this->setValue($result); + } + return $result; + } + + public function setFromString($data, $filename, $conflictResolution = null) { + $result = $this + ->getStore() + ->setFromString($data, $filename, $conflictResolution); + // Update from result + if($result) { + $this->setValue($result); + } + return $result; + } + + public function getStream() { + return $this + ->getStore() + ->getAsStream($this->Hash, $this->Filename, $this->Variant); + } + + public function getString() { + return $this + ->getStore() + ->getAsString($this->Hash, $this->Filename, $this->Variant); + } + + public function getURL() { + return $this + ->getStore() + ->getAsURL($this->Hash, $this->Filename, $this->Variant); + } + + /** + * Get the absolute URL to this resource + * + * @return type + */ + public function getAbsoluteURL() { + return Director::absoluteURL($this->getURL()); + } + + public function getMetaData() { + return $this + ->getStore() + ->getMetadata($this->Hash, $this->Filename, $this->Variant); + } + + public function getMimeType() { + return $this + ->getStore() + ->getMimeType($this->Hash, $this->Filename, $this->Variant); + } + + public function exists() { + return !empty($this->Filename); + } +} diff --git a/templates/DBFile_download.ss b/templates/DBFile_download.ss new file mode 100644 index 000000000..fc9c910db --- /dev/null +++ b/templates/DBFile_download.ss @@ -0,0 +1 @@ +download="$Basename.ATT"<% else %>download<% end_if %>/> diff --git a/templates/DBFile_image.ss b/templates/DBFile_image.ss new file mode 100644 index 000000000..07ccde96e --- /dev/null +++ b/templates/DBFile_image.ss @@ -0,0 +1 @@ +$Title.ATT diff --git a/tests/filesystem/AssetStoreTest.php b/tests/filesystem/AssetStoreTest.php new file mode 100644 index 000000000..8b1d4c594 --- /dev/null +++ b/tests/filesystem/AssetStoreTest.php @@ -0,0 +1,419 @@ +addPlugin(new FlysystemUrlPlugin()); + $backend = new AssetStoreTest_SpyStore(); + $backend->setFilesystem($filesystem); + Injector::inst()->registerService($backend, 'AssetStore'); + + // Disable legacy + Config::inst()->remove(get_class(new FlysystemAssetStore()), 'legacy_filenames'); + AssetStoreTest_SpyStore::$seekable_override = null; + + // Update base url + Config::inst()->update('Director', 'alternate_base_url', '/mysite/'); + } + + public function tearDown() { + SS_Filesystem::removeFolder(ASSETS_PATH . '/DBFileTest'); + AssetStoreTest_SpyStore::$seekable_override = null; + parent::tearDown(); + } + + /** + * @return AssetStore + */ + protected function getBackend() { + return Injector::inst()->get('AssetStore'); + } + + /** + * Test different storage methods + */ + public function testStorageMethods() { + $backend = $this->getBackend(); + + // Test setFromContent + $puppies1 = 'puppies'; + $puppies1Tuple = $backend->setFromString($puppies1, 'pets/my-puppy.txt'); + $this->assertEquals( + array ( + 'Hash' => '2a17a9cb4be918774e73ba83bd1c1e7d000fdd53', + 'Filename' => 'pets/my-puppy.txt', + 'Variant' => '', + ), + $puppies1Tuple + ); + + // Test setFromStream (seekable) + $fish1 = realpath(__DIR__ .'/../model/testimages/test_image_high-quality.jpg'); + $fish1Stream = fopen($fish1, 'r'); + $fish1Tuple = $backend->setFromStream($fish1Stream, 'parent/awesome-fish.jpg'); + fclose($fish1Stream); + $this->assertEquals( + array ( + 'Hash' => 'a870de278b475cb75f5d9f451439b2d378e13af1', + 'Filename' => 'parent/awesome-fish.jpg', + 'Variant' => '', + ), + $fish1Tuple + ); + + // Test with non-seekable streams + AssetStoreTest_SpyStore::$seekable_override = false; + $fish2 = realpath(__DIR__ .'/../model/testimages/test_image_low-quality.jpg'); + $fish2Stream = fopen($fish2, 'r'); + $fish2Tuple = $backend->setFromStream($fish2Stream, 'parent/mediocre-fish.jpg'); + fclose($fish2Stream); + + $this->assertEquals( + array ( + 'Hash' => '33be1b95cba0358fe54e8b13532162d52f97421c', + 'Filename' => 'parent/mediocre-fish.jpg', + 'Variant' => '', + ), + $fish2Tuple + ); + AssetStoreTest_SpyStore::$seekable_override = null; + } + + /** + * Test that the backend correctly resolves conflicts + */ + public function testConflictResolution() { + $backend = $this->getBackend(); + + // Put a file in + $fish1 = realpath(__DIR__ .'/../model/testimages/test_image_high-quality.jpg'); + $this->assertFileExists($fish1); + $fish1Tuple = $backend->setFromLocalFile($fish1, 'directory/lovely-fish.jpg'); + $this->assertEquals( + array( + 'Hash' => 'a870de278b475cb75f5d9f451439b2d378e13af1', + 'Filename' => 'directory/lovely-fish.jpg', + 'Variant' => '', + ), + $fish1Tuple + ); + $this->assertEquals( + '/mysite/assets/DBFileTest/directory/a870de278b/lovely-fish.jpg', + $backend->getAsURL($fish1Tuple['Hash'], $fish1Tuple['Filename']) + ); + + // Write a different file with same name. Should not detect duplicates since sha are different + $fish2 = realpath(__DIR__ .'/../model/testimages/test_image_low-quality.jpg'); + try { + $fish2Tuple = $backend->setFromLocalFile($fish2, 'directory/lovely-fish.jpg', AssetStore::CONFLICT_EXCEPTION); + } catch(Exception $ex) { + return $this->fail('Writing file with different sha to same location failed with exception'); + } + $this->assertEquals( + array( + 'Hash' => '33be1b95cba0358fe54e8b13532162d52f97421c', + 'Filename' => 'directory/lovely-fish.jpg', + 'Variant' => '', + ), + $fish2Tuple + ); + $this->assertEquals( + '/mysite/assets/DBFileTest/directory/33be1b95cb/lovely-fish.jpg', + $backend->getAsURL($fish2Tuple['Hash'], $fish2Tuple['Filename']) + ); + + // Write original file back with rename + $this->assertFileExists($fish1); + $fish3Tuple = $backend->setFromLocalFile($fish1, 'directory/lovely-fish.jpg', AssetStore::CONFLICT_RENAME); + $this->assertEquals( + array( + 'Hash' => 'a870de278b475cb75f5d9f451439b2d378e13af1', + 'Filename' => 'directory/lovely-fish-v2.jpg', + 'Variant' => '', + ), + $fish3Tuple + ); + $this->assertEquals( + '/mysite/assets/DBFileTest/directory/a870de278b/lovely-fish-v2.jpg', + $backend->getAsURL($fish3Tuple['Hash'], $fish3Tuple['Filename']) + ); + + // Write another file should increment to -v3 + $fish4Tuple = $backend->setFromLocalFile($fish1, 'directory/lovely-fish-v2.jpg', AssetStore::CONFLICT_RENAME); + $this->assertEquals( + array( + 'Hash' => 'a870de278b475cb75f5d9f451439b2d378e13af1', + 'Filename' => 'directory/lovely-fish-v3.jpg', + 'Variant' => '', + ), + $fish4Tuple + ); + $this->assertEquals( + '/mysite/assets/DBFileTest/directory/a870de278b/lovely-fish-v3.jpg', + $backend->getAsURL($fish4Tuple['Hash'], $fish4Tuple['Filename']) + ); + + // Test conflict use existing file + $fish5Tuple = $backend->setFromLocalFile($fish1, 'directory/lovely-fish.jpg', AssetStore::CONFLICT_USE_EXISTING); + $this->assertEquals( + array( + 'Hash' => 'a870de278b475cb75f5d9f451439b2d378e13af1', + 'Filename' => 'directory/lovely-fish.jpg', + 'Variant' => '', + ), + $fish5Tuple + ); + $this->assertEquals( + '/mysite/assets/DBFileTest/directory/a870de278b/lovely-fish.jpg', + $backend->getAsURL($fish5Tuple['Hash'], $fish5Tuple['Filename']) + ); + + // Test conflict use existing file + $fish6Tuple = $backend->setFromLocalFile($fish1, 'directory/lovely-fish.jpg', AssetStore::CONFLICT_OVERWRITE); + $this->assertEquals( + array( + 'Hash' => 'a870de278b475cb75f5d9f451439b2d378e13af1', + 'Filename' => 'directory/lovely-fish.jpg', + 'Variant' => '', + ), + $fish6Tuple + ); + $this->assertEquals( + '/mysite/assets/DBFileTest/directory/a870de278b/lovely-fish.jpg', + $backend->getAsURL($fish6Tuple['Hash'], $fish6Tuple['Filename']) + ); + } + + /** + * Test that flysystem can regenerate the original filename from fileID + */ + public function testGetOriginalFilename() { + $store = new AssetStoreTest_SpyStore(); + $this->assertEquals( + 'directory/lovely-fish.jpg', + $store->getOriginalFilename('directory/a870de278b/lovely-fish.jpg', $variant) + ); + $this->assertEmpty($variant); + $this->assertEquals( + 'directory/lovely-fish.jpg', + $store->getOriginalFilename('directory/a870de278b/lovely-fish__variant.jpg', $variant) + ); + $this->assertEquals('variant', $variant); + $this->assertEquals( + 'directory/lovely_fish.jpg', + $store->getOriginalFilename('directory/a870de278b/lovely_fish__vari_ant.jpg', $variant) + ); + $this->assertEquals('vari_ant', $variant); + $this->assertEquals( + 'directory/lovely_fish.jpg', + $store->getOriginalFilename('directory/a870de278b/lovely_fish.jpg', $variant) + ); + $this->assertEmpty($variant); + $this->assertEquals( + 'lovely-fish.jpg', + $store->getOriginalFilename('a870de278b/lovely-fish.jpg', $variant) + ); + $this->assertEmpty($variant); + $this->assertEquals( + 'lovely-fish.jpg', + $store->getOriginalFilename('a870de278b/lovely-fish__variant.jpg', $variant) + ); + $this->assertEquals('variant', $variant); + $this->assertEquals( + 'lovely_fish.jpg', + $store->getOriginalFilename('a870de278b/lovely_fish__vari__ant.jpg', $variant) + ); + $this->assertEquals('vari__ant', $variant); + $this->assertEquals( + 'lovely_fish.jpg', + $store->getOriginalFilename('a870de278b/lovely_fish.jpg', $variant) + ); + $this->assertEmpty($variant); + } + + /** + * Test internal file Id generation + */ + public function testGetFileID() { + $store = new AssetStoreTest_SpyStore(); + $this->assertEquals( + 'directory/2a17a9cb4b/file.jpg', + $store->getFileID(sha1('puppies'), 'directory/file.jpg') + ); + $this->assertEquals( + '2a17a9cb4b/file.jpg', + $store->getFileID(sha1('puppies'), 'file.jpg') + ); + $this->assertEquals( + 'dir_ectory/2a17a9cb4b/fil_e.jpg', + $store->getFileID(sha1('puppies'), 'dir__ectory/fil__e.jpg') + ); + $this->assertEquals( + 'directory/2a17a9cb4b/file_variant.jpg', + $store->getFileID(sha1('puppies'), 'directory/file__variant.jpg', null) + ); + $this->assertEquals( + 'directory/2a17a9cb4b/file__variant.jpg', + $store->getFileID(sha1('puppies'), 'directory/file.jpg', 'variant') + ); + $this->assertEquals( + '2a17a9cb4b/file__var__iant.jpg', + $store->getFileID(sha1('puppies'), 'file.jpg', 'var__iant') + ); + } + + public function testGetMetadata() { + $backend = $this->getBackend(); + + // jpg + $fish = realpath(__DIR__ .'/../model/testimages/test_image_high-quality.jpg'); + $fishTuple = $backend->setFromLocalFile($fish, 'parent/awesome-fish.jpg'); + $this->assertEquals( + 'image/jpeg', + $backend->getMimeType($fishTuple['Hash'], $fishTuple['Filename']) + ); + $fishMeta = $backend->getMetadata($fishTuple['Hash'], $fishTuple['Filename']); + $this->assertEquals(151889, $fishMeta['size']); + $this->assertEquals('file', $fishMeta['type']); + $this->assertNotEmpty($fishMeta['timestamp']); + + + // text + $puppies = 'puppies'; + $puppiesTuple = $backend->setFromString($puppies, 'pets/my-puppy.txt'); + $this->assertEquals( + 'text/plain', + $backend->getMimeType($puppiesTuple['Hash'], $puppiesTuple['Filename']) + ); + $puppiesMeta = $backend->getMetadata($puppiesTuple['Hash'], $puppiesTuple['Filename']); + $this->assertEquals(7, $puppiesMeta['size']); + $this->assertEquals('file', $puppiesMeta['type']); + $this->assertNotEmpty($puppiesMeta['timestamp']); + } + + /** + * Test that legacy filenames work as expected + */ + public function testLegacyFilenames() { + Config::inst()->update(get_class(new FlysystemAssetStore()), 'legacy_filenames', true); + + $backend = $this->getBackend(); + + // Put a file in + $fish1 = realpath(__DIR__ .'/../model/testimages/test_image_high-quality.jpg'); + $this->assertFileExists($fish1); + $fish1Tuple = $backend->setFromLocalFile($fish1, 'directory/lovely-fish.jpg'); + $this->assertEquals( + array( + 'Hash' => 'a870de278b475cb75f5d9f451439b2d378e13af1', + 'Filename' => 'directory/lovely-fish.jpg', + 'Variant' => '', + ), + $fish1Tuple + ); + $this->assertEquals( + '/mysite/assets/DBFileTest/directory/lovely-fish.jpg', + $backend->getAsURL($fish1Tuple['Hash'], $fish1Tuple['Filename']) + ); + + // Write a different file with same name. + // Since we are using legacy filenames, this should generate a new filename + $fish2 = realpath(__DIR__ .'/../model/testimages/test_image_low-quality.jpg'); + try { + $backend->setFromLocalFile($fish2, 'directory/lovely-fish.jpg', AssetStore::CONFLICT_EXCEPTION); + return $this->fail('Writing file with different sha to same location should throw exception'); + } catch(Exception $ex) { + // Success + } + + // Re-attempt this file write with conflict_rename + $fish3Tuple = $backend->setFromLocalFile($fish2, 'directory/lovely-fish.jpg', AssetStore::CONFLICT_RENAME); + $this->assertEquals( + array( + 'Hash' => '33be1b95cba0358fe54e8b13532162d52f97421c', + 'Filename' => 'directory/lovely-fish-v2.jpg', + 'Variant' => '', + ), + $fish3Tuple + ); + $this->assertEquals( + '/mysite/assets/DBFileTest/directory/lovely-fish-v2.jpg', + $backend->getAsURL($fish3Tuple['Hash'], $fish3Tuple['Filename']) + ); + + // Write back original file, but with CONFLICT_EXISTING. The file should not change + $fish4Tuple = $backend->setFromLocalFile($fish1, 'directory/lovely-fish-v2.jpg', AssetStore::CONFLICT_USE_EXISTING); + $this->assertEquals( + array( + 'Hash' => '33be1b95cba0358fe54e8b13532162d52f97421c', + 'Filename' => 'directory/lovely-fish-v2.jpg', + 'Variant' => '', + ), + $fish4Tuple + ); + $this->assertEquals( + '/mysite/assets/DBFileTest/directory/lovely-fish-v2.jpg', + $backend->getAsURL($fish4Tuple['Hash'], $fish4Tuple['Filename']) + ); + + // Write back original file with CONFLICT_OVERWRITE. The file sha should now be updated + $fish5Tuple = $backend->setFromLocalFile($fish1, 'directory/lovely-fish-v2.jpg', AssetStore::CONFLICT_OVERWRITE); + $this->assertEquals( + array( + 'Hash' => 'a870de278b475cb75f5d9f451439b2d378e13af1', + 'Filename' => 'directory/lovely-fish-v2.jpg', + 'Variant' => '', + ), + $fish5Tuple + ); + $this->assertEquals( + '/mysite/assets/DBFileTest/directory/lovely-fish-v2.jpg', + $backend->getAsURL($fish5Tuple['Hash'], $fish5Tuple['Filename']) + ); + } +} + +/** + * Spy! + */ +class AssetStoreTest_SpyStore extends FlysystemAssetStore { + + /** + * Set to true|false to override all isSeekableStream calls + * + * @var null|bool + */ + public static $seekable_override = null; + + public function cleanFilename($filename) { + return parent::cleanFilename($filename); + } + + public function getFileID($hash, $filename, $variant = null) { + return parent::getFileID($hash, $filename, $variant); + } + + public function getOriginalFilename($fileID, &$variant = '') { + return parent::getOriginalFilename($fileID, $variant); + } + + protected function isSeekableStream($stream) { + if(isset(self::$seekable_override)) { + return self::$seekable_override; + } + return parent::isSeekableStream($stream); + } +} diff --git a/tests/forms/DBFileTest.php b/tests/forms/DBFileTest.php new file mode 100644 index 000000000..213ae7f81 --- /dev/null +++ b/tests/forms/DBFileTest.php @@ -0,0 +1,88 @@ +addPlugin(new FlysystemUrlPlugin()); + $backend = new AssetStoreTest_SpyStore(); + $backend->setFilesystem($filesystem); + Injector::inst()->registerService($backend, 'AssetStore'); + + // Disable legacy + Config::inst()->remove(get_class(new FlysystemAssetStore()), 'legacy_filenames'); + + // Update base url + Config::inst()->update('Director', 'alternate_base_url', '/mysite/'); + } + + public function tearDown() { + SS_Filesystem::removeFolder(ASSETS_PATH . '/DBFileTest'); + parent::tearDown(); + } + + /** + * Test that images in a DBFile are rendered properly + */ + public function testRender() { + $obj = new DBFileTest_Object(); + + // Test image tag + $fish = realpath(__DIR__ .'/../model/testimages/test_image_high-quality.jpg'); + $this->assertFileExists($fish); + $obj->MyFile->setFromLocalFile($fish, 'awesome-fish.jpg'); + $this->assertEquals( + 'awesome-fish.jpg', + trim($obj->MyFile->forTemplate()) + ); + + // Test download tag + $obj->MyFile->setFromString('puppies', 'subdir/puppy-document.txt'); + $this->assertEquals( + '', + trim($obj->MyFile->forTemplate()) + ); + } + +} + +/** + * @property DBFile $MyFile + */ +class DBFileTest_Object extends DataObject implements TestOnly { + private static $db = array( + 'MyFile' => 'DBFile' + ); +} + + +class DBFileTest_Subclass extends DBFileTest_Object implements TestOnly { + private static $db = array( + 'AnotherFile' => 'DBFile' + ); +} + + +