Compare commits

...

28 Commits
3.3.0 ... 3

Author SHA1 Message Date
Guy Sartorelli
e04501cb52
Merge branch '3.5' into 3 2023-04-26 12:49:10 +12:00
Maxime Rainville
821d2858f3
Merge pull request #81 from creative-commoners/pulls/3.5/fix-constraints
MNT Revert erroneous dependency changes
2023-03-28 17:06:30 +13:00
Guy Sartorelli
de215d63f6
MNT Revert erroneous dependency changes 2023-03-28 14:55:48 +13:00
Maxime Rainville
2a260607ec
Merge pull request #80 from creative-commoners/pulls/3/dispatch-ci
MNT Use gha-dispatch-ci
2023-03-23 14:19:29 +13:00
Steve Boyd
6a92eb58e2 MNT Use gha-dispatch-ci 2023-03-21 13:42:30 +13:00
Guy Sartorelli
87869e94a6
MNT Update development dependencies 2023-03-10 16:38:07 +13:00
Guy Sartorelli
61f443d49c
MNT Update release dependencies 2023-03-10 16:38:04 +13:00
Guy Sartorelli
8f2e1d9b75
MNT Update development dependencies 2023-03-10 12:21:32 +13:00
Sabina Talipova
a281114ed2
Merge pull request #77 from creative-commoners/pulls/3/stop-using-depr
API Stop using deprecated API
2022-12-05 16:44:21 +13:00
Steve Boyd
1a0cd6d6a6 API Stop using deprecated API 2022-11-28 19:20:01 +13:00
Steve Boyd
3bfa989a7e Merge branch '3.4' into 3 2022-08-02 19:01:21 +12:00
Steve Boyd
46b7f51040 Merge branch '3.3' into 3.4 2022-08-02 19:01:18 +12:00
Guy Sartorelli
041296bda2
Merge pull request #76 from creative-commoners/pulls/3.3/standardise-modules
MNT Standardise modules
2022-08-02 15:49:18 +12:00
Steve Boyd
e8997870c5 MNT Standardise modules 2022-08-01 16:23:36 +12:00
Steve Boyd
e8061724c5 Merge branch '3.4' into 3 2022-07-25 09:43:40 +12:00
Steve Boyd
db8a36fa3e Merge branch '3.3' into 3.4 2022-07-25 09:43:36 +12:00
Guy Sartorelli
04ff0c6084
Merge pull request #74 from creative-commoners/pulls/3.3/module-standards
MNT Use GitHub Actions CI
2022-07-15 17:10:18 +12:00
Steve Boyd
e5bf4f1322 MNT Use GitHub Actions CI 2022-07-05 19:08:18 +12:00
Guy Sartorelli
4674084d0d
Merge pull request #72 from creative-commoners/pulls/3/php81
ENH PHP 8.1 compatibility
2022-04-26 17:58:43 +12:00
Steve Boyd
df8b17ab85 ENH PHP 8.1 compatibility 2022-04-13 13:51:04 +12:00
Michal Kleiner
77fecc4c53
Merge pull request #71 from GuySartorelli/patch-1
DOCS Fix incorrect PHPDoc about what null lifetime means.
2022-03-10 07:38:08 +13:00
Guy Sartorelli
d03a9f06e2
DOCS Fix incorrect PHPDoc about what null lifetime means. 2022-03-09 16:03:03 +13:00
Michal Kleiner
88e7f27c5c
Merge pull request #69 from GuySartorelli/patch-1
DOCS Fix class reference for cache class
2022-03-07 11:31:11 +13:00
Guy Sartorelli
04e4b60435
DOCS Fix class reference for cache class
The `lifetime` config variable is on the `Cache` class, not the `Database` class.
2022-03-07 11:20:01 +13:00
Maxime Rainville
25d8a55058
Merge pull request #68 from creative-commoners/pulls/3/php74
DEP Set PHP 7.4 as the minimum version
2022-02-18 22:10:02 +13:00
Steve Boyd
e8f015ddd2 DEP Set PHP 7.4 as the minimum version 2022-02-10 17:41:01 +13:00
Michal Kleiner
254c4e31f8
Merge pull request #67 from GuySartorelli/patch-1
DEP Loosen constraints for guzzlehttp/guzzle
2022-01-26 22:08:47 +13:00
GuySartorelli
7ad3fc9f13
DEP Loosen constraints for guzzlehttp/guzzle 2022-01-26 12:58:30 +13:00
17 changed files with 90 additions and 64 deletions

11
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,11 @@
name: CI
on:
push:
pull_request:
workflow_dispatch:
jobs:
ci:
name: CI
uses: silverstripe/gha-ci/.github/workflows/ci.yml@v1

16
.github/workflows/dispatch-ci.yml vendored Normal file
View File

@ -0,0 +1,16 @@
name: Dispatch CI
on:
# At 12:20 PM UTC, only on Saturday and Sunday
schedule:
- cron: '20 12 * * 6,0'
jobs:
dispatch-ci:
name: Dispatch CI
# Only run cron on the silverstripe account
if: (github.event_name == 'schedule' && github.repository_owner == 'silverstripe') || (github.event_name != 'schedule')
runs-on: ubuntu-latest
steps:
- name: Dispatch CI
uses: silverstripe/gha-dispatch-ci@v1

17
.github/workflows/keepalive.yml vendored Normal file
View File

@ -0,0 +1,17 @@
name: Keepalive
on:
workflow_dispatch:
# The 4th of every month at 10:50am UTC
schedule:
- cron: '50 10 4 * *'
jobs:
keepalive:
name: Keepalive
# Only run cron on the silverstripe account
if: (github.event_name == 'schedule' && github.repository_owner == 'silverstripe') || (github.event_name != 'schedule')
runs-on: ubuntu-latest
steps:
- name: Keepalive
uses: silverstripe/gha-keepalive@v1

View File

@ -1,13 +0,0 @@
inherit: true
checks:
php: true
build:
nodes:
analysis:
tests:
override: [php-scrutinizer-run]
filter:
paths: [src/*, tests/*]

View File

@ -1,4 +0,0 @@
version: ~> 1.0
import:
- silverstripe/silverstripe-travis-shared:config/provision/standard-jobs-range.yml

View File

@ -1,9 +1,7 @@
# Text extraction module # Text extraction module
[![Build Status](https://api.travis-ci.com/silverstripe/silverstripe-textextraction.svg?branch=3)](https://travis-ci.com/silverstripe/silverstripe-textextraction) [![CI](https://github.com/silverstripe/silverstripe-textextraction/actions/workflows/ci.yml/badge.svg)](https://github.com/silverstripe/silverstripe-textextraction/actions/workflows/ci.yml)
[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/silverstripe/silverstripe-textextraction/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/silverstripe/silverstripe-textextraction/?branch=master) [![Silverstripe supported module](https://img.shields.io/badge/silverstripe-supported-0071C4.svg)](https://www.silverstripe.org/software/addons/silverstripe-commercially-supported-module-list/)
[![codecov](https://codecov.io/gh/silverstripe/silverstripe-textextraction/branch/master/graph/badge.svg)](https://codecov.io/gh/silverstripe/silverstripe-textextraction)
[![SilverStripe supported module](https://img.shields.io/badge/silverstripe-supported-0071C4.svg)](https://www.silverstripe.org/software/addons/silverstripe-commercially-supported-module-list/)
Provides a text extraction API for file content, that can hook into different extractor Provides a text extraction API for file content, that can hook into different extractor
engines based on availability and the parsed file format. The output returned is always a string of the file content. engines based on availability and the parsed file format. The output returned is always a string of the file content.
@ -24,7 +22,7 @@ The module supports text extraction on the following file formats:
## Requirements ## Requirements
* SilverStripe ^4.0 * Silverstripe ^4.0
* (optional) [XPDF](http://www.foolabs.com/xpdf/) (`pdftotext` utility) * (optional) [XPDF](http://www.foolabs.com/xpdf/) (`pdftotext` utility)
* (optional) [Apache Solr with ExtracingRequestHandler](http://wiki.apache.org/solr/ExtractingRequestHandler) * (optional) [Apache Solr with ExtracingRequestHandler](http://wiki.apache.org/solr/ExtractingRequestHandler)
* (optional) [Apache Tika](http://tika.apache.org/) * (optional) [Apache Tika](http://tika.apache.org/)
@ -54,8 +52,8 @@ If the issue does look like a new bug:
- Create a new issue - Create a new issue
- Describe the steps required to reproduce your issue, and the expected outcome. Unit tests, screenshots - Describe the steps required to reproduce your issue, and the expected outcome. Unit tests, screenshots
and screencasts can help here. and screencasts can help here.
- Describe your environment as detailed as possible: SilverStripe version, Browser, PHP version, - Describe your environment as detailed as possible: Silverstripe version, Browser, PHP version,
Operating System, any installed SilverStripe modules. Operating System, any installed Silverstripe modules.
Please report security issues to security@silverstripe.org directly. Please don't file security issues in the bugtracker. Please report security issues to security@silverstripe.org directly. Please don't file security issues in the bugtracker.

View File

@ -20,11 +20,11 @@
} }
], ],
"require": { "require": {
"php": "^7.3 || ^8.0", "php": "^7.4 || ^8.0",
"silverstripe/framework": "^4.10", "silverstripe/framework": "^4.10",
"silverstripe/assets": "^1", "silverstripe/assets": "^1",
"silverstripe/versioned": "^1", "silverstripe/versioned": "^1",
"guzzlehttp/guzzle": "~6.3.0" "guzzlehttp/guzzle": "^6.3 || ^7.0"
}, },
"require-dev": { "require-dev": {
"squizlabs/php_codesniffer": "^3", "squizlabs/php_codesniffer": "^3",

View File

@ -30,7 +30,7 @@ SilverStripe\Core\Injector\Injector:
SilverStripe\TextExtraction\Cache\FileTextCache: SilverStripe\TextExtraction\Cache\FileTextCache:
class: SilverStripe\TextExtraction\Cache\FileTextCache\Cache class: SilverStripe\TextExtraction\Cache\FileTextCache\Cache
SilverStripe\TextExtraction\Cache\FileTextCache\Database: SilverStripe\TextExtraction\Cache\FileTextCache\Cache:
lifetime: 3600 # Number of seconds to cache content for lifetime: 3600 # Number of seconds to cache content for
``` ```

View File

@ -1,3 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<phpunit bootstrap="vendor/silverstripe/framework/tests/bootstrap.php" colors="true"> <phpunit bootstrap="vendor/silverstripe/framework/tests/bootstrap.php" colors="true">
<testsuites> <testsuites>
<testsuite name="Default"> <testsuite name="Default">

View File

@ -18,7 +18,7 @@ class Cache implements FileTextCache, Flushable
/** /**
* Lifetime of cache in seconds * Lifetime of cache in seconds
* Null is indefinite * Null defaults to 3600 (1 hour)
* *
* @var int|null * @var int|null
* @config * @config
@ -42,7 +42,7 @@ class Cache implements FileTextCache, Flushable
*/ */
protected function getKey(File $file) protected function getKey(File $file)
{ {
return md5($file->getFilename()); return md5($file->getFilename() ?? '');
} }
/** /**

View File

@ -58,7 +58,7 @@ abstract class FileTextExtractor
arsort($classPriorities); arsort($classPriorities);
// Save classes // Save classes
$sortedClasses = array_keys($classPriorities); $sortedClasses = array_keys($classPriorities ?? []);
return self::$sorted_extractor_classes = $sortedClasses; return self::$sorted_extractor_classes = $sortedClasses;
} }
@ -81,7 +81,7 @@ abstract class FileTextExtractor
*/ */
public static function for_file($file) public static function for_file($file)
{ {
if (!$file || (is_string($file) && !file_exists($file))) { if (!$file || (is_string($file) && !file_exists($file ?? ''))) {
return null; return null;
} }
@ -137,11 +137,11 @@ abstract class FileTextExtractor
} }
// Remove any existing temp files with this name // Remove any existing temp files with this name
if (file_exists($path)) { if (file_exists($path ?? '')) {
unlink($path); unlink($path ?? '');
} }
$bytesWritten = file_put_contents($path, $file->getStream()); $bytesWritten = file_put_contents($path ?? '', $file->getStream());
if (false === $bytesWritten) { if (false === $bytesWritten) {
throw new Exception(static::class . '->getPathFromFile() failed to write temporary file'); throw new Exception(static::class . '->getPathFromFile() failed to write temporary file');
} }

View File

@ -34,7 +34,7 @@ class HTMLTextExtractor extends FileTextExtractor
*/ */
public function supportsExtension($extension) public function supportsExtension($extension)
{ {
return in_array(strtolower($extension), ["html", "htm", "xhtml"]); return in_array(strtolower($extension ?? ''), ["html", "htm", "xhtml"]);
} }
/** /**
@ -43,7 +43,7 @@ class HTMLTextExtractor extends FileTextExtractor
*/ */
public function supportsMime($mime) public function supportsMime($mime)
{ {
return strtolower($mime) === 'text/html'; return strtolower($mime ?? '') === 'text/html';
} }
/** /**
@ -56,7 +56,7 @@ class HTMLTextExtractor extends FileTextExtractor
*/ */
public function getContent($file) public function getContent($file)
{ {
$content = $file instanceof File ? $file->getString() : file_get_contents($file); $content = $file instanceof File ? $file->getString() : file_get_contents($file ?? '');
// Yes, yes, regex'ing HTML is evil. // Yes, yes, regex'ing HTML is evil.
// Since we don't care about well-formedness or markup here, it does the job. // Since we don't care about well-formedness or markup here, it does the job.
@ -82,9 +82,9 @@ class HTMLTextExtractor extends FileTextExtractor
'@</?((frameset)|(frame)|(iframe))@iu', '@</?((frameset)|(frame)|(iframe))@iu',
], ],
[' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', "$0", "$0", "$0", "$0", "$0", "$0", "$0", "$0"], [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', "$0", "$0", "$0", "$0", "$0", "$0", "$0", "$0"],
$content $content ?? ''
); );
return strip_tags($content); return strip_tags($content ?? '');
} }
} }

View File

@ -33,18 +33,18 @@ class PDFTextExtractor extends FileTextExtractor
public function isAvailable() public function isAvailable()
{ {
$bin = $this->bin('pdftotext'); $bin = $this->bin('pdftotext');
return $bin && file_exists($bin) && is_executable($bin); return $bin && file_exists($bin ?? '') && is_executable($bin ?? '');
} }
public function supportsExtension($extension) public function supportsExtension($extension)
{ {
return strtolower($extension) === 'pdf'; return strtolower($extension ?? '') === 'pdf';
} }
public function supportsMime($mime) public function supportsMime($mime)
{ {
return in_array( return in_array(
strtolower($mime), strtolower($mime ?? ''),
[ [
'application/pdf', 'application/pdf',
'application/x-pdf', 'application/x-pdf',
@ -72,7 +72,7 @@ class PDFTextExtractor extends FileTextExtractor
// Find program in each path // Find program in each path
foreach ($locations as $location) { foreach ($locations as $location) {
$path = "{$location}/{$program}"; $path = "{$location}/{$program}";
if (file_exists($path)) { if (file_exists($path ?? '')) {
return $path; return $path;
} }
if (file_exists($path . '.exe')) { if (file_exists($path . '.exe')) {
@ -86,7 +86,7 @@ class PDFTextExtractor extends FileTextExtractor
public function getContent($file) public function getContent($file)
{ {
if (!$file || (is_string($file) && !file_exists($file))) { if (!$file || (is_string($file) && !file_exists($file ?? ''))) {
// no file // no file
return ''; return '';
} }
@ -108,7 +108,7 @@ class PDFTextExtractor extends FileTextExtractor
} }
$path = $file instanceof File ? $this->getPathFromFile($file) : $file; $path = $file instanceof File ? $this->getPathFromFile($file) : $file;
exec(sprintf('%s %s - 2>&1', $this->bin('pdftotext'), escapeshellarg($path)), $content, $err); exec(sprintf('%s %s - 2>&1', $this->bin('pdftotext'), escapeshellarg($path ?? '')), $content, $err);
if ($err) { if ($err) {
throw new Exception(sprintf( throw new Exception(sprintf(
@ -141,6 +141,6 @@ class PDFTextExtractor extends FileTextExtractor
'st' => 'st' 'st' => 'st'
]; ];
return str_replace(array_keys($mapping), array_values($mapping), $input); return str_replace(array_keys($mapping ?? []), array_values($mapping ?? []), $input ?? '');
} }
} }

View File

@ -80,7 +80,7 @@ class SolrCellTextExtractor extends FileTextExtractor
public function supportsExtension($extension) public function supportsExtension($extension)
{ {
return in_array( return in_array(
strtolower($extension), strtolower($extension ?? ''),
[ [
'pdf', 'doc', 'docx', 'xls', 'xlsx', 'pdf', 'doc', 'docx', 'xls', 'xlsx',
'epub', 'rtf', 'odt', 'fodt', 'ods', 'fods', 'epub', 'rtf', 'odt', 'fodt', 'ods', 'fods',
@ -106,12 +106,12 @@ class SolrCellTextExtractor extends FileTextExtractor
*/ */
public function getContent($file) public function getContent($file)
{ {
if (!$file || (is_string($file) && !file_exists($file))) { if (!$file || (is_string($file) && !file_exists($file ?? ''))) {
// no file // no file
return ''; return '';
} }
$fileName = $file instanceof File ? $file->getFilename() : basename($file); $fileName = $file instanceof File ? $file->getFilename() : basename($file ?? '');
$client = $this->getHttpClient(); $client = $this->getHttpClient();
// Get and validate base URL // Get and validate base URL
@ -121,7 +121,7 @@ class SolrCellTextExtractor extends FileTextExtractor
} }
try { try {
$stream = $file instanceof File ? $file->getStream() : fopen($file, 'r'); $stream = $file instanceof File ? $file->getStream() : fopen($file ?? '', 'r');
/** @var Response $response */ /** @var Response $response */
$response = $client $response = $client
->post($baseUrl, [ ->post($baseUrl, [
@ -154,7 +154,7 @@ class SolrCellTextExtractor extends FileTextExtractor
$matches = []; $matches = [];
// Use preg match to avoid SimpleXML running out of memory on large text nodes // Use preg match to avoid SimpleXML running out of memory on large text nodes
preg_match( preg_match(
sprintf('/\<str name\="%s"\>(.*?)\<\/str\>/s', preg_quote($fileName)), sprintf('/\<str name\="%s"\>(.*?)\<\/str\>/s', preg_quote($fileName ?? '')),
(string)$response->getBody(), (string)$response->getBody(),
$matches $matches
); );

View File

@ -86,7 +86,7 @@ class TikaServerTextExtractor extends FileTextExtractor
{ {
return $this->getServerEndpoint() return $this->getServerEndpoint()
&& $this->getClient()->isAvailable() && $this->getClient()->isAvailable()
&& version_compare($this->getVersion(), '1.7') >= 0; && version_compare($this->getVersion() ?? '', '1.7') >= 0;
} }
/** /**
@ -116,7 +116,7 @@ class TikaServerTextExtractor extends FileTextExtractor
// Check aliases // Check aliases
foreach ($this->supportedMimes as $info) { foreach ($this->supportedMimes as $info) {
if (isset($info['alias']) && in_array($mime, $info['alias'])) { if (isset($info['alias']) && in_array($mime, $info['alias'] ?? [])) {
return true; return true;
} }
} }
@ -130,7 +130,7 @@ class TikaServerTextExtractor extends FileTextExtractor
$content = $this->getClient()->tika($tempFile); $content = $this->getClient()->tika($tempFile);
//Cleanup temp file //Cleanup temp file
if ($file instanceof File) { if ($file instanceof File) {
unlink($tempFile); unlink($tempFile ?? '');
} }
return $content; return $content;
} }

View File

@ -29,7 +29,7 @@ class TikaTextExtractor extends FileTextExtractor
$code = $this->runShell('tika --version', $stdout); $code = $this->runShell('tika --version', $stdout);
// Parse output // Parse output
if (!$code && preg_match('/Apache Tika (?<version>[\.\d]+)/', $stdout, $matches)) { if (!$code && preg_match('/Apache Tika (?<version>[\.\d]+)/', $stdout ?? '', $matches)) {
return $matches['version']; return $matches['version'];
} }
@ -54,14 +54,14 @@ class TikaTextExtractor extends FileTextExtractor
]; ];
// Invoke command // Invoke command
$pipes = []; $pipes = [];
$proc = proc_open($command, $descriptorSpecs, $pipes); $proc = proc_open($command ?? '', $descriptorSpecs ?? [], $pipes);
if (!is_resource($proc)) { if (!is_resource($proc)) {
return 255; return 255;
} }
// Send content as input // Send content as input
fwrite($pipes[0], $input); fwrite($pipes[0], $input ?? '');
fclose($pipes[0]); fclose($pipes[0]);
// Get output // Get output
@ -78,11 +78,11 @@ class TikaTextExtractor extends FileTextExtractor
{ {
$mode = $this->config()->get('output_mode'); $mode = $this->config()->get('output_mode');
$path = $file instanceof File ? $this->getPathFromFile($file) : $file; $path = $file instanceof File ? $this->getPathFromFile($file) : $file;
$command = sprintf('tika %s %s', $mode, escapeshellarg($path)); $command = sprintf('tika %s %s', $mode, escapeshellarg($path ?? ''));
$code = $this->runShell($command, $output); $code = $this->runShell($command, $output);
//Cleanup temp file //Cleanup temp file
if ($file instanceof File) { if ($file instanceof File) {
unlink($path); unlink($path ?? '');
} }
if ($code == 0) { if ($code == 0) {
@ -123,8 +123,8 @@ class TikaTextExtractor extends FileTextExtractor
} }
// Check if the mime type is inside the result // Check if the mime type is inside the result
$pattern = sprintf('/\b(%s)\b/', preg_quote($mime, '/')); $pattern = sprintf('/\b(%s)\b/', preg_quote($mime ?? '', '/'));
return (bool)preg_match($pattern, $supportedTypes); return (bool)preg_match($pattern ?? '', $supportedTypes ?? '');
} }
} }

View File

@ -80,7 +80,7 @@ class TikaRestClient extends Client
// Parse output // Parse output
if ($response->getStatusCode() == 200 if ($response->getStatusCode() == 200
&& preg_match('/Apache Tika (?<version>[\.\d]+)/', $response->getBody(), $matches) && preg_match('/Apache Tika (?<version>[\.\d]+)/', $response->getBody() ?? '', $matches)
) { ) {
$version = $matches['version']; $version = $matches['version'];
} }
@ -108,7 +108,7 @@ class TikaRestClient extends Client
]) ])
); );
return $this->mimes = Convert::json2array($response->getBody()); return $this->mimes = json_decode($response->getBody(), true);
} }
/** /**
@ -129,7 +129,7 @@ class TikaRestClient extends Client
'headers' => [ 'headers' => [
'Accept' => 'text/plain', 'Accept' => 'text/plain',
], ],
'body' => file_get_contents($file), 'body' => file_get_contents($file ?? ''),
]) ])
); );
$text = $response->getBody(); $text = $response->getBody();