alexzhu/sveden-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Исправлена обработка разметки с кодировкой windows-1251

Browse Source
This commit is contained in:
Alexander 2024-08-21 14:51:01 +03:00
parent 29eeb5bf77
commit 48f540b8af
5 changed files with 22 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +0,0 @@
.log
.yaml

21
app/app.php
View File

@ -64,12 +64,12 @@ for ($i = $start; $i < count($sites); $i++) {
$client = new Client([
'allow_directs' => true,
'track_redirects' => true,
'connect_timeout' => 300.0,
'verify' => false,
// 'http_errors' => false,
'headers' => [
'User-Agent' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 YaBrowser/24.6.0.0 Safari/537.36',
'Content-Type' => 'text/html;charset=utf-8',
'timeout' => 300
'Content-Type' => 'text/html;charset=utf-8'
]
]);
@ -109,17 +109,12 @@ for ($i = $start; $i < count($sites); $i++) {
}
unset($contingent);
}
} catch (ClientException $e) {
} catch (ClientException
| RequestException
| ConnectException
| ServerException $e
) {
$message = implode(' ', $sites[$i]) . "\t" . $e->getCode() . "\t" . $e->getMessage();
Logger::log($pathLogErrorHttp, $message);
} catch (RequestException $e) {
$message = implode(' ', $sites[$i]) . "\t" . $e->getCode() . "\t" . $e->getMessage();
Logger::log($pathLogErrorHttp, $message);
} catch (ConnectException $e) {
$message = implode(' ', $sites[$i]) . "\t" . $e->getCode() . "\t" . $e->getMessage();
Logger::log($pathLogErrorHttp, $message);
} catch (ServerException $e) {
$message = implode(' ', $sites[$i]) . "\t" . $e->getCode() . "\t" . $e->getMessage();
Logger::log($pathLogErrorHttp, $message);
}
}
}

15
app/library/ContingentParser.php
View File

@ -26,8 +26,16 @@ class ContingentParser
public function __construct(string $html, string $template)
{
libxml_use_internal_errors(true);
$dom = new \DOMDocument();
$dom->loadHTML(mb_convert_encoding($html,'HTML-ENTITIES','UTF-8'));
$dom = new \DOMDocument(
encoding: "UTF-8"
);
$html = mb_convert_encoding(
$html,
'UTF-8',
mb_detect_encoding($html, 'UTF-8, windows-1251')
);
$html = str_replace('windows-1251','utf-8', $html);
$dom->loadHTML($html);
$this->xpath = new \DOMXPath($dom);
$this->template = $template;
}
@ -52,7 +60,8 @@ class ContingentParser
{
$data = $this->parse();
$records = array();
// var_dump($data);
// exit(0);
if ($data == null) return [];
$equal = $data['eduName']->length;

2
app/library/Database.php
View File

@ -69,7 +69,7 @@ class Database
Logger::log(self::$logFile, "Запрос выполнен успешно!");
} catch (PDOException $e) {
// При ошибке запроса сохраняем валидные данные в yaml-файл
$yaml = Yaml::dump($params);
$yaml = Yaml::dump([$params]);
file_put_contents(__DIR__ . '/not-recorded-in-db.yaml', $yaml, FILE_APPEND);
$message = "Ошибка запроса:" . $e->getMessage();
Logger::log(self::$logFile, $message);

2
script.php
View File

@ -31,6 +31,6 @@ if(array_depth($data) == 2){
ContingentManager::getInstance()->insertContingent($db, $data);
} else {
foreach ($data as $value) {
// ContingentManager::getInstance()->insertContingent($db, $value);
ContingentManager::getInstance()->insertContingent($db, $value);
}
}