alexzhu/sveden-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Поменял структуру проекта для расширения задачи с приемом

Browse Source
This commit is contained in:
Alexander
2024-10-10 08:38:24 +03:00
parent 2be45826c1
commit 984f6bda0a
1716 changed files with 444 additions and 139201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/Color.php Normal file
View File

@ -0,0 +1,16 @@
<?php
namespace SvedenParser;
enum Color : string
{
case WHITE = "\033[0m";
case GREEN = "\033[92m";
case RED = "\033[91m";
case BLUE = "\033[94m";
case YELLOW = "\033[33m";
public function tostring(): string
{
return $this->value;
}
}

149
src/ContingentParser/ContingentFacade.php Normal file
View File

@ -0,0 +1,149 @@
<?php
namespace SvedenParser\ContingentParser;
use SvedenParser\Http\HttpClient;
use SvedenParser\Http\UrlBuilder;
use SvedenParser\Logger\HtmlLogger;
use SvedenParser\Color;
use SvedenParser\Printer;
final class ContingentFacade
{
private ContingentRepository $contingentRepository;
private HttpClient $httpClient;
private ContingentService $contingentService;
private HtmlLogger $htmlLogger;
/**
* Конструктор
*/
public function __construct()
{
$this->contingentRepository = new ContingentRepository();
$this->httpClient = new HttpClient();
$this->contingentService = new ContingentService();
$this->htmlLogger = new HtmlLogger('log/html.log');
}
/**
* Получить массив сайтов
* @param array $params Массив сайтов, у которых нужны обновиленные URL
* @return array
*/
public function getSites(array $params = []): array
{
if (!$params) {
return $this->contingentRepository->getSitesFromNiimko();
} else {
return $this->contingentRepository->getSitesFromMiccedu($params);
}
}
/**
* Cобирает из микроразметки данные таблицы
* "Информация о численности обучающихся" в разделе "Образование"
* @param array $site Сайт содержащий id организации и URL
* @return void
*/
public function collectDataFromContingent(array $site): void
{
if ($this->isExit($site)) {
return;
}
list('org_id' => $orgId, 'site' => $url) = $site;
$url = UrlBuilder::build($url);
Printer::println(implode(' ', $site), Color::GREEN);
$html = $this->httpClient->getContentOfSite(
$url,
$site,
'sveden/education/'
);
if (!$html) {
return;
}
$uri = $this->contingentService->getLink($html);
Printer::println($uri, Color::YELLOW);
if ($uri) {
$pattern = '/^https?:\/\/(?:www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b(?:[-a-zA-Z0-9()@:%_\+.~#?&\/=]*)$/';
if (preg_match($pattern, $uri)) {
$html = $this->httpClient->getContentOfSite(
$uri,
$site
);
} else if (UrlBuilder::checkUri($uri)) {
if (0 === strpos($uri, '/')) {
$html = $this->httpClient->getContentOfSite(
$url,
$site,
$uri
);
} else {
$html = $this->httpClient->getContentOfSite(
$url,
$site,
"sveden/education/$uri"
);
}
}
}
// Получаем данные таблицы численности
$contingent = $this->contingentService->getContingent(
$html,
$this->contingentRepository->specialties(),
$orgId
);
if ($contingent
&& $this->contingentService->isValidContingent($contingent)
) {
// Заносим в базу
Printer::print_r($contingent, Color::BLUE);
// $this->contingentRepository->insertContingent($contingent);
} else {
Printer::println("No result", Color::RED);
$this->htmlLogger->log("$orgId $url");
}
Printer::println();
}
public function getOrgInOpendata(): array
{
return $this->contingentRepository->universities();
}
/**
* Условие выхода
* @param array $site
* @return bool
*/
private function isExit(array $site): bool
{
// Нет URL сайта вуза
if (!$site['site']) {
return true;
}
// Уже в базе
if (in_array($site['org_id'], $this->contingentRepository->universities())) {
return true;
}
return false;
}
public function getSitesFromLog(string $path): array
{
try {
$result = [];
$data = file($path);
foreach ($data as &$dt) {
$dt = explode(' ', $dt);
$result[] = [
'org_id' => trim($dt[0]),
'site' => trim($dt[1])
];
}
} catch (\Exception $e) {
Printer::println($e->getMessage(), Color::RED);
} finally {
return $result;
}
}
}

125
src/ContingentParser/ContingentParser.php Normal file
View File

@ -0,0 +1,125 @@
<?php
/**
* Парсер информации об образовательной организации
* с её сайта с использованием микроразметки
*/
namespace SvedenParser\ContingentParser;
use SvedenParser\Color;
use SvedenParser\Printer;
use DOMDocument;
use DOMXPath;
final class ContingentParser
{
private ?DOMXPath $xpath;
private DOMDocument $dom;
private const TEMPLATE = '//tr[@itemprop="eduChislen"]//';
private const ENCODING = "UTF-8";
private const FIELDS = [
"eduCode" => "td",
"eduName" => "td",
"eduLevel" => "td",
"eduForm" => "td",
"numberAll" => ["th", "td"]
];
public function __construct(string $html)
{
libxml_use_internal_errors(true);
$this->dom = new DOMDocument(
encoding: self::ENCODING
);
if (empty($html)) {
$this->xpath = null;
} else {
$this->setEncoding($html);
$this->dom->loadHTML($html);
$this->xpath = new DOMXPath($this->dom);
}
}
private function setEncoding(string &$html): void
{
$encoding = mb_detect_encoding($html, 'UTF-8, windows-1251');
if ($encoding != self::ENCODING) {
$html = mb_convert_encoding(
$html,
self::ENCODING,
$encoding
);
$html = str_replace('windows-1251',self::ENCODING, $html);
}
$html = mb_convert_encoding($html,'HTML-ENTITIES','UTF-8');
}
public function getDataTable(): array
{
if (empty($this->xpath)) return [];
$data = $this->parseContingent();
var_dump($data);
$records = [];
if ($data == null) return [];
$equal = $data['eduName']->length;
foreach ($data as $field) {
if ($field->length == 0) {
return [];
}
if ($field->length != $equal) {
return [];
}
}
for ($i = 0; $i < $data['eduCode']->length; $i++) {
try {
$contingentRow = new ContingentRow(
$data['eduCode']->item($i)->textContent,
$data['eduName']->item($i)->textContent,
$data['eduLevel']->item($i)->textContent,
$data['eduForm']->item($i)->textContent,
(int)$data['numberAll']->item($i)->textContent
);
$records[] = $contingentRow->getData();
} catch (\Exception $e) {
Printer::println($e->getMessage(), Color::RED);
}
}
return $records;
}
private function parseContingent(): array
{
$data = [];
foreach (self::FIELDS as $field => $tag) {
if (!is_array($tag)) {
$data[$field] = $this->xpath->query(
self::TEMPLATE . $tag . "[@itemprop=\"$field\"]"
);
} else {
$th = $this->xpath->query(
self::TEMPLATE . $tag[0] . "[@itemprop=\"$field\"]"
);
$td = $this->xpath->query(
self::TEMPLATE . $tag[1] . "[@itemprop=\"$field\"]"
);
$data[$field] = $th->length > $td->length ? $th : $td;
}
}
return $data;
}
public function getLink(): string
{
$needle = "Информация о численности обучающихся";
$data = $this->dom->getElementsByTagName('a');
for ($i = 0; $i < $data->length; $i++) {
$haystack = $data->item($i)->textContent;
$isInformationOfContingent = strpos($haystack, $needle) !== false;
if ($isInformationOfContingent) {
return $data->item($i)->getAttribute('href');
}
}
return '';
}
}

216
src/ContingentParser/ContingentRepository.php Normal file
View File

@ -0,0 +1,216 @@
<?php
namespace SvedenParser\ContingentParser;
use SvedenParser\Database\Database;
use SvedenParser\Database\DatabaseConfig;
use NilPortugues\Sql\QueryBuilder\Builder\GenericBuilder;
final class ContingentRepository
{
private Database $opendata;
private Database $niimko;
public const FILE_ADD_RECORDING = Database::FILE_ADD_RECORDING;
private array $specialties;
private array $universities;
/**
* Конструктор
*/
public function __construct()
{
$this->niimko = new Database(new DatabaseConfig('niimko'));
$this->opendata = new Database(new DatabaseConfig('opendata'));
$this->specialties = $this->getSpecialties();
$this->universities = $this->getUniversities();
}
/**
* Извлечение URL сайтов из базы данных niimko
* @return array
*/
public function getSitesFromNiimko(): array
{
/*
SELECT kod AS org_id, site FROM niimko.s_vuzes
WHERE ootype = 'vuz' AND deleted = 'n' AND fake = 'n'
*/
$builder = new GenericBuilder();
$params = ['vuz', 'n', 'n', 'RU'];
$query = $builder->select()
->setTable('s_vuzes')
->setColumns(['org_id' => 'kod', 'site'])
->where('AND')
->equals('ootype', 'vuz')
->equals('deleted', 'n')
->equals('fake', 'n')
->equals('country', 'RU')
->end();
$sql = $builder->write($query);
$sites = $this->niimko->select($sql, $params);
return $sites;
}
/**
* Извлечение сайтов базы данных opendata
* из таблицы miccedu_monitoring.
* @param array $params
* Сайты, у которых устаревшие URL
* @return array
*/
public function getSitesFromMiccedu(array $params): array
{
/*
SELECT site, vuzkod AS org_id FROM opendata.miccedu_monitoring
WHERE year = 2023 AND (vuzkod = :val1 OR vuzkod = :val2 OR ...)
*/
$builder = new GenericBuilder();
$year = 2023;
foreach ($params as $key => $org) {
$params[$key] = (int)$org['org_id'];
}
$query = $builder->select()
->setTable('miccedu_monitoring')
->setColumns(['org_id' => 'vuzkod','site'])
->where('AND')
->equals('year', $year)
->subWhere('OR');
foreach ($params as $orgId) {
$query->equals('vuzkod', $orgId);
}
$query = $query->end();
$sql = $builder->write($query);
array_unshift($params, $year);
$sites = $this->opendata->select($sql, $params);
return $sites;
}
/**
* Внесение данных численности обучающихся в базу данных opendata
* @param array $contingent
* Массив записей численности по специальностям
* @return void
*/
public function insertContingent(array $contingent): void
{
/*
INSERT INTO sveden_education_contingent
(org_id, spec_id, spec_code, spec_name, edu_level, edu_forms, contingent)
VALUES
(:v1, :v2, :v3, :v4, :v5, :v6, :v7)
...
*/
$builder = new GenericBuilder();
$countAtributes = count($contingent[0]);
$size = $countAtributes * (count($contingent) - 1);
$query = $builder->insert()
->setTable('sveden_education_contingent')
->setValues(
$contingent[0]
);
$sql = $builder->write($query);
for ($i = $countAtributes; $i <= $size;) {
$sql .= " (:v".(++$i).", :v".(++$i).", :v".(++$i).", :v"
.(++$i).", :v".(++$i).", :v".(++$i).", :v".(++$i).")\n";
}
$sql = preg_replace('/\)\s*VALUES\s*/', ') VALUES ', $sql);
$sql = preg_replace('/\)\s*\(/', '), (', $sql);
$this->opendata->insert($sql, $contingent);
}
/**
* Публичное получение специальностей
* @return array
*/
public function specialties(): array
{
return $this->specialties ? $this->specialties : [];
}
/**
* Публичное получение id вузов, занесенных в базу opendata
* @return array
*/
public function universities(): array
{
return $this->universities ? $this->universities : [];
}
/**
* Извлечение кодов специальности из базы данных niimko
* @return array
*/
private function getSpecialties(): array
{
/*
SELECT id AS spec_id, kod AS spec_code FROM niimko.s_specs
WHERE oopkodes = 'gos3p'
*/
$builder = new GenericBuilder();
$params = ['gos3p'];
$query = $builder->select()
->setTable('s_specs')
->setColumns(['spec_id' =>'id', 'spec_code' => 'kod'])
->where()
->equals('oopkodes','gos3p')
->end();
$sql = $builder->write($query);
$specialties = $this->niimko->select($sql, $params);
return $specialties;
}
/**
* Извлечение id вузов, занесенных в базу opendata
* @return array
*/
private function getUniversities(): array
{
/*
SELECT DISTINCT org_id FROM sveden_education_contingent
*/
$builder = new GenericBuilder();
$query = $builder->select()
->setTable('sveden_education_contingent')
->setColumns(['org_id'])
->where()
->greaterThan('org_id', 0)
->end();
$sql = $builder->write($query);
$sql = preg_replace("/ WHERE.*/", '', $sql);
$sql = preg_replace('/SELECT/', 'SELECT DISTINCT', $sql);
$universities = $this->opendata->select($sql);
return array_column($universities, 'org_id');
}
/**
* Обновление сайтов в базе данных niimko
* @param array $params
* Массив [['org_id' => val1, 'site' => val1,],...]
* @return void
*/
public function updateSitesOpendata(array $params): void
{
/*
UPDATE niimko.s_vuzes
SET site = CASE kod
WHEN :v1 THEN :v2
WHEN :v3 THEN :v4
...
ELSE kod
END
WHERE kod IN (:v1, :v2...)
*/
$count = count($params);
for ($i = 0; $i < $count; $i++) {
if ($i % 2 == 0) {
$params[] = $params[$i];
}
}
$sql = "UPDATE niimko.s_vuzes\nSET site = CASE kod\n";
for ($i = 0; $i < $count;) {
$sql .= "WHEN :v".++$i." THEN :v".++$i."\n";
}
$sql .= "ELSE kod\nEND\nWHERE kod in(";
for ($i = $count++; $i < count($params);) {
$sql .= ":v".++$i.",\n";
}
$sql = rtrim($sql,",\n") .")\n";
$this->opendata->update($sql, $params);
}
}

33
src/ContingentParser/ContingentRow.php Normal file
View File

@ -0,0 +1,33 @@
<?php
namespace SvedenParser\ContingentParser;
final class ContingentRow
{
public function __construct(
private string $eduCode,
private string $eduName,
private string $eduLevel,
private string $eduForm,
private int $contingent
) {
if ($contingent < 0) {
throw new \Exception("Недействительная численность обучающихся!");
}
$this->eduCode = trim($eduCode);
$this->eduName = trim($eduName);
$this->eduLevel = trim($eduLevel);
$this->eduForm = trim($eduForm);
$this->contingent = $contingent;
}
public function getData(): array
{
return [
"spec_code" => $this->eduCode,
"spec_name" => $this->eduName,
"edu_level" => $this->eduLevel,
"edu_forms"=> $this->eduForm,
"contingent" => $this->contingent
];
}
}

73
src/ContingentParser/ContingentService.php Normal file
View File

@ -0,0 +1,73 @@
<?php
namespace SvedenParser\ContingentParser;
final class ContingentService
{
/**
* Получить данные о численности
* @param string $html Разметка сайта вуза
* @param mixed $specialties Массив специальностей
* @param int $orgId Идентификатор организации
* @return array
*/
public function getContingent(
string $html,
array $specialties,
int $orgId
): array {
$parser = new ContingentParser($html);
$contingent = $parser->getDataTable();
$this->addSpecId($contingent, $specialties);
$this->addOrgId($contingent, $orgId);
return $contingent;
}
/**
* Проверка на валидность записи численнести
* @param array $contingent Массив численности по специальностям
* @return bool
*/
public function isValidContingent(array $contingent): bool
{
$count = 0;
foreach ($contingent as $value) {
$count += $value['contingent'];
}
return $count ? true : false;
}
/**
* Добавить идентификатор специальности в запись численности
* @param array $contingent Массив численности по специальностям
* @param array $specialties Массив специальностей
* @return void
*/
private function addSpecId(array &$contingent, array $specialties): void
{
$specIdMap = array_column($specialties, 'spec_id', 'spec_code');
foreach ($contingent as $key => $con) {
$contingent[$key]['spec_id'] = $specIdMap[$con['spec_code']] ?? null;
}
}
/**
* Добавить идентификатор организации в запись численности
* @param array $contingent Массив численности по специальностям
* @param int $orgId Идентифиактор организации
* @return void
*/
private function addOrgId(array &$contingent, int $orgId): void
{
foreach ($contingent as &$con) {
$con['org_id'] = $orgId;
}
}
/**
*
* @param string $html
* @return string
*/
public function getLink(string $html): string
{
$parser = new ContingentParser($html);
return $parser->getLink();
}
}

132
src/Database/Database.php Normal file
View File

@ -0,0 +1,132 @@
<?php
namespace SvedenParser\Database;
use SvedenParser\Logger\DatabaseLogger;
use Symfony\Component\Yaml\Yaml;
use PDOException;
use PDO;
final class Database
{
private PDO $pdo;
public const FILE_ADD_RECORDING ='not-recorded-in-db.yaml';
private const ERR_NO_CONNECT = "HY000";
private static $logfile = 'log/database.log';
private DatabaseConfig $databaseConfig;
private DatabaseLogger $logger;
/**
* Конструктор
* @param \SvedenParser\Database\DatabaseConfig $config
* Конфигурация подключения к базе данных
*/
public function __construct(DatabaseConfig $config)
{
$this->logger = new DatabaseLogger(self::$logfile);
$this->databaseConfig = $config;
try {
$dsn = $this->databaseConfig->getDsn();
$username = $this->databaseConfig->getUsername();
$password = $this->databaseConfig->getPassword();
$this->pdo = new PDO(
$dsn,
$username,
$password,
[PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]
);
$message = "Подключение к {$this->databaseConfig->getDBName()} успешно!";
$this->logger->log($message);
} catch (PDOException $e) {
$message = "Ошибка подключения к {$this->databaseConfig->getDBName()}: {$e->getMessage()}";
$this->logger->log($message);
}
}
/**
* Сообщение о разрыве соединения
*/
public function __destruct()
{
$message = "Подключение к {$this->databaseConfig->getDBName()} прервано!";
$this->logger->log($message);
}
/**
* Выборка данных из базы
* @param string $sql SQL-запрос
* @param array $params Параметры запроса
* @return array
*/
public function select(string $sql, array $params = []): array
{
try {
$stmt = $this->pdo->prepare($sql);
if (!empty($params)) {
for ($i = 0; $i < count($params); $i++) {
$stmt->bindParam(":v".($i+1), $params[$i]);
}
}
$stmt->execute();
$array = $stmt->fetchAll(PDO::FETCH_ASSOC);
} catch (PDOException $e) {
$message = "Ошибка запроса: " . $e->getMessage();
$this->logger->log($message);
} finally {
return $array;
}
}
/**
* Добавление данных в базу
* @param string $sql SQL-запрос
* @param array $params Параметры запроса
* @return void
*/
public function insert(string $sql, array $params): void
{
try {
$stmt = $this->pdo->prepare($sql);
$count = 1;
$size = count($params[0]);
foreach ($params as $param) {
for ($i = $count; $i <= $size; $i++) {
$param = array_values($param);
$stmt->bindParam(":v$i", $param[$i-$count]);
}
$count += count($param);
$size += count($param);
}
$stmt->execute();
$this->logger->log("Запрос выполнен успешно!");
} catch (PDOException $e) {
$message = "Ошибка запроса:" . $e->getMessage();
$this->logger->log($message);
// При ошибке запроса сохраняем валидные данные в yaml-файл
if ($e->getCode() === self::ERR_NO_CONNECT) {
$yaml = Yaml::dump($params);
file_put_contents(
self::FILE_ADD_RECORDING,
$yaml,
FILE_APPEND
);
}
}
}
/**
* Обновление данных в базе
* @param string $sql SQL-запрос
* @param array $params Параметры запроса
* @return void
*/
public function update(string $sql, array $params): void
{
try {
$stmt = $this->pdo->prepare($sql);
$count = count($params);
for ($i = 0; $i < $count; $i++) {
$stmt->bindParam(":v".($i+1), $params[$i]);
}
// $stmt->execute();
$this->logger->log("Запрос выполнен успешно!");
} catch (PDOException $e) {
$message = "Ошибка запроса:" . $e->getMessage();
$this->logger->log($message);
}
}
}

62
src/Database/DatabaseConfig.php Normal file
View File

@ -0,0 +1,62 @@
<?php
namespace SvedenParser\Database;
final class DatabaseConfig
{
private string $_driver;
private string $_host;
private string $_dbname;
private string $_port;
private string $_charset;
private string $_username;
private string $_password;
public function __construct(string $db)
{
$config = $this->getDataEnv($db);
$this->_driver = $config['DB_DRIVER'];
$this->_host = $config['DB_HOST'];
$this->_dbname = $config['DB_NAME'];
$this->_port = $config['DB_PORT'];
$this->_charset = $config["DB_CHARSET"];
$this->_username = $config['DB_USERNAME'];
$this->_password = $config['DB_PASSWORD'];
}
private function getDataEnv(string $db): array
{
$envVars = parse_ini_file('.env', true);
$db = strtoupper($db);
$config = [];
foreach ($envVars as $dbname => $dbconfig) {
if ($dbname == $db) {
$config = $dbconfig;
}
}
return $config;
}
public function getDBName(): string
{
return $this->_dbname;
}
public function getDsn(): string
{
return $this->_driver.":host=".$this->_host
.";dbname=".$this->_dbname
.";charset=".$this->_charset
.";port=".$this->_port;
}
public function getUsername(): string
{
return $this->_username;
}
public function getPassword(): string
{
return $this->_password;
}
}

106
src/Http/CurlHelper.php Normal file
View File

@ -0,0 +1,106 @@
<?php
namespace SvedenParser\Http;
use SvedenParser\Color;
use SvedenParser\Logger\HttpLogger;
use SvedenParser\Printer;
use CurlHandle;
final class CurlHelper
{
private CurlHandle|bool $curl;
private string $url;
private array $site;
private int $countRedirect;
private const MAX_REDIRECT = 5;
/**
* Коснтруктор
* Инициализация сессии
* @param string $url URL сайта
* @param array $site Идентификатор организации и базовый URL сайта
*/
public function __construct(string $url, array $site)
{
$this->countRedirect = 0;
$this->url = $url;
$this->site = $site;
$this->curl = curl_init();
curl_setopt($this->curl, CURLOPT_RETURNTRANSFER, true);
curl_setopt($this->curl, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($this->curl, CURLOPT_HEADER, true);
curl_setopt($this->curl, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($this->curl, CURLOPT_USERAGENT,
'Mozilla/5.0 (X11; Linux x86_64) '
.'AppleWebKit/537.36 (KHTML, like Gecko) '
.'Chrome/124.0.0.0 YaBrowser/24.6.0.0 Safari/537.36'
);
curl_setopt($this->curl, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($this->curl, CURLOPT_CONNECTTIMEOUT, 30);
}
/**
* Прекратить сессию
*/
public function __destruct()
{
curl_close($this->curl);
}
/**
* Получить html-разметку
* @return string
*/
public function getContent(): string
{
if ($this->countRedirect < self::MAX_REDIRECT) {
curl_setopt($this->curl, CURLOPT_URL, $this->url);
$html = curl_exec($this->curl);
if ($this->checkLocation($this->url, $html)) {
$this->countRedirect++;
$html = $this->getContent();
}
return $html;
}
return '';
}
/**
* Summary of checkLocation
* @param string $html
* @return bool
*/
private function checkLocation(string &$url, string $html): bool
{
preg_match('/location:(.*?)\n/i', $html, $matches);
if (empty($matches)) return false;
$target = $matches[1];
$target = preg_replace("/[^a-z0-9\-:.\/,]/iu", '', $target);
$url = $target ? $target : $url;
return $target ? true : false;
}
/**
* Сообщить об ошибке
* @return void
*/
public function isError(): bool
{
$httpLogger = new HttpLogger('log/http-curl.log');
$httpCode = curl_getinfo($this->curl, CURLINFO_HTTP_CODE);
if ($httpCode != 200 && $httpCode != 0) {
Printer::println("HTTP-code: $httpCode", Color::RED);
$message = implode(' ', $this->site) . ' HTTP-code (' . $httpCode.'):';
$httpLogger->log($message, $httpCode);
return true;
} else if ($httpCode == 0) {
$errno = curl_errno($this->curl);
$message = implode(' ', $this->site);
$message .= " cURL error ({$errno}): " . curl_strerror($errno);
$httpLogger->log($message);
return true;
} else {
Printer::println("HTTP-code: $httpCode", Color::BLUE);
return false;
}
}
}

107
src/Http/HttpClient.php Normal file
View File

@ -0,0 +1,107 @@
<?php
namespace SvedenParser\Http;
use SvedenParser\Color;
use SvedenParser\Printer;
use GuzzleHttp\Client;
use GuzzleHttp\TransferStats;
final class HttpClient
{
private array $config;
/**
* Обработка численности обучающихся
* @param string $url URL сайта
* @param array $site Идентификатор организации, и базовый URL
* @return string|bool
*/
public function getContentOfSite(
string $url,
array $site,
string $uri = ''
): string|bool {
try {
$client = $this->createClient($url);
if ($uri !== '') {
// Запрос по базовому uri
$response = $client->get('', [
'on_stats' => function (TransferStats $stats) use (&$redirectUrl) {
$redirectUrl = $stats->getEffectiveUri();
}
]);
if ($url !== $redirectUrl) {
Printer::println("Redirect $url -> $redirectUrl");
$url = UrlBuilder::build($redirectUrl);
}
}
$url = UrlBuilder::addUri($url , $uri);
Printer::println("Parsing for $url");
$response = $client->get($url);
$httpCode = $response->getStatusCode();
Printer::println("HTTP-code: $httpCode", Color::BLUE);
$html = $response->getBody()->getContents();
} catch (\Exception $e) {
$message = $e->getCode() ? "HTTP-code: " . $e->getCode() : "Error cURL";
Printer::println($message, Color::RED);
$html = $this->handleException($url, $site);
} finally {
return $html;
}
}
/**
* Обработка исключения
* Повторная попытка с помощью CurlHelper
* @param string $url URL сайта
* @param array $site
* @return string|bool
*/
private function handleException(string $url, array $site): string|bool
{
$curlHelper = new CurlHelper($url, $site);
$html = $curlHelper->getContent();
if ($curlHelper->isError()) {
return false;
}
return $html;
}
/**
* Создать клиента с базовым URL
* @param string $url
* @return \GuzzleHttp\Client
*/
private function createClient(string $url): Client
{
$this->config = $this->config() + ["base_uri" => $url];
return new Client($this->config);
}
/**
* Конфигурация клиента
* @return array
*/
private function config(): array
{
return [
'force_ip_resolve' => 'v4',
'debug' => fopen("log/debug-http.log", "w"),
'allow_directs' => [
'max' => 5,
'strict' => true,
'referer' => true,
'protocols' => ['http', 'https'],
'track_redirects' => true
],
'connect_timeout' => 30.0,
'verify' => false,
'headers' => [
'User-Agent' => 'Mozilla/5.0 (X11; Linux x86_64) '
.'AppleWebKit/537.36 (KHTML, like Gecko) '
.'Chrome/124.0.0.0 YaBrowser/24.6.0.0 Safari/537.36',
'Content-Type' => 'text/html;charset=utf-8'
]
];
}
}

51
src/Http/UrlBuilder.php Normal file
View File

@ -0,0 +1,51 @@
<?php
namespace SvedenParser\Http;
final class UrlBuilder
{
/**
* Строит валидный URL сайта
* @param string $url Изначальный URL
* @return string
*/
public static function build(string $url): string
{
// Строит -> https://<base_uri>
$url = trim(strtolower($url));
$url = preg_replace('/\s+/', '', $url);
$url = str_replace("www/", "www.", $url);
$url = str_replace("http:\\\\", "", $url);
if (!preg_match('#^https?://#', $url)) {
$url = "http://$url";
}
$url = str_replace("http://", "https://", $url);
$arr = parse_url($url);
$url = $arr['scheme'] . '://' . $arr['host'] . '/';
// $url = str_replace("www.", "", $url);
$url = str_replace("_", "/", $url);
return trim($url);
}
public static function addUri(string $url, string $uri): string
{
if (!$uri)
return $url;
$url .= substr($url, -1) == '/' ? '' : '/';
$url .= substr($uri, 0, 1) == '/' ? substr($uri, 1) : $uri;
return $url;
}
public static function checkUri(string $uri): bool
{
if (str_ends_with($uri, ".pdf")
|| str_ends_with($uri, ".docx")
|| str_ends_with($uri, ".doc")
|| str_starts_with($uri, "javascript")
) {
return false;
}
return true;
}
}

12
src/Logger/DatabaseLogger.php Normal file
View File

@ -0,0 +1,12 @@
<?php
namespace SvedenParser\Logger;
final class DatabaseLogger extends Logger
{
public function log(string $message): void
{
$date = date('Y-m-d H:i:s');
$logMessage = "[$date] $message\n";
file_put_contents($this->_path, $logMessage, FILE_APPEND);
}
}

12
src/Logger/HtmlLogger.php Normal file
View File

@ -0,0 +1,12 @@
<?php
namespace SvedenParser\Logger;
final class HtmlLogger extends Logger
{
public function log(string $message): void
{
$date = date('Y-m-d H:i:s');
$logMessage = "[$date] $message\n";
file_put_contents($this->_path, $logMessage, FILE_APPEND);
}
}

70
src/Logger/HttpLogger.php Normal file
View File

@ -0,0 +1,70 @@
<?php
namespace SvedenParser\Logger;
final class HttpLogger extends Logger
{
private const ARR_HTTP_STATUS_CODE = array(
100 => 'Continue',
101 => 'Switching Protocols',
200 => 'OK',
201 => 'Created',
202 => 'Accepted',
203 => 'Non-Authoritative Information',
204 => 'No Content',
205 => 'Reset Content',
206 => 'Partial Content',
300 => 'Multiple Choices',
301 => 'Moved Permanently',
302 => 'Found',
303 => 'See Other',
304 => 'Not Modified',
305 => 'Use Proxy',
306 => 'Switch Proxy',
307 => 'Temporary Redirect',
400 => 'Bad Request',
401 => 'Unauthorized',
402 => 'Payment Required',
403 => 'Forbidden',
404 => 'Not Found',
405 => 'Method Not Allowed',
406 => 'Not Acceptable',
407 => 'Proxy Authentication Required',
408 => 'Request Timeout',
409 => 'Conflict',
410 => 'Gone',
411 => 'Length Required',
412 => 'Precondition Failed',
413 => 'Payload Too Large',
414 => 'URI Too Long',
415 => 'Unsupported Media Type',
416 => 'Range Not Satisfiable',
417 => 'Expectation Failed',
418 => 'I\'m a teapot',
429 => 'Too Many Requests',
451 => 'Unavailable For Legal Reasons',
500 => 'Internal Server Error',
501 => 'Not Implemented',
502 => 'Bad Gateway',
503 => 'Service Unavailable',
504 => 'Gateway Timeout',
505 => 'HTTP Version Not Supported',
506 => 'Variant Also Negotiates',
507 => 'Insufficient Storage',
508 => 'Loop Detected',
509 => 'Bandwidth Limit Exceeded',
510 => 'Not Extended',
511 => 'Network Authentication Required'
);
public function log(string $message, int $httpCode = null): void
{
$date = date('Y-m-d H:i:s');
if (!$httpCode) {
$logMessage = "[$date] $message\n";
file_put_contents($this->_path, $logMessage, FILE_APPEND);
} else {
$logMessage = "[$date] $message "
.self::ARR_HTTP_STATUS_CODE[$httpCode]."\n";
file_put_contents($this->_path, $logMessage, FILE_APPEND);
}
}
}

12
src/Logger/Logger.php Normal file
View File

@ -0,0 +1,12 @@
<?php
namespace SvedenParser\Logger;
abstract class Logger
{
protected string $_path;
public function __construct(string $path)
{
$this->_path = $path;
}
}

45
src/Printer.php Normal file
View File

@ -0,0 +1,45 @@
<?php
namespace SvedenParser;
final class Printer
{
/**
* Вывод строки
* @param string $text Строка
* @param \SvedenParser\Color $color Цвет
* @return void
*/
public static function print(
string $text = '',
Color $color = Color::WHITE
): void {
print($color->tostring().$text.Color::WHITE->tostring());
}
/**
* Вывод строки с EOL
* @param string $text Строка
* @param \SvedenParser\Color $color Цвет
* @return void
*/
public static function println(
string $text = '',
Color $color = Color::WHITE
): void {
print($color->tostring().$text.Color::WHITE->tostring());
print(PHP_EOL);
}
/**
* Удобочитаемый вывод переменной
* @param mixed $value Переменная
* @param \SvedenParser\Color $color Цвет
* @return void
*/
public static function print_r(
mixed $value,
Color $color = Color::WHITE
): void {
print($color->tostring());
print_r($value);
print(Color::WHITE->tostring());
}
}