alexzhu/sveden-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Обновлены настройки Client, дозапись в базу

Browse Source
This commit is contained in:
Alexander
2024-08-22 17:03:25 +03:00
parent fd5fe013b0
commit 30733f294c
5 changed files with 93 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
app/app.php
View File

@ -4,6 +4,7 @@ namespace App;
use App\Library\ContingentManager;
use App\Library\DatabaseConfig;
use App\Library\Logger;
use GuzzleHttp\Psr7\Exception\MalformedUriException;
use GuzzleHttp\Exception\ClientException;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Exception\RequestException;
@ -11,6 +12,10 @@ use GuzzleHttp\Exception\ServerException;
use App\Library\ContingentParser;
use App\Library\Database;
use GuzzleHttp\Client;
use GuzzleHttp\Psr7\Request;
use Psr\Http\Message\RequestInterface;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\UriInterface;
$pathLogErrorHtml = 'error-html.log';
$pathLogErrorHttp = 'error-http-curl.log';
@ -21,29 +26,13 @@ $dbNiimko = new Database(new DatabaseConfig('niimko'));
$sites = ContingentManager::getInstance()->getSites($dbNiimko);
$specializations = ContingentManager::getInstance()->getSpecializations($dbNiimko);
$orgs = ContingentManager::getInstance()->getOrgs($dbOpendata);
// print_r($sites);
// print_r($specializations);
// print_r($org);
// $errorSites = [];
// $filename = 'error-html.log';
// $array = file($filename);
// for ($i = 0; $i < count($array); $i++) {
// $arr = explode(' ', $array[$i]);
// if (!in_array($arr[2], $orgs)) {
// $errorSites[] = $arr[2];
// }
// }
// $filename = 'error-http.log';
// $array = file($filename);
// for ($i = 0; $i < count($array); $i++) {
// $arr = explode(' ', $array[$i]);
// if (!in_array($arr[2], $orgs)) {
// $errorSites[] = $arr[2];
// }
// }
$exceptionsOrgHtml = ContingentManager::getInstance()->getExceptionsHtml('error-html.log');
$exceptionsOrgHttpCurl = ContingentManager::getInstance()->getExceptionsHttpCurl('error-http-curl.log');
$start = 999;
// print_r($exceptionsOrgHttpCurl);
$start = 0;
for ($i = $start; $i < count($sites); $i++) {
// Нет URL сайта вуза
@ -57,13 +46,33 @@ for ($i = $start; $i < count($sites); $i++) {
continue;
}
// С ошибками разметки игнорируем
// if (in_array($sites[$i]['org_id'], $errorSites)) {
// continue;
// }
if (in_array($sites[$i]['org_id'], $exceptionsOrgHtml)) {
continue;
}
if (!in_array($sites[$i]['org_id'], $exceptionsOrgHttpCurl)) {
continue;
}
try {
$baseUri = ContingentManager::getInstance()->buildBaseUri($sites[$i]['site']);
$onRedirect = function(
RequestInterface $request,
ResponseInterface $res,
UriInterface $uri
) {
echo 'Redirecting! ' . $request->getUri() . ' to ' . $uri . "\n";
};
$client = new Client([
'allow_directs' => true,
'track_redirects' => true,
'force_ip_resolve' => 'v4',
'debug' => fopen("debug-http.log", "a"),
'base_uri' => $baseUri,
'allow_directs' => [
'max' => 5,
'strict' => true,
'referer' => true,
'protocols' => ['http', 'https'],
'on_redirect' => $onRedirect,
'track_redirects' => true
],
'connect_timeout' => 300.0,
'verify' => false,
// 'http_errors' => false,
@ -72,17 +81,14 @@ for ($i = $start; $i < count($sites); $i++) {
'Content-Type' => 'text/html;charset=utf-8'
]
]);
$url = $sites[$i]['site'];
$url = ContingentManager::getInstance()->buildURL($url);
print(($i+1).". Current url: $url\n");
$response = $client->get($url);
print(($i+1). '. ' . implode(' ', $sites[$i]) . "\n");
$response = $client->get('/sveden/education/');
echo $response->getStatusCode() .PHP_EOL;
var_dump($response->getHeaderLine("'X-Guzzle-Redirect-History") . PHP_EOL);
$html = $response->getBody()->getContents();
if (empty($html)) {
$message = $sites[$i]['org_id'] . ' ' . $sites[$i]['site'];
$message = implode(' ', $sites[$i]);
Logger::log($pathLogErrorHtml, $message);
continue;
}
@ -96,7 +102,7 @@ for ($i = $start; $i < count($sites); $i++) {
ContingentManager::getInstance()->addOrgId($contingent, $sites[$i]['org_id']);
print_r($contingent);
if (empty($contingent)) {
$message = $sites[$i]['org_id'] . ' ' . $sites[$i]['site'];
$message = implode(' ', $sites[$i]);
Logger::log($pathLogErrorHtml, $message);
} else {
$set = ContingentManager::getInstance()->checkContingent($contingent);
@ -104,7 +110,7 @@ for ($i = $start; $i < count($sites); $i++) {
// Заносим в базу
ContingentManager::getInstance()->insertContingent($dbOpendata, $contingent);
} else {
$message = $sites[$i]['org_id'] . ' ' . $sites[$i]['site'];
$message = implode(' ', $sites[$i]);
Logger::log($pathLogErrorHtml, $message);
}
unset($contingent);
@ -112,7 +118,8 @@ for ($i = $start; $i < count($sites); $i++) {
} catch (ClientException
| RequestException
| ConnectException
| ServerException $e
| ServerException
| MalformedUriException $e
) {
$message = implode(' ', $sites[$i]) . "\t" . $e->getCode() . "\t" . $e->getMessage();
Logger::log($pathLogErrorHttp, $message);