alexzhu/sveden-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Добавлено логирование и исправлены некоторые ошибки в классах

Browse Source
This commit is contained in:
Alexander
2024-08-16 13:44:49 +03:00
parent f06eee3e95
commit 8462c1907b
9 changed files with 1604 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
app/app.php
View File

@ -11,20 +11,60 @@ use GuzzleHttp\RequestOptions;
use App\Library\ContingentParser;
use App\Library\Database;
use GuzzleHttp\Client;
use Symfony\Component\Yaml\Yaml;
$dbOpendata = new Database(new DatabaseConfig('opendata'));
$dbNiimko = new Database(new DatabaseConfig('niimko'));
$sites = ContingentManager::getInstance()->getSites($dbNiimko);
$specializations = ContingentManager::getInstance()->getSpecializations($dbNiimko);
$sql = 'SELECT DISTINCT org_id FROM sveden_education_contingent';
$org = $dbOpendata->selectQuery($sql);
// print_r($sites);
// print_r($specializations);
// print_r($org);
$orgs = [];
foreach ($org as $o) {
$orgs[] = $o['org_id'];
}
unset($org);
$errorSites = [];
$filename = 'error-html.log';
$array = file($filename);
for ($i = 0; $i < count($array); $i++) {
$arr = explode(' ', $array[$i]);
if (!in_array($arr[2], $orgs)) {
$errorSites[] = $arr[2];
}
}
$filename = 'error-http.log';
$array = file($filename);
for ($i = 0; $i < count($array); $i++) {
$arr = explode(' ', $array[$i]);
if (!in_array($arr[2], $orgs)) {
$errorSites[] = $arr[2];
}
}
$status = null;
$succes = 0;
$failed = array();
for ($i = 0; $i < 100; $i++) {
$start = 600;
for ($i = $start; $i < count($sites); $i++) {
// Нет URL сайта вуза
if (empty($sites[$i]['site'])) {
$log = date('Y-m-d H:i:s') . ' ' . $sites[$i]['org_id'] . ' ' . $sites[$i]['site'];
file_put_contents(__DIR__ . '/../error-http.log', $log . PHP_EOL, FILE_APPEND);
continue;
}
// Уже в базе
if (in_array($sites[$i]['org_id'], $orgs)) {
continue;
}
// С ошибками разметки игнорируем
if (in_array($sites[$i]['org_id'], $errorSites)) {
continue;
}
try {
$client = new Client([
RequestOptions::ALLOW_REDIRECTS => [
@ -32,10 +72,15 @@ for ($i = 0; $i < 100; $i++) {
'strict' => true,
'referer' => true,
'track_redirects' => true,
'headers' => [
'content-type' => 'text/html;',
'charset' => 'utf-8'
]
]
]);
$baseURL = $sites[$i]['site'];
$url = ContingentManager::getInstance()->buildURL($baseURL);
$url = $sites[$i]['site'];
$url = ContingentManager::getInstance()->buildURL($url);
print(($i+1).". Current url: $url\n");
$response = $client->get($url, ['timeout' => 300]);
@ -50,31 +95,32 @@ for ($i = 0; $i < 100; $i++) {
// Добавляем поле org_id
ContingentManager::getInstance()->addOrgId($contingent, $sites[$i]['org_id']);
print_r($contingent);
} catch (ClientException $e) {
$response = $e->getCode();
$status = 0;
} catch (RequestException $e) {
$response = $e->getCode();
$status = 0;
} catch (ConnectException $e) {
$response = $e->getCode();
$status = 0;
} catch (ServerException $e) {
$response = $e->getCode();
$status = 0;
} finally {
if ($status == 200 && !empty($contingent)) {
$status = 0;
print("Succes: ".++$succes."\n");
// TODO - здесь заносим в базу
if ($status != 200) {
$log = date('Y-m-d H:i:s') . ' ' . $sites[$i]['org_id'] . ' ' . $sites[$i]['site'];
file_put_contents(__DIR__ . '/../error-http.log', $log . PHP_EOL, FILE_APPEND);
} else if (empty($contingent)) {
$log = date('Y-m-d H:i:s') . ' ' . $sites[$i]['org_id'] . ' ' . $sites[$i]['site'];
file_put_contents(__DIR__ . '/../error-html.log', $log . PHP_EOL, FILE_APPEND);
} else {
// Сайты, которые распарсить не удолось
$failed[] = $sites[$i];
$set = ContingentManager::getInstance()->checkContingent($contingent);
if ($set) {
// Заносим в базу
ContingentManager::getInstance()->insertContingent($dbOpendata, $contingent);
} else {
$log = date('Y-m-d H:i:s') . ' ' . $sites[$i]['org_id'] . ' ' . $sites[$i]['site'];
file_put_contents(__DIR__ . '/../error-html.log', $log . PHP_EOL, FILE_APPEND);
}
unset($contingent);
}
}
}
$yaml = Yaml::dump($failed);
file_put_contents(__DIR__ . '/../failed.yaml', $yaml);
// Чтобы не дублировались в базе
// $dbOpendata->insert('sveden_education_contingent', $data);
// $dbOpendata->update('sveden_education_contingent', $specializations);
// $data = $dbOpendata->select('sveden_education_contingent');
}