alexzhu/sveden-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Curl стал основой, а также классы бд стали информативней в логах

Browse Source
This commit is contained in:
Alexander 2024-08-29 09:34:16 +03:00
parent 04406fb444
commit 5e0f59f8c5
3 changed files with 70 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app/library/Database.php
View File

@ -10,28 +10,32 @@ class Database
{
private PDO $pdo;
private static $logFile = 'database.log';
private DatabaseConfig $config;
public function __construct(DatabaseConfig $config)
{
$this->config = $config;
try {
$dsn = $config->getDsn();
$username = $config->getUsername();
$password = $config->getPassword();
$dsn = $this->config->getDsn();
$username = $this->config->getUsername();
$password = $this->config->getPassword();
$this->pdo = new PDO(
$dsn,
$username,
$password,
[PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]
);
Logger::log(self::$logFile, "Подключение успешно!");
$message = "Подключение к ". $this->config->getDBName() ." успешно!";
Logger::log(self::$logFile, $message);
} catch (PDOException $e) {
$message = "Ошибка подключения:" . $e->getMessage();
$message = "Ошибка подключения к ". $this->config->getDBName() .": " . $e->getMessage();
Logger::log(self::$logFile, $message);
}
}
public function __destruct()
{
Logger::log(self::$logFile, "Подключение прервано!");
$message = "Подключение к ". $this->config->getDBName() ." прервано!";
Logger::log(self::$logFile, $message);
}
public function selectQuery(string $sql, array $params = []) : array
@ -45,7 +49,7 @@ class Database
$stmt->execute();
$array = $stmt->fetchAll(PDO::FETCH_ASSOC);
} catch (PDOException $e) {
$message = "Ошибка запроса:" . $e->getMessage();
$message = "Ошибка запроса: " . $e->getMessage();
Logger::log(self::$logFile, $message);
} finally {
return $array;

5
app/library/DatabaseConfig.php
View File

@ -29,6 +29,11 @@ class DatabaseConfig
$this->password = $config['DB_PASSWORD'];
}
public function getDBName(): string
{
return $this->dbname;
}
public function getDsn() : string
{
return $this->driver.":host=".$this->host

62
curl-helper.php
View File

@ -8,26 +8,64 @@ use Symfony\Component\Yaml\Yaml;
require_once(dirname(__FILE__) ."/vendor/autoload.php");
$pathLogErrorHttp = __DIR__.'/../log/'. date('Y-m-d') . '/error-http-curl.log';
$pathLogErrorHtml = __DIR__.'/../log/'. date('Y-m-d') . '/error-html.log';
$
function curl_redir_exec($ch)
{
static $curl_loops = 0;
static $curl_max_loops = 20;
if ($curl_loops++ >= $curl_max_loops) {
$curl_loops = 0;
return false;
}
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$data = curl_exec($ch);
list($header, $data) = explode("\n\n", $data, 2);
$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($http_code == 301 || $http_code == 302) {
$matches = [];
preg_match('/Location:(.*?)\n/', $header, $matches);
$url = @parse_url(trim(array_pop($matches)));
if (!$url) {
//couldn't process the url to redirect to
$curl_loops = 0;
return $data;
}
$last_url = parse_url(curl_getinfo($ch, CURLINFO_EFFECTIVE_URL));
if (!$url['scheme'])
$url['scheme'] = $last_url['scheme'];
if (!$url['host'])
$url['host'] = $last_url['host'];
if (!$url['path'])
$url['path'] = $last_url['path'];
$new_url = $url['scheme'] . '://' . $url['host'] . $url['path'] . ($url['query']?'?'.$url['query']:'');
curl_setopt($ch, CURLOPT_URL, $new_url);
// debug('Redirecting to', $new_url);
return curl_redir_exec($ch);
} else {
$curl_loops=0;
return $data;
}
}
$pathLogErrorHttp = __DIR__.'/log/'. date('Y-m-d') . '/error-http-curl.log';
$pathLogErrorHtml = __DIR__.'/log/'. date('Y-m-d') . '/error-html.log';
// $sites = ContingentManager::getInstance()->getExceptionsHttpCurl('select-http-error.log');
// print_r($sites);
$dbOpendata = new Database(new DatabaseConfig('opendata'));
$dbNiimko = new Database(new DatabaseConfig('niimko'));
var_dump($dbOpendata);
// $sites = ContingentManager::getInstance()->getSites($dbNiimko);
$specializations = ContingentManager::getInstance()->getSpecializations($dbNiimko);
$orgs = ContingentManager::getInstance()->getOrgs($dbOpendata);
$sites = Yaml::parse(file_get_contents(dirname(__FILE__) ."/sites.yaml"));
print_r($sites);
// $sites = ContingentManager::getInstance()->getExceptionsHttpCurl('log/2024-08-27/error-http-curl.log');
// print_r($sites);
for ($i = 0; $i < count($sites); $i++) {
// Нет URL сайта вуза
if (empty($sites[$i]['site'])) {
// $message = implode(' ', $sites[$i]);
// Logger::log($pathLogErrorHttp, $message);
Logger::log($pathLogErrorHttp, $message);
// Logger::log($pathErrorHttp, implode(' ', $sites[$i]));
continue;
}
@ -57,9 +95,17 @@ for ($i = 0; $i < count($sites); $i++) {
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 YaBrowser/24.6.0.0 Safari/537.36');
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
$html = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($httpCode != 200) {
if ($httpCode != 200 && $httpCode != 0) {
$message = implode(' ', $sites[$i]) . ' ' . $httpCode;
Logger::log($pathLogErrorHttp, $message);
unset($httpCode);
continue;
} else if ($httpCode == 0) {
$errno = curl_errno($ch);
$message = implode(' ', $sites[$i]);
$message .= " cURL error ({$errno}): ".curl_strerror($errno);