67 lines
1.6 KiB
PHP
67 lines
1.6 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs\Tasks;
|
|
|
|
use andreskrey\Readability\Configuration;
|
|
use andreskrey\Readability\ParseException;
|
|
use andreskrey\Readability\Readability;
|
|
use App\Helpers\FirstParty\OSSUploader\OSSUploader;
|
|
use Exception;
|
|
use Illuminate\Support\Facades\Http;
|
|
|
|
class ScrapeUrlBodyTask
|
|
{
|
|
public static function handle(string $url)
|
|
{
|
|
$slug = str_slug($url);
|
|
|
|
$disk_url = '/scraped/'.$slug.'.html';
|
|
|
|
$html_content = null;
|
|
|
|
try {
|
|
$html_content = OSSUploader::readFile('r2', '/scraped/', $slug.'.html');
|
|
|
|
if (is_null($disk_url)) {
|
|
throw Exception('Not stored.');
|
|
}
|
|
} catch (Exception $e) {
|
|
$html_content = null;
|
|
}
|
|
|
|
if (is_null($html_content)) {
|
|
$proxy = 'gate.smartproxy.com:10000';
|
|
$user = 'sp5bbkzj7e';
|
|
$psw = 'yTtk2cc5kg23kIkSSr';
|
|
|
|
$response = Http::withOptions([
|
|
'proxy' => "http://$user:$psw@$proxy",
|
|
])->get($url);
|
|
|
|
if ($response->successful()) {
|
|
$html_content = $response->body();
|
|
|
|
OSSUploader::uploadFile('r2', '/scraped/', $slug.'.html', $html_content);
|
|
}
|
|
}
|
|
|
|
//dump("Initial: " . strlen($html_content));
|
|
|
|
$readability = new Readability(new Configuration());
|
|
|
|
try {
|
|
$readability->parse($html_content);
|
|
|
|
$html_content = strip_tags($readability->getContent());
|
|
//dd($readability);
|
|
} catch (ParseException $e) {
|
|
|
|
}
|
|
|
|
//dump("After: " . strlen($html_content));
|
|
|
|
return $html_content;
|
|
|
|
}
|
|
}
|