futurewalker/app/Jobs/Tasks/CrawlRssPostTask.php

<?php

namespace App\Jobs\Tasks;

use App\Jobs\ParseRssPostMetadataJob;
use App\Models\RssPost;
use Exception;
use Illuminate\Support\Facades\Http;
use League\HTMLToMarkdown\HtmlConverter;
use Symfony\Component\DomCrawler\Crawler;

class CrawlRssPostTask
{
    public static function handle(int $rss_post_id)
    {
        $rss_post = RssPost::find($rss_post_id);

        if (is_null($rss_post)) {
            return null;
        }

        try {
            $user_agent = config('platform.proxy.user_agent');

            $response = Http::withHeaders([
                'User-Agent' => $user_agent,
            ])
                ->withOptions([
                    'proxy' => get_smartproxy_rotating_server(),
                    'timeout' => 10,
                    'verify' => false,
                ])
                ->get($rss_post->post_url);

            if ($response->successful()) {
                $raw_html = $response->body();
                $costs['unblocker'] = calculate_smartproxy_cost(round(strlen($raw_html) / 1024, 2), 'rotating_global');
            } else {
                $raw_html = null;
                $response->throw();
            }

        } catch (Exception $e) {
            $raw_html = null;
        }

        if (! is_empty($raw_html)) {
            $rss_post->body = self::getMarkdownFromHtml($raw_html);
        } else {
            $rss_post->body = 'EMPTY CONTENT';
        }

        if ((is_empty($rss_post->body)) || ($rss_post->body == 'EMPTY CONTENT') || (strlen($rss_post->body) < 800)){
            $rss_post->status = 'blocked';
        }

        if ($rss_post->save()) {

            if (! in_array($rss_post->status, ['blocked', 'trashed'])) {
                ParseRssPostMetadataJob::dispatch($rss_post->id)->onConnection('default')->onQueue('default');
            }
        }
    }

    private static function getMarkdownFromHtml($html)
    {

        $converter = new HtmlConverter([
            'strip_tags' => true,
            'strip_placeholder_links' => true,
        ]);

        $html = self::cleanHtml($html);

        $markdown = $converter->convert($html);

        //dd($markdown);

        $markdown = self::reverseLTGT($markdown);

        $markdown = self::normalizeNewLines($markdown);

        $markdown = self::removeDuplicateLines($markdown);

        return html_entity_decode(markdown_to_plaintext($markdown));
    }

    private static function reverseLTGT($input)
    {
        $output = str_replace('&lt;', '<', $input);
        $output = str_replace('&gt;', '>', $output);

        return $output;
    }

    private static function removeDuplicateLines($string)
    {
        $lines = explode("\n", $string);
        $uniqueLines = array_unique($lines);

        return implode("\n", $uniqueLines);
    }

    private static function normalizeNewLines($content)
    {
        // Split the content by lines
        $lines = explode("\n", $content);

        $processedLines = [];

        for ($i = 0; $i < count($lines); $i++) {
            $line = trim($lines[$i]);

            // If the line is an image markdown
            if (preg_match("/^!\[.*\]\(.*\)$/", $line)) {
                // And if the next line is not empty and not another markdown structure
                if (isset($lines[$i + 1]) && ! empty(trim($lines[$i + 1])) && ! preg_match('/^[-=#*&_]+$/', trim($lines[$i + 1]))) {
                    $line .= ' '.trim($lines[$i + 1]);
                    $i++; // Skip the next line as we're merging it
                }
            }

            // Add line to processedLines if it's not empty
            if (! empty($line)) {
                $processedLines[] = $line;
            }
        }

        // Collapse excessive newlines
        $result = preg_replace("/\n{3,}/", "\n\n", implode("\n", $processedLines));

        // Detect and replace the pattern
        $result = preg_replace('/^(!\[.*?\]\(.*?\))\s*\n\s*([^\n!]+)/m', '$1 $2', $result);

        // Replace multiple spaces with a dash separator
        $result = preg_replace('/ {2,}/', ' - ', $result);

        return $result;
    }

    private static function cleanHtml($htmlContent)
    {
        $crawler = new Crawler($htmlContent);

        // Define tags to remove completely
        $tagsToRemove = ['script', 'style', 'svg', 'picture', 'form', 'footer', 'nav', 'aside'];

        foreach ($tagsToRemove as $tag) {
            $crawler->filter($tag)->each(function ($node) {
                foreach ($node as $child) {
                    $child->parentNode->removeChild($child);
                }
            });
        }

        // Replace <span> tags with their inner content
        $crawler->filter('span')->each(function ($node) {
            $replacement = new \DOMText($node->text());

            foreach ($node as $child) {
                $child->parentNode->replaceChild($replacement, $child);
            }
        });

        return $crawler->outerHtml();
    }
}