charlestehio/aibuddytool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update (fix): infinite loop

Browse Source
This commit is contained in:
Charles T
2023-11-29 12:46:37 +08:00
parent 8c0ae63f42
commit 6580463fcb
4 changed files with 44 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
app/Jobs/Tasks/GetUrlBodyTask.php
View File

@@ -23,12 +23,17 @@ public static function handle(int $url_to_crawl_id)
return null;
}
if (in_array($url_to_crawl->status, ['blocked', 'trashed'])) {
return;
}
$enable_proxy = false;
$url_to_crawl->is_crawling = true;
$url_to_crawl->save();
$url_to_crawl->refresh();
// try {
$user_agent = config('platform.proxy.user_agent');
@@ -66,9 +71,13 @@ public static function handle(int $url_to_crawl_id)
// //throw $e;
// }
if (! is_empty($raw_html)) {
$markdown_output = self::getMarkdownFromHtml($raw_html);
if (! is_empty($markdown_output)) {
$url_to_crawl->output_type = 'markdown';
$url_to_crawl->output = self::getMarkdownFromHtml($raw_html);
$url_to_crawl->output = $markdown_output;
} else {
$url_to_crawl->output = 'EMPTY CONTENT';
@@ -76,6 +85,7 @@ public static function handle(int $url_to_crawl_id)
}
$url_to_crawl->is_crawled = true;
$url_to_crawl->crawl_counts = $url_to_crawl->crawl_counts + 1;
if ($url_to_crawl->save()) {
if (! in_array($url_to_crawl->status, ['blocked', 'trashed'])) {