This commit is contained in:
ct
2025-08-10 21:10:33 +08:00
parent 480bd9055d
commit 583a804073
43 changed files with 7623 additions and 270 deletions

View File

@@ -0,0 +1,43 @@
<?php
namespace App\Jobs;
use App\Models\CrawlShotJob;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Facades\Log;
class CleanupOldResults implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function handle(): void
{
$cutoffTime = now()->subHours(24);
$oldJobs = CrawlShotJob::where('created_at', '<', $cutoffTime)->get();
$deletedFiles = 0;
$deletedJobs = 0;
foreach ($oldJobs as $job) {
if ($job->file_path && Storage::exists($job->file_path)) {
Storage::delete($job->file_path);
$deletedFiles++;
}
$job->delete();
$deletedJobs++;
}
Log::info("Cleanup completed", [
'deleted_files' => $deletedFiles,
'deleted_jobs' => $deletedJobs,
'cutoff_time' => $cutoffTime->toISOString()
]);
}
}

View File

@@ -0,0 +1,88 @@
<?php
namespace App\Jobs;
use App\Models\CrawlShotJob;
use App\Services\BrowsershotService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
class ProcessCrawlShotJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected string $jobUuid;
public function __construct(string $jobUuid)
{
$this->jobUuid = $jobUuid;
}
public function handle(): void
{
$job = CrawlShotJob::where('uuid', $this->jobUuid)->first();
if (!$job) {
Log::error("CrawlShotJob not found: {$this->jobUuid}");
return;
}
try {
$job->update([
'status' => 'processing',
'started_at' => now()
]);
$browsershot = new BrowsershotService();
if ($job->type === 'crawl') {
$result = $browsershot->crawlHtml($job->url, $job->parameters ?? []);
$this->saveCrawlResult($job, $result);
} elseif ($job->type === 'shot') {
$result = $browsershot->takeScreenshot($job->url, $job->parameters ?? []);
$this->saveScreenshotResult($job, $result);
}
$job->update([
'status' => 'completed',
'completed_at' => now()
]);
} catch (\Exception $e) {
Log::error("Job {$this->jobUuid} failed: " . $e->getMessage());
$job->update([
'status' => 'failed',
'error_message' => $e->getMessage(),
'completed_at' => now()
]);
}
}
private function saveCrawlResult(CrawlShotJob $job, string $html): void
{
$filename = "{$job->uuid}.html";
$path = "crawlshot/html/{$filename}";
Storage::put($path, $html);
$job->update(['file_path' => $path]);
}
private function saveScreenshotResult(CrawlShotJob $job, array $result): void
{
$parameters = $job->parameters ?? [];
$format = $parameters['format'] ?? 'jpg';
$filename = "{$job->uuid}.{$format}";
$path = "crawlshot/images/{$filename}";
Storage::put($path, $result['data']);
$job->update(['file_path' => $path]);
}
}