Update
This commit is contained in:
@@ -18,6 +18,7 @@
|
||||
|
||||
namespace Database\Seeders;
|
||||
|
||||
use App\Helpers\FirstParty\AI\CloudflareAI;
|
||||
use App\Helpers\FirstParty\MediaEngine\MediaEngine;
|
||||
use App\Models\MediaCollection;
|
||||
use App\Models\MemeMedia;
|
||||
@@ -46,43 +47,43 @@ public function run(): void
|
||||
$this->ensureMediaCollectionExists();
|
||||
|
||||
// Read CSV file
|
||||
$csvPath = database_path('seeders/data/webm_metadata.csv');
|
||||
$memeData = $this->parseCsvFile($csvPath);
|
||||
$csv_path = database_path('seeders/data/webm_metadata.csv');
|
||||
$meme_data = $this->parseCsvFile($csv_path);
|
||||
|
||||
$this->command->info('📊 Found '.count($memeData).' memes to import');
|
||||
$this->command->info('📊 Found '.count($meme_data).' memes to import');
|
||||
|
||||
// Process in chunks for memory efficiency
|
||||
$chunks = array_chunk($memeData, 50);
|
||||
$totalProcessed = 0;
|
||||
$totalSkipped = 0;
|
||||
$totalFailed = 0;
|
||||
$chunks = array_chunk($meme_data, 50);
|
||||
$total_processed = 0;
|
||||
$total_skipped = 0;
|
||||
$total_failed = 0;
|
||||
|
||||
foreach ($chunks as $chunkIndex => $chunk) {
|
||||
$this->command->info('Processing chunk '.($chunkIndex + 1).'/'.count($chunks));
|
||||
foreach ($chunks as $chunk_index => $chunk) {
|
||||
$this->command->info('Processing chunk '.($chunk_index + 1).'/'.count($chunks));
|
||||
|
||||
DB::transaction(function () use ($chunk, &$totalProcessed, &$totalSkipped, &$totalFailed) {
|
||||
foreach ($chunk as $memeRecord) {
|
||||
DB::transaction(function () use ($chunk, &$total_processed, &$total_skipped, &$total_failed) {
|
||||
foreach ($chunk as $meme_record) {
|
||||
try {
|
||||
// Extract base filename for duplicate checking
|
||||
$baseFilename = pathinfo($memeRecord['filename'], PATHINFO_FILENAME);
|
||||
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
|
||||
|
||||
// Check if already exists by checking if webm_url contains this filename
|
||||
if (MemeMedia::where('webm_url', 'like', "%/{$baseFilename}.webm")->exists()) {
|
||||
$this->command->warn("⏭️ Skipping existing: {$memeRecord['filename']} ({$memeRecord['name']})");
|
||||
$totalSkipped++;
|
||||
if (MemeMedia::where('webm_url', 'like', "%/{$base_filename}.webm")->exists()) {
|
||||
$this->command->warn("⏭️ Skipping existing: {$meme_record['filename']} ({$meme_record['name']})");
|
||||
$total_skipped++;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->importSingleMeme($memeRecord);
|
||||
$totalProcessed++;
|
||||
$this->importSingleMeme($meme_record);
|
||||
$total_processed++;
|
||||
|
||||
if ($totalProcessed % 10 === 0) {
|
||||
$this->command->info("✅ Processed {$totalProcessed} memes...");
|
||||
if ($total_processed % 10 === 0) {
|
||||
$this->command->info("✅ Processed {$total_processed} memes...");
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$totalFailed++;
|
||||
$this->command->error("❌ Failed to import: {$memeRecord['filename']} - {$e->getMessage()}");
|
||||
$total_failed++;
|
||||
$this->command->error("❌ Failed to import: {$meme_record['filename']} - {$e->getMessage()}");
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -90,34 +91,34 @@ public function run(): void
|
||||
|
||||
// Summary
|
||||
$this->command->info("\n🎯 Import Summary:");
|
||||
$this->command->info("✅ Successfully imported: {$totalProcessed} memes");
|
||||
if ($totalSkipped > 0) {
|
||||
$this->command->info("⏭️ Skipped (existing): {$totalSkipped} memes");
|
||||
$this->command->info("✅ Successfully imported: {$total_processed} memes");
|
||||
if ($total_skipped > 0) {
|
||||
$this->command->info("⏭️ Skipped (existing): {$total_skipped} memes");
|
||||
}
|
||||
if ($totalFailed > 0) {
|
||||
$this->command->error("❌ Failed: {$totalFailed} memes");
|
||||
if ($total_failed > 0) {
|
||||
$this->command->error("❌ Failed: {$total_failed} memes");
|
||||
}
|
||||
|
||||
$totalMediaRecords = $totalProcessed * 4;
|
||||
$this->command->info("📊 Created {$totalMediaRecords} media records and {$totalProcessed} meme_media records");
|
||||
$total_media_records = $total_processed * 4;
|
||||
$this->command->info("📊 Created {$total_media_records} media records and {$total_processed} meme_media records");
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse CSV file and return array of meme data
|
||||
*/
|
||||
private function parseCsvFile(string $csvPath): array
|
||||
private function parseCsvFile(string $csv_path): array
|
||||
{
|
||||
if (! file_exists($csvPath)) {
|
||||
throw new \RuntimeException("CSV file not found: {$csvPath}");
|
||||
if (! file_exists($csv_path)) {
|
||||
throw new \RuntimeException("CSV file not found: {$csv_path}");
|
||||
}
|
||||
|
||||
$csvContent = file_get_contents($csvPath);
|
||||
$lines = str_getcsv($csvContent, "\n");
|
||||
$csv_content = file_get_contents($csv_path);
|
||||
$lines = str_getcsv($csv_content, "\n");
|
||||
|
||||
// Parse header row
|
||||
$headers = str_getcsv(array_shift($lines));
|
||||
|
||||
$memeData = [];
|
||||
$meme_data = [];
|
||||
foreach ($lines as $line) {
|
||||
if (empty(trim($line))) {
|
||||
continue;
|
||||
@@ -125,32 +126,32 @@ private function parseCsvFile(string $csvPath): array
|
||||
|
||||
$row = str_getcsv($line);
|
||||
if (count($row) === count($headers)) {
|
||||
$memeData[] = array_combine($headers, $row);
|
||||
$meme_data[] = array_combine($headers, $row);
|
||||
}
|
||||
}
|
||||
|
||||
return $memeData;
|
||||
return $meme_data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a single meme with all its formats
|
||||
*/
|
||||
private function importSingleMeme(array $memeRecord): void
|
||||
private function importSingleMeme(array $meme_record): void
|
||||
{
|
||||
// Extract base filename (remove .webm extension)
|
||||
$baseFilename = pathinfo($memeRecord['filename'], PATHINFO_FILENAME);
|
||||
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
|
||||
|
||||
$mediaUuids = [];
|
||||
$mediaUrls = [];
|
||||
$media_uuids = [];
|
||||
$media_urls = [];
|
||||
|
||||
// Create MediaEngine entries for each format
|
||||
foreach (self::FORMATS as $format => $config) {
|
||||
$url = $this->generateCdnUrl($baseFilename, $config['ext']);
|
||||
$url = $this->generateCdnUrl($base_filename, $config['ext']);
|
||||
|
||||
// Create media entry using save_url mode
|
||||
$media = MediaEngine::addMedia(
|
||||
'temps', // Media collection key
|
||||
$memeRecord['type'], // video or image
|
||||
$meme_record['type'], // video or image
|
||||
'system_uploaded', // Media source
|
||||
'meme_cdn', // Media provider
|
||||
null, // No file content
|
||||
@@ -158,48 +159,56 @@ private function importSingleMeme(array $memeRecord): void
|
||||
'save_url', // Mode: just save URL reference
|
||||
null, // Auto-generate filename
|
||||
'r2', // Disk (not used for URL mode)
|
||||
trim($memeRecord['name'])." ({$format})", // Name with format
|
||||
trim($meme_record['name'])." ({$format})", // Name with format
|
||||
null, // No specific user
|
||||
$config['mime'] // MIME type
|
||||
);
|
||||
|
||||
$mediaUuids[$format.'_uuid'] = $media->uuid;
|
||||
$mediaUrls[$format.'_url'] = $url;
|
||||
$media_uuids[$format.'_uuid'] = $media->uuid;
|
||||
$media_urls[$format.'_url'] = $url;
|
||||
}
|
||||
|
||||
$embedding = CloudflareAI::getVectorEmbeddingBgeSmall($meme_record['name'].' '.$meme_record['description'].' '.$meme_record['keywords']);
|
||||
|
||||
// Create MemeMedia record
|
||||
MemeMedia::create([
|
||||
'type' => $memeRecord['type'],
|
||||
'sub_type' => $memeRecord['sub_type'],
|
||||
'name' => trim($memeRecord['name']),
|
||||
'description' => $memeRecord['description'],
|
||||
'keywords' => $memeRecord['keywords'],
|
||||
|
||||
// UUIDs from MediaEngine
|
||||
'mov_uuid' => $mediaUuids['mov_uuid'],
|
||||
'webm_uuid' => $mediaUuids['webm_uuid'],
|
||||
'gif_uuid' => $mediaUuids['gif_uuid'],
|
||||
'webp_uuid' => $mediaUuids['webp_uuid'],
|
||||
$meme_media = MemeMedia::where('original_id', $meme_record['filename'])->first();
|
||||
|
||||
// Direct CDN URLs
|
||||
'mov_url' => $mediaUrls['mov_url'],
|
||||
'webm_url' => $mediaUrls['webm_url'],
|
||||
'gif_url' => $mediaUrls['gif_url'],
|
||||
'webp_url' => $mediaUrls['webp_url'],
|
||||
if (is_null($meme_media)) {
|
||||
MemeMedia::create([
|
||||
'original_id' => $meme_record['filename'],
|
||||
'type' => $meme_record['type'],
|
||||
'sub_type' => $meme_record['sub_type'],
|
||||
'name' => trim($meme_record['name']),
|
||||
'description' => $meme_record['description'],
|
||||
'keywords' => $meme_record['keywords'],
|
||||
|
||||
// Embedding will be null initially
|
||||
'embedding' => null,
|
||||
]);
|
||||
// UUIDs from MediaEngine
|
||||
'mov_uuid' => $media_uuids['mov_uuid'],
|
||||
'webm_uuid' => $media_uuids['webm_uuid'],
|
||||
'gif_uuid' => $media_uuids['gif_uuid'],
|
||||
'webp_uuid' => $media_uuids['webp_uuid'],
|
||||
|
||||
$this->command->info('✅ Imported: '.trim($memeRecord['name']));
|
||||
// Direct CDN URLs
|
||||
'mov_url' => $media_urls['mov_url'],
|
||||
'webm_url' => $media_urls['webm_url'],
|
||||
'gif_url' => $media_urls['gif_url'],
|
||||
'webp_url' => $media_urls['webp_url'],
|
||||
|
||||
// Embedding will be null initially
|
||||
'embedding' => $embedding,
|
||||
]);
|
||||
}
|
||||
|
||||
$this->command->info('✅ Imported: '.trim($meme_record['name']));
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate CDN URL for specific format
|
||||
*/
|
||||
private function generateCdnUrl(string $baseFilename, string $extension): string
|
||||
private function generateCdnUrl(string $base_filename, string $extension): string
|
||||
{
|
||||
return self::CDN_BASE_URL."/{$extension}/{$baseFilename}.{$extension}";
|
||||
return self::CDN_BASE_URL."/{$extension}/{$base_filename}.{$extension}";
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user