This commit is contained in:
ct
2025-06-13 12:53:13 +08:00
parent 53690753c6
commit 8a1bd93b82

View File

@@ -1,7 +1,7 @@
<?php <?php
/** /**
* MemeMediaSeeder - Clean Laravel Seeder * MemeMediaSeeder - PostgreSQL Compatible Laravel Seeder
* *
* Usage: php artisan db:seed --class=MemeMediaSeeder * Usage: php artisan db:seed --class=MemeMediaSeeder
* *
@@ -14,6 +14,7 @@
* - Creates 4 MediaEngine entries per meme (webm, mov, webp, gif) * - Creates 4 MediaEngine entries per meme (webm, mov, webp, gif)
* - Creates MemeMedia records with all UUIDs and URLs * - Creates MemeMedia records with all UUIDs and URLs
* - Uses save_url mode for fast CDN references * - Uses save_url mode for fast CDN references
* - PostgreSQL transaction-safe with individual record transactions
*/ */
namespace Database\Seeders; namespace Database\Seeders;
@@ -52,42 +53,41 @@ public function run(): void
$this->command->info('📊 Found '.count($meme_data).' memes to import'); $this->command->info('📊 Found '.count($meme_data).' memes to import');
// Process in chunks for memory efficiency // Process records individually for PostgreSQL compatibility
$chunks = array_chunk($meme_data, 50);
$total_processed = 0; $total_processed = 0;
$total_skipped = 0; $total_skipped = 0;
$total_failed = 0; $total_failed = 0;
foreach ($chunks as $chunk_index => $chunk) { foreach ($meme_data as $index => $meme_record) {
$this->command->info('Processing chunk '.($chunk_index + 1).'/'.count($chunks)); $this->command->info('Processing '.($index + 1).'/'.count($meme_data).': '.$meme_record['filename']);
DB::transaction(function () use ($chunk, &$total_processed, &$total_skipped, &$total_failed) {
foreach ($chunk as $meme_record) {
try { try {
// Extract base filename for duplicate checking // Check for duplicates OUTSIDE of transaction
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME); $base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
// Check if already exists by checking if webm_url contains this filename if ($this->isDuplicate($base_filename)) {
if (MemeMedia::where('webm_url', 'like', "%/{$base_filename}.webm")->exists()) {
$this->command->warn("⏭️ Skipping existing: {$meme_record['filename']} ({$meme_record['name']})"); $this->command->warn("⏭️ Skipping existing: {$meme_record['filename']} ({$meme_record['name']})");
$total_skipped++; $total_skipped++;
continue; continue;
} }
$this->importSingleMeme($meme_record); // Process single meme in its own transaction
$total_processed++; $result = $this->importSingleMemeWithTransaction($meme_record);
if ($result) {
$total_processed++;
if ($total_processed % 10 === 0) { if ($total_processed % 10 === 0) {
$this->command->info("✅ Processed {$total_processed} memes..."); $this->command->info("✅ Processed {$total_processed} memes...");
} }
} else {
$total_failed++;
}
} catch (\Exception $e) { } catch (\Exception $e) {
$total_failed++; $total_failed++;
$this->command->error("❌ Failed to import: {$meme_record['filename']} - {$e->getMessage()}"); $this->command->error("❌ Failed to import: {$meme_record['filename']} - {$e->getMessage()}");
} }
} }
});
}
// Summary // Summary
$this->command->info("\n🎯 Import Summary:"); $this->command->info("\n🎯 Import Summary:");
@@ -103,6 +103,30 @@ public function run(): void
$this->command->info("📊 Created {$total_media_records} media records and {$total_processed} meme_media records"); $this->command->info("📊 Created {$total_media_records} media records and {$total_processed} meme_media records");
} }
/**
* Check if meme already exists (outside transaction)
*/
private function isDuplicate(string $base_filename): bool
{
try {
return MemeMedia::where('webm_url', 'like', "%/{$base_filename}.webm")->exists();
} catch (\Exception $e) {
$this->command->warn("⚠️ Could not check duplicate for {$base_filename}: {$e->getMessage()}");
return false;
}
}
/**
* Import single meme within its own transaction
*/
private function importSingleMemeWithTransaction(array $meme_record): bool
{
return DB::transaction(function () use ($meme_record) {
return $this->importSingleMeme($meme_record);
});
}
/** /**
* Parse CSV file and return array of meme data * Parse CSV file and return array of meme data
*/ */
@@ -136,8 +160,9 @@ private function parseCsvFile(string $csv_path): array
/** /**
* Import a single meme with all its formats * Import a single meme with all its formats
*/ */
private function importSingleMeme(array $meme_record): void private function importSingleMeme(array $meme_record): bool
{ {
try {
// Extract base filename (remove .webm extension) // Extract base filename (remove .webm extension)
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME); $base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
@@ -148,6 +173,7 @@ private function importSingleMeme(array $meme_record): void
foreach (self::FORMATS as $format => $config) { foreach (self::FORMATS as $format => $config) {
$url = $this->generateCdnUrl($base_filename, $config['ext']); $url = $this->generateCdnUrl($base_filename, $config['ext']);
try {
// Create media entry using save_url mode // Create media entry using save_url mode
$media = MediaEngine::addMedia( $media = MediaEngine::addMedia(
'temps', // Media collection key 'temps', // Media collection key
@@ -166,15 +192,32 @@ private function importSingleMeme(array $meme_record): void
$media_uuids[$format.'_uuid'] = $media->uuid; $media_uuids[$format.'_uuid'] = $media->uuid;
$media_urls[$format.'_url'] = $url; $media_urls[$format.'_url'] = $url;
} catch (\Exception $e) {
$this->command->error("Failed to create {$format} media for {$meme_record['filename']}: {$e->getMessage()}");
throw $e;
}
} }
$embedding = CloudflareAI::getVectorEmbeddingBgeSmall($meme_record['name'].' '.$meme_record['description'].' '.$meme_record['keywords']); // Generate embedding
try {
$embedding = CloudflareAI::getVectorEmbeddingBgeSmall(
$meme_record['name'].' '.$meme_record['description'].' '.$meme_record['keywords']
);
} catch (\Exception $e) {
$this->command->warn("Failed to generate embedding for {$meme_record['filename']}: {$e->getMessage()}");
$embedding = null; // Continue without embedding
}
// Check if record exists one more time within transaction
$existing_meme = MemeMedia::where('original_id', $meme_record['filename'])->first();
if ($existing_meme) {
$this->command->warn("Record already exists for {$meme_record['filename']}, skipping...");
return false;
}
// Create MemeMedia record // Create MemeMedia record
$meme_media = MemeMedia::where('original_id', $meme_record['filename'])->first();
if (is_null($meme_media)) {
MemeMedia::create([ MemeMedia::create([
'original_id' => $meme_record['filename'], 'original_id' => $meme_record['filename'],
'type' => $meme_record['type'], 'type' => $meme_record['type'],
@@ -195,12 +238,17 @@ private function importSingleMeme(array $meme_record): void
'gif_url' => $media_urls['gif_url'], 'gif_url' => $media_urls['gif_url'],
'webp_url' => $media_urls['webp_url'], 'webp_url' => $media_urls['webp_url'],
// Embedding will be null initially // Embedding (may be null)
'embedding' => $embedding, 'embedding' => $embedding,
]); ]);
}
$this->command->info('✅ Imported: '.trim($meme_record['name'])); $this->command->info('✅ Imported: '.trim($meme_record['name']));
return true;
} catch (\Exception $e) {
$this->command->error("Error importing {$meme_record['filename']}: {$e->getMessage()}");
throw $e;
}
} }
/** /**