charlestehio/memefast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
ct
2025-06-13 12:53:13 +08:00
parent 53690753c6
commit 8a1bd93b82
1 changed files with 109 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
database/seeders/MemeMediaSeeder.php
View File

@@ -1,7 +1,7 @@
<?php <?php
/** /**
* MemeMediaSeeder - Clean Laravel Seeder * MemeMediaSeeder - PostgreSQL Compatible Laravel Seeder
* *
* Usage: php artisan db:seed --class=MemeMediaSeeder * Usage: php artisan db:seed --class=MemeMediaSeeder
* *
@@ -14,6 +14,7 @@
* - Creates 4 MediaEngine entries per meme (webm, mov, webp, gif) * - Creates 4 MediaEngine entries per meme (webm, mov, webp, gif)
* - Creates MemeMedia records with all UUIDs and URLs * - Creates MemeMedia records with all UUIDs and URLs
* - Uses save_url mode for fast CDN references * - Uses save_url mode for fast CDN references
* - PostgreSQL transaction-safe with individual record transactions
*/ */
namespace Database\Seeders; namespace Database\Seeders;
@@ -52,41 +53,40 @@ public function run(): void
$this->command->info('📊 Found '.count($meme_data).' memes to import'); $this->command->info('📊 Found '.count($meme_data).' memes to import');
// Process in chunks for memory efficiency // Process records individually for PostgreSQL compatibility
$chunks = array_chunk($meme_data, 50);
$total_processed = 0; $total_processed = 0;
$total_skipped = 0; $total_skipped = 0;
$total_failed = 0; $total_failed = 0;
foreach ($chunks as $chunk_index => $chunk) { foreach ($meme_data as $index => $meme_record) {
$this->command->info('Processing chunk '.($chunk_index + 1).'/'.count($chunks)); $this->command->info('Processing '.($index + 1).'/'.count($meme_data).': '.$meme_record['filename']);
DB::transaction(function () use ($chunk, &$total_processed, &$total_skipped, &$total_failed) { try {
foreach ($chunk as $meme_record) { // Check for duplicates OUTSIDE of transaction
try { $base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
// Extract base filename for duplicate checking
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
// Check if already exists by checking if webm_url contains this filename if ($this->isDuplicate($base_filename)) {
if (MemeMedia::where('webm_url', 'like', "%/{$base_filename}.webm")->exists()) { $this->command->warn("⏭️ Skipping existing: {$meme_record['filename']} ({$meme_record['name']})");
$this->command->warn("⏭️ Skipping existing: {$meme_record['filename']} ({$meme_record['name']})"); $total_skipped++;
$total_skipped++;
continue; continue;
}
$this->importSingleMeme($meme_record);
$total_processed++;
if ($total_processed % 10 === 0) {
$this->command->info("✅ Processed {$total_processed} memes...");
}
} catch (\Exception $e) {
$total_failed++;
$this->command->error("❌ Failed to import: {$meme_record['filename']} - {$e->getMessage()}");
}
} }
});
// Process single meme in its own transaction
$result = $this->importSingleMemeWithTransaction($meme_record);
if ($result) {
$total_processed++;
if ($total_processed % 10 === 0) {
$this->command->info("✅ Processed {$total_processed} memes...");
}
} else {
$total_failed++;
}
} catch (\Exception $e) {
$total_failed++;
$this->command->error("❌ Failed to import: {$meme_record['filename']} - {$e->getMessage()}");
}
} }
// Summary // Summary
@@ -103,6 +103,30 @@ public function run(): void
$this->command->info("📊 Created {$total_media_records} media records and {$total_processed} meme_media records"); $this->command->info("📊 Created {$total_media_records} media records and {$total_processed} meme_media records");
} }
/**
* Check if meme already exists (outside transaction)
*/
private function isDuplicate(string $base_filename): bool
{
try {
return MemeMedia::where('webm_url', 'like', "%/{$base_filename}.webm")->exists();
} catch (\Exception $e) {
$this->command->warn("⚠️ Could not check duplicate for {$base_filename}: {$e->getMessage()}");
return false;
}
}
/**
* Import single meme within its own transaction
*/
private function importSingleMemeWithTransaction(array $meme_record): bool
{
return DB::transaction(function () use ($meme_record) {
return $this->importSingleMeme($meme_record);
});
}
/** /**
* Parse CSV file and return array of meme data * Parse CSV file and return array of meme data
*/ */
@@ -136,45 +160,64 @@ private function parseCsvFile(string $csv_path): array
/** /**
* Import a single meme with all its formats * Import a single meme with all its formats
*/ */
private function importSingleMeme(array $meme_record): void private function importSingleMeme(array $meme_record): bool
{ {
// Extract base filename (remove .webm extension) try {
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME); // Extract base filename (remove .webm extension)
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
$media_uuids = []; $media_uuids = [];
$media_urls = []; $media_urls = [];
// Create MediaEngine entries for each format // Create MediaEngine entries for each format
foreach (self::FORMATS as $format => $config) { foreach (self::FORMATS as $format => $config) {
$url = $this->generateCdnUrl($base_filename, $config['ext']); $url = $this->generateCdnUrl($base_filename, $config['ext']);
// Create media entry using save_url mode try {
$media = MediaEngine::addMedia( // Create media entry using save_url mode
'temps', // Media collection key $media = MediaEngine::addMedia(
$meme_record['type'], // video or image 'temps', // Media collection key
'system_uploaded', // Media source $meme_record['type'], // video or image
'meme_cdn', // Media provider 'system_uploaded', // Media source
null, // No file content 'meme_cdn', // Media provider
$url, // CDN URL null, // No file content
'save_url', // Mode: just save URL reference $url, // CDN URL
null, // Auto-generate filename 'save_url', // Mode: just save URL reference
'r2', // Disk (not used for URL mode) null, // Auto-generate filename
trim($meme_record['name'])." ({$format})", // Name with format 'r2', // Disk (not used for URL mode)
null, // No specific user trim($meme_record['name'])." ({$format})", // Name with format
$config['mime'] // MIME type null, // No specific user
); $config['mime'] // MIME type
);
$media_uuids[$format.'_uuid'] = $media->uuid; $media_uuids[$format.'_uuid'] = $media->uuid;
$media_urls[$format.'_url'] = $url; $media_urls[$format.'_url'] = $url;
} } catch (\Exception $e) {
$this->command->error("Failed to create {$format} media for {$meme_record['filename']}: {$e->getMessage()}");
throw $e;
}
}
$embedding = CloudflareAI::getVectorEmbeddingBgeSmall($meme_record['name'].' '.$meme_record['description'].' '.$meme_record['keywords']); // Generate embedding
try {
$embedding = CloudflareAI::getVectorEmbeddingBgeSmall(
$meme_record['name'].' '.$meme_record['description'].' '.$meme_record['keywords']
);
} catch (\Exception $e) {
$this->command->warn("Failed to generate embedding for {$meme_record['filename']}: {$e->getMessage()}");
$embedding = null; // Continue without embedding
}
// Create MemeMedia record // Check if record exists one more time within transaction
$existing_meme = MemeMedia::where('original_id', $meme_record['filename'])->first();
$meme_media = MemeMedia::where('original_id', $meme_record['filename'])->first(); if ($existing_meme) {
$this->command->warn("Record already exists for {$meme_record['filename']}, skipping...");
if (is_null($meme_media)) { return false;
}
// Create MemeMedia record
MemeMedia::create([ MemeMedia::create([
'original_id' => $meme_record['filename'], 'original_id' => $meme_record['filename'],
'type' => $meme_record['type'], 'type' => $meme_record['type'],
@@ -195,12 +238,17 @@ private function importSingleMeme(array $meme_record): void
'gif_url' => $media_urls['gif_url'], 'gif_url' => $media_urls['gif_url'],
'webp_url' => $media_urls['webp_url'], 'webp_url' => $media_urls['webp_url'],
// Embedding will be null initially // Embedding (may be null)
'embedding' => $embedding, 'embedding' => $embedding,
]); ]);
}
$this->command->info('✅ Imported: '.trim($meme_record['name'])); $this->command->info('✅ Imported: '.trim($meme_record['name']));
return true;
} catch (\Exception $e) {
$this->command->error("Error importing {$meme_record['filename']}: {$e->getMessage()}");
throw $e;
}
} }
/** /**