charlestehio/memefast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
ct
2025-06-13 12:53:13 +08:00
parent 53690753c6
commit 8a1bd93b82
1 changed files with 109 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
database/seeders/MemeMediaSeeder.php
View File

@@ -1,7 +1,7 @@
<?php
/**
* MemeMediaSeeder - Clean Laravel Seeder
* MemeMediaSeeder - PostgreSQL Compatible Laravel Seeder
*
* Usage: php artisan db:seed --class=MemeMediaSeeder
*
@@ -14,6 +14,7 @@
* - Creates 4 MediaEngine entries per meme (webm, mov, webp, gif)
* - Creates MemeMedia records with all UUIDs and URLs
* - Uses save_url mode for fast CDN references
* - PostgreSQL transaction-safe with individual record transactions
*/
namespace Database\Seeders;
@@ -52,41 +53,40 @@ public function run(): void
$this->command->info('📊 Found '.count($meme_data).' memes to import');
// Process in chunks for memory efficiency
$chunks = array_chunk($meme_data, 50);
// Process records individually for PostgreSQL compatibility
$total_processed = 0;
$total_skipped = 0;
$total_failed = 0;
foreach ($chunks as $chunk_index => $chunk) {
$this->command->info('Processing chunk '.($chunk_index + 1).'/'.count($chunks));
foreach ($meme_data as $index => $meme_record) {
$this->command->info('Processing '.($index + 1).'/'.count($meme_data).': '.$meme_record['filename']);
DB::transaction(function () use ($chunk, &$total_processed, &$total_skipped, &$total_failed) {
foreach ($chunk as $meme_record) {
try {
// Extract base filename for duplicate checking
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
try {
// Check for duplicates OUTSIDE of transaction
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
// Check if already exists by checking if webm_url contains this filename
if (MemeMedia::where('webm_url', 'like', "%/{$base_filename}.webm")->exists()) {
$this->command->warn("⏭️ Skipping existing: {$meme_record['filename']} ({$meme_record['name']})");
$total_skipped++;
if ($this->isDuplicate($base_filename)) {
$this->command->warn("⏭️ Skipping existing: {$meme_record['filename']} ({$meme_record['name']})");
$total_skipped++;
continue;
}
$this->importSingleMeme($meme_record);
$total_processed++;
if ($total_processed % 10 === 0) {
$this->command->info("✅ Processed {$total_processed} memes...");
}
} catch (\Exception $e) {
$total_failed++;
$this->command->error("❌ Failed to import: {$meme_record['filename']} - {$e->getMessage()}");
}
continue;
}
});
// Process single meme in its own transaction
$result = $this->importSingleMemeWithTransaction($meme_record);
if ($result) {
$total_processed++;
if ($total_processed % 10 === 0) {
$this->command->info("✅ Processed {$total_processed} memes...");
}
} else {
$total_failed++;
}
} catch (\Exception $e) {
$total_failed++;
$this->command->error("❌ Failed to import: {$meme_record['filename']} - {$e->getMessage()}");
}
}
// Summary
@@ -103,6 +103,30 @@ public function run(): void
$this->command->info("📊 Created {$total_media_records} media records and {$total_processed} meme_media records");
}
/**
* Check if meme already exists (outside transaction)
*/
private function isDuplicate(string $base_filename): bool
{
try {
return MemeMedia::where('webm_url', 'like', "%/{$base_filename}.webm")->exists();
} catch (\Exception $e) {
$this->command->warn("⚠️ Could not check duplicate for {$base_filename}: {$e->getMessage()}");
return false;
}
}
/**
* Import single meme within its own transaction
*/
private function importSingleMemeWithTransaction(array $meme_record): bool
{
return DB::transaction(function () use ($meme_record) {
return $this->importSingleMeme($meme_record);
});
}
/**
* Parse CSV file and return array of meme data
*/
@@ -136,45 +160,64 @@ private function parseCsvFile(string $csv_path): array
/**
* Import a single meme with all its formats
*/
private function importSingleMeme(array $meme_record): void
private function importSingleMeme(array $meme_record): bool
{
// Extract base filename (remove .webm extension)
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
try {
// Extract base filename (remove .webm extension)
$base_filename = pathinfo($meme_record['filename'], PATHINFO_FILENAME);
$media_uuids = [];
$media_urls = [];
$media_uuids = [];
$media_urls = [];
// Create MediaEngine entries for each format
foreach (self::FORMATS as $format => $config) {
$url = $this->generateCdnUrl($base_filename, $config['ext']);
// Create MediaEngine entries for each format
foreach (self::FORMATS as $format => $config) {
$url = $this->generateCdnUrl($base_filename, $config['ext']);
// Create media entry using save_url mode
$media = MediaEngine::addMedia(
'temps', // Media collection key
$meme_record['type'], // video or image
'system_uploaded', // Media source
'meme_cdn', // Media provider
null, // No file content
$url, // CDN URL
'save_url', // Mode: just save URL reference
null, // Auto-generate filename
'r2', // Disk (not used for URL mode)
trim($meme_record['name'])." ({$format})", // Name with format
null, // No specific user
$config['mime'] // MIME type
);
try {
// Create media entry using save_url mode
$media = MediaEngine::addMedia(
'temps', // Media collection key
$meme_record['type'], // video or image
'system_uploaded', // Media source
'meme_cdn', // Media provider
null, // No file content
$url, // CDN URL
'save_url', // Mode: just save URL reference
null, // Auto-generate filename
'r2', // Disk (not used for URL mode)
trim($meme_record['name'])." ({$format})", // Name with format
null, // No specific user
$config['mime'] // MIME type
);
$media_uuids[$format.'_uuid'] = $media->uuid;
$media_urls[$format.'_url'] = $url;
}
$media_uuids[$format.'_uuid'] = $media->uuid;
$media_urls[$format.'_url'] = $url;
} catch (\Exception $e) {
$this->command->error("Failed to create {$format} media for {$meme_record['filename']}: {$e->getMessage()}");
throw $e;
}
}
$embedding = CloudflareAI::getVectorEmbeddingBgeSmall($meme_record['name'].' '.$meme_record['description'].' '.$meme_record['keywords']);
// Generate embedding
try {
$embedding = CloudflareAI::getVectorEmbeddingBgeSmall(
$meme_record['name'].' '.$meme_record['description'].' '.$meme_record['keywords']
);
} catch (\Exception $e) {
$this->command->warn("Failed to generate embedding for {$meme_record['filename']}: {$e->getMessage()}");
$embedding = null; // Continue without embedding
}
// Create MemeMedia record
// Check if record exists one more time within transaction
$existing_meme = MemeMedia::where('original_id', $meme_record['filename'])->first();
$meme_media = MemeMedia::where('original_id', $meme_record['filename'])->first();
if ($existing_meme) {
$this->command->warn("Record already exists for {$meme_record['filename']}, skipping...");
if (is_null($meme_media)) {
return false;
}
// Create MemeMedia record
MemeMedia::create([
'original_id' => $meme_record['filename'],
'type' => $meme_record['type'],
@@ -195,12 +238,17 @@ private function importSingleMeme(array $meme_record): void
'gif_url' => $media_urls['gif_url'],
'webp_url' => $media_urls['webp_url'],
// Embedding will be null initially
// Embedding (may be null)
'embedding' => $embedding,
]);
}
$this->command->info('✅ Imported: '.trim($meme_record['name']));
$this->command->info('✅ Imported: '.trim($meme_record['name']));
return true;
} catch (\Exception $e) {
$this->command->error("Error importing {$meme_record['filename']}: {$e->getMessage()}");
throw $e;
}
}
/**