From 096f515f58ad2c33ad9559e28942fcb85ab87c42 Mon Sep 17 00:00:00 2001 From: ct Date: Tue, 15 Jul 2025 20:03:10 +0800 Subject: [PATCH] Update --- .env.example | 4 + app/Console/Commands/FindDuplicateImages.php | 124 ++++++++ app/Console/Commands/GenerateImageHashes.php | 84 +++++ .../FirstParty/ImageHash/ImageHashService.php | 100 ++++++ .../Controllers/AdminDuplicateController.php | 170 ++++++++++ app/Http/Middleware/BasicAuthMiddleware.php | 47 +++ app/Models/MemeMedia.php | 2 + app/Observers/MemeMediaObserver.php | 68 ++++ app/Providers/AppServiceProvider.php | 4 +- composer.json | 1 + composer.lock | 219 ++++++++++++- ...33_add_image_hash_to_meme_medias_table.php | 28 ++ .../js/pages/admin/duplicate-management.tsx | 295 ++++++++++++++++++ resources/js/ziggy.js | 2 +- routes/web.php | 16 + 15 files changed, 1161 insertions(+), 3 deletions(-) create mode 100644 app/Console/Commands/FindDuplicateImages.php create mode 100644 app/Console/Commands/GenerateImageHashes.php create mode 100644 app/Helpers/FirstParty/ImageHash/ImageHashService.php create mode 100644 app/Http/Controllers/AdminDuplicateController.php create mode 100644 app/Http/Middleware/BasicAuthMiddleware.php create mode 100644 app/Observers/MemeMediaObserver.php create mode 100644 database/migrations/2025_07_15_101333_add_image_hash_to_meme_medias_table.php create mode 100644 resources/js/pages/admin/duplicate-management.tsx diff --git a/.env.example b/.env.example index 35db1dd..ce13ac3 100644 --- a/.env.example +++ b/.env.example @@ -63,3 +63,7 @@ AWS_BUCKET= AWS_USE_PATH_STYLE_ENDPOINT=false VITE_APP_NAME="${APP_NAME}" + +# Basic Auth for Admin Tools +BASIC_AUTH_USERNAME=admin +BASIC_AUTH_PASSWORD=changeme diff --git a/app/Console/Commands/FindDuplicateImages.php b/app/Console/Commands/FindDuplicateImages.php new file mode 100644 index 0000000..6bd3430 --- /dev/null +++ b/app/Console/Commands/FindDuplicateImages.php @@ -0,0 +1,124 @@ +imageHashService = $imageHashService; + } + + /** + * Execute the console command. + */ + public function handle() + { + $threshold = (int) $this->option('threshold'); + + $records = MemeMedia::whereNotNull('image_hash') + ->whereNotNull('webp_url') + ->get(['id', 'name', 'image_hash', 'webp_url']); + + if ($records->isEmpty()) { + $this->info('No records with image hashes found. Run app:generate-image-hashes first.'); + + return; + } + + $this->info("Checking {$records->count()} records for duplicates with threshold: {$threshold}"); + + $duplicates = []; + $processed = []; + + foreach ($records as $record) { + if (in_array($record->id, $processed)) { + continue; + } + + $similarRecords = []; + + foreach ($records as $compareRecord) { + if ($record->id === $compareRecord->id || in_array($compareRecord->id, $processed)) { + continue; + } + + $distance = $this->imageHashService->calculateHammingDistance( + $record->image_hash, + $compareRecord->image_hash + ); + + if ($distance <= $threshold) { + $similarRecords[] = [ + 'id' => $compareRecord->id, + 'name' => $compareRecord->name, + 'distance' => $distance, + 'url' => $compareRecord->webp_url, + ]; + $processed[] = $compareRecord->id; + } + } + + if (! empty($similarRecords)) { + $duplicates[] = [ + 'original' => [ + 'id' => $record->id, + 'name' => $record->name, + 'url' => $record->webp_url, + ], + 'duplicates' => $similarRecords, + ]; + $processed[] = $record->id; + } + } + + if (empty($duplicates)) { + $this->info('No duplicates found.'); + + return; + } + + $this->info('Found '.count($duplicates).' duplicate groups:'); + + foreach ($duplicates as $group) { + $this->newLine(); + $this->line("Original: [{$group['original']['id']}] {$group['original']['name']}"); + $this->line("URL: {$group['original']['url']}"); + + foreach ($group['duplicates'] as $duplicate) { + $this->line(" → [{$duplicate['id']}] {$duplicate['name']} (distance: {$duplicate['distance']})"); + $this->line(" URL: {$duplicate['url']}"); + } + } + + $this->newLine(); + $this->info('Total duplicate groups: '.count($duplicates)); + + $totalDuplicates = array_sum(array_map(function ($group) { + return count($group['duplicates']); + }, $duplicates)); + + $this->info("Total duplicate records: {$totalDuplicates}"); + } +} diff --git a/app/Console/Commands/GenerateImageHashes.php b/app/Console/Commands/GenerateImageHashes.php new file mode 100644 index 0000000..8ff525d --- /dev/null +++ b/app/Console/Commands/GenerateImageHashes.php @@ -0,0 +1,84 @@ +imageHashService = $imageHashService; + } + + /** + * Execute the console command. + */ + public function handle() + { + $force = $this->option('force'); + + $query = MemeMedia::query(); + + if (! $force) { + $query->whereNull('image_hash'); + } + + $records = $query->whereNotNull('webp_url')->get(); + + if ($records->isEmpty()) { + $this->info('No records found to process.'); + + return; + } + + $this->info("Processing {$records->count()} records..."); + + $progressBar = $this->output->createProgressBar($records->count()); + $progressBar->start(); + + $processed = 0; + $failed = 0; + + foreach ($records as $record) { + $hash = $this->imageHashService->generateHashFromUrl($record->webp_url); + + if ($hash) { + $record->update(['image_hash' => $hash]); + $processed++; + } else { + $failed++; + $this->newLine(); + $this->error("Failed to generate hash for ID: {$record->id} - {$record->webp_url}"); + } + + $progressBar->advance(); + } + + $progressBar->finish(); + $this->newLine(); + + $this->info('Processing complete!'); + $this->info("Processed: {$processed}"); + $this->info("Failed: {$failed}"); + } +} diff --git a/app/Helpers/FirstParty/ImageHash/ImageHashService.php b/app/Helpers/FirstParty/ImageHash/ImageHashService.php new file mode 100644 index 0000000..76e8805 --- /dev/null +++ b/app/Helpers/FirstParty/ImageHash/ImageHashService.php @@ -0,0 +1,100 @@ +hasher = new ImageHash(new DifferenceHash); + } + + public function generateHashFromUrl(string $url): ?string + { + try { + $response = Http::timeout(30)->get($url); + + if (! $response->successful()) { + Log::warning("Failed to download image from URL: {$url}"); + + return null; + } + + $imageData = $response->body(); + + return $this->generateHashFromData($imageData); + + } catch (\Exception $e) { + Log::error("Error generating hash from URL {$url}: ".$e->getMessage()); + + return null; + } + } + + public function generateHashFromData(string $imageData): ?string + { + try { + $tempFile = tempnam(sys_get_temp_dir(), 'imagehash_'); + file_put_contents($tempFile, $imageData); + + $hash = $this->hasher->hash($tempFile); + unlink($tempFile); + + return $hash->toHex(); + + } catch (\Exception $e) { + Log::error('Error generating hash from image data: '.$e->getMessage()); + + return null; + } + } + + public function calculateHammingDistance(string $hash1, string $hash2): int + { + // Validate hashes are not empty + if (empty($hash1) || empty($hash2)) { + return PHP_INT_MAX; // Return max distance for invalid hashes + } + + // Pad shorter hash with zeros to make them equal length + $maxLength = max(strlen($hash1), strlen($hash2)); + $hash1 = str_pad($hash1, $maxLength, '0', STR_PAD_LEFT); + $hash2 = str_pad($hash2, $maxLength, '0', STR_PAD_LEFT); + + $distance = 0; + for ($i = 0; $i < $maxLength; $i++) { + if ($hash1[$i] !== $hash2[$i]) { + $distance++; + } + } + + return $distance; + } + + public function areHashesSimilar(string $hash1, string $hash2, int $threshold = 5): bool + { + return $this->calculateHammingDistance($hash1, $hash2) <= $threshold; + } + + public function findSimilarHashes(string $targetHash, array $hashes, int $threshold = 5): array + { + $similar = []; + + foreach ($hashes as $id => $hash) { + if ($this->areHashesSimilar($targetHash, $hash, $threshold)) { + $similar[$id] = $this->calculateHammingDistance($targetHash, $hash); + } + } + + asort($similar); + + return $similar; + } +} diff --git a/app/Http/Controllers/AdminDuplicateController.php b/app/Http/Controllers/AdminDuplicateController.php new file mode 100644 index 0000000..e756031 --- /dev/null +++ b/app/Http/Controllers/AdminDuplicateController.php @@ -0,0 +1,170 @@ +imageHashService = $imageHashService; + } + + public function index() + { + return Inertia::render('admin/duplicate-management', [ + 'title' => 'Duplicate Management', + ]); + } + + public function scan(Request $request) + { + $threshold = $request->input('threshold', 5); + + $records = MemeMedia::whereNotNull('image_hash') + ->whereNotNull('webp_url') + ->where('image_hash', '!=', '') + ->get(['id', 'name', 'image_hash', 'webp_url', 'group']); + + if ($records->isEmpty()) { + return response()->json([ + 'duplicates' => [], + 'message' => 'No records with image hashes found.', + ]); + } + + $duplicates = []; + $processed = []; + + foreach ($records as $record) { + if (in_array($record->id, $processed)) { + continue; + } + + // Skip if hash is empty or invalid + if (empty($record->image_hash)) { + continue; + } + + $similarRecords = []; + + foreach ($records as $compareRecord) { + if ($record->id === $compareRecord->id || in_array($compareRecord->id, $processed)) { + continue; + } + + // Skip if either hash is empty or invalid + if (empty($record->image_hash) || empty($compareRecord->image_hash)) { + continue; + } + + $distance = $this->imageHashService->calculateHammingDistance( + $record->image_hash, + $compareRecord->image_hash + ); + + // Skip if distance calculation failed (returns PHP_INT_MAX) + if ($distance === PHP_INT_MAX) { + continue; + } + + if ($distance <= $threshold) { + $similarRecords[] = [ + 'id' => $compareRecord->id, + 'name' => $compareRecord->name, + 'distance' => $distance, + 'url' => $compareRecord->webp_url, + 'group' => $compareRecord->group, + ]; + $processed[] = $compareRecord->id; + } + } + + if (! empty($similarRecords)) { + // Sort similar records to prioritize group 2, then by ID + usort($similarRecords, function ($a, $b) { + if ($a['group'] == $b['group']) { + return $a['id'] <=> $b['id']; + } + + return $b['group'] <=> $a['group']; // Higher group first + }); + + $duplicates[] = [ + 'original' => [ + 'id' => $record->id, + 'name' => $record->name, + 'url' => $record->webp_url, + 'group' => $record->group, + ], + 'duplicates' => $similarRecords, + ]; + $processed[] = $record->id; + } + } + + return response()->json([ + 'duplicates' => $duplicates, + 'total_groups' => count($duplicates), + 'total_duplicates' => array_sum(array_map(function ($group) { + return count($group['duplicates']); + }, $duplicates)), + ]); + } + + public function delete(Request $request) + { + $request->validate([ + 'id' => 'required|exists:meme_medias,id', + ]); + + $record = MemeMedia::findOrFail($request->id); + + // Soft delete the record + $record->delete(); + + return response()->json([ + 'success' => true, + 'message' => "Deleted '{$record->name}' (Group {$record->group})", + ]); + } + + public function regenerateHash(Request $request) + { + $request->validate([ + 'id' => 'required|exists:meme_medias,id', + ]); + + $record = MemeMedia::findOrFail($request->id); + + if (! $record->webp_url) { + return response()->json([ + 'success' => false, + 'message' => 'No WebP URL found for this record', + ], 400); + } + + $hash = $this->imageHashService->generateHashFromUrl($record->webp_url); + + if (! $hash) { + return response()->json([ + 'success' => false, + 'message' => 'Failed to generate hash', + ], 500); + } + + $record->update(['image_hash' => $hash]); + + return response()->json([ + 'success' => true, + 'message' => 'Hash regenerated successfully', + 'hash' => $hash, + ]); + } +} diff --git a/app/Http/Middleware/BasicAuthMiddleware.php b/app/Http/Middleware/BasicAuthMiddleware.php new file mode 100644 index 0000000..2bb3e9f --- /dev/null +++ b/app/Http/Middleware/BasicAuthMiddleware.php @@ -0,0 +1,47 @@ + 'Basic']); + } + + // Check if Authorization header is present + if (! $request->header('Authorization')) { + return response('Unauthorized', 401, ['WWW-Authenticate' => 'Basic']); + } + + // Extract credentials from Authorization header + $authHeader = $request->header('Authorization'); + if (! str_starts_with($authHeader, 'Basic ')) { + return response('Unauthorized', 401, ['WWW-Authenticate' => 'Basic']); + } + + $credentials = base64_decode(substr($authHeader, 6)); + [$inputUsername, $inputPassword] = explode(':', $credentials, 2); + + // Verify credentials + if ($inputUsername !== $username || $inputPassword !== $password) { + return response('Unauthorized', 401, ['WWW-Authenticate' => 'Basic']); + } + + return $next($request); + } +} diff --git a/app/Models/MemeMedia.php b/app/Models/MemeMedia.php index b9fe8d6..66f599e 100644 --- a/app/Models/MemeMedia.php +++ b/app/Models/MemeMedia.php @@ -68,6 +68,7 @@ class MemeMedia extends Model 'action_keywords', 'emotion_keywords', 'misc_keywords', + 'image_hash', ]; protected $hidden = [ @@ -86,6 +87,7 @@ class MemeMedia extends Model // 'mov_url', // 'webm_url', 'embedding', + 'image_hash', ]; protected $appends = [ diff --git a/app/Observers/MemeMediaObserver.php b/app/Observers/MemeMediaObserver.php new file mode 100644 index 0000000..8850d1b --- /dev/null +++ b/app/Observers/MemeMediaObserver.php @@ -0,0 +1,68 @@ +imageHashService = $imageHashService; + } + + /** + * Handle the MemeMedia "created" event. + */ + public function created(MemeMedia $memeMedia): void + { + $this->generateHashIfNeeded($memeMedia); + } + + /** + * Handle the MemeMedia "updated" event. + */ + public function updated(MemeMedia $memeMedia): void + { + if ($memeMedia->wasChanged('webp_url')) { + $this->generateHashIfNeeded($memeMedia); + } + } + + private function generateHashIfNeeded(MemeMedia $memeMedia): void + { + if ($memeMedia->webp_url && ! $memeMedia->image_hash) { + $hash = $this->imageHashService->generateHashFromUrl($memeMedia->webp_url); + if ($hash) { + $memeMedia->updateQuietly(['image_hash' => $hash]); + } + } + } + + /** + * Handle the MemeMedia "deleted" event. + */ + public function deleted(MemeMedia $memeMedia): void + { + // + } + + /** + * Handle the MemeMedia "restored" event. + */ + public function restored(MemeMedia $memeMedia): void + { + // + } + + /** + * Handle the MemeMedia "force deleted" event. + */ + public function forceDeleted(MemeMedia $memeMedia): void + { + // + } +} diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 452e6b6..1aeb057 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -2,6 +2,8 @@ namespace App\Providers; +use App\Models\MemeMedia; +use App\Observers\MemeMediaObserver; use Illuminate\Support\ServiceProvider; class AppServiceProvider extends ServiceProvider @@ -19,6 +21,6 @@ public function register(): void */ public function boot(): void { - // + MemeMedia::observe(MemeMediaObserver::class); } } diff --git a/composer.json b/composer.json index e35e87f..5247952 100644 --- a/composer.json +++ b/composer.json @@ -12,6 +12,7 @@ "php": "^8.2", "artesaos/seotools": "^1.3", "inertiajs/inertia-laravel": "^2.0", + "jenssegers/imagehash": "^0.10.0", "kalnoy/nestedset": "^6.0", "laravel/cashier": "^15.7", "laravel/framework": "^12.0", diff --git a/composer.lock b/composer.lock index 719f9cd..4336334 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2ec5e47e5246683315eb1f220ccd493b", + "content-hash": "dbe9b4012c67d84fc322f58ddb9af791", "packages": [ { "name": "artesaos/seotools", @@ -1814,6 +1814,223 @@ }, "time": "2025-04-10T15:08:36+00:00" }, + { + "name": "intervention/gif", + "version": "4.2.2", + "source": { + "type": "git", + "url": "https://github.com/Intervention/gif.git", + "reference": "5999eac6a39aa760fb803bc809e8909ee67b451a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Intervention/gif/zipball/5999eac6a39aa760fb803bc809e8909ee67b451a", + "reference": "5999eac6a39aa760fb803bc809e8909ee67b451a", + "shasum": "" + }, + "require": { + "php": "^8.1" + }, + "require-dev": { + "phpstan/phpstan": "^2.1", + "phpunit/phpunit": "^10.0 || ^11.0 || ^12.0", + "slevomat/coding-standard": "~8.0", + "squizlabs/php_codesniffer": "^3.8" + }, + "type": "library", + "autoload": { + "psr-4": { + "Intervention\\Gif\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Oliver Vogel", + "email": "oliver@intervention.io", + "homepage": "https://intervention.io/" + } + ], + "description": "Native PHP GIF Encoder/Decoder", + "homepage": "https://github.com/intervention/gif", + "keywords": [ + "animation", + "gd", + "gif", + "image" + ], + "support": { + "issues": "https://github.com/Intervention/gif/issues", + "source": "https://github.com/Intervention/gif/tree/4.2.2" + }, + "funding": [ + { + "url": "https://paypal.me/interventionio", + "type": "custom" + }, + { + "url": "https://github.com/Intervention", + "type": "github" + }, + { + "url": "https://ko-fi.com/interventionphp", + "type": "ko_fi" + } + ], + "time": "2025-03-29T07:46:21+00:00" + }, + { + "name": "intervention/image", + "version": "3.11.3", + "source": { + "type": "git", + "url": "https://github.com/Intervention/image.git", + "reference": "d0f097b8a3fa8fb758efc9440b513aa3833cda17" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Intervention/image/zipball/d0f097b8a3fa8fb758efc9440b513aa3833cda17", + "reference": "d0f097b8a3fa8fb758efc9440b513aa3833cda17", + "shasum": "" + }, + "require": { + "ext-mbstring": "*", + "intervention/gif": "^4.2", + "php": "^8.1" + }, + "require-dev": { + "mockery/mockery": "^1.6", + "phpstan/phpstan": "^2.1", + "phpunit/phpunit": "^10.0 || ^11.0 || ^12.0", + "slevomat/coding-standard": "~8.0", + "squizlabs/php_codesniffer": "^3.8" + }, + "suggest": { + "ext-exif": "Recommended to be able to read EXIF data properly." + }, + "type": "library", + "autoload": { + "psr-4": { + "Intervention\\Image\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Oliver Vogel", + "email": "oliver@intervention.io", + "homepage": "https://intervention.io/" + } + ], + "description": "PHP image manipulation", + "homepage": "https://image.intervention.io/", + "keywords": [ + "gd", + "image", + "imagick", + "resize", + "thumbnail", + "watermark" + ], + "support": { + "issues": "https://github.com/Intervention/image/issues", + "source": "https://github.com/Intervention/image/tree/3.11.3" + }, + "funding": [ + { + "url": "https://paypal.me/interventionio", + "type": "custom" + }, + { + "url": "https://github.com/Intervention", + "type": "github" + }, + { + "url": "https://ko-fi.com/interventionphp", + "type": "ko_fi" + } + ], + "time": "2025-05-22T17:26:23+00:00" + }, + { + "name": "jenssegers/imagehash", + "version": "v0.10.0", + "source": { + "type": "git", + "url": "https://github.com/jenssegers/imagehash.git", + "reference": "643d8d676c5cbe637199206476015e36548841f0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/jenssegers/imagehash/zipball/643d8d676c5cbe637199206476015e36548841f0", + "reference": "643d8d676c5cbe637199206476015e36548841f0", + "shasum": "" + }, + "require": { + "intervention/image": "^3.3", + "php": "^8.1" + }, + "require-dev": { + "php-coveralls/php-coveralls": "^2.0", + "phpunit/phpunit": "^10" + }, + "suggest": { + "ext-gd": "GD or ImageMagick is required", + "ext-gmp": "GMP results in faster comparisons", + "ext-imagick": "GD or ImageMagick is required" + }, + "type": "library", + "autoload": { + "psr-4": { + "Jenssegers\\ImageHash\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jens Segers", + "homepage": "https://jenssegers.com" + }, + { + "name": "Kenneth Rapp" + } + ], + "description": "Perceptual image hashing for PHP", + "homepage": "https://github.com/jenssegers/imagehash", + "keywords": [ + "ahash", + "dhash", + "hash", + "image hash", + "imagehash", + "perceptual", + "phash" + ], + "support": { + "issues": "https://github.com/jenssegers/imagehash/issues", + "source": "https://github.com/jenssegers/imagehash/tree/v0.10.0" + }, + "funding": [ + { + "url": "https://github.com/jenssegers", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/jenssegers/imagehash", + "type": "tidelift" + } + ], + "time": "2024-06-17T09:53:53+00:00" + }, { "name": "kalnoy/nestedset", "version": "v6.0.6", diff --git a/database/migrations/2025_07_15_101333_add_image_hash_to_meme_medias_table.php b/database/migrations/2025_07_15_101333_add_image_hash_to_meme_medias_table.php new file mode 100644 index 0000000..728fa12 --- /dev/null +++ b/database/migrations/2025_07_15_101333_add_image_hash_to_meme_medias_table.php @@ -0,0 +1,28 @@ +string('image_hash', 64)->nullable()->index(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('meme_medias', function (Blueprint $table) { + $table->dropColumn('image_hash'); + }); + } +}; diff --git a/resources/js/pages/admin/duplicate-management.tsx b/resources/js/pages/admin/duplicate-management.tsx new file mode 100644 index 0000000..3827a0f --- /dev/null +++ b/resources/js/pages/admin/duplicate-management.tsx @@ -0,0 +1,295 @@ +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { Input } from '@/components/ui/input'; +import { Label } from '@/components/ui/label'; +import { useAxios } from '@/plugins/AxiosContext'; +import { Head, usePage } from '@inertiajs/react'; +import { AlertCircle, Loader2, RotateCw, Search, Trash2 } from 'lucide-react'; +import { useState } from 'react'; +import { toast } from 'sonner'; + +interface DuplicateItem { + id: number; + name: string; + distance: number; + url: string; + group: number; +} + +interface DuplicateGroup { + original: { + id: number; + name: string; + url: string; + group: number; + } | null; + duplicates: DuplicateItem[]; +} + +interface ScanResult { + duplicates: DuplicateGroup[]; + total_groups: number; + total_duplicates: number; +} + +export default function DuplicateManagement() { + const { title } = usePage().props; + const axios = useAxios(); + + const [threshold, setThreshold] = useState(5); + const [isScanning, setIsScanning] = useState(false); + const [scanResult, setScanResult] = useState(null); + const [deletingIds, setDeletingIds] = useState([]); + + const handleScan = async () => { + setIsScanning(true); + try { + const response = await axios.post(route('admin.duplicates.scan'), { + threshold, + }); + + if (response.data.duplicates.length === 0) { + toast.info('No duplicate images were found with the current threshold.'); + } + + setScanResult(response.data); + } catch (error) { + console.error('Scan error:', error); + toast.error('Failed to scan for duplicates'); + } finally { + setIsScanning(false); + } + }; + + const handleDelete = async (id: number) => { + setDeletingIds((prev) => [...prev, id]); + try { + const response = await axios.post(route('admin.duplicates.delete'), { + id, + }); + + if (response.data.success) { + toast.success(response.data.message); + + // Remove the deleted item from results + setScanResult((prev) => { + if (!prev) return null; + + return { + ...prev, + duplicates: prev.duplicates.map((group) => { + // If the original was deleted, remove it from the group + if (group.original && group.original.id === id) { + return { + ...group, + original: null, // Mark original as deleted + }; + } + + // Remove the deleted item from duplicates + return { + ...group, + duplicates: group.duplicates.filter((item) => item.id !== id), + }; + }), + }; + }); + } + } catch (error) { + console.error('Delete error:', error); + if (error.response?.data?.message) { + toast.error(error.response.data.message); + } else { + toast.error('Failed to delete image'); + } + } finally { + setDeletingIds((prev) => prev.filter((deletingId) => deletingId !== id)); + } + }; + + const handleRegenerateHash = async (id: number) => { + try { + const response = await axios.post(route('admin.duplicates.regenerate-hash'), { + id, + }); + + if (response.data.success) { + toast.success(response.data.message); + } + } catch (error) { + console.error('Regenerate hash error:', error); + toast.error('Failed to regenerate hash'); + } + }; + + return ( + <> + + +
+
+

Duplicate Management

+

Find and manage duplicate meme images using Hamming distance comparison

+
+ + + + Scan for Duplicates + + +
+
+ + setThreshold(Number(e.target.value))} + min="0" + max="64" + className="mt-1" + /> +

Lower values = stricter matching (0-10 recommended)

+
+ +
+ + + + {scanResult && ( + + + + + Scan Results + +
+ Groups: {scanResult.total_groups} + Total Duplicates: {scanResult.total_duplicates} +
+ + + {scanResult.duplicates.length === 0 ? ( +

No duplicates found

+ ) : ( +
+ {scanResult.duplicates.map((group, index) => ( +
+
+

Duplicate Group {index + 1}

+ {group.duplicates.length} duplicates +
+ +
+ {/* Original */} + {group.original && ( +
+
+ Original + + Group {group.original.group} + +
+ {group.original.name} +

{group.original.name}

+

ID: {group.original.id}

+
+ + +
+
+ )} + + {/* Duplicates */} + {group.duplicates.map((duplicate) => ( +
+
+ Distance: {duplicate.distance} + + Group {duplicate.group} + +
+ {duplicate.name} +

{duplicate.name}

+

ID: {duplicate.id}

+
+ + +
+
+ ))} +
+
+ ))} +
+ )} + + + )} +
+ + ); +} diff --git a/resources/js/ziggy.js b/resources/js/ziggy.js index 78cd40a..97beff2 100644 --- a/resources/js/ziggy.js +++ b/resources/js/ziggy.js @@ -1,4 +1,4 @@ -const Ziggy = {"url":"https:\/\/memefa.st","port":null,"defaults":{},"routes":{"cashier.payment":{"uri":"stripe\/payment\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"cashier.webhook":{"uri":"stripe\/webhook","methods":["POST"]},"horizon.stats.index":{"uri":"horizon\/api\/stats","methods":["GET","HEAD"]},"horizon.workload.index":{"uri":"horizon\/api\/workload","methods":["GET","HEAD"]},"horizon.masters.index":{"uri":"horizon\/api\/masters","methods":["GET","HEAD"]},"horizon.monitoring.index":{"uri":"horizon\/api\/monitoring","methods":["GET","HEAD"]},"horizon.monitoring.store":{"uri":"horizon\/api\/monitoring","methods":["POST"]},"horizon.monitoring-tag.paginate":{"uri":"horizon\/api\/monitoring\/{tag}","methods":["GET","HEAD"],"parameters":["tag"]},"horizon.monitoring-tag.destroy":{"uri":"horizon\/api\/monitoring\/{tag}","methods":["DELETE"],"wheres":{"tag":".*"},"parameters":["tag"]},"horizon.jobs-metrics.index":{"uri":"horizon\/api\/metrics\/jobs","methods":["GET","HEAD"]},"horizon.jobs-metrics.show":{"uri":"horizon\/api\/metrics\/jobs\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"horizon.queues-metrics.index":{"uri":"horizon\/api\/metrics\/queues","methods":["GET","HEAD"]},"horizon.queues-metrics.show":{"uri":"horizon\/api\/metrics\/queues\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"horizon.jobs-batches.index":{"uri":"horizon\/api\/batches","methods":["GET","HEAD"]},"horizon.jobs-batches.show":{"uri":"horizon\/api\/batches\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"horizon.jobs-batches.retry":{"uri":"horizon\/api\/batches\/retry\/{id}","methods":["POST"],"parameters":["id"]},"horizon.pending-jobs.index":{"uri":"horizon\/api\/jobs\/pending","methods":["GET","HEAD"]},"horizon.completed-jobs.index":{"uri":"horizon\/api\/jobs\/completed","methods":["GET","HEAD"]},"horizon.silenced-jobs.index":{"uri":"horizon\/api\/jobs\/silenced","methods":["GET","HEAD"]},"horizon.failed-jobs.index":{"uri":"horizon\/api\/jobs\/failed","methods":["GET","HEAD"]},"horizon.failed-jobs.show":{"uri":"horizon\/api\/jobs\/failed\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"horizon.retry-jobs.show":{"uri":"horizon\/api\/jobs\/retry\/{id}","methods":["POST"],"parameters":["id"]},"horizon.jobs.show":{"uri":"horizon\/api\/jobs\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"horizon.index":{"uri":"horizon\/{view?}","methods":["GET","HEAD"],"wheres":{"view":"(.*)"},"parameters":["view"]},"sanctum.csrf-cookie":{"uri":"sanctum\/csrf-cookie","methods":["GET","HEAD"]},"api.pricing_page":{"uri":"api\/pricing","methods":["POST"]},"api.ai_hints":{"uri":"api\/ai-hints","methods":["POST"]},"api.user":{"uri":"api\/user","methods":["POST"]},"api.user.subscribe":{"uri":"api\/user\/subscribe","methods":["POST"]},"api.user.purchase":{"uri":"api\/user\/purchase","methods":["POST"]},"api.user.billing_portal":{"uri":"api\/user\/billing-portal","methods":["POST"]},"api.user.premium_export.request":{"uri":"api\/user\/premium-export\/request","methods":["POST"]},"api.user.premium_export.complete":{"uri":"api\/user\/premium-export\/complete","methods":["POST"]},"api.user.generate_meme":{"uri":"api\/user\/generate_meme","methods":["POST"]},"api.user.check_meme_job_status":{"uri":"api\/user\/generate_meme\/status","methods":["POST"]},"api.user.get_active_job":{"uri":"api\/user\/generate_meme\/active","methods":["POST"]},"api.user.get_meme_history":{"uri":"api\/user\/generate_meme\/history","methods":["POST"]},"api.app.init":{"uri":"api\/app\/init","methods":["POST"]},"api.app.memes":{"uri":"api\/app\/memes","methods":["POST"]},"api.app.search.memes":{"uri":"api\/app\/search\/memes","methods":["POST"]},"api.app.background":{"uri":"api\/app\/background","methods":["POST"]},"api.app.search.background":{"uri":"api\/app\/search\/background","methods":["POST"]},"auth.google.redirect":{"uri":"auth\/google\/redirect","methods":["GET","HEAD"]},"auth.google.callback":{"uri":"auth\/google\/callback","methods":["GET","HEAD"]},"dashboard":{"uri":"dashboard","methods":["GET","HEAD"]},"subscribe.success":{"uri":"subscribe\/success","methods":["GET","HEAD"]},"subscribe.cancelled":{"uri":"subscribe\/cancelled","methods":["GET","HEAD"]},"purchase.success":{"uri":"purchase\/success","methods":["GET","HEAD"]},"purchase.cancelled":{"uri":"purchase\/cancelled","methods":["GET","HEAD"]},"admin.dashboard":{"uri":"admin","methods":["GET","HEAD"]},"admin.background-generation":{"uri":"admin\/background-generation","methods":["GET","HEAD"]},"admin.background-generation.generate":{"uri":"admin\/background-generation\/generate","methods":["POST"]},"admin.background-generation.save":{"uri":"admin\/background-generation\/save","methods":["POST"]},"admin.background-generation.delete":{"uri":"admin\/background-generation\/delete\/{id}","methods":["POST"],"parameters":["id"]},"profile.edit":{"uri":"settings\/profile","methods":["GET","HEAD"]},"profile.update":{"uri":"settings\/profile","methods":["PATCH"]},"profile.destroy":{"uri":"settings\/profile","methods":["DELETE"]},"password.edit":{"uri":"settings\/password","methods":["GET","HEAD"]},"password.update":{"uri":"settings\/password","methods":["PUT"]},"appearance":{"uri":"settings\/appearance","methods":["GET","HEAD"]},"register":{"uri":"register","methods":["GET","HEAD"]},"login":{"uri":"login","methods":["GET","HEAD"]},"password.request":{"uri":"forgot-password","methods":["GET","HEAD"]},"password.email":{"uri":"forgot-password","methods":["POST"]},"password.reset":{"uri":"reset-password\/{token}","methods":["GET","HEAD"],"parameters":["token"]},"password.store":{"uri":"reset-password","methods":["POST"]},"verification.notice":{"uri":"verify-email","methods":["GET","HEAD"]},"verification.verify":{"uri":"verify-email\/{id}\/{hash}","methods":["GET","HEAD"],"parameters":["id","hash"]},"verification.send":{"uri":"email\/verification-notification","methods":["POST"]},"password.confirm":{"uri":"confirm-password","methods":["GET","HEAD"]},"logout":{"uri":"logout","methods":["POST"]},"home":{"uri":"\/","methods":["GET","HEAD"]},"privacy":{"uri":"privacy","methods":["GET","HEAD"]},"terms":{"uri":"terms","methods":["GET","HEAD"]},"storage.local":{"uri":"storage\/{path}","methods":["GET","HEAD"],"wheres":{"path":".*"},"parameters":["path"]}}}; +const Ziggy = {"url":"https:\/\/memefa.st","port":null,"defaults":{},"routes":{"cashier.payment":{"uri":"stripe\/payment\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"cashier.webhook":{"uri":"stripe\/webhook","methods":["POST"]},"horizon.stats.index":{"uri":"horizon\/api\/stats","methods":["GET","HEAD"]},"horizon.workload.index":{"uri":"horizon\/api\/workload","methods":["GET","HEAD"]},"horizon.masters.index":{"uri":"horizon\/api\/masters","methods":["GET","HEAD"]},"horizon.monitoring.index":{"uri":"horizon\/api\/monitoring","methods":["GET","HEAD"]},"horizon.monitoring.store":{"uri":"horizon\/api\/monitoring","methods":["POST"]},"horizon.monitoring-tag.paginate":{"uri":"horizon\/api\/monitoring\/{tag}","methods":["GET","HEAD"],"parameters":["tag"]},"horizon.monitoring-tag.destroy":{"uri":"horizon\/api\/monitoring\/{tag}","methods":["DELETE"],"wheres":{"tag":".*"},"parameters":["tag"]},"horizon.jobs-metrics.index":{"uri":"horizon\/api\/metrics\/jobs","methods":["GET","HEAD"]},"horizon.jobs-metrics.show":{"uri":"horizon\/api\/metrics\/jobs\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"horizon.queues-metrics.index":{"uri":"horizon\/api\/metrics\/queues","methods":["GET","HEAD"]},"horizon.queues-metrics.show":{"uri":"horizon\/api\/metrics\/queues\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"horizon.jobs-batches.index":{"uri":"horizon\/api\/batches","methods":["GET","HEAD"]},"horizon.jobs-batches.show":{"uri":"horizon\/api\/batches\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"horizon.jobs-batches.retry":{"uri":"horizon\/api\/batches\/retry\/{id}","methods":["POST"],"parameters":["id"]},"horizon.pending-jobs.index":{"uri":"horizon\/api\/jobs\/pending","methods":["GET","HEAD"]},"horizon.completed-jobs.index":{"uri":"horizon\/api\/jobs\/completed","methods":["GET","HEAD"]},"horizon.silenced-jobs.index":{"uri":"horizon\/api\/jobs\/silenced","methods":["GET","HEAD"]},"horizon.failed-jobs.index":{"uri":"horizon\/api\/jobs\/failed","methods":["GET","HEAD"]},"horizon.failed-jobs.show":{"uri":"horizon\/api\/jobs\/failed\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"horizon.retry-jobs.show":{"uri":"horizon\/api\/jobs\/retry\/{id}","methods":["POST"],"parameters":["id"]},"horizon.jobs.show":{"uri":"horizon\/api\/jobs\/{id}","methods":["GET","HEAD"],"parameters":["id"]},"horizon.index":{"uri":"horizon\/{view?}","methods":["GET","HEAD"],"wheres":{"view":"(.*)"},"parameters":["view"]},"sanctum.csrf-cookie":{"uri":"sanctum\/csrf-cookie","methods":["GET","HEAD"]},"api.pricing_page":{"uri":"api\/pricing","methods":["POST"]},"api.ai_hints":{"uri":"api\/ai-hints","methods":["POST"]},"api.user":{"uri":"api\/user","methods":["POST"]},"api.user.subscribe":{"uri":"api\/user\/subscribe","methods":["POST"]},"api.user.purchase":{"uri":"api\/user\/purchase","methods":["POST"]},"api.user.billing_portal":{"uri":"api\/user\/billing-portal","methods":["POST"]},"api.user.premium_export.request":{"uri":"api\/user\/premium-export\/request","methods":["POST"]},"api.user.premium_export.complete":{"uri":"api\/user\/premium-export\/complete","methods":["POST"]},"api.user.generate_meme":{"uri":"api\/user\/generate_meme","methods":["POST"]},"api.user.check_meme_job_status":{"uri":"api\/user\/generate_meme\/status","methods":["POST"]},"api.user.get_active_job":{"uri":"api\/user\/generate_meme\/active","methods":["POST"]},"api.user.get_meme_history":{"uri":"api\/user\/generate_meme\/history","methods":["POST"]},"api.app.init":{"uri":"api\/app\/init","methods":["POST"]},"api.app.memes":{"uri":"api\/app\/memes","methods":["POST"]},"api.app.search.memes":{"uri":"api\/app\/search\/memes","methods":["POST"]},"api.app.background":{"uri":"api\/app\/background","methods":["POST"]},"api.app.search.background":{"uri":"api\/app\/search\/background","methods":["POST"]},"auth.google.redirect":{"uri":"auth\/google\/redirect","methods":["GET","HEAD"]},"auth.google.callback":{"uri":"auth\/google\/callback","methods":["GET","HEAD"]},"dashboard":{"uri":"dashboard","methods":["GET","HEAD"]},"subscribe.success":{"uri":"subscribe\/success","methods":["GET","HEAD"]},"subscribe.cancelled":{"uri":"subscribe\/cancelled","methods":["GET","HEAD"]},"purchase.success":{"uri":"purchase\/success","methods":["GET","HEAD"]},"purchase.cancelled":{"uri":"purchase\/cancelled","methods":["GET","HEAD"]},"admin.dashboard":{"uri":"admin","methods":["GET","HEAD"]},"admin.background-generation":{"uri":"admin\/background-generation","methods":["GET","HEAD"]},"admin.background-generation.generate":{"uri":"admin\/background-generation\/generate","methods":["POST"]},"admin.background-generation.save":{"uri":"admin\/background-generation\/save","methods":["POST"]},"admin.background-generation.delete":{"uri":"admin\/background-generation\/delete\/{id}","methods":["POST"],"parameters":["id"]},"profile.edit":{"uri":"settings\/profile","methods":["GET","HEAD"]},"profile.update":{"uri":"settings\/profile","methods":["PATCH"]},"profile.destroy":{"uri":"settings\/profile","methods":["DELETE"]},"password.edit":{"uri":"settings\/password","methods":["GET","HEAD"]},"password.update":{"uri":"settings\/password","methods":["PUT"]},"appearance":{"uri":"settings\/appearance","methods":["GET","HEAD"]},"register":{"uri":"register","methods":["GET","HEAD"]},"login":{"uri":"login","methods":["GET","HEAD"]},"password.request":{"uri":"forgot-password","methods":["GET","HEAD"]},"password.email":{"uri":"forgot-password","methods":["POST"]},"password.reset":{"uri":"reset-password\/{token}","methods":["GET","HEAD"],"parameters":["token"]},"password.store":{"uri":"reset-password","methods":["POST"]},"verification.notice":{"uri":"verify-email","methods":["GET","HEAD"]},"verification.verify":{"uri":"verify-email\/{id}\/{hash}","methods":["GET","HEAD"],"parameters":["id","hash"]},"verification.send":{"uri":"email\/verification-notification","methods":["POST"]},"password.confirm":{"uri":"confirm-password","methods":["GET","HEAD"]},"logout":{"uri":"logout","methods":["POST"]},"home":{"uri":"\/","methods":["GET","HEAD"]},"privacy":{"uri":"privacy","methods":["GET","HEAD"]},"terms":{"uri":"terms","methods":["GET","HEAD"]},"admin.duplicates":{"uri":"duplicates","methods":["GET","HEAD"]},"admin.duplicates.scan":{"uri":"duplicates\/scan","methods":["POST"]},"admin.duplicates.delete":{"uri":"duplicates\/delete","methods":["POST"]},"admin.duplicates.regenerate-hash":{"uri":"duplicates\/regenerate-hash","methods":["POST"]},"storage.local":{"uri":"storage\/{path}","methods":["GET","HEAD"],"wheres":{"path":".*"},"parameters":["path"]}}}; if (typeof window !== 'undefined' && typeof window.Ziggy !== 'undefined') { Object.assign(Ziggy.routes, window.Ziggy.routes); } diff --git a/routes/web.php b/routes/web.php index a00e8b7..290cb69 100644 --- a/routes/web.php +++ b/routes/web.php @@ -2,12 +2,14 @@ use App\Http\Controllers\AdminBackgroundGenerationController; use App\Http\Controllers\AdminDashboardController; +use App\Http\Controllers\AdminDuplicateController; use App\Http\Controllers\FrontHomeController; use App\Http\Controllers\FrontPagesController; use App\Http\Controllers\SocialAuthController; use App\Http\Controllers\UserDashboardController; use App\Http\Controllers\UserPurchaseController; use App\Http\Middleware\AdminMiddleware; +use App\Http\Middleware\BasicAuthMiddleware; use Illuminate\Support\Facades\Route; if (App::environment('local')) { @@ -72,3 +74,17 @@ Route::get('/terms', [FrontPagesController::class, 'terms']) ->middleware('cacheResponse') ->name('terms'); + +// Admin Tools with Basic Auth +Route::prefix('duplicates')->middleware([BasicAuthMiddleware::class])->group(function () { + Route::get('/', [AdminDuplicateController::class, 'index'])->name('admin.duplicates'); + + Route::post('scan', [AdminDuplicateController::class, 'scan']) + ->name('admin.duplicates.scan'); + + Route::post('delete', [AdminDuplicateController::class, 'delete']) + ->name('admin.duplicates.delete'); + + Route::post('regenerate-hash', [AdminDuplicateController::class, 'regenerateHash']) + ->name('admin.duplicates.regenerate-hash'); +});