charlestehio/aibuddytool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add (ai gen)

Browse Source
This commit is contained in:
Charles T
2023-10-01 20:39:18 +08:00
parent c68bda42c5
commit b03ceb3868
4 changed files with 105 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
app/Jobs/Tasks/SaveShopeeSellerImagesTask.php
View File

@@ -8,68 +8,46 @@
use Illuminate\Support\Str;
use Intervention\Image\Facades\Image;
use Symfony\Component\DomCrawler\Crawler;
use Masterminds\HTML5;
class SaveShopeeSellerImagesTask
{
public static function handle($shopee_task)
{
$main_intervention_image = null;
$intervention_images = [];
$costs = [];
$main_image_url = null;
$unblocker_proxy_server = get_smartproxy_unblocker_server();
$rotating_proxy_server = get_smartproxy_rotating_server();
$costs = [];
$user_agent = config('platform.proxy.user_agent');
///////// PART 1
$main_image_url = self::getProductImageUrl($shopee_task->product_task->response->jsonld);
// If there is a main intervention image, then set in, else get the url only.
if (isset($shopee_task?->product_task?->intervention?->main_intervention_image)) {
$main_intervention_image = $shopee_task->product_task->intervention->main_intervention_image;
} else {
$main_image_url = self::getProductImageUrl($shopee_task->product_task->response->jsonld);
}
// If there is other image interventions set, then set in, else get the image urls only.
if (isset($shopee_task?->product_task?->intervention?->intervention_images)) {
$intervention_images = $shopee_task->product_task->intervention->intervention_images;
} else {
$images = self::getImages($shopee_task->product_task->response->raw_html);
$images = self::filterImages($images, $rotating_proxy_server, $user_agent, $costs, $intervention_images);
}
///////// PART 2
// Check existence and upload if image intervention is set
if (! is_null($main_intervention_image)) {
$scraped_image = ShopeeSellerScrapedImage::where('original_name', $main_intervention_image->original_name)->where('shopee_seller_scrape_id', $shopee_task->shopee_seller_scrape->id)->first();
if (is_null($scraped_image)) {
$scraped_image = self::uploadAndSaveScrapedImage($shopee_task->shopee_seller_scrape, $main_intervention_image, true);
}
}
// if there is no main image intervention but the main image url is provided
elseif (! is_empty($main_image_url)) {
if (! is_empty($main_image_url)) {
$scraped_image = ShopeeSellerScrapedImage::where('original_name', pathinfo($main_image_url, PATHINFO_BASENAME))->where('shopee_seller_scrape_id', $shopee_task->shopee_seller_scrape->id)->first();
if (is_null($scraped_image)) {
$main_image = self::getProductImage($shopee_task->product_task->response->jsonld, $rotating_proxy_server, $user_agent, $costs, $main_intervention_image);
$main_image = self::getProductImage($shopee_task->product_task->response->jsonld, $rotating_proxy_server, $user_agent, $costs);
$scraped_image = self::uploadAndSaveScrapedImage($shopee_task->shopee_seller_scrape, $main_intervention_image, true);
$scraped_image = self::uploadAndSaveScrapedImage($shopee_task->shopee_seller_scrape, $main_image, true);
}
}
/////// PART 3
/////// PART 2
if (! is_null($intervention_images) && is_array($intervention_images) && count($intervention_images) > 0) {
foreach ($intervention_images as $intervention_image) {
$scraped_image = ShopeeSellerScrapedImage::where('original_name', $intervention_image->original_name)->where('shopee_seller_scrape_id', $shopee_task->shopee_seller_scrape->id)->first();
$images = self::getFilteredImages($shopee_task->product_task->response->raw_html, $rotating_proxy_server, $user_agent, $costs);
//dd($images);
if (! is_null($images) && is_array($images) && count($images) > 0) {
foreach ($images as $image_obj) {
$scraped_image = ShopeeSellerScrapedImage::where('original_name', $image_obj->original_name)->where('shopee_seller_scrape_id', $shopee_task->shopee_seller_scrape->id)->first();
if (is_null($scraped_image)) {
$scraped_image = self::uploadAndSaveScrapedImage($shopee_task->shopee_seller_scrape, $intervention_image, false);
$scraped_image = self::uploadAndSaveScrapedImage($shopee_task->shopee_seller_scrape, $image_obj, false);
}
}
}
@@ -78,7 +56,7 @@ public static function handle($shopee_task)
}
private static function uploadAndSaveScrapedImage($shopee_seller_scrape, $intervention_image, $featured = false)
private static function uploadAndSaveScrapedImage($shopee_seller_scrape, $image_obj, $featured = false)
{
// Generate a unique filename for the uploaded file and LQIP version
$uuid = Str::uuid()->toString();
@@ -86,7 +64,7 @@ private static function uploadAndSaveScrapedImage($shopee_seller_scrape, $interv
$lqipFileName = time().'_'.$uuid.'_lqip.jpg';
// Convert the file to JPEG format using Intervention Image library
$image = $intervention_image->image;
$image = $image_obj->intervention;
// Get the original image width and height
$originalWidth = $image->width();
@@ -102,7 +80,7 @@ private static function uploadAndSaveScrapedImage($shopee_seller_scrape, $interv
// Save the original image to a temporary file and open it again
$tempImagePath = tempnam(sys_get_temp_dir(), 'temp_image');
file_put_contents($tempImagePath, $intervention_image->image->encode());
file_put_contents($tempImagePath, $image_obj->intervention->encode());
$clonedImage = Image::make($tempImagePath);
// Create the LQIP version of the image using a small size while maintaining the aspect ratio
@@ -120,7 +98,7 @@ private static function uploadAndSaveScrapedImage($shopee_seller_scrape, $interv
$scraped_image = new ShopeeSellerScrapedImage;
$scraped_image->shopee_seller_scrape_id = $shopee_seller_scrape->id;
$scraped_image->original_name = $intervention_image->original_name;
$scraped_image->original_name = $image_obj->original_name;
$scraped_image->image = $url;
$scraped_image->featured = $featured;
@@ -131,32 +109,36 @@ private static function uploadAndSaveScrapedImage($shopee_seller_scrape, $interv
return null;
}
private static function getImages(string $raw_html)
private static function getImageUrls(string $raw_html)
{
$crawler = new Crawler($raw_html);
$images = [];
$crawler->filter('img')->each(function ($node) use (&$images) {
$src = $node->attr('src');
$alt = $node->attr('alt') ?? null; // Setting a default value if alt is not present
// Pattern for extracting src and alt attributes from img tags
$pattern = '/<img\s.*?(?:src=["\'](.*?)["\']).*?(?:alt=["\'](.*?)["\'])?[^>]*>/is';
$blacklist_domain = [];
if (preg_match_all($pattern, $raw_html, $matches, PREG_SET_ORDER)) {
foreach ($matches as $match) {
$src = $match[1];
foreach ($blacklist_domain as $blacklist) {
if (! str_contains($src, $blacklist)) {
$images[] = [
'src' => $src,
'alt' => $alt,
];
}
// Check if image file name ends with '_tn' and remove it
$src = preg_replace('/_tn(\.[a-z]+)?$/i', '$1', $src);
$images[] = [
'src' => $src,
'alt' => isset($match[2]) ? $match[2] : null,
];
}
});
}
return $images;
}
private static function filterImages(array $images, string $proxy, string $user_agent, &$costs, &$intervention_images)
private static function getFilteredImages(string $raw_html, string $proxy, string $user_agent, &$costs)
{
$images = self::getImageUrls($raw_html);
//dd($images);
$filteredImages = [];
$uniqueAttributes = []; // This array will track unique width, height, mime, and size combinations
@@ -188,7 +170,7 @@ private static function filterImages(array $images, string $proxy, string $user_
$sizeKb = round(strlen($imageData) / 1024, 2);
// Check constraints
if ($width < 300 || $height < 300) {
if ($width < 800 || $height < 800 || $sizeKb < 100 || $mime !== 'image/jpeg') {
continue;
}
@@ -225,16 +207,15 @@ private static function filterImages(array $images, string $proxy, string $user_
'mime' => $mime,
'sizeKb' => $sizeKb,
];
$image['color_counts'] = self::isMostlyTextBasedOnUniqueColors($interventionImage);
$image['color_counts'] = self::getImageColorCounts($interventionImage);
$image['intervention'] = $interventionImage;
$image['original_name'] = pathinfo($src, PATHINFO_BASENAME);
//$image['img'] = $interventionImage;
$costs['count-'.$count] = calculate_smartproxy_cost($sizeKb, 'rotating_global');
$filteredImages[] = $image;
$intervention_images[] = (object) [
'image' => $interventionImage,
'original_name' => pathinfo($src, PATHINFO_BASENAME),
];
}
} catch (\Exception $e) {
// Handle exceptions related to the HTTP request
@@ -269,7 +250,14 @@ private static function filterImages(array $images, string $proxy, string $user_
return $b['sizeKb'] <=> $a['sizeKb']; // Using the spaceship operator to sort in descending order
});
return $filteredImages;
$final_images = [];
foreach ($filteredImages as $image_obj)
{
$final_images[] = (object) $image_obj;
}
return $final_images;
}
private static function getProductImageUrl(array $jsonLdData)
@@ -284,7 +272,7 @@ private static function getProductImageUrl(array $jsonLdData)
}
}
private static function getProductImage(array $jsonLdData, string $proxy, string $user_agent, &$costs, &$main_intervention_image)
private static function getProductImage(array $jsonLdData, string $proxy, string $user_agent, &$costs)
{
foreach ($jsonLdData as $data) {
// Ensure the type is "Product" before proceeding
@@ -313,14 +301,10 @@ private static function getProductImage(array $jsonLdData, string $proxy, string
$costs['product_image'] = $cost;
$main_intervention_image = (object) [
'image' => $interventionImage,
'original_name' => pathinfo($data->image, PATHINFO_BASENAME),
];
return [
return (object) [
'url' => $data->url,
//'img' => $interventionImage,
'intervention' => $interventionImage,
'original_name' => pathinfo($data->image, PATHINFO_BASENAME),
'cost' => $cost,
];
}
@@ -335,7 +319,7 @@ private static function getProductImage(array $jsonLdData, string $proxy, string
return null;
}
private static function isMostlyTextBasedOnUniqueColors($interventionImage)
private static function getImageColorCounts($interventionImage)
{
// Use Intervention to manipulate the image
$img = clone $interventionImage;