Sync

2023-11-26 18:56:40 +08:00
parent be14f5fdb1
commit 64431e7a73
144 changed files with 497072 additions and 3730 deletions
--- a/app/Exceptions/Handler.php
+++ b/app/Exceptions/Handler.php
@@ -38,7 +38,7 @@ public function register()
                    'status' => -1,
                ], 404);
            } else {
-                return redirect()->route('home', [], 301);
+                return redirect()->route('front.home', [], 301);
            }
        });
    }
--- a/app/FirstParty/DFS/DFSBacklinks.php
+++ b/app/FirstParty/DFS/DFSBacklinks.php
@@ -1,40 +0,0 @@
-<?php
-
-namespace App\Helpers\FirstParty\DFS;
-
-use Exception;
-use Http;
-
-class DFSBacklinks
-{
-    public static function backlinksPaginationLive($target, $search_after_token, $value = 1000)
-    {
-        $api_url = config('dataforseo.url');
-
-        $api_version = config('dataforseo.api_version');
-
-        $api_timeout = config('dataforseo.timeout');
-
-        $query = [
-            'target' => $target,
-            'search_after_token' => $search_after_token,
-            'value' => $value,
-            'mode' => 'as_is',
-        ];
-
-        try {
-            $response = Http::timeout($api_timeout)->withBasicAuth(config('dataforseo.login'), config('dataforseo.password'))->withBody(
-                json_encode([(object) $query])
-            )->post("{$api_url}{$api_version}backlinks/backlinks/live");
-
-            if ($response->successful()) {
-                return $response->body();
-            }
-        } catch (Exception $e) {
-            return null;
-        }
-
-        return null;
-
-    }
-}
--- a/app/Helpers/FirstParty/Aictio/Aictio.php
+++ b/app/Helpers/FirstParty/Aictio/Aictio.php
@@ -0,0 +1,54 @@
+<?php
+
+namespace App\Helpers\FirstParty\Aictio;
+
+use Exception;
+use Http;
+use Pgvector\Laravel\Vector;
+
+class Aictio
+{
+    public static function getVectorEmbedding($embedding_query)
+    {
+        $embedding_query = trim(preg_replace('/[^a-zA-Z0-9\s,\.]/', '', $embedding_query));
+
+        if (is_empty($embedding_query)) {
+            throw new Exception('Empty embedding query.');
+        }
+
+        $maxRetries = 3;
+        $currentAttempt = 0;
+
+        while ($currentAttempt < $maxRetries) {
+            try {
+                $response = Http::withHeaders([
+                    'CtioApp' => 'hireblitz260823',
+                ])->withOptions(['verify' => (app()->environment() == 'local') ? false : false])->timeout(800)
+                    ->get('https://aictio.applikuapp.com/api/embeddings',
+                        [
+                            'input' => $embedding_query,
+                        ]
+                    );
+
+                $embedding_response = json_decode($response->body(), true);
+
+                if (is_null($embedding_response)) {
+                    throw new Exception('Embedding response failed, null response');
+                }
+
+                if (isset($embedding_response['error'])) {
+                    throw new Exception($embedding_response['error']);
+                }
+
+                return new Vector(array_values($embedding_response)[0]);
+            } catch (Exception $e) {
+                $currentAttempt++;
+                if ($currentAttempt >= $maxRetries) {
+                    throw $e;
+                }
+                // Optional: Add sleep for a few seconds if you want to delay the next attempt
+                // sleep(1);
+            }
+        }
+    }
+}
--- a/app/Helpers/FirstParty/DFS/DFSBacklinks.php
+++ b/app/Helpers/FirstParty/DFS/DFSBacklinks.php
@@ -0,0 +1,82 @@
+<?php
+
+namespace App\Helpers\FirstParty\DFS;
+
+use App\Models\CandidateDomain;
+use Exception;
+use Http;
+
+class DFSBacklinks
+{
+    public static function processAndSaveToCandidateDomains($directory)
+    {
+        // Get all the JSON files in the directory
+        $jsonFiles = glob($directory.'/*.json');
+
+        dd($jsonFiles);
+
+        // Loop through each file
+        foreach ($jsonFiles as $file) {
+            // Check if the file name matches the pattern (1.json, 2.json, etc.)
+            if (preg_match('/\/(\d+)\.json$/', $file, $matches)) {
+                // Decode the JSON file
+                $data = json_decode(file_get_contents($file), false);
+
+                foreach ($data->result[0]->items as $item) {
+                    $candidate_domain = CandidateDomain::where('from_domain', $item->domain_from)->first();
+
+                    if (is_null($candidate_domain)) {
+                        $candidate_domain = new CandidateDomain;
+                        $candidate_domain->to_url = $item->url_to;
+                        $candidate_domain->from_domain = $item->domain_from;
+                        $candidate_domain->from_tld = get_tld_from_url($item->url_from);
+
+                        if (is_array($item->domain_from_platform_type)) {
+                            $candidate_domain->platform_type = implode(',', $item->domain_from_platform_type);
+                        } else {
+                            $candidate_domain->platform_type = $item->domain_from_platform_type;
+                        }
+
+                        $candidate_domain->from_url = $item->url_from;
+                        $candidate_domain->from_title = $item->page_from_title;
+                        $candidate_domain->from_image_alt = $item->alt;
+                        $candidate_domain->from_image_url = $item->image_url;
+                        $candidate_domain->save();
+
+                    }
+                }
+            }
+        }
+    }
+
+    public static function backlinksPaginationLive($target, $search_after_token, $value = 1000)
+    {
+        $api_url = config('dataforseo.url');
+
+        $api_version = config('dataforseo.api_version');
+
+        $api_timeout = config('dataforseo.timeout');
+
+        $query = [
+            'target' => $target,
+            'search_after_token' => $search_after_token,
+            'value' => $value,
+            'mode' => 'as_is',
+        ];
+
+        try {
+            $response = Http::timeout($api_timeout)->withBasicAuth(config('dataforseo.login'), config('dataforseo.password'))->withBody(
+                json_encode([(object) $query])
+            )->post("{$api_url}{$api_version}backlinks/backlinks/live");
+
+            if ($response->successful()) {
+                return $response->body();
+            }
+        } catch (Exception $e) {
+            return null;
+        }
+
+        return null;
+
+    }
+}
--- a/app/Helpers/FirstParty/DFS/DFSCommon.php
+++ b/app/Helpers/FirstParty/DFS/DFSCommon.php
@@ -0,0 +1,132 @@
+<?php
+
+namespace App\Helpers\FirstParty\DFS;
+
+use Exception;
+use Http;
+
+class DFSCommon
+{
+    public static function getTaskResult($task)
+    {
+        $task_object = new DFSResponse($task);
+
+        return (object) [
+            'type' => 'task',
+            'path' => implode('/', $task_object->getPath()),
+            'id' => $task_object->getId(),
+            'is_successful' => $task_object->isSuccessful(),
+            'result' => $task_object->getResult(),
+            'message' => $task_object->getStatusMessage(),
+        ];
+
+    }
+
+    public static function getTasks($response)
+    {
+
+        $dfs_object = new DFSResponse($response);
+
+        if ($dfs_object->isSuccessful()) {
+            return $dfs_object->getTasks();
+        }
+
+        return [];
+    }
+
+    public static function getTask($response, $key)
+    {
+        $tasks = self::getTasks($response);
+
+        if (count($tasks) > 0) {
+            if (isset($tasks[$key])) {
+                $task_object = $tasks[$key];
+
+                if (count($tasks) > 1) {
+                    $task_object->client_message = 'There are '.count($tasks).' tasks.';
+                }
+
+                return $task_object;
+            }
+        }
+
+        return null;
+    }
+
+    public static function apiCall($method, $action_url, $query)
+    {
+
+        $api_url = self::getApiUrl();
+
+        $api_version = config('dataforseo.api_version');
+
+        $api_timeout = config('dataforseo.timeout');
+
+        $full_api_url = "{$api_url}{$api_version}{$action_url}";
+
+        dump($full_api_url);
+        dump($query);
+
+        $http_object = Http::timeout($api_timeout)->withBasicAuth(config('dataforseo.login'), config('dataforseo.password'));
+
+        if (strtoupper($method) == 'GET') {
+            try {
+                $response = $http_object->withUrlParameters(
+                    json_encode([(object) $query])
+                )->get('');
+
+            } catch (Exception $e) {
+                return self::defaultFailedResponse($e);
+            }
+
+        } elseif (strtoupper($method) == 'POST') {
+            try {
+                $response = $http_object->withBody(
+                    json_encode([(object) $query])
+                )->post("{$api_url}{$api_version}{$action_url}");
+            } catch (Exception $e) {
+                return self::defaultFailedResponse($e);
+            }
+        } else {
+            throw new Exception('Invalid action method parameter.');
+        }
+
+        if ($response->successful()) {
+            return json_decode($response->body(), false);
+        }
+
+        return self::defaultFailedResponse();
+
+    }
+
+    public static function getApiUrl()
+    {
+        $api_url = config('dataforseo.url');
+
+        if (config('dataforseo.sandbox_mode')) {
+            $api_url = config('dataforseo.sandbox_url');
+        }
+
+        return $api_url;
+    }
+
+    private static function defaultFailedResponse(Exception $e = null)
+    {
+        $message = 'No such url.';
+
+        if (! is_null($e)) {
+            $message = $e->getMessage();
+        }
+
+        return (object) [
+            'version' => '-1',
+            'status_code' => -1,
+            'status_message' => $message,
+            'time' => '0 sec.',
+            'cost' => 0,
+            'tasks_count' => 0,
+            'tasks_error' => 0,
+            'tasks' => [],
+        ];
+    }
+}
--- a/app/Helpers/FirstParty/DFS/DFSOnPage.php
+++ b/app/Helpers/FirstParty/DFS/DFSOnPage.php
@@ -0,0 +1,41 @@
+<?php
+
+namespace App\Helpers\FirstParty\DFS;
+
+class DFSOnPage
+{
+    // https://docs.dataforseo.com/v3/backlinks/id_list/?bash
+    public static function listId($from_days, $to_days = 0, $limit = 1000, $offset = 0)
+    {
+
+        $query = [
+            'datetime_from' => now()->subDays($from_days)->format('Y-m-d H:i:s P'),
+            'datetime_to' => now()->subDays($to_days)->format('Y-m-d H:i:s P'),
+        ];
+
+        if ($limit != 1000) {
+            $query['limit'] = $limit;
+        }
+        if ($offset != 0) {
+            $query['offset'] = $offset;
+        }
+
+        $response = DFSCommon::apiCall('POST', 'on_page/id_list', $query);
+
+        return $response;
+    }
+
+    // https://docs.dataforseo.com/v3/on_page/task_post/
+    public static function taskPost($target, $max_crawl_pages = 1)
+    {
+
+        $query = [
+            'target' => $target,
+            'max_crawl_pages' => $max_crawl_pages,
+        ];
+
+        $response = DFSCommon::apiCall('POST', 'on_page/task_post', $query);
+
+        return $response;
+    }
+}
--- a/app/Helpers/FirstParty/DFS/DFSResponse.php
+++ b/app/Helpers/FirstParty/DFS/DFSResponse.php
@@ -0,0 +1,63 @@
+<?php
+
+namespace App\Helpers\FirstParty\DFS;
+
+class DFSResponse
+{
+    protected $response;
+
+    public function __construct($response)
+    {
+        $this->response = $response;
+    }
+
+    public function isSuccessful()
+    {
+        return ($this->response->status_code == 20000) ? true : false;
+    }
+
+    public function getStatusCode()
+    {
+        return $this->response->status_code;
+    }
+
+    public function getStatusMessage()
+    {
+        return $this->response->status_message;
+    }
+
+    public function getTasks()
+    {
+        return $this->response->tasks;
+    }
+
+    public function getResult()
+    {
+        return $this->response->result;
+    }
+
+    public function getResultCount()
+    {
+        return $this->response->result_count;
+    }
+
+    public function getId()
+    {
+        return $this->response->id;
+    }
+
+    public function getCost()
+    {
+        return $this->response->cost;
+    }
+
+    public function getPath()
+    {
+        return $this->response->path;
+    }
+
+    public function getData()
+    {
+        return $this->response->data;
+    }
+}
--- a/app/Helpers/FirstParty/DFS/DFSSerp.php
+++ b/app/Helpers/FirstParty/DFS/DFSSerp.php
@@ -9,7 +9,8 @@ class DFSSerp
 {
    public static function liveAdvanced($se, $se_type, $keyword, $location_name, $language_code, $depth, $search_param = null)
    {
-        $api_url = config('dataforseo.url');
+
+        $api_url = DFSCommon::getApiUrl();

        $api_version = config('dataforseo.api_version');

--- a/app/Helpers/FirstParty/OpenAI/OpenAI.php
+++ b/app/Helpers/FirstParty/OpenAI/OpenAI.php
@@ -8,85 +8,152 @@

 class OpenAI
 {
-    public static function writeProductArticle($excerpt, $photos, $categories)
+    public static function getSiteSummary($parent_categories, $user_prompt, $model_max_tokens = 1536, $timeout = 60)
    {
-        //$excerpt = substr($excerpt, 0, 1500);
+        $openai_config = 'openai-gpt-3-5-turbo-1106';

-        $category_str = implode('|', $categories);
+        $category_list = implode('|', $parent_categories->pluck('name')->toArray());

-        $system_prompt = '
-          You are tasked with writing a product introduction & review article using the provided excerpt. Write as if you are reviewing the product by a third party, avoiding pronouns. Emphasize the product\'s performance, features, and notable aspects. Do not mention marketplace-related information. Return the output as a minified JSON in this format:
-              {"category": "($category_str)","title": "(Start with product name, 60-70 characters)","excerpt": "(150-160 characters, do not start with a verb)","cliffhanger": "(70-80 characters, enticing sentence)","body": "(Markdown, 700-900 words)"}
+        $system_prompt = "Based on the website content containing an AI tool, return a valid JSON containing:\n{\n\"is_ai_tool\":(true|false),\n\"ai_tool_name\":\"(AI Tool Name)\",\n\"is_app_web_both\":\"(app|web|both)\",\n\"tagline\":\"(One line tagline in 6-8 words)\",\n\"summary\": \"(Summary of AI tool in 2-3 parapgraphs, 140-180 words using grade 8 US english)\",\n\"pricing_type\": \"(Free|Free Trial|Freemium|Subscription|Usage Based)\",\n\"main_category\": \"(AI Training|Art|Audio|Avatars|Business|Chatbots|Coaching|Content Generation|Data|Dating|Design|Dev|Education|Emailing|Finance|Gaming|GPTs|Health|Legal|Marketing|Music|Networking|Personal Assistance|Planning|Podcasting|Productivity|Project Management|Prompting|Reporting|Research|Sales|Security|SEO|Shopping|Simulation|Social|Speech|Support|Task|Testing|Training|Translation|UI\/UX|Video|Workflow|Writing)\",\n\"keywords\":[\"(Identify relevant keywords for this AI Tool, 1-2 words each, at least)\"],\n\"qna\":[{\"q\":\"Typical FAQ that readers want to know, up to 5 questions\",\"a\":\"Answer of the question\"}]\n}";

-              Mandatory Requirements:
-              - Language: US grade 8-9 English
-              - Use these sections when applicable:
-                -- ### Introduction
-                -- ### Overview
-                -- ### Specifications (use valid Markdown table. Two columns: Features and Specifications. Use `| --- | --- |` as a separator.)
-                -- ### Price (in given currency)
-                -- ### (Choose one: Should I Buy?, Conclusion, Final Thoughts, Our Verdict)
-              - Only use facts from the provided excerpt
-              - Don\'t use titles inside the markdown body
-              - Use \'###\' for all article sections
-              - Pick the closest provided category
-              - Do not use newline in the JSON structure
-      ';
+        return self::getChatCompletion($user_prompt, $system_prompt, $openai_config, $model_max_tokens, $timeout);
+    }

-        $user_prompt = "EXCERPT\n------------\n{$excerpt}\n";
+    private static function getChatCompletion($user_prompt, $system_prompt, $openai_config, $model_max_tokens, $timeout, $response_format = 'json_object')
+    {
+        $model = config("platform.ai.{$openai_config}.model");
+        $input_cost_per_thousand_tokens = config("platform.ai.{$openai_config}.input_cost_per_thousand_tokens");
+        $output_cost_per_thousand_tokens = config("platform.ai.{$openai_config}.output_cost_per_thousand_tokens");

-        if (count($photos) > 0) {
-            $system_prompt .= '- Include 3 markdown images with the article title as caption in every section, excluding Introduction.\n';
-            $user_prompt .= "\n\MARKDOWN IMAGES\n------------\n";
-            foreach ($photos as $photo) {
-                $user_prompt .= "{$photo}\n";
-            }
-        }
+        $output_token = 1280;

-        $output = (self::chatCompletion($system_prompt, $user_prompt, 'gpt-3.5-turbo', 1500));
+        try {

-        // dump($user_prompt);
-        // dd($output);
+            $obj = self::chatCompletionApi($system_prompt, $user_prompt, $model, $output_token, $response_format, $timeout);

-        if (! is_null($output)) {
-            try {
-                return json_decode($output, false, 512, JSON_THROW_ON_ERROR);
-            } catch (Exception $e) {
-                Log::error($output);
-                inspector()->reportException($e);
+            $input_cost = self::getCostUsage($obj->usage_detailed->prompt_tokens, $input_cost_per_thousand_tokens);
+            $output_cost = self::getCostUsage($obj->usage_detailed->completion_tokens, $output_cost_per_thousand_tokens);

-                return null;
+            $output = $obj->reply;
+
+            if ($response_format == 'json_object') {
+                $output = json_decode(self::jsonFixer($obj->reply), false, 512, JSON_THROW_ON_ERROR);
            }

+            return (object) [
+                'prompts' => (object) [
+                    'system_prompt' => $system_prompt,
+                    'user_prompt' => $user_prompt,
+                ],
+                'cost' => $input_cost + $output_cost,
+                'output' => $output,
+                'token_usage' => $obj->usage,
+                'token_usage_detailed' => $obj->usage_detailed,
+            ];
+        } catch (Exception $e) {
+            return self::getDefaultFailedResponse($system_prompt, $user_prompt, $e);
        }

-        return null;
+        return self::getDefaultFailedResponse($system_prompt, $user_prompt);

    }

-    public static function chatCompletion($system_prompt, $user_prompt, $model, $max_token = 2500)
+    private static function getDefaultFailedResponse($system_prompt, $user_prompt, $exception = null)
    {
+        $exception_message = null;
+
+        if (! is_null($exception)) {
+            $exception_message = $exception->getMessage();
+        }
+
+        return (object) [
+            'exception' => $exception_message,
+            'prompts' => (object) [
+                'system_prompt' => $system_prompt,
+                'user_prompt' => $user_prompt,
+            ],
+            'cost' => 0,
+            'output' => null,
+            'token_usage' => 0,
+            'token_usage_detailed' => (object) [
+                'completion_tokens' => 0,
+                'prompt_tokens' => 0,
+                'total_tokens' => 0,
+            ],
+        ];
+    }
+
+    private static function getCostUsage($token_usage, $cost_per_thousand_tokens)
+    {
+        $calc = $token_usage / 1000;
+
+        return $calc * $cost_per_thousand_tokens;
+    }
+
+    private static function jsonFixer($json_string)
+    {
+        $json_string = str_replace("\n", '', $json_string);
+
+        // try {
+        //   return (new JsonFixer)->fix($json_string);
+        // }
+        // catch(Exception $e) {
+
+        // }
+        return $json_string;
+
+    }
+
+    public static function chatCompletionApi($system_prompt, $user_prompt, $model, $max_token = 2500, $response_format = 'text', $timeout = 800)
+    {
+
+        if ($response_format == 'json_object') {
+            $arr = [
+                'model' => $model,
+                'max_tokens' => $max_token,
+                'response_format' => (object) [
+                    'type' => 'json_object',
+                ],
+                'messages' => [
+                    ['role' => 'system', 'content' => $system_prompt],
+                    ['role' => 'user', 'content' => $user_prompt],
+                ],
+            ];
+        } else {
+            $arr = [
+                'model' => $model,
+                'max_tokens' => $max_token,
+                'messages' => [
+                    ['role' => 'system', 'content' => $system_prompt],
+                    ['role' => 'user', 'content' => $user_prompt],
+                ],
+            ];
+        }
+
        try {
-            $response = Http::timeout(800)->withToken(config('platform.ai.openai.api_key'))
-                ->post('https://api.openai.com/v1/chat/completions', [
-                    'model' => $model,
-                    'max_tokens' => $max_token,
-                    'messages' => [
-                        ['role' => 'system', 'content' => $system_prompt],
-                        ['role' => 'user', 'content' => $user_prompt],
-                    ],
-                ]);
+            $response = Http::timeout($timeout)->withToken(config('platform.ai.openai.api_key'))
+                ->post('https://api.openai.com/v1/chat/completions', $arr);

-            //dd($response->body());
+            $json_response = json_decode($response->body());

-            $json_response = json_decode($response->body(), false, 512, JSON_THROW_ON_ERROR);
+            //dump($json_response);

-            $reply = $json_response?->choices[0]?->message?->content;
+            if (isset($json_response->error)) {
+                Log::error(serialize($json_response));
+                throw new Exception(serialize($json_response->error));
+            }

-            return $reply;
+            $obj = (object) [
+                'usage' => $json_response?->usage?->total_tokens,
+                'usage_detailed' => $json_response?->usage,
+                'reply' => $json_response?->choices[0]?->message?->content,
+
+            ];
+
+            return $obj;
        } catch (Exception $e) {
-            Log::error($response->body());
-            inspector()->reportException($e);
+            ////dd($response->body());
+            //inspector()->reportException($e);
            throw ($e);
        }

--- a/app/Helpers/Global/string_helper.php
+++ b/app/Helpers/Global/string_helper.php
@@ -9,6 +9,155 @@ function epoch_now_timestamp()
    }
 }

+if (! function_exists('markdown_to_plaintext')) {
+    function markdown_to_plaintext($markdown)
+    {
+        // Headers
+        $markdown = preg_replace('/^#.*$/m', '', $markdown);
+        // Links and images
+        $markdown = preg_replace('/\[(.*?)\]\(.*?\)/', '$1', $markdown);
+        // Bold and italic
+        $markdown = str_replace(['**', '__', '*', '_'], '', $markdown);
+        // Ordered and unordered lists
+        $markdown = preg_replace('/^[-\*].*$/m', '', $markdown);
+        // Horizontal line
+        $markdown = str_replace(['---', '***', '- - -', '* * *'], '', $markdown);
+        // Inline code and code blocks
+        $markdown = preg_replace('/`.*?`/', '', $markdown);
+        $markdown = preg_replace('/```[\s\S]*?```/', '', $markdown);
+        // Blockquotes
+        $markdown = preg_replace('/^>.*$/m', '', $markdown);
+
+        // Remove multiple spaces, leading and trailing spaces
+        $plaintext = trim(preg_replace('/\s+/', ' ', $markdown));
+
+        return $plaintext;
+    }
+}
+
+if (! function_exists('round_to_nearest_base')) {
+    function round_to_nearest_base($number, $postfix = '+', $base = null)
+    {
+        // If $base is not provided, determine the base dynamically
+        if ($base === null) {
+            $length = strlen((string) $number);
+            if ($length > 1) {
+                $base = pow(10, $length - 2);
+            } else {
+                $base = 1;
+            }
+        }
+
+        $roundedNumber = floor($number / $base) * $base;
+
+        return $roundedNumber.$postfix;
+    }
+}
+
+if (! function_exists('remove_query_parameters')) {
+    function remove_query_parameters($url)
+    {
+        // Trim the URL to remove whitespace, newline characters, and trailing slashes
+        $trimmedUrl = rtrim(trim($url), '/');
+
+        // Parse the trimmed URL
+        $parsedUrl = parse_url($trimmedUrl);
+
+        // Rebuild the URL without the query string
+        $rebuiltUrl = $parsedUrl['scheme'].'://'.$parsedUrl['host'];
+        if (isset($parsedUrl['path']) && $parsedUrl['path'] !== '/') {
+            $rebuiltUrl .= $parsedUrl['path'];
+        }
+
+        return $rebuiltUrl;
+    }
+}
+
+if (! function_exists('remove_referral_parameters')) {
+    function remove_referral_parameters($url)
+    {
+        $referralParameters = [
+            'aff', 'ref', 'via',
+            'utm_source', 'utm_medium', 'utm_campaign',
+            'utm_content', 'utm_term', 'partner',
+            'click_id', 'affiliate', 'source',
+            'referral', 'tracker', 'tracking_id',
+            'promo', 'coupon', 'discount',
+            'campaign', 'ad', 'ad_id',
+            // Add more parameters as needed
+        ];
+
+        // Parse the URL
+        $parsedUrl = parse_url($url);
+        if (! isset($parsedUrl['query'])) {
+            return $url; // No query string present, return original URL
+        }
+
+        // Parse the query string
+        parse_str($parsedUrl['query'], $queryParams);
+
+        // Remove referral parameters
+        foreach ($referralParameters as $param) {
+            unset($queryParams[$param]);
+        }
+
+        // Rebuild query string
+        $queryString = http_build_query($queryParams);
+
+        // Rebuild the URL
+        $rebuiltUrl = $parsedUrl['scheme'].'://'.$parsedUrl['host'];
+        if (isset($parsedUrl['path'])) {
+            $rebuiltUrl .= $parsedUrl['path'];
+        }
+        if (! empty($queryString)) {
+            $rebuiltUrl .= '?'.$queryString;
+        }
+
+        return $rebuiltUrl;
+    }
+}
+
+if (! function_exists('get_domain_from_url')) {
+    function get_domain_from_url($url)
+    {
+        $parse = parse_url($url);
+
+        // Check if 'host' key exists in the parsed URL array
+        if (! isset($parse['host'])) {
+            return null; // or you can throw an exception or handle this case as per your requirement
+        }
+
+        $host = $parse['host'];
+
+        // Check if the domain starts with 'www.' and remove it
+        if (substr($host, 0, 4) === 'www.') {
+            $host = substr($host, 4);
+        }
+
+        return $host;
+    }
+}
+
+if (! function_exists('get_tld_from_url')) {
+    function get_tld_from_url($url)
+    {
+        // Parse the URL and return its components
+        $parsedUrl = parse_url($url);
+
+        // Check if the 'host' part is set
+        if (isset($parsedUrl['host'])) {
+            // Split the host by dots
+            $hostParts = explode('.', $parsedUrl['host']);
+
+            // Return the last part which should be the TLD
+            return end($hostParts);
+        }
+
+        // Return false if the URL doesn't have a host component
+        return false;
+    }
+}
+
 if (! function_exists('str_first_sentence')) {
    function str_first_sentence($str)
    {
--- a/app/Http/.DS_Store
+++ b/app/Http/.DS_Store
--- a/app/Http/Controllers/.DS_Store
+++ b/app/Http/Controllers/.DS_Store
--- a/app/Http/Controllers/Front/FrontDiscoverController.php
+++ b/app/Http/Controllers/Front/FrontDiscoverController.php
@@ -0,0 +1,70 @@
+<?php
+
+namespace App\Http\Controllers\Front;
+
+use App\Http\Controllers\Controller;
+use App\Models\Category;
+use Artesaos\SEOTools\Facades\SEOTools;
+use Illuminate\Http\Request;
+use JsonLd\Context;
+
+class FrontDiscoverController extends Controller
+{
+    public function discover(Request $request, $category_slug = null)
+    {
+        $tools_count = round_to_nearest_base(700);
+
+        $category = null;
+
+        if (! is_empty($category_slug)) {
+            $category = Category::where('slug', $category_slug)->first();
+
+            if (is_null($category)) {
+                return abort(404);
+            }
+        }
+
+        if (! is_null($category)) {
+            $breadcrumbs = collect([
+                ['name' => 'Home', 'url' => route('front.home')],
+                ['name' => 'Discover AI Tools', 'url' => route('front.discover.home')],
+                ['name' => $category->name.' AI Tools', 'url' => null],
+            ]);
+
+            SEOTools::metatags();
+            SEOTools::twitter();
+            SEOTools::opengraph();
+            SEOTools::jsonLd();
+            SEOTools::setTitle($category->name.' AI Tools', false);
+            //SEOTools::setDescription($description);
+        } else {
+            $breadcrumbs = collect([
+                ['name' => 'Home', 'url' => route('front.home')],
+                ['name' => 'Discover AI Tools', 'url' => null],
+            ]);
+
+            SEOTools::metatags();
+            SEOTools::twitter();
+            SEOTools::opengraph();
+            SEOTools::jsonLd();
+            SEOTools::setTitle("{$tools_count} over AI Tools for you", false);
+            //SEOTools::setDescription($description);
+        }
+
+        // breadcrumb json ld
+        $listItems = [];
+
+        foreach ($breadcrumbs as $index => $breadcrumb) {
+            $listItems[] = [
+                'name' => $breadcrumb['name'],
+                'url' => $breadcrumb['url'],
+            ];
+        }
+
+        $breadcrumb_context = Context::create('breadcrumb_list', [
+            'itemListElement' => $listItems,
+        ]);
+
+        return view('front.discover', compact('breadcrumbs', 'breadcrumb_context', 'category'));
+    }
+}
--- a/app/Http/Controllers/Front/FrontHomeController.php
+++ b/app/Http/Controllers/Front/FrontHomeController.php
@@ -0,0 +1,83 @@
+<?php
+
+namespace App\Http\Controllers\Front;
+
+use App\Http\Controllers\Controller;
+use Artesaos\SEOTools\Facades\SEOMeta;
+use Artesaos\SEOTools\Facades\SEOTools;
+use GrahamCampbell\Markdown\Facades\Markdown;
+use Illuminate\Http\Request;
+
+class FrontHomeController extends Controller
+{
+    public function index(Request $request)
+    {
+        return view('front.home');
+    }
+
+    public function terms(Request $request)
+    {
+        $markdown = file_get_contents(resource_path('markdown/terms.md'));
+
+        $title = 'Terms of Service';
+        $description = 'Our Terms of Service outline your rights, responsibilities, and the standards we uphold for a seamless experience.';
+
+        SEOTools::metatags();
+        SEOTools::twitter();
+        SEOTools::opengraph();
+        SEOTools::jsonLd();
+        SEOTools::setTitle($title);
+        SEOTools::setDescription($description);
+        SEOMeta::setRobots('noindex');
+
+        $content = Markdown::convert($markdown)->getContent();
+
+        //$content = $this->injectTailwindClasses($content);
+
+        return view('front.pages', compact('content', 'title', 'description'));
+    }
+
+    public function privacy(Request $request)
+    {
+        $markdown = file_get_contents(resource_path('markdown/privacy.md'));
+
+        $title = 'Privacy Policy';
+        $description = 'Our Privacy Policy provides clarity about the data we collect and how we use it, ensuring your peace of mind.';
+
+        SEOTools::metatags();
+        SEOTools::twitter();
+        SEOTools::opengraph();
+        SEOTools::jsonLd();
+        SEOTools::setTitle($title);
+        SEOTools::setDescription($description);
+        SEOMeta::setRobots('noindex');
+
+        $content = Markdown::convert($markdown)->getContent();
+
+        //$content = $this->injectTailwindClasses($content);
+
+        return view('front.pages', compact('content', 'title', 'description'));
+    }
+
+    public function disclaimer(Request $request)
+    {
+        $markdown = file_get_contents(resource_path('markdown/disclaimer.md'));
+
+        $title = 'Disclaimer';
+        $description = 'AI Buddy Tool provides the content on this website purely for informational purposes and should not be interpreted as legal, financial, or medical guidance.';
+
+        SEOTools::metatags();
+        SEOTools::twitter();
+        SEOTools::opengraph();
+        SEOTools::jsonLd();
+        SEOTools::setTitle($title);
+        SEOTools::setDescription($description);
+        SEOMeta::setRobots('noindex');
+
+        $content = Markdown::convert($markdown)->getContent();
+
+        //$content = $this->injectTailwindClasses($content);
+
+        return view('front.pages', compact('content', 'title', 'description'));
+    }
+}
--- a/app/Http/Controllers/Front/FrontSearchController.php
+++ b/app/Http/Controllers/Front/FrontSearchController.php
@@ -0,0 +1,10 @@
+<?php
+
+namespace App\Http\Controllers\Front;
+
+use App\Http\Controllers\Controller;
+
+class FrontSearchController extends Controller
+{
+    //
+}
--- a/app/Http/Controllers/Front/HomeController.php
+++ b/app/Http/Controllers/Front/HomeController.php
@@ -1,14 +0,0 @@
-<?php
-
-namespace App\Http\Controllers\Front;
-
-use App\Http\Controllers\Controller;
-use Illuminate\Http\Request;
-
-class HomeController extends Controller
-{
-    public function index(Request $request)
-    {
-        return view('front.home');
-    }
-}
--- a/app/Http/Controllers/TestOnPageController.php
+++ b/app/Http/Controllers/TestOnPageController.php
@@ -0,0 +1,36 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use App\Helpers\FirstParty\DFS\DFSCommon;
+use App\Helpers\FirstParty\DFS\DFSOnPage;
+use Illuminate\Http\Request;
+
+class TestOnPageController extends Controller
+{
+    public function listId(Request $request)
+    {
+        $response = DFSOnPage::listId(7);
+
+        $tasks = DFSCommon::getTasks($response);
+
+        foreach ($tasks as $task) {
+            $task_result = DFSCommon::getTaskResult($task);
+
+            dump($task_result);
+        }
+    }
+
+    public function taskPost(Request $request)
+    {
+        $response = DFSOnPage::taskPost('dataforseo.com', 1);
+
+        $tasks = DFSCommon::getTasks($response);
+
+        foreach ($tasks as $task) {
+            $task_result = DFSCommon::getTaskResult($task);
+
+            dump($task_result);
+        }
+    }
+}
--- a/app/Http/Controllers/Tests/GeneralTestController.php
+++ b/app/Http/Controllers/Tests/GeneralTestController.php
@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Http\Controllers\Tests;
+
+use App\Helpers\FirstParty\DFS\DFSBacklinks;
+use App\Helpers\FirstParty\DFS\DFSCommon;
+use App\Helpers\FirstParty\DFS\DFSOnPage;
+use App\Http\Controllers\Controller;
+use App\Jobs\SaveDFSToCandidateDomainsJob;
+use Illuminate\Http\Request;
+
+class GeneralTestController extends Controller
+{
+    public function onPageList(Request $request)
+    {
+        $response = DFSOnPage::listId(7);
+
+        $tasks = DFSCommon::getTasks($response);
+
+        foreach ($tasks as $task) {
+            $task_result = DFSCommon::getTaskResult($task);
+
+            dump($task_result);
+        }
+    }
+
+    public function processBacklinks(Request $request)
+    {
+        SaveDFSToCandidateDomainsJob::dispatch()->onConnection('default')->onQueue('default');
+    }
+
+    public function backlinks(Request $request)
+    {
+        $dfs_results = DFSBacklinks::backlinksPaginationLive('topai.tools', null);
+
+        dd($dfs_results);
+    }
+}
--- a/app/Http/Kernel.php
+++ b/app/Http/Kernel.php
@@ -66,5 +66,8 @@ class Kernel extends HttpKernel
        'signed' => \App\Http\Middleware\ValidateSignature::class,
        'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
        'verified' => \Illuminate\Auth\Middleware\EnsureEmailIsVerified::class,
+
+        'cacheResponse' => \Spatie\ResponseCache\Middlewares\CacheResponse::class,
+        'doNotCacheResponse' => \Spatie\ResponseCache\Middlewares\DoNotCacheResponse::class,
    ];
 }
--- a/app/Jobs/GetAIToolScreenshotJob.php
+++ b/app/Jobs/GetAIToolScreenshotJob.php
@@ -0,0 +1,37 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Jobs\Tasks\GetAIToolScreenshotTask;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class GetAIToolScreenshotJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $url_to_crawl_id;
+
+    protected $ai_tool_id;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($url_to_crawl_id, $ai_tool_id)
+    {
+        $this->url_to_crawl_id = $url_to_crawl_id;
+
+        $this->ai_tool_id = $ai_tool_id;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        GetAIToolScreenshotTask::handle($this->url_to_crawl_id, $this->ai_tool_id);
+    }
+}
--- a/app/Jobs/GetUrlBodyJob.php
+++ b/app/Jobs/GetUrlBodyJob.php
@@ -0,0 +1,35 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Jobs\Tasks\GetUrlBodyTask;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class GetUrlBodyJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $url_to_crawl_id;
+
+    public $timeout = 60;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($url_to_crawl_id)
+    {
+        $this->url_to_crawl_id = $url_to_crawl_id;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        GetUrlBodyTask::handle($this->url_to_crawl_id);
+    }
+}
--- a/app/Jobs/ParseUrlBodyJob.php
+++ b/app/Jobs/ParseUrlBodyJob.php
@@ -0,0 +1,35 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Jobs\Tasks\ParseUrlBodyTask;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class ParseUrlBodyJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $url_to_crawl_id;
+
+    public $timeout = 60;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($url_to_crawl_id)
+    {
+        $this->url_to_crawl_id = $url_to_crawl_id;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        ParseUrlBodyTask::handle($this->url_to_crawl_id);
+    }
+}
--- a/app/Jobs/ShopeeSellerTopProductScraperJob.php
+++ b/app/Jobs/ShopeeSellerTopProductScraperJob.php
@@ -1,55 +0,0 @@
-<?php
-
-namespace App\Jobs;
-
-use App\Jobs\Tasks\GenerateShopeeAIArticleTask;
-use App\Jobs\Tasks\SaveShopeeSellerImagesTask;
-use App\Jobs\Tasks\ShopeeSellerTopProductScraperTask;
-use App\Models\Category;
-use Illuminate\Bus\Queueable;
-use Illuminate\Contracts\Queue\ShouldQueue;
-use Illuminate\Foundation\Bus\Dispatchable;
-use Illuminate\Queue\InteractsWithQueue;
-use Illuminate\Queue\SerializesModels;
-
-class ShopeeSellerTopProductScraperJob implements ShouldQueue
-{
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
-
-    public $timeout = 1000;
-
-    protected $seller;
-
-    protected $country_iso;
-
-    protected $category;
-
-    /**
-     * Create a new job instance.
-     */
-    public function __construct(string $seller, string $country_iso, Category $category)
-    {
-        $this->seller = $seller;
-
-        $this->country_iso = $country_iso;
-
-        $this->category = $category;
-    }
-
-    /**
-     * Execute the job.
-     */
-    public function handle(): void
-    {
-        $shopee_task = ShopeeSellerTopProductScraperTask::handle($this->seller, $this->country_iso, $this->category);
-
-        //dd($shopee_task->product_task);
-
-        if (! is_null($shopee_task)) {
-            SaveShopeeSellerImagesTask::handle($shopee_task);
-
-            GenerateShopeeAIArticleTask::handle($shopee_task->shopee_seller_scrape);
-        }
-
-    }
-}
--- a/app/Jobs/StoreSearchEmbeddingJob.php
+++ b/app/Jobs/StoreSearchEmbeddingJob.php
@@ -0,0 +1,65 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Helpers\FirstParty\Aictio\Aictio;
+use App\Models\SearchEmbedding;
+use Exception;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class StoreSearchEmbeddingJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $type;
+
+    protected $category_id;
+
+    protected $ai_tool_id;
+
+    protected $query;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($type, $category_id, $ai_tool_id, $query)
+    {
+        $this->type = $type;
+        $this->category_id = $category_id;
+        $this->ai_tool_id = $ai_tool_id;
+        $this->query = $query;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $embedding = Aictio::getVectorEmbedding(strtolower($this->query));
+
+        if (! is_null($embedding)) {
+
+            $search_embedding = SearchEmbedding::where('type', $this->type)
+                ->where('category_id', $this->category_id)
+                ->where('ai_tool_id', $this->ai_tool_id)
+                ->where('query', $this->query)
+                ->first();
+
+            if (is_null($search_embedding)) {
+                $search_embedding = new SearchEmbedding;
+                $search_embedding->type = $this->type;
+                $search_embedding->category_id = $this->category_id;
+                $search_embedding->ai_tool_id = $this->ai_tool_id;
+                $search_embedding->query = $this->query;
+                $search_embedding->embedding = $embedding;
+                $search_embedding->save();
+            }
+        } else {
+            throw new Exception('Failed vector embedding: '.$this->query);
+        }
+    }
+}
--- a/app/Jobs/Tasks/GenerateShopeeAIArticleTask.php
+++ b/app/Jobs/Tasks/GenerateShopeeAIArticleTask.php
@@ -1,278 +0,0 @@
-<?php
-
-namespace App\Jobs\Tasks;
-
-use App\Helpers\FirstParty\OpenAI\OpenAI;
-use App\Helpers\FirstParty\OSSUploader\OSSUploader;
-use App\Models\AiWriteup;
-use App\Models\Category;
-use App\Models\Post;
-use App\Models\PostCategory;
-use App\Models\ShopeeSellerCategory;
-use App\Models\ShopeeSellerScrape;
-use App\Models\ShopeeSellerScrapedImage;
-use Exception;
-use fivefilters\Readability\Configuration as ReadabilityConfiguration;
-use fivefilters\Readability\ParseException as ReadabilityParseException;
-use fivefilters\Readability\Readability;
-use Illuminate\Support\Facades\Log;
-use LaravelFreelancerNL\LaravelIndexNow\Facades\IndexNow;
-use LaravelGoogleIndexing;
-use Masterminds\HTML5;
-use Symfony\Component\DomCrawler\Crawler;
-
-class GenerateShopeeAIArticleTask
-{
-    public static function handle(ShopeeSellerScrape $shopee_seller_scrape)
-    {
-        $serialised = OSSUploader::readFile('r2', 'shopee/seller', $shopee_seller_scrape->filename);
-
-        $post = null;
-
-        $shopee_seller_scrape->load('category');
-
-        if (! is_empty($serialised)) {
-            $shopee_task = unserialize($serialised);
-            $shopee_task->shopee_seller_scrape = $shopee_seller_scrape;
-        }
-
-        // dd($shopee_task);
-
-        // dd($shopee_task->product_task->response);
-
-        $raw_html = $shopee_task->product_task->response->raw_html;
-
-        $excerpt = self::stripHtml($raw_html);
-
-        $excerpt = substr($excerpt, 0, 1500); // limit to 1500 (+1500 output token, total 3k token) characters due to OpenAI model limitations unless use 16k model, $$$$
-
-        $excerpt .= self::getProductPricingExcerpt($shopee_task->product_task->response->jsonld);
-
-        $photos = ShopeeSellerScrapedImage::where('shopee_seller_scrape_id', $shopee_seller_scrape->id)->where('featured', false)->orderByRaw('RAND()')->take(3)->get()->pluck('image')->toArray();
-
-        $ai_writeup = AiWriteup::where('source', 'shopee')->where('source_url', $shopee_task->product_task->response->url)->first();
-
-        if (is_null($ai_writeup)) {
-
-            $categories = [
-                'Beauty',
-                'Technology',
-                'Home & Living',
-                'Health',
-                'Fitness',
-            ];
-
-            $ai_output = OpenAI::writeProductArticle($excerpt, $photos, $categories);
-
-            //dd($ai_output);
-
-            if (is_null($ai_output)) {
-                $e = new Exception('Failed to write: Missing ai_output');
-
-                Log::error(serialize($ai_writeup?->toArray()));
-                inspector()->reportException($e);
-                throw ($e);
-            } else {
-
-                $picked_category = Category::where('name', $ai_output->category)->where('country_locale_id', $shopee_seller_scrape->category->country_locale_id)->first();
-
-                if (is_null($picked_category)) {
-                    $picked_category = $shopee_seller_scrape->category;
-                }
-
-                // save
-                $ai_writeup = new AiWriteup;
-                $ai_writeup->source = 'shopee';
-                $ai_writeup->source_url = $shopee_task->product_task->response->url;
-                $ai_writeup->category_id = $picked_category->id;
-                $ai_writeup->title = $ai_output->title;
-                $ai_writeup->excerpt = $ai_output->excerpt;
-                $ai_writeup->featured_image = '';
-                $ai_writeup->body = $ai_output->body;
-                $ai_writeup->cost = self::getTotalServiceCost($shopee_task);
-                $ai_writeup->editor_format = 'markdown';
-
-                if ($ai_writeup->save()) {
-                    $featured_photo = ShopeeSellerScrapedImage::where('shopee_seller_scrape_id', $shopee_seller_scrape->id)->where('featured', true)->first();
-
-                    // new post
-                    $post_data = [
-                        'publish_date' => now(),
-                        'title' => $ai_writeup->title,
-                        'slug' => str_slug($ai_writeup->title),
-                        'excerpt' => $ai_writeup->excerpt,
-                        'cliffhanger' => $ai_writeup->cliffhanger,
-                        'author_id' => 1,
-                        'featured' => false,
-                        'featured_image' => $featured_photo->image,
-                        'editor' => 'markdown',
-                        'body' => $ai_writeup->body,
-                        'post_format' => 'standard',
-                        'status' => 'publish',
-                    ];
-
-                    $post = Post::create($post_data);
-
-                    if (! is_null($post)) {
-
-                        $shopee_seller_scrape->write_counts = $shopee_seller_scrape->write_counts + 1;
-                        $shopee_seller_scrape->last_ai_written_at = now();
-                        $shopee_seller_scrape->save();
-
-                        $shopee_seller_category = ShopeeSellerCategory::where('seller', $shopee_seller_scrape->seller)->first();
-
-                        if (is_null($shopee_seller_category)) {
-                            $shopee_seller_category = new ShopeeSellerCategory;
-                            $shopee_seller_category->seller = $shopee_seller_scrape->seller;
-                            $shopee_seller_category->category_id = $shopee_seller_scrape->category_id;
-                        }
-
-                        $shopee_seller_category->last_ai_written_at = $shopee_seller_scrape->last_ai_written_at;
-                        $shopee_seller_category->write_counts = $shopee_seller_scrape->write_counts;
-
-                        $shopee_seller_category->save();
-
-                        PostCategory::create([
-                            'post_id' => $post->id,
-                            'category_id' => $picked_category->id,
-                        ]);
-
-                        if (app()->environment() == 'production') {
-                            if ($post->status == 'publish') {
-
-                                $post_url = route('home.country.post', ['country' => $post->post_category?->category?->country_locale_slug, 'post_slug' => $post->slug]);
-
-                                LaravelGoogleIndexing::create()->update($post_url);
-                                IndexNow::submit($post_url);
-                            }
-                        }
-
-                    }
-                }
-            }
-        } else {
-            $e = new Exception('Failed to write: ai_writeup found');
-            Log::error(serialize($ai_writeup?->toArray()));
-            inspector()->reportException($e);
-            throw ($e);
-        }
-
-        return $post;
-    }
-
-    private static function getProductPricingExcerpt(array $jsonLdData)
-    {
-        foreach ($jsonLdData as $data) {
-            // Ensure the type is "Product" before proceeding
-            if (isset($data->{'@type'}) && $data->{'@type'} === 'Product') {
-
-                // Extract necessary data
-                $lowPrice = $data->offers->lowPrice ?? null;
-                $highPrice = $data->offers->highPrice ?? null;
-                $price = $data->offers->price ?? null;
-                $currency = $data->offers->priceCurrency ?? null;
-                $sellerName = $data->offers->seller->name ?? 'online store';  // default to "online store" if name is not set
-
-                if (! is_empty($currency)) {
-                    if ($currency == 'MYR') {
-                        $currency = 'RM';
-                    }
-                }
-
-                // Determine and format pricing sentence
-                if ($lowPrice && $highPrice) {
-                    $lowPrice = number_format($lowPrice, 0);
-                    $highPrice = number_format($highPrice, 0);
-
-                    return "Price Range from {$currency} {$lowPrice} to {$highPrice} in {$sellerName} online store";
-                } elseif ($price) {
-                    $price = number_format($price, 0);
-
-                    return "Priced at {$currency} {$price} in {$sellerName} online store";
-                } else {
-                    return "Price not stated, refer to {$sellerName} online store";
-                }
-
-            }
-        }
-    }
-
-    private static function getTotalServiceCost($shopee_task)
-    {
-
-        $cost = 0.00;
-
-        $cost += 0.09; // chatgpt-3.5-turbo $0.03 for 1k, writing for 2k tokens
-
-        // Shopee Seller Scraping
-        if (isset($shopee_task?->seller_shop_task?->response?->total_cost)) {
-            $cost += $shopee_task?->seller_shop_task?->response?->total_cost;
-        }
-
-        // Shopee Product Scraping
-        if (isset($shopee_task?->product_task?->response?->total_cost)) {
-            $cost += $shopee_task?->product_task?->response?->total_cost;
-        }
-
-        return $cost;
-
-    }
-
-    private static function stripHtml(string $raw_html)
-    {
-
-        $html_content = '';
-
-        try {
-
-            $r_configuration = new ReadabilityConfiguration();
-            $r_configuration->setCharThreshold(20);
-
-            $readability = new Readability($r_configuration);
-
-            $readability->parse($raw_html);
-
-            $temp_html_content = $readability->getContent();
-
-            // Remove tabs
-            $temp_html_content = str_replace("\t", '', $temp_html_content);
-
-            // Replace newlines with spaces
-            $temp_html_content = str_replace(["\n", "\r\n"], ' ', $temp_html_content);
-
-            // Replace multiple spaces with a single space
-            $temp_html_content = preg_replace('/\s+/', ' ', $temp_html_content);
-
-            // Output the cleaned text
-            $temp_html_content = trim($temp_html_content); // Using trim to remove any leading or trailing spaces
-
-            $temp_html_content = strip_tags($temp_html_content);
-
-            $crawler = new Crawler($raw_html);
-
-            // Extract meta title
-            $title = $crawler->filter('title')->text();  // This assumes <title> tags are used for titles.
-
-            // Extract meta description
-            $metaDescriptionNode = $crawler->filter('meta[name="description"]');
-            $description = $metaDescriptionNode->count() > 0 ? $metaDescriptionNode->attr('content') : null;
-
-            $html_content .= $title.' ';
-            $html_content .= $description.' ';
-            $html_content .= $temp_html_content;
-
-        } catch (ReadabilityParseException|Exception $e) {
-
-            $html5 = new HTML5(['preserveWhiteSpace' => true]);
-
-            // Parse the HTML into a DOM tree.
-            $dom = $html5->loadHTML($raw_html);
-
-            // Serialize the DOM tree back to a string, formatted.
-            $html_content = strip_tags($html5->saveHTML($dom));
-
-        }
-
-        return $html_content;
-    }
-}
--- a/app/Jobs/Tasks/GetAIToolScreenshotTask.php
+++ b/app/Jobs/Tasks/GetAIToolScreenshotTask.php
@@ -0,0 +1,87 @@
+<?php
+
+namespace App\Jobs\Tasks;
+
+use App\Helpers\FirstParty\OSSUploader\OSSUploader;
+use App\Models\AiTool;
+use App\Models\BusinessProfile;
+use App\Models\SerpUrl;
+use App\Models\UrlToCrawl;
+use Exception;
+use Image;
+use Spatie\Browsershot\Browsershot;
+
+class GetAIToolScreenshotTask
+{
+
+    public static function handle($url_to_crawl_id, $ai_tool_id)
+    {
+        $url_to_crawl = UrlToCrawl::find($url_to_crawl_id);
+
+        if (is_null($url_to_crawl))
+        {
+          return ;
+        }
+
+        $ai_tool = AiTool::find($ai_tool_id);
+
+        if (is_null($ai_tool))
+        {
+          return ;
+        }
+
+        $userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36";
+
+
+        $browsershot = Browsershot::url($url_to_crawl->url)
+            ->timeout(30)
+            ->emulateMedia('screen')
+            ->userAgent($userAgent)
+            ->windowSize(1024, 576)  // Set window size
+            ->waitUntilNetworkIdle();  // Wait until all resources are loaded
+
+        if (app()->environment() == 'local') {
+            $browsershot->setNodeBinary(config('platform.general.node_binary'))->setNpmBinary(config('platform.general.npm_binary'));
+        }
+
+        $retries = 0;
+        $maxRetries = 1;
+
+        while ($retries < $maxRetries) {
+            try {
+                $image_content = $browsershot->screenshot();
+                break;  // Exit the loop if the screenshot method succeeds
+            } catch (Exception $e) {
+                $retries++;
+                if ($retries === $maxRetries) {
+                    throw new Exception("Failed to take a screenshot after $maxRetries attempts: ".$e->getMessage(), 0, $e);
+                }
+            }
+        }
+
+        $image = Image::make($image_content)
+            ->resize(1024, null, function ($constraint) {
+                $constraint->aspectRatio();
+            })
+            ->stream('webp', 80);
+
+        $image_file_name = str_slug($ai_tool->tool_name).'-'.epoch_now_timestamp().'.webp';
+
+        $upload_status = OSSUploader::uploadFile(
+            config('platform.uploads.ai_tools.screenshot.driver'),
+            config('platform.uploads.ai_tools.screenshot.path'),
+            $image_file_name,
+            $image);
+
+        if ($upload_status) {
+            $ai_tool->screenshot_img = $image_file_name;
+        }
+
+        if ($ai_tool->isDirty()) {
+            $ai_tool->save();
+        }
+
+        return $ai_tool;
+
+    }
+}
--- a/app/Jobs/Tasks/GetUrlBodyTask.php
+++ b/app/Jobs/Tasks/GetUrlBodyTask.php
@@ -0,0 +1,204 @@
+<?php
+
+namespace App\Jobs\Tasks;
+
+use App\Jobs\ParseUrlBodyJob;
+use App\Models\ServiceCostUsage;
+use App\Models\UrlToCrawl;
+use Exception;
+use fivefilters\Readability\Configuration as ReadabilityConfiguration;
+use fivefilters\Readability\ParseException as ReadabilityParseException;
+use fivefilters\Readability\Readability;
+use Illuminate\Support\Facades\Http;
+use League\HTMLToMarkdown\HtmlConverter;
+use Symfony\Component\DomCrawler\Crawler;
+
+class GetUrlBodyTask
+{
+    public static function handle(int $url_to_crawl_id)
+    {
+        $url_to_crawl = UrlToCrawl::find($url_to_crawl_id);
+
+        if (is_null($url_to_crawl)) {
+            return null;
+        }
+
+        $url_to_crawl->is_crawling = true;
+        $url_to_crawl->save();
+        $url_to_crawl->refresh();
+
+        try {
+            $user_agent = config('platform.proxy.user_agent');
+
+            $response = Http::withHeaders([
+                'User-Agent' => $user_agent,
+            ])
+                ->withOptions([
+                    'proxy' => get_smartproxy_rotating_server(),
+                    'timeout' => 10,
+                    'verify' => false,
+                ])
+                ->get($url_to_crawl->url);
+
+            if ($response->successful()) {
+                $raw_html = $response->body();
+                $cost = calculate_smartproxy_cost(round(strlen($raw_html) / 1024, 2), 'rotating_global');
+
+                $service_cost_usage = new ServiceCostUsage;
+                $service_cost_usage->cost = $cost;
+                $service_cost_usage->name = 'smartproxy-GetUrlBodyTask';
+                $service_cost_usage->reference_1 = 'url_to_crawl';
+                $service_cost_usage->reference_2 = strval($url_to_crawl_id);
+                $service_cost_usage->output = self::getMarkdownFromHtml($raw_html);
+                $service_cost_usage->save();
+
+            } else {
+                $raw_html = null;
+                $response->throw();
+            }
+
+        } catch (Exception $e) {
+            $raw_html = null;
+            //throw $e;
+        }
+
+        if (! is_empty($raw_html)) {
+            $url_to_crawl->output_type = 'markdown';
+            $url_to_crawl->output = self::getMarkdownFromHtml($raw_html);
+
+        } else {
+            $url_to_crawl->output = 'EMPTY CONTENT';
+            $url_to_crawl->status = 'blocked';
+        }
+
+        $url_to_crawl->is_crawled = true;
+
+        if ($url_to_crawl->save()) {
+            if (! in_array($url_to_crawl->status, ['blocked', 'trashed'])) {
+                ParseUrlBodyJob::dispatch($url_to_crawl->id)->onQueue('default')->onConnection('default');
+            }
+
+        }
+    }
+
+    private static function getMainImageFromHtml($html)
+    {
+        $r_configuration = new ReadabilityConfiguration();
+        $r_configuration->setCharThreshold(20);
+
+        $readability = new Readability($r_configuration);
+
+        try {
+            $readability->parse($html);
+
+            return $readability->getImage();
+            //dd($readability);
+        } catch (ReadabilityParseException $e) {
+        }
+
+        return null;
+    }
+
+    private static function getMarkdownFromHtml($html)
+    {
+
+        $converter = new HtmlConverter([
+            'strip_tags' => true,
+            'strip_placeholder_links' => true,
+        ]);
+
+        $html = self::cleanHtml($html);
+
+        $markdown = $converter->convert($html);
+
+        //dd($markdown);
+
+        $markdown = self::reverseLTGT($markdown);
+
+        $markdown = self::normalizeNewLines($markdown);
+
+        $markdown = self::removeDuplicateLines($markdown);
+
+        return html_entity_decode(markdown_to_plaintext($markdown));
+    }
+
+    private static function reverseLTGT($input)
+    {
+        $output = str_replace('&lt;', '<', $input);
+        $output = str_replace('&gt;', '>', $output);
+
+        return $output;
+    }
+
+    private static function removeDuplicateLines($string)
+    {
+        $lines = explode("\n", $string);
+        $uniqueLines = array_unique($lines);
+
+        return implode("\n", $uniqueLines);
+    }
+
+    private static function normalizeNewLines($content)
+    {
+        // Split the content by lines
+        $lines = explode("\n", $content);
+
+        $processedLines = [];
+
+        for ($i = 0; $i < count($lines); $i++) {
+            $line = trim($lines[$i]);
+
+            // If the line is an image markdown
+            if (preg_match("/^!\[.*\]\(.*\)$/", $line)) {
+                // And if the next line is not empty and not another markdown structure
+                if (isset($lines[$i + 1]) && ! empty(trim($lines[$i + 1])) && ! preg_match('/^[-=#*&_]+$/', trim($lines[$i + 1]))) {
+                    $line .= ' '.trim($lines[$i + 1]);
+                    $i++; // Skip the next line as we're merging it
+                }
+            }
+
+            // Add line to processedLines if it's not empty
+            if (! empty($line)) {
+                $processedLines[] = $line;
+            }
+        }
+
+        // Collapse excessive newlines
+        $result = preg_replace("/\n{3,}/", "\n\n", implode("\n", $processedLines));
+
+        // Detect and replace the pattern
+        $result = preg_replace('/^(!\[.*?\]\(.*?\))\s*\n\s*([^\n!]+)/m', '$1 $2', $result);
+
+        // Replace multiple spaces with a dash separator
+        $result = preg_replace('/ {2,}/', ' - ', $result);
+
+        return $result;
+    }
+
+    private static function cleanHtml($htmlContent)
+    {
+        $crawler = new Crawler($htmlContent);
+
+        // Define tags to remove completely
+        $tagsToRemove = ['script', 'style', 'svg', 'picture', 'form', 'footer', 'nav', 'aside'];
+
+        foreach ($tagsToRemove as $tag) {
+            $crawler->filter($tag)->each(function ($node) {
+                foreach ($node as $child) {
+                    $child->parentNode->removeChild($child);
+                }
+            });
+        }
+
+        // Replace <span> tags with their inner content
+        $crawler->filter('span')->each(function ($node) {
+            $replacement = new \DOMText($node->text());
+
+            foreach ($node as $child) {
+                $child->parentNode->replaceChild($replacement, $child);
+            }
+        });
+
+        return $crawler->outerHtml();
+    }
+}
--- a/app/Jobs/Tasks/ParseUrlBodyTask.php
+++ b/app/Jobs/Tasks/ParseUrlBodyTask.php
@@ -0,0 +1,197 @@
+<?php
+
+namespace App\Jobs\Tasks;
+
+use App\Helpers\FirstParty\OpenAI\OpenAI;
+use App\Jobs\GetAIToolScreenshotJob;
+use App\Jobs\ParseUrlBodyJob;
+use App\Jobs\StoreSearchEmbeddingJob;
+use App\Models\AiTool;
+use App\Models\AiToolKeyword;
+use App\Models\Category;
+use App\Models\ServiceCostUsage;
+use App\Models\UrlToCrawl;
+use Exception;
+
+class ParseUrlBodyTask
+{
+    public static function handle(int $url_to_crawl_id)
+    {
+        $url_to_crawl = UrlToCrawl::find($url_to_crawl_id);
+
+        $parent_categories = Category::whereNull('parent_id')->orderBy('name', 'ASC')->get();
+
+        if (is_null($url_to_crawl)) {
+            return;
+        }
+
+        if (in_array($url_to_crawl->status, ['blocked', 'trashed'])) {
+            return;
+        }
+
+        if (is_empty($url_to_crawl->output)) {
+            ParseUrlBodyJob::dispatch($url_to_crawl->id)->onQueue('default')->onConnection('default');
+        }
+
+        $url_meta_response = null;
+
+        if (! is_null($url_to_crawl->metadata)) {
+            $url_meta_response = $url_to_crawl->metadata;
+        } else {
+            $url_meta_response = OpenAI::getSiteSummary($parent_categories, $url_to_crawl->output, 1536, 30, true);
+
+            if ((isset($url_meta_response->output)) && (! is_null($url_meta_response->output))) {
+                $service_cost_usage = new ServiceCostUsage;
+                $service_cost_usage->cost = $url_meta_response->cost;
+                $service_cost_usage->name = 'openai-getSiteSummary';
+                $service_cost_usage->reference_1 = 'url_to_crawl';
+                $service_cost_usage->reference_2 = strval($url_to_crawl->id);
+                $service_cost_usage->output = $url_meta_response;
+                $service_cost_usage->save();
+            }
+        }
+
+        if (is_null($url_meta_response->output)) {
+            throw new Exception('OpenAI::getSiteSummary failed. Empty object');
+        }
+
+        $url_to_crawl->metadata = $url_meta_response;
+
+        // Check AI Tool
+        $ai_tool = AiTool::where('url_to_crawl_id', $url_to_crawl->id)->first();
+
+        if (is_null($ai_tool)) {
+            $ai_tool = new AiTool;
+            $ai_tool->url_to_crawl_id = $url_to_crawl->id;
+        }
+
+        // Tool Name
+        if ((isset($url_meta_response->output->tool_name)) && (! is_empty($url_meta_response->output->tool_name))) {
+            $ai_tool->tool_name = $url_meta_response->output->tool_name;
+        } else {
+            throw new Exception('OpenAI::getSiteSummary failed, no tool name');
+        }
+
+        // Is AI Tool
+        if ((isset($url_meta_response->output->is_ai_tool)) && (! is_null($url_meta_response->output->is_at_tool)) && is_bool($url_meta_response->output->is_ai_tool)) {
+            $ai_tool->is_ai_tool = $url_meta_response->output->is_ai_tool;
+        } else {
+            $ai_tool->is_ai_tool = true;
+        }
+
+        // Is App/Web/Both
+        if ((isset($url_meta_response->output->is_app_web_both)) && (is_array($url_meta_response->output->is_app_web_both)) && in_array($url_meta_response->output->is_app_web_both, ['app', 'web', 'both'])) {
+            $ai_tool->is_app_web_both = $url_meta_response->output->is_app_web_both;
+        }
+
+        // Tagline
+        if ((isset($url_meta_response->output->tagline)) && (! is_empty($url_meta_response->output->tagline))) {
+            $ai_tool->tagline = $url_meta_response->output->tagline;
+        }
+
+        // Summary
+        if ((isset($url_meta_response->output->summary)) && (! is_empty($url_meta_response->output->summary))) {
+            $ai_tool->summary = $url_meta_response->output->summary;
+        }
+
+        // Pricing Type
+        if ((isset($url_meta_response->output->pricing_type)) && (is_array($url_meta_response->output->pricing_type)) && in_array($url_meta_response->output->pricing_type, ['Free', 'Free Trial', 'Freemium', 'Subscription', 'Usage Based'])) {
+            $ai_tool->pricing_type = $url_meta_response->output->pricing_type;
+        } else {
+            $ai_tool->pricing_type = 'Free';
+        }
+
+        // Category ID
+        $has_main_category_record = false;
+
+        $main_category = null;
+
+        if ((isset($url_meta_response->output->main_category)) && (! is_empty($url_meta_response->output->main_category))) {
+            $main_category = Category::where('name', $url_meta_response->output->main_category)->first();
+        }
+
+        if (is_null($main_category)) {
+            $main_category = Category::where('name', 'Productivity')->first();
+        }
+
+        $ai_tool->category_id = $main_category->id;
+
+        // Keyword
+        if ((isset($url_meta_response->output->keywords)) && (is_array($url_meta_response->output->keywords))) {
+            $ai_tool->keyword_string = implode(',', $url_meta_response->output->keywords);
+        }
+
+        // Q&A
+        if ((isset($url_meta_response->output->qna)) && (is_array($url_meta_response->output->qna))) {
+            $ai_tool->qna = $url_meta_response->output->qna;
+        }
+
+        if ($ai_tool->save()) {
+
+            $query = $ai_tool->tool_name;
+
+            if (!is_empty($ai_tool->tagline))
+            {
+              $query .= ": " . $ai_tool->tagline; 
+            }
+
+            StoreSearchEmbeddingJob::dispatch(
+                'ai_tool',
+                $ai_tool->category_id,
+                $ai_tool->id,
+                $query
+            );
+
+            if (is_empty($ai_tool->screenshot_img)) {
+                GetAIToolScreenshotJob::dispatch($url_to_crawl->id, $ai_tool->id)->onQueue('default')->onConnection('default');
+            }
+
+            // Keyword
+            if ((isset($url_meta_response->output->keywords)) && (is_array($url_meta_response->output->keywords))) {
+                foreach ($url_meta_response->output->keywords as $keyword) {
+                    $keyword_lowercased = strtolower(trim($keyword));
+
+                    $ai_tool_keyword = AiToolKeyword::where('value_lowercased', $keyword_lowercased)
+                        ->where('ai_tool_id', $ai_tool->id)
+                        ->first();
+
+                    if (is_null($ai_tool_keyword)) {
+                        $ai_tool_keyword = new AiToolKeyword;
+                        $ai_tool_keyword->category_id = $ai_tool->category_id;
+                        $ai_tool_keyword->ai_tool_id = $ai_tool->id;
+                        $ai_tool_keyword->value = trim($keyword);
+                        $ai_tool_keyword->value_lowercased = $keyword_lowercased;
+
+                        if ($ai_tool_keyword->save()) {
+                            StoreSearchEmbeddingJob::dispatch(
+                                'ai_tool_keyword',
+                                $ai_tool->category_id,
+                                $ai_tool->id,
+                                $ai_tool_keyword->value
+                            );
+                        }
+                    }
+                }
+            }
+
+            // Q&A
+            if ((isset($url_meta_response->output->qna)) && (is_array($url_meta_response->output->qna))) {
+                foreach ($url_meta_response->output->qna as $qna)
+                {
+                    $q = $qna->q;
+                    $a = $qna->a;
+
+                    $value = "{$q} {$a}";
+
+                    StoreSearchEmbeddingJob::dispatch(
+                        'qna',
+                        $ai_tool->category_id,
+                        $ai_tool->id,
+                        ($qna->q . " " . $qna->a)
+                    );
+                }
+            }
+        }
+
+    }
+}
--- a/app/Jobs/Tasks/SaveShopeeSellerImagesTask.php
+++ b/app/Jobs/Tasks/SaveShopeeSellerImagesTask.php
@@ -1,355 +0,0 @@
-<?php
-
-namespace App\Jobs\Tasks;
-
-use App\Models\ShopeeSellerScrapedImage;
-use Illuminate\Support\Facades\Http;
-use Illuminate\Support\Facades\Storage;
-use Illuminate\Support\Str;
-use Intervention\Image\Facades\Image;
-
-class SaveShopeeSellerImagesTask
-{
-    public static function handle($shopee_task)
-    {
-
-        $unblocker_proxy_server = get_smartproxy_unblocker_server();
-        $rotating_proxy_server = get_smartproxy_rotating_server();
-        $costs = [];
-        $user_agent = config('platform.proxy.user_agent');
-
-        ///////// PART 1
-        $main_image_url = self::getProductImageUrl($shopee_task->product_task->response->jsonld);
-
-        // if there is no main image intervention but the main image url is provided
-        if (! is_empty($main_image_url)) {
-            $scraped_image = ShopeeSellerScrapedImage::where('original_name', pathinfo($main_image_url, PATHINFO_BASENAME))->where('shopee_seller_scrape_id', $shopee_task->shopee_seller_scrape->id)->first();
-
-            if (is_null($scraped_image)) {
-                $main_image = self::getProductImage($shopee_task->product_task->response->jsonld, $rotating_proxy_server, $user_agent, $costs);
-
-                $scraped_image = self::uploadAndSaveScrapedImage($shopee_task->shopee_seller_scrape, $main_image, true);
-            }
-        }
-
-        /////// PART 2
-
-        $images = self::getFilteredImages($shopee_task->product_task->response->raw_html, $rotating_proxy_server, $user_agent, $costs);
-
-        //dd($images);
-
-        if (! is_null($images) && is_array($images) && count($images) > 0) {
-            foreach ($images as $image_obj) {
-                $scraped_image = ShopeeSellerScrapedImage::where('original_name', $image_obj->original_name)->where('shopee_seller_scrape_id', $shopee_task->shopee_seller_scrape->id)->first();
-
-                if (is_null($scraped_image)) {
-                    $scraped_image = self::uploadAndSaveScrapedImage($shopee_task->shopee_seller_scrape, $image_obj, false);
-                }
-            }
-        }
-
-        //return ShopeeSellerScrapedImage::where('shopee_seller_scrape_id', $shopee_task->shopee_seller_scrape->id)->get();
-
-    }
-
-    private static function uploadAndSaveScrapedImage($shopee_seller_scrape, $image_obj, $featured = false)
-    {
-        // Generate a unique filename for the uploaded file and LQIP version
-        $uuid = Str::uuid()->toString();
-        $fileName = time().'_'.$uuid.'.jpg';
-        $lqipFileName = time().'_'.$uuid.'_lqip.jpg';
-
-        // Convert the file to JPEG format using Intervention Image library
-        $image = $image_obj->intervention;
-
-        // Get the original image width and height
-        $originalWidth = $image->width();
-        $originalHeight = $image->height();
-
-        // Compress the image to reduce file size to 50%
-        $image->encode('jpg', 50);
-
-        // Save the processed image to the 'r2' storage driver under the 'uploads' directory
-        $filePath = 'uploads/'.$fileName;
-        $lqipFilePath = 'uploads/'.$lqipFileName;
-        Storage::disk('r2')->put($filePath, $image->stream()->detach());
-
-        // Save the original image to a temporary file and open it again
-        $tempImagePath = tempnam(sys_get_temp_dir(), 'temp_image');
-        file_put_contents($tempImagePath, $image_obj->intervention->encode());
-        $clonedImage = Image::make($tempImagePath);
-
-        // Create the LQIP version of the image using a small size while maintaining the aspect ratio
-        $lqipImage = $clonedImage->fit(10, 10, function ($constraint) {
-            $constraint->aspectRatio();
-        });
-        $lqipImage->encode('jpg', 5);
-        Storage::disk('r2')->put($lqipFilePath, $lqipImage->stream()->detach());
-
-        // Cleanup the temporary image file
-        unlink($tempImagePath);
-
-        // Get the final URL of the uploaded image (non-LQIP version)
-        $url = Storage::disk('r2')->url($filePath);
-
-        $scraped_image = new ShopeeSellerScrapedImage;
-        $scraped_image->shopee_seller_scrape_id = $shopee_seller_scrape->id;
-        $scraped_image->original_name = $image_obj->original_name;
-        $scraped_image->image = $url;
-        $scraped_image->featured = $featured;
-
-        if ($scraped_image->save()) {
-            return $scraped_image;
-        }
-
-        return null;
-    }
-
-    private static function getImageUrls(string $raw_html)
-    {
-        $images = [];
-
-        // Pattern for extracting src and alt attributes from img tags
-        $pattern = '/<img\s.*?(?:src=["\'](.*?)["\']).*?(?:alt=["\'](.*?)["\'])?[^>]*>/is';
-
-        if (preg_match_all($pattern, $raw_html, $matches, PREG_SET_ORDER)) {
-            foreach ($matches as $match) {
-                $src = $match[1];
-
-                // Check if image file name ends with '_tn' and remove it
-                $src = preg_replace('/_tn(\.[a-z]+)?$/i', '$1', $src);
-
-                $images[] = [
-                    'src' => $src,
-                    'alt' => isset($match[2]) ? $match[2] : null,
-                ];
-            }
-        }
-
-        return $images;
-    }
-
-    private static function getFilteredImages(string $raw_html, string $proxy, string $user_agent, &$costs)
-    {
-        $images = self::getImageUrls($raw_html);
-
-        //dd($images);
-
-        $filteredImages = [];
-        $uniqueAttributes = [];  // This array will track unique width, height, mime, and size combinations
-
-        $count = 0;
-
-        foreach ($images as $image) {
-            $count++;
-
-            $src = $image['src'];
-
-            try {
-                $response = Http::withOptions(['proxy' => $proxy, 'verify' => false])->withHeaders(['User-Agent' => $user_agent])->get($src);
-
-                // Check if the request was successful
-                if (! $response->successful()) {
-                    continue;
-                }
-
-                $imageData = $response->body();
-
-                // Create an Intervention Image instance from the response data
-                $interventionImage = Image::make($imageData);
-
-                $width = $interventionImage->width();
-                $height = $interventionImage->height();
-                $mime = $interventionImage->mime();
-
-                // Image size in KB
-                $sizeKb = round(strlen($imageData) / 1024, 2);
-
-                // Check constraints
-                if ($width < 800 || $height < 800 || $sizeKb < 100 || $mime !== 'image/jpeg') {
-                    continue;
-                }
-
-                if ($height > $width) {
-                    continue;
-                }
-
-                $interventionImage->resize(800, null, function ($constraint) {
-                    $constraint->aspectRatio();
-                });
-                $width = $interventionImage->width();
-                $height = $interventionImage->height();
-                $mime = $interventionImage->mime();
-
-                $image['width'] = $width;
-                $image['height'] = $height;
-                $image['mime'] = $mime;
-                $image['sizeKb'] = $sizeKb;
-
-                // Check for duplicates by searching through uniqueAttributes
-                $isDuplicate = false;
-                foreach ($uniqueAttributes as $attr) {
-                    if (
-                        $attr['width'] == $width &&
-                        $attr['height'] == $height &&
-                        $attr['mime'] == $mime &&
-                        abs($attr['sizeKb'] - $sizeKb) <= 30  // Check for size within a +/- 10kB tolerance
-                    ) {
-                        $isDuplicate = true;
-                        break;
-                    }
-                }
-
-                if (! $isDuplicate) {
-                    $uniqueAttributes[] = [
-                        'width' => $width,
-                        'height' => $height,
-                        'mime' => $mime,
-                        'sizeKb' => $sizeKb,
-                    ];
-                    $image['color_counts'] = self::getImageColorCounts($interventionImage);
-
-                    $image['intervention'] = $interventionImage;
-                    $image['original_name'] = pathinfo($src, PATHINFO_BASENAME);
-
-                    //$image['img'] = $interventionImage;
-                    $costs['count-'.$count] = calculate_smartproxy_cost($sizeKb, 'rotating_global');
-
-                    $filteredImages[] = $image;
-                }
-            } catch (\Exception $e) {
-                // Handle exceptions related to the HTTP request
-                continue;
-            }
-        }
-
-        // Collect all the color counts
-        $colorCounts = [];
-        foreach ($filteredImages as $image) {
-            $colorCounts[] = $image['color_counts'];
-        }
-
-        if (! empty($colorCounts)) {
-            // Compute the median of the color counts
-            sort($colorCounts);
-            $count = count($colorCounts);
-            $middleIndex = floor($count / 2);
-            $median = $count % 2 === 0 ? ($colorCounts[$middleIndex - 1] + $colorCounts[$middleIndex]) / 2 : $colorCounts[$middleIndex];
-
-            // Use the median to filter out the low outliers
-            $threshold = 0.10 * $median;  // Adjust this percentage as needed
-            $filteredImages = array_filter($filteredImages, function ($image) use ($threshold) {
-                return $image['color_counts'] > $threshold;
-            });
-        } else {
-            // No images found
-            $filteredImages = []; // Clear the array or take any other appropriate action
-        }
-
-        usort($filteredImages, function ($a, $b) {
-            return $b['sizeKb'] <=> $a['sizeKb'];  // Using the spaceship operator to sort in descending order
-        });
-
-        $final_images = [];
-
-        foreach ($filteredImages as $image_obj) {
-            $final_images[] = (object) $image_obj;
-        }
-
-        return $final_images;
-    }
-
-    private static function getProductImageUrl(array $jsonLdData)
-    {
-        foreach ($jsonLdData as $data) {
-            // Ensure the type is "Product" before proceeding
-            if (isset($data->{'@type'}) && $data->{'@type'} === 'Product') {
-                if (isset($data->url)) {
-                    return $data->url;
-                }
-            }
-        }
-    }
-
-    private static function getProductImage(array $jsonLdData, string $proxy, string $user_agent, &$costs)
-    {
-        foreach ($jsonLdData as $data) {
-            // Ensure the type is "Product" before proceeding
-            if (isset($data->{'@type'}) && $data->{'@type'} === 'Product') {
-                if (isset($data->url) && isset($data->image)) {
-                    try {
-                        $response = Http::withOptions(['proxy' => $proxy, 'verify' => false])->withHeaders(['User-Agent' => $user_agent])->get($data->image);
-
-                        // Check if the request was successful
-                        if ($response->successful()) {
-                            $imageData = $response->body();
-
-                            // Create an Intervention Image instance from the response data
-                            $interventionImage = Image::make($imageData);
-
-                            // Resize/upscale the image to 1920x1080 maintaining the aspect ratio and cropping if needed
-                            $interventionImage->fit(1920, 1080, function ($constraint) {
-                                $constraint->upsize();
-                                $constraint->aspectRatio();
-                            });
-
-                            $sizeInKb = strlen($interventionImage->encode()) / 1024; // Convert bytes to kilobytes
-
-                            // Calculate the cost
-                            $cost = calculate_smartproxy_cost($sizeInKb, 'rotating_global');
-
-                            $costs['product_image'] = $cost;
-
-                            return (object) [
-                                'url' => $data->url,
-                                'intervention' => $interventionImage,
-                                'original_name' => pathinfo($data->image, PATHINFO_BASENAME),
-                                'cost' => $cost,
-                            ];
-                        }
-                    } catch (\Exception $e) {
-                        // Handle exceptions related to the HTTP request
-                        return null;
-                    }
-                }
-            }
-        }
-
-        return null;
-    }
-
-    private static function getImageColorCounts($interventionImage)
-    {
-        // Use Intervention to manipulate the image
-        $img = clone $interventionImage;
-
-        // Resize to a smaller dimension for faster processing (maintaining aspect ratio)
-        $img->resize(200, null, function ($constraint) {
-            $constraint->aspectRatio();
-        });
-
-        // Apply some blur
-        $img->blur(10);
-
-        $im = imagecreatefromstring($img->encode());
-
-        $width = imagesx($im);
-        $height = imagesy($im);
-
-        $uniqueColors = [];
-
-        for ($x = 0; $x < $width; $x++) {
-            for ($y = 0; $y < $height; $y++) {
-                $rgb = imagecolorat($im, $x, $y);
-                $uniqueColors[$rgb] = true;
-            }
-        }
-
-        imagedestroy($im);
-
-        // Adjust the threshold based on your dataset.
-        // Here, I'm assuming that images with less than 100 unique colors are mostly text
-        // because we've reduced the image size and applied blurring.
-        return count($uniqueColors);
-    }
-}
--- a/app/Jobs/Tasks/ShopeeSellerTopProductScraperTask.php
+++ b/app/Jobs/Tasks/ShopeeSellerTopProductScraperTask.php
@@ -1,133 +0,0 @@
-<?php
-
-namespace App\Jobs\Tasks;
-
-use App\Helpers\FirstParty\OSSUploader\OSSUploader;
-use App\Models\Category;
-use App\Models\ShopeeSellerScrape;
-use Exception;
-
-class ShopeeSellerTopProductScraperTask
-{
-    public static function handle(string $seller, string $country_iso, Category $category)
-    {
-
-        $country_iso = strtolower($country_iso);
-
-        if (is_empty($seller)) {
-            throw new Exception('Missing \'seller\' attribute.');
-        }
-
-        $shopee_seller_scrape = ShopeeSellerScrape::where('seller', $seller)
-            ->where('country_iso', $country_iso)->first();
-
-        if (! is_null($shopee_seller_scrape)) {
-            $serialised = OSSUploader::readFile('r2', 'shopee/seller', $shopee_seller_scrape->filename);
-
-            if (! is_empty($serialised)) {
-                $obj = unserialize($serialised);
-                $obj->shopee_seller_scrape = $shopee_seller_scrape;
-
-                return $obj;
-            }
-        }
-
-        $epoch = epoch_now_timestamp();
-
-        $seller_shop_url = "https://shopee.com.my/{$seller}?page=0&sortBy=sales";
-
-        $seller_shop_task = UrlCrawlerTask::handle($seller_shop_url, 'shopee/seller', $epoch, true, false);
-
-        //dd($seller_shop_task);
-
-        if (isset($seller_shop_task->response->jsonld)) {
-            $top_rank_products = self::getSortedData($seller_shop_task->response->jsonld, 400);
-
-            if (count($top_rank_products) > 0) {
-
-                $product_found = null;
-
-                foreach ($top_rank_products as $product) {
-                    $product_task = UrlCrawlerTask::handle($product->url, 'shopee/seller', $epoch, true, true);
-
-                    if ($product_task->response->status_code >= 0) {
-                        $product_found = $product_task->response;
-                        break;
-                    }
-                }
-
-                $scraped = (object) [
-                    'seller_shop_task' => (object) [
-                        'response' => $seller_shop_task->response,
-                    ],
-                    'product_task' => (object) [
-                        'response' => $product_task->response,
-                    ],
-                ];
-
-                $serialised = serialize($scraped);
-
-                $filename = $seller.'-'.$epoch.'-'.$country_iso.'.txt';
-
-                OSSUploader::uploadFile('r2', 'shopee/seller', $filename, $serialised);
-
-                $shopee_seller_scrape = new ShopeeSellerScrape;
-                $shopee_seller_scrape->seller = $seller;
-                $shopee_seller_scrape->country_iso = $country_iso;
-                $shopee_seller_scrape->epoch = $epoch;
-                $shopee_seller_scrape->filename = $filename;
-                $shopee_seller_scrape->category_id = $category->id;
-
-                if ($shopee_seller_scrape->save()) {
-                    return (object) compact('seller_shop_task', 'product_task', 'shopee_seller_scrape');
-                }
-
-            }
-        }
-
-        return null;
-    }
-
-    private static function getSortedData($data, $minValue)
-    {
-        // Filter the items of type "Product" with an offer price greater than 200
-        $filtered = array_filter($data, function ($item) use ($minValue) {
-            $isProduct = $item->{'@type'} === 'Product';
-            $lowPrice = floatval($item->offers?->lowPrice ?? 0);
-            $price = floatval($item->offers?->price ?? 0);
-
-            return $isProduct && ($lowPrice > $minValue) || ($price > $minValue);
-        });
-
-        // Sort the items based on `ratingCount` and `ratingValue` in descending order
-        usort($filtered, function ($a, $b) {
-            $ratingCountA = intval($a->aggregateRating?->ratingCount ?? 0);
-            $ratingCountB = intval($b->aggregateRating?->ratingCount ?? 0);
-
-            $ratingValueA = floatval($a->aggregateRating?->ratingValue ?? 0);
-            $ratingValueB = floatval($b->aggregateRating?->ratingValue ?? 0);
-
-            if ($ratingCountA !== $ratingCountB) {
-                return $ratingCountB - $ratingCountA;
-            }
-
-            return $ratingValueB <=> $ratingValueA;
-        });
-
-        // Map the filtered and sorted items to a new array of objects
-        return array_map(function ($item) {
-            return (object) [
-                'name' => $item->name ?? null,
-                'description' => $item->description ?? null,
-                'url' => $item->url ?? null,
-                'image' => $item->image ?? null,
-                'lowPrice' => floatval($item->offers?->lowPrice ?? 0),
-                'highPrice' => floatval($item->offers?->highPrice ?? 0),
-                'price' => floatval($item->offers?->price ?? 0),
-                'priceCurrency' => $item->offers?->priceCurrency ?? null,
-                'ratingCount' => intval($item->aggregateRating?->ratingCount ?? 0),
-                'ratingValue' => floatval($item->aggregateRating?->ratingValue ?? 0),
-            ];
-        }, $filtered);
-    }
-}
--- a/app/Jobs/Tasks/UrlCrawlerTask.php
+++ b/app/Jobs/Tasks/UrlCrawlerTask.php
@@ -1,236 +0,0 @@
-<?php
-
-namespace App\Jobs\Tasks;
-
-use App\Helpers\FirstParty\OSSUploader\OSSUploader;
-use Exception;
-use Illuminate\Support\Facades\Http;
-use Spatie\Browsershot\Browsershot;
-use Spatie\Browsershot\Exceptions\UnsuccessfulResponse;
-use Symfony\Component\DomCrawler\Crawler;
-
-class UrlCrawlerTask
-{
-    public static function handle(string $url, $directory, $postfix = null, $strip_html = false, $parse_images = false)
-    {
-        $slug = str_slug($url);
-
-        $cached_url = $url; // self::getGoogleCachedUrl($url, false);
-
-        $postfix = strval($postfix);
-
-        $driver = 'r2';
-        $filename = $slug.'-'.$postfix.'.html';
-        $user_agent = config('platform.proxy.user_agent');
-        $disk_url = $directory.$filename;
-
-        $raw_html = null;
-        $status_code = 0;
-
-        $costs = [];
-
-        $unblocker_proxy_server = get_smartproxy_unblocker_server();
-        $rotating_proxy_server = get_smartproxy_rotating_server();
-
-        try {
-            $raw_html = OSSUploader::readFile($driver, $directory, $filename);
-
-            if (is_null($raw_html)) {
-                $status_code = -1;
-                throw new Exception('Not stored.');
-            }
-        } catch (Exception $e) {
-            $raw_html = null;
-        }
-
-            if (is_null($raw_html)) {
-
-                try {
-
-                    $response = Http::withHeaders([
-                        'User-Agent' => $user_agent,
-                    ])
-                        ->withOptions([
-                            'proxy' => $unblocker_proxy_server,
-                            'timeout' => 1000,
-                            'verify' => false,
-                        ])
-                        ->get($cached_url);
-
-                    if ($response->successful()) {
-                        $raw_html = $response->body();
-                        $costs['unblocker'] = calculate_smartproxy_cost(round(strlen($raw_html) / 1024, 2), 'unblocker');
-                    } else {
-                        $raw_html = null;
-                        $status_code = -3;
-                        $response->throw();
-                    }
-
-                    // $browsershot = new Browsershot();
-
-                    // $browsershot->setUrl($cached_url)
-                    //     ->setOption('args', ['headless: "new"'])
-                    //     ->noSandbox()
-                    //     ->setOption('args', ['--disable-web-security'])
-                    //     ->userAgent($user_agent)
-                    //     ->ignoreHttpsErrors()
-                    //     ->preventUnsuccessfulResponse()
-                    //     ->timeout(10)
-                    //     ->setProxyServer($proxy_server)
-                    //     ->userAgent($user_agent);
-
-                    // if (app()->environment() == 'local') {
-                    //     $browsershot->setNodeBinary(config('platform.general.node_binary'))->setNpmBinary(config('platform.general.npm_binary'));
-                    // }
-
-                    // //dump($browsershot);
-
-                    // $raw_html = $browsershot->bodyHtml();
-
-                    // $sizeInKb = strlen($raw_html) / 1024; // Convert bytes to kilobytes
-                    // $browsershot_cost = round(calculate_smartproxy_cost($sizeInKb)) ;
-
-                    // $costs['html'] = $browsershot_cost;
-
-                } catch (UnsuccessfulResponse|Exception $e) {
-                    $raw_html = null;
-                    $status_code = -3;
-                    throw $e;
-                }
-
-                if (! is_empty($raw_html)) {
-                    OSSUploader::uploadFile($driver, $directory, $filename, $raw_html);
-                    $status_code = 1;
-                }
-            }
-
-            if (! is_null($raw_html)) {
-
-                //$raw_html = self::minifyAndCleanHtml($raw_html);
-
-                $jsonld = self::getJsonLd($raw_html);
-
-                return (object) [
-                    'response' => (object) [
-                        'url' => $url,
-                        'postfix' => $postfix,
-                        'filename' => $disk_url,
-                        'raw_html' => $raw_html,
-                        'jsonld' => $jsonld,
-                        'status_code' => $status_code,
-                        'costs' => $costs,
-                        'total_cost' => array_sum(array_values($costs)),
-                    ],
-                ];
-            }
-
-            return (object) [
-                'response' => (object) [
-                    'url' => $url,
-                    'postfix' => $postfix,
-                    'filename' => null,
-                    'raw_html' => null,
-                    'jsonld' => [],
-                    'status_code' => $status_code,
-                    'costs' => $costs,
-                    'total_cost' => 0,
-                ],
-            ];
-    }
-
-    private static function getJsonLd(string $raw_html)
-    {
-        $crawler = new Crawler($raw_html);
-
-        try {
-            $jsonld = $crawler->filter('script[type="application/ld+json"]')->each(function (Crawler $node) {
-                return $node->text();
-            });
-        } catch (Exception $e) {
-            return [];
-        }
-
-        $contents = [];
-
-        foreach ($jsonld as $content) {
-            try {
-                $contents[] = json_decode($content);
-            } catch (Exception $e) {
-            }
-        }
-
-        return $contents;
-    }
-
-    private static function minifyAndCleanHtml(string $raw_html)
-    {
-        $raw_html = self::minifyHTML($raw_html);
-
-        $crawler = new Crawler($raw_html);
-
-        // Directly loop through the DOM and remove 'class' and 'id' attributes
-        foreach ($crawler as $domElement) {
-            /** @var \DOMNodeList $nodes */
-            $nodes = $domElement->getElementsByTagName('*');
-            foreach ($nodes as $node) {
-                /** @var \DOMElement $node */
-                $node->removeAttribute('class');
-                $node->removeAttribute('id');
-                $node->removeAttribute('style');
-            }
-        }
-
-        // Remove <style> tags and their content
-        $styleTags = $domElement->getElementsByTagName('style');
-        for ($i = $styleTags->length; --$i >= 0;) {
-            $styleNode = $styleTags->item($i);
-            $styleNode->parentNode->removeChild($styleNode);
-        }
-
-        // Output the manipulated HTML
-        return $crawler->html();
-    }
-
-    private static function minifyHTML($input)
-    {
-        // Remove extra white space between HTML tags
-        $input = preg_replace('/>\s+</', '><', $input);
-
-        // Remove comments
-        $input = preg_replace('/<!--(.|\s)*?-->/', '', $input);
-
-        return $input;
-    }
-
-    private static function getGoogleCachedUrl(string $url, $stripHtml = false)
-    {
-        $url = self::stripUrlQueryParameters($url);
-        $cached_url = "https://webcache.googleusercontent.com/search?q=cache:{$url}";
-
-        if ($stripHtml) {
-            $cached_url .= '&strip=1';
-        }
-
-        return $cached_url;
-
-    }
-
-    private static function stripUrlQueryParameters(string $url)
-    {
-        // Parse the URL into its components
-        $parts = parse_url($url);
-
-        // Rebuild the URL without the query component
-        $newUrl = $parts['scheme'].'://'.$parts['host'];
-
-        if (isset($parts['path'])) {
-            $newUrl .= $parts['path'];
-        }
-
-        if (isset($parts['fragment'])) {
-            $newUrl .= '#'.$parts['fragment'];
-        }
-
-        return $newUrl;
-    }
-}
--- a/app/Models/AiTool.php
+++ b/app/Models/AiTool.php
@@ -0,0 +1,82 @@
+<?php
+
+/**
+ * Created by Reliese Model.
+ */
+
+namespace App\Models;
+
+use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Collection;
+use Illuminate\Database\Eloquent\Model;
+
+/**
+ * Class AiTool
+ * 
+ * @property int $id
+ * @property int $category_id
+ * @property int $url_to_crawl_id
+ * @property string|null $screenshot_img
+ * @property bool $is_ai_tool
+ * @property string $tool_name
+ * @property string $is_app_web_both
+ * @property string|null $tagline
+ * @property string|null $summary
+ * @property string $pricing_type
+ * @property string|null $keyword_string
+ * @property string|null $qna
+ * @property Carbon|null $created_at
+ * @property Carbon|null $updated_at
+ * 
+ * @property Category $category
+ * @property UrlToCrawl $url_to_crawl
+ * @property Collection|SearchEmbedding[] $search_embeddings
+ * @property Collection|AiToolKeyword[] $ai_tool_keywords
+ *
+ * @package App\Models
+ */
+class AiTool extends Model
+{
+	protected $table = 'ai_tools';
+
+	protected $casts = [
+		'category_id' => 'int',
+		'url_to_crawl_id' => 'int',
+		'is_ai_tool' => 'bool',
+		'qna' => 'object'
+	];
+
+	protected $fillable = [
+		'category_id',
+		'url_to_crawl_id',
+		'screenshot_img',
+		'is_ai_tool',
+		'tool_name',
+		'is_app_web_both',
+		'tagline',
+		'summary',
+		'pricing_type',
+		'keyword_string',
+		'qna'
+	];
+
+	public function category()
+	{
+		return $this->belongsTo(Category::class);
+	}
+
+	public function url_to_crawl()
+	{
+		return $this->belongsTo(UrlToCrawl::class);
+	}
+
+	public function search_embeddings()
+	{
+		return $this->hasMany(SearchEmbedding::class);
+	}
+
+	public function ai_tool_keywords()
+	{
+		return $this->hasMany(AiToolKeyword::class);
+	}
+}
--- a/app/Models/AiToolKeyword.php
+++ b/app/Models/AiToolKeyword.php
@@ -10,28 +10,32 @@
 use Illuminate\Database\Eloquent\Model;

 /**
- * Class PostCategory
+ * Class AiToolKeyword
 *
 * @property int $id
- * @property int $post_id
 * @property int $category_id
+ * @property int $ai_tool_id
+ * @property string $value
+ * @property string $value_lowercased
 * @property Carbon|null $created_at
 * @property Carbon|null $updated_at
 * @property Category $category
- * @property Post $post
+ * @property AiTool $ai_tool
 */
-class PostCategory extends Model
+class AiToolKeyword extends Model
 {
-    protected $table = 'post_categories';
+    protected $table = 'ai_tool_keywords';

    protected $casts = [
-        'post_id' => 'int',
        'category_id' => 'int',
+        'ai_tool_id' => 'int',
    ];

    protected $fillable = [
-        'post_id',
        'category_id',
+        'ai_tool_id',
+        'value',
+        'value_lowercased',
    ];

    public function category()
@@ -39,8 +43,8 @@ public function category()
        return $this->belongsTo(Category::class);
    }

-    public function post()
+    public function ai_tool()
    {
-        return $this->belongsTo(Post::class);
+        return $this->belongsTo(AiTool::class);
    }
 }
--- a/app/Models/AiWriteup.php
+++ b/app/Models/AiWriteup.php
@@ -1,55 +0,0 @@
-<?php
-
-/**
- * Created by Reliese Model.
- */
-
-namespace App\Models;
-
-use Carbon\Carbon;
-use Illuminate\Database\Eloquent\Model;
-
-/**
- * Class AiWriteup
- *
- * @property int $id
- * @property string $source
- * @property string $source_url
- * @property int $category_id
- * @property string $title
- * @property string $editor_format
- * @property string|null $excerpt
- * @property string|null $featured_image
- * @property array|null $body
- * @property float $cost
- * @property Carbon|null $created_at
- * @property Carbon|null $updated_at
- * @property Category $category
- */
-class AiWriteup extends Model
-{
-    protected $table = 'ai_writeups';
-
-    protected $casts = [
-        'category_id' => 'int',
-        'body' => 'json',
-        'cost' => 'float',
-    ];
-
-    protected $fillable = [
-        'source',
-        'source_url',
-        'category_id',
-        'title',
-        'editor_format',
-        'excerpt',
-        'featured_image',
-        'body',
-        'cost',
-    ];
-
-    public function category()
-    {
-        return $this->belongsTo(Category::class);
-    }
-}
--- a/app/Models/Author.php
+++ b/app/Models/Author.php
@@ -1,47 +0,0 @@
-<?php
-
-/**
- * Created by Reliese Model.
- */
-
-namespace App\Models;
-
-use Carbon\Carbon;
-use Illuminate\Database\Eloquent\Collection;
-use Illuminate\Database\Eloquent\Model;
-
-/**
- * Class Author
- *
- * @property int $id
- * @property string $name
- * @property string $avatar
- * @property string $bio
- * @property bool $enabled
- * @property bool $public
- * @property Carbon|null $created_at
- * @property Carbon|null $updated_at
- * @property Collection|Post[] $posts
- */
-class Author extends Model
-{
-    protected $table = 'authors';
-
-    protected $casts = [
-        'enabled' => 'bool',
-        'public' => 'bool',
-    ];
-
-    protected $fillable = [
-        'name',
-        'avatar',
-        'bio',
-        'enabled',
-        'public',
-    ];
-
-    public function posts()
-    {
-        return $this->hasMany(Post::class);
-    }
-}
--- a/app/Models/Category.php
+++ b/app/Models/Category.php
@@ -7,18 +7,18 @@
 namespace App\Models;

 use Carbon\Carbon;
-use GeneaLabs\LaravelModelCaching\Traits\Cachable;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\SoftDeletes;
+use Kalnoy\Nestedset\NodeTrait;

 /**
 * Class Category
 *
 * @property int $id
- * @property int $country_locale_id
- * @property string|null $name
- * @property string $short_name
+ * @property string $name
+ * @property string|null $emoji
 * @property string|null $slug
+ * @property string|null $description
 * @property bool $enabled
 * @property bool $is_top
 * @property int $_lft
@@ -27,16 +27,14 @@
 * @property string|null $deleted_at
 * @property Carbon|null $created_at
 * @property Carbon|null $updated_at
- * @property CountryLocale $country_locale
 */
 class Category extends Model
 {
-    use Cachable, SoftDeletes;
+    use NodeTrait, SoftDeletes;

    protected $table = 'categories';

    protected $casts = [
-        'country_locale_id' => 'int',
        'enabled' => 'bool',
        'is_top' => 'bool',
        '_lft' => 'int',
@@ -45,10 +43,8 @@ class Category extends Model
    ];

    protected $fillable = [
-        'type',
-        'country_locale_id',
        'name',
-        'short_name',
+        'emoji',
        'slug',
        'description',
        'enabled',
@@ -57,9 +53,4 @@ class Category extends Model
        '_rgt',
        'parent_id',
    ];
-
-    public function country_locale()
-    {
-        return $this->belongsTo(CountryLocale::class);
-    }
 }
--- a/app/Models/CountryLocale.php
+++ b/app/Models/CountryLocale.php
@@ -1,44 +0,0 @@
-<?php
-
-/**
- * Created by Reliese Model.
- */
-
-namespace App\Models;
-
-use Carbon\Carbon;
-use GeneaLabs\LaravelModelCaching\Traits\Cachable;
-use Illuminate\Database\Eloquent\Model;
-use Illuminate\Database\Eloquent\SoftDeletes;
-
-/**
- * Class CountryLocale
- *
- * @property int $id
- * @property string $name
- * @property string $slug
- * @property string $i18n
- * @property bool $enabled
- * @property string|null $deleted_at
- * @property Carbon|null $created_at
- * @property Carbon|null $updated_at
- */
-class CountryLocale extends Model
-{
-    use Cachable, SoftDeletes;
-
-    protected $table = 'country_locales';
-
-    protected $casts = [
-        'enabled' => 'bool',
-    ];
-
-    protected $fillable = [
-        'name',
-        'slug',
-        'i18n',
-        'lang',
-        'country_iso',
-        'enabled',
-    ];
-}
--- a/app/Models/DailyTaskSchedule.php
+++ b/app/Models/DailyTaskSchedule.php
@@ -1,33 +0,0 @@
-<?php
-
-/**
- * Created by Reliese Model.
- */
-
-namespace App\Models;
-
-use Carbon\Carbon;
-use Illuminate\Database\Eloquent\Model;
-
-/**
- * Class DailyTaskSchedule
- *
- * @property int $id
- * @property string $task
- * @property Carbon $next_run_time
- * @property Carbon|null $created_at
- * @property Carbon|null $updated_at
- */
-class DailyTaskSchedule extends Model
-{
-    protected $table = 'daily_task_schedules';
-
-    protected $casts = [
-        'next_run_time' => 'datetime',
-    ];
-
-    protected $fillable = [
-        'task',
-        'next_run_time',
-    ];
-}
--- a/app/Models/DraftPost.php
+++ b/app/Models/DraftPost.php
@@ -1,11 +0,0 @@
-<?php
-
-namespace App\Models;
-
-use Illuminate\Database\Eloquent\Factories\HasFactory;
-use Illuminate\Database\Eloquent\Model;
-
-class DraftPost extends Model
-{
-    use HasFactory;
-}
--- a/app/Models/FailedJob.php
+++ b/app/Models/FailedJob.php
@@ -1,38 +0,0 @@
-<?php
-
-namespace App\Models;
-
-use Illuminate\Database\Eloquent\Model;
-
-class FailedJob extends Model
-{
-    /**
-     * The table associated with the model.
-     *
-     * @var string
-     */
-    protected $table = 'failed_jobs';
-
-    /**
-     * The attributes that aren't mass assignable.
-     *
-     * @var array
-     */
-    protected $guarded = [];
-
-    /**
-     * Indicates if the model should be timestamped.
-     *
-     * @var bool
-     */
-    public $timestamps = false;
-
-    /**
-     * Casts attributes to native types.
-     *
-     * @var array
-     */
-    protected $casts = [
-        'failed_at' => 'datetime',
-    ];
-}
--- a/app/Models/Post.php
+++ b/app/Models/Post.php
@@ -1,236 +0,0 @@
-<?php
-
-/**
- * Created by Reliese Model.
- */
-
-namespace App\Models;
-
-use AlAminFirdows\LaravelEditorJs\Facades\LaravelEditorJs;
-use Carbon\Carbon;
-use GrahamCampbell\Markdown\Facades\Markdown;
-use Illuminate\Database\Eloquent\Collection;
-use Illuminate\Database\Eloquent\Model;
-use Spatie\Feed\Feedable;
-use Spatie\Feed\FeedItem;
-use Symfony\Component\DomCrawler\Crawler;
-
-/**
- * Class Post
- *
- * @property int $id
- * @property string|null $title
- * @property string|null $slug
- * @property string|null $excerpt
- * @property int|null $author_id
- * @property string $featured_image
- * @property string $editor
- * @property array|null $body
- * @property string $post_format
- * @property int $comment_count
- * @property int $likes_count
- * @property string $status
- * @property Carbon|null $created_at
- * @property Carbon|null $updated_at
- * @property Author|null $author
- * @property Collection|PostCategory[] $post_categories
- */
-class Post extends Model implements Feedable
-{
-    protected $table = 'posts';
-
-    protected $casts = [
-        'author_id' => 'int',
-        'body' => 'json',
-        'comment_count' => 'int',
-        'likes_count' => 'int',
-        'featured' => 'bool',
-        'publish_date' => 'datetime',
-    ];
-
-    protected $fillable = [
-        'title',
-        'slug',
-        'cliffhanger',
-        'excerpt',
-        'author_id',
-        'featured_image',
-        'editor',
-        'body',
-        'post_format',
-        'comment_count',
-        'likes_count',
-        'status',
-        'featured',
-        'publish_date',
-    ];
-
-    protected $appends = [
-        //'html_body',
-    ];
-
-    public function author()
-    {
-        return $this->belongsTo(Author::class);
-    }
-
-    public function post_categories()
-    {
-        return $this->hasMany(PostCategory::class);
-    }
-
-    public function post_category()
-    {
-        return $this->hasOne(PostCategory::class);
-    }
-
-    public function getHtmlBodyAttribute()
-    {
-        if (! is_empty($this->body)) {
-            if ($this->editor == 'editorjs') {
-                return LaravelEditorJs::render(json_encode($this->body));
-            } elseif ($this->editor == 'markdown') {
-                //dd($this->body);
-
-                $html = Markdown::convert($this->body)->getContent();
-
-                //dd($html);
-
-                $html = str_replace('\\n', "\n", $html);
-
-                $crawler = new Crawler($html);
-
-                $firstHeaderProcessed = false;
-
-                $crawler->filter('h1, h2, h3, h4, h5, h6')->each(function (Crawler $node) use (&$firstHeaderProcessed) {
-                    $element = $node->getNode(0);
-
-                    // Create a new h3 element
-                    $newElement = $element->ownerDocument->createElement('h3');
-
-                    // Copy attributes from the old header element to the new h3 element
-                    foreach ($element->attributes as $attribute) {
-                        $newElement->setAttribute($attribute->name, $attribute->value);
-                    }
-
-                    // Add the 'fw-bold' class to the new h3 element
-                    $existingClasses = $element->getAttribute('class');
-                    $newClass = 'fw-bold';
-                    $updatedClasses = ($existingClasses ? $existingClasses.' ' : '').$newClass;
-                    $newElement->setAttribute('class', $updatedClasses);
-
-                    // Move child nodes from the old header element to the new h3 element
-                    while ($element->firstChild) {
-                        $newElement->appendChild($element->firstChild);
-                    }
-
-                    // Replace the old header element with the new h3 element in the DOM
-                    $element->parentNode->replaceChild($newElement, $element);
-
-                    // Only for the first header
-                    if (! $firstHeaderProcessed) {
-                        $firstHeaderProcessed = true;
-
-                        $nextSibling = $newElement->nextSibling;
-                        while ($nextSibling) {
-                            if ($nextSibling->nodeType === XML_ELEMENT_NODE) {
-                                if ($nextSibling->nodeName === 'p' && ($nextSibling->getElementsByTagName('img')->length > 0 || $nextSibling->getElementsByTagName('figure')->length > 0)) {
-                                    // Remove <p> if it contains an <img> or <figure> directly
-                                    $nextSibling->parentNode->removeChild($nextSibling);
-                                    break;
-                                } elseif ($nextSibling->nodeName === 'img' || $nextSibling->nodeName === 'figure') {
-                                    // Direct <img> or <figure> without wrapping <p>
-                                    $newElement->parentNode->removeChild($nextSibling);
-                                    break;
-                                } else {
-                                    break;
-                                }
-                            } else {
-                                $nextSibling = $nextSibling->nextSibling;
-                            }
-                        }
-                    }
-                });
-
-                // Modify the DOM by wrapping the <img> tags inside a <figure> and adding the desired structure
-                $crawler->filter('img')->each(function (Crawler $node) {
-                    $imgElement = $node->getNode(0);
-
-                    // Update the class of the <img>
-                    $existingClasses = $imgElement->getAttribute('class');
-                    $newClasses = 'img-fluid rounded-2 shadow-sm mb-2';
-                    $updatedClasses = ($existingClasses ? $existingClasses.' ' : '').$newClasses;
-                    $imgElement->setAttribute('class', $updatedClasses);
-
-                    // Create a new <figure> element
-                    $figureElement = new \DOMElement('figure');
-                    $imgElement->parentNode->insertBefore($figureElement, $imgElement);
-
-                    // Move the <img> inside the <figure>
-                    $figureElement->appendChild($imgElement);
-                    $figureElement->setAttribute('class', 'image');
-
-                    // Create a new <footer> element inside <figure> without directly setting its content
-                    $footerElement = $imgElement->ownerDocument->createElement('footer');
-                    $figureElement->appendChild($footerElement);
-
-                    // Add the content to <footer> using createTextNode to ensure special characters are handled
-                    $footerText = $imgElement->ownerDocument->createTextNode($imgElement->getAttribute('alt'));
-                    $footerElement->appendChild($footerText);
-
-                    // Set the class attribute for <footer>
-                    $footerElement->setAttribute('class', 'image-caption');
-                });
-
-                // Add Bootstrap 5 styling to tables
-                $crawler->filter('table')->each(function (Crawler $node) {
-                    $tableElement = $node->getNode(0);
-                    $existingClasses = $tableElement->getAttribute('class');
-                    $newClass = 'table';
-                    $updatedClasses = ($existingClasses ? $existingClasses.' ' : '').$newClass;
-                    $tableElement->setAttribute('class', $updatedClasses);
-                });
-
-                // Convert the modified DOM back to HTML
-                $updatedHtml = '';
-                foreach ($crawler as $domElement) {
-                    $updatedHtml .= $domElement->ownerDocument->saveHTML($domElement);
-                }
-
-                return $updatedHtml;
-
-            }
-
-        }
-
-        return '';
-
-    }
-
-    public function getFeaturedImageLqipAttribute()
-    {
-        $featuredImage = $this->featured_image;
-
-        // Get the extension of the original featured image
-        $extension = pathinfo($featuredImage, PATHINFO_EXTENSION);
-
-        // Append "_lqip" before the extension to create the LQIP image URL
-        return str_replace(".{$extension}", "_lqip.{$extension}", $featuredImage);
-    }
-
-    public function toFeedItem(): FeedItem
-    {
-        return FeedItem::create()
-            ->id($this->id)
-            ->title($this->title)
-            ->summary($this->excerpt)
-            ->updated($this->publish_date)
-            ->link(route('home.country.post', ['country' => $this->post_category?->category?->country_locale_slug, 'post_slug' => $this->slug]))
-            ->authorName($this->author->name);
-    }
-
-    public static function getFeedItems()
-    {
-        return Post::where('status', 'publish')->latest()->get();
-    }
-}
--- a/app/Models/SearchEmbedding.php
+++ b/app/Models/SearchEmbedding.php
@@ -0,0 +1,54 @@
+<?php
+
+/**
+ * Created by Reliese Model.
+ */
+
+namespace App\Models;
+
+use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Model;
+use Pgvector\Laravel\Vector;
+
+/**
+ * Class SearchEmbedding
+ *
+ * @property int $id
+ * @property string $type
+ * @property int|null $category_id
+ * @property int|null $ai_tool_id
+ * @property string $query
+ * @property USER-DEFINED|null $embedding
+ * @property Carbon|null $created_at
+ * @property Carbon|null $updated_at
+ * @property Category|null $category
+ * @property AiTool|null $ai_tool
+ */
+class SearchEmbedding extends Model
+{
+    protected $table = 'search_embeddings';
+
+    protected $casts = [
+        'category_id' => 'int',
+        'ai_tool_id' => 'int',
+        'embedding' => Vector::class,
+    ];
+
+    protected $fillable = [
+        'type',
+        'category_id',
+        'ai_tool_id',
+        'query',
+        'embedding',
+    ];
+
+    public function category()
+    {
+        return $this->belongsTo(Category::class);
+    }
+
+    public function ai_tool()
+    {
+        return $this->belongsTo(AiTool::class);
+    }
+}
--- a/app/Models/ServiceCostUsage.php
+++ b/app/Models/ServiceCostUsage.php
@@ -0,0 +1,44 @@
+<?php
+
+/**
+ * Created by Reliese Model.
+ */
+
+namespace App\Models;
+
+use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Model;
+
+/**
+ * Class ServiceCostUsage
+ *
+ * @property int $id
+ * @property float $cost
+ * @property string $name
+ * @property string|null $reference_1
+ * @property string|null $reference_2
+ * @property string $output
+ * @property string|null $input_1
+ * @property string|null $input_2
+ * @property Carbon|null $created_at
+ * @property Carbon|null $updated_at
+ */
+class ServiceCostUsage extends Model
+{
+    protected $table = 'service_cost_usages';
+
+    protected $casts = [
+        'cost' => 'float',
+        'output' => 'binary',
+    ];
+
+    protected $fillable = [
+        'cost',
+        'name',
+        'reference_1',
+        'reference_2',
+        'output',
+        'input_1',
+        'input_2',
+    ];
+}
--- a/app/Models/ShopeeSellerCategory.php
+++ b/app/Models/ShopeeSellerCategory.php
@@ -1,45 +0,0 @@
-<?php
-
-/**
- * Created by Reliese Model.
- */
-
-namespace App\Models;
-
-use Carbon\Carbon;
-use Illuminate\Database\Eloquent\Model;
-
-/**
- * Class ShopeeSellerCategory
- *
- * @property int $id
- * @property string $seller
- * @property int $category_id
- * @property Carbon|null $last_ai_written_at
- * @property int $write_counts
- * @property Carbon|null $created_at
- * @property Carbon|null $updated_at
- * @property Category $category
- */
-class ShopeeSellerCategory extends Model
-{
-    protected $table = 'shopee_seller_categories';
-
-    protected $casts = [
-        'category_id' => 'int',
-        'last_ai_written_at' => 'datetime',
-        'write_counts' => 'int',
-    ];
-
-    protected $fillable = [
-        'seller',
-        'category_id',
-        'last_ai_written_at',
-        'write_counts',
-    ];
-
-    public function category()
-    {
-        return $this->belongsTo(Category::class);
-    }
-}
--- a/app/Models/ShopeeSellerScrape.php
+++ b/app/Models/ShopeeSellerScrape.php
@@ -1,57 +0,0 @@
-<?php
-
-/**
- * Created by Reliese Model.
- */
-
-namespace App\Models;
-
-use Carbon\Carbon;
-use Illuminate\Database\Eloquent\Collection;
-use Illuminate\Database\Eloquent\Model;
-
-/**
- * Class ShopeeSellerScrape
- *
- * @property int $id
- * @property int|null $category_id
- * @property string $seller
- * @property string $country_iso
- * @property int $epoch
- * @property string $filename
- * @property Carbon|null $created_at
- * @property Carbon|null $updated_at
- * @property Category|null $category
- * @property Collection|ShopeeSellerScrapedImage[] $shopee_seller_scraped_images
- */
-class ShopeeSellerScrape extends Model
-{
-    protected $table = 'shopee_seller_scrapes';
-
-    protected $casts = [
-        'category_id' => 'int',
-        'epoch' => 'int',
-        'last_ai_written_at' => 'datetime',
-        'write_counts' => 'int',
-    ];
-
-    protected $fillable = [
-        'category_id',
-        'seller',
-        'country_iso',
-        'epoch',
-        'filename',
-        'last_ai_written_at',
-        'write_counts',
-    ];
-
-    public function category()
-    {
-        return $this->belongsTo(Category::class);
-    }
-
-    public function shopee_seller_scraped_images()
-    {
-        return $this->hasMany(ShopeeSellerScrapedImage::class);
-    }
-}
--- a/app/Models/ShopeeSellerScrapedImage.php
+++ b/app/Models/ShopeeSellerScrapedImage.php
@@ -1,44 +0,0 @@
-<?php
-
-/**
- * Created by Reliese Model.
- */
-
-namespace App\Models;
-
-use Carbon\Carbon;
-use Illuminate\Database\Eloquent\Model;
-
-/**
- * Class ShopeeSellerScrapedImage
- *
- * @property int $id
- * @property int $shopee_seller_scrape_id
- * @property string $original_name
- * @property string|null $image
- * @property bool $featured
- * @property Carbon|null $created_at
- * @property Carbon|null $updated_at
- * @property ShopeeSellerScrape $shopee_seller_scrape
- */
-class ShopeeSellerScrapedImage extends Model
-{
-    protected $table = 'shopee_seller_scraped_images';
-
-    protected $casts = [
-        'shopee_seller_scrape_id' => 'int',
-        'featured' => 'bool',
-    ];
-
-    protected $fillable = [
-        'shopee_seller_scrape_id',
-        'original_name',
-        'image',
-        'featured',
-    ];
-
-    public function shopee_seller_scrape()
-    {
-        return $this->belongsTo(ShopeeSellerScrape::class);
-    }
-}
--- a/app/Models/UrlToCrawl.php
+++ b/app/Models/UrlToCrawl.php
@@ -0,0 +1,47 @@
+<?php
+
+/**
+ * Created by Reliese Model.
+ */
+
+namespace App\Models;
+
+use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Model;
+
+/**
+ * Class UrlToCrawl
+ *
+ * @property int $id
+ * @property string $domain
+ * @property string $url
+ * @property bool $is_crawling
+ * @property bool $is_crawled
+ * @property string|null $output_type
+ * @property string|null $output
+ * @property string|null $metadata
+ * @property string $status
+ * @property Carbon|null $created_at
+ * @property Carbon|null $updated_at
+ */
+class UrlToCrawl extends Model
+{
+    protected $table = 'url_to_crawls';
+
+    protected $casts = [
+        'is_crawling' => 'bool',
+        'is_crawled' => 'bool',
+        'metadata' => 'object',
+    ];
+
+    protected $fillable = [
+        'domain',
+        'url',
+        'is_crawling',
+        'is_crawled',
+        'output_type',
+        'output',
+        'metadata',
+        'status',
+    ];
+}
--- a/app/Providers/ViewServiceProvider.php
+++ b/app/Providers/ViewServiceProvider.php
@@ -4,6 +4,7 @@

 use App\View\Composers\CategoryComposer;
 use App\View\Composers\CountryLocaleComposer;
+use App\View\Composers\StatsComposer;
 use Illuminate\Support\Facades\View;
 use Illuminate\Support\ServiceProvider;

@@ -27,11 +28,15 @@ public function register()
    public function boot()
    {
        // Using class based composers...
-        // View::composer('layouts.front.navigation', CategoryComposer::class);
-        // View::composer('layouts.front.navigation', CountryLocaleComposer::class);
+        View::composer('front.home', CategoryComposer::class);
+        View::composer('front.home', StatsComposer::class);
+        View::composer('front.discover', StatsComposer::class);
+        View::composer('front.discover', CategoryComposer::class);

-        // View::composer('layouts.front.footer', CategoryComposer::class);
-        // View::composer('layouts.front.footer', CountryLocaleComposer::class);
+        // View::composer('front.layoutsnavigation', CountryLocaleComposer::class);
+
+        // View::composer('front.layouts.footer', CategoryComposer::class);
+        // View::composer('front.layouts.footer', CountryLocaleComposer::class);

        if (auth()->check()) {

--- a/app/View/Composers/CategoryComposer.php
+++ b/app/View/Composers/CategoryComposer.php
@@ -3,22 +3,20 @@
 namespace App\View\Composers;

 use App\Models\Category;
-use App\Models\CountryLocale;
 use Illuminate\View\View;

 class CategoryComposer
 {
    public function compose(View $view)
    {
-        $current_country_locale = request()->session()->get('view_country_locale');
+        $parent_categories = Category::whereNull('parent_id')->orderBy('name', 'ASC')->get();

-        if (is_null($current_country_locale)) {
-            $current_country_locale = CountryLocale::where('slug', config('platform.general.fallback_country_slug'))->first();
-        }
+        $top_parent_categories = clone $parent_categories->where('is_top', true);
+        $non_top_parent_categories = clone $parent_categories->where('is_top', false);

-        $categories = Category::where('country_locale_id', $current_country_locale->id)->get();
-
-        $view->with('categories', $categories);
+        $view->with('parent_categories', $parent_categories)
+            ->with('top_parent_categories', $top_parent_categories)
+            ->with('non_top_parent_categories', $non_top_parent_categories);

    }
 }
--- a/app/View/Composers/StatsComposer.php
+++ b/app/View/Composers/StatsComposer.php
@@ -0,0 +1,16 @@
+<?php
+
+namespace App\View\Composers;
+
+use Illuminate\View\View;
+
+class StatsComposer
+{
+    public function compose(View $view)
+    {
+        $tools_count = round_to_nearest_base(700);
+
+        $view->with('tools_count', $tools_count);
+
+    }
+}