diff --git a/app/Exceptions/Handler.php b/app/Exceptions/Handler.php index 690b4a1..b152e40 100644 --- a/app/Exceptions/Handler.php +++ b/app/Exceptions/Handler.php @@ -38,7 +38,7 @@ public function register() 'status' => -1, ], 404); } else { - return redirect()->route('home', [], 301); + return redirect()->route('front.home', [], 301); } }); } diff --git a/app/FirstParty/DFS/DFSBacklinks.php b/app/FirstParty/DFS/DFSBacklinks.php deleted file mode 100644 index 4276b97..0000000 --- a/app/FirstParty/DFS/DFSBacklinks.php +++ /dev/null @@ -1,40 +0,0 @@ - $target, - 'search_after_token' => $search_after_token, - 'value' => $value, - 'mode' => 'as_is', - ]; - - try { - $response = Http::timeout($api_timeout)->withBasicAuth(config('dataforseo.login'), config('dataforseo.password'))->withBody( - json_encode([(object) $query]) - )->post("{$api_url}{$api_version}backlinks/backlinks/live"); - - if ($response->successful()) { - return $response->body(); - } - } catch (Exception $e) { - return null; - } - - return null; - - } -} diff --git a/app/Helpers/FirstParty/Aictio/Aictio.php b/app/Helpers/FirstParty/Aictio/Aictio.php new file mode 100644 index 0000000..d9c1711 --- /dev/null +++ b/app/Helpers/FirstParty/Aictio/Aictio.php @@ -0,0 +1,54 @@ + 'hireblitz260823', + ])->withOptions(['verify' => (app()->environment() == 'local') ? false : false])->timeout(800) + ->get('https://aictio.applikuapp.com/api/embeddings', + [ + 'input' => $embedding_query, + ] + ); + + $embedding_response = json_decode($response->body(), true); + + if (is_null($embedding_response)) { + throw new Exception('Embedding response failed, null response'); + } + + if (isset($embedding_response['error'])) { + throw new Exception($embedding_response['error']); + } + + return new Vector(array_values($embedding_response)[0]); + } catch (Exception $e) { + $currentAttempt++; + if ($currentAttempt >= $maxRetries) { + throw $e; + } + // Optional: Add sleep for a few seconds if you want to delay the next attempt + // sleep(1); + } + } + } +} diff --git a/app/Helpers/FirstParty/DFS/DFSBacklinks.php b/app/Helpers/FirstParty/DFS/DFSBacklinks.php new file mode 100644 index 0000000..87fdaaa --- /dev/null +++ b/app/Helpers/FirstParty/DFS/DFSBacklinks.php @@ -0,0 +1,82 @@ +result[0]->items as $item) { + $candidate_domain = CandidateDomain::where('from_domain', $item->domain_from)->first(); + + if (is_null($candidate_domain)) { + $candidate_domain = new CandidateDomain; + $candidate_domain->to_url = $item->url_to; + $candidate_domain->from_domain = $item->domain_from; + $candidate_domain->from_tld = get_tld_from_url($item->url_from); + + if (is_array($item->domain_from_platform_type)) { + $candidate_domain->platform_type = implode(',', $item->domain_from_platform_type); + } else { + $candidate_domain->platform_type = $item->domain_from_platform_type; + } + + $candidate_domain->from_url = $item->url_from; + $candidate_domain->from_title = $item->page_from_title; + $candidate_domain->from_image_alt = $item->alt; + $candidate_domain->from_image_url = $item->image_url; + $candidate_domain->save(); + + } + } + } + } + } + + public static function backlinksPaginationLive($target, $search_after_token, $value = 1000) + { + $api_url = config('dataforseo.url'); + + $api_version = config('dataforseo.api_version'); + + $api_timeout = config('dataforseo.timeout'); + + $query = [ + 'target' => $target, + 'search_after_token' => $search_after_token, + 'value' => $value, + 'mode' => 'as_is', + ]; + + try { + $response = Http::timeout($api_timeout)->withBasicAuth(config('dataforseo.login'), config('dataforseo.password'))->withBody( + json_encode([(object) $query]) + )->post("{$api_url}{$api_version}backlinks/backlinks/live"); + + if ($response->successful()) { + return $response->body(); + } + } catch (Exception $e) { + return null; + } + + return null; + + } +} diff --git a/app/Helpers/FirstParty/DFS/DFSCommon.php b/app/Helpers/FirstParty/DFS/DFSCommon.php new file mode 100644 index 0000000..d00a269 --- /dev/null +++ b/app/Helpers/FirstParty/DFS/DFSCommon.php @@ -0,0 +1,132 @@ + 'task', + 'path' => implode('/', $task_object->getPath()), + 'id' => $task_object->getId(), + 'is_successful' => $task_object->isSuccessful(), + 'result' => $task_object->getResult(), + 'message' => $task_object->getStatusMessage(), + ]; + + } + + public static function getTasks($response) + { + + $dfs_object = new DFSResponse($response); + + if ($dfs_object->isSuccessful()) { + return $dfs_object->getTasks(); + } + + return []; + } + + public static function getTask($response, $key) + { + $tasks = self::getTasks($response); + + if (count($tasks) > 0) { + if (isset($tasks[$key])) { + $task_object = $tasks[$key]; + + if (count($tasks) > 1) { + $task_object->client_message = 'There are '.count($tasks).' tasks.'; + } + + return $task_object; + } + } + + return null; + } + + public static function apiCall($method, $action_url, $query) + { + + $api_url = self::getApiUrl(); + + $api_version = config('dataforseo.api_version'); + + $api_timeout = config('dataforseo.timeout'); + + $full_api_url = "{$api_url}{$api_version}{$action_url}"; + + dump($full_api_url); + dump($query); + + $http_object = Http::timeout($api_timeout)->withBasicAuth(config('dataforseo.login'), config('dataforseo.password')); + + if (strtoupper($method) == 'GET') { + try { + $response = $http_object->withUrlParameters( + json_encode([(object) $query]) + )->get(''); + + } catch (Exception $e) { + return self::defaultFailedResponse($e); + } + + } elseif (strtoupper($method) == 'POST') { + try { + $response = $http_object->withBody( + json_encode([(object) $query]) + )->post("{$api_url}{$api_version}{$action_url}"); + } catch (Exception $e) { + return self::defaultFailedResponse($e); + } + } else { + throw new Exception('Invalid action method parameter.'); + } + + if ($response->successful()) { + return json_decode($response->body(), false); + } + + return self::defaultFailedResponse(); + + } + + public static function getApiUrl() + { + $api_url = config('dataforseo.url'); + + if (config('dataforseo.sandbox_mode')) { + $api_url = config('dataforseo.sandbox_url'); + } + + return $api_url; + } + + private static function defaultFailedResponse(Exception $e = null) + { + $message = 'No such url.'; + + if (! is_null($e)) { + $message = $e->getMessage(); + } + + return (object) [ + 'version' => '-1', + 'status_code' => -1, + 'status_message' => $message, + 'time' => '0 sec.', + 'cost' => 0, + 'tasks_count' => 0, + 'tasks_error' => 0, + 'tasks' => [], + ]; + } +} diff --git a/app/Helpers/FirstParty/DFS/DFSOnPage.php b/app/Helpers/FirstParty/DFS/DFSOnPage.php new file mode 100644 index 0000000..f9782a0 --- /dev/null +++ b/app/Helpers/FirstParty/DFS/DFSOnPage.php @@ -0,0 +1,41 @@ + now()->subDays($from_days)->format('Y-m-d H:i:s P'), + 'datetime_to' => now()->subDays($to_days)->format('Y-m-d H:i:s P'), + ]; + + if ($limit != 1000) { + $query['limit'] = $limit; + } + if ($offset != 0) { + $query['offset'] = $offset; + } + + $response = DFSCommon::apiCall('POST', 'on_page/id_list', $query); + + return $response; + } + + // https://docs.dataforseo.com/v3/on_page/task_post/ + public static function taskPost($target, $max_crawl_pages = 1) + { + + $query = [ + 'target' => $target, + 'max_crawl_pages' => $max_crawl_pages, + ]; + + $response = DFSCommon::apiCall('POST', 'on_page/task_post', $query); + + return $response; + } +} diff --git a/app/Helpers/FirstParty/DFS/DFSResponse.php b/app/Helpers/FirstParty/DFS/DFSResponse.php new file mode 100644 index 0000000..9537f5d --- /dev/null +++ b/app/Helpers/FirstParty/DFS/DFSResponse.php @@ -0,0 +1,63 @@ +response = $response; + } + + public function isSuccessful() + { + return ($this->response->status_code == 20000) ? true : false; + } + + public function getStatusCode() + { + return $this->response->status_code; + } + + public function getStatusMessage() + { + return $this->response->status_message; + } + + public function getTasks() + { + return $this->response->tasks; + } + + public function getResult() + { + return $this->response->result; + } + + public function getResultCount() + { + return $this->response->result_count; + } + + public function getId() + { + return $this->response->id; + } + + public function getCost() + { + return $this->response->cost; + } + + public function getPath() + { + return $this->response->path; + } + + public function getData() + { + return $this->response->data; + } +} diff --git a/app/FirstParty/DFS/DFSSerp.php b/app/Helpers/FirstParty/DFS/DFSSerp.php similarity index 95% rename from app/FirstParty/DFS/DFSSerp.php rename to app/Helpers/FirstParty/DFS/DFSSerp.php index e531486..e66cdea 100644 --- a/app/FirstParty/DFS/DFSSerp.php +++ b/app/Helpers/FirstParty/DFS/DFSSerp.php @@ -9,7 +9,8 @@ class DFSSerp { public static function liveAdvanced($se, $se_type, $keyword, $location_name, $language_code, $depth, $search_param = null) { - $api_url = config('dataforseo.url'); + + $api_url = DFSCommon::getApiUrl(); $api_version = config('dataforseo.api_version'); diff --git a/app/Helpers/FirstParty/OpenAI/OpenAI.php b/app/Helpers/FirstParty/OpenAI/OpenAI.php index bea9760..64e732a 100644 --- a/app/Helpers/FirstParty/OpenAI/OpenAI.php +++ b/app/Helpers/FirstParty/OpenAI/OpenAI.php @@ -8,85 +8,152 @@ class OpenAI { - public static function writeProductArticle($excerpt, $photos, $categories) + public static function getSiteSummary($parent_categories, $user_prompt, $model_max_tokens = 1536, $timeout = 60) { - //$excerpt = substr($excerpt, 0, 1500); + $openai_config = 'openai-gpt-3-5-turbo-1106'; - $category_str = implode('|', $categories); + $category_list = implode('|', $parent_categories->pluck('name')->toArray()); - $system_prompt = ' - You are tasked with writing a product introduction & review article using the provided excerpt. Write as if you are reviewing the product by a third party, avoiding pronouns. Emphasize the product\'s performance, features, and notable aspects. Do not mention marketplace-related information. Return the output as a minified JSON in this format: - {"category": "($category_str)","title": "(Start with product name, 60-70 characters)","excerpt": "(150-160 characters, do not start with a verb)","cliffhanger": "(70-80 characters, enticing sentence)","body": "(Markdown, 700-900 words)"} + $system_prompt = "Based on the website content containing an AI tool, return a valid JSON containing:\n{\n\"is_ai_tool\":(true|false),\n\"ai_tool_name\":\"(AI Tool Name)\",\n\"is_app_web_both\":\"(app|web|both)\",\n\"tagline\":\"(One line tagline in 6-8 words)\",\n\"summary\": \"(Summary of AI tool in 2-3 parapgraphs, 140-180 words using grade 8 US english)\",\n\"pricing_type\": \"(Free|Free Trial|Freemium|Subscription|Usage Based)\",\n\"main_category\": \"(AI Training|Art|Audio|Avatars|Business|Chatbots|Coaching|Content Generation|Data|Dating|Design|Dev|Education|Emailing|Finance|Gaming|GPTs|Health|Legal|Marketing|Music|Networking|Personal Assistance|Planning|Podcasting|Productivity|Project Management|Prompting|Reporting|Research|Sales|Security|SEO|Shopping|Simulation|Social|Speech|Support|Task|Testing|Training|Translation|UI\/UX|Video|Workflow|Writing)\",\n\"keywords\":[\"(Identify relevant keywords for this AI Tool, 1-2 words each, at least)\"],\n\"qna\":[{\"q\":\"Typical FAQ that readers want to know, up to 5 questions\",\"a\":\"Answer of the question\"}]\n}"; - Mandatory Requirements: - - Language: US grade 8-9 English - - Use these sections when applicable: - -- ### Introduction - -- ### Overview - -- ### Specifications (use valid Markdown table. Two columns: Features and Specifications. Use `| --- | --- |` as a separator.) - -- ### Price (in given currency) - -- ### (Choose one: Should I Buy?, Conclusion, Final Thoughts, Our Verdict) - - Only use facts from the provided excerpt - - Don\'t use titles inside the markdown body - - Use \'###\' for all article sections - - Pick the closest provided category - - Do not use newline in the JSON structure - '; + return self::getChatCompletion($user_prompt, $system_prompt, $openai_config, $model_max_tokens, $timeout); + } - $user_prompt = "EXCERPT\n------------\n{$excerpt}\n"; + private static function getChatCompletion($user_prompt, $system_prompt, $openai_config, $model_max_tokens, $timeout, $response_format = 'json_object') + { + $model = config("platform.ai.{$openai_config}.model"); + $input_cost_per_thousand_tokens = config("platform.ai.{$openai_config}.input_cost_per_thousand_tokens"); + $output_cost_per_thousand_tokens = config("platform.ai.{$openai_config}.output_cost_per_thousand_tokens"); - if (count($photos) > 0) { - $system_prompt .= '- Include 3 markdown images with the article title as caption in every section, excluding Introduction.\n'; - $user_prompt .= "\n\MARKDOWN IMAGES\n------------\n"; - foreach ($photos as $photo) { - $user_prompt .= "{$photo}\n"; - } - } + $output_token = 1280; - $output = (self::chatCompletion($system_prompt, $user_prompt, 'gpt-3.5-turbo', 1500)); + try { - // dump($user_prompt); - // dd($output); + $obj = self::chatCompletionApi($system_prompt, $user_prompt, $model, $output_token, $response_format, $timeout); - if (! is_null($output)) { - try { - return json_decode($output, false, 512, JSON_THROW_ON_ERROR); - } catch (Exception $e) { - Log::error($output); - inspector()->reportException($e); + $input_cost = self::getCostUsage($obj->usage_detailed->prompt_tokens, $input_cost_per_thousand_tokens); + $output_cost = self::getCostUsage($obj->usage_detailed->completion_tokens, $output_cost_per_thousand_tokens); - return null; + $output = $obj->reply; + + if ($response_format == 'json_object') { + $output = json_decode(self::jsonFixer($obj->reply), false, 512, JSON_THROW_ON_ERROR); } + return (object) [ + 'prompts' => (object) [ + 'system_prompt' => $system_prompt, + 'user_prompt' => $user_prompt, + ], + 'cost' => $input_cost + $output_cost, + 'output' => $output, + 'token_usage' => $obj->usage, + 'token_usage_detailed' => $obj->usage_detailed, + ]; + } catch (Exception $e) { + return self::getDefaultFailedResponse($system_prompt, $user_prompt, $e); } - return null; + return self::getDefaultFailedResponse($system_prompt, $user_prompt); } - public static function chatCompletion($system_prompt, $user_prompt, $model, $max_token = 2500) + private static function getDefaultFailedResponse($system_prompt, $user_prompt, $exception = null) { + $exception_message = null; + + if (! is_null($exception)) { + $exception_message = $exception->getMessage(); + } + + return (object) [ + 'exception' => $exception_message, + 'prompts' => (object) [ + 'system_prompt' => $system_prompt, + 'user_prompt' => $user_prompt, + ], + 'cost' => 0, + 'output' => null, + 'token_usage' => 0, + 'token_usage_detailed' => (object) [ + 'completion_tokens' => 0, + 'prompt_tokens' => 0, + 'total_tokens' => 0, + ], + ]; + } + + private static function getCostUsage($token_usage, $cost_per_thousand_tokens) + { + $calc = $token_usage / 1000; + + return $calc * $cost_per_thousand_tokens; + } + + private static function jsonFixer($json_string) + { + $json_string = str_replace("\n", '', $json_string); + + // try { + // return (new JsonFixer)->fix($json_string); + // } + // catch(Exception $e) { + + // } + return $json_string; + + } + + public static function chatCompletionApi($system_prompt, $user_prompt, $model, $max_token = 2500, $response_format = 'text', $timeout = 800) + { + + if ($response_format == 'json_object') { + $arr = [ + 'model' => $model, + 'max_tokens' => $max_token, + 'response_format' => (object) [ + 'type' => 'json_object', + ], + 'messages' => [ + ['role' => 'system', 'content' => $system_prompt], + ['role' => 'user', 'content' => $user_prompt], + ], + ]; + } else { + $arr = [ + 'model' => $model, + 'max_tokens' => $max_token, + 'messages' => [ + ['role' => 'system', 'content' => $system_prompt], + ['role' => 'user', 'content' => $user_prompt], + ], + ]; + } + try { - $response = Http::timeout(800)->withToken(config('platform.ai.openai.api_key')) - ->post('https://api.openai.com/v1/chat/completions', [ - 'model' => $model, - 'max_tokens' => $max_token, - 'messages' => [ - ['role' => 'system', 'content' => $system_prompt], - ['role' => 'user', 'content' => $user_prompt], - ], - ]); + $response = Http::timeout($timeout)->withToken(config('platform.ai.openai.api_key')) + ->post('https://api.openai.com/v1/chat/completions', $arr); - //dd($response->body()); + $json_response = json_decode($response->body()); - $json_response = json_decode($response->body(), false, 512, JSON_THROW_ON_ERROR); + //dump($json_response); - $reply = $json_response?->choices[0]?->message?->content; + if (isset($json_response->error)) { + Log::error(serialize($json_response)); + throw new Exception(serialize($json_response->error)); + } - return $reply; + $obj = (object) [ + 'usage' => $json_response?->usage?->total_tokens, + 'usage_detailed' => $json_response?->usage, + 'reply' => $json_response?->choices[0]?->message?->content, + + ]; + + return $obj; } catch (Exception $e) { - Log::error($response->body()); - inspector()->reportException($e); + ////dd($response->body()); + //inspector()->reportException($e); throw ($e); } diff --git a/app/Helpers/Global/string_helper.php b/app/Helpers/Global/string_helper.php index 03ece80..b205091 100644 --- a/app/Helpers/Global/string_helper.php +++ b/app/Helpers/Global/string_helper.php @@ -9,6 +9,155 @@ function epoch_now_timestamp() } } +if (! function_exists('markdown_to_plaintext')) { + function markdown_to_plaintext($markdown) + { + // Headers + $markdown = preg_replace('/^#.*$/m', '', $markdown); + // Links and images + $markdown = preg_replace('/\[(.*?)\]\(.*?\)/', '$1', $markdown); + // Bold and italic + $markdown = str_replace(['**', '__', '*', '_'], '', $markdown); + // Ordered and unordered lists + $markdown = preg_replace('/^[-\*].*$/m', '', $markdown); + // Horizontal line + $markdown = str_replace(['---', '***', '- - -', '* * *'], '', $markdown); + // Inline code and code blocks + $markdown = preg_replace('/`.*?`/', '', $markdown); + $markdown = preg_replace('/```[\s\S]*?```/', '', $markdown); + // Blockquotes + $markdown = preg_replace('/^>.*$/m', '', $markdown); + + // Remove multiple spaces, leading and trailing spaces + $plaintext = trim(preg_replace('/\s+/', ' ', $markdown)); + + return $plaintext; + } +} + +if (! function_exists('round_to_nearest_base')) { + function round_to_nearest_base($number, $postfix = '+', $base = null) + { + // If $base is not provided, determine the base dynamically + if ($base === null) { + $length = strlen((string) $number); + if ($length > 1) { + $base = pow(10, $length - 2); + } else { + $base = 1; + } + } + + $roundedNumber = floor($number / $base) * $base; + + return $roundedNumber.$postfix; + } +} + +if (! function_exists('remove_query_parameters')) { + function remove_query_parameters($url) + { + // Trim the URL to remove whitespace, newline characters, and trailing slashes + $trimmedUrl = rtrim(trim($url), '/'); + + // Parse the trimmed URL + $parsedUrl = parse_url($trimmedUrl); + + // Rebuild the URL without the query string + $rebuiltUrl = $parsedUrl['scheme'].'://'.$parsedUrl['host']; + if (isset($parsedUrl['path']) && $parsedUrl['path'] !== '/') { + $rebuiltUrl .= $parsedUrl['path']; + } + + return $rebuiltUrl; + } +} + +if (! function_exists('remove_referral_parameters')) { + function remove_referral_parameters($url) + { + $referralParameters = [ + 'aff', 'ref', 'via', + 'utm_source', 'utm_medium', 'utm_campaign', + 'utm_content', 'utm_term', 'partner', + 'click_id', 'affiliate', 'source', + 'referral', 'tracker', 'tracking_id', + 'promo', 'coupon', 'discount', + 'campaign', 'ad', 'ad_id', + // Add more parameters as needed + ]; + + // Parse the URL + $parsedUrl = parse_url($url); + if (! isset($parsedUrl['query'])) { + return $url; // No query string present, return original URL + } + + // Parse the query string + parse_str($parsedUrl['query'], $queryParams); + + // Remove referral parameters + foreach ($referralParameters as $param) { + unset($queryParams[$param]); + } + + // Rebuild query string + $queryString = http_build_query($queryParams); + + // Rebuild the URL + $rebuiltUrl = $parsedUrl['scheme'].'://'.$parsedUrl['host']; + if (isset($parsedUrl['path'])) { + $rebuiltUrl .= $parsedUrl['path']; + } + if (! empty($queryString)) { + $rebuiltUrl .= '?'.$queryString; + } + + return $rebuiltUrl; + } +} + +if (! function_exists('get_domain_from_url')) { + function get_domain_from_url($url) + { + $parse = parse_url($url); + + // Check if 'host' key exists in the parsed URL array + if (! isset($parse['host'])) { + return null; // or you can throw an exception or handle this case as per your requirement + } + + $host = $parse['host']; + + // Check if the domain starts with 'www.' and remove it + if (substr($host, 0, 4) === 'www.') { + $host = substr($host, 4); + } + + return $host; + } +} + +if (! function_exists('get_tld_from_url')) { + function get_tld_from_url($url) + { + // Parse the URL and return its components + $parsedUrl = parse_url($url); + + // Check if the 'host' part is set + if (isset($parsedUrl['host'])) { + // Split the host by dots + $hostParts = explode('.', $parsedUrl['host']); + + // Return the last part which should be the TLD + return end($hostParts); + } + + // Return false if the URL doesn't have a host component + return false; + } +} + if (! function_exists('str_first_sentence')) { function str_first_sentence($str) { diff --git a/app/Http/.DS_Store b/app/Http/.DS_Store deleted file mode 100644 index a57b9e3..0000000 Binary files a/app/Http/.DS_Store and /dev/null differ diff --git a/app/Http/Controllers/.DS_Store b/app/Http/Controllers/.DS_Store deleted file mode 100644 index 4c7aebe..0000000 Binary files a/app/Http/Controllers/.DS_Store and /dev/null differ diff --git a/app/Http/Controllers/Front/FrontDiscoverController.php b/app/Http/Controllers/Front/FrontDiscoverController.php new file mode 100644 index 0000000..1503212 --- /dev/null +++ b/app/Http/Controllers/Front/FrontDiscoverController.php @@ -0,0 +1,70 @@ +first(); + + if (is_null($category)) { + return abort(404); + } + } + + if (! is_null($category)) { + $breadcrumbs = collect([ + ['name' => 'Home', 'url' => route('front.home')], + ['name' => 'Discover AI Tools', 'url' => route('front.discover.home')], + ['name' => $category->name.' AI Tools', 'url' => null], + ]); + + SEOTools::metatags(); + SEOTools::twitter(); + SEOTools::opengraph(); + SEOTools::jsonLd(); + SEOTools::setTitle($category->name.' AI Tools', false); + //SEOTools::setDescription($description); + } else { + $breadcrumbs = collect([ + ['name' => 'Home', 'url' => route('front.home')], + ['name' => 'Discover AI Tools', 'url' => null], + ]); + + SEOTools::metatags(); + SEOTools::twitter(); + SEOTools::opengraph(); + SEOTools::jsonLd(); + SEOTools::setTitle("{$tools_count} over AI Tools for you", false); + //SEOTools::setDescription($description); + } + + // breadcrumb json ld + $listItems = []; + + foreach ($breadcrumbs as $index => $breadcrumb) { + $listItems[] = [ + 'name' => $breadcrumb['name'], + 'url' => $breadcrumb['url'], + ]; + } + + $breadcrumb_context = Context::create('breadcrumb_list', [ + 'itemListElement' => $listItems, + ]); + + return view('front.discover', compact('breadcrumbs', 'breadcrumb_context', 'category')); + } +} diff --git a/app/Http/Controllers/Front/FrontHomeController.php b/app/Http/Controllers/Front/FrontHomeController.php new file mode 100644 index 0000000..8d0e8a4 --- /dev/null +++ b/app/Http/Controllers/Front/FrontHomeController.php @@ -0,0 +1,83 @@ +getContent(); + + //$content = $this->injectTailwindClasses($content); + + return view('front.pages', compact('content', 'title', 'description')); + } + + public function privacy(Request $request) + { + $markdown = file_get_contents(resource_path('markdown/privacy.md')); + + $title = 'Privacy Policy'; + $description = 'Our Privacy Policy provides clarity about the data we collect and how we use it, ensuring your peace of mind.'; + + SEOTools::metatags(); + SEOTools::twitter(); + SEOTools::opengraph(); + SEOTools::jsonLd(); + SEOTools::setTitle($title); + SEOTools::setDescription($description); + SEOMeta::setRobots('noindex'); + + $content = Markdown::convert($markdown)->getContent(); + + //$content = $this->injectTailwindClasses($content); + + return view('front.pages', compact('content', 'title', 'description')); + } + + public function disclaimer(Request $request) + { + $markdown = file_get_contents(resource_path('markdown/disclaimer.md')); + + $title = 'Disclaimer'; + $description = 'AI Buddy Tool provides the content on this website purely for informational purposes and should not be interpreted as legal, financial, or medical guidance.'; + + SEOTools::metatags(); + SEOTools::twitter(); + SEOTools::opengraph(); + SEOTools::jsonLd(); + SEOTools::setTitle($title); + SEOTools::setDescription($description); + SEOMeta::setRobots('noindex'); + + $content = Markdown::convert($markdown)->getContent(); + + //$content = $this->injectTailwindClasses($content); + + return view('front.pages', compact('content', 'title', 'description')); + } +} diff --git a/app/Http/Controllers/Front/FrontSearchController.php b/app/Http/Controllers/Front/FrontSearchController.php new file mode 100644 index 0000000..2bdbb19 --- /dev/null +++ b/app/Http/Controllers/Front/FrontSearchController.php @@ -0,0 +1,10 @@ +onConnection('default')->onQueue('default'); + } + + public function backlinks(Request $request) + { + $dfs_results = DFSBacklinks::backlinksPaginationLive('topai.tools', null); + + dd($dfs_results); + } +} diff --git a/app/Http/Kernel.php b/app/Http/Kernel.php index 54c41cd..5804014 100644 --- a/app/Http/Kernel.php +++ b/app/Http/Kernel.php @@ -66,5 +66,8 @@ class Kernel extends HttpKernel 'signed' => \App\Http\Middleware\ValidateSignature::class, 'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class, 'verified' => \Illuminate\Auth\Middleware\EnsureEmailIsVerified::class, + + 'cacheResponse' => \Spatie\ResponseCache\Middlewares\CacheResponse::class, + 'doNotCacheResponse' => \Spatie\ResponseCache\Middlewares\DoNotCacheResponse::class, ]; } diff --git a/app/Jobs/GetAIToolScreenshotJob.php b/app/Jobs/GetAIToolScreenshotJob.php new file mode 100644 index 0000000..2a3e70d --- /dev/null +++ b/app/Jobs/GetAIToolScreenshotJob.php @@ -0,0 +1,37 @@ +url_to_crawl_id = $url_to_crawl_id; + + $this->ai_tool_id = $ai_tool_id; + } + + /** + * Execute the job. + */ + public function handle(): void + { + GetAIToolScreenshotTask::handle($this->url_to_crawl_id, $this->ai_tool_id); + } +} diff --git a/app/Jobs/GetUrlBodyJob.php b/app/Jobs/GetUrlBodyJob.php new file mode 100644 index 0000000..7b64fc4 --- /dev/null +++ b/app/Jobs/GetUrlBodyJob.php @@ -0,0 +1,35 @@ +url_to_crawl_id = $url_to_crawl_id; + } + + /** + * Execute the job. + */ + public function handle(): void + { + GetUrlBodyTask::handle($this->url_to_crawl_id); + } +} diff --git a/app/Jobs/ParseUrlBodyJob.php b/app/Jobs/ParseUrlBodyJob.php new file mode 100644 index 0000000..0d856c1 --- /dev/null +++ b/app/Jobs/ParseUrlBodyJob.php @@ -0,0 +1,35 @@ +url_to_crawl_id = $url_to_crawl_id; + } + + /** + * Execute the job. + */ + public function handle(): void + { + ParseUrlBodyTask::handle($this->url_to_crawl_id); + } +} diff --git a/app/Jobs/ShopeeSellerTopProductScraperJob.php b/app/Jobs/ShopeeSellerTopProductScraperJob.php deleted file mode 100644 index e8ee0ba..0000000 --- a/app/Jobs/ShopeeSellerTopProductScraperJob.php +++ /dev/null @@ -1,55 +0,0 @@ -seller = $seller; - - $this->country_iso = $country_iso; - - $this->category = $category; - } - - /** - * Execute the job. - */ - public function handle(): void - { - $shopee_task = ShopeeSellerTopProductScraperTask::handle($this->seller, $this->country_iso, $this->category); - - //dd($shopee_task->product_task); - - if (! is_null($shopee_task)) { - SaveShopeeSellerImagesTask::handle($shopee_task); - - GenerateShopeeAIArticleTask::handle($shopee_task->shopee_seller_scrape); - } - - } -} diff --git a/app/Jobs/StoreSearchEmbeddingJob.php b/app/Jobs/StoreSearchEmbeddingJob.php new file mode 100644 index 0000000..a434769 --- /dev/null +++ b/app/Jobs/StoreSearchEmbeddingJob.php @@ -0,0 +1,65 @@ +type = $type; + $this->category_id = $category_id; + $this->ai_tool_id = $ai_tool_id; + $this->query = $query; + } + + /** + * Execute the job. + */ + public function handle(): void + { + $embedding = Aictio::getVectorEmbedding(strtolower($this->query)); + + if (! is_null($embedding)) { + + $search_embedding = SearchEmbedding::where('type', $this->type) + ->where('category_id', $this->category_id) + ->where('ai_tool_id', $this->ai_tool_id) + ->where('query', $this->query) + ->first(); + + if (is_null($search_embedding)) { + $search_embedding = new SearchEmbedding; + $search_embedding->type = $this->type; + $search_embedding->category_id = $this->category_id; + $search_embedding->ai_tool_id = $this->ai_tool_id; + $search_embedding->query = $this->query; + $search_embedding->embedding = $embedding; + $search_embedding->save(); + } + } else { + throw new Exception('Failed vector embedding: '.$this->query); + } + } +} diff --git a/app/Jobs/Tasks/GenerateShopeeAIArticleTask.php b/app/Jobs/Tasks/GenerateShopeeAIArticleTask.php deleted file mode 100644 index bd57a19..0000000 --- a/app/Jobs/Tasks/GenerateShopeeAIArticleTask.php +++ /dev/null @@ -1,278 +0,0 @@ -filename); - - $post = null; - - $shopee_seller_scrape->load('category'); - - if (! is_empty($serialised)) { - $shopee_task = unserialize($serialised); - $shopee_task->shopee_seller_scrape = $shopee_seller_scrape; - } - - // dd($shopee_task); - - // dd($shopee_task->product_task->response); - - $raw_html = $shopee_task->product_task->response->raw_html; - - $excerpt = self::stripHtml($raw_html); - - $excerpt = substr($excerpt, 0, 1500); // limit to 1500 (+1500 output token, total 3k token) characters due to OpenAI model limitations unless use 16k model, $$$$ - - $excerpt .= self::getProductPricingExcerpt($shopee_task->product_task->response->jsonld); - - $photos = ShopeeSellerScrapedImage::where('shopee_seller_scrape_id', $shopee_seller_scrape->id)->where('featured', false)->orderByRaw('RAND()')->take(3)->get()->pluck('image')->toArray(); - - $ai_writeup = AiWriteup::where('source', 'shopee')->where('source_url', $shopee_task->product_task->response->url)->first(); - - if (is_null($ai_writeup)) { - - $categories = [ - 'Beauty', - 'Technology', - 'Home & Living', - 'Health', - 'Fitness', - ]; - - $ai_output = OpenAI::writeProductArticle($excerpt, $photos, $categories); - - //dd($ai_output); - - if (is_null($ai_output)) { - $e = new Exception('Failed to write: Missing ai_output'); - - Log::error(serialize($ai_writeup?->toArray())); - inspector()->reportException($e); - throw ($e); - } else { - - $picked_category = Category::where('name', $ai_output->category)->where('country_locale_id', $shopee_seller_scrape->category->country_locale_id)->first(); - - if (is_null($picked_category)) { - $picked_category = $shopee_seller_scrape->category; - } - - // save - $ai_writeup = new AiWriteup; - $ai_writeup->source = 'shopee'; - $ai_writeup->source_url = $shopee_task->product_task->response->url; - $ai_writeup->category_id = $picked_category->id; - $ai_writeup->title = $ai_output->title; - $ai_writeup->excerpt = $ai_output->excerpt; - $ai_writeup->featured_image = ''; - $ai_writeup->body = $ai_output->body; - $ai_writeup->cost = self::getTotalServiceCost($shopee_task); - $ai_writeup->editor_format = 'markdown'; - - if ($ai_writeup->save()) { - $featured_photo = ShopeeSellerScrapedImage::where('shopee_seller_scrape_id', $shopee_seller_scrape->id)->where('featured', true)->first(); - - // new post - $post_data = [ - 'publish_date' => now(), - 'title' => $ai_writeup->title, - 'slug' => str_slug($ai_writeup->title), - 'excerpt' => $ai_writeup->excerpt, - 'cliffhanger' => $ai_writeup->cliffhanger, - 'author_id' => 1, - 'featured' => false, - 'featured_image' => $featured_photo->image, - 'editor' => 'markdown', - 'body' => $ai_writeup->body, - 'post_format' => 'standard', - 'status' => 'publish', - ]; - - $post = Post::create($post_data); - - if (! is_null($post)) { - - $shopee_seller_scrape->write_counts = $shopee_seller_scrape->write_counts + 1; - $shopee_seller_scrape->last_ai_written_at = now(); - $shopee_seller_scrape->save(); - - $shopee_seller_category = ShopeeSellerCategory::where('seller', $shopee_seller_scrape->seller)->first(); - - if (is_null($shopee_seller_category)) { - $shopee_seller_category = new ShopeeSellerCategory; - $shopee_seller_category->seller = $shopee_seller_scrape->seller; - $shopee_seller_category->category_id = $shopee_seller_scrape->category_id; - } - - $shopee_seller_category->last_ai_written_at = $shopee_seller_scrape->last_ai_written_at; - $shopee_seller_category->write_counts = $shopee_seller_scrape->write_counts; - - $shopee_seller_category->save(); - - PostCategory::create([ - 'post_id' => $post->id, - 'category_id' => $picked_category->id, - ]); - - if (app()->environment() == 'production') { - if ($post->status == 'publish') { - - $post_url = route('home.country.post', ['country' => $post->post_category?->category?->country_locale_slug, 'post_slug' => $post->slug]); - - LaravelGoogleIndexing::create()->update($post_url); - IndexNow::submit($post_url); - } - } - - } - } - } - } else { - $e = new Exception('Failed to write: ai_writeup found'); - Log::error(serialize($ai_writeup?->toArray())); - inspector()->reportException($e); - throw ($e); - } - - return $post; - } - - private static function getProductPricingExcerpt(array $jsonLdData) - { - foreach ($jsonLdData as $data) { - // Ensure the type is "Product" before proceeding - if (isset($data->{'@type'}) && $data->{'@type'} === 'Product') { - - // Extract necessary data - $lowPrice = $data->offers->lowPrice ?? null; - $highPrice = $data->offers->highPrice ?? null; - $price = $data->offers->price ?? null; - $currency = $data->offers->priceCurrency ?? null; - $sellerName = $data->offers->seller->name ?? 'online store'; // default to "online store" if name is not set - - if (! is_empty($currency)) { - if ($currency == 'MYR') { - $currency = 'RM'; - } - } - - // Determine and format pricing sentence - if ($lowPrice && $highPrice) { - $lowPrice = number_format($lowPrice, 0); - $highPrice = number_format($highPrice, 0); - - return "Price Range from {$currency} {$lowPrice} to {$highPrice} in {$sellerName} online store"; - } elseif ($price) { - $price = number_format($price, 0); - - return "Priced at {$currency} {$price} in {$sellerName} online store"; - } else { - return "Price not stated, refer to {$sellerName} online store"; - } - - } - } - } - - private static function getTotalServiceCost($shopee_task) - { - - $cost = 0.00; - - $cost += 0.09; // chatgpt-3.5-turbo $0.03 for 1k, writing for 2k tokens - - // Shopee Seller Scraping - if (isset($shopee_task?->seller_shop_task?->response?->total_cost)) { - $cost += $shopee_task?->seller_shop_task?->response?->total_cost; - } - - // Shopee Product Scraping - if (isset($shopee_task?->product_task?->response?->total_cost)) { - $cost += $shopee_task?->product_task?->response?->total_cost; - } - - return $cost; - - } - - private static function stripHtml(string $raw_html) - { - - $html_content = ''; - - try { - - $r_configuration = new ReadabilityConfiguration(); - $r_configuration->setCharThreshold(20); - - $readability = new Readability($r_configuration); - - $readability->parse($raw_html); - - $temp_html_content = $readability->getContent(); - - // Remove tabs - $temp_html_content = str_replace("\t", '', $temp_html_content); - - // Replace newlines with spaces - $temp_html_content = str_replace(["\n", "\r\n"], ' ', $temp_html_content); - - // Replace multiple spaces with a single space - $temp_html_content = preg_replace('/\s+/', ' ', $temp_html_content); - - // Output the cleaned text - $temp_html_content = trim($temp_html_content); // Using trim to remove any leading or trailing spaces - - $temp_html_content = strip_tags($temp_html_content); - - $crawler = new Crawler($raw_html); - - // Extract meta title - $title = $crawler->filter('title')->text(); // This assumes