Sync

app/Helpers/FirstParty/Aictio/Aictio.php

@@ -0,0 +1,54 @@
+<?php
+
+namespace App\Helpers\FirstParty\Aictio;
+
+use Exception;
+use Http;
+use Pgvector\Laravel\Vector;
+
+class Aictio
+{
+    public static function getVectorEmbedding($embedding_query)
+    {
+        $embedding_query = trim(preg_replace('/[^a-zA-Z0-9\s,\.]/', '', $embedding_query));
+
+        if (is_empty($embedding_query)) {
+            throw new Exception('Empty embedding query.');
+        }
+
+        $maxRetries = 3;
+        $currentAttempt = 0;
+
+        while ($currentAttempt < $maxRetries) {
+            try {
+                $response = Http::withHeaders([
+                    'CtioApp' => 'hireblitz260823',
+                ])->withOptions(['verify' => (app()->environment() == 'local') ? false : false])->timeout(800)
+                    ->get('https://aictio.applikuapp.com/api/embeddings',
+                        [
+                            'input' => $embedding_query,
+                        ]
+                    );
+
+                $embedding_response = json_decode($response->body(), true);
+
+                if (is_null($embedding_response)) {
+                    throw new Exception('Embedding response failed, null response');
+                }
+
+                if (isset($embedding_response['error'])) {
+                    throw new Exception($embedding_response['error']);
+                }
+
+                return new Vector(array_values($embedding_response)[0]);
+            } catch (Exception $e) {
+                $currentAttempt++;
+                if ($currentAttempt >= $maxRetries) {
+                    throw $e;
+                }
+                // Optional: Add sleep for a few seconds if you want to delay the next attempt
+                // sleep(1);
+            }
+        }
+    }
+}

app/Helpers/FirstParty/DFS/DFSBacklinks.php

@@ -0,0 +1,82 @@
+<?php
+
+namespace App\Helpers\FirstParty\DFS;
+
+use App\Models\CandidateDomain;
+use Exception;
+use Http;
+
+class DFSBacklinks
+{
+    public static function processAndSaveToCandidateDomains($directory)
+    {
+        // Get all the JSON files in the directory
+        $jsonFiles = glob($directory.'/*.json');
+
+        dd($jsonFiles);
+
+        // Loop through each file
+        foreach ($jsonFiles as $file) {
+            // Check if the file name matches the pattern (1.json, 2.json, etc.)
+            if (preg_match('/\/(\d+)\.json$/', $file, $matches)) {
+                // Decode the JSON file
+                $data = json_decode(file_get_contents($file), false);
+
+                foreach ($data->result[0]->items as $item) {
+                    $candidate_domain = CandidateDomain::where('from_domain', $item->domain_from)->first();
+
+                    if (is_null($candidate_domain)) {
+                        $candidate_domain = new CandidateDomain;
+                        $candidate_domain->to_url = $item->url_to;
+                        $candidate_domain->from_domain = $item->domain_from;
+                        $candidate_domain->from_tld = get_tld_from_url($item->url_from);
+
+                        if (is_array($item->domain_from_platform_type)) {
+                            $candidate_domain->platform_type = implode(',', $item->domain_from_platform_type);
+                        } else {
+                            $candidate_domain->platform_type = $item->domain_from_platform_type;
+                        }
+
+                        $candidate_domain->from_url = $item->url_from;
+                        $candidate_domain->from_title = $item->page_from_title;
+                        $candidate_domain->from_image_alt = $item->alt;
+                        $candidate_domain->from_image_url = $item->image_url;
+                        $candidate_domain->save();
+
+                    }
+                }
+            }
+        }
+    }
+
+    public static function backlinksPaginationLive($target, $search_after_token, $value = 1000)
+    {
+        $api_url = config('dataforseo.url');
+
+        $api_version = config('dataforseo.api_version');
+
+        $api_timeout = config('dataforseo.timeout');
+
+        $query = [
+            'target' => $target,
+            'search_after_token' => $search_after_token,
+            'value' => $value,
+            'mode' => 'as_is',
+        ];
+
+        try {
+            $response = Http::timeout($api_timeout)->withBasicAuth(config('dataforseo.login'), config('dataforseo.password'))->withBody(
+                json_encode([(object) $query])
+            )->post("{$api_url}{$api_version}backlinks/backlinks/live");
+
+            if ($response->successful()) {
+                return $response->body();
+            }
+        } catch (Exception $e) {
+            return null;
+        }
+
+        return null;
+
+    }
+}

app/Helpers/FirstParty/DFS/DFSCommon.php

@@ -0,0 +1,132 @@
+<?php
+
+namespace App\Helpers\FirstParty\DFS;
+
+use Exception;
+use Http;
+
+class DFSCommon
+{
+    public static function getTaskResult($task)
+    {
+        $task_object = new DFSResponse($task);
+
+        return (object) [
+            'type' => 'task',
+            'path' => implode('/', $task_object->getPath()),
+            'id' => $task_object->getId(),
+            'is_successful' => $task_object->isSuccessful(),
+            'result' => $task_object->getResult(),
+            'message' => $task_object->getStatusMessage(),
+        ];
+
+    }
+
+    public static function getTasks($response)
+    {
+
+        $dfs_object = new DFSResponse($response);
+
+        if ($dfs_object->isSuccessful()) {
+            return $dfs_object->getTasks();
+        }
+
+        return [];
+    }
+
+    public static function getTask($response, $key)
+    {
+        $tasks = self::getTasks($response);
+
+        if (count($tasks) > 0) {
+            if (isset($tasks[$key])) {
+                $task_object = $tasks[$key];
+
+                if (count($tasks) > 1) {
+                    $task_object->client_message = 'There are '.count($tasks).' tasks.';
+                }
+
+                return $task_object;
+            }
+        }
+
+        return null;
+    }
+
+    public static function apiCall($method, $action_url, $query)
+    {
+
+        $api_url = self::getApiUrl();
+
+        $api_version = config('dataforseo.api_version');
+
+        $api_timeout = config('dataforseo.timeout');
+
+        $full_api_url = "{$api_url}{$api_version}{$action_url}";
+
+        dump($full_api_url);
+        dump($query);
+
+        $http_object = Http::timeout($api_timeout)->withBasicAuth(config('dataforseo.login'), config('dataforseo.password'));
+
+        if (strtoupper($method) == 'GET') {
+            try {
+                $response = $http_object->withUrlParameters(
+                    json_encode([(object) $query])
+                )->get('');
+
+            } catch (Exception $e) {
+                return self::defaultFailedResponse($e);
+            }
+
+        } elseif (strtoupper($method) == 'POST') {
+            try {
+                $response = $http_object->withBody(
+                    json_encode([(object) $query])
+                )->post("{$api_url}{$api_version}{$action_url}");
+            } catch (Exception $e) {
+                return self::defaultFailedResponse($e);
+            }
+        } else {
+            throw new Exception('Invalid action method parameter.');
+        }
+
+        if ($response->successful()) {
+            return json_decode($response->body(), false);
+        }
+
+        return self::defaultFailedResponse();
+
+    }
+
+    public static function getApiUrl()
+    {
+        $api_url = config('dataforseo.url');
+
+        if (config('dataforseo.sandbox_mode')) {
+            $api_url = config('dataforseo.sandbox_url');
+        }
+
+        return $api_url;
+    }
+
+    private static function defaultFailedResponse(Exception $e = null)
+    {
+        $message = 'No such url.';
+
+        if (! is_null($e)) {
+            $message = $e->getMessage();
+        }
+
+        return (object) [
+            'version' => '-1',
+            'status_code' => -1,
+            'status_message' => $message,
+            'time' => '0 sec.',
+            'cost' => 0,
+            'tasks_count' => 0,
+            'tasks_error' => 0,
+            'tasks' => [],
+        ];
+    }
+}

app/Helpers/FirstParty/DFS/DFSOnPage.php

@@ -0,0 +1,41 @@
+<?php
+
+namespace App\Helpers\FirstParty\DFS;
+
+class DFSOnPage
+{
+    // https://docs.dataforseo.com/v3/backlinks/id_list/?bash
+    public static function listId($from_days, $to_days = 0, $limit = 1000, $offset = 0)
+    {
+
+        $query = [
+            'datetime_from' => now()->subDays($from_days)->format('Y-m-d H:i:s P'),
+            'datetime_to' => now()->subDays($to_days)->format('Y-m-d H:i:s P'),
+        ];
+
+        if ($limit != 1000) {
+            $query['limit'] = $limit;
+        }
+        if ($offset != 0) {
+            $query['offset'] = $offset;
+        }
+
+        $response = DFSCommon::apiCall('POST', 'on_page/id_list', $query);
+
+        return $response;
+    }
+
+    // https://docs.dataforseo.com/v3/on_page/task_post/
+    public static function taskPost($target, $max_crawl_pages = 1)
+    {
+
+        $query = [
+            'target' => $target,
+            'max_crawl_pages' => $max_crawl_pages,
+        ];
+
+        $response = DFSCommon::apiCall('POST', 'on_page/task_post', $query);
+
+        return $response;
+    }
+}

app/Helpers/FirstParty/DFS/DFSResponse.php

@@ -0,0 +1,63 @@
+<?php
+
+namespace App\Helpers\FirstParty\DFS;
+
+class DFSResponse
+{
+    protected $response;
+
+    public function __construct($response)
+    {
+        $this->response = $response;
+    }
+
+    public function isSuccessful()
+    {
+        return ($this->response->status_code == 20000) ? true : false;
+    }
+
+    public function getStatusCode()
+    {
+        return $this->response->status_code;
+    }
+
+    public function getStatusMessage()
+    {
+        return $this->response->status_message;
+    }
+
+    public function getTasks()
+    {
+        return $this->response->tasks;
+    }
+
+    public function getResult()
+    {
+        return $this->response->result;
+    }
+
+    public function getResultCount()
+    {
+        return $this->response->result_count;
+    }
+
+    public function getId()
+    {
+        return $this->response->id;
+    }
+
+    public function getCost()
+    {
+        return $this->response->cost;
+    }
+
+    public function getPath()
+    {
+        return $this->response->path;
+    }
+
+    public function getData()
+    {
+        return $this->response->data;
+    }
+}

app/Helpers/FirstParty/DFS/DFSSerp.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace App\Helpers\FirstParty\DFS;
+
+use Exception;
+use Http;
+
+class DFSSerp
+{
+    public static function liveAdvanced($se, $se_type, $keyword, $location_name, $language_code, $depth, $search_param = null)
+    {
+
+        $api_url = DFSCommon::getApiUrl();
+
+        $api_version = config('dataforseo.api_version');
+
+        $api_timeout = config('dataforseo.timeout');
+
+        $query = [
+            'keyword' => $keyword,
+            'location_name' => $location_name,
+            'language_code' => $language_code,
+        ];
+
+        if (! is_empty($search_param)) {
+            $query['search_param'] = $search_param;
+        }
+
+        try {
+            $response = Http::timeout($api_timeout)->withBasicAuth(config('dataforseo.login'), config('dataforseo.password'))->withBody(
+                json_encode([(object) $query])
+            )->post("{$api_url}{$api_version}serp/{$se}/{$se_type}/live/advanced");
+
+            if ($response->successful()) {
+                return $response->body();
+            }
+        } catch (Exception $e) {
+            return null;
+        }
+
+        return null;
+
+    }
+}

app/Helpers/FirstParty/OpenAI/OpenAI.php

@@ -8,85 +8,152 @@
 
 class OpenAI
 {
-    public static function writeProductArticle($excerpt, $photos, $categories)
+    public static function getSiteSummary($parent_categories, $user_prompt, $model_max_tokens = 1536, $timeout = 60)
     {
-        //$excerpt = substr($excerpt, 0, 1500);
+        $openai_config = 'openai-gpt-3-5-turbo-1106';
 
-        $category_str = implode('|', $categories);
+        $category_list = implode('|', $parent_categories->pluck('name')->toArray());
 
-        $system_prompt = '
-          You are tasked with writing a product introduction & review article using the provided excerpt. Write as if you are reviewing the product by a third party, avoiding pronouns. Emphasize the product\'s performance, features, and notable aspects. Do not mention marketplace-related information. Return the output as a minified JSON in this format:
-              {"category": "($category_str)","title": "(Start with product name, 60-70 characters)","excerpt": "(150-160 characters, do not start with a verb)","cliffhanger": "(70-80 characters, enticing sentence)","body": "(Markdown, 700-900 words)"}
+        $system_prompt = "Based on the website content containing an AI tool, return a valid JSON containing:\n{\n\"is_ai_tool\":(true|false),\n\"ai_tool_name\":\"(AI Tool Name)\",\n\"is_app_web_both\":\"(app|web|both)\",\n\"tagline\":\"(One line tagline in 6-8 words)\",\n\"summary\": \"(Summary of AI tool in 2-3 parapgraphs, 140-180 words using grade 8 US english)\",\n\"pricing_type\": \"(Free|Free Trial|Freemium|Subscription|Usage Based)\",\n\"main_category\": \"(AI Training|Art|Audio|Avatars|Business|Chatbots|Coaching|Content Generation|Data|Dating|Design|Dev|Education|Emailing|Finance|Gaming|GPTs|Health|Legal|Marketing|Music|Networking|Personal Assistance|Planning|Podcasting|Productivity|Project Management|Prompting|Reporting|Research|Sales|Security|SEO|Shopping|Simulation|Social|Speech|Support|Task|Testing|Training|Translation|UI\/UX|Video|Workflow|Writing)\",\n\"keywords\":[\"(Identify relevant keywords for this AI Tool, 1-2 words each, at least)\"],\n\"qna\":[{\"q\":\"Typical FAQ that readers want to know, up to 5 questions\",\"a\":\"Answer of the question\"}]\n}";
 
-              Mandatory Requirements:
-              - Language: US grade 8-9 English
-              - Use these sections when applicable:
-                -- ### Introduction
-                -- ### Overview
-                -- ### Specifications (use valid Markdown table. Two columns: Features and Specifications. Use `| --- | --- |` as a separator.)
-                -- ### Price (in given currency)
-                -- ### (Choose one: Should I Buy?, Conclusion, Final Thoughts, Our Verdict)
-              - Only use facts from the provided excerpt
-              - Don\'t use titles inside the markdown body
-              - Use \'###\' for all article sections
-              - Pick the closest provided category
-              - Do not use newline in the JSON structure
-      ';
+        return self::getChatCompletion($user_prompt, $system_prompt, $openai_config, $model_max_tokens, $timeout);
+    }
 
-        $user_prompt = "EXCERPT\n------------\n{$excerpt}\n";
+    private static function getChatCompletion($user_prompt, $system_prompt, $openai_config, $model_max_tokens, $timeout, $response_format = 'json_object')
+    {
+        $model = config("platform.ai.{$openai_config}.model");
+        $input_cost_per_thousand_tokens = config("platform.ai.{$openai_config}.input_cost_per_thousand_tokens");
+        $output_cost_per_thousand_tokens = config("platform.ai.{$openai_config}.output_cost_per_thousand_tokens");
 
-        if (count($photos) > 0) {
-            $system_prompt .= '- Include 3 markdown images with the article title as caption in every section, excluding Introduction.\n';
-            $user_prompt .= "\n\MARKDOWN IMAGES\n------------\n";
-            foreach ($photos as $photo) {
-                $user_prompt .= "{$photo}\n";
-            }
-        }
+        $output_token = 1280;
 
-        $output = (self::chatCompletion($system_prompt, $user_prompt, 'gpt-3.5-turbo', 1500));
+        try {
 
-        // dump($user_prompt);
-        // dd($output);
+            $obj = self::chatCompletionApi($system_prompt, $user_prompt, $model, $output_token, $response_format, $timeout);
 
-        if (! is_null($output)) {
-            try {
-                return json_decode($output, false, 512, JSON_THROW_ON_ERROR);
-            } catch (Exception $e) {
-                Log::error($output);
-                inspector()->reportException($e);
+            $input_cost = self::getCostUsage($obj->usage_detailed->prompt_tokens, $input_cost_per_thousand_tokens);
+            $output_cost = self::getCostUsage($obj->usage_detailed->completion_tokens, $output_cost_per_thousand_tokens);
 
-                return null;
+            $output = $obj->reply;
+
+            if ($response_format == 'json_object') {
+                $output = json_decode(self::jsonFixer($obj->reply), false, 512, JSON_THROW_ON_ERROR);
             }
 
+            return (object) [
+                'prompts' => (object) [
+                    'system_prompt' => $system_prompt,
+                    'user_prompt' => $user_prompt,
+                ],
+                'cost' => $input_cost + $output_cost,
+                'output' => $output,
+                'token_usage' => $obj->usage,
+                'token_usage_detailed' => $obj->usage_detailed,
+            ];
+        } catch (Exception $e) {
+            return self::getDefaultFailedResponse($system_prompt, $user_prompt, $e);
         }
 
-        return null;
+        return self::getDefaultFailedResponse($system_prompt, $user_prompt);
 
     }
 
-    public static function chatCompletion($system_prompt, $user_prompt, $model, $max_token = 2500)
+    private static function getDefaultFailedResponse($system_prompt, $user_prompt, $exception = null)
     {
+        $exception_message = null;
+
+        if (! is_null($exception)) {
+            $exception_message = $exception->getMessage();
+        }
+
+        return (object) [
+            'exception' => $exception_message,
+            'prompts' => (object) [
+                'system_prompt' => $system_prompt,
+                'user_prompt' => $user_prompt,
+            ],
+            'cost' => 0,
+            'output' => null,
+            'token_usage' => 0,
+            'token_usage_detailed' => (object) [
+                'completion_tokens' => 0,
+                'prompt_tokens' => 0,
+                'total_tokens' => 0,
+            ],
+        ];
+    }
+
+    private static function getCostUsage($token_usage, $cost_per_thousand_tokens)
+    {
+        $calc = $token_usage / 1000;
+
+        return $calc * $cost_per_thousand_tokens;
+    }
+
+    private static function jsonFixer($json_string)
+    {
+        $json_string = str_replace("\n", '', $json_string);
+
+        // try {
+        //   return (new JsonFixer)->fix($json_string);
+        // }
+        // catch(Exception $e) {
+
+        // }
+        return $json_string;
+
+    }
+
+    public static function chatCompletionApi($system_prompt, $user_prompt, $model, $max_token = 2500, $response_format = 'text', $timeout = 800)
+    {
+
+        if ($response_format == 'json_object') {
+            $arr = [
+                'model' => $model,
+                'max_tokens' => $max_token,
+                'response_format' => (object) [
+                    'type' => 'json_object',
+                ],
+                'messages' => [
+                    ['role' => 'system', 'content' => $system_prompt],
+                    ['role' => 'user', 'content' => $user_prompt],
+                ],
+            ];
+        } else {
+            $arr = [
+                'model' => $model,
+                'max_tokens' => $max_token,
+                'messages' => [
+                    ['role' => 'system', 'content' => $system_prompt],
+                    ['role' => 'user', 'content' => $user_prompt],
+                ],
+            ];
+        }
+
         try {
-            $response = Http::timeout(800)->withToken(config('platform.ai.openai.api_key'))
-                ->post('https://api.openai.com/v1/chat/completions', [
-                    'model' => $model,
-                    'max_tokens' => $max_token,
-                    'messages' => [
-                        ['role' => 'system', 'content' => $system_prompt],
-                        ['role' => 'user', 'content' => $user_prompt],
-                    ],
-                ]);
+            $response = Http::timeout($timeout)->withToken(config('platform.ai.openai.api_key'))
+                ->post('https://api.openai.com/v1/chat/completions', $arr);
 
-            //dd($response->body());
+            $json_response = json_decode($response->body());
 
-            $json_response = json_decode($response->body(), false, 512, JSON_THROW_ON_ERROR);
+            //dump($json_response);
 
-            $reply = $json_response?->choices[0]?->message?->content;
+            if (isset($json_response->error)) {
+                Log::error(serialize($json_response));
+                throw new Exception(serialize($json_response->error));
+            }
 
-            return $reply;
+            $obj = (object) [
+                'usage' => $json_response?->usage?->total_tokens,
+                'usage_detailed' => $json_response?->usage,
+                'reply' => $json_response?->choices[0]?->message?->content,
+
+            ];
+
+            return $obj;
         } catch (Exception $e) {
-            Log::error($response->body());
-            inspector()->reportException($e);
+            ////dd($response->body());
+            //inspector()->reportException($e);
             throw ($e);
         }