charlestehio/aibuddytool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync

Browse Source
This commit is contained in:
Charles T
2023-11-28 04:39:36 +08:00
parent a9ac0e48b3
commit dc37274b6c
86 changed files with 2106 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/Helpers/FirstParty/OpenAI/OpenAI.php
View File

@@ -14,7 +14,7 @@ public static function getSiteSummary($parent_categories, $user_prompt, $model_m
$category_list = implode('|', $parent_categories->pluck('name')->toArray());
$system_prompt = "Based on the website content containing an AI tool, return a valid JSON containing:\n{\n\"is_ai_tool\":(true|false),\n\"ai_tool_name\":\"(AI Tool Name)\",\n\"is_app_web_both\":\"(app|web|both)\",\n\"tagline\":\"(One line tagline in 6-8 words)\",\n\"summary\": \"(Summary of AI tool in 2-3 parapgraphs, 140-180 words using grade 8 US english)\",\n\"pricing_type\": \"(Free|Free Trial|Freemium|Subscription|Usage Based)\",\n\"main_category\": \"(AI Training|Art|Audio|Avatars|Business|Chatbots|Coaching|Content Generation|Data|Dating|Design|Dev|Education|Emailing|Finance|Gaming|GPTs|Health|Legal|Marketing|Music|Networking|Personal Assistance|Planning|Podcasting|Productivity|Project Management|Prompting|Reporting|Research|Sales|Security|SEO|Shopping|Simulation|Social|Speech|Support|Task|Testing|Training|Translation|UI\/UX|Video|Workflow|Writing)\",\n\"keywords\":[\"(Identify relevant keywords for this AI Tool, 1-2 words each, at least)\"],\n\"qna\":[{\"q\":\"Typical FAQ that readers want to know, up to 5 questions\",\"a\":\"Answer of the question\"}]\n}";
$system_prompt = "Based on the website content containing an AI tool, return a valid JSON containing:\n{\n\"is_ai_tool\":(true|false),\n\"ai_tool_name\":\"(AI Tool Name)\",\n\"is_app_web_both\":\"(app|web|both)\",\n\"tagline\":\"(One line tagline in 6-8 words)\",\n\"summary\": \"(Summary of AI tool in 2-3 parapgraphs, 200-240 words using grade 8 US english, start with AI tool name)\",\n\"pricing_type\": \"(Free|Free Trial|Freemium|Subscription|Usage Based)\",\n\"main_category\": \"(AI Training|Art|Audio|Avatars|Business|Chatbots|Coaching|Content Generation|Data|Dating|Design|Dev|Education|Emailing|Finance|Gaming|GPTs|Health|Legal|Marketing|Music|Networking|Personal Assistance|Planning|Podcasting|Productivity|Project Management|Prompting|Reporting|Research|Sales|Security|SEO|Shopping|Simulation|Social|Speech|Support|Task|Testing|Training|Translation|UI\/UX|Video|Workflow|Writing)\",\n\"keywords\":[\"(Identify relevant keywords for this AI Tool, 1-2 words each, at least)\"],\n\"qna\":[{\"q\":\"Typical FAQ that readers want to know, up to 5 questions\",\"a\":\"Answer of the question\"}]\n}";
return self::getChatCompletion($user_prompt, $system_prompt, $openai_config, $model_max_tokens, $timeout);
}

1
app/Helpers/Global/helpers.php
View File

@@ -3,3 +3,4 @@
require 'string_helper.php';
require 'geo_helper.php';
require 'proxy_helper.php';
require 'route_helper.php';

13
app/Helpers/Global/route_helper.php Normal file
View File

@@ -0,0 +1,13 @@
<?php
if (! function_exists('get_route_search_result')) {
function get_route_search_result($query)
{
return route('front.search.results',
[
'query' => strtolower(urlencode($query)),
]);
}
}

65
app/Helpers/Global/string_helper.php
View File

@@ -1,11 +1,42 @@
<?php
use Carbon\Carbon;
use Illuminate\Support\Str;
if (! function_exists('epoch_now_timestamp')) {
function epoch_now_timestamp()
if (! function_exists('dmy')) {
function dmy(Carbon $carbon)
{
return (int) round(microtime(true) * 1000);
return $carbon->format('d M Y');
}
}
if (! function_exists('epoch_now_timestamp')) {
function epoch_now_timestamp($multiplier = 1000)
{
return (int) round(microtime(true) * $multiplier);
}
}
if (! function_exists('add_params_to_url')) {
function add_params_to_url(string $url, array $newParams): string
{
$url = parse_url($url);
parse_str($url['query'] ?? '', $existingParams);
$newQuery = array_merge($existingParams, $newParams);
$newUrl = $url['scheme'].'://'.$url['host'].($url['path'] ?? '');
if ($newQuery) {
$newUrl .= '?'.http_build_query($newQuery);
}
if (isset($url['fragment'])) {
$newUrl .= '#'.$url['fragment'];
}
return $newUrl;
}
}
@@ -175,6 +206,34 @@ function str_first_sentence($str)
}
if (! function_exists('str_extract_sentences')) {
function str_extract_sentences($str, $count = 1)
{
// Split the string at ., !, or ?, including the punctuation in the result
$sentences = preg_split('/([.!?])\s*/', $str, -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY);
$extractedSentences = [];
$currentSentence = '';
foreach ($sentences as $key => $sentence) {
if ($key % 2 == 0) {
// This is a sentence fragment
$currentSentence = $sentence;
} else {
// This is a punctuation mark
$currentSentence .= $sentence;
$extractedSentences[] = trim($currentSentence);
if (count($extractedSentences) >= $count) {
break;
}
}
}
return $extractedSentences;
}
}
if (! function_exists('unslug')) {
function unslug($slug, $delimiter = '-')
{

15
app/Http/Controllers/Front/FrontDiscoverController.php
View File

@@ -3,7 +3,9 @@
namespace App\Http\Controllers\Front;
use App\Http\Controllers\Controller;
use App\Models\AiTool;
use App\Models\Category;
use Artesaos\SEOTools\Facades\SEOMeta;
use Artesaos\SEOTools\Facades\SEOTools;
use Illuminate\Http\Request;
use JsonLd\Context;
@@ -36,7 +38,7 @@ public function discover(Request $request, $category_slug = null)
SEOTools::opengraph();
SEOTools::jsonLd();
SEOTools::setTitle($category->name.' AI Tools', false);
//SEOTools::setDescription($description);
//SEOTools::setDescription($description);
} else {
$breadcrumbs = collect([
['name' => 'Home', 'url' => route('front.home')],
@@ -65,6 +67,15 @@ public function discover(Request $request, $category_slug = null)
'itemListElement' => $listItems,
]);
return view('front.discover', compact('breadcrumbs', 'breadcrumb_context', 'category'));
$ai_tools = AiTool::when(! is_null($category), function ($query) use ($category) {
$query->where('category_id', $category->id);
})
->orderBy('updated_at', 'DESC')->paginate(6);
if ($ai_tools->count() <= 0) {
SEOMeta::setRobots('noindex');
}
return view('front.discover', compact('breadcrumbs', 'breadcrumb_context', 'category', 'ai_tools'));
}
}

5
app/Http/Controllers/Front/FrontHomeController.php
View File

@@ -3,6 +3,7 @@
namespace App\Http\Controllers\Front;
use App\Http\Controllers\Controller;
use App\Models\AiTool;
use Artesaos\SEOTools\Facades\SEOMeta;
use Artesaos\SEOTools\Facades\SEOTools;
use GrahamCampbell\Markdown\Facades\Markdown;
@@ -12,7 +13,9 @@ class FrontHomeController extends Controller
{
public function index(Request $request)
{
return view('front.home');
$latest_ai_tools = AiTool::orderBy('created_at', 'DESC')->take(12)->get();
return view('front.home', compact('latest_ai_tools'));
}
public function terms(Request $request)

100
app/Http/Controllers/Front/FrontSearchController.php
View File

@@ -2,9 +2,107 @@
namespace App\Http\Controllers\Front;
use App\Helpers\FirstParty\Aictio\Aictio;
use App\Http\Controllers\Controller;
use App\Models\SearchEmbedding;
use App\Models\SearchResult;
use Artesaos\SEOTools\Facades\SEOTools;
use Illuminate\Http\Request;
use JsonLd\Context;
class FrontSearchController extends Controller
{
//
public function search(Request $request)
{
if (is_empty($request->input('query'))) {
return redirect()->back();
}
return redirect()->to(get_route_search_result($request->input('query')));
}
public function searchResult(Request $request, $query)
{
if ($request->input('page') > 10) {
abort(404);
}
if (is_empty(trim($query))) {
return abort(404);
}
$query = trim($query);
$pagination_results = 10;
$search_result = SearchResult::where('query', $query)
->orderBy('id', 'desc')
->first();
$embedding = null;
if (! is_null($search_result) && ! is_null($search_result->embedding)) {
$embedding = $search_result->embedding;
$search_result->increment('counts');
} else {
$embedding = Aictio::getVectorEmbedding($query);
$search_result = new SearchResult;
$search_result->query = $query;
$search_result->counts = 1;
$search_result->embedding = $embedding;
if ($search_result->save()) {
}
}
$results = SearchEmbedding::query()
->with('ai_tool')
->selectRaw('DISTINCT ON (ai_tool_id) ai_tool_id, embedding <-> ? AS distance', [$embedding])
->orderBy('ai_tool_id')
->orderByRaw('embedding <-> ? ASC', [$embedding])
->whereRaw('embedding <-> ? < 0.65', [$embedding])
->where('ai_tool_id', '!=', null)
->paginate($pagination_results);
//dd($results->toArray());
if ($results->total() < $pagination_results) {
// TODO: Signal Serp Crawling Engine
}
$query = strtolower(urldecode($query));
session()->put('query', $query);
SEOTools::metatags();
SEOTools::twitter();
SEOTools::opengraph();
SEOTools::jsonLd();
SEOTools::setTitle(ucwords($query).' AI Tool');
$breadcrumbs = collect([
['name' => 'Home', 'url' => route('front.home')],
['name' => 'AI Tool Search', 'url' => null],
['name' => ucwords($query), 'url' => null],
]);
// breadcrumb json ld
$listItems = [];
foreach ($breadcrumbs as $index => $breadcrumb) {
$listItems[] = [
'name' => $breadcrumb['name'],
'url' => $breadcrumb['url'],
];
}
$breadcrumb_context = Context::create('breadcrumb_list', [
'itemListElement' => $listItems,
]);
return view('front.search_results', compact('results', 'query', 'breadcrumbs', 'breadcrumb_context'));
}
}

81
app/Http/Controllers/Front/FrontToolController.php Normal file
View File

@@ -0,0 +1,81 @@
<?php
namespace App\Http\Controllers\Front;
use App\Http\Controllers\Controller;
use App\JsonLd\FAQPage;
use App\Models\AiTool;
use Illuminate\Http\Request;
use JsonLd\Context;
use Artesaos\SEOTools\Facades\SEOTools;
class FrontToolController extends Controller
{
public function show(Request $request, $ai_tool_slug)
{
$ai_tool = AiTool::where('slug', $ai_tool_slug)->first();
if (is_null($ai_tool)) {
return abort(404);
}
$ai_tool->load('category');
$breadcrumbs = collect([
['name' => 'Home', 'url' => route('front.home')],
['name' => 'AI Tools', 'url' => route('front.discover.home')],
['name' => $ai_tool->category->name, 'url' => route('front.discover.category', ['category_slug' => $ai_tool->category->slug])],
['name' => $ai_tool->tool_name, 'url' => null],
]);
// breadcrumb json ld
$listItems = [];
foreach ($breadcrumbs as $index => $breadcrumb) {
$listItems[] = [
'name' => $breadcrumb['name'],
'url' => $breadcrumb['url'],
];
}
$breadcrumb_context = Context::create('breadcrumb_list', [
'itemListElement' => $listItems,
]);
$applicationCategory = '';
if ($ai_tool->is_app_web_both == 'both') {
$applicationCategory = 'App & Web Application';
} else {
$applicationCategory = ucwords($ai_tool->is_app_web_both).' Application';
}
$qnaMainEntity = [];
foreach ($ai_tool->qna as $qna) {
$qnaMainEntity[] = [
'@type' => 'Question',
'name' => $qna->q,
'acceptedAnswer' => [
'@type' => 'Answer',
'text' => $qna->a,
],
];
}
$faqData = [
'mainEntity' => $qnaMainEntity,
];
$faq_context = Context::create(FAQPage::class, $faqData);
SEOTools::metatags();
SEOTools::twitter()->addImage($ai_tool->screenshot_img);
SEOTools::opengraph()->addImage($ai_tool->screenshot_img);
SEOTools::jsonLd()->addImage($ai_tool->screenshot_img);
//dd($faq_context);
return view('front.aitool', compact('ai_tool', 'breadcrumb_context', 'breadcrumbs', 'faq_context'));
}
}

16
app/Jobs/Tasks/GetAIToolScreenshotTask.php
View File

@@ -4,8 +4,6 @@
use App\Helpers\FirstParty\OSSUploader\OSSUploader;
use App\Models\AiTool;
use App\Models\BusinessProfile;
use App\Models\SerpUrl;
use App\Models\UrlToCrawl;
use Exception;
use Image;
@@ -13,25 +11,21 @@
class GetAIToolScreenshotTask
{
public static function handle($url_to_crawl_id, $ai_tool_id)
{
$url_to_crawl = UrlToCrawl::find($url_to_crawl_id);
if (is_null($url_to_crawl))
{
return ;
if (is_null($url_to_crawl)) {
return;
}
$ai_tool = AiTool::find($ai_tool_id);
if (is_null($ai_tool))
{
return ;
if (is_null($ai_tool)) {
return;
}
$userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36";
$userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36';
$browsershot = Browsershot::url($url_to_crawl->url)
->timeout(30)

42
app/Jobs/Tasks/GetUrlBodyTask.php
View File

@@ -23,25 +23,29 @@ public static function handle(int $url_to_crawl_id)
return null;
}
$enable_proxy = false;
$url_to_crawl->is_crawling = true;
$url_to_crawl->save();
$url_to_crawl->refresh();
try {
$user_agent = config('platform.proxy.user_agent');
// try {
$user_agent = config('platform.proxy.user_agent');
$response = Http::withHeaders([
'User-Agent' => $user_agent,
$response = Http::withHeaders([
'User-Agent' => $user_agent,
])
->withOptions([
'proxy' => ($enable_proxy) ? get_smartproxy_rotating_server() : null,
'timeout' => 10,
'verify' => false,
])
->withOptions([
'proxy' => get_smartproxy_rotating_server(),
'timeout' => 10,
'verify' => false,
])
->get($url_to_crawl->url);
->get($url_to_crawl->url);
if ($response->successful()) {
$raw_html = $response->body();
if ($response->successful()) {
$raw_html = $response->body();
if ($enable_proxy)
{
$cost = calculate_smartproxy_cost(round(strlen($raw_html) / 1024, 2), 'rotating_global');
$service_cost_usage = new ServiceCostUsage;
@@ -51,17 +55,19 @@ public static function handle(int $url_to_crawl_id)
$service_cost_usage->reference_2 = strval($url_to_crawl_id);
$service_cost_usage->output = self::getMarkdownFromHtml($raw_html);
$service_cost_usage->save();
} else {
$raw_html = null;
$response->throw();
}
} catch (Exception $e) {
} else {
$raw_html = null;
//throw $e;
$response->throw();
}
// } catch (Exception $e) {
// $raw_html = null;
// //throw $e;
// }
if (! is_empty($raw_html)) {
$url_to_crawl->output_type = 'markdown';
$url_to_crawl->output = self::getMarkdownFromHtml($raw_html);

23
app/Jobs/Tasks/ParseUrlBodyTask.php
View File

@@ -65,23 +65,28 @@ public static function handle(int $url_to_crawl_id)
$ai_tool->url_to_crawl_id = $url_to_crawl->id;
}
$ai_tool->external_url = $url_to_crawl->url;
// Tool Name
if ((isset($url_meta_response->output->tool_name)) && (! is_empty($url_meta_response->output->tool_name))) {
$ai_tool->tool_name = $url_meta_response->output->tool_name;
if ((isset($url_meta_response->output->ai_tool_name)) && (! is_empty($url_meta_response->output->ai_tool_name))) {
$ai_tool->tool_name = $url_meta_response->output->ai_tool_name;
$ai_tool->slug = epoch_now_timestamp(1).'-'.str_slug($url_meta_response->output->ai_tool_name);
} else {
throw new Exception('OpenAI::getSiteSummary failed, no tool name');
}
// Is AI Tool
if ((isset($url_meta_response->output->is_ai_tool)) && (! is_null($url_meta_response->output->is_at_tool)) && is_bool($url_meta_response->output->is_ai_tool)) {
if ((isset($url_meta_response->output->is_ai_tool)) && (! is_null($url_meta_response->output->is_ai_tool)) && is_bool($url_meta_response->output->is_ai_tool)) {
$ai_tool->is_ai_tool = $url_meta_response->output->is_ai_tool;
} else {
$ai_tool->is_ai_tool = true;
}
// Is App/Web/Both
if ((isset($url_meta_response->output->is_app_web_both)) && (is_array($url_meta_response->output->is_app_web_both)) && in_array($url_meta_response->output->is_app_web_both, ['app', 'web', 'both'])) {
if ((isset($url_meta_response->output->is_app_web_both)) && (! is_empty($url_meta_response->output->is_app_web_both)) && in_array($url_meta_response->output->is_app_web_both, ['app', 'web', 'both'])) {
$ai_tool->is_app_web_both = $url_meta_response->output->is_app_web_both;
} else {
$ai_tool->is_app_web_both = 'web';
}
// Tagline
@@ -130,9 +135,8 @@ public static function handle(int $url_to_crawl_id)
$query = $ai_tool->tool_name;
if (!is_empty($ai_tool->tagline))
{
$query .= ": " . $ai_tool->tagline;
if (! is_empty($ai_tool->tagline)) {
$query .= ': '.$ai_tool->tagline;
}
StoreSearchEmbeddingJob::dispatch(
@@ -176,8 +180,7 @@ public static function handle(int $url_to_crawl_id)
// Q&A
if ((isset($url_meta_response->output->qna)) && (is_array($url_meta_response->output->qna))) {
foreach ($url_meta_response->output->qna as $qna)
{
foreach ($url_meta_response->output->qna as $qna) {
$q = $qna->q;
$a = $qna->a;
@@ -187,7 +190,7 @@ public static function handle(int $url_to_crawl_id)
'qna',
$ai_tool->category_id,
$ai_tool->id,
($qna->q . " " . $qna->a)
($qna->q.' '.$qna->a)
);
}
}

17
app/JsonLd/FAQPage.php Normal file
View File

@@ -0,0 +1,17 @@
<?php
namespace App\JsonLd;
use JsonLd\ContextTypes\AbstractContext;
class FAQPage extends AbstractContext
{
protected $structure = [
'mainEntity' => [],
];
protected function setMainEntityAttribute($value)
{
return (array) $value;
}
}

77
app/JsonLd/SoftwareApplication.php Normal file
View File

@@ -0,0 +1,77 @@
<?php
namespace App\JsonLd;
use JsonLd\ContextTypes\AbstractContext;
use JsonLd\ContextTypes\ImageObject;
class SoftwareApplication extends AbstractContext
{
/**
* Property structure
*
* @var array
*/
protected $structure = [
'name' => null,
'description' => null,
'url' => null,
'applicationCategory' => null,
'screenshot' => ImageObject::class,
];
/**
* Set the name attribute.
*
* @param string $value
* @return string
*/
protected function setNameAttribute($value)
{
return (string) $value;
}
/**
* Set the description attribute.
*
* @param string $value
* @return string
*/
protected function setDescriptionAttribute($value)
{
return $this->truncate($value, 260);
}
/**
* Set the url attribute.
*
* @param string $value
* @return string
*/
protected function setUrlAttribute($value)
{
return (string) $value;
}
/**
* Set the application category attribute.
*
* @param string $value
* @return string
*/
protected function setApplicationCategoryAttribute($value)
{
return (string) $value;
}
/**
* Set the screenshot attribute.
*
* @param ImageObject|array $value
* @return ImageObject
*/
protected function setScreenshotAttribute($value)
{
return new ImageObject([$value]);
}
}

98
app/Models/AiTool.php
View File

@@ -7,12 +7,14 @@
namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Facades\Storage;
/**
* Class AiTool
*
*
* @property int $id
* @property int $category_id
* @property int $url_to_crawl_id
@@ -27,56 +29,72 @@
* @property string|null $qna
* @property Carbon|null $created_at
* @property Carbon|null $updated_at
*
* @property Category $category
* @property UrlToCrawl $url_to_crawl
* @property Collection|SearchEmbedding[] $search_embeddings
* @property Collection|AiToolKeyword[] $ai_tool_keywords
*
* @package App\Models
*/
class AiTool extends Model
{
protected $table = 'ai_tools';
protected $table = 'ai_tools';
protected $casts = [
'category_id' => 'int',
'url_to_crawl_id' => 'int',
'is_ai_tool' => 'bool',
'qna' => 'object'
];
protected $casts = [
'category_id' => 'int',
'url_to_crawl_id' => 'int',
'view_count' => 'int',
'is_ai_tool' => 'bool',
'qna' => 'object',
];
protected $fillable = [
'category_id',
'url_to_crawl_id',
'screenshot_img',
'is_ai_tool',
'tool_name',
'is_app_web_both',
'tagline',
'summary',
'pricing_type',
'keyword_string',
'qna'
];
protected $fillable = [
'category_id',
'url_to_crawl_id',
'screenshot_img',
'is_ai_tool',
'tool_name',
'slug',
'is_app_web_both',
'tagline',
'summary',
'pricing_type',
'keyword_string',
'view_count',
'qna',
'external_url
',
];
public function category()
{
return $this->belongsTo(Category::class);
}
protected function screenshotImg(): Attribute
{
return Attribute::make(
get: function ($value = null) {
if (! is_empty($value)) {
public function url_to_crawl()
{
return $this->belongsTo(UrlToCrawl::class);
}
return Storage::disk(config('platform.uploads.ai_tools.screenshot.driver'))->url(config('platform.uploads.ai_tools.screenshot.path').$value);
}
public function search_embeddings()
{
return $this->hasMany(SearchEmbedding::class);
}
return null;
}
);
}
public function ai_tool_keywords()
{
return $this->hasMany(AiToolKeyword::class);
}
public function category()
{
return $this->belongsTo(Category::class);
}
public function url_to_crawl()
{
return $this->belongsTo(UrlToCrawl::class);
}
public function search_embeddings()
{
return $this->hasMany(SearchEmbedding::class);
}
public function keywords()
{
return $this->hasMany(AiToolKeyword::class);
}
}

41
app/Models/SearchResult.php Normal file
View File

@@ -0,0 +1,41 @@
<?php
/**
* Created by Reliese Model.
*/
namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Model;
use Pgvector\Laravel\Vector;
/**
* Class SearchResult
*
* @property int $id
* @property string $query
* @property USER-DEFINED|null $embedding
* @property Carbon|null $indexed_at
* @property int $counts
* @property string $country_iso
* @property Carbon|null $created_at
* @property Carbon|null $updated_at
*/
class SearchResult extends Model
{
protected $table = 'search_results';
protected $casts = [
'embedding' => Vector::class,
'indexed_at' => 'datetime',
'counts' => 'int',
];
protected $fillable = [
'query',
'embedding',
'indexed_at',
'counts',
];
}

2
app/Models/ServiceCostUsage.php
View File

@@ -29,7 +29,7 @@ class ServiceCostUsage extends Model
protected $casts = [
'cost' => 'float',
'output' => 'binary',
'output' => 'object',
];
protected $fillable = [