charlestehio/futurewalker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update (copies)

Browse Source
This commit is contained in:
Charles Teh
2023-12-03 11:20:03 +08:00
parent 97dc13e785
commit 38c53b4045
13 changed files with 103 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/Helpers/FirstParty/SitemapCrawler/CustomCrawlProfile.php
View File

@@ -2,8 +2,8 @@
namespace App\Helpers\FirstParty\SitemapCrawler; namespace App\Helpers\FirstParty\SitemapCrawler;
use Spatie\Crawler\CrawlProfiles\CrawlProfile;
use Psr\Http\Message\UriInterface; use Psr\Http\Message\UriInterface;
use Spatie\Crawler\CrawlProfiles\CrawlProfile;
class CustomCrawlProfile extends CrawlProfile class CustomCrawlProfile extends CrawlProfile
{ {
@@ -21,8 +21,6 @@ public function shouldCrawl(UriInterface $url): bool
return false; return false;
} }
return ($this->callback)($url); return ($this->callback)($url);
} }
} }

10
app/Helpers/Global/string_helper.php
View File

@@ -4,7 +4,8 @@
use Illuminate\Support\Str; use Illuminate\Support\Str;
if (! function_exists('count_words')) { if (! function_exists('count_words')) {
function count_words($string) { function count_words($string)
{
// Remove punctuation and line breaks // Remove punctuation and line breaks
$cleanString = preg_replace('/[\p{P}\s]/u', ' ', $string); $cleanString = preg_replace('/[\p{P}\s]/u', ' ', $string);
@@ -16,9 +17,9 @@ function count_words($string) {
} }
} }
if (! function_exists('get_country_names')) { if (! function_exists('get_country_names')) {
function get_country_names($lowercase = false) { function get_country_names($lowercase = false)
{
$countryCodes = config('platform.country_codes'); $countryCodes = config('platform.country_codes');
$countryNames = []; $countryNames = [];
@@ -34,7 +35,6 @@ function get_country_names($lowercase = false) {
} }
} }
if (! function_exists('is_valid_url')) { if (! function_exists('is_valid_url')) {
function is_valid_url($url) function is_valid_url($url)
{ {
@@ -195,7 +195,7 @@ function get_domain_from_url($url)
$parse = parse_url($url); $parse = parse_url($url);
// Check if 'host' key exists in the parsed URL array // Check if 'host' key exists in the parsed URL array
if (!isset($parse['host'])) { if (! isset($parse['host'])) {
return null; // or you can throw an exception or handle this case as per your requirement return null; // or you can throw an exception or handle this case as per your requirement
} }

8
app/Http/Controllers/Front/FrontHomeController.php
View File

@@ -26,11 +26,15 @@ public function home(Request $request)
// $query->whereNotIn('id', $featured_posts->pluck('id')->toArray()); // $query->whereNotIn('id', $featured_posts->pluck('id')->toArray());
// })->where('status', 'publish')->where('published_at', '<=', now())->orderBy('published_at', 'desc')->limit(10)->get(); // })->where('status', 'publish')->where('published_at', '<=', now())->orderBy('published_at', 'desc')->limit(10)->get();
$rss_count = RssPost::where('status', 'published')
->where('published_at', '>=', now()->subDay())
->count();
$top_rss_keywords = HybridTopRssPostKeywords::get(1, 16); $top_rss_keywords = HybridTopRssPostKeywords::get(1, 16);
$rss_posts = RssPost::with('entities_keywords')->whereNotNull('keywords')->where('status', 'published')->orderBy('published_at', 'desc')->paginate(15); $rss_posts = RssPost::with('entities_keywords')->whereNotNull('keywords')->where('status', 'published')->orderBy('published_at', 'desc')->paginate(30);
return response(view('front.welcome', compact('rss_posts', 'top_rss_keywords')), 200); return response(view('front.welcome', compact('rss_posts', 'top_rss_keywords', 'rss_count')), 200);
} }
public function terms(Request $request) public function terms(Request $request)

8
app/Jobs/Tasks/BrowseRSSLatestNewsTask.php
View File

@@ -51,16 +51,12 @@ public static function handleSingle($rss_url, $hours = 3)
$blacklist_rss_post_keywords = array_merge($blacklist_rss_post_keywords, get_country_names(true)); $blacklist_rss_post_keywords = array_merge($blacklist_rss_post_keywords, get_country_names(true));
foreach ($blacklist_rss_post_keywords as $blacklist_keyword) foreach ($blacklist_rss_post_keywords as $blacklist_keyword) {
{ if (str_contains(strtolower($title), $blacklist_keyword)) {
if (str_contains(strtolower($title), $blacklist_keyword))
{
continue 2; continue 2;
} }
} }
$raw_posts[] = (object) [ $raw_posts[] = (object) [
'source' => $f->get_title(), 'source' => $f->get_title(),
'source_url' => $rss_url, 'source_url' => $rss_url,

13
app/Jobs/Tasks/ParseRssPostMetadataTask.php
View File

@@ -28,7 +28,6 @@ public static function handle(int $rss_post_id)
$final_content = "TITLE: {$rss_post->title}"; $final_content = "TITLE: {$rss_post->title}";
if (in_array($rss_post->status, ['blocked', 'trashed'])) { if (in_array($rss_post->status, ['blocked', 'trashed'])) {
return; return;
} }
@@ -160,12 +159,9 @@ public static function handle(int $rss_post_id)
$rss_post->status = 'published'; $rss_post->status = 'published';
if (!$rss_post->status != 'blocked') if (! $rss_post->status != 'blocked') {
{ if (isset($post_meta_response->output->is_ai_or_tech_news)) {
if (isset($post_meta_response->output->is_ai_or_tech_news)) if ($post_meta_response->output->is_ai_or_tech_news != true) {
{
if ($post_meta_response->output->is_ai_or_tech_news != true)
{
$rss_post->status = 'blocked'; $rss_post->status = 'blocked';
} }
// else // else
@@ -182,8 +178,7 @@ public static function handle(int $rss_post_id)
} }
if (count($words_to_save) <= 0) if (count($words_to_save) <= 0) {
{
$rss_post->status = 'blocked'; $rss_post->status = 'blocked';
} }

2
app/Models/HybridTopRssPostKeywords.php
View File

@@ -31,7 +31,7 @@ public static function get($days = 1, $limit = 10)
$queryResults = DB::table('rss_post_keywords') $queryResults = DB::table('rss_post_keywords')
->select('value', 'value_lowercased', DB::raw('COUNT(value_lowercased) as value_count')) ->select('value', 'value_lowercased', DB::raw('COUNT(value_lowercased) as value_count'))
->where('created_at', '>=', now()->subDays($days)) ->where('created_at', '>=', now()->subDays($days))
->whereNotIn('value_lowercased', ['techcrunch', 'the verge', 'forbes', 'producthunt', 'vox media','engadget']) ->whereNotIn('value_lowercased', ['techcrunch', 'the verge', 'forbes', 'producthunt', 'vox media', 'engadget'])
->groupBy('value', 'value_lowercased') ->groupBy('value', 'value_lowercased')
->orderBy(DB::raw('COUNT(value_lowercased)'), 'desc') ->orderBy(DB::raw('COUNT(value_lowercased)'), 'desc')
->limit($limit) ->limit($limit)

2
config/platform/global.php
View File

@@ -18,7 +18,7 @@
], ],
'blacklist_rss_post_keywords' => [ 'blacklist_rss_post_keywords' => [
'deal' 'deal',
], ],
'rss' => [ 'rss' => [

1
config/sitemap.php
View File

@@ -2,7 +2,6 @@
use App\Helpers\FirstParty\SitemapCrawler\CustomCrawlProfile; use App\Helpers\FirstParty\SitemapCrawler\CustomCrawlProfile;
use GuzzleHttp\RequestOptions; use GuzzleHttp\RequestOptions;
use Spatie\Sitemap\Crawler\Profile;
return [ return [

27
resources/views/front/partials/news_bites.blade.php
View File

@@ -1,5 +1,5 @@
@foreach ($rss_posts as $key => $post) @foreach ($rss_posts as $key => $post)
<div class="card mb-1"> <div class="card mb-1">
<div class="card-body"> <div class="card-body">
<div class="d-flex justify-content-between"> <div class="d-flex justify-content-between">
<div class=""> <div class="">
@@ -9,30 +9,28 @@
@if ($post->entities_keywords->count() > 0) @if ($post->entities_keywords->count() > 0)
<div class="d-flex flex-wrap mb-1"> <div class="d-flex flex-wrap mb-1">
{{-- @if ($post->entities) --}} {{-- @if ($post->entities) --}}
@foreach ($post->entities_keywords as $keyword) @foreach ($post->entities_keywords as $keyword)
{{-- @if ($keyword->type == 'entity') --}} {{-- @if ($keyword->type == 'entity') --}}
@if ($loop->iteration <= 2) @if ($loop->iteration <= 3)
@if ($keyword->is_main) @if ($keyword->is_main)
<h4 <h4 class="mb-0 pb-1 d-inline badge bg-secondary border-secondary text-white border me-1 small fw-bold">
class="mb-0 pb-1 d-inline badge bg-secondary border-secondary text-white border me-1 small fw-bold">
{{ $keyword->value }} {{ $keyword->value }}
</h4> </h4>
@else @else
<h4 <h4 class="mb-0 pb-1 d-inline badge text-bg-light border me-1 small fw-normal">
class="mb-0 pb-1 d-inline badge text-bg-light border me-1 small fw-normal">
{{ $keyword->value }} {{ $keyword->value }}
</h4> </h4>
@endif @endif
@elseif ($loop->iteration == 3) @elseif ($loop->iteration > 3 && count($post->entities_keywords) > 3)
<h4 <h4 class="mb-0 pb-1 d-inline badge text-bg-light border me-1 small fw-normal">
class="mb-0 pb-1 d-inline badge text-bg-light border me-1 small fw-normal">
+{{ count($post->entities_keywords) - 3 }} more +{{ count($post->entities_keywords) - 3 }} more
</h4> </h4>
@break @break
@endif @endif
{{-- @endif --}} {{-- @endif --}}
@endforeach @endforeach
{{-- @endif --}} {{-- @endif --}}
@if ($post->impact_level == 'high') @if ($post->impact_level == 'high')
@@ -73,7 +71,8 @@ class="font-family-roboto-condensed mb-0 pb-1 d-inline badge bg-danger border-da
</div> </div>
<div class=" text-end d-flex justify-content-end"> <div class=" text-end d-flex justify-content-end">
<div class="align-self-center"> <div class="align-self-center">
<a class="btn btn-outline-secondary btn-sm rounded-pill px-3 text-decoration-none" target="_blank" rel="nofollow noopener noreferrer" href="{{ $post->post_url }}">👉 Read </a> <a class="btn btn-outline-secondary btn-sm rounded-pill px-3 text-decoration-none"
target="_blank" rel="nofollow noopener noreferrer" href="{{ $post->post_url }}">👉 Read article</a>
</div> </div>
</div> </div>
</div> </div>

9
resources/views/front/welcome.blade.php
View File

@@ -5,7 +5,10 @@
<div class="container-lg"> <div class="container-lg">
<div class="text-center row justify-content-center mb-4"> <div class="text-center row justify-content-center mb-4">
<div class="col-12 col-md-10 col-lg-8"> <div class="col-12 col-md-10 col-lg-8">
<div class="display-6 fw-bold font-family-roboto-condensed mb-2">Your Future Depends<wbr> on Today's
<h1 class="fw-bolder font-family-roboto-condensed mb-3">Latest AI & tech news in 1 hyper-focused platform</h1>
{{-- <div class="display-6 fw-bold font-family-roboto-condensed mb-2">Your Future Depends<wbr> on Today's
News</div> News</div>
<h1 class="h4 fw-normal mb-4">In the fast-evolving world of AI and tech, staying updated is not <h1 class="h4 fw-normal mb-4">In the fast-evolving world of AI and tech, staying updated is not
optional—it's critical for your future success. Stay updated with hourly news optional—it's critical for your future success. Stay updated with hourly news
@@ -14,12 +17,12 @@
<a href="#latest-news" class="btn btn-primary px-4 rounded-pill text-decoration-none">Start reading <a href="#latest-news" class="btn btn-primary px-4 rounded-pill text-decoration-none">Start reading
now</a> now</a>
</div> </div> --}}
</div> </div>
<div class="text-center row justify-content-center"> <div class="text-center row justify-content-center">
<div class="col-12 col-md-10 col-lg-6"> <div class="col-12 col-md-10 col-lg-6">
<h2 class="h4 fw-semibold mb-3">📡 Monitoring top tags for the past 24 hours</h2> <p data-nosnippet class="h4 mb-3">📡 Top keywords identified for the past 24 hours across {{ $rss_count }} articles identified by GPT</p>
@foreach ($top_rss_keywords as $rss_keyword) @foreach ($top_rss_keywords as $rss_keyword)
<a href="{{ get_route_search_result($rss_keyword->value_lowercased) }}" <a href="{{ get_route_search_result($rss_keyword->value_lowercased) }}"

8
routes/tests.php
View File

@@ -4,7 +4,6 @@
use App\Jobs\AISerpGenArticleJob; use App\Jobs\AISerpGenArticleJob;
use App\Jobs\BrowseAndWriteWithAIJob; use App\Jobs\BrowseAndWriteWithAIJob;
use App\Jobs\BrowseRSSPostJob; use App\Jobs\BrowseRSSPostJob;
use App\Jobs\BrowseSingleRSSJob;
use App\Jobs\CrawlRssPostJob; use App\Jobs\CrawlRssPostJob;
use App\Jobs\FillPostMetadataJob; use App\Jobs\FillPostMetadataJob;
use App\Jobs\GenerateArticleFeaturedImageJob; use App\Jobs\GenerateArticleFeaturedImageJob;
@@ -42,15 +41,12 @@
Route::get('/fix-broken-keywords', function (Request $request) { Route::get('/fix-broken-keywords', function (Request $request) {
$rss_posts = RssPost::whereNull('bites')->take(50)->orderBy('published_at','DESC')->get(); $rss_posts = RssPost::whereNull('bites')->take(50)->orderBy('published_at', 'DESC')->get();
foreach ($rss_posts as $rss_post) foreach ($rss_posts as $rss_post) {
{
ParseRssPostMetadataJob::dispatch($rss_post->id)->onQueue('default')->onConnection('default'); ParseRssPostMetadataJob::dispatch($rss_post->id)->onQueue('default')->onConnection('default');
} }
}); });
Route::get('/seed', function (Request $request) { Route::get('/seed', function (Request $request) {

13
routes/web.php
View File

@@ -23,11 +23,20 @@
Route::get('/disclaimer', [App\Http\Controllers\Front\FrontHomeController::class, 'disclaimer'])->name('front.disclaimer')->middleware('cacheResponse:2630000'); Route::get('/disclaimer', [App\Http\Controllers\Front\FrontHomeController::class, 'disclaimer'])->name('front.disclaimer')->middleware('cacheResponse:2630000');
Route::get('/bites', [App\Http\Controllers\Front\FrontListController::class, 'index'])->name('front.all')->middleware('cacheResponse:1800');
Route::get('/bites/', function ($query) {
return redirect()->route('front.all', 301);
});
Route::get('/digest', [App\Http\Controllers\Front\FrontListController::class, 'index'])->name('front.all')->middleware('cacheResponse:1800');
Route::post('/search', [App\Http\Controllers\Front\FrontListController::class, 'search'])->name('front.search'); Route::post('/search', [App\Http\Controllers\Front\FrontListController::class, 'search'])->name('front.search');
Route::get('/bites/{query}', [App\Http\Controllers\Front\FrontListController::class, 'searchResults'])->name('front.search.results')->middleware('cacheResponse:1800'); Route::get('/bites/{query}', function ($query) {
return redirect()->route('front.search.results', ['query' => $query], 301);
});
Route::get('/digest/{query}', [App\Http\Controllers\Front\FrontListController::class, 'searchResults'])->name('front.search.results')->middleware('cacheResponse:1800');
Route::get('/{category_slug}/{slug}', [App\Http\Controllers\Front\FrontPostController::class, 'index'])->name('front.post'); Route::get('/{category_slug}/{slug}', [App\Http\Controllers\Front\FrontPostController::class, 'index'])->name('front.post');