charlestehio/futurewalker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update (keywords): new logic

Browse Source
This commit is contained in:
Charles Teh
2023-11-23 02:15:41 +08:00
parent d2c2059531
commit b2762ecef2
15 changed files with 211 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/Http/Controllers/Front/FrontHomeController.php
View File

@@ -25,7 +25,7 @@ public function home(Request $request)
// $query->whereNotIn('id', $featured_posts->pluck('id')->toArray()); // $query->whereNotIn('id', $featured_posts->pluck('id')->toArray());
// })->where('status', 'publish')->where('published_at', '<=', now())->orderBy('published_at', 'desc')->limit(10)->get(); // })->where('status', 'publish')->where('published_at', '<=', now())->orderBy('published_at', 'desc')->limit(10)->get();
$rss_posts = RssPost::where('status', 'published')->orderBy('published_at', 'desc')->paginate(15); $rss_posts = RssPost::with('entities_keywords')->where('status', 'published')->orderBy('published_at', 'desc')->paginate(15);
return response(view('front.welcome', compact('rss_posts')), 200); return response(view('front.welcome', compact('rss_posts')), 200);
} }

6
app/Http/Controllers/Front/FrontListController.php
View File

@@ -40,7 +40,7 @@ public function searchResults(Request $request, $query)
SEOTools::jsonLd(); SEOTools::jsonLd();
SEOTools::setTitle($title, false); SEOTools::setTitle($title, false);
$rss_posts = RssPost::with('category') $rss_posts = RssPost::with('category', 'entities_keywords')
->where('status', 'published') ->where('status', 'published')
->whereRaw("to_tsvector('english', title || ' ' || bites || ' ' || keyword_list) @@ plainto_tsquery('english', ?)", [trim(preg_replace('/\s+/', ' ', $query))]) ->whereRaw("to_tsvector('english', title || ' ' || bites || ' ' || keyword_list) @@ plainto_tsquery('english', ?)", [trim(preg_replace('/\s+/', ' ', $query))])
->where('published_at', '<=', now()) ->where('published_at', '<=', now())
@@ -81,7 +81,7 @@ public function index(Request $request)
SEOTools::jsonLd(); SEOTools::jsonLd();
SEOTools::setTitle($title, false); SEOTools::setTitle($title, false);
$rss_posts = RssPost::with('category')->where('status', 'published') $rss_posts = RssPost::with('category', 'entities_keywords')->where('status', 'published')
->where('published_at', '<=', now()) ->where('published_at', '<=', now())
->orderBy('published_at', 'desc') ->orderBy('published_at', 'desc')
->cursorPaginate(60) ?? collect(); ->cursorPaginate(60) ?? collect();
@@ -128,7 +128,7 @@ public function category(Request $request, $category_slug)
SEOTools::jsonLd(); SEOTools::jsonLd();
SEOTools::setTitle($title, false); SEOTools::setTitle($title, false);
$rss_posts = RssPost::with('category')->where('status', 'published') $rss_posts = RssPost::with('category', 'entities_keywords')->where('status', 'published')
->where('category_id', $category->id) ->where('category_id', $category->id)
->where('published_at', '<=', now()) ->where('published_at', '<=', now())
->orderBy('published_at', 'desc') ->orderBy('published_at', 'desc')

7
app/Http/Controllers/Tests/TestController.php
View File

@@ -25,13 +25,11 @@ public function prm(Request $request)
} }
ParseRssPostMetadataTask::handle($id); ParseRssPostMetadataTask::handle($id);
return 'ok'; return 'ok';
} }
public function crawlTask(Request $request) public function crawlTask(Request $request)
{ {
$id = $request->input('id'); $id = $request->input('id');
@@ -41,12 +39,11 @@ public function crawlTask(Request $request)
} }
CrawlRssPostTask::handle($id); CrawlRssPostTask::handle($id);
return 'ok'; return 'ok';
} }
public function opml(Request $request) public function opml(Request $request)
{ {
$raw_posts = BrowseRSSLatestNewsTask::handleSingle('https://hnrss.org/newest?q=ai', 240); $raw_posts = BrowseRSSLatestNewsTask::handleSingle('https://hnrss.org/newest?q=ai', 240);

182
app/Jobs/SaveOldKeywordsJob.php
View File

@@ -2,7 +2,6 @@
namespace App\Jobs; namespace App\Jobs;
use App\Jobs\Tasks\CrawlRssPostTask;
use App\Models\RssPost; use App\Models\RssPost;
use App\Models\RssPostKeyword; use App\Models\RssPostKeyword;
use Illuminate\Bus\Queueable; use Illuminate\Bus\Queueable;
@@ -32,114 +31,105 @@ public function __construct($rss_post_id)
*/ */
public function handle(): void public function handle(): void
{ {
$rss_post = RssPost::find($this->rss_post_id); $rss_post = RssPost::find($this->rss_post_id);
if (is_null($rss_post)) if (is_null($rss_post)) {
{ return;
return ; }
}
if ($rss_post->keyword_saved == true) if ($rss_post->keyword_saved == true) {
{ return;
return ; }
}
$words_to_add_in_keyword_list = []; $words_to_add_in_keyword_list = [];
$words_to_save = []; $words_to_save = [];
$first_keyword_found = false; $first_keyword_found = false;
// Entities // Entities
if (isset($rss_post->entities)) { if (isset($rss_post->entities)) {
if (count($rss_post->entities) > 0) { if (count($rss_post->entities) > 0) {
foreach ($rss_post->entities as $key => $word) { foreach ($rss_post->entities as $key => $word) {
$word = trim($word); $word = trim($word);
$words_to_save[] = (object) [ $words_to_save[] = (object) [
'is_main' => ($key == 0) ? true : false, 'is_main' => ($key == 0) ? true : false,
'type' => 'entity', 'type' => 'entity',
'value' => $word, 'value' => $word,
'value_lowercased' => strtolower($word), 'value_lowercased' => strtolower($word),
]; ];
$words_to_add_in_keyword_list[] = $word; $words_to_add_in_keyword_list[] = $word;
} }
}
}
// Keywords
if (isset($rss_post->keywords)) {
if (count($rss_post->keywords) > 0) {
foreach ($rss_post->keywords as $word) {
$word = trim($word);
foreach($words_to_save as $saved_word)
{
if (strtolower($word) == $saved_word->value_lowercased)
{
continue 2;
}
}
$words_to_save[] = (object) [
'is_main' => !$first_keyword_found,
'type' => 'keyword',
'value' => $word,
'value_lowercased' => strtolower($word),
];
$words_to_add_in_keyword_list[] = $word;
if ($first_keyword_found == false) {
$first_keyword_found = true;
}
}
}
}
$rss_post->keyword_list = implode(',', $words_to_add_in_keyword_list);
$rss_post->status = 'published';
if($rss_post->save())
{
$has_saved_keyword = false;
$deleted_rpk = RssPostKeyword::where('rss_post_id', $rss_post->id)->delete();
foreach ($words_to_save as $word_to_save)
{
$new_rpk = new RssPostKeyword;
$new_rpk->rss_post_id = $rss_post->id;
$new_rpk->type = $word_to_save->type;
$new_rpk->is_main = $word_to_save->is_main;
$new_rpk->value = $word_to_save->value;
$new_rpk->value_lowercased = $word_to_save->value_lowercased;
$new_rpk->created_at = $rss_post->published_at;
$new_rpk->updated_at = $rss_post->published_at;
if($new_rpk->save())
{
if (!$has_saved_keyword)
{
$has_saved_keyword = true;
} }
}
} }
if ($has_saved_keyword) // Keywords
{ if (isset($rss_post->keywords)) {
$rss_post->keyword_saved = true; if (count($rss_post->keywords) > 0) {
$rss_post->save();
foreach ($rss_post->keywords as $word) {
$word = trim($word);
foreach ($words_to_save as $saved_word) {
if (strtolower($word) == $saved_word->value_lowercased) {
continue 2;
}
}
$words_to_save[] = (object) [
'is_main' => ! $first_keyword_found,
'type' => 'keyword',
'value' => $word,
'value_lowercased' => strtolower($word),
];
$words_to_add_in_keyword_list[] = $word;
if ($first_keyword_found == false) {
$first_keyword_found = true;
}
}
}
}
$rss_post->keyword_list = implode(',', $words_to_add_in_keyword_list);
$rss_post->status = 'published';
if ($rss_post->save()) {
$has_saved_keyword = false;
$deleted_rpk = RssPostKeyword::where('rss_post_id', $rss_post->id)->delete();
foreach ($words_to_save as $word_to_save) {
$new_rpk = new RssPostKeyword;
$new_rpk->rss_post_id = $rss_post->id;
$new_rpk->type = $word_to_save->type;
$new_rpk->is_main = $word_to_save->is_main;
$new_rpk->value = $word_to_save->value;
$new_rpk->value_lowercased = $word_to_save->value_lowercased;
$new_rpk->created_at = $rss_post->published_at;
$new_rpk->updated_at = $rss_post->published_at;
if ($new_rpk->save()) {
if (! $has_saved_keyword) {
$has_saved_keyword = true;
}
}
}
if ($has_saved_keyword) {
$rss_post->keyword_saved = true;
$rss_post->save();
}
} }
}
} }
} }

76
app/Jobs/Tasks/ParseRssPostMetadataTask.php
View File

@@ -63,10 +63,10 @@ public static function handle(int $rss_post_id)
$word = trim($word); $word = trim($word);
$words_to_save[] = (object) [ $words_to_save[] = (object) [
'is_main' => ($key == 0) ? true : false, 'is_main' => ($key == 0) ? true : false,
'type' => 'entity', 'type' => 'entity',
'value' => $word, 'value' => $word,
'value_lowercased' => strtolower($word), 'value_lowercased' => strtolower($word),
]; ];
$words_to_add_in_keyword_list[] = $word; $words_to_add_in_keyword_list[] = $word;
@@ -82,26 +82,23 @@ public static function handle(int $rss_post_id)
$word = trim($word); $word = trim($word);
foreach($words_to_save as $saved_word) foreach ($words_to_save as $saved_word) {
{ if (strtolower($word) == $saved_word->value_lowercased) {
if (strtolower($word) == $saved_word->value_lowercased) continue 2;
{ }
continue 2;
}
} }
$words_to_save[] = (object) [ $words_to_save[] = (object) [
'is_main' => !$first_keyword_found, 'is_main' => ! $first_keyword_found,
'type' => 'keyword', 'type' => 'keyword',
'value' => $word, 'value' => $word,
'value_lowercased' => strtolower($word), 'value_lowercased' => strtolower($word),
]; ];
$words_to_add_in_keyword_list[] = $word; $words_to_add_in_keyword_list[] = $word;
if ($first_keyword_found == false) { if ($first_keyword_found == false) {
$first_keyword_found = true; $first_keyword_found = true;
} }
} }
@@ -145,38 +142,33 @@ public static function handle(int $rss_post_id)
$rss_post->keyword_list = implode(',', $words_to_add_in_keyword_list); $rss_post->keyword_list = implode(',', $words_to_add_in_keyword_list);
$rss_post->status = 'published'; $rss_post->status = 'published';
if($rss_post->save())
{
$has_saved_keyword = false;
$deleted_rpk = RssPostKeyword::where('rss_post_id', $rss_post->id)->delete(); if ($rss_post->save()) {
$has_saved_keyword = false;
foreach ($words_to_save as $word_to_save) $deleted_rpk = RssPostKeyword::where('rss_post_id', $rss_post->id)->delete();
{
$new_rpk = new RssPostKeyword; foreach ($words_to_save as $word_to_save) {
$new_rpk->rss_post_id = $rss_post->id;
$new_rpk->type = $word_to_save->type;
$new_rpk->is_main = $word_to_save->is_main;
$new_rpk->value = $word_to_save->value;
$new_rpk->value_lowercased = $word_to_save->value_lowercased;
if($new_rpk->save()) $new_rpk = new RssPostKeyword;
{ $new_rpk->rss_post_id = $rss_post->id;
if (!$has_saved_keyword) $new_rpk->type = $word_to_save->type;
{ $new_rpk->is_main = $word_to_save->is_main;
$has_saved_keyword = true; $new_rpk->value = $word_to_save->value;
} $new_rpk->value_lowercased = $word_to_save->value_lowercased;
if ($new_rpk->save()) {
if (! $has_saved_keyword) {
$has_saved_keyword = true;
}
}
} }
}
if ($has_saved_keyword) if ($has_saved_keyword) {
{ $rss_post->keyword_saved = true;
$rss_post->keyword_saved = true; $rss_post->save();
$rss_post->save(); }
}
} }
} }

5
app/Models/RssPost.php
View File

@@ -69,6 +69,11 @@ class RssPost extends Model implements Feedable
'keyword_saved', 'keyword_saved',
]; ];
public function entities_keywords()
{
return $this->hasMany(RssPostKeyword::class);
}
public function category() public function category()
{ {
return $this->belongsTo(Category::class); return $this->belongsTo(Category::class);

37
app/Models/RssPostKeyword.php
View File

@@ -11,7 +11,7 @@
/** /**
* Class RssPostKeyword * Class RssPostKeyword
* *
* @property int $id * @property int $id
* @property int $rss_post_id * @property int $rss_post_id
* @property string $type * @property string $type
@@ -20,30 +20,27 @@
* @property string $value_lowercased * @property string $value_lowercased
* @property Carbon|null $created_at * @property Carbon|null $created_at
* @property Carbon|null $updated_at * @property Carbon|null $updated_at
*
* @property RssPost $rss_post * @property RssPost $rss_post
*
* @package App\Models
*/ */
class RssPostKeyword extends Model class RssPostKeyword extends Model
{ {
protected $table = 'rss_post_keywords'; protected $table = 'rss_post_keywords';
protected $casts = [ protected $casts = [
'rss_post_id' => 'int', 'rss_post_id' => 'int',
'is_main' => 'bool' 'is_main' => 'bool',
]; ];
protected $fillable = [ protected $fillable = [
'rss_post_id', 'rss_post_id',
'type', 'type',
'is_main', 'is_main',
'value', 'value',
'value_lowercased' 'value_lowercased',
]; ];
public function rss_post() public function rss_post()
{ {
return $this->belongsTo(RssPost::class); return $this->belongsTo(RssPost::class);
} }
} }

2
database/migrations/2023_11_22_151403_create_rss_post_keywords_table.php
View File

@@ -14,7 +14,7 @@ public function up(): void
Schema::create('rss_post_keywords', function (Blueprint $table) { Schema::create('rss_post_keywords', function (Blueprint $table) {
$table->id(); $table->id();
$table->foreignId('rss_post_id'); $table->foreignId('rss_post_id');
$table->enum('type',['keyword','entity']); $table->enum('type', ['keyword', 'entity']);
$table->boolean('is_main')->default(false); $table->boolean('is_main')->default(false);
$table->string('value'); $table->string('value');
$table->string('value_lowercased'); $table->string('value_lowercased');

2
database/migrations/2023_11_22_154104_add_keyword_saved_to_rss_posts_table.php
View File

@@ -12,7 +12,7 @@
public function up(): void public function up(): void
{ {
Schema::table('rss_posts', function (Blueprint $table) { Schema::table('rss_posts', function (Blueprint $table) {
$table->boolean('keyword_saved')->default(false); $table->boolean('keyword_saved')->default(false);
}); });
} }

28
database/migrations/2023_11_22_181428_add_created_at_index_to_rss_post_keywords_table.php Normal file
View File

@@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('rss_post_keywords', function (Blueprint $table) {
$table->index('created_at');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('rss_post_keywords', function (Blueprint $table) {
$table->dropIndex(['created_at']);
});
}
};

4
resources/css/app-front.css
View File

@@ -101,3 +101,7 @@ @keyframes shimmer {
background-position: -400% 0; background-position: -400% 0;
} }
} }
.word-wrap-break-word {
word-wrap: break-word;
}

42
resources/views/front/partials/news_bites.blade.php
View File

@@ -12,16 +12,18 @@
<div class="d-flex flex-wrap mb-1"> <div class="d-flex flex-wrap mb-1">
@if ($post->entities) @if ($post->entities)
@foreach ($post->entities as $key => $keyword) @foreach ($post->entities_keywords as $keyword)
@if ($key == 0) @if($keyword->type == 'entity')
<h4 @if ($keyword->is_main)
class="mb-1 pb-1 d-inline badge bg-secondary border-secondary text-white border me-1 small fw-bold"> <h4
{{ $keyword }} class="mb-1 pb-1 d-inline badge bg-secondary border-secondary text-white border me-1 small fw-bold">
</h4> {{ $keyword->value }}
@else </h4>
<h4 class="mb-1 pb-1 d-inline badge text-bg-light border me-1 small fw-normal"> @else
{{ $keyword }} <h4 class="mb-1 pb-1 d-inline badge text-bg-light border me-1 small fw-normal">
</h4> {{ $keyword->value }}
</h4>
@endif
@endif @endif
@endforeach @endforeach
@endif @endif
@@ -40,12 +42,12 @@ class="font-family-roboto-condensed mb-1 pb-1 d-inline badge bg-danger border-da
@endif @endif
<span class="d-inline text-secondary small"> <span class="d-inline text-secondary small">
@if($post->published_at->isBetween(now()->subDays(1), now())) @if ($post->published_at->isBetween(now()->subDays(1), now()))
{{ $post->published_at->diffForHumans() }} {{ $post->published_at->diffForHumans() }}
@else @else
{{ $post->published_at->format('d M') }} {{ $post->published_at->format('d M') }}
@endif @endif
</span> </span>
<i class="bi bi-dot"></i> <i class="bi bi-dot"></i>
<span class="d-inline text-secondary small">{{ min_read($post->bites) }}</span> <span class="d-inline text-secondary small">{{ min_read($post->bites) }}</span>
@@ -68,10 +70,10 @@ class="font-family-roboto-condensed mb-1 pb-1 d-inline badge bg-danger border-da
</div> </div>
@endif @endif
@if ($post->entities) @if ($post->entities)
<div class="mb-2"> <div class="mb-2 text-wrap">
More about: @foreach ($post->all_keywords as $keyword) More about: @foreach ($post->entities_keywords as $keyword)
<a class="ms-1" <a class="word-wrap-break-word ms-2"
href="{{ get_route_search_result($keyword) }}">{{ $keyword }}</a> href="{{ get_route_search_result($keyword->value_lowercased) }}">#{{ $keyword->value_lowercased }}</a>
@endforeach @endforeach
</div> </div>
@endif @endif

3
resources/views/front/partials/search.blade.php
View File

@@ -1,8 +1,7 @@
<aside> <aside>
<form class="d-flex mb-3" action="{{ route('front.search') }}" method="POST"> <form class="d-flex mb-3" action="{{ route('front.search') }}" method="POST">
@csrf @csrf
<input name="query" class="form-control me-2" type="search" placeholder="Type a keyword..." <input name="query" class="form-control me-2" type="search" placeholder="Type a keyword..." aria-label="Search">
aria-label="Search">
<button class="btn btn-outline-dark border-2" type="submit"> <button class="btn btn-outline-dark border-2" type="submit">
<i class="bi bi-search"></i> Search <i class="bi bi-search"></i> Search
</button> </button>

12
routes/tests.php
View File

@@ -42,10 +42,10 @@
Route::get('/set_keywords', function (Request $request) { Route::get('/set_keywords', function (Request $request) {
$last_record = RssPost::where('keyword_saved', false)->orderBy('id','DESC')->first(); $last_record = RssPost::where('keyword_saved', false)->orderBy('id', 'DESC')->first();
for ($i= 1; $i <= $last_record->id; $i++) { for ($i = 1; $i <= $last_record->id; $i++) {
SaveOldKeywordsJob::dispatch($i)->onQueue('default')->onConnection('default'); SaveOldKeywordsJob::dispatch($i)->onQueue('default')->onConnection('default');
} }
return 'ok'; return 'ok';
@@ -59,10 +59,10 @@
return "Missing 'id'."; return "Missing 'id'.";
} }
$last_record = RssPost::orderBy('id','DESC')->first(); $last_record = RssPost::orderBy('id', 'DESC')->first();
for ($i=$id; $i < $last_record->id; $i++) { for ($i = $id; $i < $last_record->id; $i++) {
CrawlRssPostJob::dispatch($i)->onQueue('default')->onConnection('default'); CrawlRssPostJob::dispatch($i)->onQueue('default')->onConnection('default');
} }
return 'ok'; return 'ok';

1
routes/web.php
View File

@@ -17,7 +17,6 @@
Route::get('/', [App\Http\Controllers\Front\FrontHomeController::class, 'home'])->name('front.home')->middleware('cacheResponse:1800'); Route::get('/', [App\Http\Controllers\Front\FrontHomeController::class, 'home'])->name('front.home')->middleware('cacheResponse:1800');
Route::get('/terms', [App\Http\Controllers\Front\FrontHomeController::class, 'terms'])->name('front.terms')->middleware('cacheResponse:2630000'); Route::get('/terms', [App\Http\Controllers\Front\FrontHomeController::class, 'terms'])->name('front.terms')->middleware('cacheResponse:2630000');
Route::get('/privacy', [App\Http\Controllers\Front\FrontHomeController::class, 'privacy'])->name('front.privacy')->middleware('cacheResponse:2630000'); Route::get('/privacy', [App\Http\Controllers\Front\FrontHomeController::class, 'privacy'])->name('front.privacy')->middleware('cacheResponse:2630000');