58 lines
1.4 KiB
PHP
58 lines
1.4 KiB
PHP
<?php
|
|
|
|
namespace Database\Seeders;
|
|
|
|
use App\Models\UrlToCrawl;
|
|
use Illuminate\Database\Seeder;
|
|
|
|
class SampleDomainSeeder extends Seeder
|
|
{
|
|
/**
|
|
* Run the database seeds.
|
|
*/
|
|
public function run(): void
|
|
{
|
|
|
|
// Path to the text file
|
|
$filePath = resource_path('data/domains/topaitools.txt');
|
|
|
|
// Check if the file exists
|
|
if (file_exists($filePath)) {
|
|
// Open the file for reading
|
|
$file = fopen($filePath, 'r');
|
|
|
|
if ($file) {
|
|
while (($url_link = fgets($file)) !== false) {
|
|
|
|
$domain = get_domain_from_url($url_link);
|
|
|
|
// dump($url_link);
|
|
|
|
$url_link = remove_query_parameters($url_link);
|
|
|
|
// dd($url_link);
|
|
|
|
$url_to_crawl = UrlToCrawl::where('url', $url_link)->first();
|
|
|
|
if (is_null($url_to_crawl)) {
|
|
$url_to_crawl = new UrlToCrawl;
|
|
$url_to_crawl->domain = $domain;
|
|
$url_to_crawl->url = $url_link;
|
|
$url_to_crawl->save();
|
|
}
|
|
|
|
}
|
|
|
|
if (! feof($file)) {
|
|
dump('Error: unexpected fgets() fail');
|
|
}
|
|
|
|
// Close the file
|
|
fclose($file);
|
|
}
|
|
} else {
|
|
dump('The file does not exist.');
|
|
}
|
|
}
|
|
}
|