diff --git a/app/Http/Controllers/Sources/WebSiteDocumentSourceController.php b/app/Http/Controllers/Sources/ScrapeWebPageSourceController.php similarity index 81% rename from app/Http/Controllers/Sources/WebSiteDocumentSourceController.php rename to app/Http/Controllers/Sources/ScrapeWebPageSourceController.php index 4cd6307..2003fdb 100644 --- a/app/Http/Controllers/Sources/WebSiteDocumentSourceController.php +++ b/app/Http/Controllers/Sources/ScrapeWebPageSourceController.php @@ -6,12 +6,12 @@ use App\Models\Source; use App\Source\SourceTypeEnum; -class WebSiteDocumentSourceController extends BaseSourceController +class ScrapeWebPageSourceController extends BaseSourceController { public function create(Project $project) { - return inertia('Sources/WebSiteDocument/Create', [ - 'details' => config('larachain.sources.web_site_document'), + return inertia('Sources/ScrapeWebPage/Create', [ + 'details' => config('larachain.sources.scrape_web_page'), 'project' => $project, 'source' => [ 'meta_data' => [ @@ -23,8 +23,8 @@ public function create(Project $project) public function edit(Project $project, Source $source) { - return inertia('Sources/WebSiteDocument/Edit', [ - 'details' => config('larachain.sources.web_site_document'), + return inertia('Sources/ScrapeWebPage/Edit', [ + 'details' => config('larachain.sources.scrape_web_page'), 'project' => $project, 'source' => $source, ]); @@ -42,7 +42,7 @@ public function store(Project $project) 'project_id' => $project->id, 'name' => $validated['name'], 'description' => $validated['description'], - 'type' => SourceTypeEnum::WebSiteDocument, + 'type' => SourceTypeEnum::ScrapeWebPage, 'order' => 1, 'meta_data' => $validated['meta_data'], ]); @@ -68,7 +68,7 @@ public function update(Project $project, Source $source) 'project_id' => $validated['project_id'], 'name' => $validated['name'], 'description' => $validated['description'], - 'type' => SourceTypeEnum::WebSiteDocument, + 'type' => SourceTypeEnum::ScrapeWebPage, 'order' => 1, 'meta_data' => $validated['meta_data'], ]); diff --git a/app/Models/Document.php b/app/Models/Document.php index 05fcbe4..85db6a4 100644 --- a/app/Models/Document.php +++ b/app/Models/Document.php @@ -59,7 +59,6 @@ public function directoryForFile(): string|null { if (in_array($this->source->type, [ SourceTypeEnum::WebFile, - SourceTypeEnum::WebSiteDocument, ])) { return sprintf( storage_path('app/projects/%d/sources/%d'), @@ -74,7 +73,6 @@ public function pathToFile(): string|null { if (in_array($this->source->type, [ SourceTypeEnum::WebFile, - SourceTypeEnum::WebSiteDocument, ])) { return sprintf( storage_path('app/projects/%d/sources/%d/%s'), diff --git a/app/Source/SourceTypeEnum.php b/app/Source/SourceTypeEnum.php index 620f6e8..0e233dc 100644 --- a/app/Source/SourceTypeEnum.php +++ b/app/Source/SourceTypeEnum.php @@ -12,6 +12,6 @@ enum SourceTypeEnum: string use TypeTrait; //case TemplateType = 'template_type' - case WebSiteDocument = 'web_site_document'; + case ScrapeWebPage = 'scrape_web_page'; case WebFile = 'web_file'; } diff --git a/app/Source/Types/WebSiteDocument.php b/app/Source/Types/ScrapeWebPage.php similarity index 97% rename from app/Source/Types/WebSiteDocument.php rename to app/Source/Types/ScrapeWebPage.php index ad759a5..0163f4e 100644 --- a/app/Source/Types/WebSiteDocument.php +++ b/app/Source/Types/ScrapeWebPage.php @@ -8,7 +8,7 @@ use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Storage; -class WebSiteDocument extends BaseSourceType +class ScrapeWebPage extends BaseSourceType { public function handle(): Document { diff --git a/app/Source/Types/WebFile.php b/app/Source/Types/WebFile.php index f2f09d6..a948b9d 100644 --- a/app/Source/Types/WebFile.php +++ b/app/Source/Types/WebFile.php @@ -2,11 +2,11 @@ namespace App\Source\Types; -use App\Models\Document; +use App\Exceptions\SourceMissingRequiredMetaDataException; use App\Ingress\StatusEnum; +use App\Models\Document; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Storage; -use App\Exceptions\SourceMissingRequiredMetaDataException; class WebFile extends BaseSourceType { diff --git a/config/larachain.php b/config/larachain.php index a258a3c..219f7fe 100644 --- a/config/larachain.php +++ b/config/larachain.php @@ -32,10 +32,10 @@ 'active' => 0, 'background' => 'bg-red-500', ], - 'web_site_document' => [ + 'scrape_web_page' => [ 'name' => 'Web Site Document', 'description' => 'Get a web page and save as html', - 'class' => 'App\\Source\\Types\\WebSiteDocument', + 'class' => 'App\\Source\\Types\\ScrapeWebPage', 'requires' => [ ], 'active' => 1, diff --git a/database/factories/SourceFactory.php b/database/factories/SourceFactory.php index 98f80c9..4f9ece2 100644 --- a/database/factories/SourceFactory.php +++ b/database/factories/SourceFactory.php @@ -35,11 +35,11 @@ public function definition(): array ]; } - public function webDocumentMetaData() + public function scrapeWebPage() { return $this->state(function (array $attributes) { return [ - 'type' => SourceTypeEnum::WebFile, + 'type' => SourceTypeEnum::ScrapeWebPage, 'meta_data' => [ 'url' => 'https://en.wikipedia.org/wiki/Laravel', ], diff --git a/resources/js/Pages/Sources/WebSiteDocument/Create.vue b/resources/js/Pages/Sources/ScrapeWebPage/Create.vue similarity index 97% rename from resources/js/Pages/Sources/WebSiteDocument/Create.vue rename to resources/js/Pages/Sources/ScrapeWebPage/Create.vue index ff4f108..5032017 100644 --- a/resources/js/Pages/Sources/WebSiteDocument/Create.vue +++ b/resources/js/Pages/Sources/ScrapeWebPage/Create.vue @@ -59,7 +59,7 @@ const form = useForm({ }) const submit = () => { - form.post(route("sources.web_site_document.store", { + form.post(route("sources.scrape_web_page.store", { project: props.project.id }), { preserveScroll: true, diff --git a/resources/js/Pages/Sources/WebSiteDocument/Edit.vue b/resources/js/Pages/Sources/ScrapeWebPage/Edit.vue similarity index 97% rename from resources/js/Pages/Sources/WebSiteDocument/Edit.vue rename to resources/js/Pages/Sources/ScrapeWebPage/Edit.vue index fd20756..3488ecc 100644 --- a/resources/js/Pages/Sources/WebSiteDocument/Edit.vue +++ b/resources/js/Pages/Sources/ScrapeWebPage/Edit.vue @@ -60,7 +60,7 @@ const form = useForm({ }) const submit = () => { - form.put(route("sources.web_site_document.update", { + form.put(route("sources.scrape_web_page.update", { project: props.project.id, source: props.source.id }), { diff --git a/resources/js/Pages/Sources/WebSiteDocument/Partials/ResourceForm.vue b/resources/js/Pages/Sources/ScrapeWebPage/Partials/ResourceForm.vue similarity index 100% rename from resources/js/Pages/Sources/WebSiteDocument/Partials/ResourceForm.vue rename to resources/js/Pages/Sources/ScrapeWebPage/Partials/ResourceForm.vue diff --git a/routes/web.php b/routes/web.php index d3408f6..2a5b091 100644 --- a/routes/web.php +++ b/routes/web.php @@ -8,8 +8,8 @@ use App\Http\Controllers\ResponseTypes\TrimTextResponseTypeController; use App\Http\Controllers\ResponseTypes\VectorSearchResponseTypeController; use App\Http\Controllers\SortingController; +use App\Http\Controllers\Sources\ScrapeWebPageSourceController; use App\Http\Controllers\Sources\WebFileSourceController; -use App\Http\Controllers\Sources\WebSiteDocumentSourceController; use App\Http\Controllers\Transformers\EmbedTransformerController; use App\Http\Controllers\Transformers\PdfTransformerController; use Illuminate\Foundation\Application; @@ -309,18 +309,18 @@ function () { 'auth:sanctum', config('jetstream.auth_session'), 'verified', -])->controller(WebSiteDocumentSourceController::class)->group( +])->controller(ScrapeWebPageSourceController::class)->group( function () { - Route::get('/projects/{project}/sources/web_site_document/create', 'create') - ->name('sources.web_site_document.create'); - Route::get('/projects/{project}/sources/{source}/web_site_document/edit', 'edit') - ->name('sources.web_site_document.edit'); - Route::post('/projects/{project}/sources/web_site_document/store', 'store') - ->name('sources.web_site_document.store'); - Route::put('/projects/{project}/sources/{source}/web_site_document/update', 'update') - ->name('sources.web_site_document.update'); - Route::post('/projects/{project}/sources/{source}/web_site_document/run', 'run') - ->name('sources.web_site_document.run'); + Route::get('/projects/{project}/sources/scrape_web_page/create', 'create') + ->name('sources.scrape_web_page.create'); + Route::get('/projects/{project}/sources/{source}/scrape_web_page/edit', 'edit') + ->name('sources.scrape_web_page.edit'); + Route::post('/projects/{project}/sources/scrape_web_page/store', 'store') + ->name('sources.scrape_web_page.store'); + Route::put('/projects/{project}/sources/{source}/scrape_web_page/update', 'update') + ->name('sources.scrape_web_page.update'); + Route::post('/projects/{project}/sources/{source}/scrape_web_page/run', 'run') + ->name('sources.scrape_web_page.run'); } ); diff --git a/tests/Feature/Http/Controllers/WebSiteDocumentSourceControllerTest.php b/tests/Feature/Http/Controllers/ScrapeWebPageSourceControllerTest.php similarity index 91% rename from tests/Feature/Http/Controllers/WebSiteDocumentSourceControllerTest.php rename to tests/Feature/Http/Controllers/ScrapeWebPageSourceControllerTest.php index 59fda44..c4c679e 100644 --- a/tests/Feature/Http/Controllers/WebSiteDocumentSourceControllerTest.php +++ b/tests/Feature/Http/Controllers/ScrapeWebPageSourceControllerTest.php @@ -18,7 +18,7 @@ ]); $this->actingAs($user) - ->get(route('sources.web_site_document.create', [ + ->get(route('sources.scrape_web_page.create', [ 'project' => $project->id, ])) ->assertOk(); @@ -39,7 +39,7 @@ ]); $this->actingAs($user) - ->get(route('sources.web_site_document.edit', [ + ->get(route('sources.scrape_web_page.edit', [ 'project' => $project->id, 'source' => $source->id, ])) @@ -62,7 +62,7 @@ ]); $this->actingAs($user) - ->post(route('sources.web_site_document.run', [ + ->post(route('sources.scrape_web_page.run', [ 'project' => $project->id, 'source' => $source->id, ])) @@ -88,7 +88,7 @@ ]); $this->actingAs($user) - ->put(route('sources.web_site_document.update', [ + ->put(route('sources.scrape_web_page.update', [ 'project' => $project->id, 'source' => $source->id, ]), [ @@ -118,7 +118,7 @@ assertDatabaseCount('sources', 0); $this->actingAs($user) - ->post(route('sources.web_site_document.store', [ + ->post(route('sources.scrape_web_page.store', [ 'project' => $project->id, ]), [ 'name' => 'Foo', diff --git a/tests/Feature/PdfTransformerTest.php b/tests/Feature/PdfTransformerTest.php index 47e0d2b..88fda41 100644 --- a/tests/Feature/PdfTransformerTest.php +++ b/tests/Feature/PdfTransformerTest.php @@ -41,14 +41,4 @@ public function test_does_not_repeat() $transformer->handle($transformerModel); $this->assertDatabaseCount('document_chunks', 10); } - - public function test_does_not_run() - { - $this->webPageToText(); - $transformerModel = Transformer::factory()->pdfTranformer()->create(); - $this->assertDatabaseCount('document_chunks', 0); - $transformer = new PdfTransformer($this->document); - $transformer->handle($transformerModel); - $this->assertDatabaseCount('document_chunks', 0); - } } diff --git a/tests/Feature/WebSiteDocumentTest.php b/tests/Feature/ScrapeWebPageTest.php similarity index 79% rename from tests/Feature/WebSiteDocumentTest.php rename to tests/Feature/ScrapeWebPageTest.php index e5e02ea..6c92af6 100644 --- a/tests/Feature/WebSiteDocumentTest.php +++ b/tests/Feature/ScrapeWebPageTest.php @@ -4,22 +4,22 @@ use App\Models\Document; use App\Models\Source; -use App\Source\Types\WebSiteDocument; +use App\Source\Types\ScrapeWebPage; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Storage; use Mockery; use Tests\TestCase; -class WebSiteDocumentTest extends TestCase +class ScrapeWebPageTest extends TestCase { public function test_gets_file() { - $source = Source::factory()->webDocumentMetaData()->create(); + $source = Source::factory()->scrapeWebPage()->create(); Storage::fake('projects'); - $webFileSourceType = new WebSiteDocument($source); + $webFileSourceType = new ScrapeWebPage($source); $html = File::get(base_path('tests/fixtures/example.html')); @@ -39,10 +39,10 @@ public function test_gets_file() public function test_makes_document() { - $source = Source::factory()->webDocumentMetaData()->create(); + $source = Source::factory()->scrapeWebPage()->create(); Storage::fake('projects'); - $webFileSourceType = new WebSiteDocument($source); + $webFileSourceType = new ScrapeWebPage($source); Http::fake([ 'en.wikipedia.com/*' => Http::response('foo', 200), @@ -57,10 +57,10 @@ public function test_makes_document() public function test_makes_document_once_with_name() { - $source = Source::factory()->webDocumentMetaData()->create(); + $source = Source::factory()->scrapeWebPage()->create(); Storage::fake('projects'); - $webFileSourceType = new WebSiteDocument($source); + $webFileSourceType = new ScrapeWebPage($source); Http::fake([ 'wikipedia.com/*' => Http::response('foo', 200), diff --git a/tests/Feature/SharedSetupForPdfFile.php b/tests/Feature/SharedSetupForPdfFile.php index 405bb5d..2e2b6ce 100644 --- a/tests/Feature/SharedSetupForPdfFile.php +++ b/tests/Feature/SharedSetupForPdfFile.php @@ -12,35 +12,6 @@ trait SharedSetupForPdfFile public Source $source; - protected function webPageToText() - { - $source = Source::factory() - ->webDocumentMetaData() - ->create(); - - $document = Document::factory()->create([ - 'source_id' => $source->id, - 'guid' => 'example.html', - ]); - - $from = base_path('tests/fixtures/example.html'); - - if (! File::exists($document->directoryForFile())) { - File::makeDirectory( - $document->directoryForFile(), - 0755, - true - ); - File::copy( - $from, - $document->pathToFile() - ); - } - - $this->document = $document; - $this->source = $source; - } - protected function webFileDownloadSetup() { $source = Source::factory() diff --git a/tests/Feature/WebFileTest.php b/tests/Feature/WebFileTest.php index 438e8f9..6824d9f 100644 --- a/tests/Feature/WebFileTest.php +++ b/tests/Feature/WebFileTest.php @@ -2,12 +2,12 @@ namespace Tests\Feature; -use Mockery; -use Tests\TestCase; use App\Models\Source; use App\Source\Types\WebFile; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Storage; +use Mockery; +use Tests\TestCase; class WebFileTest extends TestCase {