Rename so my mind stops breaking on the Source

app/Http/Controllers/Sources/WebSiteDocumentSourceController.php → app/Http/Controllers/Sources/ScrapeWebPageSourceController.php

@@ -6,12 +6,12 @@
 use App\Models\Source;
 use App\Source\SourceTypeEnum;
 
-class WebSiteDocumentSourceController extends BaseSourceController
+class ScrapeWebPageSourceController extends BaseSourceController
 {
     public function create(Project $project)
     {
-        return inertia('Sources/WebSiteDocument/Create', [
-            'details' => config('larachain.sources.web_site_document'),
+        return inertia('Sources/ScrapeWebPage/Create', [
+            'details' => config('larachain.sources.scrape_web_page'),
             'project' => $project,
             'source' => [
                 'meta_data' => [
@@ -23,8 +23,8 @@ public function create(Project $project)
 
     public function edit(Project $project, Source $source)
     {
-        return inertia('Sources/WebSiteDocument/Edit', [
-            'details' => config('larachain.sources.web_site_document'),
+        return inertia('Sources/ScrapeWebPage/Edit', [
+            'details' => config('larachain.sources.scrape_web_page'),
             'project' => $project,
             'source' => $source,
         ]);
@@ -42,7 +42,7 @@ public function store(Project $project)
             'project_id' => $project->id,
             'name' => $validated['name'],
             'description' => $validated['description'],
-            'type' => SourceTypeEnum::WebSiteDocument,
+            'type' => SourceTypeEnum::ScrapeWebPage,
             'order' => 1,
             'meta_data' => $validated['meta_data'],
         ]);
@@ -68,7 +68,7 @@ public function update(Project $project, Source $source)
             'project_id' => $validated['project_id'],
             'name' => $validated['name'],
             'description' => $validated['description'],
-            'type' => SourceTypeEnum::WebSiteDocument,
+            'type' => SourceTypeEnum::ScrapeWebPage,
             'order' => 1,
             'meta_data' => $validated['meta_data'],
         ]);

app/Models/Document.php

@@ -59,7 +59,6 @@ public function directoryForFile(): string|null
     {
         if (in_array($this->source->type, [
             SourceTypeEnum::WebFile,
-            SourceTypeEnum::WebSiteDocument,
         ])) {
             return sprintf(
                 storage_path('app/projects/%d/sources/%d'),
@@ -74,7 +73,6 @@ public function pathToFile(): string|null
     {
         if (in_array($this->source->type, [
             SourceTypeEnum::WebFile,
-            SourceTypeEnum::WebSiteDocument,
         ])) {
             return sprintf(
                 storage_path('app/projects/%d/sources/%d/%s'),

app/Source/SourceTypeEnum.php

@@ -12,6 +12,6 @@ enum SourceTypeEnum: string
     use TypeTrait;
 
     //case TemplateType = 'template_type'
-    case WebSiteDocument = 'web_site_document';
+    case ScrapeWebPage = 'scrape_web_page';
     case WebFile = 'web_file';
 }

app/Source/Types/WebSiteDocument.php → app/Source/Types/ScrapeWebPage.php

@@ -8,7 +8,7 @@
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Storage;
 
-class WebSiteDocument extends BaseSourceType
+class ScrapeWebPage extends BaseSourceType
 {
     public function handle(): Document
     {

app/Source/Types/WebFile.php

@@ -2,11 +2,11 @@
 
 namespace App\Source\Types;
 
-use App\Models\Document;
+use App\Exceptions\SourceMissingRequiredMetaDataException;
 use App\Ingress\StatusEnum;
+use App\Models\Document;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Storage;
-use App\Exceptions\SourceMissingRequiredMetaDataException;
 
 class WebFile extends BaseSourceType
 {

config/larachain.php

@@ -32,10 +32,10 @@
             'active' => 0,
             'background' => 'bg-red-500',
         ],
-        'web_site_document' => [
+        'scrape_web_page' => [
             'name' => 'Web Site Document',
             'description' => 'Get a web page and save as html',
-            'class' => 'App\\Source\\Types\\WebSiteDocument',
+            'class' => 'App\\Source\\Types\\ScrapeWebPage',
             'requires' => [
             ],
             'active' => 1,

database/factories/SourceFactory.php

@@ -35,11 +35,11 @@ public function definition(): array
         ];
     }
 
-    public function webDocumentMetaData()
+    public function scrapeWebPage()
     {
         return $this->state(function (array $attributes) {
             return [
-                'type' => SourceTypeEnum::WebFile,
+                'type' => SourceTypeEnum::ScrapeWebPage,
                 'meta_data' => [
                     'url' => 'https://en.wikipedia.org/wiki/Laravel',
                 ],

resources/js/Pages/Sources/WebSiteDocument/Create.vue → resources/js/Pages/Sources/ScrapeWebPage/Create.vue

@@ -59,7 +59,7 @@ const form = useForm({
 })
 
 const submit = () => {
-    form.post(route("sources.web_site_document.store", {
+    form.post(route("sources.scrape_web_page.store", {
         project: props.project.id
     }), {
         preserveScroll: true,

resources/js/Pages/Sources/WebSiteDocument/Edit.vue → resources/js/Pages/Sources/ScrapeWebPage/Edit.vue

@@ -60,7 +60,7 @@ const form = useForm({
 })
 
 const submit = () => {
-    form.put(route("sources.web_site_document.update", {
+    form.put(route("sources.scrape_web_page.update", {
         project: props.project.id,
         source: props.source.id
     }), {

routes/web.php

@@ -8,8 +8,8 @@
 use App\Http\Controllers\ResponseTypes\TrimTextResponseTypeController;
 use App\Http\Controllers\ResponseTypes\VectorSearchResponseTypeController;
 use App\Http\Controllers\SortingController;
+use App\Http\Controllers\Sources\ScrapeWebPageSourceController;
 use App\Http\Controllers\Sources\WebFileSourceController;
-use App\Http\Controllers\Sources\WebSiteDocumentSourceController;
 use App\Http\Controllers\Transformers\EmbedTransformerController;
 use App\Http\Controllers\Transformers\PdfTransformerController;
 use Illuminate\Foundation\Application;
@@ -309,18 +309,18 @@ function () {
     'auth:sanctum',
     config('jetstream.auth_session'),
     'verified',
-])->controller(WebSiteDocumentSourceController::class)->group(
+])->controller(ScrapeWebPageSourceController::class)->group(
     function () {
-        Route::get('/projects/{project}/sources/web_site_document/create', 'create')
-            ->name('sources.web_site_document.create');
-        Route::get('/projects/{project}/sources/{source}/web_site_document/edit', 'edit')
-            ->name('sources.web_site_document.edit');
-        Route::post('/projects/{project}/sources/web_site_document/store', 'store')
-            ->name('sources.web_site_document.store');
-        Route::put('/projects/{project}/sources/{source}/web_site_document/update', 'update')
-            ->name('sources.web_site_document.update');
-        Route::post('/projects/{project}/sources/{source}/web_site_document/run', 'run')
-            ->name('sources.web_site_document.run');
+        Route::get('/projects/{project}/sources/scrape_web_page/create', 'create')
+            ->name('sources.scrape_web_page.create');
+        Route::get('/projects/{project}/sources/{source}/scrape_web_page/edit', 'edit')
+            ->name('sources.scrape_web_page.edit');
+        Route::post('/projects/{project}/sources/scrape_web_page/store', 'store')
+            ->name('sources.scrape_web_page.store');
+        Route::put('/projects/{project}/sources/{source}/scrape_web_page/update', 'update')
+            ->name('sources.scrape_web_page.update');
+        Route::post('/projects/{project}/sources/{source}/scrape_web_page/run', 'run')
+            ->name('sources.scrape_web_page.run');
     }
 );
 

tests/Feature/Http/Controllers/WebSiteDocumentSourceControllerTest.php → tests/Feature/Http/Controllers/ScrapeWebPageSourceControllerTest.php

@@ -18,7 +18,7 @@
     ]);
 
     $this->actingAs($user)
-        ->get(route('sources.web_site_document.create', [
+        ->get(route('sources.scrape_web_page.create', [
             'project' => $project->id,
         ]))
         ->assertOk();
@@ -39,7 +39,7 @@
     ]);
 
     $this->actingAs($user)
-        ->get(route('sources.web_site_document.edit', [
+        ->get(route('sources.scrape_web_page.edit', [
             'project' => $project->id,
             'source' => $source->id,
         ]))
@@ -62,7 +62,7 @@
     ]);
 
     $this->actingAs($user)
-        ->post(route('sources.web_site_document.run', [
+        ->post(route('sources.scrape_web_page.run', [
             'project' => $project->id,
             'source' => $source->id,
         ]))
@@ -88,7 +88,7 @@
     ]);
 
     $this->actingAs($user)
-        ->put(route('sources.web_site_document.update', [
+        ->put(route('sources.scrape_web_page.update', [
             'project' => $project->id,
             'source' => $source->id,
         ]), [
@@ -118,7 +118,7 @@
     assertDatabaseCount('sources', 0);
 
     $this->actingAs($user)
-        ->post(route('sources.web_site_document.store', [
+        ->post(route('sources.scrape_web_page.store', [
             'project' => $project->id,
         ]), [
             'name' => 'Foo',

tests/Feature/PdfTransformerTest.php

@@ -41,14 +41,4 @@ public function test_does_not_repeat()
         $transformer->handle($transformerModel);
         $this->assertDatabaseCount('document_chunks', 10);
     }
-
-    public function test_does_not_run()
-    {
-        $this->webPageToText();
-        $transformerModel = Transformer::factory()->pdfTranformer()->create();
-        $this->assertDatabaseCount('document_chunks', 0);
-        $transformer = new PdfTransformer($this->document);
-        $transformer->handle($transformerModel);
-        $this->assertDatabaseCount('document_chunks', 0);
-    }
 }

tests/Feature/WebSiteDocumentTest.php → tests/Feature/ScrapeWebPageTest.php

@@ -4,22 +4,22 @@
 
 use App\Models\Document;
 use App\Models\Source;
-use App\Source\Types\WebSiteDocument;
+use App\Source\Types\ScrapeWebPage;
 use Illuminate\Support\Facades\File;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Storage;
 use Mockery;
 use Tests\TestCase;
 
-class WebSiteDocumentTest extends TestCase
+class ScrapeWebPageTest extends TestCase
 {
     public function test_gets_file()
     {
-        $source = Source::factory()->webDocumentMetaData()->create();
+        $source = Source::factory()->scrapeWebPage()->create();
 
         Storage::fake('projects');
 
-        $webFileSourceType = new WebSiteDocument($source);
+        $webFileSourceType = new ScrapeWebPage($source);
 
         $html = File::get(base_path('tests/fixtures/example.html'));
 
@@ -39,10 +39,10 @@ public function test_gets_file()
 
     public function test_makes_document()
     {
-        $source = Source::factory()->webDocumentMetaData()->create();
+        $source = Source::factory()->scrapeWebPage()->create();
 
         Storage::fake('projects');
-        $webFileSourceType = new WebSiteDocument($source);
+        $webFileSourceType = new ScrapeWebPage($source);
 
         Http::fake([
             'en.wikipedia.com/*' => Http::response('foo', 200),
@@ -57,10 +57,10 @@ public function test_makes_document()
 
     public function test_makes_document_once_with_name()
     {
-        $source = Source::factory()->webDocumentMetaData()->create();
+        $source = Source::factory()->scrapeWebPage()->create();
 
         Storage::fake('projects');
-        $webFileSourceType = new WebSiteDocument($source);
+        $webFileSourceType = new ScrapeWebPage($source);
 
         Http::fake([
             'wikipedia.com/*' => Http::response('foo', 200),

tests/Feature/SharedSetupForPdfFile.php

@@ -12,35 +12,6 @@ trait SharedSetupForPdfFile
 
     public Source $source;
 
-    protected function webPageToText()
-    {
-        $source = Source::factory()
-            ->webDocumentMetaData()
-            ->create();
-
-        $document = Document::factory()->create([
-            'source_id' => $source->id,
-            'guid' => 'example.html',
-        ]);
-
-        $from = base_path('tests/fixtures/example.html');
-
-        if (! File::exists($document->directoryForFile())) {
-            File::makeDirectory(
-                $document->directoryForFile(),
-                0755,
-                true
-            );
-            File::copy(
-                $from,
-                $document->pathToFile()
-            );
-        }
-
-        $this->document = $document;
-        $this->source = $source;
-    }
-
     protected function webFileDownloadSetup()
     {
         $source = Source::factory()

tests/Feature/WebFileTest.php

@@ -2,12 +2,12 @@
 
 namespace Tests\Feature;
 
-use Mockery;
-use Tests\TestCase;
 use App\Models\Source;
 use App\Source\Types\WebFile;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Storage;
+use Mockery;
+use Tests\TestCase;
 
 class WebFileTest extends TestCase
 {