Fix url filtering (uniques), sort urls, readme update (tasks)

README.MD

@@ -1,10 +1,8 @@
 ToDo:
-- Add type to output (node/doctor, media/video, etc.)
 - Add a way to do an initial run, show results, maybe with checkboxes for each line pre-checked, ability for future runs to exclude those urls with the same code and/or the same error message.
 - Way to exclude certain urls
 - Way to include certain urls
-- Count success/fail and write at end of run.
 - Add configuration screen where you can enable different plugins etc.
-- Consider sitemap and menus for url sources.
+- Consider sitemap for url sources.
 - Should I cache bust by clearing all cache at the start of the run or by including url parameters? Which is more reliable/faster?
 - Add javascript console errors. Would have to use a headless browser like Puppeteer, Playwright or https://github.com/chrome-php/chrome. We should test speed and see if it makes sense to move our current url get logic over.

src/Commands/LazyTestCommands.php

@@ -20,15 +20,31 @@ public function __construct(LazyTestService $lazyTestService) {
    */
   public function run() {
     $urls = $this->lazyTestService->getAllURLs();
+
+    // Filter out empty urls.
     $urls = array_filter($urls, function($item) {
       return !empty($item['url']);
     });
-    $urls = array_map('unserialize', array_unique(array_map('serialize', $urls), SORT_REGULAR));
-    // Filter the URLs to only include those that have the same scheme and host as the current site
+
+    // Filter out non-unique urls.
+    array_multisort(array_column($urls, 'url'), SORT_ASC, $urls);
+    $urls = array_intersect_key(
+      $urls,
+      array_unique(array_column($urls, 'url'))
+    );
+
+    // Filter out external urls.
     $current_scheme_and_host = \Drupal::request()->getSchemeAndHttpHost();
     $urls = array_filter($urls, function($item) use ($current_scheme_and_host) {
       return parse_url($item['url'], PHP_URL_SCHEME) . '://' . parse_url($item['url'], PHP_URL_HOST) === $current_scheme_and_host;
     });
+
+    // Sort.
+    $source = array_column($urls, 'source');
+    $subsource = array_column($urls, 'subsource');
+    $url = array_column($urls, 'url');
+    array_multisort($source, SORT_ASC, $subsource, SORT_ASC, $url, SORT_ASC, $urls);
+
     $this->lazyTestService->checkURLs($urls);
   }
 

src/LazyTestService.php

@@ -112,7 +112,7 @@ public function checkURLs($urls) {
         if (!empty($log_messages) || $code != 200) {
           $message = [
             'source' => $url["source"],
-            'subsource' => $url["subsource"],
+            'subsource' => $url["subsource"] ?? '',
             'code' => $code,
             'url' => $url["url"],
             'message' => $log_messages,
@@ -129,7 +129,7 @@ public function checkURLs($urls) {
         $log_messages = $this->getLogMessages($url, $startTimestamp);
         $message = [
           'source' => $url["source"],
-          'subsource' => $url["subsource"],
+          'subsource' => $url["subsource"] ?? '',
           'code' => $code,
           'url' => $url["url"],
           'message' => $log_messages,

src/Plugin/URLProvider/FileURLProvider.php

@@ -48,6 +48,7 @@ public function getURLs() {
         $url = $stream_wrapper_manager->getViaUri($file_uri)->getExternalUrl();
         $urls[] = [
           'source' => "file",
+          'subsource' => '',
           'url' => $url,
         ];
       }

src/Plugin/URLProvider/UserURLProvider.php

@@ -48,6 +48,7 @@ public function getURLs() {
         $url_object->setAbsolute();
         $urls[] = [
           'source' => "user",
+          'subsource' => '',
           'url' => $url_object->toString(),
         ];
       }