fix(adaptive-crawler): if no sitemap, use recursive instead

zhanqingbin · Oct 8, 2024 · 5a4b35e · 5a4b35e
1 parent ee29be5
commit 5a4b35e
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/backend/functions/src/cloud-functions/adaptive-crawler.ts b/backend/functions/src/cloud-functions/adaptive-crawler.ts
@@ -120,9 +120,12 @@ export class AdaptiveCrawlerHost extends RPCHost {
             failed: {},
         });
 
+        let urls: string[] = [];
         if (useSitemap) {
-            const urls = await this.crawlUrlsFromSitemap(targetUrl, maxPages);
+            urls = await this.crawlUrlsFromSitemap(targetUrl, maxPages);
+        }
 
+        if (urls.length > 0) {
             await AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({
                 status: AdaptiveCrawlTaskStatus.PROCESSING,
                 statusText: `Processing 0/${urls.length}`,