Finish migration of publishing to deps

* Frontend and cmdline version use the same fns!
* Add docstrings and db tests
* Fix bug with intermittently failing asset copies
* Move publish cli to scripts since publish-spa shouldn't have graph-parser dependency
* Fix parse-graph bug noticed while testing publishing

bb.edn

@@ -34,9 +34,12 @@
   dev:build-publishing
   logseq.tasks.dev/build-publishing
 
-  dev:publish
-  (apply shell {:dir "deps/publish-spa"} "yarn -s nbb-logseq -cp src -m logseq.publish-spa.cli"
-                *command-line-args*)
+  dev:publish-spa
+  {:depends [dev:build-publishing]
+   :doc "Build publish spa app given graph and output dirs"
+   :task (apply shell {:dir "scripts"}
+           "yarn -s nbb-logseq -cp src -m logseq.tasks.dev.publishing"
+           (into ["static"] *command-line-args*))}
 
   dev:npx-cap-run-ios
   logseq.tasks.dev.mobile/npx-cap-run-ios

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -77,57 +77,58 @@ Options available:
   which may be referenced elsewhere.
 * :skip-db-transact? - Boolean which skips transacting in order to batch transactions. Default is false
 * :extract-options - Options map to pass to extract/extract"
-  [conn file content {:keys [new? delete-blocks-fn extract-options skip-db-transact?]
-                      :or {new? true
-                           delete-blocks-fn (constantly [])
-                           skip-db-transact? false}
-                      :as options}]
-  (let [format (gp-util/get-format file)
-        file-content [{:file/path file}]
-        {:keys [tx ast]}
-        (let [extract-options' (merge {:block-pattern (gp-config/get-block-pattern format)
-                                       :date-formatter "MMM do, yyyy"
-                                       :supported-formats (gp-config/supported-formats)
-                                       :uri-encoded? false
-                                       :filename-format :legacy}
-                                      extract-options
-                                      {:db @conn})
-              {:keys [pages blocks ast]
-               :or   {pages []
-                      blocks []
-                      ast []}}
-              (cond (contains? gp-config/mldoc-support-formats format)
-                (extract/extract file content extract-options')
+  ([conn file content] (parse-file conn file content {}))
+  ([conn file content {:keys [new? delete-blocks-fn extract-options skip-db-transact?]
+                       :or {new? true
+                            delete-blocks-fn (constantly [])
+                            skip-db-transact? false}
+                       :as options}]
+   (let [format (gp-util/get-format file)
+         file-content [{:file/path file}]
+         {:keys [tx ast]}
+         (let [extract-options' (merge {:block-pattern (gp-config/get-block-pattern format)
+                                        :date-formatter "MMM do, yyyy"
+                                        :supported-formats (gp-config/supported-formats)
+                                        :uri-encoded? false
+                                        :filename-format :legacy}
+                                       extract-options
+                                       {:db @conn})
+               {:keys [pages blocks ast]
+                :or   {pages []
+                       blocks []
+                       ast []}}
+               (cond (contains? gp-config/mldoc-support-formats format)
+                 (extract/extract file content extract-options')
 
-                (gp-config/whiteboard? file)
-                (extract/extract-whiteboard-edn file content extract-options')
+                 (gp-config/whiteboard? file)
+                 (extract/extract-whiteboard-edn file content extract-options')
 
-                :else nil)
-              block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
-              delete-blocks (delete-blocks-fn @conn (first pages) file block-ids)
-              block-refs-ids (->> (mapcat :block/refs blocks)
-                                  (filter (fn [ref] (and (vector? ref)
-                                                         (= :block/uuid (first ref)))))
-                                  (map (fn [ref] {:block/uuid (second ref)}))
-                                  (seq))
-              ;; To prevent "unique constraint" on datascript
-              block-ids (set/union (set block-ids) (set block-refs-ids))
-              pages (extract/with-ref-pages pages blocks)
-              pages-index (map #(select-keys % [:block/name]) pages)]
-              ;; does order matter?
-          {:tx (concat file-content pages-index delete-blocks pages block-ids blocks)
-           :ast ast})
-        tx (concat tx [(cond-> {:file/path file
-                                :file/content content}
-                         new?
-                         ;; TODO: use file system timestamp?
-                         (assoc :file/created-at (date-time-util/time-ms)))])
-        tx' (gp-util/fast-remove-nils tx)
-        result (if skip-db-transact?
-                 tx'
-                 (d/transact! conn tx' (select-keys options [:new-graph? :from-disk?])))]
-    {:tx result
-     :ast ast}))
+                 :else nil)
+               block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
+               delete-blocks (delete-blocks-fn @conn (first pages) file block-ids)
+               block-refs-ids (->> (mapcat :block/refs blocks)
+                                   (filter (fn [ref] (and (vector? ref)
+                                                          (= :block/uuid (first ref)))))
+                                   (map (fn [ref] {:block/uuid (second ref)}))
+                                   (seq))
+               ;; To prevent "unique constraint" on datascript
+               block-ids (set/union (set block-ids) (set block-refs-ids))
+               pages (extract/with-ref-pages pages blocks)
+               pages-index (map #(select-keys % [:block/name]) pages)]
+           ;; does order matter?
+           {:tx (concat file-content pages-index delete-blocks pages block-ids blocks)
+            :ast ast})
+         tx (concat tx [(cond-> {:file/path file
+                                 :file/content content}
+                                new?
+                                ;; TODO: use file system timestamp?
+                                (assoc :file/created-at (date-time-util/time-ms)))])
+         tx' (gp-util/fast-remove-nils tx)
+         result (if skip-db-transact?
+                  tx'
+                  (d/transact! conn tx' (select-keys options [:new-graph? :from-disk?])))]
+     {:tx result
+      :ast ast})))
 
 (defn filter-files
   "Filters files in preparation for parsing. Only includes files that are

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -26,9 +26,11 @@ TODO: Fail fast when process exits 1"
   "Given a git graph directory, returns allowed file paths and their contents in
   preparation for parsing"
   [dir]
-  (let [files (->> (str (.-stdout (sh ["git" "ls-files"]
+  ;; -z needed to avoid quoting unusual paths that cause slurp failures.
+  ;; See https://git-scm.com/docs/git-ls-files#_output for more
+  (let [files (->> (str (.-stdout (sh ["git" "ls-files" "-z"]
                                       {:cwd dir :stdio nil})))
-                   string/split-lines
+                   (#(string/split % #"\0"))
                    (map #(hash-map :file/path (str dir "/" %)))
                    graph-parser/filter-files)]
     (mapv #(assoc % :file/content (slurp (:file/path %))) files)))

deps/publish-spa/nbb.edn

@@ -1,8 +1,7 @@
 {:paths ["src"]
  :deps
  {logseq/graph-parser
-  ;; Nbb bug. Should just be "../graph-parser"
-  {:local/root "../../../../graph-parser"}
+  ;; Nbb bug. Should just be "../db"
+  {:local/root "../../../../db"}
   io.github.nextjournal/nbb-test-runner
-  {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"
-   #_#_:local/root "../../../../../../nbb-test-runner"}}}
+  {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}}

deps/publish-spa/package.json

@@ -6,10 +6,9 @@
     "@logseq/nbb-logseq": "^1.2.168"
   },
   "dependencies": {
-    "mldoc": "^1.5.1",
     "fs-extra": "9.1.0"
   },
   "scripts": {
-    "test": "nbb-logseq -cp src:test -m nextjournal.test-runner"
+    "test": "yarn nbb-logseq -cp test:../graph-parser/src -m nextjournal.test-runner"
   }
 }

deps/publish-spa/src/logseq/publish_spa.cljs

@@ -1,24 +1,18 @@
 (ns logseq.publish-spa
-  (:require [datascript.transit :as dt]
-            [logseq.publish-spa.html :as html]
-            [logseq.publish-spa.export :as export]
-            [logseq.publish-spa.db :as db]))
+  "This node only ns provides api fns for exporting a publishing app"
+  (:require [logseq.publish-spa.html :as publish-html]
+            [logseq.publish-spa.export :as publish-export]))
 
-(defn prep-for-export [db {:keys [app-state repo-config html-options]}]
-  (let [[db asset-filenames']
-        (if (:publishing/all-pages-public? repo-config)
-          (db/clean-export! db)
-          (db/filter-only-public-pages-and-blocks db))
-        asset-filenames (remove nil? asset-filenames')
-        db-str (dt/write-transit-str db)
-        state (assoc (select-keys app-state
-                            [:ui/theme
-                             :ui/sidebar-collapsed-blocks])
-                     :config {"local" repo-config})
-        raw-html-str (html/publishing-html db-str state html-options)]
-    {:html raw-html-str
-     :asset-filenames asset-filenames}))
-
-(defn publish [db static-dir graph-dir output-path options]
-  (let [{:keys [html asset-filenames]} (prep-for-export db options)]
-    (export/export html static-dir graph-dir output-path {:asset-filenames asset-filenames})))
+(defn export
+  "Exports the given graph-dir and db to the specific output-dir. Most of the graph
+configuration is done through logseq/config.edn. There are a few explicit options that
+can be passed:
+* :ui/theme - Theme mode that can either be 'light' or 'dark'.
+* :html-options - A map of values that are inserted into index.html. Map keys
+  can be icon, name, alias, title, description and url"
+  [db static-dir graph-dir output-dir options]
+  (let [options' (cond-> options
+                         (:ui/theme options)
+                         (assoc :app-state {:ui/theme (:ui/theme options)}))
+        {:keys [html asset-filenames]} (publish-html/build-html db options')]
+    (publish-export/create-export html static-dir graph-dir output-dir {:asset-filenames asset-filenames})))