From 657bb0959104922d2636e7ff9dd01096914814cd Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 5 May 2022 20:44:56 -0400 Subject: [PATCH 01/58] Add mldoc integration test and start mldoc split --- .clj-kondo/config.edn | 2 + src/main/frontend/components/block.cljs | 9 +-- src/main/frontend/format.cljs | 7 ++- src/main/frontend/format/block.cljs | 3 +- src/main/frontend/format/mldoc.cljs | 71 +++--------------------- src/main/frontend/handler/editor.cljs | 9 +-- src/main/frontend/handler/export.cljs | 5 +- src/main/frontend/handler/extract.cljs | 3 +- src/main/frontend/handler/plugin.cljs | 4 +- src/main/frontend/utf8.cljs | 2 +- src/main/frontend/util/drawer.cljs | 5 +- src/main/frontend/util/property.cljs | 5 +- src/main/logseq/graph_parser/mldoc.cljc | 61 ++++++++++++++++++++ src/test/frontend/format/mldoc_test.cljs | 68 ++++++++++++++++++++++- src/test/frontend/parser.cljs | 3 +- 15 files changed, 171 insertions(+), 86 deletions(-) create mode 100644 src/main/logseq/graph_parser/mldoc.cljc diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index bdff5ace2ae..3df6e13d09d 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -20,6 +20,8 @@ frontend.db.query-react query-react frontend.util util frontend.config config + frontend.format.mldoc mldoc + logseq.graph-parser.mldoc gp-mldoc logseq.graph-parser.util gp-util logseq.graph-parser.config gp-config}}} diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 1f69a120df0..625161d82d2 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -53,6 +53,7 @@ [frontend.util.drawer :as drawer] [logseq.graph-parser.config :as gp-config] [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.mldoc :as gp-mldoc] [goog.dom :as gdom] [goog.object :as gobj] [lambdaisland.glogi :as log] @@ -719,13 +720,13 @@ (inline-text {} format v)) ([config format v] (when (string? v) - (let [inline-list (mldoc/inline->edn v (mldoc/default-config format))] + (let [inline-list (mldoc/inline->edn v (gp-mldoc/default-config format))] [:div.inline.mr-1 (map-inline config inline-list)])))) (defn- render-macro [config name arguments macro-content format] (if macro-content - (let [ast (->> (mldoc/->edn macro-content (mldoc/default-config format)) + (let [ast (->> (mldoc/->edn macro-content (gp-mldoc/default-config format)) (map first)) paragraph? (and (= 1 (count ast)) (= "Paragraph" (ffirst ast)))] @@ -2707,11 +2708,11 @@ ;; (cond ;; (= lang "quote") ;; (let [content (string/trim (string/join "\n" lines))] -;; ["Quote" (first (mldoc/->edn content (mldoc/default-config :markdown)))]) +;; ["Quote" (first (mldoc/->edn content (gp-mldoc/default-config :markdown)))]) ;; (contains? #{"query" "note" "tip" "important" "caution" "warning" "pinned"} lang) ;; (let [content (string/trim (string/join "\n" lines))] -;; ["Custom" lang nil (first (mldoc/->edn content (mldoc/default-config :markdown))) content]) +;; ["Custom" lang nil (first (mldoc/->edn content (gp-mldoc/default-config :markdown))) content]) ;; :else ;; ["Src" options]))) diff --git a/src/main/frontend/format.cljs b/src/main/frontend/format.cljs index 50e8fbaee8b..ced7e00688c 100644 --- a/src/main/frontend/format.cljs +++ b/src/main/frontend/format.cljs @@ -3,6 +3,7 @@ [frontend.format.adoc :refer [->AdocMode]] [frontend.format.protocol :as protocol] [frontend.text :as text] + [logseq.graph-parser.mldoc :as gp-mldoc] [clojure.string :as string])) (set! mldoc/parse-property text/parse-property) @@ -37,9 +38,9 @@ ;; html (defn get-default-config ([format] - (mldoc/default-config format)) + (gp-mldoc/default-config format)) ([format options] - (mldoc/default-config format options))) + (gp-mldoc/default-config format options))) (defn to-html ([content format] @@ -49,7 +50,7 @@ (if (string/blank? content) "" (if-let [record (get-format-record format)] - (protocol/toHtml record content config mldoc/default-references) + (protocol/toHtml record content config gp-mldoc/default-references) content))))) (defn to-edn diff --git a/src/main/frontend/format/block.cljs b/src/main/frontend/format/block.cljs index 63c1c74490f..19474ece771 100644 --- a/src/main/frontend/format/block.cljs +++ b/src/main/frontend/format/block.cljs @@ -13,6 +13,7 @@ [frontend.util.property :as property] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.config :as gp-config] + [logseq.graph-parser.mldoc :as gp-mldoc] [lambdaisland.glogi :as log] [medley.core :as medley] [frontend.format.mldoc :as mldoc])) @@ -702,7 +703,7 @@ (str (config/get-block-pattern format) " " (string/triml content)))] (if-let [result (state/get-block-ast block-uuid content)] result - (let [ast (->> (format/to-edn content format (mldoc/default-config format)) + (let [ast (->> (format/to-edn content format (gp-mldoc/default-config format)) (map first)) title (when (heading-block? (first ast)) (:title (second (first ast)))) diff --git a/src/main/frontend/format/mldoc.cljs b/src/main/frontend/format/mldoc.cljs index 036078f5c52..ae0854234be 100644 --- a/src/main/frontend/format/mldoc.cljs +++ b/src/main/frontend/format/mldoc.cljs @@ -1,6 +1,5 @@ (ns frontend.format.mldoc - (:require [cljs-bean.core :as bean] - [clojure.string :as string] + (:require [clojure.string :as string] [frontend.format.protocol :as protocol] [frontend.utf8 :as utf8] [goog.object :as gobj] @@ -8,61 +7,15 @@ [medley.core :as medley] ["mldoc" :as mldoc :refer [Mldoc]] [linked.core :as linked] + [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.config :as gp-config])) -(defonce parseJson (gobj/get Mldoc "parseJson")) -(defonce parseInlineJson (gobj/get Mldoc "parseInlineJson")) -(defonce parseOPML (gobj/get Mldoc "parseOPML")) -(defonce export (gobj/get Mldoc "export")) (defonce anchorLink (gobj/get Mldoc "anchorLink")) +(defonce parseOPML (gobj/get Mldoc "parseOPML")) (defonce parseAndExportMarkdown (gobj/get Mldoc "parseAndExportMarkdown")) (defonce parseAndExportOPML (gobj/get Mldoc "parseAndExportOPML")) -(defonce astExportMarkdown (gobj/get Mldoc "astExportMarkdown")) - -(defn convert-export-md-remove-options [opts] - (->> - (mapv (fn [opt] - (case opt - :page-ref ["Page_ref"] - :emphasis ["Emphasis"] - [])) - opts) - (remove empty?))) - - -(defn default-config - ([format] - (default-config format {:export-heading-to-list? false})) - ([format {:keys [export-heading-to-list? export-keep-properties? export-md-indent-style export-md-remove-options parse_outline_only?]}] - (let [format (string/capitalize (name (or format :markdown)))] - (->> {:toc false - :parse_outline_only (or parse_outline_only? false) - :heading_number false - :keep_line_break true - :format format - :heading_to_list (or export-heading-to-list? false) - :exporting_keep_properties export-keep-properties? - :export_md_indent_style export-md-indent-style - :export_md_remove_options - (convert-export-md-remove-options export-md-remove-options)} - (filter #(not(nil? (second %)))) - (into {}) - (bean/->js) - (js/JSON.stringify))))) - -(def default-references - (js/JSON.stringify - (clj->js {:embed_blocks [] - :embed_pages []}))) - -(defn parse-json - [content config] - (parseJson content config)) - -(defn inline-parse-json - [text config] - (parseInlineJson text config)) +(defonce export (gobj/get Mldoc "export")) (defn parse-opml [content] @@ -72,20 +25,14 @@ [content config references] (parseAndExportMarkdown content config - (or references default-references))) + (or references gp-mldoc/default-references))) (defn parse-export-opml [content config title references] (parseAndExportOPML content config title - (or references default-references))) - -(defn ast-export-markdown - [ast config references] - (astExportMarkdown ast - config - (or references default-references))) + (or references gp-mldoc/default-references))) (defn remove-indentation-spaces [s level remove-first-line?] @@ -201,7 +148,7 @@ (if (string/blank? content) [] (-> content - (parse-json config) + (gp-mldoc/parse-json config) (gp-util/json->clj) (update-src-full-content content) (collect-page-properties parse-property))) @@ -227,7 +174,7 @@ (if (string/blank? text) {} (-> text - (inline-parse-json config) + (gp-mldoc/inline-parse-json config) (gp-util/json->clj))) (catch js/Error _e []))) @@ -263,7 +210,7 @@ (defn link? [format link] (when (string? link) - (let [[type link] (first (inline->edn link (default-config format))) + (let [[type link] (first (inline->edn link (gp-mldoc/default-config format))) [ref-type ref-value] (:url link)] (and (= "Link" type) (or diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 8c3dc66535b..fcc769b4d3f 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -53,6 +53,7 @@ [promesa.core :as p] [frontend.util.keycode :as keycode] [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.mldoc :as gp-mldoc] ["path" :as path])) ;; FIXME: should support multiple images concurrently uploading @@ -356,7 +357,7 @@ content (drawer/with-logbook block content) content (with-timetracking block content) first-block? (= left page) - ast (mldoc/->edn (string/trim content) (mldoc/default-config format)) + ast (mldoc/->edn (string/trim content) (gp-mldoc/default-config format)) first-elem-type (first (ffirst ast)) first-elem-meta (second (ffirst ast)) properties? (contains? #{"Property_Drawer" "Properties"} first-elem-type) @@ -1941,7 +1942,7 @@ props (into [] (:properties block)) content* (str (if (= :markdown format) "- " "* ") (property/insert-properties format content props)) - ast (mldoc/->edn content* (mldoc/default-config format)) + ast (mldoc/->edn content* (gp-mldoc/default-config format)) blocks (block/extract-blocks ast content* true format) fst-block (first blocks)] (assert fst-block "fst-block shouldn't be nil") @@ -2850,7 +2851,7 @@ (when-let [editing-block (state/get-edit-block)] (let [page-id (:db/id (:block/page editing-block)) blocks (block/extract-blocks - (mldoc/->edn text (mldoc/default-config format)) text true format) + (mldoc/->edn text (gp-mldoc/default-config format)) text true format) blocks' (block/with-parent-and-left page-id blocks)] (paste-blocks blocks' {})))) @@ -3168,7 +3169,7 @@ [format content semantic?] (and (string/includes? content "\n") (if semantic? - (let [ast (mldoc/->edn content (mldoc/default-config format)) + (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) first-elem-type (first (ffirst ast))] (mldoc/block-with-title? first-elem-type)) true))) diff --git a/src/main/frontend/handler/export.cljs b/src/main/frontend/handler/export.cljs index f53a7085ce5..427d5749888 100644 --- a/src/main/frontend/handler/export.cljs +++ b/src/main/frontend/handler/export.cljs @@ -16,6 +16,7 @@ [frontend.state :as state] [frontend.util :as util] [frontend.format.mldoc :as mldoc] + [logseq.graph-parser.mldoc :as gp-mldoc] [goog.dom :as gdom] [promesa.core :as p]) (:import [goog.string StringBuffer])) @@ -214,7 +215,7 @@ (let [block (db/entity [:block/uuid (uuid block-uuid)]) block-content (get-blocks-contents repo (:block/uuid block)) format (:block/format block) - ast (mldoc/->edn block-content (mldoc/default-config format)) + ast (mldoc/->edn block-content (gp-mldoc/default-config format)) embed-pages-new (get-embed-pages-from-ast ast) embed-blocks-new (get-embed-blocks-from-ast ast) block-refs-new (get-block-refs-from-ast ast) @@ -258,7 +259,7 @@ (let [page-name* (util/page-name-sanity-lc page-name) page-content (get-page-content repo page-name*) format (:block/format (db/entity [:block/name page-name*])) - ast (mldoc/->edn page-content (mldoc/default-config format)) + ast (mldoc/->edn page-content (gp-mldoc/default-config format)) embed-pages-new (get-embed-pages-from-ast ast) embed-blocks-new (get-embed-blocks-from-ast ast) block-refs-new (get-block-refs-from-ast ast) diff --git a/src/main/frontend/handler/extract.cljs b/src/main/frontend/handler/extract.cljs index 000a8a0eef8..cddde05c20e 100644 --- a/src/main/frontend/handler/extract.cljs +++ b/src/main/frontend/handler/extract.cljs @@ -12,6 +12,7 @@ [frontend.text :as text] [frontend.util :as util] [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.mldoc :as gp-mldoc] [frontend.util.property :as property] [lambdaisland.glogi :as log])) @@ -142,7 +143,7 @@ [] (let [format (format/get-format file) _ (println "Parsing start: " file) - ast (mldoc/->edn content (mldoc/default-config format + ast (mldoc/->edn content (gp-mldoc/default-config format ;; {:parse_outline_only? true} ))] (println "Parsing finished : " file) diff --git a/src/main/frontend/handler/plugin.cljs b/src/main/frontend/handler/plugin.cljs index 6179572a1dd..f278476685f 100644 --- a/src/main/frontend/handler/plugin.cljs +++ b/src/main/frontend/handler/plugin.cljs @@ -3,7 +3,7 @@ [rum.core :as rum] [frontend.util :as util] [clojure.walk :as walk] - [frontend.format.mldoc :as mldoc] + [logseq.graph-parser.mldoc :as gp-mldoc] [frontend.handler.notification :as notifications] [camel-snake-kebab.core :as csk] [frontend.state :as state] @@ -403,7 +403,7 @@ (string/replace matched link (util/node-path.join url link)) matched))) content)] - (format/to-html content :markdown (mldoc/default-config :markdown)))) + (format/to-html content :markdown (gp-mldoc/default-config :markdown)))) (catch js/Error e (log/error :parse-user-md-exception e) content))) diff --git a/src/main/frontend/utf8.cljs b/src/main/frontend/utf8.cljs index 200e65a8e6b..a464314cd74 100644 --- a/src/main/frontend/utf8.cljs +++ b/src/main/frontend/utf8.cljs @@ -1,4 +1,4 @@ -(ns frontend.utf8 +(ns ^:nbb-compatible frontend.utf8 (:require [goog.object :as gobj])) (defonce encoder diff --git a/src/main/frontend/util/drawer.cljs b/src/main/frontend/util/drawer.cljs index 67e47046a60..cf3f81c84e6 100644 --- a/src/main/frontend/util/drawer.cljs +++ b/src/main/frontend/util/drawer.cljs @@ -2,6 +2,7 @@ (:require [clojure.string :as string] [frontend.util :as util] [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.mldoc :as gp-mldoc] [frontend.util.property :as property] [frontend.format.mldoc :as mldoc])) @@ -23,7 +24,7 @@ (defn get-drawer-ast [format content typ] - (let [ast (mldoc/->edn content (mldoc/default-config format)) + (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) typ-drawer (ffirst (filter (fn [x] (mldoc/typ-drawer? x typ)) ast))] typ-drawer)) @@ -32,7 +33,7 @@ [format content typ value] (when (string? content) (try - (let [ast (mldoc/->edn content (mldoc/default-config format)) + (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) has-properties? (some (fn [x] (mldoc/properties? x)) ast) has-typ-drawer? (some (fn [x] (mldoc/typ-drawer? x typ)) ast) lines (string/split-lines content) diff --git a/src/main/frontend/util/property.cljs b/src/main/frontend/util/property.cljs index 2449ac1bdea..18402897644 100644 --- a/src/main/frontend/util/property.cljs +++ b/src/main/frontend/util/property.cljs @@ -5,6 +5,7 @@ [frontend.config :as config] [medley.core :as medley] [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.mldoc :as gp-mldoc] [frontend.format.mldoc :as mldoc] [frontend.text :as text] [frontend.util.cursor :as cursor])) @@ -187,7 +188,7 @@ properties (filter (fn [[k _v]] ((built-in-properties) k)) properties)] (if (seq properties) (let [lines (string/split-lines content) - ast (mldoc/->edn content (mldoc/default-config format)) + ast (mldoc/->edn content (gp-mldoc/default-config format)) [title body] (if (mldoc/block-with-title? (first (ffirst ast))) [(first lines) (rest lines)] [nil lines]) @@ -233,7 +234,7 @@ (insert-property format content key value false)) ([format content key value front-matter?] (when (string? content) - (let [ast (mldoc/->edn content (mldoc/default-config format)) + (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) title? (mldoc/block-with-title? (ffirst (map first ast))) has-properties? (or (and title? (mldoc/properties? (second ast))) diff --git a/src/main/logseq/graph_parser/mldoc.cljc b/src/main/logseq/graph_parser/mldoc.cljc new file mode 100644 index 00000000000..8c04a6e1d79 --- /dev/null +++ b/src/main/logseq/graph_parser/mldoc.cljc @@ -0,0 +1,61 @@ +(ns ^:nbb-compatible logseq.graph-parser.mldoc + (:require #?(:org.babashka/nbb ["mldoc$default" :refer [Mldoc]] + :default ["mldoc" :refer [Mldoc]]) + [goog.object :as gobj] + [cljs-bean.core :as bean] + [clojure.string :as string])) + +(defonce parseJson (gobj/get Mldoc "parseJson")) +(defonce parseInlineJson (gobj/get Mldoc "parseInlineJson")) +(defonce astExportMarkdown (gobj/get Mldoc "astExportMarkdown")) + +;; NOTE: All references to js/ have to have a :default conditional because of clojure +(def default-references + #?(:cljs + (js/JSON.stringify + (clj->js {:embed_blocks [] + :embed_pages []})) + :default {})) + +(defn- convert-export-md-remove-options [opts] + (->> opts + (mapv (fn [opt] + (case opt + :page-ref ["Page_ref"] + :emphasis ["Emphasis"] + []))) + (remove empty?))) + +(defn parse-json + [content config] + (parseJson content config)) + +(defn inline-parse-json + [text config] + (parseInlineJson text config)) + +(defn ast-export-markdown + [ast config references] + (astExportMarkdown ast + config + (or references default-references))) + +(defn default-config + ([format] + (default-config format {:export-heading-to-list? false})) + ([format {:keys [export-heading-to-list? export-keep-properties? export-md-indent-style export-md-remove-options parse_outline_only?]}] + (let [format (string/capitalize (name (or format :markdown)))] + (->> {:toc false + :parse_outline_only (or parse_outline_only? false) + :heading_number false + :keep_line_break true + :format format + :heading_to_list (or export-heading-to-list? false) + :exporting_keep_properties export-keep-properties? + :export_md_indent_style export-md-indent-style + :export_md_remove_options + (convert-export-md-remove-options export-md-remove-options)} + (filter #(not (nil? (second %)))) + (into {}) + (bean/->js) + #?(:cljs js/JSON.stringify :default identity))))) diff --git a/src/test/frontend/format/mldoc_test.cljs b/src/test/frontend/format/mldoc_test.cljs index 07e3d87b194..cadcb613735 100644 --- a/src/test/frontend/format/mldoc_test.cljs +++ b/src/test/frontend/format/mldoc_test.cljs @@ -1,6 +1,11 @@ (ns frontend.format.mldoc-test (:require [frontend.format.mldoc :as mldoc] - [cljs.test :refer [testing deftest are]])) + [logseq.graph-parser.mldoc :as gp-mldoc] + ["fs" :as fs] + ["child_process" :as child-process] + [clojure.string :as string] + [clojure.edn :as edn] + [cljs.test :refer [testing deftest are is]])) (deftest test-link (testing "non-link" @@ -37,3 +42,64 @@ (testing "parsing links should be finished" (are [x y] (= (mldoc/link? :markdown x) y) "[YouTube](https://www.youtube.com/watch?v=-8ym7pyUs9gL) - [Vimeo](https://vimeo.com/677920303) {{youtube https://www.youtube.com/watch?v=-8ym7pyUs9g}}" true))) + +;; TODO: Reuse with repo-test fns +(defn- slurp + "Like clojure.core/slurp" + [file] + (str (fs/readFileSync file))) + +(defn- sh + "Run shell cmd synchronously and print to inherited streams by default. Aims + to be similar to babashka.tasks/shell" + [cmd opts] + (child-process/spawnSync (first cmd) + (clj->js (rest cmd)) + (clj->js (merge {:stdio "inherit"} opts)))) + +(defn- build-graph-files + [dir] + (let [files (->> (str (.-stdout (sh ["git" "ls-files"] + {:cwd dir :stdio nil}))) + string/split-lines + (filter #(re-find #"^(pages|journals)" %)) + (map #(str dir "/" %)))] + (mapv #(hash-map :file/path % :file/content (slurp %)) files))) + +;; TODO: Add clone docs step +(deftest ^:integration test->edn + (let [graph-dir "src/test/docs" + files (build-graph-files graph-dir) + asts-by-file (->> files + (map (fn [{:file/keys [path content]}] + (let [format (if (string/ends-with? path ".org") + :org :markdown)] + [path + (mldoc/->edn content + (gp-mldoc/default-config format))]))) + (into {}))] + (is (= {"CommentBlock" 1, + "Custom" 41, + "Displayed_Math" 1, + "Drawer" 1, + "Example" 20, + "Footnote_Definition" 2, + "Heading" 3493, + "Hiccup" 15, + "List" 36, + "Paragraph" 411, + "Properties" 104, + "Property_Drawer" 188, + "Quote" 9, + "Raw_Html" 12, + "Src" 56, + "Table" 4} + (->> asts-by-file (mapcat val) (map ffirst) frequencies)) + "AST node type counts") + + ;; This is just temporary + (is (= (edn/read-string (slurp "mldoc-asts.edn")) + asts-by-file) + "Matches initial AST") + #_(println "Wrote asts for" (count asts-by-file) "files") + #_(fs/writeFileSync "mldoc-asts.edn" (pr-str asts-by-file)))) diff --git a/src/test/frontend/parser.cljs b/src/test/frontend/parser.cljs index c617d3aad54..f4df43ff339 100644 --- a/src/test/frontend/parser.cljs +++ b/src/test/frontend/parser.cljs @@ -1,8 +1,9 @@ (ns frontend.parser (:require [cljs.test :refer [is deftest]] + [logseq.graph-parser.mldoc :as gp-mldoc] [frontend.format.mldoc :as mldoc :refer [->edn]])) -(def md-config (mldoc/default-config :markdown)) +(def md-config (gp-mldoc/default-config :markdown)) (deftest src-test (is (= From 67461aaedeb7018ca8e669c597688e4cea10de03 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Fri, 6 May 2022 16:14:50 -0400 Subject: [PATCH 02/58] Necessary parts of mldoc are nbb compatible Also merged in parser tests which were not running --- .carve/ignore | 2 +- src/main/frontend/components/block.cljs | 8 +- src/main/frontend/format.cljs | 2 +- src/main/frontend/format/block.cljs | 7 +- src/main/frontend/format/mldoc.cljs | 155 +--------------- src/main/frontend/handler/editor.cljs | 8 +- src/main/frontend/handler/export.cljs | 5 +- src/main/frontend/handler/extract.cljs | 3 +- src/main/frontend/mobile/intent.cljs | 6 +- src/main/frontend/text.cljs | 4 +- src/main/frontend/util/drawer.cljs | 4 +- src/main/frontend/util/property.cljs | 4 +- src/main/logseq/graph_parser/log.cljs | 6 + src/main/logseq/graph_parser/mldoc.cljc | 172 +++++++++++++++++- src/test/frontend/parser.cljs | 29 --- .../graph_parser}/mldoc_test.cljs | 48 +++-- 16 files changed, 234 insertions(+), 229 deletions(-) create mode 100644 src/main/logseq/graph_parser/log.cljs delete mode 100644 src/test/frontend/parser.cljs rename src/test/{frontend/format => logseq/graph_parser}/mldoc_test.cljs (70%) diff --git a/.carve/ignore b/.carve/ignore index ac491435a65..c4f4fb05d46 100644 --- a/.carve/ignore +++ b/.carve/ignore @@ -29,7 +29,7 @@ frontend.extensions.zotero.api/item ;; For repl frontend.external.roam/reset-state! ;; For repl -frontend.format.mldoc/ast-export-markdown +logseq.graph-parser.mldoc/ast-export-markdown ;; Protocol fn wrapper that could be used frontend.fs/readdir ;; Referenced in TODO diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 625161d82d2..8e56a5b573b 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -720,13 +720,13 @@ (inline-text {} format v)) ([config format v] (when (string? v) - (let [inline-list (mldoc/inline->edn v (gp-mldoc/default-config format))] + (let [inline-list (gp-mldoc/inline->edn v (gp-mldoc/default-config format))] [:div.inline.mr-1 (map-inline config inline-list)])))) (defn- render-macro [config name arguments macro-content format] (if macro-content - (let [ast (->> (mldoc/->edn macro-content (gp-mldoc/default-config format)) + (let [ast (->> (gp-mldoc/->edn macro-content (gp-mldoc/default-config format)) (map first)) paragraph? (and (= 1 (count ast)) (= "Paragraph" (ffirst ast)))] @@ -2708,11 +2708,11 @@ ;; (cond ;; (= lang "quote") ;; (let [content (string/trim (string/join "\n" lines))] -;; ["Quote" (first (mldoc/->edn content (gp-mldoc/default-config :markdown)))]) +;; ["Quote" (first (gp-mldoc/->edn content (gp-mldoc/default-config :markdown)))]) ;; (contains? #{"query" "note" "tip" "important" "caution" "warning" "pinned"} lang) ;; (let [content (string/trim (string/join "\n" lines))] -;; ["Custom" lang nil (first (mldoc/->edn content (gp-mldoc/default-config :markdown))) content]) +;; ["Custom" lang nil (first (gp-mldoc/->edn content (gp-mldoc/default-config :markdown))) content]) ;; :else ;; ["Src" options]))) diff --git a/src/main/frontend/format.cljs b/src/main/frontend/format.cljs index ced7e00688c..e9c24b45f04 100644 --- a/src/main/frontend/format.cljs +++ b/src/main/frontend/format.cljs @@ -6,7 +6,7 @@ [logseq.graph-parser.mldoc :as gp-mldoc] [clojure.string :as string])) -(set! mldoc/parse-property text/parse-property) +(set! gp-mldoc/parse-property text/parse-property) (defonce mldoc-record (->MldocMode)) (defonce adoc-record (->AdocMode)) diff --git a/src/main/frontend/format/block.cljs b/src/main/frontend/format/block.cljs index 19474ece771..28e03948053 100644 --- a/src/main/frontend/format/block.cljs +++ b/src/main/frontend/format/block.cljs @@ -15,8 +15,7 @@ [logseq.graph-parser.config :as gp-config] [logseq.graph-parser.mldoc :as gp-mldoc] [lambdaisland.glogi :as log] - [medley.core :as medley] - [frontend.format.mldoc :as mldoc])) + [medley.core :as medley])) (defn heading-block? [block] @@ -166,7 +165,7 @@ (map last) (map (fn [v] (when (and (string? v) - (not (mldoc/link? format v))) + (not (gp-mldoc/link? format v))) (let [v (string/trim v) result (text/split-page-refs-without-brackets v {:un-brackets? false})] (if (coll? result) @@ -430,7 +429,7 @@ (if (or (:pre-block? block) (= (:format block) :org)) content - (mldoc/remove-indentation-spaces content (inc (:level block)) false))))] + (gp-mldoc/remove-indentation-spaces content (inc (:level block)) false))))] (if (= format :org) content (property/->new-properties content)))) diff --git a/src/main/frontend/format/mldoc.cljs b/src/main/frontend/format/mldoc.cljs index ae0854234be..dc980225aee 100644 --- a/src/main/frontend/format/mldoc.cljs +++ b/src/main/frontend/format/mldoc.cljs @@ -1,15 +1,11 @@ (ns frontend.format.mldoc (:require [clojure.string :as string] [frontend.format.protocol :as protocol] - [frontend.utf8 :as utf8] [goog.object :as gobj] [lambdaisland.glogi :as log] - [medley.core :as medley] ["mldoc" :as mldoc :refer [Mldoc]] - [linked.core :as linked] [logseq.graph-parser.mldoc :as gp-mldoc] - [logseq.graph-parser.util :as gp-util] - [logseq.graph-parser.config :as gp-config])) + [logseq.graph-parser.util :as gp-util])) (defonce anchorLink (gobj/get Mldoc "anchorLink")) (defonce parseOPML (gobj/get Mldoc "parseOPML")) @@ -34,104 +30,6 @@ title (or references gp-mldoc/default-references))) -(defn remove-indentation-spaces - [s level remove-first-line?] - (let [lines (string/split-lines s) - [f & r] lines - body (map (fn [line] - (if (string/blank? (gp-util/safe-subs line 0 level)) - (gp-util/safe-subs line level) - line)) - (if remove-first-line? lines r)) - content (if remove-first-line? body (cons f body))] - (string/join "\n" content))) - -(defn- ->vec - [s] - (if (string? s) [s] s)) - -(defn- ->vec-concat - [& coll] - (->> (map ->vec coll) - (remove nil?) - (apply concat) - (distinct))) - -(defn collect-page-properties - [ast parse-property] - (if (seq ast) - (let [original-ast ast - ast (map first ast) ; without position meta - directive? (fn [[item _]] (= "directive" (string/lower-case (first item)))) - grouped-ast (group-by directive? original-ast) - directive-ast (take-while directive? original-ast) - [properties-ast other-ast] (if (= "Property_Drawer" (ffirst ast)) - [(last (first ast)) - (rest original-ast)] - [(->> (map first directive-ast) - (map rest)) - (get grouped-ast false)]) - properties (->> - properties-ast - (map (fn [[k v]] - (let [k (keyword (string/lower-case k)) - v (if (contains? #{:title :description :filters :macro} k) - v - (parse-property k v))] - [k v])))) - properties (into (linked/map) properties) - macro-properties (filter (fn [x] (= :macro (first x))) properties) - macros (if (seq macro-properties) - (->> - (map - (fn [[_ v]] - (let [[k v] (gp-util/split-first " " v)] - (mapv - string/trim - [k v]))) - macro-properties) - (into {})) - {}) - properties (->> (remove (fn [x] (= :macro (first x))) properties) - (into (linked/map))) - properties (cond-> properties - (seq macros) - (assoc :macros macros)) - alias (:alias properties) - alias (when alias - (if (coll? alias) - (remove string/blank? alias) - [alias])) - filetags (when-let [org-file-tags (:filetags properties)] - (->> (string/split org-file-tags ":") - (remove string/blank?))) - tags (:tags properties) - tags (->> (->vec-concat tags filetags) - (remove string/blank?)) - properties (assoc properties :tags tags :alias alias) - properties (-> properties - (update :filetags (constantly filetags))) - properties (medley/remove-kv (fn [_k v] (or (nil? v) (and (coll? v) (empty? v)))) properties)] - (if (seq properties) - (cons [["Properties" properties] nil] other-ast) - original-ast)) - ast)) - -(defn update-src-full-content - [ast content] - (let [content (utf8/encode content)] - (map (fn [[block pos-meta]] - (if (and (vector? block) - (= "Src" (first block))) - (let [{:keys [start_pos end_pos]} pos-meta - content (utf8/substring content start_pos end_pos) - spaces (re-find #"^[\t ]+" (first (string/split-lines content))) - content (if spaces (remove-indentation-spaces content (count spaces) true) - content) - block ["Src" (assoc (second block) :full_content content)]] - [block pos-meta]) - [block pos-meta])) ast))) - (defn block-with-title? [type] (contains? #{"Paragraph" @@ -139,50 +37,21 @@ "Hiccup" "Heading"} type)) -(def parse-property nil) - -(defn ->edn - [content config] - (if (string? content) - (try - (if (string/blank? content) - [] - (-> content - (gp-mldoc/parse-json config) - (gp-util/json->clj) - (update-src-full-content content) - (collect-page-properties parse-property))) - (catch js/Error e - (js/console.error e) - [])) - (log/error :edn/wrong-content-type content))) - (defn opml->edn [content] (try (if (string/blank? content) {} (let [[headers blocks] (-> content (parse-opml) (gp-util/json->clj))] - [headers (collect-page-properties blocks parse-property)])) + [headers (gp-mldoc/collect-page-properties blocks gp-mldoc/parse-property)])) (catch js/Error e (log/error :edn/convert-failed e) []))) -(defn inline->edn - [text config] - (try - (if (string/blank? text) - {} - (-> text - (gp-mldoc/inline-parse-json config) - (gp-util/json->clj))) - (catch js/Error _e - []))) - (defrecord MldocMode [] protocol/Format (toEdn [_this content config] - (->edn content config)) + (gp-mldoc/->edn content config)) (toHtml [_this content config references] (export "html" content config references)) (loaded? [_this] @@ -206,21 +75,3 @@ [ast typ] (and (contains? #{"Drawer"} (ffirst ast)) (= typ (second (first ast))))) - -(defn link? - [format link] - (when (string? link) - (let [[type link] (first (inline->edn link (gp-mldoc/default-config format))) - [ref-type ref-value] (:url link)] - (and (= "Link" type) - (or - ;; 1. url - (not (contains? #{"Page_ref" "Block_ref"} ref-type)) - - (and (contains? #{"Page_ref"} ref-type) - (or - ;; 2. excalidraw link - (gp-config/draw? ref-value) - - ;; 3. local asset link - (boolean (gp-config/local-asset? ref-value))))))))) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index fcc769b4d3f..4782e476a3e 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -357,7 +357,7 @@ content (drawer/with-logbook block content) content (with-timetracking block content) first-block? (= left page) - ast (mldoc/->edn (string/trim content) (gp-mldoc/default-config format)) + ast (gp-mldoc/->edn (string/trim content) (gp-mldoc/default-config format)) first-elem-type (first (ffirst ast)) first-elem-meta (second (ffirst ast)) properties? (contains? #{"Property_Drawer" "Properties"} first-elem-type) @@ -1942,7 +1942,7 @@ props (into [] (:properties block)) content* (str (if (= :markdown format) "- " "* ") (property/insert-properties format content props)) - ast (mldoc/->edn content* (gp-mldoc/default-config format)) + ast (gp-mldoc/->edn content* (gp-mldoc/default-config format)) blocks (block/extract-blocks ast content* true format) fst-block (first blocks)] (assert fst-block "fst-block shouldn't be nil") @@ -2851,7 +2851,7 @@ (when-let [editing-block (state/get-edit-block)] (let [page-id (:db/id (:block/page editing-block)) blocks (block/extract-blocks - (mldoc/->edn text (gp-mldoc/default-config format)) text true format) + (gp-mldoc/->edn text (gp-mldoc/default-config format)) text true format) blocks' (block/with-parent-and-left page-id blocks)] (paste-blocks blocks' {})))) @@ -3169,7 +3169,7 @@ [format content semantic?] (and (string/includes? content "\n") (if semantic? - (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) + (let [ast (gp-mldoc/->edn content (gp-mldoc/default-config format)) first-elem-type (first (ffirst ast))] (mldoc/block-with-title? first-elem-type)) true))) diff --git a/src/main/frontend/handler/export.cljs b/src/main/frontend/handler/export.cljs index 427d5749888..50f27864df5 100644 --- a/src/main/frontend/handler/export.cljs +++ b/src/main/frontend/handler/export.cljs @@ -15,7 +15,6 @@ [frontend.publishing.html :as html] [frontend.state :as state] [frontend.util :as util] - [frontend.format.mldoc :as mldoc] [logseq.graph-parser.mldoc :as gp-mldoc] [goog.dom :as gdom] [promesa.core :as p]) @@ -215,7 +214,7 @@ (let [block (db/entity [:block/uuid (uuid block-uuid)]) block-content (get-blocks-contents repo (:block/uuid block)) format (:block/format block) - ast (mldoc/->edn block-content (gp-mldoc/default-config format)) + ast (gp-mldoc/->edn block-content (gp-mldoc/default-config format)) embed-pages-new (get-embed-pages-from-ast ast) embed-blocks-new (get-embed-blocks-from-ast ast) block-refs-new (get-block-refs-from-ast ast) @@ -259,7 +258,7 @@ (let [page-name* (util/page-name-sanity-lc page-name) page-content (get-page-content repo page-name*) format (:block/format (db/entity [:block/name page-name*])) - ast (mldoc/->edn page-content (gp-mldoc/default-config format)) + ast (gp-mldoc/->edn page-content (gp-mldoc/default-config format)) embed-pages-new (get-embed-pages-from-ast ast) embed-blocks-new (get-embed-blocks-from-ast ast) block-refs-new (get-block-refs-from-ast ast) diff --git a/src/main/frontend/handler/extract.cljs b/src/main/frontend/handler/extract.cljs index cddde05c20e..20bb7e08a5e 100644 --- a/src/main/frontend/handler/extract.cljs +++ b/src/main/frontend/handler/extract.cljs @@ -7,7 +7,6 @@ [frontend.db :as db] [frontend.format :as format] [frontend.format.block :as block] - [frontend.format.mldoc :as mldoc] [frontend.state :as state] [frontend.text :as text] [frontend.util :as util] @@ -143,7 +142,7 @@ [] (let [format (format/get-format file) _ (println "Parsing start: " file) - ast (mldoc/->edn content (gp-mldoc/default-config format + ast (gp-mldoc/->edn content (gp-mldoc/default-config format ;; {:parse_outline_only? true} ))] (println "Parsing finished : " file) diff --git a/src/main/frontend/mobile/intent.cljs b/src/main/frontend/mobile/intent.cljs index fa84298429a..5516d9833ea 100644 --- a/src/main/frontend/mobile/intent.cljs +++ b/src/main/frontend/mobile/intent.cljs @@ -10,7 +10,7 @@ [frontend.date :as date] [frontend.util :as util] [frontend.config :as config] - [frontend.format.mldoc :as mldoc] + [logseq.graph-parser.mldoc :as gp-mldoc] ["path" :as path] [frontend.mobile.util :as mobile-util] [frontend.handler.notification :as notification] @@ -23,11 +23,11 @@ (string/lower-case (date/journal-name))) format (db/get-page-format page) time (date/get-current-time) - url (if (and (mldoc/link? format title) (not url)) + url (if (and (gp-mldoc/link? format title) (not url)) title url) text (if (= url title) nil title) - [text url] (if (or (mldoc/link? format url) (not url)) + [text url] (if (or (gp-mldoc/link? format url) (not url)) [text url] (string/split url "\"\n")) text (some-> text (string/replace #"^\"" "")) diff --git a/src/main/frontend/text.cljs b/src/main/frontend/text.cljs index 92b59f00c50..e8da33b87f0 100644 --- a/src/main/frontend/text.cljs +++ b/src/main/frontend/text.cljs @@ -2,8 +2,8 @@ (:require [frontend.config :as config] [frontend.util :as util] [clojure.string :as string] - [frontend.format.mldoc :as mldoc] [clojure.set :as set] + [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.util :as gp-util] [frontend.state :as state])) @@ -366,7 +366,7 @@ (contains? @non-parsing-properties (string/lower-case k)) v - (mldoc/link? format v) + (gp-mldoc/link? format v) v :else diff --git a/src/main/frontend/util/drawer.cljs b/src/main/frontend/util/drawer.cljs index cf3f81c84e6..3c742c3ec34 100644 --- a/src/main/frontend/util/drawer.cljs +++ b/src/main/frontend/util/drawer.cljs @@ -24,7 +24,7 @@ (defn get-drawer-ast [format content typ] - (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) + (let [ast (gp-mldoc/->edn content (gp-mldoc/default-config format)) typ-drawer (ffirst (filter (fn [x] (mldoc/typ-drawer? x typ)) ast))] typ-drawer)) @@ -33,7 +33,7 @@ [format content typ value] (when (string? content) (try - (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) + (let [ast (gp-mldoc/->edn content (gp-mldoc/default-config format)) has-properties? (some (fn [x] (mldoc/properties? x)) ast) has-typ-drawer? (some (fn [x] (mldoc/typ-drawer? x typ)) ast) lines (string/split-lines content) diff --git a/src/main/frontend/util/property.cljs b/src/main/frontend/util/property.cljs index 18402897644..7944cb48c79 100644 --- a/src/main/frontend/util/property.cljs +++ b/src/main/frontend/util/property.cljs @@ -188,7 +188,7 @@ properties (filter (fn [[k _v]] ((built-in-properties) k)) properties)] (if (seq properties) (let [lines (string/split-lines content) - ast (mldoc/->edn content (gp-mldoc/default-config format)) + ast (gp-mldoc/->edn content (gp-mldoc/default-config format)) [title body] (if (mldoc/block-with-title? (first (ffirst ast))) [(first lines) (rest lines)] [nil lines]) @@ -234,7 +234,7 @@ (insert-property format content key value false)) ([format content key value front-matter?] (when (string? content) - (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) + (let [ast (gp-mldoc/->edn content (gp-mldoc/default-config format)) title? (mldoc/block-with-title? (ffirst (map first ast))) has-properties? (or (and title? (mldoc/properties? (second ast))) diff --git a/src/main/logseq/graph_parser/log.cljs b/src/main/logseq/graph_parser/log.cljs new file mode 100644 index 00000000000..60ef312ae63 --- /dev/null +++ b/src/main/logseq/graph_parser/log.cljs @@ -0,0 +1,6 @@ +(ns logseq.graph-parser.log + "Minimal logging ns that implements basic lambdaisland.glogi fns. May use + glogi later if this ns is used more") + +(defn error [& msgs] + (apply js/console.error (map clj->js msgs))) diff --git a/src/main/logseq/graph_parser/mldoc.cljc b/src/main/logseq/graph_parser/mldoc.cljc index 8c04a6e1d79..a1ac73e054b 100644 --- a/src/main/logseq/graph_parser/mldoc.cljc +++ b/src/main/logseq/graph_parser/mldoc.cljc @@ -1,21 +1,27 @@ (ns ^:nbb-compatible logseq.graph-parser.mldoc + ;; Disable clj linters since we don't support clj + #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off} + :unresolved-symbol {:level :off}}}}) (:require #?(:org.babashka/nbb ["mldoc$default" :refer [Mldoc]] - :default ["mldoc" :refer [Mldoc]]) + :default ["mldoc" :refer [Mldoc]]) + #?(:org.babashka/nbb [logseq.graph-parser.log :as log] + :default [lambdaisland.glogi :as log]) [goog.object :as gobj] [cljs-bean.core :as bean] - [clojure.string :as string])) + [frontend.utf8 :as utf8] + [clojure.string :as string] + [linked.core :as linked] + [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.config :as gp-config])) (defonce parseJson (gobj/get Mldoc "parseJson")) (defonce parseInlineJson (gobj/get Mldoc "parseInlineJson")) (defonce astExportMarkdown (gobj/get Mldoc "astExportMarkdown")) -;; NOTE: All references to js/ have to have a :default conditional because of clojure (def default-references - #?(:cljs - (js/JSON.stringify - (clj->js {:embed_blocks [] - :embed_pages []})) - :default {})) + (js/JSON.stringify + (clj->js {:embed_blocks [] + :embed_pages []}))) (defn- convert-export-md-remove-options [opts] (->> opts @@ -58,4 +64,152 @@ (filter #(not (nil? (second %)))) (into {}) (bean/->js) - #?(:cljs js/JSON.stringify :default identity))))) + js/JSON.stringify)))) + +(defn remove-indentation-spaces + [s level remove-first-line?] + (let [lines (string/split-lines s) + [f & r] lines + body (map (fn [line] + (if (string/blank? (gp-util/safe-subs line 0 level)) + (gp-util/safe-subs line level) + line)) + (if remove-first-line? lines r)) + content (if remove-first-line? body (cons f body))] + (string/join "\n" content))) + +(defn- update-src-full-content + [ast content] + (let [content (utf8/encode content)] + (map (fn [[block pos-meta]] + (if (and (vector? block) + (= "Src" (first block))) + (let [{:keys [start_pos end_pos]} pos-meta + content (utf8/substring content start_pos end_pos) + spaces (re-find #"^[\t ]+" (first (string/split-lines content))) + content (if spaces (remove-indentation-spaces content (count spaces) true) + content) + block ["Src" (assoc (second block) :full_content content)]] + [block pos-meta]) + [block pos-meta])) ast))) + +(defn- ->vec + [s] + (if (string? s) [s] s)) + +(defn- ->vec-concat + [& coll] + (->> (map ->vec coll) + (remove nil?) + (apply concat) + (distinct))) + +(defn collect-page-properties + [ast parse-property] + (if (seq ast) + (let [original-ast ast + ast (map first ast) ; without position meta + directive? (fn [[item _]] (= "directive" (string/lower-case (first item)))) + grouped-ast (group-by directive? original-ast) + directive-ast (take-while directive? original-ast) + [properties-ast other-ast] (if (= "Property_Drawer" (ffirst ast)) + [(last (first ast)) + (rest original-ast)] + [(->> (map first directive-ast) + (map rest)) + (get grouped-ast false)]) + properties (->> + properties-ast + (map (fn [[k v]] + (let [k (keyword (string/lower-case k)) + v (if (contains? #{:title :description :filters :macro} k) + v + (parse-property k v))] + [k v])))) + properties (into (linked/map) properties) + macro-properties (filter (fn [x] (= :macro (first x))) properties) + macros (if (seq macro-properties) + (->> + (map + (fn [[_ v]] + (let [[k v] (gp-util/split-first " " v)] + (mapv + string/trim + [k v]))) + macro-properties) + (into {})) + {}) + properties (->> (remove (fn [x] (= :macro (first x))) properties) + (into (linked/map))) + properties (cond-> properties + (seq macros) + (assoc :macros macros)) + alias (:alias properties) + alias (when alias + (if (coll? alias) + (remove string/blank? alias) + [alias])) + filetags (when-let [org-file-tags (:filetags properties)] + (->> (string/split org-file-tags ":") + (remove string/blank?))) + tags (:tags properties) + tags (->> (->vec-concat tags filetags) + (remove string/blank?)) + properties (assoc properties :tags tags :alias alias) + properties (-> properties + (update :filetags (constantly filetags))) + properties (into (linked/map) + (remove (fn [[_k v]] + (or (nil? v) (and (coll? v) (empty? v)))) + properties))] + (if (seq properties) + (cons [["Properties" properties] nil] other-ast) + original-ast)) + ast)) + +(def parse-property nil) + +(defn ->edn + [content config] + (if (string? content) + (try + (if (string/blank? content) + [] + (-> content + (parse-json config) + (gp-util/json->clj) + (update-src-full-content content) + (collect-page-properties parse-property))) + (catch js/Error e + (js/console.error e) + [])) + (log/error :edn/wrong-content-type content))) + +(defn inline->edn + [text config] + (try + (if (string/blank? text) + {} + (-> text + (inline-parse-json config) + (gp-util/json->clj))) + (catch js/Error _e + []))) + +(defn link? + [format link] + (when (string? link) + (let [[type link] (first (inline->edn link (default-config format))) + [ref-type ref-value] (:url link)] + (and (= "Link" type) + (or + ;; 1. url + (not (contains? #{"Page_ref" "Block_ref"} ref-type)) + + (and (contains? #{"Page_ref"} ref-type) + (or + ;; 2. excalidraw link + (gp-config/draw? ref-value) + + ;; 3. local asset link + (boolean (gp-config/local-asset? ref-value))))))))) diff --git a/src/test/frontend/parser.cljs b/src/test/frontend/parser.cljs deleted file mode 100644 index f4df43ff339..00000000000 --- a/src/test/frontend/parser.cljs +++ /dev/null @@ -1,29 +0,0 @@ -(ns frontend.parser - (:require [cljs.test :refer [is deftest]] - [logseq.graph-parser.mldoc :as gp-mldoc] - [frontend.format.mldoc :as mldoc :refer [->edn]])) - -(def md-config (gp-mldoc/default-config :markdown)) - -(deftest src-test - (is (= - (first (->edn "``` -: hello -```" md-config)) - [["Src" - {:lines [": hello" "\n"], - :pos_meta {:start_pos 4, :end_pos 12}, - :full_content "```\n: hello\n```"}] - {:start_pos 0, :end_pos 15}]))) - -(deftest name-definition-test - (is (= - (first (->edn "term -: definition" md-config)) - [["List" - [{:content [["Paragraph" [["Plain" "definition"]]]], - :items [], - :name [["Plain" "term"]], - :indent 0, - :ordered false}]] - {:start_pos 0, :end_pos 17}]))) diff --git a/src/test/frontend/format/mldoc_test.cljs b/src/test/logseq/graph_parser/mldoc_test.cljs similarity index 70% rename from src/test/frontend/format/mldoc_test.cljs rename to src/test/logseq/graph_parser/mldoc_test.cljs index cadcb613735..693e6ced385 100644 --- a/src/test/frontend/format/mldoc_test.cljs +++ b/src/test/logseq/graph_parser/mldoc_test.cljs @@ -1,38 +1,38 @@ -(ns frontend.format.mldoc-test - (:require [frontend.format.mldoc :as mldoc] - [logseq.graph-parser.mldoc :as gp-mldoc] +(ns logseq.graph-parser.mldoc-test + (:require [logseq.graph-parser.mldoc :as gp-mldoc] ["fs" :as fs] ["child_process" :as child-process] [clojure.string :as string] [clojure.edn :as edn] + [frontend.format] [cljs.test :refer [testing deftest are is]])) (deftest test-link (testing "non-link" - (are [x y] (= (mldoc/link? :markdown x) y) + (are [x y] (= (gp-mldoc/link? :markdown x) y) "google.com" false)) (testing "plain links" - (are [x y] (= (mldoc/link? :markdown x) y) + (are [x y] (= (gp-mldoc/link? :markdown x) y) "http://www.google.com" true "http://google.com" true)) (testing "org links with labels" - (are [x y] (= (mldoc/link? :org x) y) + (are [x y] (= (gp-mldoc/link? :org x) y) "[[http://www.google.com][google]]" true "[[http://google.com][google]]" true "[[https://www.google.com][google]]" true "[[https://google.com][google]]" true)) (testing "org links without labels" - (are [x y] (= (mldoc/link? :org x) y) + (are [x y] (= (gp-mldoc/link? :org x) y) "[[http://www.google.com]]" true "[[https://www.google.com]]" true "[[draws/2022-03-06-15-00-28.excalidraw]]" true "[[assets/2022-03-06-15-00-28.pdf]]" true)) (testing "markdown links" - (are [x y] (= (mldoc/link? :markdown x) y) + (are [x y] (= (gp-mldoc/link? :markdown x) y) "[google](http://www.google.com)" true "[google](https://www.google.com)" true "[[draws/2022-03-06-15-00-28.excalidraw]]" true @@ -40,7 +40,7 @@ ;; https://github.com/logseq/logseq/issues/4308 (testing "parsing links should be finished" - (are [x y] (= (mldoc/link? :markdown x) y) + (are [x y] (= (gp-mldoc/link? :markdown x) y) "[YouTube](https://www.youtube.com/watch?v=-8ym7pyUs9gL) - [Vimeo](https://vimeo.com/677920303) {{youtube https://www.youtube.com/watch?v=-8ym7pyUs9g}}" true))) ;; TODO: Reuse with repo-test fns @@ -75,8 +75,8 @@ (let [format (if (string/ends-with? path ".org") :org :markdown)] [path - (mldoc/->edn content - (gp-mldoc/default-config format))]))) + (gp-mldoc/->edn content + (gp-mldoc/default-config format))]))) (into {}))] (is (= {"CommentBlock" 1, "Custom" 41, @@ -97,9 +97,35 @@ (->> asts-by-file (mapcat val) (map ffirst) frequencies)) "AST node type counts") + ; (println :DIFF) + ; (prn (butlast (clojure.data/diff (edn/read-string (slurp "mldoc-asts.edn")) + ; asts-by-file))) ;; This is just temporary (is (= (edn/read-string (slurp "mldoc-asts.edn")) asts-by-file) "Matches initial AST") #_(println "Wrote asts for" (count asts-by-file) "files") #_(fs/writeFileSync "mldoc-asts.edn" (pr-str asts-by-file)))) + +(def md-config (gp-mldoc/default-config :markdown)) + +(deftest src-test + (is (= [["Src" + {:lines [": hello" "\n"], + :pos_meta {:start_pos 4, :end_pos 12}, + :full_content "```\n: hello\n```"}] + {:start_pos 0, :end_pos 15}] + (first (gp-mldoc/->edn "``` +: hello +```" md-config))))) + +(deftest name-definition-test + (is (= [["List" + [{:content [["Paragraph" [["Plain" "definition"]]]], + :items [], + :name [["Plain" "term"]], + :indent 0, + :ordered false}]] + {:start_pos 0, :end_pos 17}] + (first (gp-mldoc/->edn "term +: definition" md-config))))) From f529777d782ff90ac0859f414b2e86fe4875a9f1 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Fri, 6 May 2022 17:12:11 -0400 Subject: [PATCH 03/58] frontend.text conversion WIP Removed extract-level-spaces b/c it's not used --- src/main/frontend/format/block.cljs | 2 +- src/main/frontend/handler/editor.cljs | 2 +- src/main/frontend/handler/route.cljs | 3 ++- src/main/frontend/handler/search.cljs | 5 +++-- src/main/frontend/text.cljs | 24 +++++++----------------- src/test/frontend/text_test.cljs | 22 +++++----------------- 6 files changed, 19 insertions(+), 39 deletions(-) diff --git a/src/main/frontend/format/block.cljs b/src/main/frontend/format/block.cljs index 28e03948053..d7c64e7b841 100644 --- a/src/main/frontend/format/block.cljs +++ b/src/main/frontend/format/block.cljs @@ -425,7 +425,7 @@ (utf8/substring utf8-content (:start_pos meta))) content (when content - (let [content (text/remove-level-spaces content format)] + (let [content (text/remove-level-spaces content format (config/get-block-pattern format))] (if (or (:pre-block? block) (= (:format block) :org)) content diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 4782e476a3e..f19f2e0be9c 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -1314,7 +1314,7 @@ (defn- clean-content! [format content] - (->> (text/remove-level-spaces content format) + (->> (text/remove-level-spaces content format (config/get-block-pattern format)) (drawer/remove-logbook) (property/remove-properties format) string/trim)) diff --git a/src/main/frontend/handler/route.cljs b/src/main/frontend/handler/route.cljs index ff132808a53..84359477748 100644 --- a/src/main/frontend/handler/route.cljs +++ b/src/main/frontend/handler/route.cljs @@ -1,5 +1,6 @@ (ns frontend.handler.route (:require [clojure.string :as string] + [frontend.config :as config] [frontend.date :as date] [frontend.db :as db] [frontend.handler.ui :as ui-handler] @@ -82,7 +83,7 @@ (if block? (if-let [block (db/entity [:block/uuid (medley/uuid name)])] (let [content (text/remove-level-spaces (:block/content block) - (:block/format block))] + (:block/format block) (config/get-block-pattern (:block/format block)))] (if (> (count content) 48) (str (subs content 0 48) "...") content)) diff --git a/src/main/frontend/handler/search.cljs b/src/main/frontend/handler/search.cljs index e8571edd6db..ddeaf150947 100644 --- a/src/main/frontend/handler/search.cljs +++ b/src/main/frontend/handler/search.cljs @@ -1,5 +1,6 @@ (ns frontend.handler.search (:require [clojure.string :as string] + [frontend.config :as config] [frontend.db :as db] [frontend.handler.notification :as notification-handler] [frontend.search :as search] @@ -21,7 +22,7 @@ (defn sanity-search-content "Convert a block to the display contents for searching" [format content] - (->> (text/remove-level-spaces content format) + (->> (text/remove-level-spaces content format (config/get-block-pattern format)) (drawer/remove-logbook) (property/remove-built-in-properties format))) @@ -78,4 +79,4 @@ (p/let [_ (search/rebuild-indices! repo)] (notification-handler/show! "Stale search cache detected. Search indices rebuilt successfully!" - :success)))))) \ No newline at end of file + :success)))))) diff --git a/src/main/frontend/text.cljs b/src/main/frontend/text.cljs index e8da33b87f0..2bb662d1bc8 100644 --- a/src/main/frontend/text.cljs +++ b/src/main/frontend/text.cljs @@ -1,6 +1,5 @@ (ns frontend.text - (:require [frontend.config :as config] - [frontend.util :as util] + (:require [frontend.util :as util] [clojure.string :as string] [clojure.set :as set] [logseq.graph-parser.mldoc :as gp-mldoc] @@ -187,15 +186,6 @@ :else s))) -(defn extract-level-spaces - [text format] - (if-not (string/blank? text) - (let [pattern (util/format - "^[%s]+\\s?" - (config/get-block-pattern format))] - (gp-util/safe-re-find (re-pattern pattern) text)) - "")) - (defn- remove-level-space-aux! [text pattern space? trim-left?] (let [pattern (util/format @@ -207,11 +197,11 @@ (string/replace-first text (re-pattern pattern) ""))) (defn remove-level-spaces - ([text format] - (remove-level-spaces text format false true)) - ([text format space?] - (remove-level-spaces text format space? true)) - ([text format space? trim-left?] + ([text format block-pattern] + (remove-level-spaces text format block-pattern false true)) + ([text format block-pattern space?] + (remove-level-spaces text format block-pattern space? true)) + ([text format block-pattern space? trim-left?] (when format (cond (string/blank? text) @@ -222,7 +212,7 @@ text :else - (remove-level-space-aux! text (config/get-block-pattern format) space? trim-left?))))) + (remove-level-space-aux! text block-pattern space? trim-left?))))) (defn build-data-value [col] diff --git a/src/test/frontend/text_test.cljs b/src/test/frontend/text_test.cljs index 9364de27133..387baa6d902 100644 --- a/src/test/frontend/text_test.cljs +++ b/src/test/frontend/text_test.cljs @@ -1,5 +1,6 @@ (ns frontend.text-test (:require [cljs.test :refer [are deftest testing]] + [frontend.config :as config] [frontend.text :as text])) (deftest test-get-page-name @@ -74,35 +75,22 @@ "#tag1,#tag2" #{"tag1" "tag2"} "[[Jan 26th, 2021]], hello" #{"hello" "Jan 26th, 2021"})) -(deftest extract-level-spaces - [] - (testing "markdown" - (are [x y] (= (text/extract-level-spaces x :markdown) y) - "- foobar" "- " - "-- foobar" "-- " - "--------------------- foobar" "--------------------- ")) - (testing "org mode" - (are [x y] (= (text/extract-level-spaces x :org) y) - "* foobar" "* " - "** foobar" "** " - "********************* foobar" "********************* "))) - (deftest remove-level-spaces [] (testing "markdown" - (are [x y] (= (text/remove-level-spaces x :markdown true) y) + (are [x y] (= (text/remove-level-spaces x :markdown (config/get-block-pattern :markdown) true) y) "- foobar" "foobar" " - foobar" "foobar")) (testing "markdown without spaces between the `#` and title" - (are [x y] (= (text/remove-level-spaces x :markdown) y) + (are [x y] (= (text/remove-level-spaces x :markdown (config/get-block-pattern :markdown)) y) "-foobar" "foobar")) (testing "org" - (are [x y] (= (text/remove-level-spaces x :org true) y) + (are [x y] (= (text/remove-level-spaces x :org (config/get-block-pattern :org) true) y) "* foobar" "foobar" "** foobar" "foobar" "********************* foobar" "foobar")) (testing "org without spaces between the `#` and title" - (are [x y] (= (text/remove-level-spaces x :org) y) + (are [x y] (= (text/remove-level-spaces x :org (config/get-block-pattern :org)) y) "*foobar" "foobar" "**foobar" "foobar" "*********************foobar" "foobar"))) From 65804b418391427d3c9a3d8fbe04d704d8ecce9c Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Mon, 9 May 2022 14:45:28 -0400 Subject: [PATCH 04/58] Split out text ns to graph-parser --- docs/dev-practices.md | 6 +-- src/main/frontend/components/block.cljs | 14 +++--- src/main/frontend/components/hierarchy.cljs | 2 +- src/main/frontend/components/journal.cljs | 2 +- src/main/frontend/components/page.cljs | 6 +-- src/main/frontend/components/repo.cljs | 2 +- src/main/frontend/components/select.cljs | 2 +- src/main/frontend/db/conn.cljs | 2 +- src/main/frontend/db/query_dsl.cljs | 7 +-- src/main/frontend/db/query_react.cljs | 2 +- src/main/frontend/diff.cljs | 2 +- src/main/frontend/external/roam.cljs | 2 +- src/main/frontend/format.cljs | 3 -- src/main/frontend/format/block.cljs | 4 +- src/main/frontend/format/mldoc.cljs | 10 ++++- src/main/frontend/handler/editor.cljs | 16 +++---- src/main/frontend/handler/export.cljs | 5 ++- src/main/frontend/handler/extract.cljs | 7 +-- src/main/frontend/handler/route.cljs | 2 +- src/main/frontend/handler/search.cljs | 2 +- src/main/frontend/util.cljc | 18 -------- src/main/frontend/util/drawer.cljs | 4 +- src/main/frontend/util/property.cljs | 6 +-- src/main/frontend/util/thingatpt.cljs | 2 +- src/main/logseq/graph_parser/mldoc.cljc | 8 ++-- .../graph_parser}/text.cljs | 44 +++++++++++-------- src/main/logseq/graph_parser/util.cljs | 32 ++++++++++++++ src/test/logseq/graph_parser/mldoc_test.cljs | 11 ++--- .../graph_parser}/text_test.cljs | 11 ++--- 29 files changed, 131 insertions(+), 103 deletions(-) rename src/main/{frontend => logseq/graph_parser}/text.cljs (90%) rename src/test/{frontend => logseq/graph_parser}/text_test.cljs (93%) diff --git a/docs/dev-practices.md b/docs/dev-practices.md index 86eed48a8b6..904df42fc0c 100644 --- a/docs/dev-practices.md +++ b/docs/dev-practices.md @@ -101,9 +101,9 @@ For this workflow: 1. Add `^:focus` metadata flags to tests e.g. `(deftest ^:focus test-name ...)`. 2. In another shell, run `node static/tests.js -i focus` to only run those tests. To run all tests except those tests run `node static/tests.js -e focus`. -3. Or focus namespaces: Using the regex option `-r`, run tests for `frontend.text-test` with `node static/tests.js -r text`. +3. Or focus namespaces: Using the regex option `-r`, run tests for `frontend.util.page-property-test` with `node static/tests.js -r page-property`. -Multiple options can be specified to AND selections. For example, to run all `frontend.text-test` tests except for the focused one: `node static/tests.js -r text -e focus` +Multiple options can be specified to AND selections. For example, to run all `frontend.util.page-property-test` tests except for the focused one: `node static/tests.js -r page-property -e focus` For help on more options, run `node static/tests.js -h`. @@ -114,7 +114,7 @@ shadow-cljs watch test --config-merge '{:autorun true}'`. The test output may appear where shadow-cljs was first invoked e.g. where `yarn watch` is running. Specific namespace(s) can be auto run with the `:ns-regexp` option e.g. `npx shadow-cljs watch test --config-merge '{:autorun true :ns-regexp -"frontend.text-test"}'`. +"frontend.util.page-property-test"}'`. ## Logging diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 8e56a5b573b..d2531084c07 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -45,7 +45,7 @@ [frontend.security :as security] [frontend.state :as state] [frontend.template :as template] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.ui :as ui] [frontend.util :as util] [frontend.util.clock :as clock] @@ -726,7 +726,7 @@ (defn- render-macro [config name arguments macro-content format] (if macro-content - (let [ast (->> (gp-mldoc/->edn macro-content (gp-mldoc/default-config format)) + (let [ast (->> (mldoc/->edn macro-content (gp-mldoc/default-config format)) (map first)) paragraph? (and (= 1 (count ast)) (= "Paragraph" (ffirst ast)))] @@ -853,7 +853,7 @@ (not (string/includes? s ".")) (page-reference (:html-export? config) s config label) - (util/url? s) + (gp-util/url? s) (->elem :a {:href s :data-href s :target "_blank"} @@ -1703,8 +1703,8 @@ (for [elem elems] (rum/with-key elem (str (random-uuid))))) - (and (string? v) (util/wrapped-by-quotes? v)) - (util/unquote-string v) + (and (string? v) (gp-util/wrapped-by-quotes? v)) + (gp-util/unquote-string v) :else (inline-text config (:block/format block) (str v)))])) @@ -2708,11 +2708,11 @@ ;; (cond ;; (= lang "quote") ;; (let [content (string/trim (string/join "\n" lines))] -;; ["Quote" (first (gp-mldoc/->edn content (gp-mldoc/default-config :markdown)))]) +;; ["Quote" (first (mldoc/->edn content (gp-mldoc/default-config :markdown)))]) ;; (contains? #{"query" "note" "tip" "important" "caution" "warning" "pinned"} lang) ;; (let [content (string/trim (string/join "\n" lines))] -;; ["Custom" lang nil (first (gp-mldoc/->edn content (gp-mldoc/default-config :markdown))) content]) +;; ["Custom" lang nil (first (mldoc/->edn content (gp-mldoc/default-config :markdown))) content]) ;; :else ;; ["Src" options]))) diff --git a/src/main/frontend/components/hierarchy.cljs b/src/main/frontend/components/hierarchy.cljs index 4d2a3facf30..a5e74d1c33d 100644 --- a/src/main/frontend/components/hierarchy.cljs +++ b/src/main/frontend/components/hierarchy.cljs @@ -4,7 +4,7 @@ [frontend.db :as db] [frontend.db.model :as db-model] [frontend.state :as state] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.ui :as ui] [medley.core :as medley] [rum.core :as rum] diff --git a/src/main/frontend/components/journal.cljs b/src/main/frontend/components/journal.cljs index e9496d66710..7ffda09f93a 100644 --- a/src/main/frontend/components/journal.cljs +++ b/src/main/frontend/components/journal.cljs @@ -8,7 +8,7 @@ [frontend.db.model :as model] [frontend.handler.page :as page-handler] [frontend.state :as state] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.ui :as ui] [frontend.util :as util] [goog.object :as gobj] diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 3b4224b8cd7..b19cce3a9e0 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -26,7 +26,7 @@ [frontend.handler.route :as route-handler] [frontend.mixins :as mixins] [frontend.state :as state] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.search :as search] [frontend.ui :as ui] [frontend.util :as util] @@ -215,8 +215,8 @@ (reset! *edit? false) (notification/show! "Illegal page name, can not rename!" :warning)) blur-fn (fn [e] - (when (util/wrapped-by-quotes? @*title-value) - (swap! *title-value util/unquote-string) + (when (gp-util/wrapped-by-quotes? @*title-value) + (swap! *title-value gp-util/unquote-string) (gobj/set (rum/deref input-ref) "value" @*title-value)) (state/set-state! :editor/editing-page-title? false) (cond diff --git a/src/main/frontend/components/repo.cljs b/src/main/frontend/components/repo.cljs index 4c3c99d202f..91349d1419b 100644 --- a/src/main/frontend/components/repo.cljs +++ b/src/main/frontend/components/repo.cljs @@ -14,7 +14,7 @@ [reitit.frontend.easy :as rfe] [rum.core :as rum] [frontend.mobile.util :as mobile-util] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [promesa.core :as p] [electron.ipc :as ipc] [goog.object :as gobj] diff --git a/src/main/frontend/components/select.cljs b/src/main/frontend/components/select.cljs index 01fc86af1d9..3f2ee11ad0e 100644 --- a/src/main/frontend/components/select.cljs +++ b/src/main/frontend/components/select.cljs @@ -10,7 +10,7 @@ [frontend.ui :as ui] [frontend.util :as util] [frontend.db :as db] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [rum.core :as rum] [frontend.config :as config] [frontend.handler.repo :as repo-handler] diff --git a/src/main/frontend/db/conn.cljs b/src/main/frontend/db/conn.cljs index dbed4e096af..bd87007dde1 100644 --- a/src/main/frontend/db/conn.cljs +++ b/src/main/frontend/db/conn.cljs @@ -7,7 +7,7 @@ [frontend.mobile.util :as mobile-util] [frontend.state :as state] [frontend.config :as config] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [logseq.graph-parser.util :as gp-util] [datascript.core :as d])) diff --git a/src/main/frontend/db/query_dsl.cljs b/src/main/frontend/db/query_dsl.cljs index 368e7a447c3..af90dbaa575 100644 --- a/src/main/frontend/db/query_dsl.cljs +++ b/src/main/frontend/db/query_dsl.cljs @@ -6,13 +6,14 @@ [clojure.set :as set] [clojure.string :as string] [clojure.walk :as walk] + [frontend.state :as state] [frontend.date :as date] [frontend.db.model :as model] [frontend.db.query-react :as query-react] [frontend.db.utils :as db-utils] [frontend.db.rules :as rules] [frontend.template :as template] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.util :as util] [logseq.graph-parser.util :as gp-util])) @@ -238,7 +239,7 @@ (let [k (string/replace (name (nth e 1)) "_" "-") v (nth e 2) v (if-not (nil? v) - (text/parse-property k v) + (text/parse-property k v (state/get-config)) v) v (if (coll? v) (first v) v)] {:query (list 'property '?b (keyword k) v) @@ -283,7 +284,7 @@ (let [[k v] (rest e) k (string/replace (name k) "_" "-")] (if (some? v) - (let [v' (text/parse-property k v) + (let [v' (text/parse-property k v (state/get-config)) val (if (coll? v') (first v') v')] {:query (list 'page-property '?p (keyword k) val) :rules [:page-property]}) diff --git a/src/main/frontend/db/query_react.cljs b/src/main/frontend/db/query_react.cljs index 44ba56e082b..a2bc02c684a 100644 --- a/src/main/frontend/db/query_react.cljs +++ b/src/main/frontend/db/query_react.cljs @@ -10,7 +10,7 @@ [frontend.debug :as debug] [frontend.extensions.sci :as sci] [frontend.state :as state] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.util :as util] [logseq.graph-parser.util :as gp-util] [lambdaisland.glogi :as log])) diff --git a/src/main/frontend/diff.cljs b/src/main/frontend/diff.cljs index 52970480a2a..53b52a0547d 100644 --- a/src/main/frontend/diff.cljs +++ b/src/main/frontend/diff.cljs @@ -6,7 +6,7 @@ [cljs-bean.core :as bean] [frontend.util :as util] [logseq.graph-parser.util :as gp-util] - [frontend.text :as text])) + [logseq.graph-parser.text :as text])) (defn diff [s1 s2] diff --git a/src/main/frontend/external/roam.cljs b/src/main/frontend/external/roam.cljs index 11c6c6b0ab7..d5e378dbcb1 100644 --- a/src/main/frontend/external/roam.cljs +++ b/src/main/frontend/external/roam.cljs @@ -7,7 +7,7 @@ [clojure.string :as string] [frontend.util :as util] [logseq.graph-parser.util :as gp-util] - [frontend.text :as text])) + [logseq.graph-parser.text :as text])) (defonce all-refed-uids (atom #{})) (defonce uid->uuid (atom {})) diff --git a/src/main/frontend/format.cljs b/src/main/frontend/format.cljs index e9c24b45f04..166a29e4ec3 100644 --- a/src/main/frontend/format.cljs +++ b/src/main/frontend/format.cljs @@ -2,12 +2,9 @@ (:require [frontend.format.mldoc :refer [->MldocMode] :as mldoc] [frontend.format.adoc :refer [->AdocMode]] [frontend.format.protocol :as protocol] - [frontend.text :as text] [logseq.graph-parser.mldoc :as gp-mldoc] [clojure.string :as string])) -(set! gp-mldoc/parse-property text/parse-property) - (defonce mldoc-record (->MldocMode)) (defonce adoc-record (->AdocMode)) diff --git a/src/main/frontend/format/block.cljs b/src/main/frontend/format/block.cljs index d7c64e7b841..f086bdbb590 100644 --- a/src/main/frontend/format/block.cljs +++ b/src/main/frontend/format/block.cljs @@ -7,7 +7,7 @@ [frontend.db :as db] [frontend.format :as format] [frontend.state :as state] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.utf8 :as utf8] [frontend.util :as util] [frontend.util.property :as property] @@ -185,7 +185,7 @@ (remove string/blank? v) (if (string/blank? v) nil - (text/parse-property format k v))) + (text/parse-property format k v (state/get-config)))) k (keyword k) v (if (and (string? v) diff --git a/src/main/frontend/format/mldoc.cljs b/src/main/frontend/format/mldoc.cljs index dc980225aee..a79a3142d74 100644 --- a/src/main/frontend/format/mldoc.cljs +++ b/src/main/frontend/format/mldoc.cljs @@ -1,6 +1,7 @@ (ns frontend.format.mldoc (:require [clojure.string :as string] [frontend.format.protocol :as protocol] + [frontend.state :as state] [goog.object :as gobj] [lambdaisland.glogi :as log] ["mldoc" :as mldoc :refer [Mldoc]] @@ -43,15 +44,20 @@ (if (string/blank? content) {} (let [[headers blocks] (-> content (parse-opml) (gp-util/json->clj))] - [headers (gp-mldoc/collect-page-properties blocks gp-mldoc/parse-property)])) + [headers (gp-mldoc/collect-page-properties blocks gp-mldoc/parse-property (state/get-config))])) (catch js/Error e (log/error :edn/convert-failed e) []))) +(defn ->edn + "Wrapper around gp-mldoc/->edn which provides config state" + [content config] + (gp-mldoc/->edn content config (state/get-config))) + (defrecord MldocMode [] protocol/Format (toEdn [_this content config] - (gp-mldoc/->edn content config)) + (->edn content config)) (toHtml [_this content config references] (export "html" content config references)) (loaded? [_this] diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index f19f2e0be9c..b5b9b7dcbb9 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -34,7 +34,7 @@ [frontend.search :as search] [frontend.state :as state] [frontend.template :as template] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.utf8 :as utf8] [frontend.util :as util :refer [profile]] [frontend.util.clock :as clock] @@ -357,7 +357,7 @@ content (drawer/with-logbook block content) content (with-timetracking block content) first-block? (= left page) - ast (gp-mldoc/->edn (string/trim content) (gp-mldoc/default-config format)) + ast (mldoc/->edn (string/trim content) (gp-mldoc/default-config format)) first-elem-type (first (ffirst ast)) first-elem-meta (second (ffirst ast)) properties? (contains? #{"Property_Drawer" "Properties"} first-elem-type) @@ -1942,7 +1942,7 @@ props (into [] (:properties block)) content* (str (if (= :markdown format) "- " "* ") (property/insert-properties format content props)) - ast (gp-mldoc/->edn content* (gp-mldoc/default-config format)) + ast (mldoc/->edn content* (gp-mldoc/default-config format)) blocks (block/extract-blocks ast content* true format) fst-block (first blocks)] (assert fst-block "fst-block shouldn't be nil") @@ -2851,7 +2851,7 @@ (when-let [editing-block (state/get-edit-block)] (let [page-id (:db/id (:block/page editing-block)) blocks (block/extract-blocks - (gp-mldoc/->edn text (gp-mldoc/default-config format)) text true format) + (mldoc/->edn text (gp-mldoc/default-config format)) text true format) blocks' (block/with-parent-and-left page-id blocks)] (paste-blocks blocks' {})))) @@ -2902,17 +2902,17 @@ (state/set-copied-full-blocks! blocks) (paste-blocks blocks {}))) - (and (util/url? text) + (and (gp-util/url? text) (not (string/blank? (util/get-selected-text)))) (html-link-format! text) - (and (util/url? text) + (and (gp-util/url? text) (or (string/includes? text "youtube.com") (string/includes? text "youtu.be")) (mobile-util/is-native-platform?)) (commands/simple-insert! (state/get-edit-input-id) (util/format "{{youtube %s}}" text) nil) - (and (util/url? text) + (and (gp-util/url? text) (string/includes? text "twitter.com") (mobile-util/is-native-platform?)) (commands/simple-insert! (state/get-edit-input-id) (util/format "{{twitter %s}}" text) nil) @@ -3169,7 +3169,7 @@ [format content semantic?] (and (string/includes? content "\n") (if semantic? - (let [ast (gp-mldoc/->edn content (gp-mldoc/default-config format)) + (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) first-elem-type (first (ffirst ast))] (mldoc/block-with-title? first-elem-type)) true))) diff --git a/src/main/frontend/handler/export.cljs b/src/main/frontend/handler/export.cljs index 50f27864df5..427d5749888 100644 --- a/src/main/frontend/handler/export.cljs +++ b/src/main/frontend/handler/export.cljs @@ -15,6 +15,7 @@ [frontend.publishing.html :as html] [frontend.state :as state] [frontend.util :as util] + [frontend.format.mldoc :as mldoc] [logseq.graph-parser.mldoc :as gp-mldoc] [goog.dom :as gdom] [promesa.core :as p]) @@ -214,7 +215,7 @@ (let [block (db/entity [:block/uuid (uuid block-uuid)]) block-content (get-blocks-contents repo (:block/uuid block)) format (:block/format block) - ast (gp-mldoc/->edn block-content (gp-mldoc/default-config format)) + ast (mldoc/->edn block-content (gp-mldoc/default-config format)) embed-pages-new (get-embed-pages-from-ast ast) embed-blocks-new (get-embed-blocks-from-ast ast) block-refs-new (get-block-refs-from-ast ast) @@ -258,7 +259,7 @@ (let [page-name* (util/page-name-sanity-lc page-name) page-content (get-page-content repo page-name*) format (:block/format (db/entity [:block/name page-name*])) - ast (gp-mldoc/->edn page-content (gp-mldoc/default-config format)) + ast (mldoc/->edn page-content (gp-mldoc/default-config format)) embed-pages-new (get-embed-pages-from-ast ast) embed-blocks-new (get-embed-blocks-from-ast ast) block-refs-new (get-block-refs-from-ast ast) diff --git a/src/main/frontend/handler/extract.cljs b/src/main/frontend/handler/extract.cljs index 20bb7e08a5e..93fd5fdcdfa 100644 --- a/src/main/frontend/handler/extract.cljs +++ b/src/main/frontend/handler/extract.cljs @@ -7,8 +7,9 @@ [frontend.db :as db] [frontend.format :as format] [frontend.format.block :as block] + [frontend.format.mldoc :as mldoc] [frontend.state :as state] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.util :as util] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] @@ -142,7 +143,7 @@ [] (let [format (format/get-format file) _ (println "Parsing start: " file) - ast (gp-mldoc/->edn content (gp-mldoc/default-config format + ast (mldoc/->edn content (gp-mldoc/default-config format ;; {:parse_outline_only? true} ))] (println "Parsing finished : " file) @@ -151,7 +152,7 @@ (->> (last first-block) (map (fn [[x y]] [x (if (string? y) - (text/parse-property format x y) + (text/parse-property format x y (state/get-config)) y)])) (into {}) (walk/keywordize-keys)))] diff --git a/src/main/frontend/handler/route.cljs b/src/main/frontend/handler/route.cljs index 84359477748..53bfc9147a3 100644 --- a/src/main/frontend/handler/route.cljs +++ b/src/main/frontend/handler/route.cljs @@ -7,7 +7,7 @@ [frontend.handler.recent :as recent-handler] [frontend.handler.search :as search-handler] [frontend.state :as state] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.util :as util] [logseq.graph-parser.util :as gp-util] [medley.core :as medley] diff --git a/src/main/frontend/handler/search.cljs b/src/main/frontend/handler/search.cljs index ddeaf150947..41d991bfd15 100644 --- a/src/main/frontend/handler/search.cljs +++ b/src/main/frontend/handler/search.cljs @@ -7,7 +7,7 @@ [frontend.state :as state] [frontend.util :as util] [promesa.core :as p] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.util.drawer :as drawer] [frontend.util.property :as property])) diff --git a/src/main/frontend/util.cljc b/src/main/frontend/util.cljc index adb3cb253ca..cb91e56d7e3 100644 --- a/src/main/frontend/util.cljc +++ b/src/main/frontend/util.cljc @@ -1258,14 +1258,6 @@ (defn meta-key-name [] (if mac? "Cmd" "Ctrl"))) -(defn wrapped-by-quotes? - [v] - (and (string? v) (>= (count v) 2) (= "\"" (first v) (last v)))) - -(defn unquote-string - [v] - (string/trim (subs v 1 (dec (count v))))) - #?(:cljs (defn right-click? [e] @@ -1274,16 +1266,6 @@ (or (= which 3) (= button 2))))) -#?(:cljs - (defn url? - [s] - (and (string? s) - (try - (js/URL. s) - true - (catch js/Error _e - false))))) - #?(:cljs (defn make-el-into-center-viewport [^js/HTMLElement el] diff --git a/src/main/frontend/util/drawer.cljs b/src/main/frontend/util/drawer.cljs index 3c742c3ec34..cf3f81c84e6 100644 --- a/src/main/frontend/util/drawer.cljs +++ b/src/main/frontend/util/drawer.cljs @@ -24,7 +24,7 @@ (defn get-drawer-ast [format content typ] - (let [ast (gp-mldoc/->edn content (gp-mldoc/default-config format)) + (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) typ-drawer (ffirst (filter (fn [x] (mldoc/typ-drawer? x typ)) ast))] typ-drawer)) @@ -33,7 +33,7 @@ [format content typ value] (when (string? content) (try - (let [ast (gp-mldoc/->edn content (gp-mldoc/default-config format)) + (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) has-properties? (some (fn [x] (mldoc/properties? x)) ast) has-typ-drawer? (some (fn [x] (mldoc/typ-drawer? x typ)) ast) lines (string/split-lines content) diff --git a/src/main/frontend/util/property.cljs b/src/main/frontend/util/property.cljs index 7944cb48c79..7bcacf12ed2 100644 --- a/src/main/frontend/util/property.cljs +++ b/src/main/frontend/util/property.cljs @@ -7,7 +7,7 @@ [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] [frontend.format.mldoc :as mldoc] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [frontend.util.cursor :as cursor])) (defonce properties-start ":PROPERTIES:") @@ -188,7 +188,7 @@ properties (filter (fn [[k _v]] ((built-in-properties) k)) properties)] (if (seq properties) (let [lines (string/split-lines content) - ast (gp-mldoc/->edn content (gp-mldoc/default-config format)) + ast (mldoc/->edn content (gp-mldoc/default-config format)) [title body] (if (mldoc/block-with-title? (first (ffirst ast))) [(first lines) (rest lines)] [nil lines]) @@ -234,7 +234,7 @@ (insert-property format content key value false)) ([format content key value front-matter?] (when (string? content) - (let [ast (gp-mldoc/->edn content (gp-mldoc/default-config format)) + (let [ast (mldoc/->edn content (gp-mldoc/default-config format)) title? (mldoc/block-with-title? (ffirst (map first ast))) has-properties? (or (and title? (mldoc/properties? (second ast))) diff --git a/src/main/frontend/util/thingatpt.cljs b/src/main/frontend/util/thingatpt.cljs index 1f66fb0aedb..e0690328de1 100644 --- a/src/main/frontend/util/thingatpt.cljs +++ b/src/main/frontend/util/thingatpt.cljs @@ -4,7 +4,7 @@ [frontend.util.property :as property-util] [frontend.util.cursor :as cursor] [frontend.config :as config] - [frontend.text :as text] + [logseq.graph-parser.text :as text] [cljs.reader :as reader] [goog.object :as gobj])) diff --git a/src/main/logseq/graph_parser/mldoc.cljc b/src/main/logseq/graph_parser/mldoc.cljc index a1ac73e054b..133446fe37d 100644 --- a/src/main/logseq/graph_parser/mldoc.cljc +++ b/src/main/logseq/graph_parser/mldoc.cljc @@ -105,7 +105,7 @@ (distinct))) (defn collect-page-properties - [ast parse-property] + [ast parse-property config-state] (if (seq ast) (let [original-ast ast ast (map first ast) ; without position meta @@ -124,7 +124,7 @@ (let [k (keyword (string/lower-case k)) v (if (contains? #{:title :description :filters :macro} k) v - (parse-property k v))] + (parse-property k v config-state))] [k v])))) properties (into (linked/map) properties) macro-properties (filter (fn [x] (= :macro (first x))) properties) @@ -170,7 +170,7 @@ (def parse-property nil) (defn ->edn - [content config] + [content config config-state] (if (string? content) (try (if (string/blank? content) @@ -179,7 +179,7 @@ (parse-json config) (gp-util/json->clj) (update-src-full-content content) - (collect-page-properties parse-property))) + (collect-page-properties parse-property config-state))) (catch js/Error e (js/console.error e) [])) diff --git a/src/main/frontend/text.cljs b/src/main/logseq/graph_parser/text.cljs similarity index 90% rename from src/main/frontend/text.cljs rename to src/main/logseq/graph_parser/text.cljs index 2bb662d1bc8..4e0402ff782 100644 --- a/src/main/frontend/text.cljs +++ b/src/main/logseq/graph_parser/text.cljs @@ -1,10 +1,10 @@ -(ns frontend.text - (:require [frontend.util :as util] +(ns ^:nbb-compatible logseq.graph-parser.text + (:require ["path" :as path] + [goog.string :as gstring] [clojure.string :as string] [clojure.set :as set] [logseq.graph-parser.mldoc :as gp-mldoc] - [logseq.graph-parser.util :as gp-util] - [frontend.state :as state])) + [logseq.graph-parser.util :as gp-util])) (def page-ref-re-0 #"\[\[(.*)\]\]") (def org-page-ref-re #"\[\[(file:.*)\]\[.+?\]\]") @@ -13,7 +13,8 @@ (defn get-file-basename [path] (when-not (string/blank? path) - (util/node-path.name path))) + ;; Same as util/node-path.name + (.-name (path/parse (string/replace path "+" "/"))))) (defn get-page-name [s] @@ -122,14 +123,15 @@ (string/split s #"(\"[^\"]*\")")) (def markdown-link #"\[([^\[]+)\](\(.*\))") + (defn split-page-refs-without-brackets ([s] (split-page-refs-without-brackets s {})) ([s {:keys [un-brackets?] :or {un-brackets? true}}] (cond - (and (string? s) (util/wrapped-by-quotes? s)) - (util/unquote-string s) + (and (string? s) (gp-util/wrapped-by-quotes? s)) + (gp-util/unquote-string s) (and (string? s) (re-find markdown-link s)) s @@ -141,11 +143,11 @@ (let [result (->> (sep-by-quotes s) (mapcat (fn [s] - (when-not (util/wrapped-by-quotes? (string/trim s)) + (when-not (gp-util/wrapped-by-quotes? (string/trim s)) (string/split s page-ref-re-2)))) (mapcat (fn [s] (cond - (util/wrapped-by-quotes? s) + (gp-util/wrapped-by-quotes? s) nil (string/includes? (string/trimr s) "]],") @@ -165,7 +167,7 @@ (remove string/blank?) (mapcat (fn [s] (cond - (util/wrapped-by-quotes? s) + (gp-util/wrapped-by-quotes? s) nil (page-ref? s) @@ -188,7 +190,7 @@ (defn- remove-level-space-aux! [text pattern space? trim-left?] - (let [pattern (util/format + (let [pattern (gstring/format (if space? "^[%s]+\\s+" "^[%s]+\\s?") @@ -217,7 +219,7 @@ (defn build-data-value [col] (let [items (map (fn [item] (str "\"" item "\"")) col)] - (util/format "[%s]" + (gstring/format "[%s]" (string/join ", " items)))) (defn media-link? @@ -230,7 +232,7 @@ (string/includes? p "/") (not (string/starts-with? p "../")) (not (string/starts-with? p "./")) - (not (util/url? p)))) + (not (gp-util/url? p)))) (defn add-timestamp [content key value] @@ -330,16 +332,16 @@ (atom #{"background-color" "background_color"})) (defn parse-property - ([k v] - (parse-property :markdown k v)) - ([format k v] + ([k v config-state] + (parse-property :markdown k v config-state)) + ([format k v config-state] (let [k (name k) v (if (or (symbol? v) (keyword? v)) (name v) (str v)) v (string/trim v)] (cond (contains? (set/union #{"title" "filters"} - (get (state/get-config) :ignored-page-references-keywords)) k) + (get config-state :ignored-page-references-keywords)) k) v (= v "true") @@ -348,9 +350,9 @@ false (and (not= k "alias") (gp-util/safe-re-find #"^\d+$" v)) - (util/safe-parse-int v) + (gp-util/safe-parse-int v) - (util/wrapped-by-quotes? v) ; wrapped in "" + (gp-util/wrapped-by-quotes? v) ; wrapped in "" v (contains? @non-parsing-properties (string/lower-case k)) @@ -361,3 +363,7 @@ :else (split-page-refs-without-brackets v))))) + +;; TODO: Properly fix this circular dependency: +;; mldoc/->edn > text/parse-property > mldoc/link? ->mldoc/inline->edn + mldoc/default-config +(set! gp-mldoc/parse-property parse-property) diff --git a/src/main/logseq/graph_parser/util.cljs b/src/main/logseq/graph_parser/util.cljs index b3e431708c8..d98898049a9 100644 --- a/src/main/logseq/graph_parser/util.cljs +++ b/src/main/logseq/graph_parser/util.cljs @@ -57,6 +57,38 @@ (let [c (count s)] (subs s (min c start) (min c end))))) +(defn unquote-string + [v] + (string/trim (subs v 1 (dec (count v))))) + +(defn wrapped-by-quotes? + [v] + (and (string? v) (>= (count v) 2) (= "\"" (first v) (last v)))) + +(defn parse-int + "Copy of frontend.util/parse-int. Don't want to couple to main app too much" + [x] + (if (string? x) + (js/parseInt x) + x)) + +(defn safe-parse-int + "Copy of frontend.util/safe-parse-int. Don't want to couple to main app too much" + [x] + (let [result (parse-int x)] + (if (js/isNaN result) + nil + result))) + +(defn url? + [s] + (and (string? s) + (try + (js/URL. s) + true + (catch js/Error _e + false)))) + (defn json->clj [json-string] (-> json-string diff --git a/src/test/logseq/graph_parser/mldoc_test.cljs b/src/test/logseq/graph_parser/mldoc_test.cljs index 693e6ced385..31ada8f5ef6 100644 --- a/src/test/logseq/graph_parser/mldoc_test.cljs +++ b/src/test/logseq/graph_parser/mldoc_test.cljs @@ -76,7 +76,8 @@ :org :markdown)] [path (gp-mldoc/->edn content - (gp-mldoc/default-config format))]))) + (gp-mldoc/default-config format) + {})]))) (into {}))] (is (= {"CommentBlock" 1, "Custom" 41, @@ -102,8 +103,8 @@ ; asts-by-file))) ;; This is just temporary (is (= (edn/read-string (slurp "mldoc-asts.edn")) - asts-by-file) - "Matches initial AST") + asts-by-file) + "Matches initial AST") #_(println "Wrote asts for" (count asts-by-file) "files") #_(fs/writeFileSync "mldoc-asts.edn" (pr-str asts-by-file)))) @@ -117,7 +118,7 @@ {:start_pos 0, :end_pos 15}] (first (gp-mldoc/->edn "``` : hello -```" md-config))))) +```" md-config {}))))) (deftest name-definition-test (is (= [["List" @@ -128,4 +129,4 @@ :ordered false}]] {:start_pos 0, :end_pos 17}] (first (gp-mldoc/->edn "term -: definition" md-config))))) +: definition" md-config {}))))) diff --git a/src/test/frontend/text_test.cljs b/src/test/logseq/graph_parser/text_test.cljs similarity index 93% rename from src/test/frontend/text_test.cljs rename to src/test/logseq/graph_parser/text_test.cljs index 387baa6d902..2db2cb42d46 100644 --- a/src/test/frontend/text_test.cljs +++ b/src/test/logseq/graph_parser/text_test.cljs @@ -1,7 +1,8 @@ -(ns frontend.text-test +(ns logseq.graph-parser.text-test (:require [cljs.test :refer [are deftest testing]] [frontend.config :as config] - [frontend.text :as text])) + [frontend.state :as state] + [logseq.graph-parser.text :as text])) (deftest test-get-page-name [] @@ -127,7 +128,7 @@ (deftest test-parse-property (testing "parse-property" - (are [k v y] (= (text/parse-property k v) y) + (are [k v y] (= (text/parse-property k v (state/get-config)) y) :tags "foo" "foo" :tags "foo, bar" #{"foo" "bar"} :tags "foo,bar" #{"foo" "bar"} @@ -139,9 +140,9 @@ :tags "[[foo [[bar]]]]" #{"foo [[bar]]"} :tags "[[foo [[bar]]]], baz" #{"baz" "foo [[bar]]"})) (testing "parse-property with quoted strings" - (are [k v y] (= (text/parse-property k v) y) + (are [k v y] (= (text/parse-property k v (state/get-config)) y) :tags "\"foo, bar\"" "\"foo, bar\"" :tags "\"[[foo]], [[bar]]\"" "\"[[foo]], [[bar]]\"" :tags "baz, \"[[foo]], [[bar]]\"" #{"baz"}))) -#_(cljs.test/test-ns 'frontend.text-test) +#_(cljs.test/test-ns 'logseq.graph-parser.text-test) From 20b8b9624d0750e5dcafd162306e5c2f87b58fcb Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Mon, 9 May 2022 16:38:59 -0400 Subject: [PATCH 05/58] Moved nbb ns into graph-parser. nbb mldoc tests wip --- src/main/frontend/encrypt.cljs | 2 +- src/main/frontend/extensions/code.cljs | 2 +- src/main/frontend/format/block.cljs | 2 +- src/main/frontend/handler/editor.cljs | 2 +- src/main/logseq/graph_parser/mldoc.cljc | 2 +- src/main/logseq/graph_parser/text.cljs | 1 + src/main/{frontend => logseq/graph_parser}/utf8.cljs | 2 +- src/test/logseq/graph_parser/mldoc_test.cljs | 5 +++-- src/test/logseq/test/nbb_test_runner.cljs | 11 +++++++++++ 9 files changed, 21 insertions(+), 8 deletions(-) rename src/main/{frontend => logseq/graph_parser}/utf8.cljs (90%) create mode 100644 src/test/logseq/test/nbb_test_runner.cljs diff --git a/src/main/frontend/encrypt.cljs b/src/main/frontend/encrypt.cljs index b5be1631539..60d651da2e8 100644 --- a/src/main/frontend/encrypt.cljs +++ b/src/main/frontend/encrypt.cljs @@ -1,5 +1,5 @@ (ns frontend.encrypt - (:require [frontend.utf8 :as utf8] + (:require [logseq.graph-parser.utf8 :as utf8] [frontend.db.utils :as db-utils] [frontend.db :as db] [promesa.core :as p] diff --git a/src/main/frontend/extensions/code.cljs b/src/main/frontend/extensions/code.cljs index f8227b20d00..206735a9e23 100644 --- a/src/main/frontend/extensions/code.cljs +++ b/src/main/frontend/extensions/code.cljs @@ -132,7 +132,7 @@ [frontend.handler.editor :as editor-handler] [frontend.handler.file :as file-handler] [frontend.state :as state] - [frontend.utf8 :as utf8] + [logseq.graph-parser.utf8 :as utf8] [frontend.util :as util] [frontend.config :as config] [goog.dom :as gdom] diff --git a/src/main/frontend/format/block.cljs b/src/main/frontend/format/block.cljs index f086bdbb590..9da64520aec 100644 --- a/src/main/frontend/format/block.cljs +++ b/src/main/frontend/format/block.cljs @@ -8,7 +8,7 @@ [frontend.format :as format] [frontend.state :as state] [logseq.graph-parser.text :as text] - [frontend.utf8 :as utf8] + [logseq.graph-parser.utf8 :as utf8] [frontend.util :as util] [frontend.util.property :as property] [logseq.graph-parser.util :as gp-util] diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index b5b9b7dcbb9..70dc0ae9a2a 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -35,7 +35,7 @@ [frontend.state :as state] [frontend.template :as template] [logseq.graph-parser.text :as text] - [frontend.utf8 :as utf8] + [logseq.graph-parser.utf8 :as utf8] [frontend.util :as util :refer [profile]] [frontend.util.clock :as clock] [frontend.util.cursor :as cursor] diff --git a/src/main/logseq/graph_parser/mldoc.cljc b/src/main/logseq/graph_parser/mldoc.cljc index 133446fe37d..fa4d9b446d7 100644 --- a/src/main/logseq/graph_parser/mldoc.cljc +++ b/src/main/logseq/graph_parser/mldoc.cljc @@ -8,7 +8,7 @@ :default [lambdaisland.glogi :as log]) [goog.object :as gobj] [cljs-bean.core :as bean] - [frontend.utf8 :as utf8] + [logseq.graph-parser.utf8 :as utf8] [clojure.string :as string] [linked.core :as linked] [logseq.graph-parser.util :as gp-util] diff --git a/src/main/logseq/graph_parser/text.cljs b/src/main/logseq/graph_parser/text.cljs index 4e0402ff782..fa0730cdbd7 100644 --- a/src/main/logseq/graph_parser/text.cljs +++ b/src/main/logseq/graph_parser/text.cljs @@ -367,3 +367,4 @@ ;; TODO: Properly fix this circular dependency: ;; mldoc/->edn > text/parse-property > mldoc/link? ->mldoc/inline->edn + mldoc/default-config (set! gp-mldoc/parse-property parse-property) +; (alter-var-root #'gp-mldoc/parse-property (constantly parse-property)) diff --git a/src/main/frontend/utf8.cljs b/src/main/logseq/graph_parser/utf8.cljs similarity index 90% rename from src/main/frontend/utf8.cljs rename to src/main/logseq/graph_parser/utf8.cljs index a464314cd74..9f310091ca2 100644 --- a/src/main/frontend/utf8.cljs +++ b/src/main/logseq/graph_parser/utf8.cljs @@ -1,4 +1,4 @@ -(ns ^:nbb-compatible frontend.utf8 +(ns ^:nbb-compatible logseq.graph-parser.utf8 (:require [goog.object :as gobj])) (defonce encoder diff --git a/src/test/logseq/graph_parser/mldoc_test.cljs b/src/test/logseq/graph_parser/mldoc_test.cljs index 31ada8f5ef6..6b1e79682a7 100644 --- a/src/test/logseq/graph_parser/mldoc_test.cljs +++ b/src/test/logseq/graph_parser/mldoc_test.cljs @@ -4,7 +4,8 @@ ["child_process" :as child-process] [clojure.string :as string] [clojure.edn :as edn] - [frontend.format] + ;; hack needed for parse-property to exist + [logseq.graph-parser.text] [cljs.test :refer [testing deftest are is]])) (deftest test-link @@ -102,7 +103,7 @@ ; (prn (butlast (clojure.data/diff (edn/read-string (slurp "mldoc-asts.edn")) ; asts-by-file))) ;; This is just temporary - (is (= (edn/read-string (slurp "mldoc-asts.edn")) + #_(is (= (edn/read-string (slurp "mldoc-asts.edn")) asts-by-file) "Matches initial AST") #_(println "Wrote asts for" (count asts-by-file) "files") diff --git a/src/test/logseq/test/nbb_test_runner.cljs b/src/test/logseq/test/nbb_test_runner.cljs new file mode 100644 index 00000000000..5d18e3898bd --- /dev/null +++ b/src/test/logseq/test/nbb_test_runner.cljs @@ -0,0 +1,11 @@ +(ns logseq.test.nbb-test-runner + (:require [cljs.test :as t] + [logseq.graph-parser.mldoc-test])) + +(defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m] + (when-not (cljs.test/successful? m) + (set! (.-exitCode js/process) 1))) + +;; run this function with: nbb-logseq -m logseq.test.nbb-test-runner/run-tests +(defn run-tests [] + (t/run-tests 'logseq.graph-parser.mldoc-test)) From b79b1ca36e91787fd77b697ef0868ecea0905328 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 10 May 2022 12:20:02 -0400 Subject: [PATCH 06/58] Mldoc tests run under nbb --- .clj-kondo/config.edn | 1 + .github/workflows/build.yml | 4 +- src/main/frontend/format.cljs | 5 ++ src/main/logseq/graph_parser/text.cljs | 5 -- src/test/logseq/graph_parser/mldoc_test.cljs | 3 +- .../logseq/graph_parser/nbb_test_runner.cljs | 21 ++++++++ src/test/logseq/graph_parser/text_test.cljs | 48 ++++++++++--------- src/test/logseq/test/nbb_test_runner.cljs | 11 ----- 8 files changed, 56 insertions(+), 42 deletions(-) create mode 100644 src/test/logseq/graph_parser/nbb_test_runner.cljs delete mode 100644 src/test/logseq/test/nbb_test_runner.cljs diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 3df6e13d09d..c671f8b73f7 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -21,6 +21,7 @@ frontend.util util frontend.config config frontend.format.mldoc mldoc + logseq.graph-parser.text text logseq.graph-parser.mldoc gp-mldoc logseq.graph-parser.util gp-util logseq.graph-parser.config gp-config}}} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a43d55f5b86..257130b4a66 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,11 +74,13 @@ jobs: - name: Fetch yarn deps run: yarn install --frozen-lockfile - - name: Run ClojureScript test + - name: Run ClojureScript tests run: | yarn cljs:test node static/tests.js + - name: Run nbb tests for graph-parser + run: yarn nbb-logseq -cp src/main:src/test -m logseq.graph-parser.nbb-test-runner/run-tests # In this job because it depends on an npm package - name: Load nbb compatible namespaces run: bb test:load-nbb-compatible-namespaces diff --git a/src/main/frontend/format.cljs b/src/main/frontend/format.cljs index 166a29e4ec3..6638ce1ebfa 100644 --- a/src/main/frontend/format.cljs +++ b/src/main/frontend/format.cljs @@ -3,8 +3,13 @@ [frontend.format.adoc :refer [->AdocMode]] [frontend.format.protocol :as protocol] [logseq.graph-parser.mldoc :as gp-mldoc] + [logseq.graph-parser.text :as text] [clojure.string :as string])) +;; TODO: Properly fix this circular dependency: +;; mldoc/->edn > text/parse-property > mldoc/link? ->mldoc/inline->edn + mldoc/default-config +(set! gp-mldoc/parse-property text/parse-property) + (defonce mldoc-record (->MldocMode)) (defonce adoc-record (->AdocMode)) diff --git a/src/main/logseq/graph_parser/text.cljs b/src/main/logseq/graph_parser/text.cljs index fa0730cdbd7..a25d5b14b66 100644 --- a/src/main/logseq/graph_parser/text.cljs +++ b/src/main/logseq/graph_parser/text.cljs @@ -363,8 +363,3 @@ :else (split-page-refs-without-brackets v))))) - -;; TODO: Properly fix this circular dependency: -;; mldoc/->edn > text/parse-property > mldoc/link? ->mldoc/inline->edn + mldoc/default-config -(set! gp-mldoc/parse-property parse-property) -; (alter-var-root #'gp-mldoc/parse-property (constantly parse-property)) diff --git a/src/test/logseq/graph_parser/mldoc_test.cljs b/src/test/logseq/graph_parser/mldoc_test.cljs index 6b1e79682a7..347a90d4c74 100644 --- a/src/test/logseq/graph_parser/mldoc_test.cljs +++ b/src/test/logseq/graph_parser/mldoc_test.cljs @@ -3,7 +3,6 @@ ["fs" :as fs] ["child_process" :as child-process] [clojure.string :as string] - [clojure.edn :as edn] ;; hack needed for parse-property to exist [logseq.graph-parser.text] [cljs.test :refer [testing deftest are is]])) @@ -103,7 +102,7 @@ ; (prn (butlast (clojure.data/diff (edn/read-string (slurp "mldoc-asts.edn")) ; asts-by-file))) ;; This is just temporary - #_(is (= (edn/read-string (slurp "mldoc-asts.edn")) + #_(is (= (clojure.edn/read-string (slurp "mldoc-asts.edn")) asts-by-file) "Matches initial AST") #_(println "Wrote asts for" (count asts-by-file) "files") diff --git a/src/test/logseq/graph_parser/nbb_test_runner.cljs b/src/test/logseq/graph_parser/nbb_test_runner.cljs new file mode 100644 index 00000000000..1a174997b15 --- /dev/null +++ b/src/test/logseq/graph_parser/nbb_test_runner.cljs @@ -0,0 +1,21 @@ +(ns logseq.graph-parser.nbb-test-runner + "Nbb tests for graph-parser" + (:require [cljs.test :as t] + [logseq.graph-parser.mldoc :as gp-mldoc] + [logseq.graph-parser.text :as text] + ; [logseq.graph-parser.text-test] + [logseq.graph-parser.mldoc-test])) + +(defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m] + (when-not (cljs.test/successful? m) + (set! (.-exitCode js/process) 1))) + +;; run this function with: nbb-logseq -m logseq.test.nbb-test-runner/run-tests +(defn run-tests [] + ;; This hack is the same as the one in frontend.format. This has to be in an nbb only + ;; ns since alter-var-root doesn't exist in cljs and nbb doesn't support set! yet + #_:clj-kondo/ignore + (alter-var-root #'gp-mldoc/parse-property (constantly text/parse-property)) + (t/run-tests 'logseq.graph-parser.mldoc-test + ;; TODO: Enable when https://github.com/babashka/nbb/issues/187 works + #_'logseq.graph-parser.text-test)) diff --git a/src/test/logseq/graph_parser/text_test.cljs b/src/test/logseq/graph_parser/text_test.cljs index 2db2cb42d46..d56cc664bad 100644 --- a/src/test/logseq/graph_parser/text_test.cljs +++ b/src/test/logseq/graph_parser/text_test.cljs @@ -1,32 +1,30 @@ (ns logseq.graph-parser.text-test (:require [cljs.test :refer [are deftest testing]] - [frontend.config :as config] - [frontend.state :as state] [logseq.graph-parser.text :as text])) (deftest test-get-page-name [] (are [x y] (= (text/get-page-name x) y) - "[[page]]" "page" - "[[another page]]" "another page" - "[single bracket]" nil - "no brackets" nil + "[[page]]" "page" + "[[another page]]" "another page" + "[single bracket]" nil + "no brackets" nil - "[[another page]]" "another page" - "[[nested [[page]]]]" "nested [[page]]" + "[[another page]]" "another page" + "[[nested [[page]]]]" "nested [[page]]" - "[[file:./page.org][page]]" "page" - "[[file:./pages/page.org][page]]" "page" + "[[file:./page.org][page]]" "page" + "[[file:./pages/page.org][page]]" "page" - "[[file:./namespace.page.org][namespace/page]]" "namespace/page" - "[[file:./pages/namespace.page.org][namespace/page]]" "namespace/page" - "[[file:./pages/namespace.page.org][please don't change me]]" "namespace/page" + "[[file:./namespace.page.org][namespace/page]]" "namespace/page" + "[[file:./pages/namespace.page.org][namespace/page]]" "namespace/page" + "[[file:./pages/namespace.page.org][please don't change me]]" "namespace/page" - "[page](file:./page.md)" "page" - "[page](file:.pages/page.md)" "page" + "[page](file:./page.md)" "page" + "[page](file:.pages/page.md)" "page" - "[logseq/page](file:./logseq.page.md)" "logseq/page" - "[logseq/page](file:./pages/logseq.page.md)" "logseq/page")) + "[logseq/page](file:./logseq.page.md)" "logseq/page" + "[logseq/page](file:./pages/logseq.page.md)" "logseq/page")) (deftest page-ref? [] @@ -76,22 +74,26 @@ "#tag1,#tag2" #{"tag1" "tag2"} "[[Jan 26th, 2021]], hello" #{"hello" "Jan 26th, 2021"})) +(def block-patterns + {:markdown "-" + :org "*"}) + (deftest remove-level-spaces [] (testing "markdown" - (are [x y] (= (text/remove-level-spaces x :markdown (config/get-block-pattern :markdown) true) y) + (are [x y] (= (text/remove-level-spaces x :markdown (block-patterns :markdown) true) y) "- foobar" "foobar" " - foobar" "foobar")) (testing "markdown without spaces between the `#` and title" - (are [x y] (= (text/remove-level-spaces x :markdown (config/get-block-pattern :markdown)) y) + (are [x y] (= (text/remove-level-spaces x :markdown (block-patterns :markdown)) y) "-foobar" "foobar")) (testing "org" - (are [x y] (= (text/remove-level-spaces x :org (config/get-block-pattern :org) true) y) + (are [x y] (= (text/remove-level-spaces x :org (block-patterns :org) true) y) "* foobar" "foobar" "** foobar" "foobar" "********************* foobar" "foobar")) (testing "org without spaces between the `#` and title" - (are [x y] (= (text/remove-level-spaces x :org (config/get-block-pattern :org)) y) + (are [x y] (= (text/remove-level-spaces x :org (block-patterns :org)) y) "*foobar" "foobar" "**foobar" "foobar" "*********************foobar" "foobar"))) @@ -128,7 +130,7 @@ (deftest test-parse-property (testing "parse-property" - (are [k v y] (= (text/parse-property k v (state/get-config)) y) + (are [k v y] (= (text/parse-property k v {}) y) :tags "foo" "foo" :tags "foo, bar" #{"foo" "bar"} :tags "foo,bar" #{"foo" "bar"} @@ -140,7 +142,7 @@ :tags "[[foo [[bar]]]]" #{"foo [[bar]]"} :tags "[[foo [[bar]]]], baz" #{"baz" "foo [[bar]]"})) (testing "parse-property with quoted strings" - (are [k v y] (= (text/parse-property k v (state/get-config)) y) + (are [k v y] (= (text/parse-property k v {}) y) :tags "\"foo, bar\"" "\"foo, bar\"" :tags "\"[[foo]], [[bar]]\"" "\"[[foo]], [[bar]]\"" :tags "baz, \"[[foo]], [[bar]]\"" #{"baz"}))) diff --git a/src/test/logseq/test/nbb_test_runner.cljs b/src/test/logseq/test/nbb_test_runner.cljs deleted file mode 100644 index 5d18e3898bd..00000000000 --- a/src/test/logseq/test/nbb_test_runner.cljs +++ /dev/null @@ -1,11 +0,0 @@ -(ns logseq.test.nbb-test-runner - (:require [cljs.test :as t] - [logseq.graph-parser.mldoc-test])) - -(defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m] - (when-not (cljs.test/successful? m) - (set! (.-exitCode js/process) 1))) - -;; run this function with: nbb-logseq -m logseq.test.nbb-test-runner/run-tests -(defn run-tests [] - (t/run-tests 'logseq.graph-parser.mldoc-test)) From ede88bb3906b117d03a3908f1cda79c88ef4b168 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 10 May 2022 12:32:33 -0400 Subject: [PATCH 07/58] Fix carve errors --- .carve/config.edn | 4 +++- .carve/ignore | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.carve/config.edn b/.carve/config.edn index 287c10c6eb4..943afa8e135 100644 --- a/.carve/config.edn +++ b/.carve/config.edn @@ -5,5 +5,7 @@ ;; Ignore b/c too many false positives frontend.db ;; Used for debugging - frontend.db.debug] + frontend.db.debug + ;; carve doesn't detect nbb only usage + logseq.graph-parser.log] :report {:format :ignore}} diff --git a/.carve/ignore b/.carve/ignore index c4f4fb05d46..a79ef044741 100644 --- a/.carve/ignore +++ b/.carve/ignore @@ -72,5 +72,7 @@ frontend.util/trace! frontend.util.pool/terminate-pool! ;; Repl fn frontend.util.property/add-page-properties -;; Used by shadow +;; Test runner used by shadow frontend.test.node-test-runner/main +;; Test runner for nbb +logseq.graph-parser.nbb-test-runner/run-tests From a184b1ccac5be03b96148808a077cd74e57af8be Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 10 May 2022 16:50:35 -0400 Subject: [PATCH 08/58] Reuse helpers b/n repo and mldoc --- src/test/frontend/handler/repo_test.cljs | 36 ++----------------- src/test/frontend/test/docs_graph_helper.cljs | 34 ++++++++++++++++++ src/test/logseq/graph_parser/mldoc_test.cljs | 30 ++-------------- 3 files changed, 40 insertions(+), 60 deletions(-) create mode 100644 src/test/frontend/test/docs_graph_helper.cljs diff --git a/src/test/frontend/handler/repo_test.cljs b/src/test/frontend/handler/repo_test.cljs index 1a03d87ae92..7f7f82bb901 100644 --- a/src/test/frontend/handler/repo_test.cljs +++ b/src/test/frontend/handler/repo_test.cljs @@ -1,44 +1,14 @@ (ns frontend.handler.repo-test (:require [cljs.test :refer [deftest use-fixtures is testing]] - [clojure.string :as string] - ["fs" :as fs] - ["child_process" :as child-process] [frontend.handler.repo :as repo-handler] [frontend.test.helper :as test-helper] + [frontend.test.docs-graph-helper :as docs-graph-helper] [datascript.core :as d] [frontend.db.conn :as conn])) (use-fixtures :each {:before test-helper/start-test-db! :after test-helper/destroy-test-db!}) -(defn- slurp - "Like clojure.core/slurp" - [file] - (str (fs/readFileSync file))) - -(defn- sh - "Run shell cmd synchronously and print to inherited streams by default. Aims - to be similar to babashka.tasks/shell" - [cmd opts] - (child-process/spawnSync (first cmd) - (clj->js (rest cmd)) - (clj->js (merge {:stdio "inherit"} opts)))) - -(defn- build-graph-files - [dir] - (let [files (->> (str (.-stdout (sh ["git" "ls-files"] - {:cwd dir :stdio nil}))) - string/split-lines - (filter #(re-find #"^(pages|journals)" %)) - (map #(str dir "/" %)))] - (mapv #(hash-map :file/path % :file/content (slurp %)) files))) - -(defn- clone-docs-repo-if-not-exists - [dir] - (when-not (.existsSync fs dir) - (sh ["git" "clone" "--depth" "1" "-b" "v0.6.7" "-c" "advice.detachedHead=false" - "https://github.com/logseq/docs" dir] {}))) - (defn- get-top-block-properties [db] (->> (d/q '[:find (pull ?b [*]) @@ -67,8 +37,8 @@ ;; Integration test that test parsing a large graph like docs (deftest ^:integration parse-and-load-files-to-db (let [graph-dir "src/test/docs" - _ (clone-docs-repo-if-not-exists graph-dir) - files (build-graph-files graph-dir) + _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir) + files (docs-graph-helper/build-graph-files graph-dir) _ (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false}) db (conn/get-db test-helper/test-db)] diff --git a/src/test/frontend/test/docs_graph_helper.cljs b/src/test/frontend/test/docs_graph_helper.cljs new file mode 100644 index 00000000000..25f64eca638 --- /dev/null +++ b/src/test/frontend/test/docs_graph_helper.cljs @@ -0,0 +1,34 @@ +(ns ^:nbb-compatible frontend.test.docs-graph-helper + "Helper fns for running tests against docs graph" + (:require ["fs" :as fs] + ["child_process" :as child-process] + [clojure.string :as string])) + + +(defn- slurp + "Like clojure.core/slurp" + [file] + (str (fs/readFileSync file))) + +(defn- sh + "Run shell cmd synchronously and print to inherited streams by default. Aims + to be similar to babashka.tasks/shell" + [cmd opts] + (child-process/spawnSync (first cmd) + (clj->js (rest cmd)) + (clj->js (merge {:stdio "inherit"} opts)))) + +(defn build-graph-files + [dir] + (let [files (->> (str (.-stdout (sh ["git" "ls-files"] + {:cwd dir :stdio nil}))) + string/split-lines + (filter #(re-find #"^(pages|journals)" %)) + (map #(str dir "/" %)))] + (mapv #(hash-map :file/path % :file/content (slurp %)) files))) + +(defn clone-docs-repo-if-not-exists + [dir] + (when-not (.existsSync fs dir) + (sh ["git" "clone" "--depth" "1" "-b" "v0.6.7" "-c" "advice.detachedHead=false" + "https://github.com/logseq/docs" dir] {}))) diff --git a/src/test/logseq/graph_parser/mldoc_test.cljs b/src/test/logseq/graph_parser/mldoc_test.cljs index 347a90d4c74..ef71e1f259e 100644 --- a/src/test/logseq/graph_parser/mldoc_test.cljs +++ b/src/test/logseq/graph_parser/mldoc_test.cljs @@ -3,8 +3,7 @@ ["fs" :as fs] ["child_process" :as child-process] [clojure.string :as string] - ;; hack needed for parse-property to exist - [logseq.graph-parser.text] + [frontend.test.docs-graph-helper :as docs-graph-helper] [cljs.test :refer [testing deftest are is]])) (deftest test-link @@ -43,33 +42,10 @@ (are [x y] (= (gp-mldoc/link? :markdown x) y) "[YouTube](https://www.youtube.com/watch?v=-8ym7pyUs9gL) - [Vimeo](https://vimeo.com/677920303) {{youtube https://www.youtube.com/watch?v=-8ym7pyUs9g}}" true))) -;; TODO: Reuse with repo-test fns -(defn- slurp - "Like clojure.core/slurp" - [file] - (str (fs/readFileSync file))) - -(defn- sh - "Run shell cmd synchronously and print to inherited streams by default. Aims - to be similar to babashka.tasks/shell" - [cmd opts] - (child-process/spawnSync (first cmd) - (clj->js (rest cmd)) - (clj->js (merge {:stdio "inherit"} opts)))) - -(defn- build-graph-files - [dir] - (let [files (->> (str (.-stdout (sh ["git" "ls-files"] - {:cwd dir :stdio nil}))) - string/split-lines - (filter #(re-find #"^(pages|journals)" %)) - (map #(str dir "/" %)))] - (mapv #(hash-map :file/path % :file/content (slurp %)) files))) - -;; TODO: Add clone docs step (deftest ^:integration test->edn (let [graph-dir "src/test/docs" - files (build-graph-files graph-dir) + _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir) + files (docs-graph-helper/build-graph-files graph-dir) asts-by-file (->> files (map (fn [{:file/keys [path content]}] (let [format (if (string/ends-with? path ".org") From 44bc0e8f40497d2994b966e305584a38c910f6ae Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 10 May 2022 17:41:37 -0400 Subject: [PATCH 09/58] Add a few more mldoc unit tests Also remove local integration test --- src/test/logseq/graph_parser/mldoc_test.cljs | 94 ++++++++++++-------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/src/test/logseq/graph_parser/mldoc_test.cljs b/src/test/logseq/graph_parser/mldoc_test.cljs index ef71e1f259e..4dd56bbb48b 100644 --- a/src/test/logseq/graph_parser/mldoc_test.cljs +++ b/src/test/logseq/graph_parser/mldoc_test.cljs @@ -1,7 +1,5 @@ (ns logseq.graph-parser.mldoc-test (:require [logseq.graph-parser.mldoc :as gp-mldoc] - ["fs" :as fs] - ["child_process" :as child-process] [clojure.string :as string] [frontend.test.docs-graph-helper :as docs-graph-helper] [cljs.test :refer [testing deftest are is]])) @@ -42,6 +40,63 @@ (are [x y] (= (gp-mldoc/link? :markdown x) y) "[YouTube](https://www.youtube.com/watch?v=-8ym7pyUs9gL) - [Vimeo](https://vimeo.com/677920303) {{youtube https://www.youtube.com/watch?v=-8ym7pyUs9g}}" true))) +(def md-config (gp-mldoc/default-config :markdown)) + +(deftest src-test + (is (= [["Src" + {:lines [": hello" "\n"], + :pos_meta {:start_pos 4, :end_pos 12}, + :full_content "```\n: hello\n```"}] + {:start_pos 0, :end_pos 15}] + (first (gp-mldoc/->edn "``` +: hello +```" md-config {}))) + "Basic src example") + + (is (= [["Src" + {:lines [" hello" "\n" " world" "\n"], + :pos_meta {:start_pos 7, :end_pos 25}, + :full_content "```\nhello\nworld\n```"}] + {:start_pos 1, :end_pos 29}] + (second (gp-mldoc/->edn " + ``` + hello + world + ``` +" md-config {}))) + "Src example with leading whitespace")) + +(deftest properties-test + (are [x y] (= [["Properties" y] nil] + (first (gp-mldoc/->edn x md-config {}))) + + ;; comma separates values + "property:: foo, bar" + {:property #{"foo" "bar"}} + + ;; alias property + "alias:: foo,, bar" + {:alias ["foo" "bar"]} + + ;; tags property + "tags:: foo,bar,foo" + {:tags ["foo" "bar"]} + + ;; title property + "title:: comma, is ok" + {:title "comma, is ok"})) + +(deftest name-definition-test + (is (= [["List" + [{:content [["Paragraph" [["Plain" "definition"]]]], + :items [], + :name [["Plain" "term"]], + :indent 0, + :ordered false}]] + {:start_pos 0, :end_pos 17}] + (first (gp-mldoc/->edn "term +: definition" md-config {}))))) + (deftest ^:integration test->edn (let [graph-dir "src/test/docs" _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir) @@ -72,37 +127,4 @@ "Src" 56, "Table" 4} (->> asts-by-file (mapcat val) (map ffirst) frequencies)) - "AST node type counts") - - ; (println :DIFF) - ; (prn (butlast (clojure.data/diff (edn/read-string (slurp "mldoc-asts.edn")) - ; asts-by-file))) - ;; This is just temporary - #_(is (= (clojure.edn/read-string (slurp "mldoc-asts.edn")) - asts-by-file) - "Matches initial AST") - #_(println "Wrote asts for" (count asts-by-file) "files") - #_(fs/writeFileSync "mldoc-asts.edn" (pr-str asts-by-file)))) - -(def md-config (gp-mldoc/default-config :markdown)) - -(deftest src-test - (is (= [["Src" - {:lines [": hello" "\n"], - :pos_meta {:start_pos 4, :end_pos 12}, - :full_content "```\n: hello\n```"}] - {:start_pos 0, :end_pos 15}] - (first (gp-mldoc/->edn "``` -: hello -```" md-config {}))))) - -(deftest name-definition-test - (is (= [["List" - [{:content [["Paragraph" [["Plain" "definition"]]]], - :items [], - :name [["Plain" "term"]], - :indent 0, - :ordered false}]] - {:start_pos 0, :end_pos 17}] - (first (gp-mldoc/->edn "term -: definition" md-config {}))))) + "AST node type counts"))) From 31032beadd4fb9dac294521fa7439bbf99ec6bd7 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 10 May 2022 20:35:31 -0400 Subject: [PATCH 10/58] Fix logging and error handling --- src/main/logseq/graph_parser/mldoc.cljc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/logseq/graph_parser/mldoc.cljc b/src/main/logseq/graph_parser/mldoc.cljc index fa4d9b446d7..c229f9a9561 100644 --- a/src/main/logseq/graph_parser/mldoc.cljc +++ b/src/main/logseq/graph_parser/mldoc.cljc @@ -3,9 +3,9 @@ #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off} :unresolved-symbol {:level :off}}}}) (:require #?(:org.babashka/nbb ["mldoc$default" :refer [Mldoc]] - :default ["mldoc" :refer [Mldoc]]) + :default ["mldoc" :refer [Mldoc]]) #?(:org.babashka/nbb [logseq.graph-parser.log :as log] - :default [lambdaisland.glogi :as log]) + :default [lambdaisland.glogi :as log]) [goog.object :as gobj] [cljs-bean.core :as bean] [logseq.graph-parser.utf8 :as utf8] @@ -180,8 +180,8 @@ (gp-util/json->clj) (update-src-full-content content) (collect-page-properties parse-property config-state))) - (catch js/Error e - (js/console.error e) + (catch :default e + (log/error :unexpected-error e) [])) (log/error :edn/wrong-content-type content))) @@ -193,7 +193,7 @@ (-> text (inline-parse-json config) (gp-util/json->clj))) - (catch js/Error _e + (catch :default _e []))) (defn link? From 43e8f07f40343a623bc80d31cadb2c3603bcf38b Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 11 May 2022 12:53:04 +0800 Subject: [PATCH 11/58] fix: journal templates not working when re-index related to #5190 --- src/main/frontend/db/model.cljs | 10 ---------- src/main/frontend/handler/page.cljs | 2 -- src/main/frontend/handler/repo.cljs | 7 +------ 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index 0f82da6e93b..43f39c43fd8 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -970,16 +970,6 @@ (reverse) (take n)))))) -(defn journal-day-exists? - [graph day] - (d/q - '[:find ?p . - :in $ ?day - :where - [?p :block/journal-day ?day]] - (conn/get-db graph) - day)) - ;; get pages that this page referenced (defn get-page-referenced-pages [repo page] diff --git a/src/main/frontend/handler/page.cljs b/src/main/frontend/handler/page.cljs index 32d053d6018..cf4cf98c8fd 100644 --- a/src/main/frontend/handler/page.cljs +++ b/src/main/frontend/handler/page.cljs @@ -739,8 +739,6 @@ file-content (when file-exists? (fs/read-file repo-dir file-path))] (when (and (db/page-empty? repo today-page) - (not (model/journal-day-exists? repo - (date/journal-title->int (date/today)))) (or (not file-exists?) (and file-exists? (string/blank? file-content)))) (create! title {:redirect? false diff --git a/src/main/frontend/handler/repo.cljs b/src/main/frontend/handler/repo.cljs index c00fc15edbb..694e3165304 100644 --- a/src/main/frontend/handler/repo.cljs +++ b/src/main/frontend/handler/repo.cljs @@ -212,6 +212,7 @@ (ui-handler/re-render-root! re-render-opts)) (state/pub-event! [:graph/added repo-url opts]) (state/reset-parsing-state!) + (state/set-loading-files! repo-url false) (async/offer! graph-added-chan true)) (defn- parse-files-and-create-default-files-inner! @@ -262,15 +263,9 @@ (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts)) (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts))) -(defn- update-parsing-state! - [repo-url] - (state/set-loading-files! repo-url false)) - (defn parse-files-and-load-to-db! [repo-url files {:keys [delete-files delete-blocks re-render? re-render-opts _refresh?] :as opts :or {re-render? true}}] - (update-parsing-state! repo-url) - (let [file-paths (map :file/path files) metadata-file (config/get-metadata-path) metadata-content (some #(when (= (:file/path %) metadata-file) From e29dc7373e557b0999aca1facc62abae974cc080 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 11 May 2022 14:55:05 +0800 Subject: [PATCH 12/58] enhance: use journal template if exists for future days --- src/main/frontend/components/page.cljs | 11 +++++++++-- src/main/frontend/handler/events.cljs | 10 ++++++++++ src/main/frontend/handler/page.cljs | 11 +++-------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 3b4224b8cd7..b8f05738aac 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -109,8 +109,15 @@ [:a.add-button-link.block (ui/icon "circle-plus")]]]]) -(rum/defc page-blocks-cp < rum/reactive - db-mixins/query +(rum/defc page-blocks-cp < rum/reactive db-mixins/query + {:will-mount (fn [state] + (let [page-e (second (:rum/args state)) + page-name (:block/name page-e)] + (when (and (db/journal-page? page-name) + (>= (date/journal-title->int page-name) + (date/journal-title->int (date/today)))) + (state/pub-event! [:journal/insert-template page-name]))) + state)} [repo page-e {:keys [sidebar?] :as config}] (when page-e (let [page-name (or (:block/name page-e) diff --git a/src/main/frontend/handler/events.cljs b/src/main/frontend/handler/events.cljs index 46953d6760e..2e5091090cc 100644 --- a/src/main/frontend/handler/events.cljs +++ b/src/main/frontend/handler/events.cljs @@ -388,6 +388,16 @@ db-encrypted-secret close-fn))) +(defmethod handle :journal/insert-template [[_ page-name]] + (let [page-name (util/page-name-sanity-lc page-name)] + (when-let [page (db/pull [:block/name page-name])] + (when (db/page-empty? (state/get-current-repo) page-name) + (when-let [template (state/get-default-journal-template)] + (editor-handler/insert-template! + nil + template + {:target page})))))) + (defn run! [] (let [chan (state/get-events-chan)] diff --git a/src/main/frontend/handler/page.cljs b/src/main/frontend/handler/page.cljs index cf4cf98c8fd..ba026d26e19 100644 --- a/src/main/frontend/handler/page.cljs +++ b/src/main/frontend/handler/page.cljs @@ -728,13 +728,13 @@ (and (= "local" repo) (not (mobile-util/is-native-platform?)))) (let [title (date/today) today-page (util/page-name-sanity-lc title) - template (state/get-default-journal-template) format (state/get-preferred-format repo) file-name (date/journal-title->default title) path (str (config/get-journals-directory) "/" file-name "." (config/get-file-extension format)) file-path (str "/" path) - repo-dir (config/get-repo-dir repo)] + repo-dir (config/get-repo-dir repo) + template (state/get-default-journal-template)] (p/let [file-exists? (fs/file-exists? repo-dir file-path) file-content (when file-exists? (fs/read-file repo-dir file-path))] @@ -745,12 +745,7 @@ :split-namespace? false :create-first-block? (not template) :journal? true}) - (when template - (let [page (db/pull [:block/name today-page])] - (editor-handler/insert-template! - nil - template - {:target page}))) + (state/pub-event! [:journal/insert-template today-page]) (ui-handler/re-render-root!)))))))) (defn open-today-in-sidebar From f7ba2b5864f8b3279ccb571ffce2e4887150abe0 Mon Sep 17 00:00:00 2001 From: Andelf Date: Wed, 11 May 2022 17:45:27 +0800 Subject: [PATCH 13/58] chore: rm fs-watcher debug print --- src/main/frontend/handler/events.cljs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/frontend/handler/events.cljs b/src/main/frontend/handler/events.cljs index 2e5091090cc..50a455d0f83 100644 --- a/src/main/frontend/handler/events.cljs +++ b/src/main/frontend/handler/events.cljs @@ -346,12 +346,11 @@ (defmethod handle :file-watcher/changed [[_ ^js event]] (let [type (.-event event) - payload (js->clj event :keywordize-keys true) - payload' (-> payload - (update :path js/decodeURI))] - (prn ::fs-watcher payload) - (fs-watcher/handle-changed! type payload') - (sync/file-watch-handler type payload'))) + payload (-> event + (js->clj :keywordize-keys true) + (update :path js/decodeURI))] + (fs-watcher/handle-changed! type payload) + (sync/file-watch-handler type payload))) (defmethod handle :rebuild-slash-commands-list [[_]] (page-handler/rebuild-slash-commands-list!)) From c480e1808bfce435f165766a09bfd2590396ad20 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Wed, 11 May 2022 13:19:23 +0800 Subject: [PATCH 14/58] enhance(sync): add ::queued-local->remote-files file-sync state --- src/main/frontend/fs/sync.cljs | 185 +++++++++++++++++---------------- 1 file changed, 94 insertions(+), 91 deletions(-) diff --git a/src/main/frontend/fs/sync.cljs b/src/main/frontend/fs/sync.cljs index 3b70489de73..1e059d4253b 100644 --- a/src/main/frontend/fs/sync.cljs +++ b/src/main/frontend/fs/sync.cljs @@ -71,6 +71,7 @@ (s/def ::sync-state (s/keys :req-un [::state ::current-local->remote-files ::current-remote->local-files + ::queued-local->remote-files ::history])) ;; diff @@ -837,6 +838,71 @@ not) (go (>! local-changes-chan (->FileChangeEvent type dir path stat))))) +;;; ### sync state + + +(defn sync-state + "create a new sync-state" + [] + {:post [(s/valid? ::sync-state %)]} + {:state ::idle + :current-local->remote-files #{} + :current-remote->local-files #{} + :queued-local->remote-files #{} + :history '()}) + +(defn- sync-state--update-state + [sync-state next-state] + {:pre [(s/valid? ::state next-state)] + :post [(s/valid? ::sync-state %)]} + (assoc sync-state :state next-state)) + +(defn sync-state--add-current-remote->local-files + [sync-state paths] + {:post [(s/valid? ::sync-state %)]} + (update sync-state :current-remote->local-files into paths)) + +(defn sync-state--add-current-local->remote-files + [sync-state paths] + {:post [(s/valid? ::sync-state %)]} + (update sync-state :current-local->remote-files into paths)) + +(defn sync-state--update-queued-local->remote-files + [sync-state paths] + {:post [(s/valid? ::sync-state %)]} + (update sync-state :queued-local->remote-files (fn [_ n] (set n)) paths)) + +(defn- add-history-items + [history paths now] + (sequence + (comp + ;; only reserve the latest one of same-path-items + (dedupe-by :path) + ;; reserve the latest 20 history items + (take 20)) + (into history + (map (fn [path] {:path path :time now}) paths)))) + +(defn sync-state--remove-current-remote->local-files + [sync-state paths] + {:post [(s/valid? ::sync-state %)]} + (let [now (t/now)] + (-> sync-state + (update :current-remote->local-files set/difference paths) + (update :history add-history-items paths now)))) + +(defn sync-state--remove-current-local->remote-files + [sync-state paths] + {:post [(s/valid? ::sync-state %)]} + (let [now (t/now)] + (-> sync-state + (update :current-local->remote-files set/difference paths) + (update :history add-history-items paths now)))) + +(defn sync-state--stopped? + [sync-state] + (= ::stop (:state sync-state))) + ;;; ### remote->local syncer & local->remote syncer (defprotocol IRemote->LocalSync @@ -953,32 +1019,26 @@ (defn- contains-path? [regexps path] (reduce #(when (re-find %2 path) (reduced true)) false regexps)) -(defn- filter-local-changes +(defn- filter-local-changes-pred "filter local-change events: - for 'unlink' event - when related file exists on local dir, ignore this event - for 'add' | 'change' event - when related file's content is same as remote file, ignore it" - [to-ch from-ch basepath graph-uuid] - (async/pipeline-async - 1 to-ch - (fn [^FileChangeEvent e result] - (go - (case (.-type e) - "unlink" - (let [r ( r ex-cause ;; str (string/index-of "No such file or directory") - ) - (>! result e))) - - ("add" "change") - (let [path (relative-path e)] - (when (and (! result e)))) - (async/close! result))) - from-ch false)) - + [^FileChangeEvent e basepath graph-uuid] + (go + (let [r-path (relative-path e)] + (case (.-type e) + "unlink" + (let [r ( r ex-cause)) + + ("add" "change") + ;; 1. local file exists + ;; 2. compare with remote file, and changed + (and (RemoteSyncer [user-uuid graph-uuid base-path repo *sync-state @@ -1009,7 +1069,7 @@ (let [c (.filtered-chan this 10000) filter-e-fn (.filter-file-change-events-fn this)] (go-loop [timeout-c (timeout rate) - tcoll (transient [])] + coll []] (let [{:keys [timeout ^FileChangeEvent e stop]} (async/alt! timeout-c {:timeout true} from-chan ([e] {:e e}) @@ -1019,22 +1079,22 @@ (async/close! c) timeout - (let [from-c (chan 10000)] - (remote-files nil) + (recur (async/timeout rate) [])) (some? e) - (do - (when (filter-e-fn e) - (conj! tcoll e)) - (recur timeout-c tcoll)) + (if (and (filter-e-fn e) + (remote-files + (mapv relative-path coll*)) + (recur timeout-c coll*)) + (recur timeout-c coll)) (nil? e) - (do - (println "close ratelimit chan") - (async/close! c))))) + (do (println "close ratelimit chan") + (async/close! c))))) c)) @@ -1114,64 +1174,7 @@ (or need-sync-remote unknown) r))))))))) -;;; ### sync state - - -(defn sync-state - "create a new sync-state" - [] - {:post [(s/valid? ::sync-state %)]} - {:state ::idle - :current-local->remote-files #{} - :current-remote->local-files #{} - :history '()}) - -(defn- sync-state--update-state - [sync-state next-state] - {:pre [(s/valid? ::state next-state)] - :post [(s/valid? ::sync-state %)]} - (assoc sync-state :state next-state)) - -(defn sync-state--add-current-remote->local-files - [sync-state paths] - {:post [(s/valid? ::sync-state %)]} - (update sync-state :current-remote->local-files into paths)) - -(defn sync-state--add-current-local->remote-files - [sync-state paths] - {:post [(s/valid? ::sync-state %)]} - (update sync-state :current-local->remote-files into paths)) - -(defn- add-history-items - [history paths now] - (sequence - (comp - ;; only reserve the latest one of same-path-items - (dedupe-by :path) - ;; reserve the latest 20 history items - (take 20)) - (into history - (map (fn [path] {:path path :time now}) paths)))) - -(defn sync-state--remove-current-remote->local-files - [sync-state paths] - {:post [(s/valid? ::sync-state %)]} - (let [now (t/now)] - (-> sync-state - (update :current-remote->local-files set/difference paths) - (update :history add-history-items paths now)))) - -(defn sync-state--remove-current-local->remote-files - [sync-state paths] - {:post [(s/valid? ::sync-state %)]} - (let [now (t/now)] - (-> sync-state - (update :current-local->remote-files set/difference paths) - (update :history add-history-items paths now)))) -(defn sync-state--stopped? - [sync-state] - (= ::stop (:state sync-state))) ;;; ### put all stuff together From 112e6e2d9773d769d620aa36d38658ef127c2dab Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 11 May 2022 12:33:02 -0400 Subject: [PATCH 15/58] Test second nbb ns thanks to upstream nbb fix :) --- package.json | 2 +- src/test/logseq/graph_parser/nbb_test_runner.cljs | 4 ++-- yarn.lock | 10 ++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index b4cc0cb66f9..20e9f8479c8 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "del": "^6.0.0", "gulp": "^4.0.2", "gulp-clean-css": "^4.3.0", - "@logseq/nbb-logseq": "^0.3.10", + "@logseq/nbb-logseq": "^0.3.99", "npm-run-all": "^4.1.5", "playwright": "^1.19.2", "postcss": "8.2.13", diff --git a/src/test/logseq/graph_parser/nbb_test_runner.cljs b/src/test/logseq/graph_parser/nbb_test_runner.cljs index 1a174997b15..c9dc7c4d154 100644 --- a/src/test/logseq/graph_parser/nbb_test_runner.cljs +++ b/src/test/logseq/graph_parser/nbb_test_runner.cljs @@ -3,7 +3,7 @@ (:require [cljs.test :as t] [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.text :as text] - ; [logseq.graph-parser.text-test] + [logseq.graph-parser.text-test] [logseq.graph-parser.mldoc-test])) (defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m] @@ -18,4 +18,4 @@ (alter-var-root #'gp-mldoc/parse-property (constantly text/parse-property)) (t/run-tests 'logseq.graph-parser.mldoc-test ;; TODO: Enable when https://github.com/babashka/nbb/issues/187 works - #_'logseq.graph-parser.text-test)) + 'logseq.graph-parser.text-test)) diff --git a/yarn.lock b/yarn.lock index aba33629c4d..e944ffe7a2b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -716,10 +716,10 @@ resolved "https://registry.yarnpkg.com/@kanru/rage-wasm/-/rage-wasm-0.2.1.tgz#dd8fdd3133992c42bf68c0086d8cad40a13bc329" integrity sha512-sYi4F2mL6Mpcz7zbS4myasw11xLBEbgZkDMRVg9jNxTKt6Ct/LT7/vCHDmEzAFcPcPqixD5De6Ql3bJijAX0/w== -"@logseq/nbb-logseq@^0.3.10": - version "0.3.10" - resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-0.3.10.tgz#74534f9d263eb2105a41143a55425d0910d43eb8" - integrity sha512-y7/VJ99WCoNpQMqQOOBobKzNrRz4IcG85HggAkIaUtNwJ/Adb5fA28ZKP5ttEpqWLXTbAyqbBObD5q/xJLLD8w== +"@logseq/nbb-logseq@^0.3.99": + version "0.3.99" + resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-0.3.99.tgz#cf6c05c559963e4e0fb92f214a63228972ef87d3" + integrity sha512-Msa6Ck6wqt7sYGExQZgUT/uUG/z5jGaPlytVdgGkCrVogZb2yaChbWeMOCfCsCTi6kbHo15hC3aOeDAXcfnFzw== dependencies: import-meta-resolve "^1.1.1" @@ -6826,6 +6826,8 @@ react-icons@2.2.7: version "2.2.7" resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-2.2.7.tgz#d7860826b258557510dac10680abea5ca23cf650" integrity sha512-0n4lcGqzJFcIQLoQytLdJCE0DKSA9dkwEZRYoGrIDJZFvIT6Hbajx5mv9geqhqFiNjUgtxg8kPyDfjlhymbGFg== + dependencies: + react-icon-base "2.1.0" react-is@^16.13.1, react-is@^16.3.1, react-is@^16.7.0: version "16.13.1" From 3625f6f29bd54f3cd14f701829002a62b5601ab2 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 12 May 2022 07:20:56 +0800 Subject: [PATCH 16/58] fix: copy paste not working across graphs --- src/main/frontend/handler/editor.cljs | 2 ++ src/main/frontend/state.cljs | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 8c3dc66535b..2bd2a00d8c6 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -2883,11 +2883,13 @@ [text e] (let [copied-blocks (state/get-copied-blocks) copied-block-ids (:copy/block-ids copied-blocks) + copied-graph (:copy/graph copied-blocks) input (state/get-input) *stop-event? (atom true)] (cond ;; Internal blocks by either copy or cut blocks (and + (= copied-graph (state/get-current-repo)) (or (seq copied-block-ids) (seq (:copy/full-blocks copied-blocks))) text diff --git a/src/main/frontend/state.cljs b/src/main/frontend/state.cljs index 29844e6a231..8f335fb2ed3 100644 --- a/src/main/frontend/state.cljs +++ b/src/main/frontend/state.cljs @@ -182,7 +182,9 @@ :graph/parsing-state {} ;; copied blocks - :copy/blocks {:copy/content nil :copy/block-ids nil} + :copy/blocks {:copy/content nil + :copy/block-ids nil + :copy/graph nil} :copy/export-block-text-indent-style (or (storage/get :copy/export-block-text-indent-style) "dashes") @@ -1426,13 +1428,15 @@ (defn set-copied-blocks [content ids] - (set-state! :copy/blocks {:copy/content content + (set-state! :copy/blocks {:copy/graph (get-current-repo) + :copy/content content :copy/block-ids ids :copy/full-blocks nil})) (defn set-copied-full-blocks [content blocks] - (set-state! :copy/blocks {:copy/content content + (set-state! :copy/blocks {:copy/graph (get-current-repo) + :copy/content content :copy/full-blocks blocks})) (defn set-copied-full-blocks! From f29aea9e55c190fc66c060626d85296c685c1bd2 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Wed, 11 May 2022 22:52:35 +0800 Subject: [PATCH 17/58] enhance(sync): update prod config, add :pre for update-graphs-txid --- src/main/frontend/config.cljs | 1 + src/main/frontend/fs/sync.cljs | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/config.cljs b/src/main/frontend/config.cljs index c8af545844e..5e17c106189 100644 --- a/src/main/frontend/config.cljs +++ b/src/main/frontend/config.cljs @@ -23,6 +23,7 @@ ;; (goog-define LOGIN-URL ;; "https://logseq.auth.us-east-1.amazoncognito.com/login?client_id=7ns5v1pu8nrbs04rvdg67u4a7c&response_type=code&scope=email+openid+phone&redirect_uri=logseq%3A%2F%2Fauth-callback") ;; (goog-define API-DOMAIN "api-prod.logseq.com") +;; (goog-define WS-URL "wss://b2rp13onu2.execute-api.us-east-1.amazonaws.com/production?graphuuid=%s") ;; dev env (goog-define FILE-SYNC-PROD? false) diff --git a/src/main/frontend/fs/sync.cljs b/src/main/frontend/fs/sync.cljs index 1e059d4253b..7f5a1222ced 100644 --- a/src/main/frontend/fs/sync.cljs +++ b/src/main/frontend/fs/sync.cljs @@ -107,7 +107,9 @@ (def graphs-txid (persist-var/persist-var nil "graphs-txid")) -(defn update-graphs-txid! [latest-txid graph-uuid user-uuid repo] +(defn update-graphs-txid! + [latest-txid graph-uuid user-uuid repo] + {:pre [(int? latest-txid) (>= latest-txid 0)]} (persist-var/-reset-value! graphs-txid [user-uuid graph-uuid latest-txid] repo) (persist-var/persist-save graphs-txid)) From 3ca8228a746dcc01c474fd664f11502ec6bef470 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Mon, 9 May 2022 06:48:31 +0800 Subject: [PATCH 18/58] Only render references when it's visible in the window viewport --- package.json | 3 +- resources/css/common.css | 5 ++ src/main/frontend/components/reference.cljs | 11 ++- src/main/frontend/db/model.cljs | 78 ++++++--------------- src/main/frontend/ui.cljs | 17 +++++ yarn.lock | 14 ++-- 6 files changed, 61 insertions(+), 67 deletions(-) diff --git a/package.json b/package.json index b4cc0cb66f9..4c4123c1371 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "static/electron.js", "devDependencies": { "@capacitor/cli": "3.2.2", + "@logseq/nbb-logseq": "^0.3.10", "@playwright/test": "^1.19.2", "@tailwindcss/ui": "0.7.2", "@types/gulp": "^4.0.7", @@ -13,7 +14,6 @@ "del": "^6.0.0", "gulp": "^4.0.2", "gulp-clean-css": "^4.3.0", - "@logseq/nbb-logseq": "^0.3.10", "npm-run-all": "^4.1.5", "playwright": "^1.19.2", "postcss": "8.2.13", @@ -113,6 +113,7 @@ "react-textarea-autosize": "8.3.3", "react-tippy": "1.4.0", "react-transition-group": "4.3.0", + "react-visibility-sensor": "^5.1.1", "reakit": "0.11.1", "remove-accents": "0.4.2", "send-intent": "3.0.11", diff --git a/resources/css/common.css b/resources/css/common.css index 96f219c096b..71433775125 100644 --- a/resources/css/common.css +++ b/resources/css/common.css @@ -1169,3 +1169,8 @@ html[data-theme='dark'] .keyboard-shortcut > code { overflow: hidden; text-overflow: ellipsis; } + +.lazy-visibility { + min-width: 1px; + min-height: 1px; +} diff --git a/src/main/frontend/components/reference.cljs b/src/main/frontend/components/reference.cljs index 623c1eb32d9..092685f2537 100644 --- a/src/main/frontend/components/reference.cljs +++ b/src/main/frontend/components/reference.cljs @@ -184,11 +184,16 @@ {:default-collapsed? default-collapsed? :title-trigger? true}))]])))) -(rum/defc references - [page-name] +(rum/defcs references < + (rum/local false ::visible?) + [state page-name] (ui/catch-error (ui/component-error "Linked References: Unexpected error") - (references* page-name))) + (ui/lazy-visible + (::visible? state) + "loading references..." + (fn [] + (references* page-name))))) (rum/defcs unlinked-references-aux < rum/reactive db-mixins/query diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index 43f39c43fd8..c48191236d0 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -1072,38 +1072,23 @@ (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)])) pages (page-alias-set repo page) aliases (set/difference pages #{page-id}) - query-result (if (seq aliases) - (let [rules '[[(find-blocks ?block ?ref-page ?pages ?alias ?aliases) - [?block :block/page ?alias] - [(contains? ?aliases ?alias)]] - [(find-blocks ?block ?ref-page ?pages ?alias ?aliases) - [?block :block/refs ?ref-page] - [(contains? ?pages ?ref-page)]]]] - (react/q repo - [:frontend.db.react/page<-blocks-or-block<-blocks page-id] - {} - '[:find [(pull ?block ?block-attrs) ...] - :in $ % ?pages ?aliases ?block-attrs - :where - (find-blocks ?block ?ref-page ?pages ?alias ?aliases)] - rules - pages - aliases - block-attrs)) - (react/q repo - [:frontend.db.react/page<-blocks-or-block<-blocks page-id] - {:use-cache? false} - '[:find [(pull ?ref-block ?block-attrs) ...] - :in $ ?page ?block-attrs - :where - [?ref-block :block/refs ?page]] - page-id - block-attrs)) + query-result (react/q repo + [:frontend.db.react/page<-blocks-or-block<-blocks page-id] + {} + '[:find [(pull ?block ?block-attrs) ...] + :in $ [?ref-page ...] ?block-attrs + :where + [?block :block/refs ?ref-page]] + pages + block-attrs + ;; TODO: :block/_parent slow recursive query + ;; (butlast block-attrs) + ) result (->> query-result react - (sort-by-left-recursive) (remove (fn [block] (= page-id (:db/id (:block/page block))))) + (sort-by-left-recursive) db-utils/group-by-page (map (fn [[k blocks]] (let [k (if (contains? aliases (:db/id k)) @@ -1119,35 +1104,14 @@ ([repo page] (when repo (when-let [db (conn/get-db repo)] - (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)])) - pages (page-alias-set repo page) - aliases (set/difference pages #{page-id}) - query-result (if (seq aliases) - (let [rules '[[(find-blocks ?block ?ref-page ?pages ?alias ?aliases) - [?block :block/page ?alias] - [(contains? ?aliases ?alias)]] - [(find-blocks ?block ?ref-page ?pages ?alias ?aliases) - [?block :block/refs ?ref-page] - [(contains? ?pages ?ref-page)]]]] - (d/q - '[:find ?block - :in $ % ?pages ?aliases ?block-attrs - :where - (find-blocks ?block ?ref-page ?pages ?alias ?aliases)] - db - rules - pages - aliases - block-attrs)) - (d/q - '[:find ?ref-block - :in $ ?page ?block-attrs - :where - [?ref-block :block/refs ?page]] - db - page-id - block-attrs))] - query-result))))) + (let [pages (page-alias-set repo page)] + (d/q + '[:find ?block + :in $ [?ref-page ...] + :where + [?block :block/refs ?ref-page]] + db + pages)))))) (defn get-date-scheduled-or-deadlines [journal-title] diff --git a/src/main/frontend/ui.cljs b/src/main/frontend/ui.cljs index 5fe756fab43..9c58dfa3f40 100644 --- a/src/main/frontend/ui.cljs +++ b/src/main/frontend/ui.cljs @@ -25,6 +25,7 @@ ["react-tippy" :as react-tippy] ["react-transition-group" :refer [CSSTransition TransitionGroup]] ["@logseq/react-tweet-embed" :as react-tweet-embed] + ["react-visibility-sensor" :as rvs] [rum.core :as rum] [frontend.db-mixins :as db-mixins] [frontend.mobile.util :as mobile-util] @@ -37,6 +38,7 @@ (def resize-consumer (r/adapt-class (gobj/get Resize "ResizeConsumer"))) (def Tippy (r/adapt-class (gobj/get react-tippy "Tooltip"))) (def ReactTweetEmbed (r/adapt-class react-tweet-embed)) +(def visibility-sensor (r/adapt-class (gobj/get rvs "default"))) (defn reset-ios-whole-page-offset! [] @@ -908,3 +910,18 @@ [:span.text-sm.font-medium label-right]] (progress-bar width)]) + +(rum/defc lazy-visible-inner + [visible? content-fn loading-label] + [:div.lazy-visibility + (if visible? + (when (fn? content-fn) (content-fn)) + (loading (or loading-label "loading...")))]) + +(rum/defc lazy-visible < rum/reactive + [*visible? loading-label content-fn] + (let [visible? (rum/react *visible?)] + (visibility-sensor + {:on-change #(reset! *visible? %) + :partialVisibility true} + (lazy-visible-inner visible? content-fn loading-label)))) diff --git a/yarn.lock b/yarn.lock index aba33629c4d..37f86e1e1d2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6624,7 +6624,7 @@ prompts@^2.3.2: kleur "^3.0.3" sisteransi "^1.0.5" -prop-types@15.x, prop-types@^15.5.4, prop-types@^15.6.0, prop-types@^15.6.2: +prop-types@15.x, prop-types@^15.5.4, prop-types@^15.6.0, prop-types@^15.6.2, prop-types@^15.7.2: version "15.8.1" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5" integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg== @@ -6812,11 +6812,6 @@ react-grid-layout@0.16.6: react-draggable "3.x" react-resizable "1.x" -react-icon-base@2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/react-icon-base/-/react-icon-base-2.1.0.tgz#a196e33fdf1e7aaa1fda3aefbb68bdad9e82a79d" - integrity sha1-oZbjP98eeqof2jrvu2i9rZ6Cp50= - react-icon-base@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/react-icon-base/-/react-icon-base-2.1.2.tgz#a17101dad9c1192652356096860a9ab43a0766c7" @@ -6886,6 +6881,13 @@ react-transition-group@4.3.0: loose-envify "^1.4.0" prop-types "^15.6.2" +react-visibility-sensor@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/react-visibility-sensor/-/react-visibility-sensor-5.1.1.tgz#5238380960d3a0b2be0b7faddff38541e337f5a9" + integrity sha512-cTUHqIK+zDYpeK19rzW6zF9YfT4486TIgizZW53wEZ+/GPBbK7cNS0EHyJVyHYacwFEvvHLEKfgJndbemWhB/w== + dependencies: + prop-types "^15.7.2" + react@17.0.2: version "17.0.2" resolved "https://registry.yarnpkg.com/react/-/react-17.0.2.tgz#d0b5cc516d29eb3eee383f75b62864cfb6800037" From 2614310ea887a81ca177505489a61b244ace7465 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 10 May 2022 13:11:43 +0800 Subject: [PATCH 19/58] Lazy load blocks in references and queries --- src/main/frontend/components/block.cljs | 92 +++++++++++++-------- src/main/frontend/components/journal.cljs | 2 +- src/main/frontend/components/page.cljs | 2 +- src/main/frontend/components/reference.cljs | 28 ++----- src/main/frontend/db/model.cljs | 19 ++++- src/main/frontend/db/react.cljs | 33 ++++---- src/main/frontend/handler/editor.cljs | 17 ++-- src/main/frontend/ui.cljs | 12 ++- 8 files changed, 115 insertions(+), 90 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 1f69a120df0..2fe2bd7627e 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -2225,37 +2225,11 @@ (= (:id config) (str (:block/uuid block))))) -(rum/defcs ^:large-vars/cleanup-todo block-container < rum/reactive - {:init (fn [state] - (let [[config block] (:rum/args state) - block-id (:block/uuid block)] - (cond - (root-block? config block) - (state/set-collapsed-block! block-id false) - - (:ref? config) - (state/set-collapsed-block! block-id - (editor-handler/block-default-collapsed? block config)) - - :else - nil) - (assoc state - ::control-show? (atom false) - ::navigating-block (atom (:block/uuid block))))) - :should-update (fn [old-state new-state] - (let [compare-keys [:block/uuid :block/content :block/parent :block/collapsed? - :block/properties :block/left :block/children :block/_refs :ui/selected?] - config-compare-keys [:show-cloze?] - b1 (second (:rum/args old-state)) - b2 (second (:rum/args new-state)) - result (or - (not= (select-keys b1 compare-keys) - (select-keys b2 compare-keys)) - (not= (select-keys (first (:rum/args old-state)) config-compare-keys) - (select-keys (first (:rum/args new-state)) config-compare-keys)))] - (boolean result)))} - [state config block] - (let [repo (state/get-current-repo) +(rum/defc block-container-inner < rum/reactive db-mixins/query + [state repo config block] + (let [ref? (:ref? config) + custom-query? (boolean (:custom-query? config)) + ref-or-custom-query? (or ref? custom-query?) *navigating-block (get state ::navigating-block) navigating-block (rum/react *navigating-block) navigated? (and (not= (:block/uuid block) navigating-block) navigating-block) @@ -2277,17 +2251,22 @@ config) heading? (or (= type :heading) (and heading-level (<= heading-level 6))) *control-show? (get state ::control-show?) - ref? (:ref? config) db-collapsed? (util/collapsed? block) collapsed? (cond - (or ref? (root-block? config block)) + (or ref? custom-query? (root-block? config block)) (state/sub-collapsed uuid) :else db-collapsed?) + children (if (and (or ref? custom-query?) + (not collapsed?)) + (map + (fn [b] (assoc b + :block/level (inc (:block/level block)))) + (model/sub-block-direct-children repo uuid)) + children) breadcrumb-show? (:breadcrumb-show? config) slide? (boolean (:slide? config)) - custom-query? (boolean (:custom-query? config)) doc-mode? (:document/mode? config) embed? (:embed? config) reference? (:reference? config) @@ -2357,6 +2336,51 @@ (dnd-separator-wrapper block block-id slide? false false)])) +(rum/defcs ^:large-vars/cleanup-todo block-container < rum/reactive + (rum/local false ::visible?) + {:init (fn [state] + (let [[config block] (:rum/args state) + block-id (:block/uuid block)] + (cond + (root-block? config block) + (state/set-collapsed-block! block-id false) + + (:ref? config) + (state/set-collapsed-block! block-id + (editor-handler/block-default-collapsed? block config)) + + :else + nil) + (assoc state + ::control-show? (atom false) + ::navigating-block (atom (:block/uuid block))))) + :should-update (fn [old-state new-state] + (let [compare-keys [:block/uuid :block/content :block/parent :block/collapsed? + :block/properties :block/left :block/children :block/_refs :ui/selected?] + config-compare-keys [:show-cloze?] + b1 (second (:rum/args old-state)) + b2 (second (:rum/args new-state)) + result (or + (not= (select-keys b1 compare-keys) + (select-keys b2 compare-keys)) + (not= (select-keys (first (:rum/args old-state)) config-compare-keys) + (select-keys (first (:rum/args new-state)) config-compare-keys)))] + (boolean result)))} + [state config block] + (let [repo (state/get-current-repo) + ref? (:ref? config) + custom-query? (boolean (:custom-query? config)) + ref-or-custom-query? (or ref? custom-query?)] + (if ref-or-custom-query? + (let [*visible? (::visible? state)] + (ui/lazy-visible + *visible? + nil + (fn [] + (block-container-inner state repo config block)) + nil)) + (block-container-inner state repo config block)))) + (defn divide-lists [[f & l]] (loop [l l diff --git a/src/main/frontend/components/journal.cljs b/src/main/frontend/components/journal.cljs index e9496d66710..d3bf65e0709 100644 --- a/src/main/frontend/components/journal.cljs +++ b/src/main/frontend/components/journal.cljs @@ -60,7 +60,7 @@ (page/today-queries repo today? false) (rum/with-key - (reference/references title) + (reference/references title false) (str title "-refs"))])) (rum/defc journals < rum/reactive diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index b8f05738aac..aa8de6bb1dc 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -394,7 +394,7 @@ ;; referenced blocks [:div {:key "page-references"} (rum/with-key - (reference/references route-page-name) + (reference/references route-page-name sidebar?) (str route-page-name "-refs"))] (when-not block? diff --git a/src/main/frontend/components/reference.cljs b/src/main/frontend/components/reference.cljs index 092685f2537..b34169ebd2c 100644 --- a/src/main/frontend/components/reference.cljs +++ b/src/main/frontend/components/reference.cljs @@ -49,24 +49,6 @@ (fn [close-fn] (filter-dialog-inner filters-atom close-fn references page-name))) -(defn- block-with-ref-level - [block level] - (if (:block/children block) - (-> (update block :block/children - (fn [blocks] - (map (fn [block] - (let [level (inc level) - block (assoc block :ref/level level)] - (block-with-ref-level block level))) blocks))) - (assoc :ref/level level)) - (assoc block :ref/level level))) - -(defn- blocks-with-ref-level - [page-blocks] - (map (fn [[page blocks]] - [page (map #(block-with-ref-level % 1) blocks)]) - page-blocks)) - (rum/defc block-linked-references < rum/reactive db-mixins/query [block-id] (let [refed-blocks-ids (model-db/get-referenced-blocks-ids (str block-id))] @@ -163,8 +145,7 @@ filters (when (seq filter-state) (->> (group-by second filter-state) (medley/map-vals #(map first %)))) - filtered-ref-blocks (->> (block-handler/filter-blocks repo ref-blocks filters true) - blocks-with-ref-level) + filtered-ref-blocks (block-handler/filter-blocks repo ref-blocks filters true) n-ref (apply + (for [[_ rfs] filtered-ref-blocks] (count rfs)))] @@ -186,14 +167,15 @@ (rum/defcs references < (rum/local false ::visible?) - [state page-name] + [state page-name sidebar?] (ui/catch-error (ui/component-error "Linked References: Unexpected error") (ui/lazy-visible (::visible? state) - "loading references..." + (if sidebar? nil "loading references...") (fn [] - (references* page-name))))) + (references* page-name)) + nil))) (rum/defcs unlinked-references-aux < rum/reactive db-mixins/query diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index c48191236d0..7f620300d24 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -815,6 +815,20 @@ block-uuid) (sort-by-left (db-utils/entity [:block/uuid block-uuid]))))) +(defn sub-block-direct-children + "Doesn't include nested children." + [repo block-uuid] + (when-let [db (conn/get-db repo)] + (-> (react/q repo [:frontend.db.react/block-direct-children block-uuid] {} + '[:find [(pull ?b [*]) ...] + :in $ ?parent-id + :where + [?parent :block/uuid ?parent-id] + [?b :block/parent ?parent]] + block-uuid) + react + (sort-by-left (db-utils/entity [:block/uuid block-uuid]))))) + (defn get-block-children "Including nested children." [repo block-uuid] @@ -1080,10 +1094,7 @@ :where [?block :block/refs ?ref-page]] pages - block-attrs - ;; TODO: :block/_parent slow recursive query - ;; (butlast block-attrs) - ) + (butlast block-attrs)) result (->> query-result react (remove (fn [block] diff --git a/src/main/frontend/db/react.cljs b/src/main/frontend/db/react.cljs index 98dd3cd3fca..6f674d5ff2b 100644 --- a/src/main/frontend/db/react.cljs +++ b/src/main/frontend/db/react.cljs @@ -24,6 +24,8 @@ ;; ::block-and-children ;; get block&children react-query (s/def ::block-and-children (s/tuple #(= ::block-and-children %) uuid?)) + +(s/def ::block-direct-children (s/tuple #(= ::block-direct-children %) uuid?)) ;; ::journals ;; get journal-list react-query (s/def ::journals (s/tuple #(= ::journals %))) @@ -48,6 +50,7 @@ (s/def ::react-query-keys (s/or :block ::block :page-blocks ::page-blocks :block-and-children ::block-and-children + :block-direct-children ::block-direct-children :journals ::journals :page->pages ::page->pages :page<-pages ::page<-pages @@ -238,7 +241,8 @@ blocks [[::block (:block/uuid block)]] others (when page-id [[::page-blocks page-id] - [::page->pages page-id]])] + [::page->pages page-id] + [::block-direct-children (:block/uuid (:block/parent block))]])] (concat blocks others))))) blocks) @@ -307,19 +311,20 @@ (or (get affected-keys (vec (rest k))) custom? kv?)) - (let [{:keys [query query-fn]} cache] - (when (or query query-fn) - (try - (let [f #(execute-query! repo-url db k tx cache)] - ;; Detects whether user is editing in a custom query, if so, execute the query immediately - (if (and custom? - ;; modifying during cards review need to be executed immediately - (not (:cards-query? (meta query))) - (not (state/edit-in-query-component))) - (async/put! (state/get-reactive-custom-queries-chan) [f query]) - (f))) - (catch js/Error e - (js/console.error e))))))))))) + (util/profile (str "refresh! " (second k)) + (let [{:keys [query query-fn]} cache] + (when (or query query-fn) + (try + (let [f #(execute-query! repo-url db k tx cache)] + ;; Detects whether user is editing in a custom query, if so, execute the query immediately + (if (and custom? + ;; modifying during cards review need to be executed immediately + (not (:cards-query? (meta query))) + (not (state/edit-in-query-component))) + (async/put! (state/get-reactive-custom-queries-chan) [f query]) + (f))) + (catch js/Error e + (js/console.error e)))))))))))) (defn set-key-value [repo-url key value] diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 2bd2a00d8c6..7b976bb2b9f 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -3475,12 +3475,11 @@ 1. References. 2. Custom queries." [block config] - (if (or (:ref? config) - (:custom-query? config)) - (and - (seq (:block/children block)) - (or - (:custom-query? config) - (>= (:ref/level block) - (state/get-ref-open-blocks-level)))) - (util/collapsed? block))) + (or + (and + (:ref? config) + (>= (inc (:block/level block)) + (state/get-ref-open-blocks-level)) + ;; has children + (first (:block/_parent (db/entity (:db/id block))))) + (util/collapsed? block))) diff --git a/src/main/frontend/ui.cljs b/src/main/frontend/ui.cljs index 9c58dfa3f40..df6574eec22 100644 --- a/src/main/frontend/ui.cljs +++ b/src/main/frontend/ui.cljs @@ -916,12 +916,16 @@ [:div.lazy-visibility (if visible? (when (fn? content-fn) (content-fn)) - (loading (or loading-label "loading...")))]) + (when loading-label (loading loading-label)))]) (rum/defc lazy-visible < rum/reactive - [*visible? loading-label content-fn] + [*visible? loading-label content-fn sensor-opts] (let [visible? (rum/react *visible?)] (visibility-sensor - {:on-change #(reset! *visible? %) - :partialVisibility true} + (merge + {:on-change #(reset! *visible? %) + :partialVisibility true + :offset {:top -300 + :bottom -300}} + sensor-opts) (lazy-visible-inner visible? content-fn loading-label)))) From d1c6d4ce885c223c3198a6d22c272147fc161809 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 10 May 2022 15:12:09 +0800 Subject: [PATCH 20/58] fix: indent/outdent blocks not updated --- src/main/frontend/components/block.cljs | 8 ++------ src/main/frontend/db/react.cljs | 17 +++++++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 2fe2bd7627e..aaa1debfb20 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -2271,11 +2271,7 @@ embed? (:embed? config) reference? (:reference? config) block-id (str "ls-block-" blocks-container-id "-" uuid) - has-child? (boolean - (and - (not pre-block?) - (coll? children) - (seq children))) + has-child? (first (:block/_parent (db/entity (:db/id block)))) attrs (on-drag-and-mouse-attrs block uuid top? block-id *move-to) children-refs (get-children-refs children) data-refs (build-refs-data-value children-refs) @@ -2371,7 +2367,7 @@ ref? (:ref? config) custom-query? (boolean (:custom-query? config)) ref-or-custom-query? (or ref? custom-query?)] - (if ref-or-custom-query? + (if (and ref-or-custom-query? (not (:ref-child? config))) (let [*visible? (::visible? state)] (ui/lazy-visible *visible? diff --git a/src/main/frontend/db/react.cljs b/src/main/frontend/db/react.cljs index 6f674d5ff2b..16e41985375 100644 --- a/src/main/frontend/db/react.cljs +++ b/src/main/frontend/db/react.cljs @@ -216,9 +216,8 @@ (defn get-affected-queries-keys "Get affected queries through transaction datoms." - [{:keys [tx-data]}] + [{:keys [tx-data tx-meta db-before]}] {:post [(s/valid? ::affected-keys %)]} - (let [blocks (->> (filter (fn [datom] (contains? #{:block/left :block/parent :block/page} (:a datom))) tx-data) (map :v) (distinct)) @@ -240,9 +239,15 @@ (:db/id (:block/page block))) blocks [[::block (:block/uuid block)]] others (when page-id - [[::page-blocks page-id] - [::page->pages page-id] - [::block-direct-children (:block/uuid (:block/parent block))]])] + (let [db-after-parent-uuid (:block/uuid (:block/parent block)) + db-before-parent-uuid (:block/uuid (:block/parent (d/entity db-before + [:block/uuid (:block/uuid block)])))] + [[::page-blocks page-id] + [::page->pages page-id] + [::block-direct-children db-after-parent-uuid] + (when (and db-before-parent-uuid + (not= db-before-parent-uuid db-after-parent-uuid)) + [::block-direct-children db-before-parent-uuid])]))] (concat blocks others))))) blocks) @@ -255,7 +260,7 @@ (if (:block/name entity) ; page [::page-blocks ref] [::page-blocks (:db/id (:block/page entity))]))) - refs)) + refs)) others (->> (keys @query-state) (filter (fn [ks] From 8c2e07275b0468a75e14da0eed4a05beba45e286 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 10 May 2022 16:06:53 +0800 Subject: [PATCH 21/58] Lazy loading journals and queries --- src/main/frontend/components/block.cljs | 28 +++++++++++---------- src/main/frontend/components/journal.cljs | 6 ++++- src/main/frontend/components/reference.cljs | 10 ++++---- src/main/frontend/handler/editor.cljs | 9 ++++--- src/main/frontend/ui.cljs | 14 ++++++----- 5 files changed, 38 insertions(+), 29 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index aaa1debfb20..1014df8e109 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -1403,6 +1403,7 @@ (rum/defc block-children < rum/reactive [config children collapsed?] (let [ref? (:ref? config) + query? (:custom-query? config) children (and (coll? children) (filter map? children))] (when (and (coll? children) (seq children) @@ -1418,8 +1419,8 @@ (-> config (assoc :block/uuid (:block/uuid child)) (dissoc :breadcrumb-show? :embed-parent)) - ref? - (assoc :ref-child? true))] + (or ref? query?) + (assoc :ref-query-child? true))] (rum/with-key (block-container config child) (:block/uuid child)))))]])))) @@ -2081,7 +2082,8 @@ (when show? (let [page-name-props (when show-page? [page - (or page-original-name page-name)]) + (page-cp config page) + {:block/name (or page-original-name page-name)}]) parents-props (doall (for [{:block/keys [uuid name content] :as block} parents] (when-not name ; not page @@ -2333,7 +2335,6 @@ (dnd-separator-wrapper block block-id slide? false false)])) (rum/defcs ^:large-vars/cleanup-todo block-container < rum/reactive - (rum/local false ::visible?) {:init (fn [state] (let [[config block] (:rum/args state) block-id (:block/uuid block)] @@ -2367,14 +2368,12 @@ ref? (:ref? config) custom-query? (boolean (:custom-query? config)) ref-or-custom-query? (or ref? custom-query?)] - (if (and ref-or-custom-query? (not (:ref-child? config))) - (let [*visible? (::visible? state)] - (ui/lazy-visible - *visible? - nil - (fn [] - (block-container-inner state repo config block)) - nil)) + (if (and ref-or-custom-query? (not (:ref-query-child? config))) + (ui/lazy-visible + nil + (fn [] + (block-container-inner state repo config block)) + nil) (block-container-inner state repo config block)))) (defn divide-lists @@ -2701,7 +2700,10 @@ [config q] (ui/catch-error (ui/block-error "Query Error:" {:content (:query q)}) - (custom-query* config q))) + (ui/lazy-visible + "loading ..." + (fn [] (custom-query* config q)) + nil))) (defn admonition [config type result] diff --git a/src/main/frontend/components/journal.cljs b/src/main/frontend/components/journal.cljs index d3bf65e0709..07a47c46240 100644 --- a/src/main/frontend/components/journal.cljs +++ b/src/main/frontend/components/journal.cljs @@ -21,7 +21,7 @@ (when-let [page-e (db/pull [:block/name (util/page-name-sanity-lc page)])] (page/page-blocks-cp repo page-e {}))) -(rum/defc journal-cp < rum/reactive +(rum/defc journal-cp-inner < rum/reactive [[title format]] (let [;; Don't edit the journal title page (string/lower-case title) @@ -63,6 +63,10 @@ (reference/references title false) (str title "-refs"))])) +(rum/defc journal-cp + [journal] + (ui/lazy-visible nil (fn [] (journal-cp-inner journal)) nil)) + (rum/defc journals < rum/reactive [latest-journals] [:div#journals diff --git a/src/main/frontend/components/reference.cljs b/src/main/frontend/components/reference.cljs index b34169ebd2c..9e25e611d10 100644 --- a/src/main/frontend/components/reference.cljs +++ b/src/main/frontend/components/reference.cljs @@ -165,14 +165,14 @@ {:default-collapsed? default-collapsed? :title-trigger? true}))]])))) -(rum/defcs references < - (rum/local false ::visible?) - [state page-name sidebar?] +(rum/defc references + [page-name sidebar?] (ui/catch-error (ui/component-error "Linked References: Unexpected error") (ui/lazy-visible - (::visible? state) - (if sidebar? nil "loading references...") + (if (or sidebar? (gp-util/uuid-string? page-name)) + nil + "loading references...") (fn [] (references* page-name)) nil))) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 7b976bb2b9f..18a21795b9c 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -453,11 +453,12 @@ (declare save-current-block!) (defn outliner-insert-block! [config current-block new-block {:keys [sibling? keep-uuid? replace-empty-target?]}] - (let [ref-top-block? (and (:ref? config) - (not (:ref-child? config))) + (let [ref-query-top-block? (and (:ref? config) + (:custom-query? config) + (not (:ref-or-query? config))) has-children? (db/has-children? (:block/uuid current-block)) sibling? (cond - ref-top-block? + ref-query-top-block? false (boolean? sibling?) @@ -3477,7 +3478,7 @@ [block config] (or (and - (:ref? config) + (or (:ref? config) (:custom-query? config)) (>= (inc (:block/level block)) (state/get-ref-open-blocks-level)) ;; has children diff --git a/src/main/frontend/ui.cljs b/src/main/frontend/ui.cljs index df6574eec22..f60219091fa 100644 --- a/src/main/frontend/ui.cljs +++ b/src/main/frontend/ui.cljs @@ -684,7 +684,7 @@ :margin-left -30}} (not title-trigger?) (assoc :on-mouse-down on-mouse-down)) - [:span {:class (if @control? "control-show cursor-pointer" "control-hide")} + [:span {:class (if (or @control? @collapsed?) "control-show cursor-pointer" "control-hide")} (rotating-arrow @collapsed?)]]) (if (fn? header) (header @collapsed?) @@ -916,11 +916,13 @@ [:div.lazy-visibility (if visible? (when (fn? content-fn) (content-fn)) - (when loading-label (loading loading-label)))]) + (when loading-label [:span.text-sm.font-medium + loading-label]))]) -(rum/defc lazy-visible < rum/reactive - [*visible? loading-label content-fn sensor-opts] - (let [visible? (rum/react *visible?)] +(rum/defcs lazy-visible < + (rum/local false ::visible?) + [state loading-label content-fn sensor-opts] + (let [*visible? (::visible? state)] (visibility-sensor (merge {:on-change #(reset! *visible? %) @@ -928,4 +930,4 @@ :offset {:top -300 :bottom -300}} sensor-opts) - (lazy-visible-inner visible? content-fn loading-label)))) + (lazy-visible-inner @*visible? content-fn loading-label)))) From 278f2ff1f952857877219c9c641f794d5ad5b4a8 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 10 May 2022 18:16:17 +0800 Subject: [PATCH 22/58] fix: lint warnings --- src/main/frontend/components/block.cljs | 25 +++++++++++++++++-------- src/main/frontend/db/react.cljs | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 1014df8e109..9b8a7dfe228 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -489,8 +489,16 @@ page-name-in-block page-name redirect-page-name page-entity contents-page? children html-export? label)] - (if (and (not (util/mobile?)) (not preview?)) + (cond + (:breadcrumb? config) + (or (:block/original-name page) + (:block/name page)) + + (and (not (util/mobile?)) + (not preview?)) (page-preview-trigger (assoc config :children inner) page-name) + + :else inner)))) (rum/defc asset-reference @@ -2078,11 +2086,12 @@ (rest parents) parents) more? (> (count parents) level-limit) - parents (if more? (take-last level-limit parents) parents)] + parents (if more? (take-last level-limit parents) parents) + config (assoc config :breadcrumb? true)] (when show? (let [page-name-props (when show-page? [page - (page-cp config page) + (page-cp (dissoc config :breadcrumb? true) page) {:block/name (or page-original-name page-name)}]) parents-props (doall (for [{:block/keys [uuid name content] :as block} parents] @@ -2227,7 +2236,7 @@ (= (:id config) (str (:block/uuid block))))) -(rum/defc block-container-inner < rum/reactive db-mixins/query +(rum/defc ^:large-vars/cleanup-todo block-container-inner < rum/reactive db-mixins/query [state repo config block] (let [ref? (:ref? config) custom-query? (boolean (:custom-query? config)) @@ -2255,12 +2264,12 @@ *control-show? (get state ::control-show?) db-collapsed? (util/collapsed? block) collapsed? (cond - (or ref? custom-query? (root-block? config block)) + (or ref-or-custom-query? (root-block? config block)) (state/sub-collapsed uuid) :else db-collapsed?) - children (if (and (or ref? custom-query?) + children (if (and ref-or-custom-query? (not collapsed?)) (map (fn [b] (assoc b @@ -2334,7 +2343,7 @@ (dnd-separator-wrapper block block-id slide? false false)])) -(rum/defcs ^:large-vars/cleanup-todo block-container < rum/reactive +(rum/defcs block-container < rum/reactive {:init (fn [state] (let [[config block] (:rum/args state) block-id (:block/uuid block)] @@ -2342,7 +2351,7 @@ (root-block? config block) (state/set-collapsed-block! block-id false) - (:ref? config) + (or (:ref? config) (:custom-query? config)) (state/set-collapsed-block! block-id (editor-handler/block-default-collapsed? block config)) diff --git a/src/main/frontend/db/react.cljs b/src/main/frontend/db/react.cljs index 16e41985375..30208aa1f6b 100644 --- a/src/main/frontend/db/react.cljs +++ b/src/main/frontend/db/react.cljs @@ -216,7 +216,7 @@ (defn get-affected-queries-keys "Get affected queries through transaction datoms." - [{:keys [tx-data tx-meta db-before]}] + [{:keys [tx-data db-before]}] {:post [(s/valid? ::affected-keys %)]} (let [blocks (->> (filter (fn [datom] (contains? #{:block/left :block/parent :block/page} (:a datom))) tx-data) (map :v) From 9042aa47c55aa78ade8446507fbc54886d87d795 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 10 May 2022 18:59:38 +0800 Subject: [PATCH 23/58] fix: block wrong indentation in queries --- src/main/frontend/db/react.cljs | 27 +++++++++++++-------------- src/main/frontend/handler/editor.cljs | 6 +++--- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/main/frontend/db/react.cljs b/src/main/frontend/db/react.cljs index 30208aa1f6b..4508bee685b 100644 --- a/src/main/frontend/db/react.cljs +++ b/src/main/frontend/db/react.cljs @@ -316,20 +316,19 @@ (or (get affected-keys (vec (rest k))) custom? kv?)) - (util/profile (str "refresh! " (second k)) - (let [{:keys [query query-fn]} cache] - (when (or query query-fn) - (try - (let [f #(execute-query! repo-url db k tx cache)] - ;; Detects whether user is editing in a custom query, if so, execute the query immediately - (if (and custom? - ;; modifying during cards review need to be executed immediately - (not (:cards-query? (meta query))) - (not (state/edit-in-query-component))) - (async/put! (state/get-reactive-custom-queries-chan) [f query]) - (f))) - (catch js/Error e - (js/console.error e)))))))))))) + (let [{:keys [query query-fn]} cache] + (when (or query query-fn) + (try + (let [f #(execute-query! repo-url db k tx cache)] + ;; Detects whether user is editing in a custom query, if so, execute the query immediately + (if (and custom? + ;; modifying during cards review need to be executed immediately + (not (:cards-query? (meta query))) + (not (state/edit-in-query-component))) + (async/put! (state/get-reactive-custom-queries-chan) [f query]) + (f))) + (catch js/Error e + (js/console.error e))))))))))) (defn set-key-value [repo-url key value] diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 18a21795b9c..d60e53b7957 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -453,9 +453,9 @@ (declare save-current-block!) (defn outliner-insert-block! [config current-block new-block {:keys [sibling? keep-uuid? replace-empty-target?]}] - (let [ref-query-top-block? (and (:ref? config) - (:custom-query? config) - (not (:ref-or-query? config))) + (let [ref-query-top-block? (and (or (:ref? config) + (:custom-query? config)) + (not (:ref-query-child? config))) has-children? (db/has-children? (:block/uuid current-block)) sibling? (cond ref-query-top-block? From ce9eff332d07b4e7041dca348ac6b3b30e268b2d Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 12 May 2022 08:22:15 +0800 Subject: [PATCH 24/58] fix: prefer block template if there're other page templates with the same name --- src/main/frontend/db/model.cljs | 5 +++-- src/main/frontend/modules/outliner/datascript.cljc | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index 7f620300d24..90e2b9d6820 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -1321,7 +1321,7 @@ [name] (when (string? name) (->> (d/q - '[:find (pull ?b [*]) + '[:find [(pull ?b [*]) ...] :in $ ?name :where [?b :block/properties ?p] @@ -1329,7 +1329,8 @@ [(= ?t ?name)]] (conn/get-db) name) - ffirst))) + (sort-by :block/name) + (first)))) (defonce blocks-count-cache (atom nil)) diff --git a/src/main/frontend/modules/outliner/datascript.cljc b/src/main/frontend/modules/outliner/datascript.cljc index 244f0e08cc3..f1da4d05243 100644 --- a/src/main/frontend/modules/outliner/datascript.cljc +++ b/src/main/frontend/modules/outliner/datascript.cljc @@ -45,7 +45,7 @@ txs (map (fn [m] (if (map? m) (dissoc m :block/children :block/meta :block/top? :block/bottom? :block/anchor - :block/title :block/body :block/level :block/container) + :block/title :block/body :block/level :block/container :db/other-tx) m)) txs)] (when (and (seq txs) (not (:skip-transact? opts))) From 8fcf5b5d60e32c4a0445660e3f83d93056a29338 Mon Sep 17 00:00:00 2001 From: charlie Date: Thu, 5 May 2022 18:06:21 +0800 Subject: [PATCH 25/58] enhance(ui): show network proxy hints on plugins page if it's settled --- src/main/frontend/components/plugins.cljs | 64 +++++++++++++++++++++- src/main/frontend/components/settings.cljs | 38 +------------ src/main/frontend/handler/events.cljs | 6 ++ 3 files changed, 69 insertions(+), 39 deletions(-) diff --git a/src/main/frontend/components/plugins.cljs b/src/main/frontend/components/plugins.cljs index 3f7a3d8eefe..102d6a4201d 100644 --- a/src/main/frontend/components/plugins.cljs +++ b/src/main/frontend/components/plugins.cljs @@ -8,6 +8,7 @@ [frontend.search :as search] [frontend.util :as util] [frontend.mixins :as mixins] + [electron.ipc :as ipc] [promesa.core :as p] [frontend.components.svg :as svg] [frontend.components.plugins-settings :as plugins-settings] @@ -334,10 +335,45 @@ :intent "logseq" :target "_blank")) +(rum/defc user-proxy-settings-panel + [{:keys [protocol] :as agent-opts}] + (let [[opts set-opts!] (rum/use-state agent-opts) + disabled? (string/blank? (:protocol opts))] + [:div.cp__settings-network-proxy-panel + [:h1.mb-2.text-2xl.font-bold (t :settings-page/network-proxy)] + [:div.p-2 + [:p [:label [:strong (t :type)] + (ui/select [{:label "Disabled" :value "" :selected disabled?} + {:label "http" :value "http" :selected (= protocol "http")} + {:label "https" :value "https" :selected (= protocol "https")} + {:label "socks5" :value "socks5" :selected (= protocol "socks5")}] + #(set-opts! + (assoc opts :protocol (if (= "disabled" (util/safe-lower-case %)) nil %))) nil)]] + [:p.flex + [:label.pr-4 [:strong (t :host)] + [:input.form-input.is-small + {:value (:host opts) :disabled disabled? + :on-change #(set-opts! + (assoc opts :host (util/trim-safe (util/evalue %))))}]] + + [:label [:strong (t :port)] + [:input.form-input.is-small + {:value (:port opts) :type "number" :disabled disabled? + :on-change #(set-opts! + (assoc opts :port (util/trim-safe (util/evalue %))))}]]] + + [:p.pt-2 + (ui/button (t :save) + :on-click (fn [] + (p/let [_ (ipc/ipc :setHttpsAgent opts)] + (state/set-state! [:electron/user-cfgs :settings/agent] opts) + (state/close-sub-modal! :https-proxy-panel))))]]])) + (rum/defc ^:large-vars/cleanup-todo panel-control-tabs < rum/static [search-key *search-key category *category sort-by *sort-by filter-by *filter-by - selected-unpacked-pkg market? develop-mode? reload-market-fn] + selected-unpacked-pkg market? develop-mode? + reload-market-fn agent-opts] (let [*search-ref (rum/create-ref)] [:div.mb-2.flex.justify-between.control-tabs.relative @@ -357,6 +393,16 @@ (unpacked-plugin-loader selected-unpacked-pkg)])] [:div.flex.items-center.r + ;; extra info + (let [{:keys [protocol host port]} agent-opts] + (when (every? not-empty [protocol host port]) + (ui/button + [:span.flex.items-center.text-indigo-500 + (ui/icon "world-download") (str protocol "://" host ":" port)] + :small? true + :intent "link" + :on-click #(state/pub-event! [:go/proxy-settings agent-opts])))) + ;; search (panel-tab-search search-key *search-key *search-ref) @@ -440,6 +486,10 @@ :options {:on-click #(reload-market-fn)}}] [{:title [:span (ui/icon "rotate-clockwise") (t :plugin/check-all-updates)] :options {:on-click #(plugin-handler/check-enabled-for-updates (not= :plugins category))}}]) + + [{:title [:span (ui/icon "world") (t :settings-page/network-proxy)] + :options {:on-click #(state/pub-event! [:go/proxy-settings agent-opts])}}] + (when (state/developer-mode?) [{:hr true} {:title [:span (ui/icon "file-code") "Open Preferences"] @@ -481,6 +531,7 @@ installing (state/sub :plugin/installing) online? (state/sub :network/online?) develop-mode? (state/sub :ui/developer-mode?) + agent-opts (state/sub [:electron/user-cfgs :settings/agent]) *search-key (::search-key state) *category (::category state) *sort-by (::sort-by state) @@ -526,7 +577,8 @@ @*search-key *search-key @*category *category @*sort-by *sort-by @*filter-by *filter-by - nil true develop-mode? (::reload state)) + nil true develop-mode? (::reload state) + agent-opts) (cond (not online?) @@ -568,6 +620,7 @@ develop-mode? (state/sub :ui/developer-mode?) selected-unpacked-pkg (state/sub :plugin/selected-unpacked-pkg) coming-updates (state/sub :plugin/updates-coming) + agent-opts (state/sub [:electron/user-cfgs :settings/agent]) *filter-by (::filter-by state) *sort-by (::sort-by state) *search-key (::search-key state) @@ -611,7 +664,8 @@ @*sort-by *sort-by @*filter-by *filter-by selected-unpacked-pkg - false develop-mode? nil) + false develop-mode? nil + agent-opts) [:div.cp__plugins-item-lists.grid-cols-1.md:grid-cols-2.lg:grid-cols-3 (for [item sorted-plugins] @@ -752,6 +806,10 @@ market? (= active :marketplace) *el-ref (rum/create-ref)] + (rum/use-effect! + #(state/load-app-user-cfgs) + []) + [:div.cp__plugins-page {:ref *el-ref :tab-index "-1"} diff --git a/src/main/frontend/components/settings.cljs b/src/main/frontend/components/settings.cljs index d324c778ccc..3f6fd7723c5 100644 --- a/src/main/frontend/components/settings.cljs +++ b/src/main/frontend/components/settings.cljs @@ -1,6 +1,7 @@ (ns frontend.components.settings (:require [clojure.string :as string] [frontend.components.svg :as svg] + [frontend.components.plugins :as plugins] [frontend.config :as config] [frontend.context.i18n :refer [t]] [frontend.storage :as storage] @@ -134,7 +135,6 @@ (mobile-util/is-native-platform?)) [:div.text-sm desc])]]) - (defn edit-config-edn [] (row-with-button-action {:left-label (t :settings-page/custom-configuration) @@ -485,40 +485,6 @@ :on-click #(js/logseq.api.relaunch) :small? true :intent "logseq")]])])) -(rum/defc user-proxy-settings-panel - [{:keys [protocol] :as agent-opts}] - (let [[opts set-opts!] (rum/use-state agent-opts) - disabled? (string/blank? (:protocol opts))] - [:div.cp__settings-network-proxy-panel - [:h1.mb-2.text-2xl.font-bold (t :settings-page/network-proxy)] - [:div.p-2 - [:p [:label [:strong (t :type)] - (ui/select [{:label "Disabled" :value "" :selected disabled?} - {:label "http" :value "http" :selected (= protocol "http")} - {:label "https" :value "https" :selected (= protocol "https")} - {:label "socks5" :value "socks5" :selected (= protocol "socks5")}] - #(set-opts! - (assoc opts :protocol (if (= "disabled" (util/safe-lower-case %)) nil %))) nil)]] - [:p.flex - [:label.pr-4 [:strong (t :host)] - [:input.form-input.is-small - {:value (:host opts) :disabled disabled? - :on-change #(set-opts! - (assoc opts :host (util/trim-safe (util/evalue %))))}]] - - [:label [:strong (t :port)] - [:input.form-input.is-small - {:value (:port opts) :type "number" :disabled disabled? - :on-change #(set-opts! - (assoc opts :port (util/trim-safe (util/evalue %))))}]]] - - [:p.pt-2 - (ui/button (t :save) - :on-click (fn [] - (p/let [_ (ipc/ipc :setHttpsAgent opts)] - (state/set-state! [:electron/user-cfgs :settings/agent] opts) - (state/close-sub-modal! :https-proxy-panel))))]]])) - (rum/defc user-proxy-settings [{:keys [protocol host port] :as agent-opts}] (ui/button [:span @@ -526,7 +492,7 @@ [:strong.pr-1 e]) (ui/icon "edit")] :on-click #(state/set-sub-modal! - (fn [_] (user-proxy-settings-panel agent-opts)) + (fn [_] (plugins/user-proxy-settings-panel agent-opts)) {:id :https-proxy-panel :center? true}))) (defn plugin-system-switcher-row [] diff --git a/src/main/frontend/handler/events.cljs b/src/main/frontend/handler/events.cljs index 50a455d0f83..451e698b980 100644 --- a/src/main/frontend/handler/events.cljs +++ b/src/main/frontend/handler/events.cljs @@ -269,6 +269,12 @@ (plugin/open-focused-settings-modal! title)) (state/close-sub-modal! "ls-focused-settings-modal"))) +(defmethod handle :go/proxy-settings [[_ agent-opts]] + (js/console.log agent-opts) + (state/set-sub-modal! + (fn [_] (plugin/user-proxy-settings-panel agent-opts)) + {:id :https-proxy-panel :center? true})) + (defmethod handle :redirect-to-home [_] (page-handler/create-today-journal!)) From da4d507bcc438158dca95e28c9cb4488937db359 Mon Sep 17 00:00:00 2001 From: charlie Date: Fri, 6 May 2022 11:32:39 +0800 Subject: [PATCH 26/58] enhance(ux): test url for proxy settings --- src/electron/electron/handler.cljs | 4 +++ src/electron/electron/utils.cljs | 2 +- src/main/frontend/components/plugins.cljs | 32 ++++++++++++++++++++--- src/main/frontend/handler/events.cljs | 1 - 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs index d3e682cf1e2..13f2823a808 100644 --- a/src/electron/electron/handler.cljs +++ b/src/electron/electron/handler.cljs @@ -314,6 +314,10 @@ (defmethod handle :getLogseqDotDirRoot [] (utils/get-ls-dotdir-root)) +(defmethod handle :testProxyUrl [win [_ url]] + (p/let [_ (utils/fetch url)] + (utils/send-to-renderer win :notification {:type "success" :payload (str "Successfully: " url)}))) + (defmethod handle :getUserDefaultPlugins [] (utils/get-ls-default-plugins)) diff --git a/src/electron/electron/utils.cljs b/src/electron/electron/utils.cljs index f24e9053ddb..a8e7ef4c9c4 100644 --- a/src/electron/electron/utils.cljs +++ b/src/electron/electron/utils.cljs @@ -118,7 +118,7 @@ ([window kind payload] (when window (.. ^js window -webContents - (send kind (bean/->js payload)))))) + (send (name kind) (bean/->js payload)))))) (defn get-graph-dir [graph-name] diff --git a/src/main/frontend/components/plugins.cljs b/src/main/frontend/components/plugins.cljs index 102d6a4201d..5ad38b3231a 100644 --- a/src/main/frontend/components/plugins.cljs +++ b/src/main/frontend/components/plugins.cljs @@ -338,6 +338,8 @@ (rum/defc user-proxy-settings-panel [{:keys [protocol] :as agent-opts}] (let [[opts set-opts!] (rum/use-state agent-opts) + [testing? set-testing?!] (rum/use-state false) + *test-input (rum/create-ref) disabled? (string/blank? (:protocol opts))] [:div.cp__settings-network-proxy-panel [:h1.mb-2.text-2xl.font-bold (t :settings-page/network-proxy)] @@ -362,12 +364,34 @@ :on-change #(set-opts! (assoc opts :port (util/trim-safe (util/evalue %))))}]]] + [:hr] + [:p.flex.items-center.space-x-2 + [:span.w-60 + [:input.form-input.is-small + {:ref *test-input + :placeholder "http://" + :on-change #(set-opts! + (assoc opts :test (util/trim-safe (util/evalue %)))) + :value (:test opts)}]] + + (ui/button (if testing? (ui/loading "Testing") "Test URL") + :intent "logseq" :large? false + :style {:margin-top 0 :padding "5px 15px"} + :on-click #(let [val (util/trim-safe (.-value (rum/deref *test-input)))] + (when (and (not testing?) (not (string/blank? val))) + (set-testing?! true) + (-> (p/let [_ (ipc/ipc :setHttpsAgent opts) + _ (ipc/ipc :testProxyUrl val)]) + (p/catch (fn [e] (notification/show! (str e) :error))) + (p/finally (fn [] (set-testing?! false))))) + ))] + [:p.pt-2 (ui/button (t :save) - :on-click (fn [] - (p/let [_ (ipc/ipc :setHttpsAgent opts)] - (state/set-state! [:electron/user-cfgs :settings/agent] opts) - (state/close-sub-modal! :https-proxy-panel))))]]])) + :on-click (fn [] + (p/let [_ (ipc/ipc :setHttpsAgent opts)] + (state/set-state! [:electron/user-cfgs :settings/agent] opts) + (state/close-sub-modal! :https-proxy-panel))))]]])) (rum/defc ^:large-vars/cleanup-todo panel-control-tabs < rum/static [search-key *search-key category *category diff --git a/src/main/frontend/handler/events.cljs b/src/main/frontend/handler/events.cljs index 451e698b980..50f16811684 100644 --- a/src/main/frontend/handler/events.cljs +++ b/src/main/frontend/handler/events.cljs @@ -270,7 +270,6 @@ (state/close-sub-modal! "ls-focused-settings-modal"))) (defmethod handle :go/proxy-settings [[_ agent-opts]] - (js/console.log agent-opts) (state/set-sub-modal! (fn [_] (plugin/user-proxy-settings-panel agent-opts)) {:id :https-proxy-panel :center? true})) From b29e9cfce298d995e4ec69f394094d7711cfa609 Mon Sep 17 00:00:00 2001 From: charlie Date: Wed, 11 May 2022 14:56:20 +0800 Subject: [PATCH 27/58] fix(lint): large vars --- src/main/frontend/components/plugins.cljs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/frontend/components/plugins.cljs b/src/main/frontend/components/plugins.cljs index 5ad38b3231a..085a7b03d68 100644 --- a/src/main/frontend/components/plugins.cljs +++ b/src/main/frontend/components/plugins.cljs @@ -606,16 +606,13 @@ (cond (not online?) - [:p.flex.justify-center.pt-20.opacity-50 - (svg/offline 30)] + [:p.flex.justify-center.pt-20.opacity-50 (svg/offline 30)] @*fetching - [:p.flex.justify-center.pt-20 - svg/loading] + [:p.flex.justify-center.pt-20 svg/loading] @*error - [:p.flex.justify-center.pt-20.opacity-50 - "Remote error: " (.-message @*error)] + [:p.flex.justify-center.pt-20.opacity-50 "Remote error: " (.-message @*error)] :else [:div.cp__plugins-marketplace-cnt From 2d2a05228a2b6b390ef9d4a402b6813ca80cb03f Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 12 May 2022 08:42:43 +0800 Subject: [PATCH 28/58] fix: parsing UI out of sync with actual state close #5254 --- src/main/frontend/handler/repo.cljs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/frontend/handler/repo.cljs b/src/main/frontend/handler/repo.cljs index 694e3165304..e52cc65d74a 100644 --- a/src/main/frontend/handler/repo.cljs +++ b/src/main/frontend/handler/repo.cljs @@ -185,8 +185,6 @@ (defn- parse-and-load-file! [repo-url file new-graph?] - (state/set-parsing-state! (fn [m] - (assoc m :current-parsing-file (:file/path file)))) (try (file-handler/alter-file repo-url (:file/path file) @@ -241,13 +239,17 @@ (if util/node-test? (do (doseq [file support-files'] + (state/set-parsing-state! (fn [m] + (assoc m :current-parsing-file (:file/path file)))) (parse-and-load-file! repo-url file new-graph?)) (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan)) (async/go-loop [] (if-let [file (async/ Date: Thu, 12 May 2022 10:44:30 +0800 Subject: [PATCH 29/58] fix: grouped transaction close #5277 --- src/main/frontend/handler/editor.cljs | 7 ++-- .../modules/outliner/transaction.cljc | 35 +++++++++++-------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index d60e53b7957..51d30a541fb 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -471,7 +471,7 @@ (not has-children?))] (outliner-tx/transact! {:outliner-op :insert-blocks} - (save-current-block!) + (save-current-block! {:current-block current-block}) (outliner-core/insert-blocks! [new-block] current-block {:sibling? sibling? :keep-uuid? keep-uuid? :replace-empty-target? replace-empty-target?})))) @@ -1274,7 +1274,7 @@ "skip-properties? if set true, when editing block is likely be properties, skip saving" ([] (save-current-block! {})) - ([{:keys [force? skip-properties?] :as opts}] + ([{:keys [force? skip-properties? current-block] :as opts}] ;; non English input method (when-not (state/editor-in-composition?) (when (state/get-current-repo) @@ -1295,7 +1295,8 @@ db-content (:block/content db-block) db-content-without-heading (and db-content (gp-util/safe-subs db-content (:block/level db-block))) - value (and elem (gobj/get elem "value"))] + value (or (:block/content current-block) + (and elem (gobj/get elem "value")))] (cond force? (save-block-aux! db-block value opts) diff --git a/src/main/frontend/modules/outliner/transaction.cljc b/src/main/frontend/modules/outliner/transaction.cljc index 229e47617f9..a838dc5e491 100644 --- a/src/main/frontend/modules/outliner/transaction.cljc +++ b/src/main/frontend/modules/outliner/transaction.cljc @@ -18,18 +18,23 @@ (delete-blocks! ...))" [opts & body] (assert (map? opts)) - `(if (some? frontend.modules.outliner.core/*transaction-data*) - (do ~@body) - (binding [frontend.modules.outliner.core/*transaction-data* (transient [])] - ~@body - (let [r# (persistent! frontend.modules.outliner.core/*transaction-data*) - tx# (mapcat :tx-data r#) - ;; FIXME: should we merge all the tx-meta? - tx-meta# (first (map :tx-meta r#)) - all-tx# (concat tx# (:additional-tx ~opts)) - opts# (merge (dissoc ~opts :additional-tx) tx-meta#)] - (when (seq all-tx#) - (let [result# (frontend.modules.outliner.datascript/transact! all-tx# opts#)] - {:tx-report result# - :tx-data all-tx# - :tx-meta tx-meta#})))))) + `(let [transact-data# frontend.modules.outliner.core/*transaction-data* + opts# (if transact-data# + (assoc ~opts :nested-transaction? true) + ~opts)] + (if transact-data# + (do ~@body) + (binding [frontend.modules.outliner.core/*transaction-data* (transient [])] + ~@body + (let [r# (persistent! frontend.modules.outliner.core/*transaction-data*) + tx# (mapcat :tx-data r#) + ;; FIXME: should we merge all the tx-meta? + tx-meta# (first (map :tx-meta r#)) + all-tx# (concat tx# (:additional-tx opts#)) + opts## (merge (dissoc opts# :additional-tx) tx-meta#)] + (when (seq all-tx#) + (when-not (:nested-transaction? opts#) ; transact only for the whole transaction + (let [result# (frontend.modules.outliner.datascript/transact! all-tx# opts##)] + {:tx-report result# + :tx-data all-tx# + :tx-meta tx-meta#})))))))) From 8af6a3d368b957761460798bead077e4bdbed325 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 12 May 2022 11:18:16 +0800 Subject: [PATCH 30/58] fix: #2021/09/22 , create files named .md close #2887 --- src/main/frontend/date.cljs | 7 +++++++ src/main/frontend/modules/file/core.cljs | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/date.cljs b/src/main/frontend/date.cljs index ebf6e873076..801ed9bf857 100644 --- a/src/main/frontend/date.cljs +++ b/src/main/frontend/date.cljs @@ -231,6 +231,13 @@ default-journal-title-formatter)] (journal-title-> journal-title #(tf/unparse formatter %)))) +(defn date->file-name + [date] + (let [formatter (if-let [format (state/get-journal-file-name-format)] + (tf/formatter format) + default-journal-title-formatter)] + (tf/unparse formatter date))) + (defn journal-title->custom-format [journal-title] (journal-title-> journal-title format)) diff --git a/src/main/frontend/modules/file/core.cljs b/src/main/frontend/modules/file/core.cljs index faf0d046313..ef4344c1a96 100644 --- a/src/main/frontend/modules/file/core.cljs +++ b/src/main/frontend/modules/file/core.cljs @@ -125,7 +125,7 @@ (config/get-pages-directory)) "/" (if journal-page? - (date/journal-title->default title) + (date/date->file-name journal-page?) (-> (or (:block/original-name page) (:block/name page)) (util/file-name-sanity))) "." (if (= format "markdown") "md" format)) From 6073fa1f0ae0b3d62c61184ffac173b3787d4782 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 12 May 2022 20:01:49 +0800 Subject: [PATCH 31/58] fix: upsert properties close #5219 --- src/main/frontend/util/property.cljs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/util/property.cljs b/src/main/frontend/util/property.cljs index 2449ac1bdea..c89ebd10b7e 100644 --- a/src/main/frontend/util/property.cljs +++ b/src/main/frontend/util/property.cljs @@ -236,7 +236,12 @@ (let [ast (mldoc/->edn content (mldoc/default-config format)) title? (mldoc/block-with-title? (ffirst (map first ast))) has-properties? (or (and title? - (mldoc/properties? (second ast))) + (or (mldoc/properties? (second ast)) + (mldoc/properties? (second + (remove + (fn [[x _]] + (= "Hiccup" (first x))) + ast))))) (mldoc/properties? (first ast))) lines (string/split-lines content) [title body] (if title? From 60fad0d3ba55a8d1b2d1423f1c7d14ce45ab42be Mon Sep 17 00:00:00 2001 From: llcc Date: Thu, 12 May 2022 14:05:30 +0800 Subject: [PATCH 32/58] open PDF with external app on mobile --- android/app/capacitor.build.gradle | 1 + .../src/main/assets/capacitor.plugins.json | 4 ++ android/capacitor.settings.gradle | 3 ++ ios/App/Podfile | 1 + package.json | 1 + src/main/frontend/components/block.cljs | 50 ++++++++++++++----- 6 files changed, 48 insertions(+), 12 deletions(-) diff --git a/android/app/capacitor.build.gradle b/android/app/capacitor.build.gradle index 8a5d26fdb41..7cc799a90b1 100644 --- a/android/app/capacitor.build.gradle +++ b/android/app/capacitor.build.gradle @@ -13,6 +13,7 @@ dependencies { implementation project(':capacitor-camera') implementation project(':capacitor-filesystem') implementation project(':capacitor-keyboard') + implementation project(':capacitor-share') implementation project(':capacitor-splash-screen') implementation project(':capacitor-status-bar') implementation project(':capacitor-voice-recorder') diff --git a/android/app/src/main/assets/capacitor.plugins.json b/android/app/src/main/assets/capacitor.plugins.json index ed46c72e401..d0b24a53520 100644 --- a/android/app/src/main/assets/capacitor.plugins.json +++ b/android/app/src/main/assets/capacitor.plugins.json @@ -15,6 +15,10 @@ "pkg": "@capacitor/keyboard", "classpath": "com.capacitorjs.plugins.keyboard.KeyboardPlugin" }, + { + "pkg": "@capacitor/share", + "classpath": "com.capacitorjs.plugins.share.SharePlugin" + }, { "pkg": "@capacitor/splash-screen", "classpath": "com.capacitorjs.plugins.splashscreen.SplashScreenPlugin" diff --git a/android/capacitor.settings.gradle b/android/capacitor.settings.gradle index 24f78ee913b..262ed3f8b32 100644 --- a/android/capacitor.settings.gradle +++ b/android/capacitor.settings.gradle @@ -14,6 +14,9 @@ project(':capacitor-filesystem').projectDir = new File('../node_modules/@capacit include ':capacitor-keyboard' project(':capacitor-keyboard').projectDir = new File('../node_modules/@capacitor/keyboard/android') +include ':capacitor-share' +project(':capacitor-share').projectDir = new File('../node_modules/@capacitor/share/android') + include ':capacitor-splash-screen' project(':capacitor-splash-screen').projectDir = new File('../node_modules/@capacitor/splash-screen/android') diff --git a/ios/App/Podfile b/ios/App/Podfile index 41184099d36..856b15756e4 100644 --- a/ios/App/Podfile +++ b/ios/App/Podfile @@ -14,6 +14,7 @@ def capacitor_pods pod 'CapacitorClipboard', :path => '../../node_modules/@capacitor/clipboard' pod 'CapacitorFilesystem', :path => '../../node_modules/@capacitor/filesystem' pod 'CapacitorKeyboard', :path => '../../node_modules/@capacitor/keyboard' + pod 'CapacitorShare', :path => '../../node_modules/@capacitor/share' pod 'CapacitorSplashScreen', :path => '../../node_modules/@capacitor/splash-screen' pod 'CapacitorStatusBar', :path => '../../node_modules/@capacitor/status-bar' pod 'CapacitorVoiceRecorder', :path => '../../node_modules/capacitor-voice-recorder' diff --git a/package.json b/package.json index 4c4123c1371..264db8280c4 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "@capacitor/filesystem": "1.0.6", "@capacitor/ios": "3.2.2", "@capacitor/keyboard": "^1.2.0", + "@capacitor/share": "^1.1.2", "@capacitor/splash-screen": "1.1.3", "@capacitor/status-bar": "1.0.6", "@excalidraw/excalidraw": "0.10.0", diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 9b8a7dfe228..2e6aa4d9929 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -1,6 +1,7 @@ (ns frontend.components.block (:refer-clojure :exclude [range]) (:require ["/frontend/utils" :as utils] + ["@capacitor/share" :refer [^js Share]] [cljs-bean.core :as bean] [cljs.core.match :refer [match]] [cljs.reader :as reader] @@ -265,10 +266,28 @@ (p/then (editor-handler/make-asset-url href) #(reset! src %))) (when @src - (let [ext (util/get-file-ext @src)] - (if (contains? (set (map name config/audio-formats)) ext) + (let [ext (keyword (util/get-file-ext @src))] + (cond + (contains? config/audio-formats ext) (audio-cp @src) - (resizable-image config title @src metadata full_text true)))))) + + (contains? (config/img-formats) ext) + (resizable-image config title @src metadata full_text true) + + (= ext :pdf) + [:a.asset-ref.is-pdf + {:href @src + :on-click + (fn [event] + (util/stop event) + (when (mobile-util/is-native-platform?) + (p/let [url (str (config/get-repo-dir (state/get-current-repo)) href)] + (.share Share #js {:url url + :title "Open PDF fils with your favorite app"}))))} + title] + + :else + [:a.asset-ref {:ref @src} title]))))) (defn ar-url->http-url [href] @@ -829,15 +848,22 @@ (not (contains? #{"pdf" "mp4" "webm" "mov"} ext)) (image-link config url s label metadata full_text) - (util/electron?) - (if (= (util/get-file-ext s) "pdf") - [:a.asset-ref.is-pdf - {:href "javascript:void(0);" - :on-mouse-down (fn [_event] - (when-let [current (pdf-assets/inflate-asset s)] - (state/set-state! :pdf/current current)))} - (get-label-text label)] - (asset-reference config label s))))) + (= (util/get-file-ext s) "pdf") + (let [label-text (get-label-text label)] + (cond + (util/electron?) + [:a.asset-ref.is-pdf + {:href "javascript:void(0);" + :on-mouse-down (fn [_event] + (when-let [current (pdf-assets/inflate-asset s)] + (state/set-state! :pdf/current current)))} + label-text] + + (mobile-util/is-native-platform?) + (asset-link config label-text s metadata full_text))) + + :else + (asset-reference config label s)))) (defn- search-link-cp [config url s label title metadata full_text] From 8332be2bd99f44fca16fcdf5d16ccfc8b6d622a3 Mon Sep 17 00:00:00 2001 From: llcc Date: Thu, 12 May 2022 15:12:23 +0800 Subject: [PATCH 33/58] fix lints --- yarn.lock | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/yarn.lock b/yarn.lock index 37f86e1e1d2..a1241a5b3a1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -569,6 +569,11 @@ resolved "https://registry.yarnpkg.com/@capacitor/keyboard/-/keyboard-1.2.2.tgz#7be5d2e4956b77e5eb7235ec020d131c9f87f30a" integrity sha512-dOZSXJTY/tTbRQ+Neiny72BIXN2Hvf/2AgPpMdTErDfaQM7C2MMgtJrm+Mi+YUeT6AnJFmt68nHJGImAL4lzmA== +"@capacitor/share@^1.1.2": + version "1.1.2" + resolved "https://registry.npmmirror.com/@capacitor/share/-/share-1.1.2.tgz#7e35c2c3bccc9955e852d3dd401afe7bd5a69b7e" + integrity sha512-FUTdjA7MAiD1tkGVZ+C3gs7a4fyEhXojDO2HkZ954oupG1cQ51dEJ1xTNnR9BAmCwUJO4sa91cxy7SMyCDPuGg== + "@capacitor/splash-screen@1.1.3": version "1.1.3" resolved "https://registry.yarnpkg.com/@capacitor/splash-screen/-/splash-screen-1.1.3.tgz#024bc91b2b521b9f1d5e86f16a114f294bf326fa" @@ -6821,6 +6826,8 @@ react-icons@2.2.7: version "2.2.7" resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-2.2.7.tgz#d7860826b258557510dac10680abea5ca23cf650" integrity sha512-0n4lcGqzJFcIQLoQytLdJCE0DKSA9dkwEZRYoGrIDJZFvIT6Hbajx5mv9geqhqFiNjUgtxg8kPyDfjlhymbGFg== + dependencies: + react-icon-base "2.1.0" react-is@^16.13.1, react-is@^16.3.1, react-is@^16.7.0: version "16.13.1" From e1fdef43b3de5795bb54bdc47c4c9b86219f4146 Mon Sep 17 00:00:00 2001 From: Kan-Ru Chen Date: Thu, 12 May 2022 22:23:36 +0900 Subject: [PATCH 34/58] chore: update electron-deeplink to 1.0.10 to fix deeplink on Linux Fixes #4984 --- resources/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/package.json b/resources/package.json index 8a67f992866..2bae8c05ee1 100644 --- a/resources/package.json +++ b/resources/package.json @@ -37,7 +37,7 @@ "@sentry/electron": "2.5.1", "posthog-js": "1.10.2", "@logseq/rsapi": "0.0.11", - "electron-deeplink": "1.0.9" + "electron-deeplink": "1.0.10" }, "devDependencies": { "@electron-forge/cli": "^6.0.0-beta.57", From d66ab6aa8220989c5f1635609519233e3800427a Mon Sep 17 00:00:00 2001 From: llcc Date: Thu, 12 May 2022 20:55:47 +0800 Subject: [PATCH 35/58] fix hidden files removing failure on mobile --- src/main/frontend/handler/repo.cljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/frontend/handler/repo.cljs b/src/main/frontend/handler/repo.cljs index e52cc65d74a..250f4ea18c0 100644 --- a/src/main/frontend/handler/repo.cljs +++ b/src/main/frontend/handler/repo.cljs @@ -295,7 +295,7 @@ (common-handler/read-config content))) relate-path-fn (fn [m k] (some-> (get m k) - (string/replace (str (config/get-local-dir repo-url) "/") ""))) + (string/replace (js/decodeURI (config/get-local-dir repo-url)) ""))) nfs-files (common-handler/remove-hidden-files nfs-files config #(relate-path-fn % :file/path)) diffs (common-handler/remove-hidden-files diffs config #(relate-path-fn % :path)) load-contents (fn [files option] From 22d41664a4dbf23879d2c57afa93f01c20104cde Mon Sep 17 00:00:00 2001 From: yshwaker Date: Sat, 30 Apr 2022 16:52:57 +0800 Subject: [PATCH 36/58] fix: modal position, getting editor width from editor-wrapper --- src/main/frontend/components/editor.cljs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/frontend/components/editor.cljs b/src/main/frontend/components/editor.cljs index 5c29fc178a0..be96d174724 100644 --- a/src/main/frontend/components/editor.cljs +++ b/src/main/frontend/components/editor.cljs @@ -408,9 +408,9 @@ :z-index 11} (when set-default-width? {:width max-width}) - (let [^js/HTMLElement textarea - (js/document.querySelector "textarea.ls-textarea")] - (if (<= (.-clientWidth textarea) (+ left (if set-default-width? max-width 500))) + (let [^js/HTMLElement editor + (js/document.querySelector ".editor-wrapper")] + (if (<= (.-clientWidth editor) (+ left (if set-default-width? max-width 500))) {:right 0} {:left (if (and y-diff (= y-diff 0)) left 0)})))} cp])) @@ -477,7 +477,6 @@ (let [content (if content (str content) "")] ;; as the function is binding to the editor content, optimization is welcome (str - "ls-textarea " (if (or (> (.-length content) 1000) (string/includes? content "\n")) "multiline-block" From 22ddae89981e713167989fe49127b5d20afd3565 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 11 May 2022 14:23:37 -0400 Subject: [PATCH 37/58] Move graph-parser related block code to proper ns and nbb migration wip --- .clj-kondo/config.edn | 2 + src/main/frontend/components/block.cljs | 3 +- src/main/frontend/components/page.cljs | 20 +- src/main/frontend/format/block.cljs | 592 +----------------- src/main/frontend/format/mldoc.cljs | 1 + src/main/frontend/handler/block.cljs | 4 +- src/main/frontend/handler/editor.cljs | 9 +- src/main/frontend/handler/external.cljs | 4 +- src/main/frontend/handler/extract.cljs | 11 +- src/main/frontend/handler/page.cljs | 6 +- src/main/logseq/graph_parser/block.cljs | 584 +++++++++++++++++ .../graph_parser}/block_test.cljs | 8 +- 12 files changed, 626 insertions(+), 618 deletions(-) create mode 100644 src/main/logseq/graph_parser/block.cljs rename src/test/{frontend/format => logseq/graph_parser}/block_test.cljs (88%) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index c671f8b73f7..d3e9e3a31bc 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -21,7 +21,9 @@ frontend.util util frontend.config config frontend.format.mldoc mldoc + frontend.format.block block logseq.graph-parser.text text + logseq.graph-parser.block gp-block logseq.graph-parser.mldoc gp-mldoc logseq.graph-parser.util gp-util logseq.graph-parser.config gp-config}}} diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 85c7fc14b6c..130c6776cc7 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -55,6 +55,7 @@ [logseq.graph-parser.config :as gp-config] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] + [logseq.graph-parser.block :as gp-block] [goog.dom :as gdom] [goog.object :as gobj] [lambdaisland.glogi :as log] @@ -1307,7 +1308,7 @@ (->elem :sub (map-inline config l)) ["Tag" _] - (when-let [s (block/get-tag item)] + (when-let [s (gp-block/get-tag item)] (let [s (text/page-ref-un-brackets! s)] (page-cp (assoc config :tag? true) {:block/name s}))) diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 9bc9f83234d..a3b177268ef 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -1,6 +1,6 @@ (ns frontend.components.page (:require [clojure.string :as string] - [frontend.components.block :as block] + [frontend.components.block :as component-block] [frontend.components.content :as content] [frontend.components.editor :as editor] [frontend.components.hierarchy :as hierarchy] @@ -15,7 +15,6 @@ [frontend.db.model :as model] [frontend.extensions.graph :as graph] [frontend.extensions.pdf.assets :as pdf-assets] - [frontend.format.block :as format-block] [frontend.handler.common :as common-handler] [frontend.handler.config :as config-handler] [frontend.handler.editor :as editor-handler] @@ -35,6 +34,7 @@ [medley.core :as medley] [rum.core :as rum] [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.block :as gp-block] [frontend.mobile.util :as mobile-util])) (defn- get-page-name @@ -139,7 +139,7 @@ :document/mode? document-mode?} config) hiccup-config (common-handler/config-with-document-mode hiccup-config) - hiccup (block/->hiccup page-blocks hiccup-config {})] + hiccup (component-block/->hiccup page-blocks hiccup-config {})] [:div (page-blocks-inner page-name page-blocks hiccup sidebar? block-id) (when-not config/publishing? @@ -163,9 +163,9 @@ (rum/with-key (ui/catch-error (ui/component-error "Failed default query:" {:content (pr-str query)}) - (block/custom-query {:attr {:class "mt-10"} - :editor-box editor/box - :page page} query)) + (component-block/custom-query {:attr {:class "mt-10"} + :editor-box editor/box + :page page} query)) (str repo "-custom-query-" (:query query))))])))) (defn tagged-pages @@ -332,7 +332,7 @@ (db/entity repo)) (do (when-not (db/entity repo [:block/name page-name]) - (let [m (format-block/page-name->map path-page-name true)] + (let [m (gp-block/page-name->map path-page-name true)] (db/transact! repo [m]))) (db/pull [:block/name page-name]))) {:keys [icon]} (:block/properties page) @@ -377,7 +377,7 @@ (let [config {:id "block-parent" :block? true}] [:div.mb-4 - (block/breadcrumb config repo block-id {:level-limit 3})])) + (component-block/breadcrumb config repo block-id {:level-limit 3})])) ;; blocks (let [page (if block? @@ -712,7 +712,7 @@ [:tr {:key name} [:td.n.w-12 [:span.opacity-70 (str (inc n) ".")]] [:td.name [:a {:href (rfe/href :page {:name (:block/name page)})} - (block/page-cp {} page)]] + (component-block/page-cp {} page)]] [:td.backlinks [:span (or backlinks "0")]] (when-not orphaned-pages? [:td.created-at [:span (if created-at (date/int->local-time-2 created-at) "Unknown")]]) (when-not orphaned-pages? [:td.updated-at [:span (if updated-at (date/int->local-time-2 updated-at) "Unknown")]])])]] @@ -954,7 +954,7 @@ (:db/id page) :page)))) :href (rfe/href :page {:name (:block/name page)})} - (block/page-cp {} page)]] + (component-block/page-cp {} page)]] (when-not mobile? [:td.backlinks [:span backlinks]]) diff --git a/src/main/frontend/format/block.cljs b/src/main/frontend/format/block.cljs index 9da64520aec..72dd1bd8fac 100644 --- a/src/main/frontend/format/block.cljs +++ b/src/main/frontend/format/block.cljs @@ -1,595 +1,13 @@ (ns frontend.format.block + "Block code needed by app but not graph-parser" (:require [clojure.string :as string] - [clojure.walk :as walk] - [cljs.core.match :as match] + [logseq.graph-parser.block :as gp-block] [frontend.config :as config] - [frontend.date :as date] [frontend.db :as db] [frontend.format :as format] [frontend.state :as state] - [logseq.graph-parser.text :as text] - [logseq.graph-parser.utf8 :as utf8] - [frontend.util :as util] [frontend.util.property :as property] - [logseq.graph-parser.util :as gp-util] - [logseq.graph-parser.config :as gp-config] - [logseq.graph-parser.mldoc :as gp-mldoc] - [lambdaisland.glogi :as log] - [medley.core :as medley])) - -(defn heading-block? - [block] - (and - (vector? block) - (= "Heading" (first block)))) - -(defn get-tag - [block] - (when-let [tag-value (and (vector? block) - (= "Tag" (first block)) - (second block))] - (-> - (map (fn [e] - (match/match e - ["Plain" s] - s - ["Link" t] - (let [{full_text :full_text} t] - full_text) - ["Nested_link" t] - (let [ {content :content} t] - content) - :else - "" - )) tag-value) - (string/join)))) - -(defn get-page-reference - [block] - (let [page (cond - (and (vector? block) (= "Link" (first block))) - (let [typ (first (:url (second block))) - value (second (:url (second block)))] - ;; {:url ["File" "file:../pages/hello_world.org"], :label [["Plain" "hello world"]], :title nil} - (or - (and - (= typ "Page_ref") - (and (string? value) - (not (or (gp-config/local-asset? value) - (gp-config/draw? value)))) - value) - - (and - (= typ "Search") - (text/page-ref? value) - (text/page-ref-un-brackets! value)) - - (and - (= typ "Search") - (not (contains? #{\# \* \/ \[} (first value))) - (let [ext (some-> (util/get-file-ext value) keyword)] - (when (and (not (util/starts-with? value "http:")) - (not (util/starts-with? value "https:")) - (not (util/starts-with? value "file:")) - (not (gp-config/local-asset? value)) - (or (= ext :excalidraw) - (not (contains? (config/supported-formats) ext)))) - value))) - - (and - (= typ "Complex") - (= (:protocol value) "file") - (:link value)) - - (and - (= typ "File") - (second (first (:label (second block))))))) - - (and (vector? block) (= "Nested_link" (first block))) - (let [content (:content (last block))] - (subs content 2 (- (count content) 2))) - - (and (vector? block) - (= "Macro" (first block))) - (let [{:keys [name arguments]} (second block) - argument (string/join ", " arguments)] - (when (= name "embed") - (text/page-ref-un-brackets! argument))) - - (and (vector? block) - (= "Tag" (first block))) - (let [text (get-tag block)] - (text/page-ref-un-brackets! text)) - - :else - nil)] - (text/block-ref-un-brackets! page))) - -(defn get-block-reference - [block] - (when-let [block-id (cond - (and (vector? block) - (= "Block_reference" (first block))) - (last block) - - (and (vector? block) - (= "Link" (first block)) - (map? (second block)) - (= "Block_ref" (first (:url (second block))))) - (second (:url (second block))) - - (and (vector? block) - (= "Macro" (first block))) - (let [{:keys [name arguments]} (second block)] - (when (and (= name "embed") - (string? (first arguments)) - (string/starts-with? (first arguments) "((") - (string/ends-with? (first arguments) "))")) - (subs (first arguments) 2 (- (count (first arguments)) 2)))) - - (and (vector? block) - (= "Link" (first block)) - (map? (second block))) - (if (= "id" (:protocol (second (:url (second block))))) - (:link (second (:url (second block)))) - (let [id (second (:url (second block)))] - (text/block-ref-un-brackets! id))) - - :else - nil)] - (when (and block-id - (gp-util/uuid-string? block-id)) - block-id))) - -(defn paragraph-block? - [block] - (and - (vector? block) - (= "Paragraph" (first block)))) - -(defn timestamp-block? - [block] - (and - (vector? block) - (= "Timestamp" (first block)))) - -;; TODO: we should move this to mldoc -(defn extract-properties - [format properties] - (when (seq properties) - (let [properties (seq properties) - page-refs (->> - properties - (remove (fn [[k _]] - (contains? #{:background-color :background_color} (keyword k)))) - (map last) - (map (fn [v] - (when (and (string? v) - (not (gp-mldoc/link? format v))) - (let [v (string/trim v) - result (text/split-page-refs-without-brackets v {:un-brackets? false})] - (if (coll? result) - (map text/page-ref-un-brackets! result) - []))))) - (apply concat) - (remove string/blank?)) - properties (->> properties - (map (fn [[k v]] - (let [k (-> (string/lower-case (name k)) - (string/replace " " "-") - (string/replace "_" "-")) - k (if (contains? #{"custom_id" "custom-id"} k) - "id" - k) - v (if (coll? v) - (remove string/blank? v) - (if (string/blank? v) - nil - (text/parse-property format k v (state/get-config)))) - k (keyword k) - v (if (and - (string? v) - (contains? #{:alias :aliases :tags} k)) - (set [v]) - v) - v (if (coll? v) (set v) v)] - [k v]))) - (remove #(nil? (second %))))] - {:properties (into {} properties) - :properties-order (map first properties) - :page-refs page-refs}))) - -(defn- paragraph-timestamp-block? - [block] - (and (paragraph-block? block) - (or (timestamp-block? (first (second block))) - (timestamp-block? (second (second block)))))) - -(defn extract-timestamps - [block] - (some->> - (second block) - (filter timestamp-block?) - (map last) - (into {}))) - -;; {"Deadline" {:date {:year 2020, :month 10, :day 20}, :wday "Tue", :time {:hour 8, :min 0}, :repetition [["DoublePlus"] ["Day"] 1], :active true}} -(defn timestamps->scheduled-and-deadline - [timestamps] - (let [timestamps (medley/map-keys (comp keyword string/lower-case) timestamps) - m (some->> (select-keys timestamps [:scheduled :deadline]) - (map (fn [[k v]] - (let [{:keys [date repetition]} v - {:keys [year month day]} date - day (js/parseInt (str year (util/zero-pad month) (util/zero-pad day)))] - (cond-> - (case k - :scheduled - {:scheduled day} - :deadline - {:deadline day}) - repetition - (assoc :repeated? true))))))] - (apply merge m))) - -(defn convert-page-if-journal - "Convert journal file name to user' custom date format" - [original-page-name] - (when original-page-name - (let [page-name (util/page-name-sanity-lc original-page-name) - day (date/journal-title->int page-name)] - (if day - (let [original-page-name (date/int->journal-title day)] - [original-page-name (util/page-name-sanity-lc original-page-name) day]) - [original-page-name page-name day])))) - -(defn page-name->map - "Create a page's map structure given a original page name (string). - map as input is supported for legacy compatibility. - with-timestamp?: assign timestampes to the map structure. - Useful when creating new pages from references or namespaces, - as there's no chance to introduce timestamps via editing in page" - ([original-page-name with-id?] - (page-name->map original-page-name with-id? true)) - ([original-page-name with-id? with-timestamp?] - (cond - (and original-page-name (string? original-page-name)) - (let [original-page-name (util/remove-boundary-slashes original-page-name) - [original-page-name page-name journal-day] (convert-page-if-journal original-page-name) - namespace? (and (not (boolean (text/get-nested-page-name original-page-name))) - (text/namespace-page? original-page-name)) - page-entity (db/entity [:block/name page-name]) - original-page-name (or (:block/original-name page-entity) original-page-name)] - (merge - {:block/name page-name - :block/original-name original-page-name} - (when with-id? - (if page-entity - {:block/uuid (:block/uuid page-entity)} - {:block/uuid (db/new-block-id)})) - (when namespace? - (let [namespace (first (gp-util/split-last "/" original-page-name))] - (when-not (string/blank? namespace) - {:block/namespace {:block/name (util/page-name-sanity-lc namespace)}}))) - (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity - (let [current-ms (util/time-ms)] - {:block/created-at current-ms - :block/updated-at current-ms})) - (if journal-day - {:block/journal? true - :block/journal-day journal-day} - {:block/journal? false}))) - - (and (map? original-page-name) (:block/uuid original-page-name)) - original-page-name - - (and (map? original-page-name) with-id?) - (assoc original-page-name :block/uuid (db/new-block-id)) - - :else - nil))) - -(defn with-page-refs - [{:keys [title body tags refs marker priority] :as block} with-id?] - (let [refs (->> (concat tags refs [marker priority]) - (remove string/blank?) - (distinct)) - refs (atom refs)] - (walk/prewalk - (fn [form] - ;; skip custom queries - (when-not (and (vector? form) - (= (first form) "Custom") - (= (second form) "query")) - (when-let [page (get-page-reference form)] - (swap! refs conj page)) - (when-let [tag (get-tag form)] - (let [tag (text/page-ref-un-brackets! tag)] - (when (gp-util/tag-valid? tag) - (swap! refs conj tag)))) - form)) - (concat title body)) - (let [refs (remove string/blank? @refs) - children-pages (->> (mapcat (fn [p] - (let [p (if (map? p) - (:block/original-name p) - p)] - (when (string? p) - (let [p (or (text/get-nested-page-name p) p)] - (when (text/namespace-page? p) - (util/split-namespace-pages p)))))) - refs) - (remove string/blank?) - (distinct)) - refs (->> (distinct (concat refs children-pages)) - (remove nil?)) - refs (map (fn [ref] (page-name->map ref with-id?)) refs)] - (assoc block :refs refs)))) - -(defn with-block-refs - [{:keys [title body] :as block}] - (let [ref-blocks (atom nil)] - (walk/postwalk - (fn [form] - (when-let [block (get-block-reference form)] - (swap! ref-blocks conj block)) - form) - (concat title body)) - (let [ref-blocks (->> @ref-blocks - (filter gp-util/uuid-string?)) - ref-blocks (map - (fn [id] - [:block/uuid (medley/uuid id)]) - ref-blocks) - refs (distinct (concat (:refs block) ref-blocks))] - (assoc block :refs refs)))) - -(defn- block-keywordize - [block] - (medley/map-keys - (fn [k] - (if (namespace k) - k - (keyword "block" k))) - block)) - -(defn- sanity-blocks-data - [blocks] - (map (fn [block] - (if (map? block) - (block-keywordize (gp-util/remove-nils block)) - block)) - blocks)) - -(defn with-path-refs - [blocks] - (loop [blocks blocks - acc [] - parents []] - (if (empty? blocks) - acc - (let [block (first blocks) - cur-level (:block/level block) - level-diff (- cur-level - (get (last parents) :block/level 0)) - [path-refs parents] - (cond - (zero? level-diff) ; sibling - (let [path-refs (mapcat :block/refs (drop-last parents)) - parents (conj (vec (butlast parents)) block)] - [path-refs parents]) - - (> level-diff 0) ; child - (let [path-refs (mapcat :block/refs parents)] - [path-refs (conj parents block)]) - - (< level-diff 0) ; new parent - (let [parents (vec (take-while (fn [p] (< (:block/level p) cur-level)) parents)) - path-refs (mapcat :block/refs parents)] - [path-refs (conj parents block)])) - path-ref-pages (->> path-refs - (concat (:block/refs block)) - (map (fn [ref] - (cond - (map? ref) - (:block/name ref) - - :else - ref))) - (remove string/blank?) - (map (fn [ref] - (if (string? ref) - {:block/name (util/page-name-sanity-lc ref)} - ref))) - (remove vector?) - (remove nil?) - (distinct))] - (recur (rest blocks) - (conj acc (assoc block :block/path-refs path-ref-pages)) - parents))))) - -(defn block-tags->pages - [{:keys [tags] :as block}] - (if (seq tags) - (assoc block :tags (map (fn [tag] - (let [tag (text/page-ref-un-brackets! tag)] - [:block/name (util/page-name-sanity-lc tag)])) tags)) - block)) - -(defn- get-block-content - [utf8-content block format meta] - (let [content (if-let [end-pos (:end_pos meta)] - (utf8/substring utf8-content - (:start_pos meta) - end-pos) - (utf8/substring utf8-content - (:start_pos meta))) - content (when content - (let [content (text/remove-level-spaces content format (config/get-block-pattern format))] - (if (or (:pre-block? block) - (= (:format block) :org)) - content - (gp-mldoc/remove-indentation-spaces content (inc (:level block)) false))))] - (if (= format :org) - content - (property/->new-properties content)))) - -(defn get-custom-id-or-new-id - [properties] - (or (when-let [custom-id (or (get-in properties [:properties :custom-id]) - (get-in properties [:properties :custom_id]) - (get-in properties [:properties :id]))] - (let [custom-id (and (string? custom-id) (string/trim custom-id))] - (when (and custom-id (gp-util/uuid-string? custom-id)) - (uuid custom-id)))) - (db/new-block-id))) - -(defn get-page-refs-from-properties - [properties] - (let [page-refs (mapcat (fn [v] (cond - (coll? v) - v - - (text/page-ref? v) - [(text/page-ref-un-brackets! v)] - - :else - nil)) (vals properties)) - page-refs (remove string/blank? page-refs)] - (map (fn [page] (page-name->map page true)) page-refs))) - -(defn with-page-block-refs - [block with-id?] - (some-> block - (with-page-refs with-id?) - with-block-refs - block-tags->pages - (update :refs (fn [col] (remove nil? col))))) - -(defn with-pre-block-if-exists - [blocks body pre-block-properties encoded-content] - (let [first-block (first blocks) - first-block-start-pos (get-in first-block [:block/meta :start_pos]) - - ;; Add pre-block - blocks (if (or (> first-block-start-pos 0) - (empty? blocks)) - (cons - (merge - (let [content (utf8/substring encoded-content 0 first-block-start-pos) - {:keys [properties properties-order]} pre-block-properties - id (get-custom-id-or-new-id {:properties properties}) - property-refs (->> (get-page-refs-from-properties properties) - (map :block/original-name)) - block {:uuid id - :content content - :level 1 - :properties properties - :properties-order properties-order - :refs property-refs - :pre-block? true - :unordered true - :body body} - block (with-page-block-refs block false)] - (block-keywordize block)) - (select-keys first-block [:block/format :block/page])) - blocks) - blocks)] - (with-path-refs blocks))) - -(defn- construct-block - [block properties timestamps body encoded-content format pos-meta with-id?] - (let [id (get-custom-id-or-new-id properties) - ref-pages-in-properties (->> (:page-refs properties) - (remove string/blank?)) - block (second block) - unordered? (:unordered block) - markdown-heading? (and (:size block) (= :markdown format)) - block (if markdown-heading? - (assoc block - :type :heading - :level (if unordered? (:level block) 1) - :heading-level (or (:size block) 6)) - block) - block (cond-> - (assoc block - :uuid id - :refs ref-pages-in-properties - :format format - :meta pos-meta) - (seq (:properties properties)) - (assoc :properties (:properties properties)) - - (seq (:properties-order properties)) - (assoc :properties-order (:properties-order properties))) - block (if (get-in block [:properties :collapsed]) - (assoc block :collapsed? true) - block) - block (assoc block - :content (get-block-content encoded-content block format pos-meta)) - block (if (seq timestamps) - (merge block (timestamps->scheduled-and-deadline timestamps)) - block) - block (assoc block :body body) - block (with-page-block-refs block with-id?) - {:keys [created-at updated-at]} (:properties properties) - block (cond-> block - (and created-at (integer? created-at)) - (assoc :block/created-at created-at) - - (and updated-at (integer? updated-at)) - (assoc :block/updated-at updated-at))] - (dissoc block :title :body :anchor))) - -(defn extract-blocks - "Extract headings from mldoc ast. - Args: - `blocks`: mldoc ast. - `content`: markdown or org-mode text. - `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids. - `format`: content's format, it could be either :markdown or :org-mode." - [blocks content with-id? format] - {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]} - (try - (let [encoded-content (utf8/encode content) - [blocks body pre-block-properties] - (loop [headings [] - blocks (reverse blocks) - timestamps {} - properties {} - body []] - (if (seq blocks) - (let [[block pos-meta] (first blocks) - ;; fix start_pos - pos-meta (assoc pos-meta :end_pos - (if (seq headings) - (get-in (last headings) [:meta :start_pos]) - nil))] - (cond - (paragraph-timestamp-block? block) - (let [timestamps (extract-timestamps block) - timestamps' (merge timestamps timestamps)] - (recur headings (rest blocks) timestamps' properties body)) - - (property/properties-ast? block) - (let [properties (extract-properties format (second block))] - (recur headings (rest blocks) timestamps properties body)) - - (heading-block? block) - (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id?)] - (recur (conj headings block) (rest blocks) {} {} [])) - - :else - (recur headings (rest blocks) timestamps properties (conj body block)))) - [(-> (reverse headings) - sanity-blocks-data) - body - properties])) - result (with-pre-block-if-exists blocks body pre-block-properties encoded-content)] - (map #(dissoc % :block/meta) result)) - (catch js/Error e - (js/console.error "extract-blocks-failed") - (log/error :exception e)))) + [logseq.graph-parser.mldoc :as gp-mldoc])) (defn with-parent-and-left [page-id blocks] @@ -669,7 +87,7 @@ (when-not (string/blank? content) (let [block (dissoc block :block/pre-block?) ast (format/to-edn content format nil) - blocks (extract-blocks ast content with-id? format) + blocks (gp-block/extract-blocks ast content with-id? format) new-block (first blocks) parent-refs (->> (db/get-block-parent (state/get-current-repo) uuid) :block/path-refs @@ -704,7 +122,7 @@ result (let [ast (->> (format/to-edn content format (gp-mldoc/default-config format)) (map first)) - title (when (heading-block? (first ast)) + title (when (gp-block/heading-block? (first ast)) (:title (second (first ast)))) body (vec (if title (rest ast) ast)) body (drop-while property/properties-ast? body) diff --git a/src/main/frontend/format/mldoc.cljs b/src/main/frontend/format/mldoc.cljs index a79a3142d74..23e6497835d 100644 --- a/src/main/frontend/format/mldoc.cljs +++ b/src/main/frontend/format/mldoc.cljs @@ -1,4 +1,5 @@ (ns frontend.format.mldoc + "Mldoc code needed by app but not graph-parser" (:require [clojure.string :as string] [frontend.format.protocol :as protocol] [frontend.state :as state] diff --git a/src/main/frontend/handler/block.cljs b/src/main/frontend/handler/block.cljs index 6676f81da37..d2b7acb31e7 100644 --- a/src/main/frontend/handler/block.cljs +++ b/src/main/frontend/handler/block.cljs @@ -5,7 +5,7 @@ [frontend.db.model :as db-model] [frontend.db.react :as react] [frontend.state :as state] - [frontend.format.block :as block] + [logseq.graph-parser.block :as gp-block] [frontend.util :as util])) @@ -83,7 +83,7 @@ [block typ] (walk-block block (fn [x] - (and (block/timestamp-block? x) + (and (gp-block/timestamp-block? x) (= typ (first (second x))))) #(second (second %)))) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index d397c76a81f..341d7372880 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -54,6 +54,7 @@ [frontend.util.keycode :as keycode] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] + [logseq.graph-parser.block :as gp-block] ["path" :as path])) ;; FIXME: should support multiple images concurrently uploading @@ -265,7 +266,7 @@ (if (and (:block/pre-block? block) (seq (:block/properties block))) (let [page-properties (:block/properties block) - str->page (fn [n] (block/page-name->map n true)) + str->page (fn [n] (gp-block/page-name->map n true)) refs (->> page-properties (filter (fn [[_ v]] (coll? v))) (vals) @@ -669,7 +670,7 @@ (defn properties-block [properties format page] (let [content (property/insert-properties format "" properties) - refs (block/get-page-refs-from-properties properties)] + refs (gp-block/get-page-refs-from-properties properties)] {:block/pre-block? true :block/uuid (db/new-block-id) :block/properties properties @@ -1945,7 +1946,7 @@ content* (str (if (= :markdown format) "- " "* ") (property/insert-properties format content props)) ast (mldoc/->edn content* (gp-mldoc/default-config format)) - blocks (block/extract-blocks ast content* true format) + blocks (gp-block/extract-blocks ast content* true format) fst-block (first blocks)] (assert fst-block "fst-block shouldn't be nil") (assoc fst-block :block/level (:block/level block))))))) @@ -2852,7 +2853,7 @@ [format text] (when-let [editing-block (state/get-edit-block)] (let [page-id (:db/id (:block/page editing-block)) - blocks (block/extract-blocks + blocks (gp-block/extract-blocks (mldoc/->edn text (gp-mldoc/default-config format)) text true format) blocks' (block/with-parent-and-left page-id blocks)] (paste-blocks blocks' {})))) diff --git a/src/main/frontend/handler/external.cljs b/src/main/frontend/handler/external.cljs index 6f840f4c27b..ce8e4e63235 100644 --- a/src/main/frontend/handler/external.cljs +++ b/src/main/frontend/handler/external.cljs @@ -8,7 +8,7 @@ [clojure.string :as string] [frontend.db :as db] [frontend.format.mldoc :as mldoc] - [frontend.format.block :as block] + [logseq.graph-parser.block :as gp-block] [frontend.handler.page :as page] [frontend.handler.editor :as editor] [frontend.util :as util])) @@ -73,7 +73,7 @@ (when-let [repo (state/get-current-repo)] (let [[headers parsed-blocks] (mldoc/opml->edn data) parsed-blocks (->> - (block/extract-blocks parsed-blocks "" true :markdown) + (gp-block/extract-blocks parsed-blocks "" true :markdown) (mapv editor/wrap-parse-block)) page-name (:title headers)] (when (not (db/page-exists? page-name)) diff --git a/src/main/frontend/handler/extract.cljs b/src/main/frontend/handler/extract.cljs index 93fd5fdcdfa..0b8a89f13c8 100644 --- a/src/main/frontend/handler/extract.cljs +++ b/src/main/frontend/handler/extract.cljs @@ -13,6 +13,7 @@ [frontend.util :as util] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] + [logseq.graph-parser.block :as gp-block] [frontend.util.property :as property] [lambdaisland.glogi :as log])) @@ -22,7 +23,7 @@ (let [ast (map first ast)] (if (string/includes? file "pages/contents.") "Contents" - (let [first-block (last (first (filter block/heading-block? ast))) + (let [first-block (last (first (filter gp-block/heading-block? ast))) property-name (when (and (contains? #{"Properties" "Property_Drawer"} (ffirst ast)) (not (string/blank? (:title (last (first ast)))))) (:title (last (first ast)))) @@ -47,8 +48,8 @@ [repo-url format ast properties file content] (try (let [page (get-page-name file ast) - [_original-page-name page-name _journal-day] (block/convert-page-if-journal page) - blocks (->> (block/extract-blocks ast content false format) + [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page) + blocks (->> (gp-block/extract-blocks ast content false format) (block/with-parent-and-left {:block/name page-name})) ref-pages (atom #{}) ref-tags (atom #{}) @@ -94,7 +95,7 @@ (cond-> (gp-util/remove-nils (assoc - (block/page-name->map page false) + (gp-block/page-name->map page false) :block/file {:file/path (gp-util/path-normalize file)})) (seq properties) (assoc :block/properties properties) @@ -114,7 +115,7 @@ (when (text/namespace-page? page) (->> (util/split-namespace-pages page) (map (fn [page] - (-> (block/page-name->map page true) + (-> (gp-block/page-name->map page true) (assoc :block/format format))))))) pages (->> (concat [page-entity] diff --git a/src/main/frontend/handler/page.cljs b/src/main/frontend/handler/page.cljs index ba026d26e19..ebeceeaf344 100644 --- a/src/main/frontend/handler/page.cljs +++ b/src/main/frontend/handler/page.cljs @@ -11,7 +11,6 @@ [frontend.db.model :as model] [frontend.db.utils :as db-utils] [frontend.db.conn :as conn] - [frontend.format.block :as block] [frontend.fs :as fs] [frontend.handler.common :as common-handler] [frontend.handler.editor :as editor-handler] @@ -35,6 +34,7 @@ [frontend.mobile.util :as mobile-util] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.config :as gp-config] + [logseq.graph-parser.block :as gp-block] [goog.functions :refer [debounce]])) (defn- get-directory @@ -72,7 +72,7 @@ (let [p (common-handler/get-page-default-properties title) ps (merge p properties) content (page-property/insert-properties format "" ps) - refs (block/get-page-refs-from-properties properties)] + refs (gp-block/get-page-refs-from-properties properties)] {:block/uuid (db/new-block-id) :block/properties ps :block/properties-order (keys ps) @@ -127,7 +127,7 @@ [title]) format (or format (state/get-preferred-format)) pages (map (fn [page] - (-> (block/page-name->map page true) + (-> (gp-block/page-name->map page true) (assoc :block/format format))) pages) txs (->> pages diff --git a/src/main/logseq/graph_parser/block.cljs b/src/main/logseq/graph_parser/block.cljs new file mode 100644 index 00000000000..de169410670 --- /dev/null +++ b/src/main/logseq/graph_parser/block.cljs @@ -0,0 +1,584 @@ +(ns logseq.graph-parser.block + "Block related code needed for graph-parser" + (:require [clojure.string :as string] + [clojure.walk :as walk] + [frontend.config :as config] + [frontend.date :as date] + [frontend.db :as db] + [frontend.state :as state] + [logseq.graph-parser.text :as text] + [logseq.graph-parser.utf8 :as utf8] + [frontend.util :as util] + [frontend.util.property :as property] + [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.config :as gp-config] + [logseq.graph-parser.mldoc :as gp-mldoc] + [lambdaisland.glogi :as log] + [medley.core :as medley])) + +(defn heading-block? + [block] + (and + (vector? block) + (= "Heading" (first block)))) + +(defn get-tag + [block] + (when-let [tag-value (and (vector? block) + (= "Tag" (first block)) + (second block))] + (->> tag-value + (map (fn [[elem value]] + (case elem + "Plain" value + "Link" (:full_text value) + "Nested_link" (:content value) + ""))) + (string/join)))) + +(defn get-page-reference + [block] + (let [page (cond + (and (vector? block) (= "Link" (first block))) + (let [typ (first (:url (second block))) + value (second (:url (second block)))] + ;; {:url ["File" "file:../pages/hello_world.org"], :label [["Plain" "hello world"]], :title nil} + (or + (and + (= typ "Page_ref") + (and (string? value) + (not (or (gp-config/local-asset? value) + (gp-config/draw? value)))) + value) + + (and + (= typ "Search") + (text/page-ref? value) + (text/page-ref-un-brackets! value)) + + (and + (= typ "Search") + (not (contains? #{\# \* \/ \[} (first value))) + (let [ext (some-> (util/get-file-ext value) keyword)] + (when (and (not (util/starts-with? value "http:")) + (not (util/starts-with? value "https:")) + (not (util/starts-with? value "file:")) + (not (gp-config/local-asset? value)) + (or (= ext :excalidraw) + (not (contains? (config/supported-formats) ext)))) + value))) + + (and + (= typ "Complex") + (= (:protocol value) "file") + (:link value)) + + (and + (= typ "File") + (second (first (:label (second block))))))) + + (and (vector? block) (= "Nested_link" (first block))) + (let [content (:content (last block))] + (subs content 2 (- (count content) 2))) + + (and (vector? block) + (= "Macro" (first block))) + (let [{:keys [name arguments]} (second block) + argument (string/join ", " arguments)] + (when (= name "embed") + (text/page-ref-un-brackets! argument))) + + (and (vector? block) + (= "Tag" (first block))) + (let [text (get-tag block)] + (text/page-ref-un-brackets! text)) + + :else + nil)] + (text/block-ref-un-brackets! page))) + +(defn get-block-reference + [block] + (when-let [block-id (cond + (and (vector? block) + (= "Block_reference" (first block))) + (last block) + + (and (vector? block) + (= "Link" (first block)) + (map? (second block)) + (= "Block_ref" (first (:url (second block))))) + (second (:url (second block))) + + (and (vector? block) + (= "Macro" (first block))) + (let [{:keys [name arguments]} (second block)] + (when (and (= name "embed") + (string? (first arguments)) + (string/starts-with? (first arguments) "((") + (string/ends-with? (first arguments) "))")) + (subs (first arguments) 2 (- (count (first arguments)) 2)))) + + (and (vector? block) + (= "Link" (first block)) + (map? (second block))) + (if (= "id" (:protocol (second (:url (second block))))) + (:link (second (:url (second block)))) + (let [id (second (:url (second block)))] + (text/block-ref-un-brackets! id))) + + :else + nil)] + (when (and block-id + (gp-util/uuid-string? block-id)) + block-id))) + +(defn paragraph-block? + [block] + (and + (vector? block) + (= "Paragraph" (first block)))) + +(defn timestamp-block? + [block] + (and + (vector? block) + (= "Timestamp" (first block)))) + +;; TODO: we should move this to mldoc +(defn extract-properties + [format properties] + (when (seq properties) + (let [properties (seq properties) + page-refs (->> + properties + (remove (fn [[k _]] + (contains? #{:background-color :background_color} (keyword k)))) + (map last) + (map (fn [v] + (when (and (string? v) + (not (gp-mldoc/link? format v))) + (let [v (string/trim v) + result (text/split-page-refs-without-brackets v {:un-brackets? false})] + (if (coll? result) + (map text/page-ref-un-brackets! result) + []))))) + (apply concat) + (remove string/blank?)) + properties (->> properties + (map (fn [[k v]] + (let [k (-> (string/lower-case (name k)) + (string/replace " " "-") + (string/replace "_" "-")) + k (if (contains? #{"custom_id" "custom-id"} k) + "id" + k) + v (if (coll? v) + (remove string/blank? v) + (if (string/blank? v) + nil + (text/parse-property format k v (state/get-config)))) + k (keyword k) + v (if (and + (string? v) + (contains? #{:alias :aliases :tags} k)) + (set [v]) + v) + v (if (coll? v) (set v) v)] + [k v]))) + (remove #(nil? (second %))))] + {:properties (into {} properties) + :properties-order (map first properties) + :page-refs page-refs}))) + +(defn- paragraph-timestamp-block? + [block] + (and (paragraph-block? block) + (or (timestamp-block? (first (second block))) + (timestamp-block? (second (second block)))))) + +(defn extract-timestamps + [block] + (some->> + (second block) + (filter timestamp-block?) + (map last) + (into {}))) + +;; {"Deadline" {:date {:year 2020, :month 10, :day 20}, :wday "Tue", :time {:hour 8, :min 0}, :repetition [["DoublePlus"] ["Day"] 1], :active true}} +(defn timestamps->scheduled-and-deadline + [timestamps] + (let [timestamps (medley/map-keys (comp keyword string/lower-case) timestamps) + m (some->> (select-keys timestamps [:scheduled :deadline]) + (map (fn [[k v]] + (let [{:keys [date repetition]} v + {:keys [year month day]} date + day (js/parseInt (str year (util/zero-pad month) (util/zero-pad day)))] + (cond-> + (case k + :scheduled + {:scheduled day} + :deadline + {:deadline day}) + repetition + (assoc :repeated? true))))))] + (apply merge m))) + +(defn convert-page-if-journal + "Convert journal file name to user' custom date format" + [original-page-name] + (when original-page-name + (let [page-name (util/page-name-sanity-lc original-page-name) + day (date/journal-title->int page-name)] + (if day + (let [original-page-name (date/int->journal-title day)] + [original-page-name (util/page-name-sanity-lc original-page-name) day]) + [original-page-name page-name day])))) + +(defn page-name->map + "Create a page's map structure given a original page name (string). + map as input is supported for legacy compatibility. + with-timestamp?: assign timestampes to the map structure. + Useful when creating new pages from references or namespaces, + as there's no chance to introduce timestamps via editing in page" + ([original-page-name with-id?] + (page-name->map original-page-name with-id? true)) + ([original-page-name with-id? with-timestamp?] + (cond + (and original-page-name (string? original-page-name)) + (let [original-page-name (util/remove-boundary-slashes original-page-name) + [original-page-name page-name journal-day] (convert-page-if-journal original-page-name) + namespace? (and (not (boolean (text/get-nested-page-name original-page-name))) + (text/namespace-page? original-page-name)) + page-entity (db/entity [:block/name page-name]) + original-page-name (or (:block/original-name page-entity) original-page-name)] + (merge + {:block/name page-name + :block/original-name original-page-name} + (when with-id? + (if page-entity + {:block/uuid (:block/uuid page-entity)} + {:block/uuid (db/new-block-id)})) + (when namespace? + (let [namespace (first (gp-util/split-last "/" original-page-name))] + (when-not (string/blank? namespace) + {:block/namespace {:block/name (util/page-name-sanity-lc namespace)}}))) + (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity + (let [current-ms (util/time-ms)] + {:block/created-at current-ms + :block/updated-at current-ms})) + (if journal-day + {:block/journal? true + :block/journal-day journal-day} + {:block/journal? false}))) + + (and (map? original-page-name) (:block/uuid original-page-name)) + original-page-name + + (and (map? original-page-name) with-id?) + (assoc original-page-name :block/uuid (db/new-block-id)) + + :else + nil))) + +(defn with-page-refs + [{:keys [title body tags refs marker priority] :as block} with-id?] + (let [refs (->> (concat tags refs [marker priority]) + (remove string/blank?) + (distinct)) + refs (atom refs)] + (walk/prewalk + (fn [form] + ;; skip custom queries + (when-not (and (vector? form) + (= (first form) "Custom") + (= (second form) "query")) + (when-let [page (get-page-reference form)] + (swap! refs conj page)) + (when-let [tag (get-tag form)] + (let [tag (text/page-ref-un-brackets! tag)] + (when (gp-util/tag-valid? tag) + (swap! refs conj tag)))) + form)) + (concat title body)) + (let [refs (remove string/blank? @refs) + children-pages (->> (mapcat (fn [p] + (let [p (if (map? p) + (:block/original-name p) + p)] + (when (string? p) + (let [p (or (text/get-nested-page-name p) p)] + (when (text/namespace-page? p) + (util/split-namespace-pages p)))))) + refs) + (remove string/blank?) + (distinct)) + refs (->> (distinct (concat refs children-pages)) + (remove nil?)) + refs (map (fn [ref] (page-name->map ref with-id?)) refs)] + (assoc block :refs refs)))) + +(defn with-block-refs + [{:keys [title body] :as block}] + (let [ref-blocks (atom nil)] + (walk/postwalk + (fn [form] + (when-let [block (get-block-reference form)] + (swap! ref-blocks conj block)) + form) + (concat title body)) + (let [ref-blocks (->> @ref-blocks + (filter gp-util/uuid-string?)) + ref-blocks (map + (fn [id] + [:block/uuid (medley/uuid id)]) + ref-blocks) + refs (distinct (concat (:refs block) ref-blocks))] + (assoc block :refs refs)))) + +(defn- block-keywordize + [block] + (medley/map-keys + (fn [k] + (if (namespace k) + k + (keyword "block" k))) + block)) + +(defn- sanity-blocks-data + [blocks] + (map (fn [block] + (if (map? block) + (block-keywordize (gp-util/remove-nils block)) + block)) + blocks)) + +(defn with-path-refs + [blocks] + (loop [blocks blocks + acc [] + parents []] + (if (empty? blocks) + acc + (let [block (first blocks) + cur-level (:block/level block) + level-diff (- cur-level + (get (last parents) :block/level 0)) + [path-refs parents] + (cond + (zero? level-diff) ; sibling + (let [path-refs (mapcat :block/refs (drop-last parents)) + parents (conj (vec (butlast parents)) block)] + [path-refs parents]) + + (> level-diff 0) ; child + (let [path-refs (mapcat :block/refs parents)] + [path-refs (conj parents block)]) + + (< level-diff 0) ; new parent + (let [parents (vec (take-while (fn [p] (< (:block/level p) cur-level)) parents)) + path-refs (mapcat :block/refs parents)] + [path-refs (conj parents block)])) + path-ref-pages (->> path-refs + (concat (:block/refs block)) + (map (fn [ref] + (cond + (map? ref) + (:block/name ref) + + :else + ref))) + (remove string/blank?) + (map (fn [ref] + (if (string? ref) + {:block/name (util/page-name-sanity-lc ref)} + ref))) + (remove vector?) + (remove nil?) + (distinct))] + (recur (rest blocks) + (conj acc (assoc block :block/path-refs path-ref-pages)) + parents))))) + +(defn block-tags->pages + [{:keys [tags] :as block}] + (if (seq tags) + (assoc block :tags (map (fn [tag] + (let [tag (text/page-ref-un-brackets! tag)] + [:block/name (util/page-name-sanity-lc tag)])) tags)) + block)) + +(defn- get-block-content + [utf8-content block format meta] + (let [content (if-let [end-pos (:end_pos meta)] + (utf8/substring utf8-content + (:start_pos meta) + end-pos) + (utf8/substring utf8-content + (:start_pos meta))) + content (when content + (let [content (text/remove-level-spaces content format (config/get-block-pattern format))] + (if (or (:pre-block? block) + (= (:format block) :org)) + content + (gp-mldoc/remove-indentation-spaces content (inc (:level block)) false))))] + (if (= format :org) + content + (property/->new-properties content)))) + +(defn get-custom-id-or-new-id + [properties] + (or (when-let [custom-id (or (get-in properties [:properties :custom-id]) + (get-in properties [:properties :custom_id]) + (get-in properties [:properties :id]))] + (let [custom-id (and (string? custom-id) (string/trim custom-id))] + (when (and custom-id (gp-util/uuid-string? custom-id)) + (uuid custom-id)))) + (db/new-block-id))) + +(defn get-page-refs-from-properties + [properties] + (let [page-refs (mapcat (fn [v] (cond + (coll? v) + v + + (text/page-ref? v) + [(text/page-ref-un-brackets! v)] + + :else + nil)) (vals properties)) + page-refs (remove string/blank? page-refs)] + (map (fn [page] (page-name->map page true)) page-refs))) + +(defn with-page-block-refs + [block with-id?] + (some-> block + (with-page-refs with-id?) + with-block-refs + block-tags->pages + (update :refs (fn [col] (remove nil? col))))) + +(defn with-pre-block-if-exists + [blocks body pre-block-properties encoded-content] + (let [first-block (first blocks) + first-block-start-pos (get-in first-block [:block/meta :start_pos]) + + ;; Add pre-block + blocks (if (or (> first-block-start-pos 0) + (empty? blocks)) + (cons + (merge + (let [content (utf8/substring encoded-content 0 first-block-start-pos) + {:keys [properties properties-order]} pre-block-properties + id (get-custom-id-or-new-id {:properties properties}) + property-refs (->> (get-page-refs-from-properties properties) + (map :block/original-name)) + block {:uuid id + :content content + :level 1 + :properties properties + :properties-order properties-order + :refs property-refs + :pre-block? true + :unordered true + :body body} + block (with-page-block-refs block false)] + (block-keywordize block)) + (select-keys first-block [:block/format :block/page])) + blocks) + blocks)] + (with-path-refs blocks))) + +(defn- construct-block + [block properties timestamps body encoded-content format pos-meta with-id?] + (let [id (get-custom-id-or-new-id properties) + ref-pages-in-properties (->> (:page-refs properties) + (remove string/blank?)) + block (second block) + unordered? (:unordered block) + markdown-heading? (and (:size block) (= :markdown format)) + block (if markdown-heading? + (assoc block + :type :heading + :level (if unordered? (:level block) 1) + :heading-level (or (:size block) 6)) + block) + block (cond-> + (assoc block + :uuid id + :refs ref-pages-in-properties + :format format + :meta pos-meta) + (seq (:properties properties)) + (assoc :properties (:properties properties)) + + (seq (:properties-order properties)) + (assoc :properties-order (:properties-order properties))) + block (if (get-in block [:properties :collapsed]) + (assoc block :collapsed? true) + block) + block (assoc block + :content (get-block-content encoded-content block format pos-meta)) + block (if (seq timestamps) + (merge block (timestamps->scheduled-and-deadline timestamps)) + block) + block (assoc block :body body) + block (with-page-block-refs block with-id?) + {:keys [created-at updated-at]} (:properties properties) + block (cond-> block + (and created-at (integer? created-at)) + (assoc :block/created-at created-at) + + (and updated-at (integer? updated-at)) + (assoc :block/updated-at updated-at))] + (dissoc block :title :body :anchor))) + +(defn extract-blocks + "Extract headings from mldoc ast. + Args: + `blocks`: mldoc ast. + `content`: markdown or org-mode text. + `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids. + `format`: content's format, it could be either :markdown or :org-mode." + [blocks content with-id? format] + {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]} + (try + (let [encoded-content (utf8/encode content) + [blocks body pre-block-properties] + (loop [headings [] + blocks (reverse blocks) + timestamps {} + properties {} + body []] + (if (seq blocks) + (let [[block pos-meta] (first blocks) + ;; fix start_pos + pos-meta (assoc pos-meta :end_pos + (if (seq headings) + (get-in (last headings) [:meta :start_pos]) + nil))] + (cond + (paragraph-timestamp-block? block) + (let [timestamps (extract-timestamps block) + timestamps' (merge timestamps timestamps)] + (recur headings (rest blocks) timestamps' properties body)) + + (property/properties-ast? block) + (let [properties (extract-properties format (second block))] + (recur headings (rest blocks) timestamps properties body)) + + (heading-block? block) + (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id?)] + (recur (conj headings block) (rest blocks) {} {} [])) + + :else + (recur headings (rest blocks) timestamps properties (conj body block)))) + [(-> (reverse headings) + sanity-blocks-data) + body + properties])) + result (with-pre-block-if-exists blocks body pre-block-properties encoded-content)] + (map #(dissoc % :block/meta) result)) + (catch js/Error e + (js/console.error "extract-blocks-failed") + (log/error :exception e)))) diff --git a/src/test/frontend/format/block_test.cljs b/src/test/logseq/graph_parser/block_test.cljs similarity index 88% rename from src/test/frontend/format/block_test.cljs rename to src/test/logseq/graph_parser/block_test.cljs index 7b326426651..35a82846393 100644 --- a/src/test/frontend/format/block_test.cljs +++ b/src/test/logseq/graph_parser/block_test.cljs @@ -1,9 +1,9 @@ -(ns frontend.format.block-test - (:require [frontend.format.block :as block] +(ns logseq.graph-parser.block-test + (:require [logseq.graph-parser.block :as gp-block] [cljs.test :refer [deftest are]])) (deftest test-extract-properties - (are [x y] (= (:properties (block/extract-properties :markdown x)) y) + (are [x y] (= (:properties (gp-block/extract-properties :markdown x)) y) [["year" "1000"]] {:year 1000} [["year" "\"1000\""]] {:year "\"1000\""} [["background-color" "#000000"]] {:background-color "#000000"} @@ -23,7 +23,7 @@ [["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}} [["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}}) - (are [x y] (= (vec (:page-refs (block/extract-properties :markdown x))) y) + (are [x y] (= (vec (:page-refs (gp-block/extract-properties :markdown x))) y) [["year" "1000"]] [] [["year" "\"1000\""]] [] [["foo" "[[bar]] test"]] ["bar" "test"] From b83f6bbd19e2600d0b0678a4a5a6f45ec2cf7e31 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 11 May 2022 17:37:03 -0400 Subject: [PATCH 38/58] Migrate half the block deps to be nbb compatible --- .clj-kondo/config.edn | 2 + src/main/frontend/components/block.cljs | 2 +- src/main/frontend/format/block.cljs | 4 +- src/main/frontend/handler/external.cljs | 3 +- src/main/frontend/handler/extract.cljs | 6 +- src/main/frontend/handler/page.cljs | 10 +-- src/main/frontend/util.cljc | 56 +++------------ src/main/frontend/util/drawer.cljs | 6 +- src/main/frontend/util/property.cljs | 53 ++------------ src/main/frontend/util/thingatpt.cljs | 6 +- src/main/logseq/graph_parser/block.cljs | 70 +++++++++---------- .../logseq/graph_parser/date_time_util.cljs | 9 +++ src/main/logseq/graph_parser/property.cljs | 48 +++++++++++++ src/main/logseq/graph_parser/util.cljs | 65 ++++++++++++++++- src/test/frontend/test/docs_graph_helper.cljs | 2 +- src/test/frontend/util/property_test.cljs | 23 ------ .../logseq/graph_parser/nbb_test_runner.cljs | 7 +- .../logseq/graph_parser/property_test.cljs | 26 +++++++ 18 files changed, 223 insertions(+), 175 deletions(-) create mode 100644 src/main/logseq/graph_parser/date_time_util.cljs create mode 100644 src/main/logseq/graph_parser/property.cljs create mode 100644 src/test/logseq/graph_parser/property_test.cljs diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index d3e9e3a31bc..4a91e0a99d9 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -19,6 +19,7 @@ frontend.db.react react frontend.db.query-react query-react frontend.util util + frontend.util.property property frontend.config config frontend.format.mldoc mldoc frontend.format.block block @@ -26,6 +27,7 @@ logseq.graph-parser.block gp-block logseq.graph-parser.mldoc gp-mldoc logseq.graph-parser.util gp-util + logseq.graph-parser.property gp-property logseq.graph-parser.config gp-config}}} :hooks {:analyze-call {rum.core/defc hooks.rum/defc diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 130c6776cc7..2e11406e03a 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -499,7 +499,7 @@ "Accepts {:block/name sanitized / unsanitized page-name}" [{:keys [html-export? redirect-page-name label children contents-page? preview?] :as config} page] (when-let [page-name-in-block (:block/name page)] - (let [page-name-in-block (util/remove-boundary-slashes page-name-in-block) + (let [page-name-in-block (gp-util/remove-boundary-slashes page-name-in-block) page-name (util/page-name-sanity-lc page-name-in-block) page-entity (db/entity [:block/name page-name]) redirect-page-name (or (and (= :org (state/get-preferred-format)) diff --git a/src/main/frontend/format/block.cljs b/src/main/frontend/format/block.cljs index 72dd1bd8fac..ab20658bf11 100644 --- a/src/main/frontend/format/block.cljs +++ b/src/main/frontend/format/block.cljs @@ -6,7 +6,7 @@ [frontend.db :as db] [frontend.format :as format] [frontend.state :as state] - [frontend.util.property :as property] + [logseq.graph-parser.property :as gp-property] [logseq.graph-parser.mldoc :as gp-mldoc])) (defn with-parent-and-left @@ -125,7 +125,7 @@ title (when (gp-block/heading-block? (first ast)) (:title (second (first ast)))) body (vec (if title (rest ast) ast)) - body (drop-while property/properties-ast? body) + body (drop-while gp-property/properties-ast? body) result (cond-> (if (seq body) {:block/body body} {}) title diff --git a/src/main/frontend/handler/external.cljs b/src/main/frontend/handler/external.cljs index ce8e4e63235..3436ef5b115 100644 --- a/src/main/frontend/handler/external.cljs +++ b/src/main/frontend/handler/external.cljs @@ -9,6 +9,7 @@ [frontend.db :as db] [frontend.format.mldoc :as mldoc] [logseq.graph-parser.block :as gp-block] + [logseq.graph-parser.util :as gp-util] [frontend.handler.page :as page] [frontend.handler.editor :as editor] [frontend.util :as util])) @@ -26,7 +27,7 @@ (when journal? (date/journal-title->default title)) (string/replace title "/" "-")) - title (-> (util/page-name-sanity title) + title (-> (gp-util/page-name-sanity title) (string/replace "\n" " ")) path (str (if journal? (config/get-journals-directory) diff --git a/src/main/frontend/handler/extract.cljs b/src/main/frontend/handler/extract.cljs index 0b8a89f13c8..2c463ed944a 100644 --- a/src/main/frontend/handler/extract.cljs +++ b/src/main/frontend/handler/extract.cljs @@ -14,7 +14,7 @@ [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.block :as gp-block] - [frontend.util.property :as property] + [logseq.graph-parser.property :as gp-property] [lambdaisland.glogi :as log])) (defn get-page-name @@ -113,7 +113,7 @@ tags))))) namespace-pages (let [page (:block/original-name page-entity)] (when (text/namespace-page? page) - (->> (util/split-namespace-pages page) + (->> (gp-util/split-namespace-pages page) (map (fn [page] (-> (gp-block/page-name->map page true) (assoc :block/format format))))))) @@ -149,7 +149,7 @@ ))] (println "Parsing finished : " file) (let [first-block (ffirst ast) - properties (let [properties (and (property/properties-ast? first-block) + properties (let [properties (and (gp-property/properties-ast? first-block) (->> (last first-block) (map (fn [[x y]] [x (if (string? y) diff --git a/src/main/frontend/handler/page.cljs b/src/main/frontend/handler/page.cljs index ebeceeaf344..9819469544b 100644 --- a/src/main/frontend/handler/page.cljs +++ b/src/main/frontend/handler/page.cljs @@ -47,7 +47,7 @@ [journal? title] (when-let [s (if journal? (date/journal-title->default title) - (util/page-name-sanity (string/lower-case title)))] + (gp-util/page-name-sanity (string/lower-case title)))] ;; Win10 file path has a length limit of 260 chars (gp-util/safe-subs s 0 200))) @@ -118,12 +118,12 @@ properties nil split-namespace? true}}] (let [title (string/trim title) - title (util/remove-boundary-slashes title) + title (gp-util/remove-boundary-slashes title) page-name (util/page-name-sanity-lc title) repo (state/get-current-repo)] (when (db/page-empty? repo page-name) (let [pages (if split-namespace? - (util/split-namespace-pages title) + (gp-util/split-namespace-pages title) [title]) format (or format (state/get-preferred-format)) pages (map (fn [page] @@ -170,7 +170,7 @@ (defn- compute-new-file-path [old-path new-name] (let [result (string/split old-path "/") - file-name (util/page-name-sanity new-name true) + file-name (gp-util/page-name-sanity new-name true) ext (last (string/split (last result) ".")) new-file (str file-name "." ext) parts (concat (butlast result) [new-file])] @@ -398,7 +398,7 @@ ;; If page name changed after sanitization (when (or (util/create-title-property? new-page-name) - (not= (util/page-name-sanity new-name false) new-name)) + (not= (gp-util/page-name-sanity new-name false) new-name)) (page-property/add-property! new-page-name :title new-name)) (when (and file (not journal?)) diff --git a/src/main/frontend/util.cljc b/src/main/frontend/util.cljc index cb91e56d7e3..eab914c4643 100644 --- a/src/main/frontend/util.cljc +++ b/src/main/frontend/util.cljc @@ -907,44 +907,18 @@ (string/includes? s "%") (string/includes? s "#")))))) -(defn remove-boundary-slashes - [s] - (when (string? s) - (let [s (if (= \/ (first s)) - (subs s 1) - s)] - (if (= \/ (last s)) - (subs s 0 (dec (count s))) - s)))) - -(defn normalize - [s] - (.normalize s "NFC")) - #?(:cljs (defn search-normalize "Normalize string for searching (loose)" [s] (removeAccents (.normalize (string/lower-case s) "NFKC")))) -(defn page-name-sanity - "Sanitize the page-name." - ([page-name] - (page-name-sanity page-name false)) - ([page-name replace-slash?] - (let [page (some-> page-name - (remove-boundary-slashes) - (normalize))] - (if replace-slash? - (string/replace page #"/" "%2A") - page)))) - #?(:cljs (defn file-name-sanity "Sanitize page-name for file name (strict), for file writing." [page-name] (some-> page-name - page-name-sanity + gp-util/page-name-sanity ;; for android filesystem compatiblity (string/replace #"[\\#|%]+" url-encode) ;; Windows reserved path characters @@ -952,15 +926,16 @@ (string/replace #"/" url-encode) (string/replace "*" "%2A")))) -(defn page-name-sanity-lc - "Sanitize the query string for a page name (mandate for :block/name)" - [s] - (page-name-sanity (string/lower-case s))) +#?(:cljs + (def page-name-sanity-lc + "Delegate to gp-util to loosely couple app usages to graph-parser" + gp-util/page-name-sanity-lc)) -(defn safe-page-name-sanity-lc - [s] - (if (string? s) - (page-name-sanity-lc s) s)) +#?(:cljs + (defn safe-page-name-sanity-lc + [s] + (if (string? s) + (page-name-sanity-lc s) s))) (defn get-page-original-name [page] @@ -1240,17 +1215,6 @@ [text] (string/join (replace regex-char-esc-smap text))) -(defn split-namespace-pages - [title] - (let [parts (string/split title "/")] - (loop [others (rest parts) - result [(first parts)]] - (if (seq others) - (let [prev (last result)] - (recur (rest others) - (conj result (str prev "/" (first others))))) - result)))) - (comment (re-matches (re-pattern (regex-escape "$u^8(d)+w.*[dw]d?")) "$u^8(d)+w.*[dw]d?")) diff --git a/src/main/frontend/util/drawer.cljs b/src/main/frontend/util/drawer.cljs index cf3f81c84e6..a07ec2a225c 100644 --- a/src/main/frontend/util/drawer.cljs +++ b/src/main/frontend/util/drawer.cljs @@ -3,7 +3,7 @@ [frontend.util :as util] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] - [frontend.util.property :as property] + [logseq.graph-parser.property :as gp-property] [frontend.format.mldoc :as mldoc])) (defn drawer-start @@ -55,8 +55,8 @@ (if has-properties? (cond (= :org format) - (let [prop-start-idx (.indexOf body-without-timestamps property/properties-start) - prop-end-idx (.indexOf body-without-timestamps property/properties-end) + (let [prop-start-idx (.indexOf body-without-timestamps gp-property/properties-start) + prop-end-idx (.indexOf body-without-timestamps gp-property/properties-end) properties (subvec body-without-timestamps prop-start-idx (inc prop-end-idx)) after (subvec body-without-timestamps (inc prop-end-idx))] (string/join "\n" (concat [title] scheduled deadline properties [drawer] after))) diff --git a/src/main/frontend/util/property.cljs b/src/main/frontend/util/property.cljs index c62d075bf66..c9c6138ab95 100644 --- a/src/main/frontend/util/property.cljs +++ b/src/main/frontend/util/property.cljs @@ -1,4 +1,5 @@ (ns frontend.util.property + "Property fns needed by the rest of the app and not graph-parser" (:require [clojure.string :as string] [frontend.util :as util] [clojure.set :as set] @@ -6,15 +7,11 @@ [medley.core :as medley] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] + [logseq.graph-parser.property :as gp-property :refer [properties-start properties-end]] [frontend.format.mldoc :as mldoc] [logseq.graph-parser.text :as text] [frontend.util.cursor :as cursor])) -(defonce properties-start ":PROPERTIES:") -(defonce properties-end ":END:") -(defonce properties-end-pattern - (re-pattern (util/format "%s[\t\r ]*\n|(%s\\s*$)" properties-end properties-end))) - (def built-in-extended-properties (atom #{})) (defn register-built-in-properties [props] @@ -36,15 +33,9 @@ built-in-properties-set (built-in-properties)] (every? built-in-properties-set ks)))) -(defn contains-properties? - [content] - (when content - (and (string/includes? content properties-start) - (gp-util/safe-re-find properties-end-pattern content)))) - (defn remove-empty-properties [content] - (if (contains-properties? content) + (if (gp-property/contains-properties? content) (string/replace content (re-pattern ":PROPERTIES:\n+:END:\n*") "") @@ -114,7 +105,7 @@ (defn get-property-keys [format content] (cond - (contains-properties? content) + (gp-property/contains-properties? content) (get-org-property-keys content) (= :markdown format) @@ -134,7 +125,7 @@ (defn remove-properties [format content] (cond - (contains-properties? content) + (gp-property/contains-properties? content) (let [lines (string/split-lines content) [title-lines properties&body] (split-with #(-> (string/triml %) string/upper-case @@ -348,7 +339,7 @@ (let [format (or format :markdown) key (string/lower-case (name key)) remove-f (if first? util/remove-first remove)] - (if (and (= format :org) (not (contains-properties? content))) + (if (and (= format :org) (not (gp-property/contains-properties? content))) content (let [lines (->> (string/split-lines content) (remove-f (fn [line] @@ -371,31 +362,6 @@ (string/replace-first content (re-pattern ":PROPERTIES:\n:END:\n*") "") content))) -(defn ->new-properties - "New syntax: key:: value" - [content] - (if (contains-properties? content) - (let [lines (string/split-lines content) - start-idx (.indexOf lines properties-start) - end-idx (.indexOf lines properties-end)] - (if (and (>= start-idx 0) (> end-idx 0) (> end-idx start-idx)) - (let [before (subvec lines 0 start-idx) - middle (->> (subvec lines (inc start-idx) end-idx) - (map (fn [text] - (let [[k v] (gp-util/split-first ":" (subs text 1))] - (if (and k v) - (let [k (string/replace k "_" "-") - compare-k (keyword (string/lower-case k)) - k (if (contains? #{:id :custom_id :custom-id} compare-k) "id" k) - k (if (contains? #{:last-modified-at} compare-k) "updated-at" k)] - (str k ":: " (string/trim v))) - text))))) - after (subvec lines (inc end-idx)) - lines (concat before middle after)] - (string/join "\n" lines)) - content)) - content)) - (defn add-page-properties [page-format properties-content properties] (let [properties (medley/map-keys name properties) @@ -435,10 +401,3 @@ (util/format (config/properties-wrapper-pattern page-format) (string/join "\n" lines)))) - -(defn properties-ast? - [block] - (and - (vector? block) - (contains? #{"Property_Drawer" "Properties"} - (first block)))) diff --git a/src/main/frontend/util/thingatpt.cljs b/src/main/frontend/util/thingatpt.cljs index e0690328de1..beff6833e92 100644 --- a/src/main/frontend/util/thingatpt.cljs +++ b/src/main/frontend/util/thingatpt.cljs @@ -1,10 +1,10 @@ (ns frontend.util.thingatpt (:require [clojure.string :as string] [frontend.state :as state] - [frontend.util.property :as property-util] [frontend.util.cursor :as cursor] [frontend.config :as config] [logseq.graph-parser.text :as text] + [logseq.graph-parser.property :as gp-property] [cljs.reader :as reader] [goog.object :as gobj])) @@ -69,8 +69,8 @@ (when-let [properties (case (state/get-preferred-format) ;; TODO fix me to block's format :org (thing-at-point - [property-util/properties-start - property-util/properties-end] + [gp-property/properties-start + gp-property/properties-end] input) (when-let [line (line-at-point input)] (when (re-matches #"^[^\s.]+:: .*$" (:raw-content line)) diff --git a/src/main/logseq/graph_parser/block.cljs b/src/main/logseq/graph_parser/block.cljs index de169410670..e718c22d7e9 100644 --- a/src/main/logseq/graph_parser/block.cljs +++ b/src/main/logseq/graph_parser/block.cljs @@ -5,16 +5,16 @@ [frontend.config :as config] [frontend.date :as date] [frontend.db :as db] + [datascript.core :as d] [frontend.state :as state] [logseq.graph-parser.text :as text] [logseq.graph-parser.utf8 :as utf8] - [frontend.util :as util] - [frontend.util.property :as property] + [logseq.graph-parser.property :as gp-property] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.config :as gp-config] [logseq.graph-parser.mldoc :as gp-mldoc] - [lambdaisland.glogi :as log] - [medley.core :as medley])) + [logseq.graph-parser.date-time-util :as date-time-util] + [lambdaisland.glogi :as log])) (defn heading-block? [block] @@ -59,10 +59,10 @@ (and (= typ "Search") (not (contains? #{\# \* \/ \[} (first value))) - (let [ext (some-> (util/get-file-ext value) keyword)] - (when (and (not (util/starts-with? value "http:")) - (not (util/starts-with? value "https:")) - (not (util/starts-with? value "file:")) + (let [ext (some-> (gp-util/get-file-ext value) keyword)] + (when (and (not (string/starts-with? value "http:")) + (not (string/starts-with? value "https:")) + (not (string/starts-with? value "file:")) (not (gp-config/local-asset? value)) (or (= ext :excalidraw) (not (contains? (config/supported-formats) ext)))) @@ -97,7 +97,7 @@ nil)] (text/block-ref-un-brackets! page))) -(defn get-block-reference +(defn- get-block-reference [block] (when-let [block-id (cond (and (vector? block) @@ -133,7 +133,7 @@ (gp-util/uuid-string? block-id)) block-id))) -(defn paragraph-block? +(defn- paragraph-block? [block] (and (vector? block) @@ -197,7 +197,7 @@ (or (timestamp-block? (first (second block))) (timestamp-block? (second (second block)))))) -(defn extract-timestamps +(defn- extract-timestamps [block] (some->> (second block) @@ -208,12 +208,12 @@ ;; {"Deadline" {:date {:year 2020, :month 10, :day 20}, :wday "Tue", :time {:hour 8, :min 0}, :repetition [["DoublePlus"] ["Day"] 1], :active true}} (defn timestamps->scheduled-and-deadline [timestamps] - (let [timestamps (medley/map-keys (comp keyword string/lower-case) timestamps) + (let [timestamps (gp-util/map-keys (comp keyword string/lower-case) timestamps) m (some->> (select-keys timestamps [:scheduled :deadline]) (map (fn [[k v]] (let [{:keys [date repetition]} v {:keys [year month day]} date - day (js/parseInt (str year (util/zero-pad month) (util/zero-pad day)))] + day (js/parseInt (str year (gp-util/zero-pad month) (gp-util/zero-pad day)))] (cond-> (case k :scheduled @@ -228,11 +228,11 @@ "Convert journal file name to user' custom date format" [original-page-name] (when original-page-name - (let [page-name (util/page-name-sanity-lc original-page-name) + (let [page-name (gp-util/page-name-sanity-lc original-page-name) day (date/journal-title->int page-name)] (if day (let [original-page-name (date/int->journal-title day)] - [original-page-name (util/page-name-sanity-lc original-page-name) day]) + [original-page-name (gp-util/page-name-sanity-lc original-page-name) day]) [original-page-name page-name day])))) (defn page-name->map @@ -246,7 +246,7 @@ ([original-page-name with-id? with-timestamp?] (cond (and original-page-name (string? original-page-name)) - (let [original-page-name (util/remove-boundary-slashes original-page-name) + (let [original-page-name (gp-util/remove-boundary-slashes original-page-name) [original-page-name page-name journal-day] (convert-page-if-journal original-page-name) namespace? (and (not (boolean (text/get-nested-page-name original-page-name))) (text/namespace-page? original-page-name)) @@ -258,13 +258,13 @@ (when with-id? (if page-entity {:block/uuid (:block/uuid page-entity)} - {:block/uuid (db/new-block-id)})) + {:block/uuid (d/squuid)})) (when namespace? (let [namespace (first (gp-util/split-last "/" original-page-name))] (when-not (string/blank? namespace) - {:block/namespace {:block/name (util/page-name-sanity-lc namespace)}}))) + {:block/namespace {:block/name (gp-util/page-name-sanity-lc namespace)}}))) (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity - (let [current-ms (util/time-ms)] + (let [current-ms (date-time-util/time-ms)] {:block/created-at current-ms :block/updated-at current-ms})) (if journal-day @@ -276,12 +276,12 @@ original-page-name (and (map? original-page-name) with-id?) - (assoc original-page-name :block/uuid (db/new-block-id)) + (assoc original-page-name :block/uuid (d/squuid)) :else nil))) -(defn with-page-refs +(defn- with-page-refs [{:keys [title body tags refs marker priority] :as block} with-id?] (let [refs (->> (concat tags refs [marker priority]) (remove string/blank?) @@ -309,7 +309,7 @@ (when (string? p) (let [p (or (text/get-nested-page-name p) p)] (when (text/namespace-page? p) - (util/split-namespace-pages p)))))) + (gp-util/split-namespace-pages p)))))) refs) (remove string/blank?) (distinct)) @@ -318,7 +318,7 @@ refs (map (fn [ref] (page-name->map ref with-id?)) refs)] (assoc block :refs refs)))) -(defn with-block-refs +(defn- with-block-refs [{:keys [title body] :as block}] (let [ref-blocks (atom nil)] (walk/postwalk @@ -331,14 +331,14 @@ (filter gp-util/uuid-string?)) ref-blocks (map (fn [id] - [:block/uuid (medley/uuid id)]) + [:block/uuid (uuid id)]) ref-blocks) refs (distinct (concat (:refs block) ref-blocks))] (assoc block :refs refs)))) (defn- block-keywordize [block] - (medley/map-keys + (gp-util/map-keys (fn [k] (if (namespace k) k @@ -353,7 +353,7 @@ block)) blocks)) -(defn with-path-refs +(defn- with-path-refs [blocks] (loop [blocks blocks acc [] @@ -391,7 +391,7 @@ (remove string/blank?) (map (fn [ref] (if (string? ref) - {:block/name (util/page-name-sanity-lc ref)} + {:block/name (gp-util/page-name-sanity-lc ref)} ref))) (remove vector?) (remove nil?) @@ -400,12 +400,12 @@ (conj acc (assoc block :block/path-refs path-ref-pages)) parents))))) -(defn block-tags->pages +(defn- block-tags->pages [{:keys [tags] :as block}] (if (seq tags) (assoc block :tags (map (fn [tag] (let [tag (text/page-ref-un-brackets! tag)] - [:block/name (util/page-name-sanity-lc tag)])) tags)) + [:block/name (gp-util/page-name-sanity-lc tag)])) tags)) block)) (defn- get-block-content @@ -424,9 +424,9 @@ (gp-mldoc/remove-indentation-spaces content (inc (:level block)) false))))] (if (= format :org) content - (property/->new-properties content)))) + (gp-property/->new-properties content)))) -(defn get-custom-id-or-new-id +(defn- get-custom-id-or-new-id [properties] (or (when-let [custom-id (or (get-in properties [:properties :custom-id]) (get-in properties [:properties :custom_id]) @@ -434,7 +434,7 @@ (let [custom-id (and (string? custom-id) (string/trim custom-id))] (when (and custom-id (gp-util/uuid-string? custom-id)) (uuid custom-id)))) - (db/new-block-id))) + (d/squuid))) (defn get-page-refs-from-properties [properties] @@ -450,7 +450,7 @@ page-refs (remove string/blank? page-refs)] (map (fn [page] (page-name->map page true)) page-refs))) -(defn with-page-block-refs +(defn- with-page-block-refs [block with-id?] (some-> block (with-page-refs with-id?) @@ -458,7 +458,7 @@ block-tags->pages (update :refs (fn [col] (remove nil? col))))) -(defn with-pre-block-if-exists +(defn- with-pre-block-if-exists [blocks body pre-block-properties encoded-content] (let [first-block (first blocks) first-block-start-pos (get-in first-block [:block/meta :start_pos]) @@ -563,7 +563,7 @@ timestamps' (merge timestamps timestamps)] (recur headings (rest blocks) timestamps' properties body)) - (property/properties-ast? block) + (gp-property/properties-ast? block) (let [properties (extract-properties format (second block))] (recur headings (rest blocks) timestamps properties body)) diff --git a/src/main/logseq/graph_parser/date_time_util.cljs b/src/main/logseq/graph_parser/date_time_util.cljs new file mode 100644 index 00000000000..0bcbe18d395 --- /dev/null +++ b/src/main/logseq/graph_parser/date_time_util.cljs @@ -0,0 +1,9 @@ +(ns ^:nbb-compatible logseq.graph-parser.date-time-util + "cljs-time util fns for graph-parser" + (:require [cljs-time.coerce :as tc] + [cljs-time.core :as t])) + +(defn time-ms + "Copy of util/time-ms. Too basic to couple this to main app" + [] + (tc/to-long (t/now))) diff --git a/src/main/logseq/graph_parser/property.cljs b/src/main/logseq/graph_parser/property.cljs new file mode 100644 index 00000000000..3d5adb9b029 --- /dev/null +++ b/src/main/logseq/graph_parser/property.cljs @@ -0,0 +1,48 @@ +(ns logseq.graph-parser.property + "Property fns needed by graph-parser" + (:require [logseq.graph-parser.util :as gp-util] + [clojure.string :as string] + [goog.string :as gstring])) + +(defn properties-ast? + [block] + (and + (vector? block) + (contains? #{"Property_Drawer" "Properties"} + (first block)))) + +(defonce properties-start ":PROPERTIES:") +(defonce properties-end ":END:") +(defonce properties-end-pattern + (re-pattern (gstring/format "%s[\t\r ]*\n|(%s\\s*$)" properties-end properties-end))) + +(defn contains-properties? + [content] + (when content + (and (string/includes? content properties-start) + (gp-util/safe-re-find properties-end-pattern content)))) + +(defn ->new-properties + "New syntax: key:: value" + [content] + (if (contains-properties? content) + (let [lines (string/split-lines content) + start-idx (.indexOf lines properties-start) + end-idx (.indexOf lines properties-end)] + (if (and (>= start-idx 0) (> end-idx 0) (> end-idx start-idx)) + (let [before (subvec lines 0 start-idx) + middle (->> (subvec lines (inc start-idx) end-idx) + (map (fn [text] + (let [[k v] (gp-util/split-first ":" (subs text 1))] + (if (and k v) + (let [k (string/replace k "_" "-") + compare-k (keyword (string/lower-case k)) + k (if (contains? #{:id :custom_id :custom-id} compare-k) "id" k) + k (if (contains? #{:last-modified-at} compare-k) "updated-at" k)] + (str k ":: " (string/trim v))) + text))))) + after (subvec lines (inc end-idx)) + lines (concat before middle after)] + (string/join "\n" lines)) + content)) + content)) diff --git a/src/main/logseq/graph_parser/util.cljs b/src/main/logseq/graph_parser/util.cljs index d98898049a9..821168a66a1 100644 --- a/src/main/logseq/graph_parser/util.cljs +++ b/src/main/logseq/graph_parser/util.cljs @@ -66,14 +66,14 @@ (and (string? v) (>= (count v) 2) (= "\"" (first v) (last v)))) (defn parse-int - "Copy of frontend.util/parse-int. Don't want to couple to main app too much" + "Copy of frontend.util/parse-int. Too basic to couple to main app" [x] (if (string? x) (js/parseInt x) x)) (defn safe-parse-int - "Copy of frontend.util/safe-parse-int. Don't want to couple to main app too much" + "Copy of frontend.util/safe-parse-int. Too basic to couple to main app" [x] (let [result (parse-int x)] (if (js/isNaN result) @@ -94,3 +94,64 @@ (-> json-string (js/JSON.parse) (js->clj :keywordize-keys true))) + +;; TODO: Use update-keys once its available in cljs and nbb +(defn map-keys + "Maps function `f` over the keys of map `m` to produce a new map." + [f m] + (reduce-kv + (fn [m_ k v] + (assoc m_ (f k) v)) {} m)) + +(defn zero-pad + "Copy of frontend.util/zero-pad. Too basic to couple to main app" + [n] + (if (< n 10) + (str "0" n) + (str n))) + +(defn get-file-ext + "Copy of frontend.util/get-file-ext. Too basic to couple to main app" + [file] + (and + (string? file) + (string/includes? file ".") + (some-> (last (string/split file #"\.")) string/lower-case))) + +(defn remove-boundary-slashes + [s] + (when (string? s) + (let [s (if (= \/ (first s)) + (subs s 1) + s)] + (if (= \/ (last s)) + (subs s 0 (dec (count s))) + s)))) + +(defn split-namespace-pages + [title] + (let [parts (string/split title "/")] + (loop [others (rest parts) + result [(first parts)]] + (if (seq others) + (let [prev (last result)] + (recur (rest others) + (conj result (str prev "/" (first others))))) + result)))) + +(defn page-name-sanity + "Sanitize the page-name." + ([page-name] + (page-name-sanity page-name false)) + ([page-name replace-slash?] + (let [page (some-> page-name + (remove-boundary-slashes) + (path-normalize))] + (if replace-slash? + (string/replace page #"/" "%2A") + page)))) + +(defn page-name-sanity-lc + "Sanitize the query string for a page name (mandate for :block/name)" + [s] + (page-name-sanity (string/lower-case s))) diff --git a/src/test/frontend/test/docs_graph_helper.cljs b/src/test/frontend/test/docs_graph_helper.cljs index 25f64eca638..ec98c6c039a 100644 --- a/src/test/frontend/test/docs_graph_helper.cljs +++ b/src/test/frontend/test/docs_graph_helper.cljs @@ -5,7 +5,7 @@ [clojure.string :as string])) -(defn- slurp +(defn slurp "Like clojure.core/slurp" [file] (str (fs/readFileSync file))) diff --git a/src/test/frontend/util/property_test.cljs b/src/test/frontend/util/property_test.cljs index 036745dfc62..3bffc2960fb 100644 --- a/src/test/frontend/util/property_test.cljs +++ b/src/test/frontend/util/property_test.cljs @@ -146,29 +146,6 @@ SCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:END:\nworld\n" "c" "d") (property/insert-properties :markdown "" {:foo "\"bar, baz\""}) "foo:: \"bar, baz\"")) -(deftest test->new-properties - (are [x y] (= (property/->new-properties x) y) - ":PROPERTIES:\n:foo: bar\n:END:" - "foo:: bar" - - "hello\n:PROPERTIES:\n:foo: bar\n:END:" - "hello\nfoo:: bar" - - "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:" - "hello\nfoo:: bar\nnice:: bingo" - - "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:" - "hello\nfoo:: bar\nnice:: bingo" - - "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:\nnice" - "hello\nfoo:: bar\nnice:: bingo\nnice" - - "hello\n:PROPERTIES:\n:foo: bar\n:nice:\n:END:\nnice" - "hello\nfoo:: bar\nnice:: \nnice" - - "hello\n:PROPERTIES:\n:foo: bar\n:nice\n:END:\nnice" - "hello\nfoo:: bar\n:nice\nnice")) - (deftest test-build-properties-str (are [x y] (= (property/build-properties-str :mardown x) y) {:title "a"} diff --git a/src/test/logseq/graph_parser/nbb_test_runner.cljs b/src/test/logseq/graph_parser/nbb_test_runner.cljs index c9dc7c4d154..31a4f94837a 100644 --- a/src/test/logseq/graph_parser/nbb_test_runner.cljs +++ b/src/test/logseq/graph_parser/nbb_test_runner.cljs @@ -4,7 +4,8 @@ [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.text :as text] [logseq.graph-parser.text-test] - [logseq.graph-parser.mldoc-test])) + [logseq.graph-parser.mldoc-test] + [logseq.graph-parser.property-test])) (defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m] (when-not (cljs.test/successful? m) @@ -17,5 +18,5 @@ #_:clj-kondo/ignore (alter-var-root #'gp-mldoc/parse-property (constantly text/parse-property)) (t/run-tests 'logseq.graph-parser.mldoc-test - ;; TODO: Enable when https://github.com/babashka/nbb/issues/187 works - 'logseq.graph-parser.text-test)) + 'logseq.graph-parser.text-test + 'logseq.graph-parser.property-test)) diff --git a/src/test/logseq/graph_parser/property_test.cljs b/src/test/logseq/graph_parser/property_test.cljs new file mode 100644 index 00000000000..4630d636bdc --- /dev/null +++ b/src/test/logseq/graph_parser/property_test.cljs @@ -0,0 +1,26 @@ +(ns logseq.graph-parser.property-test + (:require [cljs.test :refer [are deftest]] + [logseq.graph-parser.property :as gp-property])) + +(deftest test->new-properties + (are [x y] (= (gp-property/->new-properties x) y) + ":PROPERTIES:\n:foo: bar\n:END:" + "foo:: bar" + + "hello\n:PROPERTIES:\n:foo: bar\n:END:" + "hello\nfoo:: bar" + + "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:" + "hello\nfoo:: bar\nnice:: bingo" + + "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:" + "hello\nfoo:: bar\nnice:: bingo" + + "hello\n:PROPERTIES:\n:foo: bar\n:nice: bingo\n:END:\nnice" + "hello\nfoo:: bar\nnice:: bingo\nnice" + + "hello\n:PROPERTIES:\n:foo: bar\n:nice:\n:END:\nnice" + "hello\nfoo:: bar\nnice:: \nnice" + + "hello\n:PROPERTIES:\n:foo: bar\n:nice\n:END:\nnice" + "hello\nfoo:: bar\n:nice\nnice")) From 2c725ca5af5c12a59c138daaab1464ade3e8d528 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 12 May 2022 10:47:49 -0400 Subject: [PATCH 39/58] Remove unused test namespaces frontend.react was only used in a test and never used elsewhere --- src/test/frontend/db/conn.clj | 3 - src/test/frontend/handler/block_test.cljs | 9 --- src/test/frontend/react.cljc | 67 ----------------------- src/test/frontend/react_test.cljs | 64 ---------------------- 4 files changed, 143 deletions(-) delete mode 100644 src/test/frontend/db/conn.clj delete mode 100644 src/test/frontend/handler/block_test.cljs delete mode 100644 src/test/frontend/react.cljc delete mode 100644 src/test/frontend/react_test.cljs diff --git a/src/test/frontend/db/conn.clj b/src/test/frontend/db/conn.clj deleted file mode 100644 index 1285eaabbc5..00000000000 --- a/src/test/frontend/db/conn.clj +++ /dev/null @@ -1,3 +0,0 @@ -(ns frontend.db.conn) - - diff --git a/src/test/frontend/handler/block_test.cljs b/src/test/frontend/handler/block_test.cljs deleted file mode 100644 index 5d0e7d4fccb..00000000000 --- a/src/test/frontend/handler/block_test.cljs +++ /dev/null @@ -1,9 +0,0 @@ -(ns frontend.handler.block-test) - - - -(comment - (defn clip-block [x] - (map #(select-keys % [:block/parent :block/left :block/pre-block? :block/uuid :block/level - :db/id]) - x))) diff --git a/src/test/frontend/react.cljc b/src/test/frontend/react.cljc deleted file mode 100644 index ed8e298162d..00000000000 --- a/src/test/frontend/react.cljc +++ /dev/null @@ -1,67 +0,0 @@ -(ns frontend.react - "To facilitate testing, imitate the behavior of react. - Note: don't run component parallel" - #?(:cljs (:require-macros [frontend.react]))) - -#_{:component-key {:result nil - :watches [] - :root-info nil}} -(def react-components (atom {})) -(def ^:dynamic *with-key* nil) -(def ^:dynamic *comp-key* nil) -(def ^:dynamic *root-info* nil) - -(defn react - [react-ref] - (let [comp-key *comp-key* - component (get @react-components comp-key)] - (cond - (some? component) - (do - (when-not ((:watches component) react-ref) - (let [new-component (update component :watches conj react-ref)] - (swap! react-components assoc comp-key new-component) - (add-watch react-ref comp-key - (fn [_key _atom old-state new-state] - (when-not (= old-state new-state) - (let [root-info (get-in @react-components [comp-key :root-info]) - {:keys [f comp-key]} root-info] - (binding [*with-key* comp-key - *root-info* root-info] - (let [component (get @react-components comp-key)] - (reset! (:result component) (f)))))))))) - @react-ref) - - ;; Sometime react is not used in component by accident, return the val. - :else - @react-ref))) - -(defn set-comp-and-calc-result - [f] - (let [{result :result :as component} (get @react-components *comp-key*)] - (if component - (do (reset! result (f)) result) - (let [result (atom nil)] - (binding [*root-info* (if *root-info* *root-info* {:f f :comp-key *comp-key*})] - (let [component {:result result - :watches #{} - :root-info *root-info*}] - (swap! react-components assoc *comp-key* component)) - (reset! result (f)) - result))))) - -#?(:clj (defmacro defc - [sym args & body] - `(defn ~sym ~args - (let [f# (fn [] - (binding [*comp-key* *with-key* - ;; inner component should specify own *with-key* - *with-key* nil] - ~@body))] - (binding [*comp-key* *with-key*] - (set-comp-and-calc-result f#)))))) - -#?(:clj (defmacro with-key - [key & body] - `(binding [*with-key* ~key] - ~@body))) diff --git a/src/test/frontend/react_test.cljs b/src/test/frontend/react_test.cljs deleted file mode 100644 index 4419632ae07..00000000000 --- a/src/test/frontend/react_test.cljs +++ /dev/null @@ -1,64 +0,0 @@ -(ns frontend.react-test - ;; namespace local config for r/defc tests - {:clj-kondo/config {:linters {:inline-def {:level :off}}}} - (:require [frontend.react :as r] - [cljs.test :refer [deftest is use-fixtures]] - [frontend.test.fixtures :as fixtures])) - -(use-fixtures :each - fixtures/react-components) - -(deftest simple-react-test - (let [react-ref (atom 1)] - - (r/defc simple-component - [] - (+ 2 (r/react react-ref))) - - (let [result (r/with-key 1 (simple-component))] - - (is (= 3 @result)) - (reset! react-ref 2) - (is (= 4 @result))))) - -(deftest nest-component-test - (let [a (atom 1) - b (atom 2)] - - (r/defc inner - [] - - (let [r (r/react b)] - r)) - - (r/defc out - [] - (let [out (r/react a) - inner-result (r/with-key "1" (inner))] - (+ out @inner-result))) - - (let [out-result (r/with-key "2" (out))] - (is (= 3 @out-result)) - (reset! b 4) - (is (= 5 @out-result))))) - -#_(deftest defc-params-test - (let [a (atom 1) - b (atom 2)] - - (r/defc inner-1 - [c] - (+ c (r/react b))) - - (r/defc out-1 - [] - (let [out (r/react a) - inner-result (r/with-key 1 (inner-1 5))] - (+ out @inner-result))) - - (let [out-result (r/with-key 2 (out-1))] - (is (= 8 @out-result)) - - (reset! b 4) - - (is (= 10 @out-result))))) From b1e226745d1d54033b9a96f06b34eb5681d1b756 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 12 May 2022 16:58:26 -0400 Subject: [PATCH 40/58] Finish migration block ns to nbb --- .clj-kondo/config.edn | 3 +- src/main/frontend/components/journal.cljs | 3 +- src/main/frontend/components/page.cljs | 4 +- src/main/frontend/date.cljs | 46 ++---- src/main/frontend/format/block.cljs | 19 ++- src/main/frontend/handler/editor.cljs | 10 +- src/main/frontend/handler/external.cljs | 7 +- src/main/frontend/handler/extract.cljs | 80 +++++----- src/main/frontend/handler/page.cljs | 7 +- src/main/frontend/util.cljc | 5 - .../graph_parser/{block.cljs => block.cljc} | 151 +++++++++--------- .../logseq/graph_parser/date_time_util.cljs | 44 ++++- src/main/logseq/graph_parser/util.cljs | 6 + src/test/logseq/graph_parser/block_test.cljs | 4 +- .../logseq/graph_parser/nbb_test_runner.cljs | 4 +- 15 files changed, 218 insertions(+), 175 deletions(-) rename src/main/logseq/graph_parser/{block.cljs => block.cljc} (84%) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 4a91e0a99d9..ca3f6d717ee 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -28,7 +28,8 @@ logseq.graph-parser.mldoc gp-mldoc logseq.graph-parser.util gp-util logseq.graph-parser.property gp-property - logseq.graph-parser.config gp-config}}} + logseq.graph-parser.config gp-config + logseq.graph-parser.date-time-util date-time-util}}} :hooks {:analyze-call {rum.core/defc hooks.rum/defc rum.core/defcs hooks.rum/defcs}} diff --git a/src/main/frontend/components/journal.cljs b/src/main/frontend/components/journal.cljs index 0c6b50c2392..790269a5543 100644 --- a/src/main/frontend/components/journal.cljs +++ b/src/main/frontend/components/journal.cljs @@ -9,6 +9,7 @@ [frontend.handler.page :as page-handler] [frontend.state :as state] [logseq.graph-parser.text :as text] + [logseq.graph-parser.util :as gp-util] [frontend.ui :as ui] [frontend.util :as util] [goog.object :as gobj] @@ -51,7 +52,7 @@ :page)) (.preventDefault e)))} [:h1.title - (util/capitalize-all title)]] + (gp-util/capitalize-all title)]] (blocks-cp repo page format) diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index a3b177268ef..2e2171ea113 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -34,7 +34,7 @@ [medley.core :as medley] [rum.core :as rum] [logseq.graph-parser.util :as gp-util] - [logseq.graph-parser.block :as gp-block] + [frontend.format.block :as block] [frontend.mobile.util :as mobile-util])) (defn- get-page-name @@ -332,7 +332,7 @@ (db/entity repo)) (do (when-not (db/entity repo [:block/name page-name]) - (let [m (gp-block/page-name->map path-page-name true)] + (let [m (block/page-name->map path-page-name true)] (db/transact! repo [m]))) (db/pull [:block/name page-name]))) {:keys [icon]} (:block/properties page) diff --git a/src/main/frontend/date.cljs b/src/main/frontend/date.cljs index 801ed9bf857..182222233b2 100644 --- a/src/main/frontend/date.cljs +++ b/src/main/frontend/date.cljs @@ -5,9 +5,10 @@ [cljs-time.core :as t] [cljs-time.format :as tf] [cljs-time.local :as tl] - [clojure.string :as string] [frontend.state :as state] [frontend.util :as util] + [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.date-time-util :as date-time-util] [goog.object :as gobj] [lambdaisland.glogi :as log])) @@ -16,11 +17,6 @@ (when (string? s) ((gobj/get chrono "parseDate") s))) -(defn format - [date] - (when-let [formatter-string (state/get-date-formatter)] - (tf/unparse (tf/formatter formatter-string) date))) - (def custom-formatter (tf/formatter "yyyy-MM-dd'T'HH:mm:ssZZ")) (defn journal-title-formatters @@ -55,13 +51,6 @@ "yyyy年MM月dd日"} (state/get-date-formatter))) -;; (tf/parse (tf/formatter "dd.MM.yyyy") "2021Q4") => 20040120T000000 -(defn safe-journal-title-formatters - [] - (->> [(state/get-date-formatter) "yyyy-MM-dd" "yyyy_MM_dd"] - (remove string/blank?) - distinct)) - (defn get-date-time-string ([] (get-date-time-string (t/now))) @@ -115,7 +104,7 @@ ([] (journal-name (tl/local-now))) ([date] - (format date))) + (date-time-util/format date (state/get-date-formatter)))) (defn journal-name-s [s] (try @@ -183,34 +172,19 @@ (defn valid-journal-title? [title] (and title - (valid? (util/capitalize-all title)))) + (valid? (gp-util/capitalize-all title)))) (defn journal-title-> ([journal-title then-fn] - (journal-title-> journal-title then-fn (safe-journal-title-formatters))) + (journal-title-> journal-title then-fn (date-time-util/safe-journal-title-formatters (state/get-date-formatter)))) ([journal-title then-fn formatters] - (when-not (string/blank? journal-title) - (when-let [time (->> (map - (fn [formatter] - (try - (tf/parse (tf/formatter formatter) (util/capitalize-all journal-title)) - (catch js/Error _e - nil))) - formatters) - (filter some?) - first)] - (then-fn time))))) + (date-time-util/journal-title-> journal-title then-fn formatters))) (defn journal-title->int [journal-title] - (when journal-title - (let [journal-title (util/capitalize-all journal-title)] - (journal-title-> journal-title #(util/parse-int (tf/unparse (tf/formatter "yyyyMMdd") %)))))) - -(defn int->journal-title - [day] - (when day - (format (tf/parse (tf/formatter "yyyyMMdd") (str day))))) + (date-time-util/journal-title->int + journal-title + (date-time-util/safe-journal-title-formatters (state/get-date-formatter)))) (defn journal-day->ts [day] @@ -240,7 +214,7 @@ (defn journal-title->custom-format [journal-title] - (journal-title-> journal-title format)) + (journal-title-> journal-title #(date-time-util/format % (state/get-date-formatter)))) (defn int->local-time-2 [n] diff --git a/src/main/frontend/format/block.cljs b/src/main/frontend/format/block.cljs index ab20658bf11..4d2ff354dd9 100644 --- a/src/main/frontend/format/block.cljs +++ b/src/main/frontend/format/block.cljs @@ -9,6 +9,23 @@ [logseq.graph-parser.property :as gp-property] [logseq.graph-parser.mldoc :as gp-mldoc])) +(defn extract-blocks + "Wrapper around logseq.graph-parser.block/extract-blocks that adds in system state" + [blocks content with-id? format] + (gp-block/extract-blocks blocks content with-id? format + {:user-config (state/get-config) + :block-pattern (config/get-block-pattern format) + :supported-formats (config/supported-formats) + :db (db/get-db (state/get-current-repo)) + :date-formatter (state/get-date-formatter)})) + +(defn page-name->map + "Wrapper around logseq.graph-parser.block/page-name->map that adds in db" + ([original-page-name with-id?] + (page-name->map original-page-name with-id? true)) + ([original-page-name with-id? with-timestamp?] + (gp-block/page-name->map original-page-name with-id? (db/get-db (state/get-current-repo)) with-timestamp? (state/get-date-formatter)))) + (defn with-parent-and-left [page-id blocks] (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks) @@ -87,7 +104,7 @@ (when-not (string/blank? content) (let [block (dissoc block :block/pre-block?) ast (format/to-edn content format nil) - blocks (gp-block/extract-blocks ast content with-id? format) + blocks (extract-blocks ast content with-id? format) new-block (first blocks) parent-refs (->> (db/get-block-parent (state/get-current-repo) uuid) :block/path-refs diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 341d7372880..de151363bf6 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -266,7 +266,7 @@ (if (and (:block/pre-block? block) (seq (:block/properties block))) (let [page-properties (:block/properties block) - str->page (fn [n] (gp-block/page-name->map n true)) + str->page (fn [n] (block/page-name->map n true)) refs (->> page-properties (filter (fn [[_ v]] (coll? v))) (vals) @@ -670,7 +670,9 @@ (defn properties-block [properties format page] (let [content (property/insert-properties format "" properties) - refs (gp-block/get-page-refs-from-properties properties)] + refs (gp-block/get-page-refs-from-properties properties + (db/get-db (state/get-current-repo)) + (state/get-date-formatter))] {:block/pre-block? true :block/uuid (db/new-block-id) :block/properties properties @@ -1946,7 +1948,7 @@ content* (str (if (= :markdown format) "- " "* ") (property/insert-properties format content props)) ast (mldoc/->edn content* (gp-mldoc/default-config format)) - blocks (gp-block/extract-blocks ast content* true format) + blocks (block/extract-blocks ast content* true format) fst-block (first blocks)] (assert fst-block "fst-block shouldn't be nil") (assoc fst-block :block/level (:block/level block))))))) @@ -2853,7 +2855,7 @@ [format text] (when-let [editing-block (state/get-edit-block)] (let [page-id (:db/id (:block/page editing-block)) - blocks (gp-block/extract-blocks + blocks (block/extract-blocks (mldoc/->edn text (gp-mldoc/default-config format)) text true format) blocks' (block/with-parent-and-left page-id blocks)] (paste-blocks blocks' {})))) diff --git a/src/main/frontend/handler/external.cljs b/src/main/frontend/handler/external.cljs index 3436ef5b115..ac17adcc000 100644 --- a/src/main/frontend/handler/external.cljs +++ b/src/main/frontend/handler/external.cljs @@ -8,8 +8,9 @@ [clojure.string :as string] [frontend.db :as db] [frontend.format.mldoc :as mldoc] - [logseq.graph-parser.block :as gp-block] + [frontend.format.block :as block] [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.date-time-util :as date-time-util] [frontend.handler.page :as page] [frontend.handler.editor :as editor] [frontend.util :as util])) @@ -50,7 +51,7 @@ (map (fn [title] (let [day (date/journal-title->int title) - page-name (util/page-name-sanity-lc (date/int->journal-title day))] + page-name (util/page-name-sanity-lc (date-time-util/int->journal-title day (state/get-date-formatter)))] {:block/name page-name :block/journal? true :block/journal-day day})) @@ -74,7 +75,7 @@ (when-let [repo (state/get-current-repo)] (let [[headers parsed-blocks] (mldoc/opml->edn data) parsed-blocks (->> - (gp-block/extract-blocks parsed-blocks "" true :markdown) + (block/extract-blocks parsed-blocks "" true :markdown) (mapv editor/wrap-parse-block)) page-name (:title headers)] (when (not (db/page-exists? page-name)) diff --git a/src/main/frontend/handler/extract.cljs b/src/main/frontend/handler/extract.cljs index 2c463ed944a..1feb5f98ba9 100644 --- a/src/main/frontend/handler/extract.cljs +++ b/src/main/frontend/handler/extract.cljs @@ -48,8 +48,8 @@ [repo-url format ast properties file content] (try (let [page (get-page-name file ast) - [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page) - blocks (->> (gp-block/extract-blocks ast content false format) + [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page (state/get-date-formatter)) + blocks (->> (block/extract-blocks ast content false format) (block/with-parent-and-left {:block/name page-name})) ref-pages (atom #{}) ref-tags (atom #{}) @@ -66,7 +66,7 @@ :block/page [:block/name page-name] :block/refs block-ref-pages :block/path-refs block-path-ref-pages)))) - blocks) + blocks) page-entity (let [alias (:alias properties) alias (if (string? alias) [alias] alias) aliases (and alias @@ -75,56 +75,56 @@ alias))) aliases (->> (map - (fn [alias] - (let [page-name (util/page-name-sanity-lc alias) - aliases (distinct - (conj - (remove #{alias} aliases) - page)) - aliases (when (seq aliases) - (map - (fn [alias] - {:block/name (util/page-name-sanity-lc alias)}) - aliases))] - (if (seq aliases) - {:block/name page-name - :block/alias aliases} - {:block/name page-name}))) - aliases) + (fn [alias] + (let [page-name (util/page-name-sanity-lc alias) + aliases (distinct + (conj + (remove #{alias} aliases) + page)) + aliases (when (seq aliases) + (map + (fn [alias] + {:block/name (util/page-name-sanity-lc alias)}) + aliases))] + (if (seq aliases) + {:block/name page-name + :block/alias aliases} + {:block/name page-name}))) + aliases) (remove nil?))] (cond-> - (gp-util/remove-nils - (assoc - (gp-block/page-name->map page false) - :block/file {:file/path (gp-util/path-normalize file)})) - (seq properties) - (assoc :block/properties properties) + (gp-util/remove-nils + (assoc + (block/page-name->map page false) + :block/file {:file/path (gp-util/path-normalize file)})) + (seq properties) + (assoc :block/properties properties) - (seq aliases) - (assoc :block/alias aliases) + (seq aliases) + (assoc :block/alias aliases) - (:tags properties) - (assoc :block/tags (let [tags (:tags properties) - tags (if (string? tags) [tags] tags) - tags (remove string/blank? tags)] - (swap! ref-tags set/union (set tags)) - (map (fn [tag] {:block/name (util/page-name-sanity-lc tag) - :block/original-name tag}) - tags))))) + (:tags properties) + (assoc :block/tags (let [tags (:tags properties) + tags (if (string? tags) [tags] tags) + tags (remove string/blank? tags)] + (swap! ref-tags set/union (set tags)) + (map (fn [tag] {:block/name (util/page-name-sanity-lc tag) + :block/original-name tag}) + tags))))) namespace-pages (let [page (:block/original-name page-entity)] (when (text/namespace-page? page) (->> (gp-util/split-namespace-pages page) (map (fn [page] - (-> (gp-block/page-name->map page true) + (-> (block/page-name->map page true) (assoc :block/format format))))))) pages (->> (concat [page-entity] @ref-pages (map - (fn [page] - {:block/original-name page - :block/name (util/page-name-sanity-lc page)}) - @ref-tags) + (fn [page] + {:block/original-name page + :block/name (util/page-name-sanity-lc page)}) + @ref-tags) namespace-pages) ;; remove block references (remove vector?) diff --git a/src/main/frontend/handler/page.cljs b/src/main/frontend/handler/page.cljs index 9819469544b..86b98cb7734 100644 --- a/src/main/frontend/handler/page.cljs +++ b/src/main/frontend/handler/page.cljs @@ -35,6 +35,7 @@ [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.config :as gp-config] [logseq.graph-parser.block :as gp-block] + [frontend.format.block :as block] [goog.functions :refer [debounce]])) (defn- get-directory @@ -72,7 +73,9 @@ (let [p (common-handler/get-page-default-properties title) ps (merge p properties) content (page-property/insert-properties format "" ps) - refs (gp-block/get-page-refs-from-properties properties)] + refs (gp-block/get-page-refs-from-properties properties + (db/get-db (state/get-current-repo)) + (state/get-date-formatter))] {:block/uuid (db/new-block-id) :block/properties ps :block/properties-order (keys ps) @@ -127,7 +130,7 @@ [title]) format (or format (state/get-preferred-format)) pages (map (fn [page] - (-> (gp-block/page-name->map page true) + (-> (block/page-name->map page true) (assoc :block/format format))) pages) txs (->> pages diff --git a/src/main/frontend/util.cljc b/src/main/frontend/util.cljc index eab914c4643..5ec8248d0c3 100644 --- a/src/main/frontend/util.cljc +++ b/src/main/frontend/util.cljc @@ -713,11 +713,6 @@ (defn drop-nth [n coll] (keep-indexed #(when (not= %1 n) %2) coll)) -(defn capitalize-all [s] - (some->> (string/split s #" ") - (map string/capitalize) - (string/join " "))) - #?(:cljs (defn react [ref] diff --git a/src/main/logseq/graph_parser/block.cljs b/src/main/logseq/graph_parser/block.cljc similarity index 84% rename from src/main/logseq/graph_parser/block.cljs rename to src/main/logseq/graph_parser/block.cljc index e718c22d7e9..c02cda92c87 100644 --- a/src/main/logseq/graph_parser/block.cljs +++ b/src/main/logseq/graph_parser/block.cljc @@ -1,12 +1,11 @@ -(ns logseq.graph-parser.block +(ns ^:nbb-compatible logseq.graph-parser.block + ;; Disable clj linters since we don't support clj + #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off} + :unresolved-symbol {:level :off}}}}) "Block related code needed for graph-parser" (:require [clojure.string :as string] [clojure.walk :as walk] - [frontend.config :as config] - [frontend.date :as date] - [frontend.db :as db] [datascript.core :as d] - [frontend.state :as state] [logseq.graph-parser.text :as text] [logseq.graph-parser.utf8 :as utf8] [logseq.graph-parser.property :as gp-property] @@ -14,7 +13,8 @@ [logseq.graph-parser.config :as gp-config] [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.date-time-util :as date-time-util] - [lambdaisland.glogi :as log])) + #?(:org.babashka/nbb [logseq.graph-parser.log :as log] + :default [lambdaisland.glogi :as log]))) (defn heading-block? [block] @@ -36,8 +36,8 @@ ""))) (string/join)))) -(defn get-page-reference - [block] +(defn- get-page-reference + [block supported-formats] (let [page (cond (and (vector? block) (= "Link" (first block))) (let [typ (first (:url (second block))) @@ -65,7 +65,7 @@ (not (string/starts-with? value "file:")) (not (gp-config/local-asset? value)) (or (= ext :excalidraw) - (not (contains? (config/supported-formats) ext)))) + (not (contains? supported-formats ext)))) value))) (and @@ -147,7 +147,7 @@ ;; TODO: we should move this to mldoc (defn extract-properties - [format properties] + [format properties user-config] (when (seq properties) (let [properties (seq properties) page-refs (->> @@ -177,7 +177,7 @@ (remove string/blank? v) (if (string/blank? v) nil - (text/parse-property format k v (state/get-config)))) + (text/parse-property format k v user-config))) k (keyword k) v (if (and (string? v) @@ -226,12 +226,12 @@ (defn convert-page-if-journal "Convert journal file name to user' custom date format" - [original-page-name] + [original-page-name date-formatter] (when original-page-name (let [page-name (gp-util/page-name-sanity-lc original-page-name) - day (date/journal-title->int page-name)] + day (date-time-util/journal-title->int page-name (date-time-util/safe-journal-title-formatters date-formatter))] (if day - (let [original-page-name (date/int->journal-title day)] + (let [original-page-name (date-time-util/int->journal-title day date-formatter)] [original-page-name (gp-util/page-name-sanity-lc original-page-name) day]) [original-page-name page-name day])))) @@ -241,48 +241,46 @@ with-timestamp?: assign timestampes to the map structure. Useful when creating new pages from references or namespaces, as there's no chance to introduce timestamps via editing in page" - ([original-page-name with-id?] - (page-name->map original-page-name with-id? true)) - ([original-page-name with-id? with-timestamp?] - (cond - (and original-page-name (string? original-page-name)) - (let [original-page-name (gp-util/remove-boundary-slashes original-page-name) - [original-page-name page-name journal-day] (convert-page-if-journal original-page-name) - namespace? (and (not (boolean (text/get-nested-page-name original-page-name))) - (text/namespace-page? original-page-name)) - page-entity (db/entity [:block/name page-name]) - original-page-name (or (:block/original-name page-entity) original-page-name)] - (merge - {:block/name page-name - :block/original-name original-page-name} - (when with-id? - (if page-entity - {:block/uuid (:block/uuid page-entity)} - {:block/uuid (d/squuid)})) - (when namespace? - (let [namespace (first (gp-util/split-last "/" original-page-name))] - (when-not (string/blank? namespace) - {:block/namespace {:block/name (gp-util/page-name-sanity-lc namespace)}}))) - (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity - (let [current-ms (date-time-util/time-ms)] - {:block/created-at current-ms - :block/updated-at current-ms})) - (if journal-day - {:block/journal? true - :block/journal-day journal-day} - {:block/journal? false}))) - - (and (map? original-page-name) (:block/uuid original-page-name)) - original-page-name - - (and (map? original-page-name) with-id?) - (assoc original-page-name :block/uuid (d/squuid)) - - :else - nil))) + [original-page-name with-id? db with-timestamp? date-formatter] + (cond + (and original-page-name (string? original-page-name)) + (let [original-page-name (gp-util/remove-boundary-slashes original-page-name) + [original-page-name page-name journal-day] (convert-page-if-journal original-page-name date-formatter) + namespace? (and (not (boolean (text/get-nested-page-name original-page-name))) + (text/namespace-page? original-page-name)) + page-entity (some-> db (d/entity [:block/name page-name])) + original-page-name (or (:block/original-name page-entity) original-page-name)] + (merge + {:block/name page-name + :block/original-name original-page-name} + (when with-id? + (if page-entity + {:block/uuid (:block/uuid page-entity)} + {:block/uuid (d/squuid)})) + (when namespace? + (let [namespace (first (gp-util/split-last "/" original-page-name))] + (when-not (string/blank? namespace) + {:block/namespace {:block/name (gp-util/page-name-sanity-lc namespace)}}))) + (when (and with-timestamp? (not page-entity)) ;; Only assign timestamp on creating new entity + (let [current-ms (date-time-util/time-ms)] + {:block/created-at current-ms + :block/updated-at current-ms})) + (if journal-day + {:block/journal? true + :block/journal-day journal-day} + {:block/journal? false}))) + + (and (map? original-page-name) (:block/uuid original-page-name)) + original-page-name + + (and (map? original-page-name) with-id?) + (assoc original-page-name :block/uuid (d/squuid)) + + :else + nil)) (defn- with-page-refs - [{:keys [title body tags refs marker priority] :as block} with-id?] + [{:keys [title body tags refs marker priority] :as block} with-id? supported-formats db date-formatter] (let [refs (->> (concat tags refs [marker priority]) (remove string/blank?) (distinct)) @@ -293,7 +291,7 @@ (when-not (and (vector? form) (= (first form) "Custom") (= (second form) "query")) - (when-let [page (get-page-reference form)] + (when-let [page (get-page-reference form supported-formats)] (swap! refs conj page)) (when-let [tag (get-tag form)] (let [tag (text/page-ref-un-brackets! tag)] @@ -315,7 +313,7 @@ (distinct)) refs (->> (distinct (concat refs children-pages)) (remove nil?)) - refs (map (fn [ref] (page-name->map ref with-id?)) refs)] + refs (map (fn [ref] (page-name->map ref with-id? db true date-formatter)) refs)] (assoc block :refs refs)))) (defn- with-block-refs @@ -409,7 +407,7 @@ block)) (defn- get-block-content - [utf8-content block format meta] + [utf8-content block format meta block-pattern] (let [content (if-let [end-pos (:end_pos meta)] (utf8/substring utf8-content (:start_pos meta) @@ -417,7 +415,7 @@ (utf8/substring utf8-content (:start_pos meta))) content (when content - (let [content (text/remove-level-spaces content format (config/get-block-pattern format))] + (let [content (text/remove-level-spaces content format block-pattern)] (if (or (:pre-block? block) (= (:format block) :org)) content @@ -437,7 +435,7 @@ (d/squuid))) (defn get-page-refs-from-properties - [properties] + [properties db date-formatter] (let [page-refs (mapcat (fn [v] (cond (coll? v) v @@ -448,18 +446,18 @@ :else nil)) (vals properties)) page-refs (remove string/blank? page-refs)] - (map (fn [page] (page-name->map page true)) page-refs))) + (map (fn [page] (page-name->map page true db true date-formatter)) page-refs))) (defn- with-page-block-refs - [block with-id?] + [block with-id? supported-formats db date-formatter] (some-> block - (with-page-refs with-id?) + (with-page-refs with-id? supported-formats db date-formatter) with-block-refs block-tags->pages (update :refs (fn [col] (remove nil? col))))) (defn- with-pre-block-if-exists - [blocks body pre-block-properties encoded-content] + [blocks body pre-block-properties encoded-content {:keys [supported-formats db date-formatter]}] (let [first-block (first blocks) first-block-start-pos (get-in first-block [:block/meta :start_pos]) @@ -471,7 +469,7 @@ (let [content (utf8/substring encoded-content 0 first-block-start-pos) {:keys [properties properties-order]} pre-block-properties id (get-custom-id-or-new-id {:properties properties}) - property-refs (->> (get-page-refs-from-properties properties) + property-refs (->> (get-page-refs-from-properties properties db date-formatter) (map :block/original-name)) block {:uuid id :content content @@ -482,7 +480,7 @@ :pre-block? true :unordered true :body body} - block (with-page-block-refs block false)] + block (with-page-block-refs block false supported-formats db date-formatter)] (block-keywordize block)) (select-keys first-block [:block/format :block/page])) blocks) @@ -490,7 +488,7 @@ (with-path-refs blocks))) (defn- construct-block - [block properties timestamps body encoded-content format pos-meta with-id?] + [block properties timestamps body encoded-content format pos-meta with-id? {:keys [block-pattern supported-formats db date-formatter]}] (let [id (get-custom-id-or-new-id properties) ref-pages-in-properties (->> (:page-refs properties) (remove string/blank?)) @@ -518,12 +516,12 @@ (assoc block :collapsed? true) block) block (assoc block - :content (get-block-content encoded-content block format pos-meta)) + :content (get-block-content encoded-content block format pos-meta block-pattern)) block (if (seq timestamps) (merge block (timestamps->scheduled-and-deadline timestamps)) block) block (assoc block :body body) - block (with-page-block-refs block with-id?) + block (with-page-block-refs block with-id? supported-formats db date-formatter) {:keys [created-at updated-at]} (:properties properties) block (cond-> block (and created-at (integer? created-at)) @@ -539,8 +537,10 @@ `blocks`: mldoc ast. `content`: markdown or org-mode text. `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids. - `format`: content's format, it could be either :markdown or :org-mode." - [blocks content with-id? format] + `format`: content's format, it could be either :markdown or :org-mode. + `options`: Options supported are :user-config, :block-pattern :supported-formats, + :date-formatter and :db" + [blocks content with-id? format {:keys [user-config] :as options}] {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]} (try (let [encoded-content (utf8/encode content) @@ -564,11 +564,11 @@ (recur headings (rest blocks) timestamps' properties body)) (gp-property/properties-ast? block) - (let [properties (extract-properties format (second block))] + (let [properties (extract-properties format (second block) user-config)] (recur headings (rest blocks) timestamps properties body)) (heading-block? block) - (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id?)] + (let [block (construct-block block properties timestamps body encoded-content format pos-meta with-id? options)] (recur (conj headings block) (rest blocks) {} {} [])) :else @@ -577,8 +577,7 @@ sanity-blocks-data) body properties])) - result (with-pre-block-if-exists blocks body pre-block-properties encoded-content)] + result (with-pre-block-if-exists blocks body pre-block-properties encoded-content options)] (map #(dissoc % :block/meta) result)) - (catch js/Error e - (js/console.error "extract-blocks-failed") - (log/error :exception e)))) + (catch :default e + (log/error :extract-blocks-failure e)))) diff --git a/src/main/logseq/graph_parser/date_time_util.cljs b/src/main/logseq/graph_parser/date_time_util.cljs index 0bcbe18d395..26c98c8b340 100644 --- a/src/main/logseq/graph_parser/date_time_util.cljs +++ b/src/main/logseq/graph_parser/date_time_util.cljs @@ -1,9 +1,51 @@ (ns ^:nbb-compatible logseq.graph-parser.date-time-util "cljs-time util fns for graph-parser" (:require [cljs-time.coerce :as tc] - [cljs-time.core :as t])) + [cljs-time.core :as t] + [cljs-time.format :as tf] + [clojure.string :as string] + [logseq.graph-parser.util :as gp-util])) (defn time-ms "Copy of util/time-ms. Too basic to couple this to main app" [] (tc/to-long (t/now))) + +;; (tf/parse (tf/formatter "dd.MM.yyyy") "2021Q4") => 20040120T000000 +(defn safe-journal-title-formatters + [date-formatter] + (->> [date-formatter "MMM do, yyyy" "yyyy-MM-dd" "yyyy_MM_dd"] + (remove string/blank?) + distinct)) + +(defn journal-title-> + [journal-title then-fn formatters] + (when-not (string/blank? journal-title) + (when-let [time (->> (map + (fn [formatter] + (try + (tf/parse (tf/formatter formatter) (gp-util/capitalize-all journal-title)) + (catch js/Error _e + nil))) + formatters) + (filter some?) + first)] + (then-fn time)))) + +(defn journal-title->int + [journal-title formatters] + (when journal-title + (let [journal-title (gp-util/capitalize-all journal-title)] + (journal-title-> journal-title + #(gp-util/parse-int (tf/unparse (tf/formatter "yyyyMMdd") %)) + formatters)))) + +(defn format + [date date-formatter] + (when date-formatter + (tf/unparse (tf/formatter date-formatter) date))) + +(defn int->journal-title + [day date-formatter] + (when day + (format (tf/parse (tf/formatter "yyyyMMdd") (str day)) date-formatter))) diff --git a/src/main/logseq/graph_parser/util.cljs b/src/main/logseq/graph_parser/util.cljs index 821168a66a1..3baf3feca04 100644 --- a/src/main/logseq/graph_parser/util.cljs +++ b/src/main/logseq/graph_parser/util.cljs @@ -155,3 +155,9 @@ "Sanitize the query string for a page name (mandate for :block/name)" [s] (page-name-sanity (string/lower-case s))) + +(defn capitalize-all + [s] + (some->> (string/split s #" ") + (map string/capitalize) + (string/join " "))) diff --git a/src/test/logseq/graph_parser/block_test.cljs b/src/test/logseq/graph_parser/block_test.cljs index 35a82846393..c73fe11d27c 100644 --- a/src/test/logseq/graph_parser/block_test.cljs +++ b/src/test/logseq/graph_parser/block_test.cljs @@ -3,7 +3,7 @@ [cljs.test :refer [deftest are]])) (deftest test-extract-properties - (are [x y] (= (:properties (gp-block/extract-properties :markdown x)) y) + (are [x y] (= (:properties (gp-block/extract-properties :markdown x {})) y) [["year" "1000"]] {:year 1000} [["year" "\"1000\""]] {:year "\"1000\""} [["background-color" "#000000"]] {:background-color "#000000"} @@ -23,7 +23,7 @@ [["foo" "bar, [[baz, test]]"]] {:foo #{"bar" "baz, test"}} [["foo" "bar, [[baz, test, [[nested]]]]"]] {:foo #{"bar" "baz, test, [[nested]]"}}) - (are [x y] (= (vec (:page-refs (gp-block/extract-properties :markdown x))) y) + (are [x y] (= (vec (:page-refs (gp-block/extract-properties :markdown x {}))) y) [["year" "1000"]] [] [["year" "\"1000\""]] [] [["foo" "[[bar]] test"]] ["bar" "test"] diff --git a/src/test/logseq/graph_parser/nbb_test_runner.cljs b/src/test/logseq/graph_parser/nbb_test_runner.cljs index 31a4f94837a..85c1a6365e3 100644 --- a/src/test/logseq/graph_parser/nbb_test_runner.cljs +++ b/src/test/logseq/graph_parser/nbb_test_runner.cljs @@ -5,6 +5,7 @@ [logseq.graph-parser.text :as text] [logseq.graph-parser.text-test] [logseq.graph-parser.mldoc-test] + [logseq.graph-parser.block-test] [logseq.graph-parser.property-test])) (defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m] @@ -19,4 +20,5 @@ (alter-var-root #'gp-mldoc/parse-property (constantly text/parse-property)) (t/run-tests 'logseq.graph-parser.mldoc-test 'logseq.graph-parser.text-test - 'logseq.graph-parser.property-test)) + 'logseq.graph-parser.property-test + 'logseq.graph-parser.block-test)) From 8a8e3ee26c5d536f66338811ad40a21e04c80bee Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 13 May 2022 10:00:38 +0800 Subject: [PATCH 41/58] fix: cache visible component heights --- externs.js | 3 +++ src/main/frontend/ui.cljs | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/externs.js b/externs.js index 36017644f2d..3bf6ca9961a 100644 --- a/externs.js +++ b/externs.js @@ -127,6 +127,9 @@ dummy.getNodesObjects = function() {}; dummy.getEdgesObjects = function() {}; dummy.alphaTarget = function() {}; dummy.restart = function() {}; +dummy.observe = function() {}; +dummy.contentRect = function() {}; +dummy.height = function() {}; /** * @typedef {{ diff --git a/src/main/frontend/ui.cljs b/src/main/frontend/ui.cljs index f60219091fa..7de3e5c7f18 100644 --- a/src/main/frontend/ui.cljs +++ b/src/main/frontend/ui.cljs @@ -911,9 +911,23 @@ label-right]] (progress-bar width)]) -(rum/defc lazy-visible-inner - [visible? content-fn loading-label] - [:div.lazy-visibility +(rum/defcs lazy-visible-inner < + {:init (fn [state] + (assoc state + ::ref (atom nil) + ::height (atom 1))) + :did-mount (fn [state] + (let [observer (js/ResizeObserver. (fn [entries] + (let [entry (first entries) + *height (::height state) + height' (.-height (.-contentRect entry))] + (when (> height' @*height) + (reset! *height height')))))] + (.observe observer @(::ref state))) + state)} + [state visible? content-fn loading-label] + [:div.lazy-visibility {:ref #(reset! (::ref state) %) + :style {:min-height @(::height state)}} (if visible? (when (fn? content-fn) (content-fn)) (when loading-label [:span.text-sm.font-medium From 4d4ade5c96dde14e0d3a70650c9ef83a5756fec0 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 13 May 2022 10:15:37 +0800 Subject: [PATCH 42/58] enhance: add skeleton loading --- resources/css/common.css | 5 +++++ src/main/frontend/components/block.cljs | 6 ++---- src/main/frontend/components/journal.cljs | 4 ++-- src/main/frontend/components/page.cljs | 2 +- src/main/frontend/components/reference.cljs | 5 +---- src/main/frontend/ui.cljs | 20 ++++++++++++++------ 6 files changed, 25 insertions(+), 17 deletions(-) diff --git a/resources/css/common.css b/resources/css/common.css index 71433775125..77379276e2a 100644 --- a/resources/css/common.css +++ b/resources/css/common.css @@ -715,6 +715,11 @@ li p:last-child, background-color: var(--ls-primary-background-color, #fff); } +.bg-base-4 { + background-color: var(--ls-tertiary-background-color); +} + + .pre-white-space { white-space: pre; } diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 85c7fc14b6c..6dac4212ed4 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -286,7 +286,7 @@ (.share Share #js {:url url :title "Open PDF fils with your favorite app"}))))} title] - + :else [:a.asset-ref {:ref @src} title]))))) @@ -862,7 +862,7 @@ (mobile-util/is-native-platform?) (asset-link config label-text s metadata full_text))) - + :else (asset-reference config label s)))) @@ -2406,7 +2406,6 @@ ref-or-custom-query? (or ref? custom-query?)] (if (and ref-or-custom-query? (not (:ref-query-child? config))) (ui/lazy-visible - nil (fn [] (block-container-inner state repo config block)) nil) @@ -2737,7 +2736,6 @@ (ui/catch-error (ui/block-error "Query Error:" {:content (:query q)}) (ui/lazy-visible - "loading ..." (fn [] (custom-query* config q)) nil))) diff --git a/src/main/frontend/components/journal.cljs b/src/main/frontend/components/journal.cljs index 0c6b50c2392..4727bbcb301 100644 --- a/src/main/frontend/components/journal.cljs +++ b/src/main/frontend/components/journal.cljs @@ -60,12 +60,12 @@ (page/today-queries repo today? false) (rum/with-key - (reference/references title false) + (reference/references title) (str title "-refs"))])) (rum/defc journal-cp [journal] - (ui/lazy-visible nil (fn [] (journal-cp-inner journal)) nil)) + (ui/lazy-visible (fn [] (journal-cp-inner journal)) nil)) (rum/defc journals < rum/reactive [latest-journals] diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 9bc9f83234d..d8556299afa 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -394,7 +394,7 @@ ;; referenced blocks [:div {:key "page-references"} (rum/with-key - (reference/references route-page-name sidebar?) + (reference/references route-page-name) (str route-page-name "-refs"))] (when-not block? diff --git a/src/main/frontend/components/reference.cljs b/src/main/frontend/components/reference.cljs index 9e25e611d10..a053cdd776d 100644 --- a/src/main/frontend/components/reference.cljs +++ b/src/main/frontend/components/reference.cljs @@ -166,13 +166,10 @@ :title-trigger? true}))]])))) (rum/defc references - [page-name sidebar?] + [page-name] (ui/catch-error (ui/component-error "Linked References: Unexpected error") (ui/lazy-visible - (if (or sidebar? (gp-util/uuid-string? page-name)) - nil - "loading references...") (fn [] (references* page-name)) nil))) diff --git a/src/main/frontend/ui.cljs b/src/main/frontend/ui.cljs index 7de3e5c7f18..1e6e41ffc0f 100644 --- a/src/main/frontend/ui.cljs +++ b/src/main/frontend/ui.cljs @@ -915,7 +915,7 @@ {:init (fn [state] (assoc state ::ref (atom nil) - ::height (atom 1))) + ::height (atom 100))) :did-mount (fn [state] (let [observer (js/ResizeObserver. (fn [entries] (let [entry (first entries) @@ -925,17 +925,25 @@ (reset! *height height')))))] (.observe observer @(::ref state))) state)} - [state visible? content-fn loading-label] + [state visible? content-fn] [:div.lazy-visibility {:ref #(reset! (::ref state) %) :style {:min-height @(::height state)}} + (if visible? (when (fn? content-fn) (content-fn)) - (when loading-label [:span.text-sm.font-medium - loading-label]))]) + [:div.shadow.rounded-md.p-4.w-full.mx-auto + [:div.animate-pulse.flex.space-x-4 + [:div.flex-1.space-y-3.py-1 + [:div.h-2.bg-base-4.rounded] + [:div.space-y-3 + [:div.grid.grid-cols-3.gap-4 + [:div.h-2.bg-base-4.rounded.col-span-2] + [:div.h-2.bg-base-4.rounded.col-span-1]] + [:div.h-2.bg-base-4.rounded]]]]])]) (rum/defcs lazy-visible < (rum/local false ::visible?) - [state loading-label content-fn sensor-opts] + [state content-fn sensor-opts] (let [*visible? (::visible? state)] (visibility-sensor (merge @@ -944,4 +952,4 @@ :offset {:top -300 :bottom -300}} sensor-opts) - (lazy-visible-inner @*visible? content-fn loading-label)))) + (lazy-visible-inner @*visible? content-fn)))) From fb8cd6791733ddb2b82f38ae05c182f568ac6cc4 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 13 May 2022 10:50:47 +0800 Subject: [PATCH 43/58] fix: disable lazy loading for query blocks temporally --- src/main/frontend/components/block.cljs | 8 +++--- src/main/frontend/components/journal.cljs | 2 +- src/main/frontend/components/reference.cljs | 3 ++- src/main/frontend/ui.cljs | 30 ++++++++++++--------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 6dac4212ed4..feb0484807d 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -2404,11 +2404,12 @@ ref? (:ref? config) custom-query? (boolean (:custom-query? config)) ref-or-custom-query? (or ref? custom-query?)] - (if (and ref-or-custom-query? (not (:ref-query-child? config))) + (if (and ref? (not custom-query?) (not (:ref-query-child? config))) (ui/lazy-visible (fn [] (block-container-inner state repo config block)) - nil) + nil + false) (block-container-inner state repo config block)))) (defn divide-lists @@ -2737,7 +2738,8 @@ (ui/block-error "Query Error:" {:content (:query q)}) (ui/lazy-visible (fn [] (custom-query* config q)) - nil))) + nil + true))) (defn admonition [config type result] diff --git a/src/main/frontend/components/journal.cljs b/src/main/frontend/components/journal.cljs index 4727bbcb301..40aea4e72c4 100644 --- a/src/main/frontend/components/journal.cljs +++ b/src/main/frontend/components/journal.cljs @@ -65,7 +65,7 @@ (rum/defc journal-cp [journal] - (ui/lazy-visible (fn [] (journal-cp-inner journal)) nil)) + (ui/lazy-visible (fn [] (journal-cp-inner journal)) nil true)) (rum/defc journals < rum/reactive [latest-journals] diff --git a/src/main/frontend/components/reference.cljs b/src/main/frontend/components/reference.cljs index a053cdd776d..cb6dfd1e1d6 100644 --- a/src/main/frontend/components/reference.cljs +++ b/src/main/frontend/components/reference.cljs @@ -172,7 +172,8 @@ (ui/lazy-visible (fn [] (references* page-name)) - nil))) + nil + false))) (rum/defcs unlinked-references-aux < rum/reactive db-mixins/query diff --git a/src/main/frontend/ui.cljs b/src/main/frontend/ui.cljs index 1e6e41ffc0f..ebaa8b77dfd 100644 --- a/src/main/frontend/ui.cljs +++ b/src/main/frontend/ui.cljs @@ -915,23 +915,25 @@ {:init (fn [state] (assoc state ::ref (atom nil) - ::height (atom 100))) + ::height (atom 26))) :did-mount (fn [state] - (let [observer (js/ResizeObserver. (fn [entries] - (let [entry (first entries) - *height (::height state) - height' (.-height (.-contentRect entry))] - (when (> height' @*height) - (reset! *height height')))))] - (.observe observer @(::ref state))) + (when (last (:rum/args state)) + (let [observer (js/ResizeObserver. (fn [entries] + (let [entry (first entries) + *height (::height state) + height' (.-height (.-contentRect entry))] + (when (and (> height' @*height) + (not= height' 64)) + (reset! *height height')))))] + (.observe observer @(::ref state)))) state)} - [state visible? content-fn] + [state visible? content-fn reset-height?] [:div.lazy-visibility {:ref #(reset! (::ref state) %) :style {:min-height @(::height state)}} (if visible? (when (fn? content-fn) (content-fn)) - [:div.shadow.rounded-md.p-4.w-full.mx-auto + [:div.shadow.rounded-md.p-4.w-full.mx-auto {:style {:height 64}} [:div.animate-pulse.flex.space-x-4 [:div.flex-1.space-y-3.py-1 [:div.h-2.bg-base-4.rounded] @@ -943,13 +945,15 @@ (rum/defcs lazy-visible < (rum/local false ::visible?) - [state content-fn sensor-opts] + [state content-fn sensor-opts reset-height?] (let [*visible? (::visible? state)] (visibility-sensor (merge {:on-change #(reset! *visible? %) :partialVisibility true :offset {:top -300 - :bottom -300}} + :bottom -300} + :scrollCheck true + :scrollThrottle 1} sensor-opts) - (lazy-visible-inner @*visible? content-fn)))) + (lazy-visible-inner @*visible? content-fn reset-height?)))) From 8d400bcaf0916a2f58dcf6f69927a0a0d4326ba2 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 13 May 2022 11:02:29 +0800 Subject: [PATCH 44/58] fix: lint warnings --- src/main/frontend/components/block.cljs | 3 +-- src/main/frontend/ui.cljs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index feb0484807d..b04175b11f0 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -2402,8 +2402,7 @@ [state config block] (let [repo (state/get-current-repo) ref? (:ref? config) - custom-query? (boolean (:custom-query? config)) - ref-or-custom-query? (or ref? custom-query?)] + custom-query? (boolean (:custom-query? config))] (if (and ref? (not custom-query?) (not (:ref-query-child? config))) (ui/lazy-visible (fn [] diff --git a/src/main/frontend/ui.cljs b/src/main/frontend/ui.cljs index ebaa8b77dfd..6ba20aedcb7 100644 --- a/src/main/frontend/ui.cljs +++ b/src/main/frontend/ui.cljs @@ -927,7 +927,7 @@ (reset! *height height')))))] (.observe observer @(::ref state)))) state)} - [state visible? content-fn reset-height?] + [state visible? content-fn _reset-height?] [:div.lazy-visibility {:ref #(reset! (::ref state) %) :style {:min-height @(::height state)}} From 3150e06a1b5cb333827ba60cc9b1571ed145197c Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 13 May 2022 11:19:36 +0800 Subject: [PATCH 45/58] Disable lazy loading for journals on mobile --- src/main/frontend/components/journal.cljs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/frontend/components/journal.cljs b/src/main/frontend/components/journal.cljs index 40aea4e72c4..610a1922e44 100644 --- a/src/main/frontend/components/journal.cljs +++ b/src/main/frontend/components/journal.cljs @@ -13,7 +13,8 @@ [frontend.util :as util] [goog.object :as gobj] [reitit.frontend.easy :as rfe] - [rum.core :as rum])) + [rum.core :as rum] + [frontend.mobile.util :refer [is-native-platform?]])) (rum/defc blocks-cp < rum/reactive db-mixins/query {} @@ -65,7 +66,9 @@ (rum/defc journal-cp [journal] - (ui/lazy-visible (fn [] (journal-cp-inner journal)) nil true)) + (if (or (util/mobile?) (is-native-platform?)) + (journal-cp-inner journal) + (ui/lazy-visible (fn [] (journal-cp-inner journal)) nil true))) (rum/defc journals < rum/reactive [latest-journals] From 3cbd92a213bbd8e5ab4d52be255d4dfc7aac164e Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 13 May 2022 11:33:59 +0800 Subject: [PATCH 46/58] Disable lazy visibility completely on mobile --- src/main/frontend/components/journal.cljs | 7 ++---- src/main/frontend/ui.cljs | 27 +++++++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/frontend/components/journal.cljs b/src/main/frontend/components/journal.cljs index 610a1922e44..40aea4e72c4 100644 --- a/src/main/frontend/components/journal.cljs +++ b/src/main/frontend/components/journal.cljs @@ -13,8 +13,7 @@ [frontend.util :as util] [goog.object :as gobj] [reitit.frontend.easy :as rfe] - [rum.core :as rum] - [frontend.mobile.util :refer [is-native-platform?]])) + [rum.core :as rum])) (rum/defc blocks-cp < rum/reactive db-mixins/query {} @@ -66,9 +65,7 @@ (rum/defc journal-cp [journal] - (if (or (util/mobile?) (is-native-platform?)) - (journal-cp-inner journal) - (ui/lazy-visible (fn [] (journal-cp-inner journal)) nil true))) + (ui/lazy-visible (fn [] (journal-cp-inner journal)) nil true)) (rum/defc journals < rum/reactive [latest-journals] diff --git a/src/main/frontend/ui.cljs b/src/main/frontend/ui.cljs index 6ba20aedcb7..d66c91a1934 100644 --- a/src/main/frontend/ui.cljs +++ b/src/main/frontend/ui.cljs @@ -29,7 +29,8 @@ [rum.core :as rum] [frontend.db-mixins :as db-mixins] [frontend.mobile.util :as mobile-util] - [goog.functions :refer [debounce]])) + [goog.functions :refer [debounce]] + [frontend.mobile.util :refer [is-native-platform?]])) (defonce transition-group (r/adapt-class TransitionGroup)) (defonce css-transition (r/adapt-class CSSTransition)) @@ -946,14 +947,16 @@ (rum/defcs lazy-visible < (rum/local false ::visible?) [state content-fn sensor-opts reset-height?] - (let [*visible? (::visible? state)] - (visibility-sensor - (merge - {:on-change #(reset! *visible? %) - :partialVisibility true - :offset {:top -300 - :bottom -300} - :scrollCheck true - :scrollThrottle 1} - sensor-opts) - (lazy-visible-inner @*visible? content-fn reset-height?)))) + (if (or (util/mobile?) (is-native-platform?)) + (content-fn) + (let [*visible? (::visible? state)] + (visibility-sensor + (merge + {:on-change #(reset! *visible? %) + :partialVisibility true + :offset {:top -300 + :bottom -300} + :scrollCheck true + :scrollThrottle 1} + sensor-opts) + (lazy-visible-inner @*visible? content-fn reset-height?))))) From 9dd63cf7114aabc156a8f7dff9f551eb2c4e5f0f Mon Sep 17 00:00:00 2001 From: rcmerci Date: Thu, 12 May 2022 20:43:56 +0800 Subject: [PATCH 47/58] enhance(sync): auto start sync when switching graphs - auto start sync when switching graphs - add state ::starting --- src/main/frontend/fs/sync.cljs | 25 ++++++++++++++++++++----- src/main/frontend/handler/events.cljs | 5 ++--- src/main/frontend/state.cljs | 1 + 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/main/frontend/fs/sync.cljs b/src/main/frontend/fs/sync.cljs index 7f5a1222ced..065269c8f99 100644 --- a/src/main/frontend/fs/sync.cljs +++ b/src/main/frontend/fs/sync.cljs @@ -52,7 +52,12 @@ ;; and re-produce a new same-file-delete diff. ;;; ### specs -(s/def ::state #{::idle +(s/def ::state #{;; do following jobs when ::starting: + ;; - wait seconds for file-change-events from file-watcher + ;; - drop redundant file-change-events + ;; - setup states in `frontend.state` + ::starting + ::idle ;; sync local-changed files ::local->remote ;; sync remote latest-transactions @@ -847,7 +852,7 @@ "create a new sync-state" [] {:post [(s/valid? ::sync-state %)]} - {:state ::idle + {:state ::starting :current-local->remote-files #{} :current-remote->local-files #{} :queued-local->remote-files #{} @@ -1417,24 +1422,34 @@ ;; 1. if remote graph has been deleted, clear graphs-txid.edn ;; 2. if graphs-txid.edn's content isn't [user-uuid graph-uuid txid], clear it (if (not= 3 (count @graphs-txid)) - (clear-graphs-txid! repo) + (do (clear-graphs-txid! repo) + (state/set-file-sync-state nil)) (when (check-graph-belong-to-current-user current-user-uuid user-uuid) (if-not (local-sync-chan) + ;; update global state when *sync-state changes (add-watch *sync-state ::update-global-state (fn [_ _ _ n] (state/set-file-sync-state n))) (.start sm) - (state/set-file-sync-manager sm) + (offer! remote->local-sync-chan true) (offer! full-sync-chan true) diff --git a/src/main/frontend/handler/events.cljs b/src/main/frontend/handler/events.cljs index 50f16811684..f700dcb547d 100644 --- a/src/main/frontend/handler/events.cljs +++ b/src/main/frontend/handler/events.cljs @@ -59,8 +59,8 @@ (defn- file-sync-stop-when-switch-graph [] (p/do! (persist-var/load-vars) (sync/sync-stop) - ;; trigger rerender file-sync-header - (state/set-file-sync-state nil))) + (sync/sync-start) +)) (defn- graph-switch [graph] (state/set-current-repo! graph) @@ -98,7 +98,6 @@ (graph-switch graph)))) (defmethod handle :graph/switch [[_ graph]] - (file-sync-stop-when-switch-graph) (if (outliner-file/writes-finished?) (graph-switch-on-persisted graph) (notification/show! diff --git a/src/main/frontend/state.cljs b/src/main/frontend/state.cljs index 8f335fb2ed3..31b08ca9c7c 100644 --- a/src/main/frontend/state.cljs +++ b/src/main/frontend/state.cljs @@ -1668,6 +1668,7 @@ (defn get-file-sync-manager [] (:file-sync/sync-manager @state)) + (defn get-file-sync-state [] (:file-sync/sync-state @state)) From 3d725f8acb2d7bcf08b9f07fc79c79a5b76e64ba Mon Sep 17 00:00:00 2001 From: llcc Date: Fri, 13 May 2022 16:00:17 +0800 Subject: [PATCH 48/58] allow sharing office files into Logseq on mobile (#5292) * allow share office files into logseq on mobile --- src/main/frontend/components/block.cljs | 64 +++++++++++++------------ src/main/frontend/components/block.css | 13 +++++ src/main/frontend/config.cljs | 9 ++++ src/main/frontend/mobile/intent.cljs | 3 +- src/main/frontend/ui.cljs | 5 +- 5 files changed, 60 insertions(+), 34 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index b04175b11f0..26de8aac57f 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -265,9 +265,15 @@ href (config/get-local-asset-absolute-path href)] (when (or granted? (util/electron?) (mobile-util/is-native-platform?)) (p/then (editor-handler/make-asset-url href) #(reset! src %))) - + (when @src - (let [ext (keyword (util/get-file-ext @src))] + (let [ext (keyword (util/get-file-ext @src)) + share-fn (fn [event] + (util/stop event) + (when (mobile-util/is-native-platform?) + (p/let [url (str (config/get-repo-dir (state/get-current-repo)) href)] + (.share Share #js {:url url + :title "Open file with your favorite app"}))))] (cond (contains? config/audio-formats ext) (audio-cp @src) @@ -276,19 +282,14 @@ (resizable-image config title @src metadata full_text true) (= ext :pdf) - [:a.asset-ref.is-pdf - {:href @src - :on-click - (fn [event] - (util/stop event) - (when (mobile-util/is-native-platform?) - (p/let [url (str (config/get-repo-dir (state/get-current-repo)) href)] - (.share Share #js {:url url - :title "Open PDF fils with your favorite app"}))))} + [:a.asset-ref.is-pdf {:href @src + :on-click share-fn} title] :else - [:a.asset-ref {:ref @src} title]))))) + [:a.asset-ref.is-doc {:ref @src + :on-click share-fn} + title]))))) (defn ar-url->http-url [href] @@ -659,7 +660,7 @@ (and (= 1 (count label)) (let [label (first label)] (string? (last label)) - (last label)))) + (js/decodeURIComponent (last label))))) (defn- get-page [label] @@ -841,27 +842,30 @@ (defn- media-link [config url s label metadata full_text] - (let [ext (util/get-file-ext s)] + (let [ext (keyword (util/get-file-ext s)) + label-text (get-label-text label)] (cond - (contains? (set (map name config/audio-formats)) ext) + (contains? config/audio-formats ext) (audio-link config url s label metadata full_text) - (not (contains? #{"pdf" "mp4" "webm" "mov"} ext)) - (image-link config url s label metadata full_text) + (contains? (config/doc-formats) ext) + (asset-link config label-text s metadata full_text) - (= (util/get-file-ext s) "pdf") - (let [label-text (get-label-text label)] - (cond - (util/electron?) - [:a.asset-ref.is-pdf - {:href "javascript:void(0);" - :on-mouse-down (fn [_event] - (when-let [current (pdf-assets/inflate-asset s)] - (state/set-state! :pdf/current current)))} - label-text] - - (mobile-util/is-native-platform?) - (asset-link config label-text s metadata full_text))) + (= ext :pdf) + (cond + (util/electron?) + [:a.asset-ref.is-pdf + {:href "javascript:void(0);" + :on-mouse-down (fn [_event] + (when-let [current (pdf-assets/inflate-asset s)] + (state/set-state! :pdf/current current)))} + label-text] + + (mobile-util/is-native-platform?) + (asset-link config label-text s metadata full_text)) + + (not (contains? #{:mp4 :webm :mov} ext)) + (image-link config url s label metadata full_text) :else (asset-reference config label s)))) diff --git a/src/main/frontend/components/block.css b/src/main/frontend/components/block.css index 89d50634116..97ec94abd9c 100644 --- a/src/main/frontend/components/block.css +++ b/src/main/frontend/components/block.css @@ -240,6 +240,19 @@ html.is-native-android { align-items: center; } } + + &.is-doc { + &:before { + content: "[[📜"; + opacity: .7; + margin-right: 4px; + } + + &:after { + content: "]]"; + opacity: .7; + } + } } .embed-page { diff --git a/src/main/frontend/config.cljs b/src/main/frontend/config.cljs index 5e17c106189..894916c041a 100644 --- a/src/main/frontend/config.cljs +++ b/src/main/frontend/config.cljs @@ -85,6 +85,15 @@ config-formats #{:gif :svg :jpeg :ico :png :jpg :bmp :webp}))) +(defn doc-formats + [] + (let [config-formats (some->> (get-in @state/state [:config :document-formats]) + (map :keyword) + (set))] + (set/union + config-formats + #{:doc :docx :xls :xlsx :ppt :pptx :one :epub}))) + (def audio-formats #{:mp3 :ogg :mpeg :wav :m4a :flac :wma :aac}) (def media-formats (set/union (img-formats) audio-formats)) diff --git a/src/main/frontend/mobile/intent.cljs b/src/main/frontend/mobile/intent.cljs index 5516d9833ea..c0bf1f7ec83 100644 --- a/src/main/frontend/mobile/intent.cljs +++ b/src/main/frontend/mobile/intent.cljs @@ -112,7 +112,8 @@ (config/mldoc-support? application-type) (embed-text-file url title) - (contains? (set/union #{:pdf} config/media-formats) (keyword application-type)) + (contains? (set/union (config/doc-formats) config/media-formats) + (keyword application-type)) (embed-asset-file url format) :else diff --git a/src/main/frontend/ui.cljs b/src/main/frontend/ui.cljs index d66c91a1934..cb33c3f03f1 100644 --- a/src/main/frontend/ui.cljs +++ b/src/main/frontend/ui.cljs @@ -29,8 +29,7 @@ [rum.core :as rum] [frontend.db-mixins :as db-mixins] [frontend.mobile.util :as mobile-util] - [goog.functions :refer [debounce]] - [frontend.mobile.util :refer [is-native-platform?]])) + [goog.functions :refer [debounce]])) (defonce transition-group (r/adapt-class TransitionGroup)) (defonce css-transition (r/adapt-class CSSTransition)) @@ -947,7 +946,7 @@ (rum/defcs lazy-visible < (rum/local false ::visible?) [state content-fn sensor-opts reset-height?] - (if (or (util/mobile?) (is-native-platform?)) + (if (or (util/mobile?) (mobile-util/is-native-platform?)) (content-fn) (let [*visible? (::visible? state)] (visibility-sensor From a832564782e22db8868e7fc580e8e57a53c44063 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Fri, 13 May 2022 11:52:35 -0400 Subject: [PATCH 49/58] Migrate most of extract ns to nbb --- src/main/frontend/components/block.cljs | 2 +- src/main/frontend/components/content.cljs | 6 +-- src/main/frontend/components/file.cljs | 4 +- src/main/frontend/config.cljs | 8 --- src/main/frontend/db/model.cljs | 3 +- src/main/frontend/format.cljs | 16 +----- src/main/frontend/handler/export.cljs | 3 +- src/main/frontend/handler/extract.cljs | 55 ++++++++++----------- src/main/frontend/handler/file.cljs | 15 ++++-- src/main/frontend/handler/repo.cljs | 6 +-- src/main/frontend/mobile/intent.cljs | 3 +- src/main/frontend/search.cljs | 6 +-- src/main/logseq/graph_parser/config.cljs | 8 +++ src/main/logseq/graph_parser/util.cljs | 24 +++++++++ src/test/frontend/handler/extract_test.cljs | 2 +- 15 files changed, 88 insertions(+), 73 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 2e11406e03a..496fa6b94c6 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -981,7 +981,7 @@ show-brackets? (state/show-brackets?)] (if (and page (when-let [ext (util/get-file-ext href)] - (config/mldoc-support? ext))) + (gp-config/mldoc-support? ext))) [:span.page-reference (when show-brackets? [:span.text-gray-500 "[["]) (page-cp config page) diff --git a/src/main/frontend/components/content.cljs b/src/main/frontend/components/content.cljs index 93ca7beda9a..a9ca9f27394 100644 --- a/src/main/frontend/components/content.cljs +++ b/src/main/frontend/components/content.cljs @@ -36,7 +36,7 @@ (defn- lazy-load [format] - (let [format (format/normalize format)] + (let [format (gp-util/normalize-format format)] (when-let [record (format/get-format-record format)] (when-not (protocol/loaded? record) (set-format-js-loading! format true) @@ -388,7 +388,7 @@ :format format} id config) - (let [format (format/normalize format) + (let [format (gp-util/normalize-format format) loading? (get loading format) markup? (contains? config/html-render-formats format) on-click (fn [e] @@ -446,5 +446,5 @@ (if hiccup [:div (hiccup-content id option)] - (let [format (format/normalize format)] + (let [format (gp-util/normalize-format format)] (non-hiccup-content id content on-click on-hide config format)))) diff --git a/src/main/frontend/components/file.cljs b/src/main/frontend/components/file.cljs index deff736389f..a18da917e1b 100644 --- a/src/main/frontend/components/file.cljs +++ b/src/main/frontend/components/file.cljs @@ -9,11 +9,11 @@ [frontend.context.i18n :refer [t]] [frontend.date :as date] [frontend.db :as db] - [frontend.format :as format] [frontend.handler.export :as export-handler] [frontend.state :as state] [frontend.util :as util] [logseq.graph-parser.config :as gp-config] + [logseq.graph-parser.util :as gp-util] [goog.object :as gobj] [reitit.frontend.easy :as rfe] [rum.core :as rum])) @@ -71,7 +71,7 @@ state)} [state] (let [path (get-path state) - format (format/get-format path) + format (gp-util/get-format path) original-name (db/get-file-page path) random-id (str (d/squuid))] [:div.file {:id (str "file-edit-wrapper-" random-id)} diff --git a/src/main/frontend/config.cljs b/src/main/frontend/config.cljs index 5e17c106189..7d3f09865a9 100644 --- a/src/main/frontend/config.cljs +++ b/src/main/frontend/config.cljs @@ -97,14 +97,6 @@ (set/union (text-formats) (img-formats))) -;; TODO: rename -(defonce mldoc-support-formats - #{:org :markdown :md}) - -(defn mldoc-support? - [format] - (contains? mldoc-support-formats (keyword format))) - (def mobile? (when-not util/node-test? (gp-util/safe-re-find #"Mobi" js/navigator.userAgent))) diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index 90e2b9d6820..c441628fea9 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -12,7 +12,6 @@ [frontend.db.conn :as conn] [frontend.db.react :as react] [frontend.db.utils :as db-utils] - [frontend.format :as format] [frontend.state :as state] [frontend.util :as util :refer [react]] [logseq.graph-parser.util :as gp-util] @@ -294,7 +293,7 @@ (:block/format page) (when-let [file (:block/file page)] (when-let [path (:file/path (db-utils/entity (:db/id file)))] - (format/get-format path))))) + (gp-util/get-format path))))) (state/get-preferred-format) :markdown)) diff --git a/src/main/frontend/format.cljs b/src/main/frontend/format.cljs index 6638ce1ebfa..a9e0b0f106c 100644 --- a/src/main/frontend/format.cljs +++ b/src/main/frontend/format.cljs @@ -4,6 +4,7 @@ [frontend.format.protocol :as protocol] [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.text :as text] + [logseq.graph-parser.util :as gp-util] [clojure.string :as string])) ;; TODO: Properly fix this circular dependency: @@ -13,22 +14,9 @@ (defonce mldoc-record (->MldocMode)) (defonce adoc-record (->AdocMode)) -(defn normalize - [format] - (case (keyword format) - :md :markdown - :asciidoc :adoc - ;; default - (keyword format))) - -(defn get-format - [file] - (when file - (normalize (keyword (string/lower-case (last (string/split file #"\."))))))) - (defn get-format-record [format] - (case (normalize format) + (case (gp-util/normalize-format format) :org mldoc-record :markdown diff --git a/src/main/frontend/handler/export.cljs b/src/main/frontend/handler/export.cljs index 427d5749888..736b7d88adc 100644 --- a/src/main/frontend/handler/export.cljs +++ b/src/main/frontend/handler/export.cljs @@ -17,6 +17,7 @@ [frontend.util :as util] [frontend.format.mldoc :as mldoc] [logseq.graph-parser.mldoc :as gp-mldoc] + [logseq.graph-parser.util :as gp-util] [goog.dom :as gdom] [promesa.core :as p]) (:import [goog.string StringBuffer])) @@ -379,7 +380,7 @@ [?e2 :block/file ?e] [?e2 :block/name ?n] [?e2 :block/original-name ?n2]] db path) - :format (f/get-format path)}))))) + :format (gp-util/get-format path)}))))) (defn export-repo-as-markdown! diff --git a/src/main/frontend/handler/extract.cljs b/src/main/frontend/handler/extract.cljs index 1feb5f98ba9..49b3c3b6407 100644 --- a/src/main/frontend/handler/extract.cljs +++ b/src/main/frontend/handler/extract.cljs @@ -3,22 +3,18 @@ (:require [clojure.set :as set] [clojure.string :as string] [clojure.walk :as walk] - [frontend.config :as config] - [frontend.db :as db] - [frontend.format :as format] + [datascript.core :as d] [frontend.format.block :as block] - [frontend.format.mldoc :as mldoc] - [frontend.state :as state] [logseq.graph-parser.text :as text] - [frontend.util :as util] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.block :as gp-block] [logseq.graph-parser.property :as gp-property] + [logseq.graph-parser.config :as gp-config] [lambdaisland.glogi :as log])) -(defn get-page-name - [file ast] +(defn- get-page-name + [file ast page-name-order] ;; headline (let [ast (map first ast)] (if (string/includes? file "pages/contents.") @@ -33,11 +29,11 @@ title)) file-name (when-let [file-name (last (string/split file #"/"))] (let [result (first (gp-util/split-last "." file-name))] - (if (config/mldoc-support? (string/lower-case (util/get-file-ext file))) - (util/url-decode (string/replace result "." "/")) + (if (gp-config/mldoc-support? (string/lower-case (gp-util/get-file-ext file))) + (js/decodeURIComponent (string/replace result "." "/")) result)))] (or property-name - (if (= (state/page-name-order) "heading") + (if (= page-name-order "heading") (or first-block-name file-name) (or file-name first-block-name))))))) @@ -45,10 +41,10 @@ ;; TODO: performance improvement (defn- extract-pages-and-blocks #_:clj-kondo/ignore - [repo-url format ast properties file content] + [repo-url format ast properties file content date-formatter page-name-order] (try - (let [page (get-page-name file ast) - [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page (state/get-date-formatter)) + (let [page (get-page-name file ast page-name-order) + [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter) blocks (->> (block/extract-blocks ast content false format) (block/with-parent-and-left {:block/name page-name})) ref-pages (atom #{}) @@ -70,13 +66,13 @@ page-entity (let [alias (:alias properties) alias (if (string? alias) [alias] alias) aliases (and alias - (seq (remove #(or (= page-name (util/page-name-sanity-lc %)) + (seq (remove #(or (= page-name (gp-util/page-name-sanity-lc %)) (string/blank? %)) ;; disable blank alias alias))) aliases (->> (map (fn [alias] - (let [page-name (util/page-name-sanity-lc alias) + (let [page-name (gp-util/page-name-sanity-lc alias) aliases (distinct (conj (remove #{alias} aliases) @@ -84,7 +80,7 @@ aliases (when (seq aliases) (map (fn [alias] - {:block/name (util/page-name-sanity-lc alias)}) + {:block/name (gp-util/page-name-sanity-lc alias)}) aliases))] (if (seq aliases) {:block/name page-name @@ -108,7 +104,7 @@ tags (if (string? tags) [tags] tags) tags (remove string/blank? tags)] (swap! ref-tags set/union (set tags)) - (map (fn [tag] {:block/name (util/page-name-sanity-lc tag) + (map (fn [tag] {:block/name (gp-util/page-name-sanity-lc tag) :block/original-name tag}) tags))))) namespace-pages (let [page (:block/original-name page-entity)] @@ -123,15 +119,15 @@ (map (fn [page] {:block/original-name page - :block/name (util/page-name-sanity-lc page)}) + :block/name (gp-util/page-name-sanity-lc page)}) @ref-tags) namespace-pages) ;; remove block references (remove vector?) (remove nil?)) - pages (util/distinct-by :block/name pages) + pages (gp-util/distinct-by :block/name pages) pages (remove nil? pages) - pages (map (fn [page] (assoc page :block/uuid (db/new-block-id))) pages) + pages (map (fn [page] (assoc page :block/uuid (d/squuid))) pages) blocks (->> (remove nil? blocks) (map (fn [b] (dissoc b :block/title :block/body :block/level :block/children :block/meta :block/anchor))))] [pages blocks]) @@ -139,21 +135,22 @@ (log/error :exception e)))) (defn extract-blocks-pages - [repo-url file content] + [repo-url file content {:keys [user-config date-formatter page-name-order]}] (if (string/blank? content) [] - (let [format (format/get-format file) + (let [format (gp-util/get-format file) _ (println "Parsing start: " file) - ast (mldoc/->edn content (gp-mldoc/default-config format + ast (gp-mldoc/->edn content (gp-mldoc/default-config format ;; {:parse_outline_only? true} - ))] + ) + user-config)] (println "Parsing finished : " file) (let [first-block (ffirst ast) properties (let [properties (and (gp-property/properties-ast? first-block) (->> (last first-block) (map (fn [[x y]] [x (if (string? y) - (text/parse-property format x y (state/get-config)) + (text/parse-property format x y user-config) y)])) (into {}) (walk/keywordize-keys)))] @@ -166,15 +163,15 @@ (extract-pages-and-blocks repo-url format ast properties - file content))))) + file content date-formatter page-name-order))))) (defn with-block-uuid [pages] - (->> (util/distinct-by :block/name pages) + (->> (gp-util/distinct-by :block/name pages) (map (fn [page] (if (:block/uuid page) page - (assoc page :block/uuid (db/new-block-id))))))) + (assoc page :block/uuid (d/squuid))))))) (defn with-ref-pages [pages blocks] diff --git a/src/main/frontend/handler/file.cljs b/src/main/frontend/handler/file.cljs index 2629b78db75..e6b3b112c62 100644 --- a/src/main/frontend/handler/file.cljs +++ b/src/main/frontend/handler/file.cljs @@ -8,7 +8,6 @@ [clojure.core.async :as async] [frontend.config :as config] [frontend.db :as db] - [frontend.format :as format] [frontend.fs :as fs] [frontend.fs.nfs :as nfs] [frontend.handler.common :as common-handler] @@ -17,6 +16,7 @@ [frontend.state :as state] [frontend.util :as util] [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.config :as gp-config] [lambdaisland.glogi :as log] [promesa.core :as p] [frontend.mobile.util :as mobile] @@ -43,7 +43,7 @@ [files formats] (filter (fn [file] - (let [format (format/get-format file)] + (let [format (gp-util/get-format file)] (contains? formats format))) files)) @@ -119,10 +119,15 @@ file (gp-util/path-normalize file) new? (nil? (db/entity [:file/path file]))] (db/set-file-content! repo-url file content) - (let [format (format/get-format file) + (let [format (gp-util/get-format file) file-content [{:file/path file}] - tx (if (contains? config/mldoc-support-formats format) - (let [[pages blocks] (extract-handler/extract-blocks-pages repo-url file content) + tx (if (contains? gp-config/mldoc-support-formats format) + (let [[pages blocks] + (extract-handler/extract-blocks-pages + repo-url file content + {:user-config (state/get-config) + :date-formatter (state/get-date-formatter) + :page-name-order (state/page-name-order)}) first-page (first pages) delete-blocks (-> (concat diff --git a/src/main/frontend/handler/repo.cljs b/src/main/frontend/handler/repo.cljs index 250f4ea18c0..d88d7b5f7f1 100644 --- a/src/main/frontend/handler/repo.cljs +++ b/src/main/frontend/handler/repo.cljs @@ -5,7 +5,6 @@ [frontend.context.i18n :refer [t]] [frontend.date :as date] [frontend.db :as db] - [frontend.format :as format] [frontend.fs :as fs] [frontend.fs.nfs :as nfs] [frontend.handler.common :as common-handler] @@ -23,6 +22,7 @@ [shadow.resource :as rc] [frontend.db.persist :as db-persist] [logseq.graph-parser.util :as gp-util] + [logseq.graph-parser.config :as gp-config] [electron.ipc :as ipc] [clojure.set :as set] [clojure.core.async :as async] @@ -217,8 +217,8 @@ [repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts] (let [support-files (filter (fn [file] - (let [format (format/get-format (:file/path file))] - (contains? (set/union #{:edn :css} config/mldoc-support-formats) format))) + (let [format (gp-util/get-format (:file/path file))] + (contains? (set/union #{:edn :css} gp-config/mldoc-support-formats) format))) files) support-files (sort-by :file/path support-files) {journals true non-journals false} (group-by (fn [file] (string/includes? (:file/path file) "journals/")) support-files) diff --git a/src/main/frontend/mobile/intent.cljs b/src/main/frontend/mobile/intent.cljs index 5516d9833ea..bccb3ffd3dd 100644 --- a/src/main/frontend/mobile/intent.cljs +++ b/src/main/frontend/mobile/intent.cljs @@ -11,6 +11,7 @@ [frontend.util :as util] [frontend.config :as config] [logseq.graph-parser.mldoc :as gp-mldoc] + [logseq.graph-parser.config :as gp-config] ["path" :as path] [frontend.mobile.util :as mobile-util] [frontend.handler.notification :as notification] @@ -109,7 +110,7 @@ format (db/get-page-format page) application-type (last (string/split type "/")) content (cond - (config/mldoc-support? application-type) + (gp-config/mldoc-support? application-type) (embed-text-file url title) (contains? (set/union #{:pdf} config/media-formats) (keyword application-type)) diff --git a/src/main/frontend/search.cljs b/src/main/frontend/search.cljs index 835ddde41cc..b2351d5c309 100644 --- a/src/main/frontend/search.cljs +++ b/src/main/frontend/search.cljs @@ -1,7 +1,7 @@ (ns frontend.search (:require [cljs-bean.core :as bean] [clojure.string :as string] - [frontend.config :as config] + [logseq.graph-parser.config :as gp-config] [frontend.db :as db] [frontend.regex :as regex] [frontend.search.browser :as search-browser] @@ -143,7 +143,7 @@ ([q limit] (let [q (clean-str q)] (when-not (string/blank? q) - (let [mldoc-exts (set (map name config/mldoc-support-formats)) + (let [mldoc-exts (set (map name gp-config/mldoc-support-formats)) files (->> (db/get-files (state/get-current-repo)) (map first) (remove (fn [file] @@ -237,4 +237,4 @@ (defn cache-stale? [repo] (when-let [engine (get-engine repo)] - (protocol/cache-stale? engine repo))) \ No newline at end of file + (protocol/cache-stale? engine repo))) diff --git a/src/main/logseq/graph_parser/config.cljs b/src/main/logseq/graph_parser/config.cljs index 68b1b1f7f07..2d5b3dfa727 100644 --- a/src/main/logseq/graph_parser/config.cljs +++ b/src/main/logseq/graph_parser/config.cljs @@ -14,3 +14,11 @@ (defn draw? [path] (string/starts-with? path default-draw-directory)) + +;; TODO: rename +(defonce mldoc-support-formats + #{:org :markdown :md}) + +(defn mldoc-support? + [format] + (contains? mldoc-support-formats (keyword format))) diff --git a/src/main/logseq/graph_parser/util.cljs b/src/main/logseq/graph_parser/util.cljs index 3baf3feca04..e6097aa264e 100644 --- a/src/main/logseq/graph_parser/util.cljs +++ b/src/main/logseq/graph_parser/util.cljs @@ -161,3 +161,27 @@ (some->> (string/split s #" ") (map string/capitalize) (string/join " "))) + +(defn distinct-by + "Copy of frontend.util/distinct-by. Too basic to couple to main app" + [f col] + (reduce + (fn [acc x] + (if (some #(= (f x) (f %)) acc) + acc + (vec (conj acc x)))) + [] + col)) + +(defn normalize-format + [format] + (case (keyword format) + :md :markdown + :asciidoc :adoc + ;; default + (keyword format))) + +(defn get-format + [file] + (when file + (normalize-format (keyword (string/lower-case (last (string/split file #"\."))))))) diff --git a/src/test/frontend/handler/extract_test.cljs b/src/test/frontend/handler/extract_test.cljs index 84474b2f21d..7f9b21e9f14 100644 --- a/src/test/frontend/handler/extract_test.cljs +++ b/src/test/frontend/handler/extract_test.cljs @@ -6,7 +6,7 @@ (defn- extract [text] - (p/let [result (extract/extract-blocks-pages "repo" "a.md" text) + (p/let [result (extract/extract-blocks-pages "repo" "a.md" text {}) result (last result) lefts (map (juxt :block/parent :block/left) result)] (if (not= (count lefts) (count (distinct lefts))) From b9bbd0c6056d170c7e04c555030a9ebcf9c7a062 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Fri, 13 May 2022 15:47:54 -0400 Subject: [PATCH 50/58] Finish migrating extract to nbb --- .clj-kondo/config.edn | 1 + src/main/frontend/format/block.cljs | 70 ------------------- src/main/frontend/fs/watcher_handler.cljs | 2 +- src/main/frontend/handler/editor.cljs | 4 +- src/main/frontend/handler/file.cljs | 14 ++-- src/main/logseq/graph_parser/block.cljc | 70 +++++++++++++++++++ .../graph_parser/extract.cljc} | 31 ++++---- .../frontend/modules/outliner/core_test.cljs | 4 +- .../graph_parser}/extract_test.cljs | 10 +-- .../logseq/graph_parser/nbb_test_runner.cljs | 6 +- 10 files changed, 110 insertions(+), 102 deletions(-) rename src/main/{frontend/handler/extract.cljs => logseq/graph_parser/extract.cljc} (89%) rename src/test/{frontend/handler => logseq/graph_parser}/extract_test.cljs (76%) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index ca3f6d717ee..c0eb3506075 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -23,6 +23,7 @@ frontend.config config frontend.format.mldoc mldoc frontend.format.block block + frontend.handler.extract extract logseq.graph-parser.text text logseq.graph-parser.block gp-block logseq.graph-parser.mldoc gp-mldoc diff --git a/src/main/frontend/format/block.cljs b/src/main/frontend/format/block.cljs index 4d2ff354dd9..7f20ea7b7c9 100644 --- a/src/main/frontend/format/block.cljs +++ b/src/main/frontend/format/block.cljs @@ -26,76 +26,6 @@ ([original-page-name with-id? with-timestamp?] (gp-block/page-name->map original-page-name with-id? (db/get-db (state/get-current-repo)) with-timestamp? (state/get-date-formatter)))) -(defn with-parent-and-left - [page-id blocks] - (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks) - parents [{:page/id page-id ; db id or a map {:block/name "xxx"} - :block/level 0 - :block/level-spaces 0}] - result []] - (if (empty? blocks) - (map #(dissoc % :block/level-spaces) result) - (let [[block & others] blocks - level-spaces (:block/level-spaces block) - {:block/keys [uuid level parent] :as last-parent} (last parents) - parent-spaces (:block/level-spaces last-parent) - [blocks parents result] - (cond - (= level-spaces parent-spaces) ; sibling - (let [block (assoc block - :block/parent parent - :block/left [:block/uuid uuid] - :block/level level) - parents' (conj (vec (butlast parents)) block) - result' (conj result block)] - [others parents' result']) - - (> level-spaces parent-spaces) ; child - (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent)) - block (cond-> - (assoc block - :block/parent parent - :block/left parent) - ;; fix block levels with wrong order - ;; For example: - ;; - a - ;; - b - ;; What if the input indentation is two spaces instead of 4 spaces - (>= (- level-spaces parent-spaces) 1) - (assoc :block/level (inc level))) - parents' (conj parents block) - result' (conj result block)] - [others parents' result']) - - (< level-spaces parent-spaces) - (cond - (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent - (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)) - left (last parents') - blocks (cons (assoc (first blocks) - :block/level (dec level) - :block/left [:block/uuid (:block/uuid left)]) - (rest blocks))] - [blocks parents' result]) - - :else - (let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents) - left (first r) - parent-id (if-let [block-id (:block/uuid (last f))] - [:block/uuid block-id] - page-id) - block (cond-> - (assoc block - :block/parent parent-id - :block/left [:block/uuid (:block/uuid left)] - :block/level (:block/level left) - :block/level-spaces (:block/level-spaces left))) - - parents' (->> (concat f [block]) vec) - result' (conj result block)] - [others parents' result'])))] - (recur blocks parents result))))) - (defn parse-block ([block] (parse-block block nil)) diff --git a/src/main/frontend/fs/watcher_handler.cljs b/src/main/frontend/fs/watcher_handler.cljs index 195f0b170ce..ca6735f448a 100644 --- a/src/main/frontend/fs/watcher_handler.cljs +++ b/src/main/frontend/fs/watcher_handler.cljs @@ -4,7 +4,7 @@ [frontend.db :as db] [frontend.db.model :as model] [frontend.handler.editor :as editor] - [frontend.handler.extract :as extract] + [logseq.graph-parser.extract :as extract] [frontend.handler.file :as file-handler] [frontend.handler.page :as page-handler] [frontend.handler.repo :as repo-handler] diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index de151363bf6..ac24c325c5d 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -1960,7 +1960,7 @@ (let [blocks (block-tree->blocks tree-vec format) target-block (db/pull target-block-id) page-id (:db/id (:block/page target-block)) - blocks (block/with-parent-and-left page-id blocks)] + blocks (gp-block/with-parent-and-left page-id blocks)] (paste-blocks blocks {:target-block target-block @@ -2857,7 +2857,7 @@ (let [page-id (:db/id (:block/page editing-block)) blocks (block/extract-blocks (mldoc/->edn text (gp-mldoc/default-config format)) text true format) - blocks' (block/with-parent-and-left page-id blocks)] + blocks' (gp-block/with-parent-and-left page-id blocks)] (paste-blocks blocks' {})))) (defn- paste-segmented-text diff --git a/src/main/frontend/handler/file.cljs b/src/main/frontend/handler/file.cljs index e6b3b112c62..a6a374243f0 100644 --- a/src/main/frontend/handler/file.cljs +++ b/src/main/frontend/handler/file.cljs @@ -11,7 +11,7 @@ [frontend.fs :as fs] [frontend.fs.nfs :as nfs] [frontend.handler.common :as common-handler] - [frontend.handler.extract :as extract-handler] + [logseq.graph-parser.extract :as extract] [frontend.handler.ui :as ui-handler] [frontend.state :as state] [frontend.util :as util] @@ -123,11 +123,15 @@ file-content [{:file/path file}] tx (if (contains? gp-config/mldoc-support-formats format) (let [[pages blocks] - (extract-handler/extract-blocks-pages - repo-url file content + (extract/extract-blocks-pages + file + content {:user-config (state/get-config) :date-formatter (state/get-date-formatter) - :page-name-order (state/page-name-order)}) + :page-name-order (state/page-name-order) + :block-pattern (config/get-block-pattern format) + :supported-formats (config/supported-formats) + :db (db/get-db (state/get-current-repo))}) first-page (first pages) delete-blocks (-> (concat @@ -149,7 +153,7 @@ (seq)) ;; To prevent "unique constraint" on datascript block-ids (set/union (set block-ids) (set block-refs-ids)) - pages (extract-handler/with-ref-pages pages blocks) + pages (extract/with-ref-pages pages blocks) pages-index (map #(select-keys % [:block/name]) pages)] ;; does order matter? (concat file-content pages-index delete-blocks pages block-ids blocks)) diff --git a/src/main/logseq/graph_parser/block.cljc b/src/main/logseq/graph_parser/block.cljc index c02cda92c87..7cc7d437216 100644 --- a/src/main/logseq/graph_parser/block.cljc +++ b/src/main/logseq/graph_parser/block.cljc @@ -581,3 +581,73 @@ (map #(dissoc % :block/meta) result)) (catch :default e (log/error :extract-blocks-failure e)))) + +(defn with-parent-and-left + [page-id blocks] + (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks) + parents [{:page/id page-id ; db id or a map {:block/name "xxx"} + :block/level 0 + :block/level-spaces 0}] + result []] + (if (empty? blocks) + (map #(dissoc % :block/level-spaces) result) + (let [[block & others] blocks + level-spaces (:block/level-spaces block) + {:block/keys [uuid level parent] :as last-parent} (last parents) + parent-spaces (:block/level-spaces last-parent) + [blocks parents result] + (cond + (= level-spaces parent-spaces) ; sibling + (let [block (assoc block + :block/parent parent + :block/left [:block/uuid uuid] + :block/level level) + parents' (conj (vec (butlast parents)) block) + result' (conj result block)] + [others parents' result']) + + (> level-spaces parent-spaces) ; child + (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent)) + block (cond-> + (assoc block + :block/parent parent + :block/left parent) + ;; fix block levels with wrong order + ;; For example: + ;; - a + ;; - b + ;; What if the input indentation is two spaces instead of 4 spaces + (>= (- level-spaces parent-spaces) 1) + (assoc :block/level (inc level))) + parents' (conj parents block) + result' (conj result block)] + [others parents' result']) + + (< level-spaces parent-spaces) + (cond + (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent + (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)) + left (last parents') + blocks (cons (assoc (first blocks) + :block/level (dec level) + :block/left [:block/uuid (:block/uuid left)]) + (rest blocks))] + [blocks parents' result]) + + :else + (let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents) + left (first r) + parent-id (if-let [block-id (:block/uuid (last f))] + [:block/uuid block-id] + page-id) + block (cond-> + (assoc block + :block/parent parent-id + :block/left [:block/uuid (:block/uuid left)] + :block/level (:block/level left) + :block/level-spaces (:block/level-spaces left))) + + parents' (->> (concat f [block]) vec) + result' (conj result block)] + [others parents' result'])))] + (recur blocks parents result))))) diff --git a/src/main/frontend/handler/extract.cljs b/src/main/logseq/graph_parser/extract.cljc similarity index 89% rename from src/main/frontend/handler/extract.cljs rename to src/main/logseq/graph_parser/extract.cljc index 49b3c3b6407..36d25f30864 100644 --- a/src/main/frontend/handler/extract.cljs +++ b/src/main/logseq/graph_parser/extract.cljc @@ -1,17 +1,19 @@ -(ns frontend.handler.extract - "Extract helper." +(ns ^:nbb-compatible logseq.graph-parser.extract + ;; Disable clj linters since we don't support clj + #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off} + :unresolved-symbol {:level :off}}}}) (:require [clojure.set :as set] [clojure.string :as string] [clojure.walk :as walk] [datascript.core :as d] - [frontend.format.block :as block] [logseq.graph-parser.text :as text] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.block :as gp-block] [logseq.graph-parser.property :as gp-property] [logseq.graph-parser.config :as gp-config] - [lambdaisland.glogi :as log])) + #?(:org.babashka/nbb [logseq.graph-parser.log :as log] + :default [lambdaisland.glogi :as log]))) (defn- get-page-name [file ast page-name-order] @@ -40,13 +42,13 @@ ;; TODO: performance improvement (defn- extract-pages-and-blocks - #_:clj-kondo/ignore - [repo-url format ast properties file content date-formatter page-name-order] + [format ast properties file content {:keys [date-formatter page-name-order db] :as options}] (try + #_:clj-kondo/ignore ;;clj-kondo bug (let [page (get-page-name file ast page-name-order) [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter) - blocks (->> (block/extract-blocks ast content false format) - (block/with-parent-and-left {:block/name page-name})) + blocks (->> (gp-block/extract-blocks ast content false format (dissoc options :page-name-order)) + (gp-block/with-parent-and-left {:block/name page-name})) ref-pages (atom #{}) ref-tags (atom #{}) blocks (map (fn [block] @@ -91,7 +93,7 @@ (cond-> (gp-util/remove-nils (assoc - (block/page-name->map page false) + (gp-block/page-name->map page false db true date-formatter) :block/file {:file/path (gp-util/path-normalize file)})) (seq properties) (assoc :block/properties properties) @@ -111,7 +113,7 @@ (when (text/namespace-page? page) (->> (gp-util/split-namespace-pages page) (map (fn [page] - (-> (block/page-name->map page true) + (-> (gp-block/page-name->map page true db true date-formatter) (assoc :block/format format))))))) pages (->> (concat [page-entity] @@ -131,11 +133,11 @@ blocks (->> (remove nil? blocks) (map (fn [b] (dissoc b :block/title :block/body :block/level :block/children :block/meta :block/anchor))))] [pages blocks]) - (catch js/Error e + (catch :default e (log/error :exception e)))) (defn extract-blocks-pages - [repo-url file content {:keys [user-config date-formatter page-name-order]}] + [file content {:keys [user-config] :as options}] (if (string/blank? content) [] (let [format (gp-util/get-format file) @@ -161,11 +163,10 @@ (string/replace (or v "") "\\" ""))) properties)))] (extract-pages-and-blocks - repo-url format ast properties - file content date-formatter page-name-order))))) + file content options))))) -(defn with-block-uuid +(defn- with-block-uuid [pages] (->> (gp-util/distinct-by :block/name pages) (map (fn [page] diff --git a/src/test/frontend/modules/outliner/core_test.cljs b/src/test/frontend/modules/outliner/core_test.cljs index 07200720167..59a2b1c6a63 100644 --- a/src/test/frontend/modules/outliner/core_test.cljs +++ b/src/test/frontend/modules/outliner/core_test.cljs @@ -8,7 +8,7 @@ [frontend.db :as db] [frontend.db.model :as db-model] [clojure.walk :as walk] - [frontend.format.block :as block] + [logseq.graph-parser.block :as gp-block] [datascript.core :as d] [frontend.test.helper :as helper])) @@ -65,7 +65,7 @@ (defn- build-blocks [tree] - (block/with-parent-and-left 1 (build-node-tree tree))) + (gp-block/with-parent-and-left 1 (build-node-tree tree))) (defn transact-tree! [tree] diff --git a/src/test/frontend/handler/extract_test.cljs b/src/test/logseq/graph_parser/extract_test.cljs similarity index 76% rename from src/test/frontend/handler/extract_test.cljs rename to src/test/logseq/graph_parser/extract_test.cljs index 7f9b21e9f14..dc00d1647d3 100644 --- a/src/test/frontend/handler/extract_test.cljs +++ b/src/test/logseq/graph_parser/extract_test.cljs @@ -1,17 +1,17 @@ -(ns frontend.handler.extract-test +(ns logseq.graph-parser.extract-test (:require [cljs.test :refer [async deftest is]] - [frontend.handler.extract :as extract] - [frontend.util :as util] + [logseq.graph-parser.extract :as extract] + [clojure.pprint :as pprint] [promesa.core :as p])) (defn- extract [text] - (p/let [result (extract/extract-blocks-pages "repo" "a.md" text {}) + (p/let [result (extract/extract-blocks-pages "a.md" text {:block-pattern "-"}) result (last result) lefts (map (juxt :block/parent :block/left) result)] (if (not= (count lefts) (count (distinct lefts))) (do - (util/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result)) + (pprint/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result)) (throw (js/Error. ":block/parent && :block/left conflicts"))) (mapv :block/content result)))) diff --git a/src/test/logseq/graph_parser/nbb_test_runner.cljs b/src/test/logseq/graph_parser/nbb_test_runner.cljs index 85c1a6365e3..0f8ac48687b 100644 --- a/src/test/logseq/graph_parser/nbb_test_runner.cljs +++ b/src/test/logseq/graph_parser/nbb_test_runner.cljs @@ -6,7 +6,8 @@ [logseq.graph-parser.text-test] [logseq.graph-parser.mldoc-test] [logseq.graph-parser.block-test] - [logseq.graph-parser.property-test])) + [logseq.graph-parser.property-test] + [logseq.graph-parser.extract-test])) (defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m] (when-not (cljs.test/successful? m) @@ -21,4 +22,5 @@ (t/run-tests 'logseq.graph-parser.mldoc-test 'logseq.graph-parser.text-test 'logseq.graph-parser.property-test - 'logseq.graph-parser.block-test)) + 'logseq.graph-parser.block-test + 'logseq.graph-parser.extract-test)) From 5b0d5fb8b5c8a851b685c5de9cdfbc985a1c6fb8 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Fri, 13 May 2022 16:17:43 -0400 Subject: [PATCH 51/58] Decouple graph-parser util from app util usage For tiny util heavily used fns like safe-re-find and uuid-string?, decouple graph-parser from so much of the app --- src/main/frontend/commands.cljs | 6 ++-- src/main/frontend/components/block.cljs | 16 +++++----- src/main/frontend/components/content.cljs | 2 +- src/main/frontend/components/page.cljs | 6 ++-- src/main/frontend/components/page_menu.cljs | 3 +- src/main/frontend/components/reference.cljs | 5 ++-- src/main/frontend/components/search.cljs | 3 +- src/main/frontend/config.cljs | 2 +- src/main/frontend/db/model.cljs | 4 +-- src/main/frontend/db/query_dsl.cljs | 5 ++-- src/main/frontend/db/query_react.cljs | 5 ++-- src/main/frontend/db/react.cljs | 3 +- src/main/frontend/extensions/html_parser.cljs | 3 +- src/main/frontend/handler/editor.cljs | 24 +++++++-------- src/main/frontend/handler/graph.cljs | 5 ++-- src/main/frontend/handler/page.cljs | 4 +-- src/main/frontend/handler/route.cljs | 3 +- src/main/frontend/handler/ui.cljs | 3 +- src/main/frontend/modules/outliner/tree.cljs | 4 +-- src/main/frontend/security.cljs | 4 +-- src/main/frontend/state.cljs | 3 +- src/main/frontend/util.cljc | 29 +++++++++++++++---- src/main/frontend/util/drawer.cljs | 5 ++-- src/main/frontend/util/marker.cljs | 7 ++--- src/main/frontend/util/priority.cljs | 5 ++-- src/main/frontend/util/property.cljs | 10 +++---- src/main/logseq/graph_parser/util.cljs | 2 ++ 27 files changed, 88 insertions(+), 83 deletions(-) diff --git a/src/main/frontend/commands.cljs b/src/main/frontend/commands.cljs index c5bfd4c39ee..0bb20d020eb 100644 --- a/src/main/frontend/commands.cljs +++ b/src/main/frontend/commands.cljs @@ -517,7 +517,7 @@ (defn compute-pos-delta-when-change-marker [edit-content marker pos] - (let [old-marker (some->> (first (gp-util/safe-re-find marker/bare-marker-pattern edit-content)) + (let [old-marker (some->> (first (util/safe-re-find marker/bare-marker-pattern edit-content)) (string/trim)) pos-delta (- (count marker) (count old-marker)) @@ -542,7 +542,7 @@ (if-let [matches (seq (util/re-pos new-line-re-pattern prefix))] (let [[start-pos content] (last matches)] (+ start-pos (count content))) - (count (gp-util/safe-re-find re-pattern prefix)))) + (count (util/safe-re-find re-pattern prefix)))) new-value (str (subs edit-content 0 pos) (string/replace-first (subs edit-content pos) (marker/marker-pattern format) @@ -583,7 +583,7 @@ (let [edit-content (gobj/get current-input "value") heading-pattern #"^#+\s+" new-value (cond - (gp-util/safe-re-find heading-pattern edit-content) + (util/safe-re-find heading-pattern edit-content) (string/replace-first edit-content heading-pattern (str heading " ")) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 496fa6b94c6..a267649ff38 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -683,7 +683,7 @@ [config id label] (when (and (not (string/blank? id)) - (gp-util/uuid-string? id)) + (util/uuid-string? id)) (let [block-id (uuid id) block (db/pull-block block-id) block-type (keyword (get-in block [:block/properties :ls-type])) @@ -961,7 +961,7 @@ (= "Complex" protocol) (= (string/lower-case (:protocol path)) "id") (string? (:link path)) - (gp-util/uuid-string? (:link path))) ; org mode id + (util/uuid-string? (:link path))) ; org mode id (let [id (uuid (:link path)) block (db/entity [:block/uuid id])] (if (:block/pre-block? block) @@ -1077,7 +1077,7 @@ string/trim)] (when-let [id (and s (let [s (string/trim s)] - (and (gp-util/uuid-string? s) + (and (util/uuid-string? s) (uuid s))))] (block-embed (assoc config :link-depth (inc link-depth)) id))) @@ -1088,7 +1088,7 @@ [_config arguments] (when-let [url (first arguments)] (let [Vimeo-regex #"^((?:https?:)?//)?((?:www).)?((?:player.vimeo.com|vimeo.com)?)((?:/video/)?)([\w-]+)(\S+)?$"] - (when-let [vimeo-id (nth (gp-util/safe-re-find Vimeo-regex url) 5)] + (when-let [vimeo-id (nth (util/safe-re-find Vimeo-regex url) 5)] (when-not (string/blank? vimeo-id) (let [width (min (- (util/get-width) 96) 560) @@ -1109,7 +1109,7 @@ (when-let [id (cond (<= (count url) 15) url :else - (last (gp-util/safe-re-find id-regex url)))] + (last (util/safe-re-find id-regex url)))] (when-not (string/blank? id) (let [width (min (- (util/get-width) 96) 560) @@ -1239,7 +1239,7 @@ (when-let [youtube-id (cond (== 11 (count url)) url :else - (nth (gp-util/safe-re-find YouTube-regex url) 5))] + (nth (util/safe-re-find YouTube-regex url) 5))] (when-not (string/blank? youtube-id) (youtube/youtube-video youtube-id))))) @@ -1270,7 +1270,7 @@ (when-let [id (cond (<= (count url) 15) url :else - (last (gp-util/safe-re-find id-regex url)))] + (last (util/safe-re-find id-regex url)))] (ui/tweet-embed id)))) (= name "embed") @@ -2858,7 +2858,7 @@ ["Paragraph" l] ;; TODO: speedup - (if (gp-util/safe-re-find #"\"Export_Snippet\" \"embed\"" (str l)) + (if (util/safe-re-find #"\"Export_Snippet\" \"embed\"" (str l)) (->elem :div (map-inline config l)) (->elem :div.is-paragraph (map-inline config l))) diff --git a/src/main/frontend/components/content.cljs b/src/main/frontend/components/content.cljs index a9ca9f27394..d83b8d66cde 100644 --- a/src/main/frontend/components/content.cljs +++ b/src/main/frontend/components/content.cljs @@ -363,7 +363,7 @@ e (custom-context-menu-content)) - (and block-id (gp-util/uuid-string? block-id)) + (and block-id (util/uuid-string? block-id)) (let [block (.closest target ".ls-block")] (when block (util/select-highlight! [block])) diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 2e2171ea113..b9efba5739e 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -122,7 +122,7 @@ (when page-e (let [page-name (or (:block/name page-e) (str (:block/uuid page-e))) - block? (gp-util/uuid-string? page-name) + block? (util/uuid-string? page-name) block-id (and block? (uuid page-name)) page-blocks (get-blocks repo page-name block-id)] (if (empty? page-blocks) @@ -317,7 +317,7 @@ (let [current-repo (state/sub :git/current-repo) repo (or repo current-repo) page-name (util/page-name-sanity-lc path-page-name) - block? (gp-util/uuid-string? page-name) + block? (util/uuid-string? page-name) block-id (and block? (uuid page-name)) format (let [page (if block-id (:block/name (:block/page (db/entity [:block/uuid block-id]))) @@ -640,7 +640,7 @@ (date/today)) theme (:ui/theme @state/state) dark? (= theme "dark") - graph (if (gp-util/uuid-string? page) + graph (if (util/uuid-string? page) (graph-handler/build-block-graph (uuid page) theme) (graph-handler/build-page-graph page theme))] (when (seq (:nodes graph)) diff --git a/src/main/frontend/components/page_menu.cljs b/src/main/frontend/components/page_menu.cljs index 981fafff5d8..4da04fc7cb5 100644 --- a/src/main/frontend/components/page_menu.cljs +++ b/src/main/frontend/components/page_menu.cljs @@ -14,7 +14,6 @@ [frontend.handler.shell :as shell] [frontend.handler.plugin :as plugin-handler] [frontend.mobile.util :as mobile-util] - [logseq.graph-parser.util :as gp-util] [electron.ipc :as ipc] [frontend.config :as config] [frontend.handler.user :as user-handler] @@ -64,7 +63,7 @@ repo (state/sub :git/current-repo) page (db/entity repo [:block/name page-name]) page-original-name (:block/original-name page) - block? (and page (gp-util/uuid-string? page-name)) + block? (and page (util/uuid-string? page-name)) contents? (= page-name "contents") properties (:block/properties page) public? (true? (:public properties)) diff --git a/src/main/frontend/components/reference.cljs b/src/main/frontend/components/reference.cljs index 9e25e611d10..5f426388339 100644 --- a/src/main/frontend/components/reference.cljs +++ b/src/main/frontend/components/reference.cljs @@ -12,7 +12,6 @@ [frontend.state :as state] [frontend.ui :as ui] [frontend.util :as util] - [logseq.graph-parser.util :as gp-util] [medley.core :as medley] [rum.core :as rum])) @@ -83,7 +82,7 @@ default-collapsed? (>= (count refed-blocks-ids) threshold) filters-atom (get state ::filters) filter-state (rum/react filters-atom) - block? (gp-util/uuid-string? page-name) + block? (util/uuid-string? page-name) block-id (and block? (uuid page-name)) page-name (string/lower-case page-name) journal? (date/valid-journal-title? (string/capitalize page-name)) @@ -170,7 +169,7 @@ (ui/catch-error (ui/component-error "Linked References: Unexpected error") (ui/lazy-visible - (if (or sidebar? (gp-util/uuid-string? page-name)) + (if (or sidebar? (util/uuid-string? page-name)) nil "loading references...") (fn [] diff --git a/src/main/frontend/components/search.cljs b/src/main/frontend/components/search.cljs index 978b6bdd1ee..94bbd926852 100644 --- a/src/main/frontend/components/search.cljs +++ b/src/main/frontend/components/search.cljs @@ -20,7 +20,6 @@ [clojure.string :as string] [frontend.context.i18n :refer [t]] [frontend.date :as date] - [logseq.graph-parser.util :as gp-util] [reitit.frontend.easy :as rfe] [frontend.modules.shortcut.core :as shortcut])) @@ -33,7 +32,7 @@ lc-content (util/search-normalize content) lc-q (util/search-normalize q)] (if (and (string/includes? lc-content lc-q) - (not (gp-util/safe-re-find #" " q))) + (not (util/safe-re-find #" " q))) (let [i (string/index-of lc-content lc-q) [before after] [(subs content 0 i) (subs content (+ i (count q)))]] [:div diff --git a/src/main/frontend/config.cljs b/src/main/frontend/config.cljs index 7d3f09865a9..97d0bd952d2 100644 --- a/src/main/frontend/config.cljs +++ b/src/main/frontend/config.cljs @@ -99,7 +99,7 @@ (def mobile? (when-not util/node-test? - (gp-util/safe-re-find #"Mobi" js/navigator.userAgent))) + (util/safe-re-find #"Mobi" js/navigator.userAgent))) ;; TODO: protocol design for future formats support diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index c441628fea9..693fd961e76 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -912,7 +912,7 @@ (defn get-page [page-name] - (if (gp-util/uuid-string? page-name) + (if (util/uuid-string? page-name) (db-utils/entity [:block/uuid (uuid page-name)]) (db-utils/entity [:block/name (util/page-name-sanity-lc page-name)]))) @@ -1221,7 +1221,7 @@ (defn get-referenced-blocks-ids [page-name-or-block-uuid] - (if (gp-util/uuid-string? (str page-name-or-block-uuid)) + (if (util/uuid-string? (str page-name-or-block-uuid)) (let [id (uuid page-name-or-block-uuid)] (get-block-referenced-blocks-ids id)) (get-page-referenced-blocks-ids page-name-or-block-uuid))) diff --git a/src/main/frontend/db/query_dsl.cljs b/src/main/frontend/db/query_dsl.cljs index af90dbaa575..2710d912a2a 100644 --- a/src/main/frontend/db/query_dsl.cljs +++ b/src/main/frontend/db/query_dsl.cljs @@ -14,8 +14,7 @@ [frontend.db.rules :as rules] [frontend.template :as template] [logseq.graph-parser.text :as text] - [frontend.util :as util] - [logseq.graph-parser.util :as gp-util])) + [frontend.util :as util])) ;; Query fields: @@ -449,7 +448,7 @@ Some bindings in this fn: (remove string/blank?) (map (fn [x] (if (or (contains? #{"+" "-"} (first x)) - (and (gp-util/safe-re-find #"\d" (first x)) + (and (util/safe-re-find #"\d" (first x)) (some #(string/ends-with? x %) ["y" "m" "d" "h" "min"]))) (keyword (name x)) x))) diff --git a/src/main/frontend/db/query_react.cljs b/src/main/frontend/db/query_react.cljs index a2bc02c684a..c913bd67325 100644 --- a/src/main/frontend/db/query_react.cljs +++ b/src/main/frontend/db/query_react.cljs @@ -12,7 +12,6 @@ [frontend.state :as state] [logseq.graph-parser.text :as text] [frontend.util :as util] - [logseq.graph-parser.util :as gp-util] [lambdaisland.glogi :as log])) (defn resolve-input @@ -32,12 +31,12 @@ ;; This sometimes runs when there isn't a current page e.g. :home route (some-> (state/get-current-page) string/lower-case) (and (keyword? input) - (gp-util/safe-re-find #"^\d+d(-before)?$" (name input))) + (util/safe-re-find #"^\d+d(-before)?$" (name input))) (let [input (name input) days (util/parse-int (subs input 0 (dec (count input))))] (date->int (t/minus (t/today) (t/days days)))) (and (keyword? input) - (gp-util/safe-re-find #"^\d+d(-after)?$" (name input))) + (util/safe-re-find #"^\d+d(-after)?$" (name input))) (let [input (name input) days (util/parse-int (subs input 0 (dec (count input))))] (date->int (t/plus (t/today) (t/days days)))) diff --git a/src/main/frontend/db/react.cljs b/src/main/frontend/db/react.cljs index 4508bee685b..25572352a02 100644 --- a/src/main/frontend/db/react.cljs +++ b/src/main/frontend/db/react.cljs @@ -10,7 +10,6 @@ [frontend.db.utils :as db-utils] [frontend.state :as state] [frontend.util :as util :refer [react]] - [logseq.graph-parser.util :as gp-util] [cljs.spec.alpha :as s] [clojure.core.async :as async])) @@ -230,7 +229,7 @@ affected-keys (concat (mapcat (fn [block-id] - (let [block-id (if (and (string? block-id) (gp-util/uuid-string? block-id)) + (let [block-id (if (and (string? block-id) (util/uuid-string? block-id)) [:block/uuid block-id] block-id)] (when-let [block (db-utils/entity block-id)] diff --git a/src/main/frontend/extensions/html_parser.cljs b/src/main/frontend/extensions/html_parser.cljs index 3ca5c5ff511..0e7fb1e4a9a 100644 --- a/src/main/frontend/extensions/html_parser.cljs +++ b/src/main/frontend/extensions/html_parser.cljs @@ -4,7 +4,6 @@ [clojure.walk :as walk] [frontend.config :as config] [frontend.util :as util] - [logseq.graph-parser.util :as gp-util] [hickory.core :as hickory])) (defonce *inside-pre? (atom false)) @@ -75,7 +74,7 @@ :h6 (block-transform 6 children) :a (let [href (:href attrs) label (map-join children) - has-img-tag? (gp-util/safe-re-find #"\[:img" (str x))] + has-img-tag? (util/safe-re-find #"\[:img" (str x))] (if has-img-tag? (export-hiccup x) (case format diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index ac24c325c5d..9a89df0137b 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -257,7 +257,7 @@ (defn- another-block-with-same-id-exists? [current-id block-id] (and (string? block-id) - (gp-util/uuid-string? block-id) + (util/uuid-string? block-id) (not= current-id (cljs.core/uuid block-id)) (db/entity [:block/uuid (cljs.core/uuid block-id)]))) @@ -338,7 +338,7 @@ (if (and (state/enable-timetracking?) (not= (:block/content block) value)) (let [format (:block/format block) - new-marker (last (gp-util/safe-re-find (marker/marker-pattern format) (or value ""))) + new-marker (last (util/safe-re-find (marker/marker-pattern format) (or value ""))) new-value (with-marker-time value block format new-marker (:block/marker block))] @@ -483,10 +483,10 @@ (let [current-page (state/get-current-page) block-id (or (and (:id config) - (gp-util/uuid-string? (:id config)) + (util/uuid-string? (:id config)) (:id config)) (and current-page - (gp-util/uuid-string? current-page) + (util/uuid-string? current-page) current-page))] (= uuid (and block-id (medley/uuid block-id))))) @@ -1147,7 +1147,7 @@ [] (when-let [page (get-nearest-page)] (let [page-name (string/lower-case page) - block? (gp-util/uuid-string? page-name)] + block? (util/uuid-string? page-name)] (when-let [page (db/get-page page-name)] (if block? (state/sidebar-add-block! @@ -1177,7 +1177,7 @@ (let [page (state/get-current-page) block-id (and (string? page) - (gp-util/uuid-string? page) + (util/uuid-string? page) (medley/uuid page))] (when block-id (let [block-parent (db/get-block-parent block-id)] @@ -2040,7 +2040,7 @@ (defn- last-top-level-child? [{:keys [id]} current-node] (when id - (when-let [entity (if (gp-util/uuid-string? (str id)) + (when-let [entity (if (util/uuid-string? (str id)) (db/entity [:block/uuid (uuid id)]) (db/entity [:block/name (util/page-name-sanity-lc id)]))] (= (:block/uuid entity) (tree/-get-parent-id current-node))))) @@ -2867,7 +2867,7 @@ (string/join "\n" (mapv (fn [p] (->> (string/trim p) ((fn [p] - (if (gp-util/safe-re-find (if (= format :org) + (if (util/safe-re-find (if (= format :org) #"\s*\*+\s+" #"\s*-\s+") p) p @@ -2932,9 +2932,9 @@ ;; from external (let [format (or (db/get-page-format (state/get-current-page)) :markdown)] (match [format - (nil? (gp-util/safe-re-find #"(?m)^\s*(?:[-+*]|#+)\s+" text)) - (nil? (gp-util/safe-re-find #"(?m)^\s*\*+\s+" text)) - (nil? (gp-util/safe-re-find #"(?:\r?\n){2,}" text))] + (nil? (util/safe-re-find #"(?m)^\s*(?:[-+*]|#+)\s+" text)) + (nil? (util/safe-re-find #"(?m)^\s*\*+\s+" text)) + (nil? (util/safe-re-find #"(?:\r?\n){2,}" text))] [:markdown false _ _] (paste-text-parseable format text) @@ -3219,7 +3219,7 @@ :or {collapse? false expanded? false incremental? true root-block nil}}] (when-let [page (or (state/get-current-page) (date/today))] - (let [block? (gp-util/uuid-string? page) + (let [block? (util/uuid-string? page) block-id (or root-block (and block? (uuid page))) blocks (if block-id (db/get-block-and-children (state/get-current-repo) block-id) diff --git a/src/main/frontend/handler/graph.cljs b/src/main/frontend/handler/graph.cljs index b923fd6c9ff..510b35036ed 100644 --- a/src/main/frontend/handler/graph.cljs +++ b/src/main/frontend/handler/graph.cljs @@ -4,8 +4,7 @@ [frontend.db :as db] [frontend.db.default :as default-db] [frontend.state :as state] - [frontend.util :as util] - [logseq.graph-parser.util :as gp-util])) + [frontend.util :as util])) (defn- build-links [links] @@ -46,7 +45,7 @@ ;; slow (defn- uuid-or-asset? [id] - (or (gp-util/uuid-string? id) + (or (util/uuid-string? id) (string/starts-with? id "../assets/") (= id "..") (string/starts-with? id "assets/") diff --git a/src/main/frontend/handler/page.cljs b/src/main/frontend/handler/page.cljs index 86b98cb7734..ef16780cb80 100644 --- a/src/main/frontend/handler/page.cljs +++ b/src/main/frontend/handler/page.cljs @@ -637,7 +637,7 @@ (->> (db/get-all-pages repo) (remove (fn [p] (let [name (:block/name p)] - (or (gp-util/uuid-string? name) + (or (util/uuid-string? name) (gp-config/draw? name) (db/built-in-pages-names (string/upper-case name)))))) (common-handler/fix-pages-timestamps))) @@ -691,7 +691,7 @@ chosen (if (string/starts-with? chosen "New page: ") ;; FIXME: What if a page named "New page: XXX"? (subs chosen 10) chosen) - chosen (if (and (gp-util/safe-re-find #"\s+" chosen) (not wrapped?)) + chosen (if (and (util/safe-re-find #"\s+" chosen) (not wrapped?)) (util/format "[[%s]]" chosen) chosen) q (if @editor-handler/*selected-text "" q) diff --git a/src/main/frontend/handler/route.cljs b/src/main/frontend/handler/route.cljs index 53bfc9147a3..241cf4b9db5 100644 --- a/src/main/frontend/handler/route.cljs +++ b/src/main/frontend/handler/route.cljs @@ -9,7 +9,6 @@ [frontend.state :as state] [logseq.graph-parser.text :as text] [frontend.util :as util] - [logseq.graph-parser.util :as gp-util] [medley.core :as medley] [reitit.frontend.easy :as rfe])) @@ -79,7 +78,7 @@ "Create a new page" :page (let [name (:name path-params) - block? (gp-util/uuid-string? name)] + block? (util/uuid-string? name)] (if block? (if-let [block (db/entity [:block/uuid (medley/uuid name)])] (let [content (text/remove-level-spaces (:block/content block) diff --git a/src/main/frontend/handler/ui.cljs b/src/main/frontend/handler/ui.cljs index 53c63f0ef0a..f3595301f27 100644 --- a/src/main/frontend/handler/ui.cljs +++ b/src/main/frontend/handler/ui.cljs @@ -14,7 +14,6 @@ [clojure.string :as string] [rum.core :as rum] [frontend.mobile.util :as mobile] - [logseq.graph-parser.util :as gp-util] [electron.ipc :as ipc])) (defn- get-css-var-value @@ -112,7 +111,7 @@ (let [id (and (> (count fragment) 36) (subs fragment (- (count fragment) 36)))] - (if (and id (gp-util/uuid-string? id)) + (if (and id (util/uuid-string? id)) (let [elements (array-seq (js/document.getElementsByClassName id))] (when (first elements) (util/scroll-to-element (gobj/get (first elements) "id"))) diff --git a/src/main/frontend/modules/outliner/tree.cljs b/src/main/frontend/modules/outliner/tree.cljs index 74bde33cc7a..65cd4073cc9 100644 --- a/src/main/frontend/modules/outliner/tree.cljs +++ b/src/main/frontend/modules/outliner/tree.cljs @@ -1,6 +1,6 @@ (ns frontend.modules.outliner.tree (:require [frontend.db :as db] - [logseq.graph-parser.util :as gp-util] + [frontend.util :as util] [clojure.string :as string] [frontend.state :as state])) @@ -45,7 +45,7 @@ (defn- get-root-and-page [repo root-id] (if (string? root-id) - (if (gp-util/uuid-string? root-id) + (if (util/uuid-string? root-id) [false (db/entity repo [:block/uuid (uuid root-id)])] [true (db/entity repo [:block/name (string/lower-case root-id)])]) [false root-id])) diff --git a/src/main/frontend/security.cljs b/src/main/frontend/security.cljs index d62af3637a8..6936b5fb7b8 100644 --- a/src/main/frontend/security.cljs +++ b/src/main/frontend/security.cljs @@ -1,6 +1,6 @@ (ns frontend.security (:require [clojure.walk :as walk] - [logseq.graph-parser.util :as gp-util])) + [frontend.util :as util])) ;; To prevent from cross-site scripting vulnerability, we should add security checks for both hiccup and raw html. ;; Hiccup: [:a {:href "javascript:alert('hei')"} "click me"] @@ -12,7 +12,7 @@ (= :a (first f)) (:href (second f)) (:href (second f)) - (gp-util/safe-re-find #"(?i)javascript" (:href (second f))))) + (util/safe-re-find #"(?i)javascript" (:href (second f))))) (defn remove-javascript-links-in-href [hiccup] diff --git a/src/main/frontend/state.cljs b/src/main/frontend/state.cljs index 8f335fb2ed3..ee4e33f8caa 100644 --- a/src/main/frontend/state.cljs +++ b/src/main/frontend/state.cljs @@ -13,7 +13,6 @@ [goog.object :as gobj] [promesa.core :as p] [rum.core :as rum] - [logseq.graph-parser.util :as gp-util] [frontend.mobile.util :as mobile-util])) (defonce ^:large-vars/data-var state @@ -452,7 +451,7 @@ (or (when-let [workflow (:preferred-workflow (get-config))] (let [workflow (name workflow)] - (if (gp-util/safe-re-find #"now|NOW" workflow) + (if (util/safe-re-find #"now|NOW" workflow) :now :todo))) (get-in @state [:me :preferred_workflow] :now)))) diff --git a/src/main/frontend/util.cljc b/src/main/frontend/util.cljc index 5ec8248d0c3..2ff42982643 100644 --- a/src/main/frontend/util.cljc +++ b/src/main/frontend/util.cljc @@ -39,6 +39,23 @@ #?(:cljs (defn app-scroll-container-node [] (gdom/getElement "main-content-container"))) +#?(:cljs + (defn safe-re-find + [pattern s] + (when-not (string? s) + ;; TODO: sentry + (js/console.trace)) + (when (string? s) + (re-find pattern s)))) + +#?(:cljs + (do + (def uuid-pattern "[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}") + (defonce exactly-uuid-pattern (re-pattern (str "(?i)^" uuid-pattern "$"))) + (defn uuid-string? + [s] + (safe-re-find exactly-uuid-pattern s)))) + #?(:cljs (defn ios? [] @@ -55,7 +72,7 @@ (defn mobile? [] (when-not node-test? - (gp-util/safe-re-find #"Mobi" js/navigator.userAgent)))) + (safe-re-find #"Mobi" js/navigator.userAgent)))) #?(:cljs (defn electron? @@ -349,7 +366,7 @@ #?(:cljs (defn scroll-to-element [elem-id] - (when-not (gp-util/safe-re-find #"^/\d+$" elem-id) + (when-not (safe-re-find #"^/\d+$" elem-id) (when elem-id (when-let [elem (gdom/getElement elem-id)] (.scroll (app-scroll-container-node) @@ -829,8 +846,8 @@ [] (let [user-agent js/navigator.userAgent vendor js/navigator.vendor] - (and (gp-util/safe-re-find #"Chrome" user-agent) - (gp-util/safe-re-find #"Google Inc" vendor))))) + (and (safe-re-find #"Chrome" user-agent) + (safe-re-find #"Google Inc" vendor))))) #?(:cljs (defn indexeddb-check? @@ -871,7 +888,7 @@ [block-id] (when block-id (let [block-id (str block-id)] - (when (gp-util/uuid-string? block-id) + (when (uuid-string? block-id) (first (array-seq (js/document.getElementsByClassName block-id)))))))) #?(:cljs @@ -890,7 +907,7 @@ (do (defn include-windows-reserved-chars? [s] - (gp-util/safe-re-find windows-reserved-chars s)) + (safe-re-find windows-reserved-chars s)) (defn create-title-property? [s] diff --git a/src/main/frontend/util/drawer.cljs b/src/main/frontend/util/drawer.cljs index a07ec2a225c..27129de6132 100644 --- a/src/main/frontend/util/drawer.cljs +++ b/src/main/frontend/util/drawer.cljs @@ -1,7 +1,6 @@ (ns frontend.util.drawer (:require [clojure.string :as string] [frontend.util :as util] - [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.property :as gp-property] [frontend.format.mldoc :as mldoc])) @@ -88,8 +87,8 @@ (defn contains-logbook? [content] - (and (gp-util/safe-re-find (re-pattern (str "(?i)" logbook-start)) content) - (gp-util/safe-re-find (re-pattern (str "(?i)" drawer-end)) content))) + (and (util/safe-re-find (re-pattern (str "(?i)" logbook-start)) content) + (util/safe-re-find (re-pattern (str "(?i)" drawer-end)) content))) ;; TODO: DRY (defn remove-logbook diff --git a/src/main/frontend/util/marker.cljs b/src/main/frontend/util/marker.cljs index 35f73c1af67..73b0c812018 100644 --- a/src/main/frontend/util/marker.cljs +++ b/src/main/frontend/util/marker.cljs @@ -1,7 +1,6 @@ (ns frontend.util.marker (:require [clojure.string :as string] - [frontend.util :as util] - [logseq.graph-parser.util :as gp-util])) + [frontend.util :as util])) (defn marker-pattern [format] (re-pattern @@ -21,7 +20,7 @@ (if-let [matches (seq (util/re-pos new-line-re-pattern content))] (let [[start-pos content] (last matches)] (+ start-pos (count content))) - (count (gp-util/safe-re-find re-pattern content))) + (count (util/safe-re-find re-pattern content))) new-content (str (subs content 0 pos) (string/replace-first (subs content pos) @@ -60,6 +59,6 @@ (let [content (string/triml content) new-marker (or new-marker (cycle-marker-state (or marker - (last (gp-util/safe-re-find (marker-pattern format) content))) ; Returns the last matching group (last vec) + (last (util/safe-re-find (marker-pattern format) content))) ; Returns the last matching group (last vec) preferred-workflow))] [(add-or-update-marker content format new-marker) new-marker])) diff --git a/src/main/frontend/util/priority.cljs b/src/main/frontend/util/priority.cljs index edf1a83afcb..106c468a7e7 100644 --- a/src/main/frontend/util/priority.cljs +++ b/src/main/frontend/util/priority.cljs @@ -1,13 +1,12 @@ (ns frontend.util.priority (:require [clojure.string :as string] [frontend.util :as util] - [logseq.graph-parser.util :as gp-util] [frontend.util.marker :as marker])) (defn cycle-priority-state [content] (let [priority-reg #"\[#([ABC]{1})\]\s{1}" - priority (last (gp-util/safe-re-find priority-reg content)) + priority (last (util/safe-re-find priority-reg content)) next-priority (case priority "A" "B" @@ -29,7 +28,7 @@ (if-let [matches (seq (util/re-pos new-line-re-pattern content))] (let [[start-pos content] (last matches)] (+ start-pos (count content))) - (count (gp-util/safe-re-find re-pattern content))) + (count (util/safe-re-find re-pattern content))) skip-marker-pos (if-let [matches (seq (util/re-pos marker/bare-marker-pattern (subs content skip-hash-pos)))] (let [[start-pos content] (last matches)] diff --git a/src/main/frontend/util/property.cljs b/src/main/frontend/util/property.cljs index c9c6138ab95..57c5f82e86f 100644 --- a/src/main/frontend/util/property.cljs +++ b/src/main/frontend/util/property.cljs @@ -45,28 +45,28 @@ [line] (boolean (and (string? line) - (gp-util/safe-re-find #"^\s?[^ ]+:: " line)))) + (util/safe-re-find #"^\s?[^ ]+:: " line)))) (defn front-matter-property? [line] (boolean (and (string? line) - (gp-util/safe-re-find #"^\s*[^ ]+: " line)))) + (util/safe-re-find #"^\s*[^ ]+: " line)))) (defn get-property-key [line format] (and (string? line) (when-let [key (last (if (= format :org) - (gp-util/safe-re-find #"^\s*:([^: ]+): " line) - (gp-util/safe-re-find #"^\s*([^ ]+):: " line)))] + (util/safe-re-find #"^\s*:([^: ]+): " line) + (util/safe-re-find #"^\s*([^ ]+):: " line)))] (keyword key)))) (defn org-property? [line] (boolean (and (string? line) - (gp-util/safe-re-find #"^\s*:[^: ]+: " line) + (util/safe-re-find #"^\s*:[^: ]+: " line) (when-let [key (get-property-key line :org)] (not (contains? #{:PROPERTIES :END} key)))))) diff --git a/src/main/logseq/graph_parser/util.cljs b/src/main/logseq/graph_parser/util.cljs index e6097aa264e..f708d5dc1f7 100644 --- a/src/main/logseq/graph_parser/util.cljs +++ b/src/main/logseq/graph_parser/util.cljs @@ -8,6 +8,7 @@ (defonce exactly-uuid-pattern (re-pattern (str "(?i)^" uuid-pattern "$"))) (defn safe-re-find + "Copy of frontend.util/safe-re-find. Too basic to couple to main app" [pattern s] (when-not (string? s) ;; TODO: sentry @@ -16,6 +17,7 @@ (re-find pattern s))) (defn uuid-string? + "Copy of frontend.util/uuid-string?. Too basic to couple to main app" [s] (safe-re-find exactly-uuid-pattern s)) From 9b9fcceddb64f8797c2ae885ccf99e8b5307614c Mon Sep 17 00:00:00 2001 From: queeup Date: Sat, 14 May 2022 14:52:43 +0300 Subject: [PATCH 52/58] fix: update Turkish translation --- src/main/frontend/dicts.cljc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/dicts.cljc b/src/main/frontend/dicts.cljc index 98ad39f53f7..53b10a3f07c 100644 --- a/src/main/frontend/dicts.cljc +++ b/src/main/frontend/dicts.cljc @@ -3542,7 +3542,7 @@ :settings-page/spell-checker "Yazım denetleyici" :settings-page/auto-updater "Otomatik güncelleme" :settings-page/disable-sentry "Kullanım verilerini ve tanılamayı Logseq'e gönderin" - :settings-page/preferred-outdenting "mantıksal girinti" + :settings-page/preferred-outdenting "Mantıksal girinti" :settings-page/custom-date-format "Tercih edilen tarih biçimi" :settings-page/preferred-file-format "Tercih edilen dosya biçimi" :settings-page/preferred-workflow "Tercih edilen iş akışı" @@ -3551,6 +3551,7 @@ :settings-page/enable-tooltip "Araç ipuçları" :settings-page/enable-journals "Günlük" :settings-page/enable-all-pages-public "Yayımlanan tüm sayfaları herkese açık yap" + :settings-page/enable-encryption "Şifreleme" :settings-page/customize-shortcuts "Klavye kısayolları" :settings-page/shortcut-settings "Kısayolları özelleştir" :settings-page/home-default-page "Varsayılan ana sayfayı ayarla" From 0f23c7b7b5319079484fc97998dbc24a92520c1a Mon Sep 17 00:00:00 2001 From: Eduardo Vedes Date: Sun, 15 May 2022 02:14:45 +0100 Subject: [PATCH 53/58] fix: render preview page on-hover --- src/main/frontend/ui.cljs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/frontend/ui.cljs b/src/main/frontend/ui.cljs index cb33c3f03f1..ace07b5d097 100644 --- a/src/main/frontend/ui.cljs +++ b/src/main/frontend/ui.cljs @@ -829,11 +829,9 @@ :open (if manual open? @*mounted?) :trigger (if manual "manual" "mouseenter focus") ;; See https://github.com/tvkhoa/react-tippy/issues/13 - :popperOptions (if fixed-position? - {:modifiers {:flip {:enabled false} - :hide {:enabled false} - :preventOverflow {:enabled false}}} - {}) + :popperOptions {:modifiers {:flip {:enabled (not fixed-position?)} + :hide {:enabled false} + :preventOverflow {:enabled false}}} :onShow #(reset! *mounted? true) :onHide #(reset! *mounted? false)} opts) From 9ad2ce417bf2ce6ace4bf6589babb7d41c2e104e Mon Sep 17 00:00:00 2001 From: charlie Date: Sat, 14 May 2022 16:15:24 +0800 Subject: [PATCH 54/58] fix(ui): wrong position of page preview tip #5268 --- src/main/frontend/components/block.cljs | 79 ++++++++++++++++--------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 26de8aac57f..c914d85ba74 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -457,41 +457,66 @@ (rum/defc page-preview-trigger [{:keys [children sidebar? tippy-position tippy-distance fixed-position? open? manual?] :as config} page-name] - (let [page-name (util/page-name-sanity-lc page-name) + (let [*tippy-ref (rum/create-ref) + page-name (util/page-name-sanity-lc page-name) redirect-page-name (or (model/get-redirect-page-name page-name (:block/alias? config)) page-name) page-original-name (model/get-page-original-name redirect-page-name) - html-template (fn [] - (when redirect-page-name - [:div.tippy-wrapper.overflow-y-auto.p-4 - {:style {:width 600 - :text-align "left" - :font-weight 500 - :max-height 600 - :padding-bottom 64}} - (if (and (string? page-original-name) (string/includes? page-original-name "/")) - [:div.my-2 - (->> - (for [page (string/split page-original-name #"/")] - (when (and (string? page) page) - (page-reference false page {} nil))) - (interpose [:span.mx-2.opacity-30 "/"]))] - [:h2.font-bold.text-lg (if (= page-name redirect-page-name) - page-original-name - [:span - [:span.text-sm.mr-2 "Alias:"] - page-original-name])]) - (let [page (db/entity [:block/name (util/page-name-sanity-lc redirect-page-name)])] - (editor-handler/insert-first-page-block-if-not-exists! redirect-page-name {:redirect? false}) - (when-let [f (state/get-page-blocks-cp)] - (f (state/get-current-repo) page {:sidebar? sidebar? :preview? true})))]))] + html-template (rum/defc _ [] + (let [*el-popup (rum/use-ref nil)] + + (rum/use-effect! + (fn [] + (let [el-popup (rum/deref *el-popup) + cb (fn [^js e] + (when-not (:editor/editing? @state/state) + ;; Esc + (and (= e.which 27) + (when-let [tp (rum/deref *tippy-ref)] + (.hideTooltip tp)))))] + + (js/setTimeout #(.focus el-popup)) + (.addEventListener el-popup "keyup" cb) + #(.removeEventListener el-popup "keyup" cb))) + []) + + (when redirect-page-name + [:div.tippy-wrapper.overflow-y-auto.p-4.outline-none + {:ref *el-popup + :tab-index -1 + :style {:width 600 + :text-align "left" + :font-weight 500 + :max-height 600 + :padding-bottom 64}} + (if (and (string? page-original-name) (string/includes? page-original-name "/")) + [:div.my-2 + (->> + (for [page (string/split page-original-name #"/")] + (when (and (string? page) page) + (page-reference false page {} nil))) + (interpose [:span.mx-2.opacity-30 "/"]))] + [:h2.font-bold.text-lg (if (= page-name redirect-page-name) + page-original-name + [:span + [:span.text-sm.mr-2 "Alias:"] + page-original-name])]) + (let [page (db/entity [:block/name (util/page-name-sanity-lc redirect-page-name)])] + (editor-handler/insert-first-page-block-if-not-exists! redirect-page-name {:redirect? false}) + (when-let [f (state/get-page-blocks-cp)] + (f (state/get-current-repo) page {:sidebar? sidebar? :preview? true})))])))] + (if (or (not manual?) open?) - (ui/tippy {:html html-template + (ui/tippy {:ref *tippy-ref + :html html-template :interactive true :delay [1000, 100] :fixed-position? fixed-position? :position (or tippy-position "top") - :distance (or tippy-distance 10)} + :distance (or tippy-distance 10) + :popperOptions {:modifiers {:preventOverflow + {:enabled true + :boundariesElement "viewport"}}}} children) children))) From 5a8b4a6dc7f7d68bf37b09bd7261111b480bd705 Mon Sep 17 00:00:00 2001 From: charlie Date: Sat, 14 May 2022 16:38:05 +0800 Subject: [PATCH 55/58] fix: lint --- src/main/frontend/components/block.cljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index c914d85ba74..00039b883d9 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -462,7 +462,7 @@ redirect-page-name (or (model/get-redirect-page-name page-name (:block/alias? config)) page-name) page-original-name (model/get-page-original-name redirect-page-name) - html-template (rum/defc _ [] + html-template #_:clj-kondo/ignore (rum/defc _ [] (let [*el-popup (rum/use-ref nil)] (rum/use-effect! From 12eecde29fe464d1f84baef8b0a153a0646bd8b2 Mon Sep 17 00:00:00 2001 From: charlie Date: Sat, 14 May 2022 21:06:40 +0800 Subject: [PATCH 56/58] fix(lint): carve --- src/main/frontend/components/block.cljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 00039b883d9..9c21beae101 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -462,7 +462,7 @@ redirect-page-name (or (model/get-redirect-page-name page-name (:block/alias? config)) page-name) page-original-name (model/get-page-original-name redirect-page-name) - html-template #_:clj-kondo/ignore (rum/defc _ [] + _ #_:clj-kondo/ignore (rum/defc html-template [] (let [*el-popup (rum/use-ref nil)] (rum/use-effect! From 7dac9e2748706369ce48e194380616ef7593329b Mon Sep 17 00:00:00 2001 From: Alex L Date: Fri, 13 May 2022 23:15:50 +0200 Subject: [PATCH 57/58] Improved Italian localization Improved localization, fixed many grammar mistakes. Also "graph" in Logseq context is "grafo" not "diagramma" or "grafico": search Google Images in Italian for confirmation. Also "Graph Theory" (branch of Math) is known as "Teoria dei Grafi" in Italian. --- src/main/frontend/dicts.cljc | 180 +++++++++++++++++------------------ 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/src/main/frontend/dicts.cljc b/src/main/frontend/dicts.cljc index 53b10a3f07c..0e4ec4727cd 100644 --- a/src/main/frontend/dicts.cljc +++ b/src/main/frontend/dicts.cljc @@ -3073,28 +3073,28 @@ :default "tutorial-en.md") :tutorial/dummy-notes #?(:cljs (rc/inline "dummy-notes-en.md") :default "dummy-notes-en.md") - :on-boarding/demo-graph "Questo è un diagramma dimostrativo, le modifiche non saranno salvate finchè non aprirai una cartella locale." - :on-boarding/add-graph "Aggiungi un diagramma" + :on-boarding/demo-graph "Questo è un grafo dimostrativo, le modifiche non saranno salvate finché non aprirai una cartella locale." + :on-boarding/add-graph "Aggiungi un grafo" :on-boarding/open-local-dir "Apri una cartella locale" - :on-boarding/new-graph-desc-1 "Logseq supporta sia Markdown che Org-mode. Puoi aprire una cartella esistente o crearne una nuova sul tuo dispositivo. I tuoi dati saranno salvati in questo dispositivo." + :on-boarding/new-graph-desc-1 "Logseq supporta sia Markdown che Org-mode. Puoi aprire una cartella esistente o crearne una nuova sul tuo dispositivo. I tuoi dati saranno salvati solo sul tuo dispositivo." :on-boarding/new-graph-desc-2 "Dopo che hai aperto la tua cartella, saranno create al suo interno tre cartelle:" - :on-boarding/new-graph-desc-3 "/journals - conserva le tue pagine quotidiane" - :on-boarding/new-graph-desc-4 "/pages - conserva le altre pagine" - :on-boarding/new-graph-desc-5 "/logseq - conserva la configurazione, custom.css, e alcuni metadati." - :help/start "Per cominciare" - :help/about "Riguardo a Logseq" + :on-boarding/new-graph-desc-3 "/journals - contiene le pagine del diario giornaliero" + :on-boarding/new-graph-desc-4 "/pages - contiene le altre pagine" + :on-boarding/new-graph-desc-5 "/logseq - contiene i dati di configurazione, custom.css, e alcuni metadati." + :help/start "Per iniziare" + :help/about "Informazioni su Logseq" :help/roadmap "Roadmap" - :help/bug "Riportare un errore" - :help/feature "Richiedere una funzione" + :help/bug "Segnala un problema" + :help/feature "Richiedi una funzionalità" :help/changelog "Registro delle modifiche" :help/blog "blog di Logseq" :help/docs "Documentazione" :help/privacy "Politica sulla riservatezza" - :help/terms "Termini" + :help/terms "Termini di Servizio" :help/community "Comunità su Discord" - :help/awesome-logseq "Fantastico Logseq" + :help/awesome-logseq "Awesome Logseq" :help/shortcuts "Scorciatoie da tastiera" - :help/shortcuts-triggers "Inneschi" + :help/shortcuts-triggers "Attivazione delle scorciatoie" :help/shortcut "Scorciatoia" :help/slash-autocomplete "Barra di completamento automatico" :help/block-content-autocomplete "Autocompletamento del contenuto di blocco" @@ -3106,12 +3106,12 @@ :undo "Annulla" :redo "Ripeti" :general "Generale" - :more "Ancora" + :more "Altro" :search/result-for "Cerca i risultati per " :search/items "oggetti" - :search/page-names "Cerca nomi di pagina" + :search/page-names "Cerca pagine per nome" :help/context-menu "Menu contestuale del blocco" - :help/fold-unfold "Comprimi/Decomprimi blocchi (quando non sei in modalità di modifica)" + :help/fold-unfold "Comprimi/Espandi blocchi (quando non sei in modalità di modifica)" :help/markdown-syntax "Sintassi Markdown" :help/org-mode-syntax "Sintassi Org mode" :bold "Grassetto" @@ -3131,17 +3131,17 @@ :right-side-bar/block-ref "Riferimento di Blocco" :right-side-bar/graph-view "Vista del grafico" :right-side-bar/all-pages "Tutte le pagine" - :right-side-bar/flashcards "Carte flash" + :right-side-bar/flashcards "Flashcard" :right-side-bar/new-page "Nuova pagina" - :left-side-bar/journals "Diari" + :left-side-bar/journals "Diario" :left-side-bar/new-page "Nuova pagina" :left-side-bar/nav-favorites "Preferiti" - :left-side-bar/nav-shortcuts "Scrciatoie" + :left-side-bar/nav-shortcuts "Scorciatoie" :left-side-bar/nav-recent-pages "Recenti" - :format/preferred-mode "Qual'è la tua modalità preferita?" + :format/preferred-mode "Qual è la tua modalità preferita?" :format/markdown "Markdown" :format/org-mode "Org mode" - :reference/linked "Riferimento collegato" + :reference/linked "Riferimenti collegati" :reference/unlinked-ref "Riferimenti non collegati" :page/presentation-mode "Presentazione" :page/edit-properties-placeholder "Proprietà" @@ -3152,41 +3152,41 @@ :page/copy-to-json "Copia l'intera pagina in JSON" :page/rename "Rinomina pagina" :page/open-in-finder "Apri nella cartella" - :page/open-with-default-app "Apri con la app predefinita" + :page/open-with-default-app "Apri con l'app predefinita" :page/action-publish "Pubblica" - :page/make-public "Rendi pubblico per la pubblicazione" + :page/make-public "Segna come pubblico per la pubblicazione" :page/version-history "Controlla la cronologia della pagina" :page/open-backup-directory "Apri la cartella dei backup delle pagine" :page/file-sync-versions "Versioni delle pagine" - :page/make-private "Rendi privato" + :page/make-private "Segna come privato" :page/delete "Elimina pagina" :page/add-to-favorites "Aggiungi ai Preferiti" :page/unfavorite "Rimuovi la pagina dai Preferiti" - :page/show-journals "Mostra Diari" + :page/show-journals "Mostra diario" :page/show-name "Mostra il nome della pagina" :page/hide-name "Nascondi il nome della pagina" :block/name "Nome pagina" :page/last-modified "Ultima modifica alle" - :page/new-title "Qual'è il titolo della tua nuova pagina?" + :page/new-title "Qual è il titolo della tua nuova pagina?" :page/earlier "Prima" :page/no-more-journals "Non ci sono altri diari" :page/copy-page-url "Copia URL pagina" - :journal/multiple-files-with-different-formats "Sembra che tu abbia più file Diario (con formati diversi) per lo stesso mese, per favore conserva un solo file journal per ogni mese." + :journal/multiple-files-with-different-formats "Sembra che tu abbia più file diario (con formati diversi) per lo stesso mese, per favore conserva un solo file diario per ogni mese." :journal/go-to "Vai ai file" :file/name "Nome File" :file/file "File: " :file/last-modified-at "Ultima modifica alle" - :file/no-data "No dati" + :file/no-data "Nessun dato" :file/format-not-supported "Il formato .{1} non è supportato." :page/created-at "Creato alle" :page/updated-at "Aggiornato alle" :page/backlinks "Collegamenti a ritroso" :editor/block-search "Cerca un blocco" - :editor/image-uploading "Caricando" + :editor/image-uploading "Caricamento" :draw/invalid-file "Non ho potuto caricare questo file excalidraw" :draw/specify-title "Per favore specifica un titolo prima!" :draw/rename-success "Il file è stato rinominato con successo!" - :draw/rename-failure "La rinomina del file è fallita, ragione: " + :draw/rename-failure "La rinominazione del file è fallita, ragione: " :draw/title-placeholder "Senza titolo" :draw/save "Salva" :draw/save-changes "Salva modifiche" @@ -3197,7 +3197,7 @@ :draw/back-to-logseq "Torna a logseq" :text/image "Immagine" :asset/confirm-delete "Sei sicuro di voler eliminare questo {1}?" - :asset/physical-delete "Rimuovi anche i file (nota che non può essere ripristinato)" + :asset/physical-delete "Rimuovi anche i file (non possono essere ripristinati)" :content/copy "Copia" :content/cut "Taglia" :content/make-todos "Crea {1}" @@ -3219,19 +3219,19 @@ :settings-page/spell-checker "Correttore ortografico" :settings-page/auto-updater "Aggiornamento automatico" :settings-page/disable-sentry "Invia dati di utilizzo e diagnostica a Logseq" - :settings-page/preferred-outdenting "Distacco logico" + :settings-page/preferred-outdenting "Indentamento logico" :settings-page/custom-date-format "Formato data preferito" :settings-page/preferred-file-format "Formato file preferito" :settings-page/preferred-workflow "Flusso di lavoro preferito" :settings-page/enable-shortcut-tooltip "Abilita suggerimenti scorciatoie" :settings-page/enable-timetracking "Tracciamento del tempo" :settings-page/enable-tooltip "Suggerimenti" - :settings-page/enable-journals "Diari" + :settings-page/enable-journals "Diario" :settings-page/enable-all-pages-public "Tutte le pagine pubbliche durante la pubblicazione" - :settings-page/customize-shortcuts "Scorciatoie di tastiera" + :settings-page/customize-shortcuts "Scorciatoie da tastiera" :settings-page/shortcut-settings "Personalizza scorciatoie" :settings-page/home-default-page "Imposta la home page predefinita" - :settings-page/enable-block-time "Marche temporali sui blocchi" + :settings-page/enable-block-time "Indicatori temporali sui blocchi" :settings-page/clear-cache "Pulisci cache" :settings-page/clear "Pulisci" :settings-page/developer-mode "Modalità sviluppatore" @@ -3239,19 +3239,19 @@ :settings-page/disable-developer-mode "Disabilita modalità sviluppatore" :settings-page/developer-mode-desc "La modalità sviluppatore aiuta i contributori e gli sviluppatori di estensioni a testare le loro integrazioni con Logseq in modo più efficiente." :settings-page/current-version "Versione attuale" - :settings-page/current-graph "Diagramma attuale" + :settings-page/current-graph "Grafo attuale" :settings-page/tab-general "Generale" :settings-page/tab-editor "Editor" :settings-page/tab-shortcuts "Scorciatoie" - :settings-page/tab-version-control "Controllo versione" + :settings-page/tab-version-control "Controllo di versione" :settings-page/tab-advanced "Avanzate" - :settings-page/plugin-system "Sistema dei Plug-in" + :settings-page/plugin-system "Sistema di plugin" :settings-page/network-proxy "Proxy di rete" :logseq "Logseq" :on "ON" :more-options "Più opzioni" :to "a" - :yes "Si" + :yes "Sì" :no "No" :submit "Invia" :cancel "Annulla" @@ -3262,63 +3262,63 @@ :host "Host" :port "Porta" :re-index "Re-indicizza" - :re-index-detail "Ricostruisci il diagramma" - :re-index-multiple-windows-warning "È necessario chiudere le altre finestre prima di reindicizzare questo diagramma." - :re-index-discard-unsaved-changes-warning "Reindicizza elimina il diagramma corrente, quindi elabora nuovamente tutti i file, poiché sono attualmente archiviati su disco. Perderai le modifiche non salvate e potrebbe volerci del tempo. Continuare?" + :re-index-detail "Ricostruisci il grafo" + :re-index-multiple-windows-warning "È necessario chiudere le altre finestre prima di reindicizzare questo grafo." + :re-index-discard-unsaved-changes-warning "La reindicizzazione elimina il grafo corrente, quindi elabora nuovamente tutti i file poiché sono attualmente archiviati su disco. Perderai le modifiche non salvate e potrebbe volerci del tempo. Continuare?" :open-new-window "Nuova finestra" :sync-from-local-files "Ricarica" :sync-from-local-files-detail "Importa cambiamenti da un file locale" - :sync-from-local-changes-detected "Ricarica rileva ed elabora i file modificati sul disco e divergenti dal contenuto effettivo della pagina Logseq. Continuare?" + :sync-from-local-changes-detected "Il ricaricamento rileva ed elabora i file modificati sul disco e divergenti dal contenuto effettivo della pagina Logseq. Continuare?" :unlink "disconnetti" :search/publishing "Cerca" :search "Cerca o crea una pagina" :page-search "Cerca nella pagina corrente" - :graph-search "Cerca diagramma" + :graph-search "Cerca nel grafo" :new-page "Nuova pagina" :new-file "Nuovo file" - :new-graph "Aggiungi nuovo diagramma" - :graph "Diagramma" - :graph-view "Visualizza diagramma" - :graph/persist "Logseq sta sincronizzando lo stato interno, per favore aspetta molti secondi." + :new-graph "Aggiungi nuovo grafo" + :graph "Grafo" + :graph-view "Visualizza grafo" + :graph/persist "Logseq sta sincronizzando lo stato interno, per favore attendi alcuni secondi." :graph/persist-error "Sincronizzazione dello stato interno fallita." - :graph/save "Salvando..." + :graph/save "Salvataggio..." :graph/save-success "Salvato con successo" :graph/save-error "Salvataggio fallito" - :cards-view "Visualizza carte" + :cards-view "Visualizza flashcard" :publishing "Pubblicazione" :export "Esporta" - :export-graph "Esporta diagramma" + :export-graph "Esporta grafo" :export-page "Esporta pagina" - :export-markdown "Esporta come Markdown standard (nessuna proprietà di blocco)" + :export-markdown "Esporta come Markdown standard (senza le proprietà dei blocchi)" :export-opml "Esporta come OPML" - :export-public-pages "Esporta pagine pubbliche" + :export-public-pages "Esporta le pagine pubbliche" :export-json "Esporta come JSON" :export-roam-json "Esporta come Roam JSON" :export-edn "Esporta come EDN" :export-datascript-edn "Esporta datascript EDN" :convert-markdown "Converti le intestazioni di Markdown in elenchi non ordinati (# -> -)" - :all-graphs "Tutti i diagrammi" + :all-graphs "Tutti i grafi" :all-pages "Tutte le pagine" :all-files "Tutti i file" :remove-orphaned-pages "Rimuovi pagine orfane" - :all-journals "Tutti i diari" + :all-journals "Tutte le pagine di diario" :my-publishing "Le mie pubblicazioni" :settings "Impostazioni" :settings-of-plugins "Impostazioni plugin" :plugins "Plugin" :themes "Temi" - :developer-mode-alert "È necessario riavviare l'app per abilitare il plug-in. Vuoi riavviarlo ora?" - :relaunch-confirm-to-work "Bisogna riavviare l'app per farla funzionare. Vuoi riavviarla ora?" + :developer-mode-alert "È necessario riavviare l'app per abilitare il plugin. Vuoi riavviarla ora?" + :relaunch-confirm-to-work "È necessario riavviare l'app per farla funzionare. Vuoi riavviarla ora?" :import "Importa" :join-community "Unisciti alla comunità" - :sponsor-us "Sponsorizzaci" + :sponsor-us "Supportaci" :discord-title "Il nostro gruppo Discord!" - :help-shortcut-title "Clicca per controllare le scorciatoie e altri suggerimenti" - :loading "Caricando" - :cloning "Clonando" - :parsing-files "Analizzando i file" - :loading-files "Caricando i file" + :help-shortcut-title "Clicca per conoscere le scorciatoie e altri suggerimenti" + :loading "Caricamento" + :cloning "Clonazione" + :parsing-files "Analisi dei file" + :loading-files "Caricamento dei file" :login "Accedi" :logout "Esci" :go-to "Vai a " @@ -3331,41 +3331,41 @@ :remove-background "Rimuovi lo sfondo" :open "Apri" :open-a-directory "Apri una cartella locale" - :user/delete-account "Elimina account" - :user/delete-your-account "Elimina il tuo account" + :user/delete-account "Elimina profilo" + :user/delete-your-account "Elimina il tuo profilo" :user/delete-account-notice "Tutte le tue pagine pubblicate su logseq.com saranno eliminate." - :help/shortcut-page-title "Scorciatoie di tastiera" + :help/shortcut-page-title "Scorciatoie da tastiera" :plugin/installed "Installato" :plugin/not-installed "Non installato" - :plugin/installing "Installando" + :plugin/installing "Installazione" :plugin/install "Installa" :plugin/reload "Ricarica" :plugin/update "Aggiorna" :plugin/check-update "Controlla aggiornamenti" :plugin/check-all-updates "Controlla tutti gli aggiornamenti" - :plugin/refresh-lists "Ricarica liste" + :plugin/refresh-lists "Ricarica lista" :plugin/enabled "Abilitato" :plugin/disabled "Disabilitato" :plugin/update-available "Aggiornamento disponibile" - :plugin/updating "Aggiornando" - :plugin/uninstall "Disinstallare" - :plugin/marketplace "Mercato" - :plugin/downloads "Downloads" + :plugin/updating "Aggiornamento" + :plugin/uninstall "Disinstalla" + :plugin/marketplace "Libreria" + :plugin/downloads "Numero di scaricamenti" :plugin/stars "Stelle" :plugin/title "Titolo" :plugin/all "Tutti" - :plugin/unpacked "Spacchettati" - :plugin/delete-alert "Sei sicuro di disinstallare [{1}]?" + :plugin/unpacked "Non pacchettizzati" + :plugin/delete-alert "Sei sicuro di voler disinstallare [{1}]?" :plugin/open-settings "Apri impostazioni" :plugin/open-package "Apri pacchetto" - :plugin/load-unpacked "Carica plugin non impacchettato" - :plugin/open-preferences "Apri il file preferenze del plugin" - :plugin/restart "Riavvia App" + :plugin/load-unpacked "Carica plugin non pacchettizzato" + :plugin/open-preferences "Apri il file delle preferenze del plugin" + :plugin/restart "Riavvia app" :plugin/unpacked-tips "Seleziona la cartella del plugin" - :plugin/contribute "✨ Scrivi e sottoponi un nuovo plugin" - :plugin/marketplace-tips "Se il plug-in non funziona correttamente alla prima installazione, provare a riavviare Logseq." + :plugin/contribute "✨ Svilupppa e sottoponici un nuovo plugin" + :plugin/marketplace-tips "Se il plugin non funziona correttamente alla prima installazione, provare a riavviare Logseq." :plugin/up-to-date "È aggiornato" :plugin/custom-js-alert "Trovato il file custom.js, è consentito eseguirlo? (Se non si comprende il contenuto di questo file, si consiglia di non consentire l'esecuzione, che presenta alcuni rischi per la sicurezza.)" @@ -3374,23 +3374,23 @@ :pdf/linked-ref "Riferimenti collegati" :pdf/toggle-dashed "Stile tratteggiato per evidenziare l'area" - :updater/new-version-install "Una nuova versione è stata caricata." - :updater/quit-and-install "Riavvia per installare" + :updater/new-version-install "Una nuova versione è stata scaricata." + :updater/quit-and-install "Riavvia per installarla" :paginates/pages "Totale {1} pagine" :paginates/prev "Precedente" - :paginates/next "Prossimo" + :paginates/next "Successivo" - :tips/all-done "Tutto Fatto" + :tips/all-done "Completato" - :command-palette/prompt "Scrivi un comando" - :select/default-prompt "Seleziona uno" - :select.graph/prompt "Seleziona un diagramma" - :select.graph/empty-placeholder-description "Non ci sono diagrammi corrispondenti. Vuoi aggiungerne uno nuovo?" - :select.graph/add-graph "Si, aggiungi un nuovo diagramma" + :command-palette/prompt "Digita un comando" + :select/default-prompt "Selezionane uno" + :select.graph/prompt "Seleziona un grafo" + :select.graph/empty-placeholder-description "Non ci sono grafi corrispondenti. Vuoi aggiungerne uno nuovo?" + :select.graph/add-graph "Sì, aggiungi un nuovo grafo" - :file-sync/other-user-graph "Il diagramma locale attuale è associato al diagramma remoto di un altro utente. Quindi non è possibile avviare la sincronizzazione." - :file-sync/graph-deleted "Il diagramma attuale è stato eliminato"} + :file-sync/other-user-graph "Il grafo locale attuale è associato al grafo remoto di un altro utente. Non è quindi possibile avviare la sincronizzazione." + :file-sync/graph-deleted "Il grafo attuale è stato eliminato"} :tr {:tutorial/text #?(:cljs (rc/inline "tutorial-tr.md") :default "tutorial-tr.md") From 7f256b8872c74718b970d437ed4973f8e1721db6 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Mon, 16 May 2022 15:35:43 +0800 Subject: [PATCH 58/58] fix: sample query --- src/main/frontend/db/query_dsl.cljs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/frontend/db/query_dsl.cljs b/src/main/frontend/db/query_dsl.cljs index 2710d912a2a..91453fefa94 100644 --- a/src/main/frontend/db/query_dsl.cljs +++ b/src/main/frontend/db/query_dsl.cljs @@ -310,8 +310,8 @@ [e sample] (when-let [num (second e)] (when (integer? num) - (reset! sample num)) - nil)) + (reset! sample num) + {:query [['?p :block/uuid]]}))) (defn- build-sort-by [e sort-by_]