Setup CI jobs that are only for graph-parser

Also move all non graph-parser fns to frontend thanks to carve lint

.clj-kondo/config.edn

@@ -26,6 +26,7 @@
              frontend.db.query-react query-react
              frontend.util util
              frontend.util.property property
+             frontend.util.text text-util
              frontend.config config
              frontend.format.mldoc mldoc
              frontend.format.block block
@@ -40,7 +41,7 @@
              logseq.graph-parser.date-time-util date-time-util}}}
 
  :hooks {:analyze-call {rum.core/defc hooks.rum/defc
-                         rum.core/defcs hooks.rum/defcs}}
+                        rum.core/defcs hooks.rum/defcs}}
  :lint-as {promesa.core/let clojure.core/let
            promesa.core/loop clojure.core/loop
            promesa.core/recur clojure.core/recur

.github/workflows/graph-parser.yml

@@ -0,0 +1,56 @@
+name: logseq graph-parser CI
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - 'deps/graph-parser/**'
+      - '!deps/graph-parser/**.md'
+  pull_request:
+    branches: [master]
+    paths:
+      - 'deps/graph-parser/**'
+      - '!deps/graph-parser/**.md'
+
+env:
+  CLOJURE_VERSION: '1.10.1.727'
+  # setup-java@v2 dropped support for legacy Java version syntax.
+  # This is the same as 1.8.
+  JAVA_VERSION: '8'
+  # This is the latest node version we can run.
+  NODE_VERSION: '16'
+  BABASHKA_VERSION: '0.8.1'
+
+jobs:
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Java
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: ${{ env.JAVA_VERSION }}
+
+      - name: Set up Clojure
+        uses: DeLaGuardo/setup-clojure@master
+        with:
+          cli: ${{ env.CLOJURE_VERSION }}
+
+      - name: Setup Babashka
+        uses: turtlequeue/[email protected]
+        with:
+          babashka-version: ${{ env.BABASHKA_VERSION }}
+
+      - name: Run clj-kondo lint
+        run: cd deps/graph-parser && clojure -M:clj-kondo --parallel --lint src test
+
+      - name: Lint for vars that are too large
+        run: scripts/large_vars.clj deps/graph-parser/src
+
+      - name: Carve lint for unused vars
+        run: cd deps/graph-parser && ../../scripts/carve.clj

deps/graph-parser/.carve/config.edn

@@ -1,5 +1,7 @@
 {:paths ["src"]
  :api-namespaces [
                   ;; carve doesn't detect nbb only usage
-                  logseq.graph-parser.log]
+                  logseq.graph-parser.log
+                  ;; Used by logseq but not worth splitting up
+                  logseq.graph-parser.db.schema]
  :report {:format :ignore}}

deps/graph-parser/.carve/ignore

@@ -0,0 +1,6 @@
+;; For CLI
+logseq.graph-parser.cli/parse
+;; For CLI
+logseq.graph-parser.db/start-conn
+;; For CLI
+logseq.graph-parser.mldoc/ast-export-markdown

deps/graph-parser/.clj-kondo/config.edn

@@ -0,0 +1,20 @@
+{:linters
+ {:unresolved-symbol {:exclude [;; TODO: Remove parse-* and update-* when https://github.com/clj-kondo/clj-kondo/issues/1694 is done
+                                parse-long
+                                parse-double
+                                parse-uuid
+                                update-keys
+                                update-vals]}
+
+  :consistent-alias
+  {:aliases {datascript.core d
+             logseq.graph-parser graph-parser
+             logseq.graph-parser.text text
+             logseq.graph-parser.block gp-block
+             logseq.graph-parser.mldoc gp-mldoc
+             logseq.graph-parser.util gp-util
+             logseq.graph-parser.property gp-property
+             logseq.graph-parser.config gp-config
+             logseq.graph-parser.date-time-util date-time-util}}}
+ :lint-as {promesa.core/let clojure.core/let}
+ :skip-comments true}

deps/graph-parser/.gitignore

@@ -0,0 +1 @@
+.clj-kondo/.cache

deps/graph-parser/deps.edn

@@ -7,4 +7,8 @@
   frankiesardo/linked {:mvn/version "1.3.0"}
   com.andrewmcveigh/cljs-time {:git/url "https://github.com/logseq/cljs-time" ;; fork
                                :sha     "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"}
-  com.lambdaisland/glogi {:mvn/version "1.1.144"}}}
+  com.lambdaisland/glogi {:mvn/version "1.1.144"}}
+
+ :aliases
+ {:clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2022.04.25"}}
+              :main-opts  ["-m" "clj-kondo.main"]}}}

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -183,10 +183,6 @@
          (map (partial apply merge))
          (with-block-uuid))))
 
-(defn extract-all-block-refs
-  [content]
-  (map second (re-seq #"\(\(([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})\)\)" content)))
-
 #?(:org.babashka/nbb
    (alter-var-root #'gp-mldoc/parse-property (constantly text/parse-property))
    :default

deps/graph-parser/src/logseq/graph_parser/text.cljs

@@ -51,8 +51,6 @@
 
 (def page-ref-re-without-nested #"\[\[([^\[\]]+)\]\]")
 
-(defonce between-re #"\(between ([^\)]+)\)")
-
 (defn page-ref-un-brackets!
   [s]
   (or (get-page-name s) s))
@@ -122,18 +120,6 @@
   [s]
   (string/split s #"(\"[^\"]*\")"))
 
-(def bilibili-regex #"^((?:https?:)?//)?((?:www).)?((?:bilibili.com))(/(?:video/)?)([\w-]+)(\S+)?$")
-(def loom-regex #"^((?:https?:)?//)?((?:www).)?((?:loom.com))(/(?:share/|embed/))([\w-]+)(\S+)?$")
-(def vimeo-regex #"^((?:https?:)?//)?((?:www).)?((?:player.vimeo.com|vimeo.com))(/(?:video/)?)([\w-]+)(\S+)?$")
-(def youtube-regex #"^((?:https?:)?//)?((?:www|m).)?((?:youtube.com|youtu.be|y2u.be|youtube-nocookie.com))(/(?:[\w-]+\?v=|embed/|v/)?)([\w-]+)([\S^\?]+)?$")
-
-(defn get-matched-video
-  [url]
-  (or (re-find youtube-regex url)
-      (re-find loom-regex url)
-      (re-find vimeo-regex url)
-      (re-find bilibili-regex url)))
-
 (def markdown-link #"\[([^\[]+)\](\(.*\))")
 
 (defn split-page-refs-without-brackets
@@ -228,16 +214,6 @@
        :else
        (remove-level-space-aux! text block-pattern space? trim-left?)))))
 
-(defn build-data-value
-  [col]
-  (let [items (map (fn [item] (str "\"" item "\"")) col)]
-    (gstring/format "[%s]"
-                 (string/join ", " items))))
-
-(defn media-link?
-  [media-formats s]
-  (some (fn [fmt] (gp-util/safe-re-find (re-pattern (str "(?i)\\." fmt "(?:\\?([^#]*))?(?:#(.*))?$")) s)) media-formats))
-
 (defn namespace-page?
   [p]
   (and (string? p)
@@ -246,100 +222,6 @@
        (not (string/starts-with? p "./"))
        (not (gp-util/url? p))))
 
-(defn add-timestamp
-  [content key value]
-  (let [new-line (str (string/upper-case key) ": " value)
-        lines (string/split-lines content)
-        new-lines (map (fn [line]
-                         (string/trim
-                          (if (string/starts-with? (string/lower-case line) key)
-                            new-line
-                            line)))
-                    lines)
-        new-lines (if (not= (map string/trim lines) new-lines)
-                    new-lines
-                    (cons (first new-lines) ;; title
-                          (cons
-                           new-line
-                           (rest new-lines))))]
-    (string/join "\n" new-lines)))
-
-(defn remove-timestamp
-  [content key]
-  (let [lines (string/split-lines content)
-        new-lines (filter (fn [line]
-                            (not (string/starts-with? (string/lower-case line) key)))
-                          lines)]
-    (string/join "\n" new-lines)))
-
-(defn get-current-line-by-pos
-  [s pos]
-  (let [lines (string/split-lines s)
-        result (reduce (fn [acc line]
-                         (let [new-pos (+ acc (count line))]
-                           (if (>= new-pos pos)
-                             (reduced line)
-                             (inc new-pos)))) 0 lines)]
-    (when (string? result)
-      result)))
-
-(defn get-string-all-indexes
-  "Get all indexes of `value` in the string `s`."
-  [s value]
-  (loop [acc []
-         i 0]
-    (if-let [i (string/index-of s value i)]
-      (recur (conj acc i) (+ i (count value)))
-      acc)))
-
-(defn surround-by?
-  "`pos` must be surrounded by `before` and `and` in string `value`, e.g. ((|))"
-  [value pos before end]
-  (let [start-pos (if (= :start before) 0 (- pos (count before)))
-        end-pos (if (= :end end) (count value) (+ pos (count end)))]
-    (when (>= (count value) end-pos)
-      (= (cond
-           (and (= :end end) (= :start before))
-           ""
-
-           (= :end end)
-           before
-
-           (= :start before)
-           end
-
-           :else
-           (str before end))
-         (subs value start-pos end-pos)))))
-
-(defn wrapped-by?
-  "`pos` must be wrapped by `before` and `and` in string `value`, e.g. ((a|b))"
-  [value pos before end]
-  (let [before-matches (->> (get-string-all-indexes value before)
-                            (map (fn [i] [i :before])))
-        end-matches (->> (get-string-all-indexes value end)
-                         (map (fn [i] [i :end])))
-        indexes (sort-by first (concat before-matches end-matches [[pos :between]]))
-        ks (map second indexes)
-        q [:before :between :end]]
-    (true?
-     (reduce (fn [acc k]
-               (if (= q (conj acc k))
-                 (reduced true)
-                 (vec (take-last 2 (conj acc k)))))
-             []
-             ks))))
-
-(defn get-graph-name-from-path
-  [path]
-  (when (string? path)
-    (let [parts (->> (string/split path #"/")
-                     (take-last 2))]
-      (-> (if (not= (first parts) "0")
-            (string/join "/" parts)
-            (last parts))
-          js/decodeURI))))
-
 (defonce non-parsing-properties
   (atom #{"background-color" "background_color"}))
 

deps/graph-parser/src/logseq/graph_parser/utf8.cljs

@@ -1,5 +1,4 @@
-(ns ^:nbb-compatible logseq.graph-parser.utf8
-  (:require [goog.object :as gobj]))
+(ns ^:nbb-compatible logseq.graph-parser.utf8)
 
 (defonce encoder
   (js/TextEncoder. "utf-8"))
@@ -22,7 +21,3 @@
    (if end
      (decode (.subarray arr start end))
      (decode (.subarray arr start)))))
-
-(defn length
-  [arr]
-  (gobj/get arr "length"))

deps/graph-parser/test/logseq/graph_parser/text_test.cljs

@@ -98,36 +98,6 @@
       "**foobar" "foobar"
       "*********************foobar" "foobar")))
 
-(deftest test-add-timestamp
-  []
-  (are [x y] (= x y)
-    (text/add-timestamp "LATER hello world\nhello"
-                        "scheduled"
-                        "<2021-08-25 Wed>")
-    "LATER hello world\nSCHEDULED: <2021-08-25 Wed>\nhello"
-
-    (text/add-timestamp "LATER hello world "
-                        "scheduled"
-                        "<2021-08-25 Wed>")
-    "LATER hello world\nSCHEDULED: <2021-08-25 Wed>"
-
-    (text/add-timestamp "LATER hello world\nfoo:: bar\ntest"
-                        "scheduled"
-                        "<2021-08-25 Wed>")
-    "LATER hello world\nSCHEDULED: <2021-08-25 Wed>\nfoo:: bar\ntest"))
-
-(deftest get-string-all-indexes
-  []
-  (are [x y] (= x y)
-    (text/get-string-all-indexes "[[hello]] [[world]]" "[[")
-    [0 10]
-
-    (text/get-string-all-indexes "abc abc ab" "ab")
-    [0 4 8]
-
-    (text/get-string-all-indexes "a.c a.c ab" "a.")
-    [0 4]))
-
 (deftest test-parse-property
   (testing "parse-property"
     (are [k v y] (= (text/parse-property k v {}) y)