Localize bb-tasks for deps

Deps need to be independent to not introduce testing bugs. These bb tasks are good candidates to move into bb-tasks gitlib
shangjingmin · Jun 10, 2022 · ee9f390 · ee9f390
1 parent 17d2462
commit ee9f390
Show file tree

Hide file tree

Showing 14 changed files with 162 additions and 212 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -101,10 +101,10 @@ jobs:
         run: clojure -M:clj-kondo --parallel --lint src
 
       - name: Carve lint for unused vars
-        run: scripts/carve.clj
+        run: bb lint:carve
 
       - name: Lint for vars that are too large
-        run: scripts/large_vars.clj
+        run: bb lint:large-vars
 
       - name: Lint invalid translation entries
         run: bb lang:invalid-translations

diff --git a/.github/workflows/db.yml b/.github/workflows/db.yml
@@ -24,7 +24,7 @@ env:
   JAVA_VERSION: '8'
   # This is the latest node version we can run.
   NODE_VERSION: '16'
-  BABASHKA_VERSION: '0.8.2'
+  BABASHKA_VERSION: '0.8.156'
 
 jobs:
   test:
@@ -58,7 +58,7 @@ jobs:
 
       # In this job because it depends on an npm package
       - name: Load namespaces into nbb-logseq
-        run: cd ../.. && bb test:load-all-namespaces-with-nbb deps/db
+        run: bb test:load-all-namespaces-with-nbb .
 
   lint:
     runs-on: ubuntu-latest
@@ -83,10 +83,10 @@ jobs:
         run: clojure -M:clj-kondo --parallel --lint src
 
       - name: Carve lint for unused vars
-        run: ../../scripts/carve.clj
+        run: bb lint:carve
 
       - name: Lint for vars that are too large
-        run: ../../scripts/large_vars.clj src '{:max-lines-count 30}'
+        run: bb lint:large-vars
 
       - name: Lint datalog rules
-        run: ../../scripts/lint_rules.clj
+        run: bb lint:rules
diff --git a/.github/workflows/graph-parser.yml b/.github/workflows/graph-parser.yml
@@ -17,14 +17,18 @@ on:
       - 'deps/db/**'
       - '!deps/graph-parser/**.md'
 
+defaults:
+  run:
+    working-directory: deps/graph-parser
+
 env:
   CLOJURE_VERSION: '1.10.1.727'
   # setup-java@v2 dropped support for legacy Java version syntax.
   # This is the same as 1.8.
   JAVA_VERSION: '8'
   # This is the latest node version we can run.
   NODE_VERSION: '16'
-  BABASHKA_VERSION: '0.8.2'
+  BABASHKA_VERSION: '0.8.156'
 
 jobs:
   test:
@@ -69,20 +73,20 @@ jobs:
 
       - name: Fetch Clojure deps
         if: steps.clojure-deps.outputs.cache-hit != 'true'
-        run: cd deps/graph-parser && clojure -A:test -P
+        run: clojure -A:test -P
 
       - name: Fetch yarn deps
-        run: cd deps/graph-parser && yarn install --frozen-lockfile
+        run: yarn install --frozen-lockfile
 
       - name: Run ClojureScript tests
-        run: cd deps/graph-parser && clojure -M:test
+        run: clojure -M:test
 
       - name: Run nbb-logseq tests
-        run: cd deps/graph-parser && yarn nbb-logseq -cp src:test:../db/src -m logseq.graph-parser.nbb-test-runner/run-tests
+        run: yarn nbb-logseq -cp src:test:../db/src -m logseq.graph-parser.nbb-test-runner/run-tests
 
       # In this job because it depends on an npm package
       - name: Load namespaces into nbb-logseq
-        run: bb test:load-all-namespaces-with-nbb deps/graph-parser
+        run: bb test:load-all-namespaces-with-nbb .
 
   lint:
     runs-on: ubuntu-latest
@@ -108,10 +112,10 @@ jobs:
           babashka-version: ${{ env.BABASHKA_VERSION }}
 
       - name: Run clj-kondo lint
-        run: cd deps/graph-parser && clojure -M:clj-kondo --parallel --lint src test
+        run: clojure -M:clj-kondo --parallel --lint src test
 
       - name: Carve lint for unused vars
-        run: cd deps/graph-parser && ../../scripts/carve.clj
+        run: bb lint:carve
 
       - name: Lint for vars that are too large
-        run: scripts/large_vars.clj deps/graph-parser/src '{:max-lines-count 75}'
+        run: bb lint:large-vars
diff --git a/bb.edn b/bb.edn
@@ -6,7 +6,7 @@
   logseq/bb-tasks
   #_{:local/root "../bb-tasks"}
   {:git/url "https://github.com/logseq/bb-tasks"
-   :git/sha "4b3e623fb475cacb992425aa9dac770d6dd63e38"}
+   :git/sha "abb32ccd26405d56fd28a29d56f3cb902b8c4334"}
   logseq/graph-parser
   {:local/root "deps/graph-parser"}}
  :pods
@@ -33,18 +33,18 @@
   dev:lint
   logseq.tasks.dev/lint
 
+  lint:large-vars
+  logseq.bb-tasks.lint.large-vars/-main
+
+  lint:carve
+  logseq.bb-tasks.lint.carve/-main
+
   nbb:watch
   logseq.bb-tasks.nbb.watch/watch
 
   nbb:portal-watch
   logseq.bb-tasks.nbb.watch/portal-watch
 
-  test:load-namespaces-with-nbb
-  logseq.tasks.nbb/load-compatible-namespaces
-
-  test:load-all-namespaces-with-nbb
-  logseq.tasks.nbb/load-all-namespaces
-
   lang:list
   logseq.tasks.lang/list-langs
 
@@ -58,4 +58,13 @@
   logseq.tasks.lang/invalid-translations
 
   file-sync:integration-tests
-  logseq.tasks.file-sync/integration-tests}}
+  logseq.tasks.file-sync/integration-tests}
+
+ :tasks/config
+ {:large-vars
+  ;; TODO: Get to a smaller max-lines-count
+  {:max-lines-count 100
+   ;; TODO: Address vars tagged with cleanup-todo. These
+   ;; are left mostly because they are not high priority
+   ;; or not well understood
+   :metadata-exceptions #{:large-vars/cleanup-todo}}}}
diff --git a/deps/db/README.md b/deps/db/README.md
@@ -0,0 +1,52 @@
+## Description
+
+This library provides a minimal API for using a
+[datascript](https://github.com/tonsky/datascript) database from the Logseq app
+and the CLI. This library is compatible with ClojureScript and with
+[nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide
+frontend and commandline functionality.
+
+## API
+
+This library is under the parent namespace `logseq.db`. This library provides
+two main namespaces, `logseq.db` and `logseq.db.rules`.
+
+## Usage
+
+See usage in `deps/graph-parser` and in the Logseq app.
+
+## Dev
+
+This follows the practices that [the Logseq frontend
+follows](/docs/dev-practices.md). Most of the same linters are used, with
+configurations that are specific to this library. See [this library's CI
+file](/.github/workflows/db.yml) for linting examples.
+
+### Setup
+
+To run linters, you'll want to install yarn dependencies once:
+```
+yarn install
+```
+
+This step is not needed if you're just running the application.
+
+## Linting
+
+### Datalog linting
+
+Our rules are linted through a script that also uses the datalog-parser. To run this linter:
+```
+bb lint:rules
+```
+
+
+### Managing dependencies
+
+The package.json dependencies are just for testing and should be updated if there is
+new behavior to test.
+
+The deps.edn dependecies are used by both ClojureScript and nbb-logseq. Their
+versions should be backwards compatible with each other with priority given to
+the frontend. _No new dependency_ should be introduced to this library without
+an understanding of the tradeoffs of adding this to nbb-logseq.
diff --git a/deps/db/bb.edn b/deps/db/bb.edn
@@ -0,0 +1,35 @@
+{:paths ["src"]
+ :min-bb-version "0.8.156"
+ :deps
+ {logseq/bb-tasks
+  #_{:local/root "../../../bb-tasks"}
+  {:git/url "https://github.com/logseq/bb-tasks"
+   :git/sha "abb32ccd26405d56fd28a29d56f3cb902b8c4334"}}
+
+ :pods
+ {clj-kondo/clj-kondo {:version "2022.02.09"}}
+
+ :tasks
+ {test:load-all-namespaces-with-nbb
+  logseq.bb-tasks.nbb.test/load-all-namespaces
+
+  lint:large-vars
+  logseq.bb-tasks.lint.large-vars/-main
+
+  lint:carve
+  logseq.bb-tasks.lint.carve/-main
+
+  lint:rules
+  {:requires ([logseq.bb-tasks.lint.datalog :as datalog]
+              [logseq.db.rules :as rules])
+   :doc "Lint datalog rules for parsability and unbound variables"
+   :task (datalog/lint-rules
+          (into rules/rules
+                (-> rules/query-dsl-rules
+                    ;; TODO: Update linter to handle false positive on ?str-val
+                    (dissoc :property)
+                    vals)))}}
+
+ :tasks/config
+ {:large-vars
+  {:max-lines-count 30}}}
diff --git a/deps/db/src/logseq/db/rules.cljc b/deps/db/src/logseq/db/rules.cljc
@@ -1,4 +1,5 @@
-(ns ^:bb-compatible logseq.db.rules)
+(ns ^:bb-compatible logseq.db.rules
+  "Datalog rules for use with logseq.db.schema")
 
 (def ^:large-vars/data-var rules
   ;; rule "parent" is optimized for child node -> parent node nesting queries

diff --git a/deps/graph-parser/bb.edn b/deps/graph-parser/bb.edn
@@ -0,0 +1,23 @@
+{:min-bb-version "0.8.156"
+ :deps
+ {logseq/bb-tasks
+  #_{:local/root "../../../bb-tasks"}
+  {:git/url "https://github.com/logseq/bb-tasks"
+   :git/sha "abb32ccd26405d56fd28a29d56f3cb902b8c4334"}}
+
+ :pods
+ {clj-kondo/clj-kondo {:version "2022.02.09"}}
+
+ :tasks
+ {test:load-all-namespaces-with-nbb
+  logseq.bb-tasks.nbb.test/load-all-namespaces
+
+  lint:large-vars
+  logseq.bb-tasks.lint.large-vars/-main
+
+  lint:carve
+  logseq.bb-tasks.lint.carve/-main}
+
+ :tasks/config
+ {:large-vars
+  {:max-lines-count 75}}}
diff --git a/docs/dev-practices.md b/docs/dev-practices.md
@@ -27,15 +27,15 @@ We use https://github.com/borkdude/carve to detect unused vars in our codebase.
 
 To run this linter:
 ```
-scripts/carve.clj
+bb lint:carve
 ```
 
 By default, the script runs in CI mode which prints unused vars if they are
 found. The script can be run in an interactive mode which prompts for keeping
 (ignoring) an unused var or removing it. Run this mode with:
 
 ```
-scripts/carve.clj '{:interactive true}'
+bb lint:carve '{:interactive true}'
 ```
 
 When a var is ignored, it is added to `.carve/ignore`. Please add a comment for
@@ -46,17 +46,19 @@ why a var is ignored to help others understand why it's unused.
 Large vars have a lot of complexity and make it hard for the team to maintain
 and understand them. To run this linter:
 ```
-scripts/large_vars.clj
+bb lint:large-vars
 ```
 
 To configure the linter, see its `config` var.
 
 ### Datalog linting
 
-We use [datascript](https://github.com/tonsky/datascript)'s datalog to power our modeling and querying layer. Since datalog is concise, it is easy to write something invalid. To avoid typos and other preventable mistakes, we lint our queries and rules. Our queries are linted through clj-kondo and [datalog-parser](https://github.com/lambdaforge/datalog-parser). clj-kondo will error if it detects an invalid query. Our rules are linted through a script that also uses the datalog-parser. To run this linter:
-```
-scripts/lint_rules.clj
-```
+We use [datascript](https://github.com/tonsky/datascript)'s datalog to power our
+modeling and querying layer. Since datalog is concise, it is easy to write
+something invalid. To avoid typos and other preventable mistakes, we lint our
+queries and rules. Our queries are linted through clj-kondo and
+[datalog-parser](https://github.com/lambdaforge/datalog-parser). clj-kondo will
+error if it detects an invalid query.
 
 ## Testing
 

diff --git a/scripts/carve.clj b/scripts/carve.clj