forked from logseq/logseq
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request logseq#5420 from logseq/enhance/graph-parser-part-…
…four Enhance: Finish up graph-parser
- Loading branch information
Showing
81 changed files
with
1,536 additions
and
851 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| name: logseq graph-parser CI | ||
|
|
||
| on: | ||
| # Path filters ensure jobs only kick off if a change is made to graph-parser | ||
| push: | ||
| branches: [master] | ||
| paths: | ||
| - 'deps/graph-parser/**' | ||
| - '!deps/graph-parser/**.md' | ||
| pull_request: | ||
| branches: [master] | ||
| paths: | ||
| - 'deps/graph-parser/**' | ||
| - '!deps/graph-parser/**.md' | ||
|
|
||
| env: | ||
| CLOJURE_VERSION: '1.10.1.727' | ||
| # setup-java@v2 dropped support for legacy Java version syntax. | ||
| # This is the same as 1.8. | ||
| JAVA_VERSION: '8' | ||
| # This is the latest node version we can run. | ||
| NODE_VERSION: '16' | ||
| BABASHKA_VERSION: '0.8.2' | ||
|
|
||
| jobs: | ||
| test: | ||
| runs-on: ubuntu-latest | ||
|
|
||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v2 | ||
|
|
||
| - name: Set up Node | ||
| uses: actions/setup-node@v2 | ||
| with: | ||
| node-version: ${{ env.NODE_VERSION }} | ||
| cache: 'yarn' | ||
| cache-dependency-path: deps/graph-parser/yarn.lock | ||
|
|
||
| - name: Set up Java | ||
| uses: actions/setup-java@v2 | ||
| with: | ||
| distribution: 'zulu' | ||
| java-version: ${{ env.JAVA_VERSION }} | ||
|
|
||
| - name: Set up Clojure | ||
| uses: DeLaGuardo/setup-clojure@master | ||
| with: | ||
| cli: ${{ env.CLOJURE_VERSION }} | ||
|
|
||
| - name: Setup Babashka | ||
| uses: turtlequeue/[email protected] | ||
| with: | ||
| babashka-version: ${{ env.BABASHKA_VERSION }} | ||
|
|
||
| - name: Clojure cache | ||
| uses: actions/cache@v2 | ||
| id: clojure-deps | ||
| with: | ||
| path: | | ||
| ~/.m2/repository | ||
| ~/.gitlibs | ||
| key: ${{ runner.os }}-clojure-deps-${{ hashFiles('deps.edn') }} | ||
| restore-keys: ${{ runner.os }}-clojure-deps- | ||
|
|
||
| - name: Fetch Clojure deps | ||
| if: steps.clojure-deps.outputs.cache-hit != 'true' | ||
| run: cd deps/graph-parser && clojure -A:test -P | ||
|
|
||
| - name: Fetch yarn deps | ||
| run: cd deps/graph-parser && yarn install --frozen-lockfile | ||
|
|
||
| - name: Run ClojureScript tests | ||
| run: cd deps/graph-parser && clojure -M:test | ||
|
|
||
| - name: Run nbb-logseq tests | ||
| run: cd deps/graph-parser && yarn nbb-logseq -cp src:test -m logseq.graph-parser.nbb-test-runner/run-tests | ||
|
|
||
| # In this job because it depends on an npm package | ||
| - name: Load namespaces into nbb-logseq | ||
| run: bb test:load-all-namespaces-with-nbb deps/graph-parser src | ||
|
|
||
| lint: | ||
| runs-on: ubuntu-latest | ||
|
|
||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v2 | ||
|
|
||
| - name: Set up Java | ||
| uses: actions/setup-java@v2 | ||
| with: | ||
| distribution: 'zulu' | ||
| java-version: ${{ env.JAVA_VERSION }} | ||
|
|
||
| - name: Set up Clojure | ||
| uses: DeLaGuardo/setup-clojure@master | ||
| with: | ||
| cli: ${{ env.CLOJURE_VERSION }} | ||
|
|
||
| - name: Setup Babashka | ||
| uses: turtlequeue/[email protected] | ||
| with: | ||
| babashka-version: ${{ env.BABASHKA_VERSION }} | ||
|
|
||
| - name: Run clj-kondo lint | ||
| run: cd deps/graph-parser && clojure -M:clj-kondo --parallel --lint src test | ||
|
|
||
| - name: Carve lint for unused vars | ||
| run: cd deps/graph-parser && ../../scripts/carve.clj | ||
|
|
||
| - name: Lint for vars that are too large | ||
| run: scripts/large_vars.clj deps/graph-parser/src '{:max-lines-count 75}' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| {:paths ["src"] | ||
| :api-namespaces [ | ||
| ;; carve doesn't detect nbb only usage | ||
| logseq.graph-parser.log | ||
| ;; Used by logseq but not worth splitting up | ||
| logseq.graph-parser.db.schema | ||
| ;; Used in tests | ||
| logseq.graph-parser.test.docs-graph-helper] | ||
| :report {:format :ignore}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| ;; For CLI | ||
| logseq.graph-parser.cli/parse-graph | ||
| ;; For CLI | ||
| logseq.graph-parser.mldoc/ast-export-markdown |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| {:linters | ||
| {:consistent-alias | ||
| {:aliases {datascript.core d | ||
| logseq.graph-parser graph-parser | ||
| logseq.graph-parser.text text | ||
| logseq.graph-parser.block gp-block | ||
| logseq.graph-parser.mldoc gp-mldoc | ||
| logseq.graph-parser.util gp-util | ||
| logseq.graph-parser.property gp-property | ||
| logseq.graph-parser.config gp-config | ||
| logseq.graph-parser.date-time-util date-time-util}}} | ||
| :skip-comments true | ||
| :output {:progress true}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| /.clj-kondo/.cache | ||
| cljs-test-runner-out | ||
| /test/docs |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| ## Description | ||
|
|
||
| This library parses a logseq graph directory and returns it as a datascript | ||
| database connection. This library powers the Logseq app and also runs from the | ||
| commandline, _independent_ of the app. This is powerful as this can run anywhere | ||
| that a Node.js script has access to a Logseq graph e.g. on CI processes like | ||
| Github Actions. This library is compatible with ClojureScript and with | ||
| [nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide | ||
| frontend and commandline functionality. | ||
|
|
||
| ## API | ||
|
|
||
| This library is under the parent namespace `logseq.graph-parser`. This library | ||
| provides two main namespaces for parsing, `logseq.graph-parser` and | ||
| `logseq.graph-parser.cli`. `logseq.graph-parser/parse-file` is the main fn for | ||
| the frontend. `logseq.graph-parser.cli/parse-graph` is the main fn for node.js | ||
| CLIs. | ||
|
|
||
| ## Usage | ||
|
|
||
| See `logseq.graph-parser.cli-test` for now. A real world example is coming soon. | ||
|
|
||
| ## Dev | ||
|
|
||
| This follows the practices that [the Logseq frontend | ||
| follows](/docs/dev-practices.md). Most of the same linters are used, with | ||
| configurations that are specific to this library. See [this library's CI | ||
| file](/.github/workflows/graph-parser.yml) for linting examples. | ||
|
|
||
| ### Setup | ||
|
|
||
| To run linters and tests, you'll want to install yarn dependencies once: | ||
| ``` | ||
| yarn install | ||
| ``` | ||
|
|
||
| This step is not needed if you're just running the application. | ||
|
|
||
| ### Testing | ||
|
|
||
| Since this file is compatible with cljs and nbb-logseq, tests are run against both languages. | ||
|
|
||
| ClojureScript tests use https://github.com/Olical/cljs-test-runner. To run tests: | ||
| ``` | ||
| clojure -M:test | ||
| ``` | ||
|
|
||
| To see available options that can run specific tests or namespaces: `clojure -M:test --help` | ||
|
|
||
| To run nbb-logseq tests: | ||
| ``` | ||
| yarn nbb-logseq -cp src:test -m logseq.graph-parser.nbb-test-runner/run-tests | ||
| ``` | ||
|
|
||
| ### Managing dependencies | ||
|
|
||
| The package.json dependencies are just for testing and should be updated if there is | ||
| new behavior to test. | ||
|
|
||
| The deps.edn dependecies are used by both ClojureScript and nbb-logseq. Their | ||
| versions should be backwards compatible with each other with priority given to | ||
| the frontend. _No new dependency_ should be introduced to this library without | ||
| an understanding of the tradeoffs of adding this to nbb-logseq. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| {:paths ["src"] | ||
| :deps | ||
| ;; Deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn | ||
| {datascript/datascript {:mvn/version "1.3.8"} | ||
| frankiesardo/linked {:mvn/version "1.3.0"} | ||
| com.andrewmcveigh/cljs-time {:git/url "https://github.com/logseq/cljs-time" ;; fork | ||
| :sha "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"} | ||
| ;; stubbed in nbb | ||
| com.lambdaisland/glogi {:mvn/version "1.1.144"} | ||
| ;; built in to nbb | ||
| cljs-bean/cljs-bean {:mvn/version "1.5.0"}} | ||
|
|
||
| :aliases | ||
| ;; This runs tests with nodejs. Would be nice to run this with in a browser env | ||
| ;; since this is how its normally run in the app but this requires more setup | ||
| ;; with karma, shadow-cljs.edn and headless mode on CI | ||
| {:test {:extra-paths ["test"] | ||
| :extra-deps {olical/cljs-test-runner {:mvn/version "3.8.0"} | ||
| org.clojure/clojurescript {:mvn/version "1.11.54"}} | ||
| :main-opts ["-m" "cljs-test-runner.main"]} | ||
|
|
||
| :clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2022.05.28"}} | ||
| :main-opts ["-m" "clj-kondo.main"]}}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| { | ||
| "name": "@logseq/graph-parser", | ||
| "version": "1.0.0", | ||
| "private": true, | ||
| "devDependencies": { | ||
| "@logseq/nbb-logseq": "^0.5.103" | ||
| }, | ||
| "dependencies": { | ||
| "mldoc": "^1.3.3" | ||
| } | ||
| } |
Oops, something went wrong.