Skip to content

Commit

Permalink
feat(sync): support concurrent full-sync local->remote
Browse files Browse the repository at this point in the history
  • Loading branch information
@RCmerci
RCmerci committed Apr 4, 2022
1 parent 3517ae9 commit 85bb55a
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 54 deletions.
3 changes: 2 additions & 1 deletion resources/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@
"diff-match-patch": "1.0.5",
"https-proxy-agent": "5.0.0",
"@sentry/electron": "2.5.1",
"posthog-js": "1.10.2"
"posthog-js": "1.10.2",
"@andelf/rsapi": "0.0.2"
},
"devDependencies": {
"@electron-forge/cli": "^6.0.0-beta.57",
Expand Down
2 changes: 1 addition & 1 deletion src/electron/electron/file_sync_rsapi.cljs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
(ns electron.file-sync-rsapi
(:require ["rsapi" :as rsapi]))
(:require ["@andelf/rsapi" :as rsapi]))

(defn get-local-files-meta [graph-uuid base-path file-paths]
(rsapi/getLocalFilesMeta graph-uuid base-path (clj->js file-paths)))
Expand Down
123 changes: 71 additions & 52 deletions src/main/frontend/fs/sync.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,7 @@

(def local-changes-chan (chan 100))

;;; type = "change" | "add" | "unlink"
(deftype FileChangeEvent [type dir path stat]
IRelativePath
(-relative-path [_] (remove-dir-prefix dir path))
Expand All @@ -633,6 +634,20 @@
(-pr-writer [_ w _opts]
(write-all w (str {:type type :base-path dir :path path}))))

(defn- partition-file-change-events
"return transducer.
partition `FileChangeEvent`s, at most N file-change-events in each partition.
only one type in a partition."
[n]
(comp
(partition-by (fn [^FileChangeEvent e]
(case (.-type e)
("add" "change") :add-or-change
"unlink" :unlink)))
(map #(partition-all n %))
cat))


(defn file-watch-handler
[type {:keys [dir path _content stat] :as _payload}]
(go
Expand All @@ -652,7 +667,7 @@
(stop-local->remote! [this])
(ratelimit [this from-chan] "get watched local file-change events from FROM-CHAN,
return chan returning events with rate limited")
(sync-local->remote! [this ^FileChangeEvent e])
(sync-local->remote! [this es] "es is a sequence of `FileChangeEvent`, all items have same type.")
(sync-local->remote-all-files! [this stop-chan] "compare all local files to remote ones, sync if not equal.
ensure local-txid = remote-txid before calling this func"))

Expand Down Expand Up @@ -756,44 +771,42 @@
(async/close! c)))))
c))

(sync-local->remote! [this ^FileChangeEvent e]
(let [type (.-type e)]
(if (contains-path? (get-ignore-files this) (relative-path e))
(go {:succ true}) ; ignore
(do
(prn "sync-local->remote!" e)
(let [path* (relative-path e)
r
(cond
(or (= "add" type) (= "change" type))
(update-remote-file rsapi graph-uuid base-path path* @*txid)

(= "unlink" type)
(delete-remote-files rsapi graph-uuid base-path [path*] @*txid)

;; (= "rename" type)
;; (rename-local-file)
)]
(go
(let [_ (.add-current-local->remote-files! sync-state [path*])
r* (<! r)
_ (.remove-current-local->remote-files! sync-state [path*])]
(cond
(need-sync-remote? r*)
{:need-sync-remote true}

(number? r*) ; succ
(do
(println "sync-local->remote! update txid" r*)
;; persist txid
(update-graphs-txid! r* graph-uuid repo)
(reset! *txid r*)
{:succ true})

:else
(do
(println "sync-local->remote unknown:" r*)
{:unknown r*})))))))))
(sync-local->remote! [this es]
(if (empty? es)
{:succ true}
(let [type (.-type ^FileChangeEvent (first es))
ignore-files (get-ignore-files this)
es->paths-xf (comp
(map #(relative-path %))
(filter #(not (contains-path? ignore-files %))))
paths (sequence es->paths-xf es)]
(println "sync-local->remote" paths)
(let [r (case type
("add" "change")
(update-remote-files rsapi graph-uuid base-path paths @*txid)

"unlink"
(delete-remote-files rsapi graph-uuid base-path paths @*txid))]
(go
(let [_ (.add-current-local->remote-files! sync-state paths)
r* (<! r)
_ (.remove-current-local->remote-files! sync-state paths)]
(cond
(need-sync-remote? r*)
{:need-sync-remote true}

(number? r*) ; succ
(do
(println "sync-local->remote! update txid" r*)
;; persist txid
(update-graphs-txid! r* graph-uuid repo)
(reset! *txid r*)
{:succ true})

:else
(do
(println "sync-local->remote unknown:" r*)
{:unknown r*}))))))))

(sync-local->remote-all-files! [this stop-chan]
(go
Expand All @@ -804,23 +817,29 @@
diff-local-files (set/difference local-all-files-meta remote-all-files-meta)
ignore-files (get-ignore-files this)
monitored-dirs (get-monitored-dirs this)
change-events (->> diff-local-files
(mapv
#(->FileChangeEvent "change" base-path (.get-normalized-path ^FileMetadata %) nil))
(filterv #(let [path (relative-path %)]
(and (not (contains-path? ignore-files path))
(contains-path? monitored-dirs path)))))]
(println "[full-sync]" (count change-events) "files need to sync to remote")
(loop [es change-events]
(if (empty? es)
change-events-partitions
(sequence
(comp
;; convert to FileChangeEvent
(map #(->FileChangeEvent "change" base-path (.get-normalized-path ^FileMetadata %) nil))
;; filter ignore-files & monitored-dirs
(filter #(let [path (relative-path %)]
(and (not (contains-path? ignore-files path))
(contains-path? monitored-dirs path))))
;; partition FileChangeEvents
(partition-file-change-events 5))
diff-local-files)]
(println "[full-sync]" (count (flatten change-events-partitions)) "files need to sync to remote")
(loop [es-partitions change-events-partitions]
(if (empty? es-partitions)
{:succ true}
(if (async/poll! stop-chan)
{:stop true}
(let [e (first es)
{:keys [succ need-sync-remote unknown] :as r} (<! (sync-local->remote! this e))]
(let [{:keys [succ need-sync-remote unknown] :as r}
(<! (sync-local->remote! this (first es-partitions)))]
(cond
succ
(recur (next es))
(recur (next es-partitions))

(or need-sync-remote unknown) r)))))))))

Expand Down Expand Up @@ -960,7 +979,7 @@
(assert (some? local-change))
(go
(let [{:keys [succ need-sync-remote unknown]}
(<! (sync-local->remote! local->remote-syncer local-change))]
(<! (sync-local->remote! local->remote-syncer [local-change]))]
(cond
succ
(.schedule this ::idle)
Expand Down
30 changes: 30 additions & 0 deletions static/yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,36 @@
resolved "https://registry.yarnpkg.com/7zip-bin/-/7zip-bin-5.1.1.tgz#9274ec7460652f9c632c59addf24efb1684ef876"
integrity sha512-sAP4LldeWNz0lNzmTird3uWfFDWWTeg6V/MsmyyLR9X1idwKBWIgt/ZvinqQldJm3LecKEs1emkbquO6PCiLVQ==

"@andelf/[email protected]":
version "0.0.2"
resolved "https://registry.npmmirror.com/@andelf/rsapi-darwin-arm64/download/@andelf/rsapi-darwin-arm64-0.0.2.tgz#c89a025fbc2e632ef1ed2a96fa64787e29e4bab8"
integrity sha512-WfsVD+5EZkGD7NMyJ0Akwrowgf3IF2BcckERwx0PHQzHi09NZLxZtMr2NN2s6q8QOvGUfbqHYEATZ8zaZMU7XQ==

"@andelf/[email protected]":
version "0.0.2"
resolved "https://registry.npmmirror.com/@andelf/rsapi-darwin-x64/download/@andelf/rsapi-darwin-x64-0.0.2.tgz#d9f71946122dc4bb056e512fa9fc820d30c7159b"
integrity sha512-r5F9odlNt0dD2W4eKYlJnqqoim/5kn/lzch7kHmvR9SKbxIe8jn/XwGqk5Kj5zIlhwpksT24dF47ppnZ8gJZRw==

"@andelf/[email protected]":
version "0.0.2"
resolved "https://registry.npmmirror.com/@andelf/rsapi-linux-x64-gnu/download/@andelf/rsapi-linux-x64-gnu-0.0.2.tgz#f90d8402cbb7e07fcc5defc5c3743d2a591cc22b"
integrity sha512-olB2TYRzqB5e8brtyvj0+YnbdLyWCvsQjbDqV8nKyvUThSEvBPUgREBigdpb26wSETPCX7q/IEQNDDOLdXjzmQ==

"@andelf/[email protected]":
version "0.0.2"
resolved "https://registry.npmmirror.com/@andelf/rsapi-win32-x64-msvc/download/@andelf/rsapi-win32-x64-msvc-0.0.2.tgz#cee77358a4a3601d8e5a6aa5199ef537eadbd6f3"
integrity sha512-qgVZY4lnnlVFu5cNV/pEkvlI/hGokpfpIAGAipYLKVcAmuZ/H9+3aD8PcSi4uRIuexFlSBIcNBi7FtnNahMUcA==

"@andelf/[email protected]":
version "0.0.2"
resolved "https://registry.npmmirror.com/@andelf/rsapi/download/@andelf/rsapi-0.0.2.tgz#8b24e9792123131f5f4529f00fb8f380333acc26"
integrity sha512-/CVyYM3+fRFvO6Dqyd8TJv6IOZsaFpKAkX07Vs2FbnSzR50XtGujYpcjyKjmq3zLPmtJYuvD80qwKqixvzCPPw==
optionalDependencies:
"@andelf/rsapi-darwin-arm64" "0.0.2"
"@andelf/rsapi-darwin-x64" "0.0.2"
"@andelf/rsapi-linux-x64-gnu" "0.0.2"
"@andelf/rsapi-win32-x64-msvc" "0.0.2"

"@develar/schema-utils@~2.1.0":
version "2.1.0"
resolved "https://registry.yarnpkg.com/@develar/schema-utils/-/schema-utils-2.1.0.tgz#eceb1695bfbed6f6bb84666d5d3abe5e1fd54e17"
Expand Down

0 comments on commit 85bb55a

Please sign in to comment.