Merge branch 'master' into enhance/graph-parser-part-two

package.json

@@ -5,6 +5,7 @@
     "main": "static/electron.js",
     "devDependencies": {
         "@capacitor/cli": "3.2.2",
+        "@logseq/nbb-logseq": "^0.3.99",
         "@playwright/test": "^1.19.2",
         "@tailwindcss/ui": "0.7.2",
         "@types/gulp": "^4.0.7",
@@ -13,7 +14,6 @@
         "del": "^6.0.0",
         "gulp": "^4.0.2",
         "gulp-clean-css": "^4.3.0",
-        "@logseq/nbb-logseq": "^0.3.99",
         "npm-run-all": "^4.1.5",
         "playwright": "^1.19.2",
         "postcss": "8.2.13",
@@ -113,6 +113,7 @@
         "react-textarea-autosize": "8.3.3",
         "react-tippy": "1.4.0",
         "react-transition-group": "4.3.0",
+        "react-visibility-sensor": "^5.1.1",
         "reakit": "0.11.1",
         "remove-accents": "0.4.2",
         "send-intent": "3.0.11",

resources/css/common.css

@@ -1169,3 +1169,8 @@ html[data-theme='dark'] .keyboard-shortcut > code {
     overflow: hidden;
     text-overflow: ellipsis;
 }
+
+.lazy-visibility {
+    min-width: 1px;
+    min-height: 1px;
+}

src/main/frontend/components/block.cljs

@@ -490,8 +490,16 @@
                             page-name-in-block
                             page-name
                             redirect-page-name page-entity contents-page? children html-export? label)]
-      (if (and (not (util/mobile?)) (not preview?))
+      (cond
+        (:breadcrumb? config)
+        (or (:block/original-name page)
+            (:block/name page))
+
+        (and (not (util/mobile?))
+             (not preview?))
         (page-preview-trigger (assoc config :children inner) page-name)
+
+        :else
         inner))))
 
 (rum/defc asset-reference
@@ -1404,6 +1412,7 @@
 (rum/defc block-children < rum/reactive
   [config children collapsed?]
   (let [ref? (:ref? config)
+        query? (:custom-query? config)
         children (and (coll? children) (filter map? children))]
     (when (and (coll? children)
                (seq children)
@@ -1419,8 +1428,8 @@
                             (-> config
                                 (assoc :block/uuid (:block/uuid child))
                                 (dissoc :breadcrumb-show? :embed-parent))
-                             ref?
-                             (assoc :ref-child? true))]
+                             (or ref? query?)
+                             (assoc :ref-query-child? true))]
                 (rum/with-key (block-container config child)
                   (:block/uuid child)))))]]))))
 
@@ -2078,11 +2087,13 @@
                   (rest parents)
                   parents)
         more? (> (count parents) level-limit)
-        parents (if more? (take-last level-limit parents) parents)]
+        parents (if more? (take-last level-limit parents) parents)
+        config (assoc config :breadcrumb? true)]
     (when show?
       (let [page-name-props (when show-page?
                               [page
-                               (or page-original-name page-name)])
+                               (page-cp (dissoc config :breadcrumb? true) page)
+                               {:block/name (or page-original-name page-name)}])
             parents-props (doall
                            (for [{:block/keys [uuid name content] :as block} parents]
                              (when-not name ; not page
@@ -2226,37 +2237,11 @@
        (= (:id config)
           (str (:block/uuid block)))))
 
-(rum/defcs ^:large-vars/cleanup-todo block-container < rum/reactive
-  {:init (fn [state]
-           (let [[config block] (:rum/args state)
-                 block-id (:block/uuid block)]
-             (cond
-               (root-block? config block)
-               (state/set-collapsed-block! block-id false)
-
-               (:ref? config)
-               (state/set-collapsed-block! block-id
-                                           (editor-handler/block-default-collapsed? block config))
-
-               :else
-               nil)
-             (assoc state
-                    ::control-show? (atom false)
-                    ::navigating-block (atom (:block/uuid block)))))
-   :should-update (fn [old-state new-state]
-                    (let [compare-keys [:block/uuid :block/content :block/parent :block/collapsed?
-                                        :block/properties :block/left :block/children :block/_refs :ui/selected?]
-                          config-compare-keys [:show-cloze?]
-                          b1 (second (:rum/args old-state))
-                          b2 (second (:rum/args new-state))
-                          result (or
-                                  (not= (select-keys b1 compare-keys)
-                                        (select-keys b2 compare-keys))
-                                  (not= (select-keys (first (:rum/args old-state)) config-compare-keys)
-                                        (select-keys (first (:rum/args new-state)) config-compare-keys)))]
-                      (boolean result)))}
-  [state config block]
-  (let [repo (state/get-current-repo)
+(rum/defc ^:large-vars/cleanup-todo block-container-inner < rum/reactive db-mixins/query
+  [state repo config block]
+  (let [ref? (:ref? config)
+        custom-query? (boolean (:custom-query? config))
+        ref-or-custom-query? (or ref? custom-query?)
         *navigating-block (get state ::navigating-block)
         navigating-block (rum/react *navigating-block)
         navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)
@@ -2278,26 +2263,27 @@
                  config)
         heading? (or (= type :heading) (and heading-level (<= heading-level 6)))
         *control-show? (get state ::control-show?)
-        ref? (:ref? config)
         db-collapsed? (util/collapsed? block)
         collapsed? (cond
-                     (or ref? (root-block? config block))
+                     (or ref-or-custom-query? (root-block? config block))
                      (state/sub-collapsed uuid)
 
                      :else
                      db-collapsed?)
+        children (if (and ref-or-custom-query?
+                          (not collapsed?))
+                   (map
+                     (fn [b] (assoc b
+                                    :block/level (inc (:block/level block))))
+                     (model/sub-block-direct-children repo uuid))
+                   children)
         breadcrumb-show? (:breadcrumb-show? config)
         slide? (boolean (:slide? config))
-        custom-query? (boolean (:custom-query? config))
         doc-mode? (:document/mode? config)
         embed? (:embed? config)
         reference? (:reference? config)
         block-id (str "ls-block-" blocks-container-id "-" uuid)
-        has-child? (boolean
-                    (and
-                     (not pre-block?)
-                     (coll? children)
-                     (seq children)))
+        has-child? (first (:block/_parent (db/entity (:db/id block))))
         attrs (on-drag-and-mouse-attrs block uuid top? block-id *move-to)
         children-refs (get-children-refs children)
         data-refs (build-refs-data-value children-refs)
@@ -2358,6 +2344,48 @@
 
      (dnd-separator-wrapper block block-id slide? false false)]))
 
+(rum/defcs block-container < rum/reactive
+  {:init (fn [state]
+           (let [[config block] (:rum/args state)
+                 block-id (:block/uuid block)]
+             (cond
+               (root-block? config block)
+               (state/set-collapsed-block! block-id false)
+
+               (or (:ref? config) (:custom-query? config))
+               (state/set-collapsed-block! block-id
+                                           (editor-handler/block-default-collapsed? block config))
+
+               :else
+               nil)
+             (assoc state
+                    ::control-show? (atom false)
+                    ::navigating-block (atom (:block/uuid block)))))
+   :should-update (fn [old-state new-state]
+                    (let [compare-keys [:block/uuid :block/content :block/parent :block/collapsed?
+                                        :block/properties :block/left :block/children :block/_refs :ui/selected?]
+                          config-compare-keys [:show-cloze?]
+                          b1 (second (:rum/args old-state))
+                          b2 (second (:rum/args new-state))
+                          result (or
+                                  (not= (select-keys b1 compare-keys)
+                                        (select-keys b2 compare-keys))
+                                  (not= (select-keys (first (:rum/args old-state)) config-compare-keys)
+                                        (select-keys (first (:rum/args new-state)) config-compare-keys)))]
+                      (boolean result)))}
+  [state config block]
+  (let [repo (state/get-current-repo)
+        ref? (:ref? config)
+        custom-query? (boolean (:custom-query? config))
+        ref-or-custom-query? (or ref? custom-query?)]
+    (if (and ref-or-custom-query? (not (:ref-query-child? config)))
+      (ui/lazy-visible
+       nil
+       (fn []
+         (block-container-inner state repo config block))
+       nil)
+      (block-container-inner state repo config block))))
+
 (defn divide-lists
   [[f & l]]
   (loop [l l
@@ -2682,7 +2710,10 @@
   [config q]
   (ui/catch-error
    (ui/block-error "Query Error:" {:content (:query q)})
-   (custom-query* config q)))
+   (ui/lazy-visible
+    "loading ..."
+    (fn [] (custom-query* config q))
+    nil)))
 
 (defn admonition
   [config type result]

src/main/frontend/components/journal.cljs

@@ -21,7 +21,7 @@
   (when-let [page-e (db/pull [:block/name (util/page-name-sanity-lc page)])]
     (page/page-blocks-cp repo page-e {})))
 
-(rum/defc journal-cp < rum/reactive
+(rum/defc journal-cp-inner < rum/reactive
   [[title format]]
   (let [;; Don't edit the journal title
         page (string/lower-case title)
@@ -60,9 +60,13 @@
      (page/today-queries repo today? false)
 
      (rum/with-key
-       (reference/references title)
+       (reference/references title false)
        (str title "-refs"))]))
 
+(rum/defc journal-cp
+  [journal]
+  (ui/lazy-visible nil (fn [] (journal-cp-inner journal)) nil))
+
 (rum/defc journals < rum/reactive
   [latest-journals]
   [:div#journals

src/main/frontend/components/page.cljs

@@ -109,8 +109,15 @@
      [:a.add-button-link.block
       (ui/icon "circle-plus")]]]])
 
-(rum/defc page-blocks-cp < rum/reactive
-  db-mixins/query
+(rum/defc page-blocks-cp < rum/reactive db-mixins/query
+  {:will-mount (fn [state]
+                 (let [page-e (second (:rum/args state))
+                       page-name (:block/name page-e)]
+                   (when (and (db/journal-page? page-name)
+                              (>= (date/journal-title->int page-name)
+                                  (date/journal-title->int (date/today))))
+                     (state/pub-event! [:journal/insert-template page-name])))
+                 state)}
   [repo page-e {:keys [sidebar?] :as config}]
   (when page-e
     (let [page-name (or (:block/name page-e)
@@ -387,7 +394,7 @@
        ;; referenced blocks
        [:div {:key "page-references"}
         (rum/with-key
-          (reference/references route-page-name)
+          (reference/references route-page-name sidebar?)
           (str route-page-name "-refs"))]
 
        (when-not block?

src/main/frontend/components/reference.cljs

@@ -49,24 +49,6 @@
   (fn [close-fn]
     (filter-dialog-inner filters-atom close-fn references page-name)))
 
-(defn- block-with-ref-level
-  [block level]
-  (if (:block/children block)
-    (-> (update block :block/children
-                (fn [blocks]
-                  (map (fn [block]
-                         (let [level (inc level)
-                               block (assoc block :ref/level level)]
-                           (block-with-ref-level block level))) blocks)))
-        (assoc :ref/level level))
-    (assoc block :ref/level level)))
-
-(defn- blocks-with-ref-level
-  [page-blocks]
-  (map (fn [[page blocks]]
-         [page (map #(block-with-ref-level % 1) blocks)])
-    page-blocks))
-
 (rum/defc block-linked-references < rum/reactive db-mixins/query
   [block-id]
   (let [refed-blocks-ids (model-db/get-referenced-blocks-ids (str block-id))]
@@ -163,8 +145,7 @@
                      filters (when (seq filter-state)
                                (->> (group-by second filter-state)
                                     (medley/map-vals #(map first %))))
-                     filtered-ref-blocks (->> (block-handler/filter-blocks repo ref-blocks filters true)
-                                              blocks-with-ref-level)
+                     filtered-ref-blocks (block-handler/filter-blocks repo ref-blocks filters true)
                      n-ref (apply +
                              (for [[_ rfs] filtered-ref-blocks]
                                (count rfs)))]
@@ -185,10 +166,16 @@
               :title-trigger? true}))]]))))
 
 (rum/defc references
-  [page-name]
+  [page-name sidebar?]
   (ui/catch-error
    (ui/component-error "Linked References: Unexpected error")
-   (references* page-name)))
+   (ui/lazy-visible
+    (if (or sidebar? (gp-util/uuid-string? page-name))
+      nil
+      "loading references...")
+    (fn []
+      (references* page-name))
+    nil)))
 
 (rum/defcs unlinked-references-aux
   < rum/reactive db-mixins/query

src/main/frontend/config.cljs

@@ -23,6 +23,7 @@
 ;; (goog-define LOGIN-URL
 ;;              "https://logseq.auth.us-east-1.amazoncognito.com/login?client_id=7ns5v1pu8nrbs04rvdg67u4a7c&response_type=code&scope=email+openid+phone&redirect_uri=logseq%3A%2F%2Fauth-callback")
 ;; (goog-define API-DOMAIN "api-prod.logseq.com")
+;; (goog-define WS-URL "wss://b2rp13onu2.execute-api.us-east-1.amazonaws.com/production?graphuuid=%s")
 
 ;; dev env
 (goog-define FILE-SYNC-PROD? false)

src/main/frontend/db/model.cljs

@@ -815,6 +815,20 @@
           block-uuid)
         (sort-by-left (db-utils/entity [:block/uuid block-uuid])))))
 
+(defn sub-block-direct-children
+  "Doesn't include nested children."
+  [repo block-uuid]
+  (when-let [db (conn/get-db repo)]
+    (-> (react/q repo [:frontend.db.react/block-direct-children block-uuid] {}
+          '[:find [(pull ?b [*]) ...]
+            :in $ ?parent-id
+            :where
+            [?parent :block/uuid ?parent-id]
+            [?b :block/parent ?parent]]
+          block-uuid)
+        react
+        (sort-by-left (db-utils/entity [:block/uuid block-uuid])))))
+
 (defn get-block-children
   "Including nested children."
   [repo block-uuid]
@@ -970,16 +984,6 @@
         (reverse)
         (take n))))))
 
-(defn journal-day-exists?
-  [graph day]
-  (d/q
-    '[:find ?p .
-      :in $ ?day
-      :where
-      [?p :block/journal-day ?day]]
-    (conn/get-db graph)
-    day))
-
 ;; get pages that this page referenced
 (defn get-page-referenced-pages
   [repo page]
@@ -1082,38 +1086,20 @@
        (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)]))
              pages (page-alias-set repo page)
              aliases (set/difference pages #{page-id})
-             query-result (if (seq aliases)
-                            (let [rules '[[(find-blocks ?block ?ref-page ?pages ?alias ?aliases)
-                                           [?block :block/page ?alias]
-                                           [(contains? ?aliases ?alias)]]
-                                          [(find-blocks ?block ?ref-page ?pages ?alias ?aliases)
-                                           [?block :block/refs ?ref-page]
-                                           [(contains? ?pages ?ref-page)]]]]
-                              (react/q repo
-                                       [:frontend.db.react/page<-blocks-or-block<-blocks page-id]
-                                       {}
-                                       '[:find [(pull ?block ?block-attrs) ...]
-                                         :in $ % ?pages ?aliases ?block-attrs
-                                         :where
-                                         (find-blocks ?block ?ref-page ?pages ?alias ?aliases)]
-                                       rules
-                                       pages
-                                       aliases
-                                       block-attrs))
-                            (react/q repo
-                                     [:frontend.db.react/page<-blocks-or-block<-blocks page-id]
-                                     {:use-cache? false}
-                                     '[:find [(pull ?ref-block ?block-attrs) ...]
-                                       :in $ ?page ?block-attrs
-                                       :where
-                                       [?ref-block :block/refs ?page]]
-                                     page-id
-                                     block-attrs))
+             query-result (react/q repo
+                            [:frontend.db.react/page<-blocks-or-block<-blocks page-id]
+                            {}
+                            '[:find [(pull ?block ?block-attrs) ...]
+                              :in $ [?ref-page ...] ?block-attrs
+                              :where
+                              [?block :block/refs ?ref-page]]
+                            pages
+                            (butlast block-attrs))
              result (->> query-result
                          react
-                         (sort-by-left-recursive)
                          (remove (fn [block]
                                    (= page-id (:db/id (:block/page block)))))
+                         (sort-by-left-recursive)
                          db-utils/group-by-page
                          (map (fn [[k blocks]]
                                 (let [k (if (contains? aliases (:db/id k))
@@ -1129,35 +1115,14 @@
   ([repo page]
    (when repo
      (when-let [db (conn/get-db repo)]
-       (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)]))
-             pages (page-alias-set repo page)
-             aliases (set/difference pages #{page-id})
-             query-result (if (seq aliases)
-                            (let [rules '[[(find-blocks ?block ?ref-page ?pages ?alias ?aliases)
-                                           [?block :block/page ?alias]
-                                           [(contains? ?aliases ?alias)]]
-                                          [(find-blocks ?block ?ref-page ?pages ?alias ?aliases)
-                                           [?block :block/refs ?ref-page]
-                                           [(contains? ?pages ?ref-page)]]]]
-                              (d/q
-                                '[:find ?block
-                                  :in $ % ?pages ?aliases ?block-attrs
-                                  :where
-                                  (find-blocks ?block ?ref-page ?pages ?alias ?aliases)]
-                                db
-                                rules
-                                pages
-                                aliases
-                                block-attrs))
-                            (d/q
-                              '[:find ?ref-block
-                                :in $ ?page ?block-attrs
-                                :where
-                                [?ref-block :block/refs ?page]]
-                              db
-                              page-id
-                              block-attrs))]
-         query-result)))))
+       (let [pages (page-alias-set repo page)]
+         (d/q
+           '[:find ?block
+             :in $ [?ref-page ...]
+             :where
+             [?block :block/refs ?ref-page]]
+           db
+           pages))))))
 
 (defn get-date-scheduled-or-deadlines
   [journal-title]

src/main/frontend/db/react.cljs

@@ -24,6 +24,8 @@
 ;; ::block-and-children
 ;; get block&children react-query
 (s/def ::block-and-children (s/tuple #(= ::block-and-children %) uuid?))
+
+(s/def ::block-direct-children (s/tuple #(= ::block-direct-children %) uuid?))
 ;; ::journals
 ;; get journal-list react-query
 (s/def ::journals (s/tuple #(= ::journals %)))
@@ -48,6 +50,7 @@
 (s/def ::react-query-keys (s/or :block ::block
                                 :page-blocks ::page-blocks
                                 :block-and-children ::block-and-children
+                                :block-direct-children ::block-direct-children
                                 :journals ::journals
                                 :page->pages ::page->pages
                                 :page<-pages ::page<-pages
@@ -213,9 +216,8 @@
 
 (defn get-affected-queries-keys
   "Get affected queries through transaction datoms."
-  [{:keys [tx-data]}]
+  [{:keys [tx-data db-before]}]
   {:post [(s/valid? ::affected-keys %)]}
-
   (let [blocks (->> (filter (fn [datom] (contains? #{:block/left :block/parent :block/page} (:a datom))) tx-data)
                     (map :v)
                     (distinct))
@@ -237,8 +239,15 @@
                                              (:db/id (:block/page block)))
                                     blocks [[::block (:block/uuid block)]]
                                     others (when page-id
-                                             [[::page-blocks page-id]
-                                              [::page->pages page-id]])]
+                                             (let [db-after-parent-uuid (:block/uuid (:block/parent block))
+                                                   db-before-parent-uuid (:block/uuid (:block/parent (d/entity db-before
+                                                                                                               [:block/uuid (:block/uuid block)])))]
+                                               [[::page-blocks page-id]
+                                                [::page->pages page-id]
+                                                [::block-direct-children db-after-parent-uuid]
+                                                (when (and db-before-parent-uuid
+                                                           (not= db-before-parent-uuid db-after-parent-uuid))
+                                                  [::block-direct-children db-before-parent-uuid])]))]
                                 (concat blocks others)))))
                         blocks)
 
@@ -251,7 +260,7 @@
                                 (if (:block/name entity) ; page
                                   [::page-blocks ref]
                                   [::page-blocks (:db/id (:block/page entity))])))
-                            refs))
+                         refs))
         others (->>
                 (keys @query-state)
                 (filter (fn [ks]

src/main/frontend/fs/sync.cljs

@@ -71,6 +71,7 @@
 (s/def ::sync-state (s/keys :req-un [::state
                                      ::current-local->remote-files
                                      ::current-remote->local-files
+                                     ::queued-local->remote-files
                                      ::history]))
 
 ;; diff
@@ -106,7 +107,9 @@
 
 (def graphs-txid (persist-var/persist-var nil "graphs-txid"))
 
-(defn update-graphs-txid! [latest-txid graph-uuid user-uuid repo]
+(defn update-graphs-txid!
+  [latest-txid graph-uuid user-uuid repo]
+  {:pre [(int? latest-txid) (>= latest-txid 0)]}
   (persist-var/-reset-value! graphs-txid [user-uuid graph-uuid latest-txid] repo)
   (persist-var/persist-save graphs-txid))
 
@@ -837,6 +840,71 @@
                 not)
     (go (>! local-changes-chan (->FileChangeEvent type dir path stat)))))
 
+;;; ### sync state
+
+
+(defn sync-state
+  "create a new sync-state"
+  []
+  {:post [(s/valid? ::sync-state %)]}
+  {:state ::idle
+   :current-local->remote-files #{}
+   :current-remote->local-files #{}
+   :queued-local->remote-files #{}
+   :history '()})
+
+(defn- sync-state--update-state
+  [sync-state next-state]
+  {:pre [(s/valid? ::state next-state)]
+   :post [(s/valid? ::sync-state %)]}
+  (assoc sync-state :state next-state))
+
+(defn sync-state--add-current-remote->local-files
+  [sync-state paths]
+  {:post [(s/valid? ::sync-state %)]}
+  (update sync-state :current-remote->local-files into paths))
+
+(defn sync-state--add-current-local->remote-files
+  [sync-state paths]
+  {:post [(s/valid? ::sync-state %)]}
+  (update sync-state :current-local->remote-files into paths))
+
+(defn sync-state--update-queued-local->remote-files
+  [sync-state paths]
+  {:post [(s/valid? ::sync-state %)]}
+  (update sync-state :queued-local->remote-files (fn [_ n] (set n)) paths))
+
+(defn- add-history-items
+  [history paths now]
+  (sequence
+   (comp
+    ;; only reserve the latest one of same-path-items
+    (dedupe-by :path)
+    ;; reserve the latest 20 history items
+    (take 20))
+   (into history
+         (map (fn [path] {:path path :time now}) paths))))
+
+(defn sync-state--remove-current-remote->local-files
+  [sync-state paths]
+  {:post [(s/valid? ::sync-state %)]}
+  (let [now (t/now)]
+    (-> sync-state
+        (update :current-remote->local-files set/difference paths)
+        (update :history add-history-items paths now))))
+
+(defn sync-state--remove-current-local->remote-files
+  [sync-state paths]
+  {:post [(s/valid? ::sync-state %)]}
+  (let [now (t/now)]
+    (-> sync-state
+        (update :current-local->remote-files set/difference paths)
+        (update :history add-history-items paths now))))
+
+(defn sync-state--stopped?
+  [sync-state]
+  (= ::stop (:state sync-state)))
+
 ;;; ### remote->local syncer & local->remote syncer
 
 (defprotocol IRemote->LocalSync
@@ -953,32 +1021,26 @@
 (defn- contains-path? [regexps path]
   (reduce #(when (re-find %2 path) (reduced true)) false regexps))
 
-(defn- filter-local-changes
+(defn- filter-local-changes-pred
   "filter local-change events:
   - for 'unlink' event
     - when related file exists on local dir, ignore this event
   - for 'add' | 'change' event
     - when related file's content is same as remote file, ignore it"
-  [to-ch from-ch basepath graph-uuid]
-  (async/pipeline-async
-   1 to-ch
-   (fn [^FileChangeEvent e result]
-     (go
-       (case (.-type e)
-         "unlink"
-         (let [r (<! (get-local-files-meta rsapi "" basepath [(relative-path e)]))]
-           (when (some-> r ex-cause ;; str (string/index-of "No such file or directory")
-                         )
-             (>! result e)))
-
-         ("add" "change")
-         (let [path (relative-path e)]
-           (when (and (<! (local-file-exists? path basepath))
-                      (<! (file-changed? graph-uuid path basepath)))
-             (>! result e))))
-       (async/close! result)))
-   from-ch false))
-
+  [^FileChangeEvent e basepath graph-uuid]
+  (go
+    (let [r-path (relative-path e)]
+      (case (.-type e)
+        "unlink"
+        (let [r (<! (get-local-files-meta rsapi "" basepath [r-path]))]
+          ;; keep this e when it's not found
+          (some-> r ex-cause))
+
+        ("add" "change")
+        ;; 1. local file exists
+        ;; 2. compare with remote file, and changed
+        (and (<! (local-file-exists? r-path basepath))
+             (<! (file-changed? graph-uuid r-path basepath)))))))
 
 (defrecord ^:large-vars/cleanup-todo
     Local->RemoteSyncer [user-uuid graph-uuid base-path repo *sync-state
@@ -1009,7 +1071,7 @@
       (let [c (.filtered-chan this 10000)
             filter-e-fn (.filter-file-change-events-fn this)]
         (go-loop [timeout-c (timeout rate)
-                  tcoll (transient [])]
+                  coll []]
           (let [{:keys [timeout ^FileChangeEvent e stop]}
                 (async/alt! timeout-c {:timeout true}
                             from-chan ([e] {:e e})
@@ -1019,22 +1081,22 @@
               (async/close! c)
 
               timeout
-              (let [from-c (chan 10000)]
-                (<! (async/onto-chan! from-c (distinct (persistent! tcoll)) false))
-                (filter-local-changes c from-c base-path graph-uuid)
-                (async/close! from-c)
-                (recur (async/timeout rate) (transient [])))
+              (do (async/onto-chan! c coll false)
+                  (swap! *sync-state sync-state--update-queued-local->remote-files nil)
+                  (recur (async/timeout rate) []))
 
               (some? e)
-              (do
-                (when (filter-e-fn e)
-                  (conj! tcoll e))
-                (recur timeout-c tcoll))
+              (if (and (filter-e-fn e)
+                           (<! (filter-local-changes-pred e base-path graph-uuid)))
+                    (let [coll* (distinct (conj coll e))]
+                      (swap! *sync-state sync-state--update-queued-local->remote-files
+                             (mapv relative-path coll*))
+                      (recur timeout-c coll*))
+                    (recur timeout-c coll))
 
               (nil? e)
-              (do
-                (println "close ratelimit chan")
-                (async/close! c)))))
+              (do (println "close ratelimit chan")
+                  (async/close! c)))))
         c))
 
 
@@ -1114,64 +1176,7 @@
                     (or need-sync-remote unknown) r)))))))))
 
 
-;;; ### sync state
-
-
-(defn sync-state
-  "create a new sync-state"
-  []
-  {:post [(s/valid? ::sync-state %)]}
-  {:state ::idle
-   :current-local->remote-files #{}
-   :current-remote->local-files #{}
-   :history '()})
-
-(defn- sync-state--update-state
-  [sync-state next-state]
-  {:pre [(s/valid? ::state next-state)]
-   :post [(s/valid? ::sync-state %)]}
-  (assoc sync-state :state next-state))
-
-(defn sync-state--add-current-remote->local-files
-  [sync-state paths]
-  {:post [(s/valid? ::sync-state %)]}
-  (update sync-state :current-remote->local-files into paths))
-
-(defn sync-state--add-current-local->remote-files
-  [sync-state paths]
-  {:post [(s/valid? ::sync-state %)]}
-  (update sync-state :current-local->remote-files into paths))
-
-(defn- add-history-items
-  [history paths now]
-  (sequence
-   (comp
-    ;; only reserve the latest one of same-path-items
-    (dedupe-by :path)
-    ;; reserve the latest 20 history items
-    (take 20))
-   (into history
-         (map (fn [path] {:path path :time now}) paths))))
-
-(defn sync-state--remove-current-remote->local-files
-  [sync-state paths]
-  {:post [(s/valid? ::sync-state %)]}
-  (let [now (t/now)]
-    (-> sync-state
-        (update :current-remote->local-files set/difference paths)
-        (update :history add-history-items paths now))))
-
-(defn sync-state--remove-current-local->remote-files
-  [sync-state paths]
-  {:post [(s/valid? ::sync-state %)]}
-  (let [now (t/now)]
-    (-> sync-state
-        (update :current-local->remote-files set/difference paths)
-        (update :history add-history-items paths now))))
 
-(defn sync-state--stopped?
-  [sync-state]
-  (= ::stop (:state sync-state)))
 
 
 ;;; ### put all stuff together

src/main/frontend/handler/editor.cljs

@@ -454,11 +454,12 @@
 (declare save-current-block!)
 (defn outliner-insert-block!
   [config current-block new-block {:keys [sibling? keep-uuid? replace-empty-target?]}]
-  (let [ref-top-block? (and (:ref? config)
-                            (not (:ref-child? config)))
+  (let [ref-query-top-block? (and (or (:ref? config)
+                                      (:custom-query? config))
+                                  (not (:ref-query-child? config)))
         has-children? (db/has-children? (:block/uuid current-block))
         sibling? (cond
-                   ref-top-block?
+                   ref-query-top-block?
                    false
 
                    (boolean? sibling?)
@@ -2884,11 +2885,13 @@
   [text e]
   (let [copied-blocks (state/get-copied-blocks)
         copied-block-ids (:copy/block-ids copied-blocks)
+        copied-graph (:copy/graph copied-blocks)
         input (state/get-input)
         *stop-event? (atom true)]
     (cond
       ;; Internal blocks by either copy or cut blocks
       (and
+       (= copied-graph (state/get-current-repo))
        (or (seq copied-block-ids)
            (seq (:copy/full-blocks copied-blocks)))
        text
@@ -3474,12 +3477,11 @@
   1. References.
   2. Custom queries."
   [block config]
-  (if (or (:ref? config)
-          (:custom-query? config))
-    (and
-     (seq (:block/children block))
-     (or
-      (:custom-query? config)
-      (>= (:ref/level block)
-          (state/get-ref-open-blocks-level))))
-    (util/collapsed? block)))
+  (or
+   (and
+    (or (:ref? config) (:custom-query? config))
+    (>= (inc (:block/level block))
+        (state/get-ref-open-blocks-level))
+    ;; has children
+    (first (:block/_parent (db/entity (:db/id block)))))
+   (util/collapsed? block)))

src/main/frontend/handler/events.cljs

@@ -346,12 +346,11 @@
 
 (defmethod handle :file-watcher/changed [[_ ^js event]]
   (let [type (.-event event)
-        payload (js->clj event :keywordize-keys true)
-        payload' (-> payload
-                     (update :path js/decodeURI))]
-    (prn ::fs-watcher payload)
-    (fs-watcher/handle-changed! type payload')
-    (sync/file-watch-handler type payload')))
+        payload (-> event
+                    (js->clj :keywordize-keys true)
+                    (update :path js/decodeURI))]
+    (fs-watcher/handle-changed! type payload)
+    (sync/file-watch-handler type payload)))
 
 (defmethod handle :rebuild-slash-commands-list [[_]]
   (page-handler/rebuild-slash-commands-list!))
@@ -388,6 +387,16 @@
     db-encrypted-secret
     close-fn)))
 
+(defmethod handle :journal/insert-template [[_ page-name]]
+  (let [page-name (util/page-name-sanity-lc page-name)]
+    (when-let [page (db/pull [:block/name page-name])]
+      (when (db/page-empty? (state/get-current-repo) page-name)
+        (when-let [template (state/get-default-journal-template)]
+          (editor-handler/insert-template!
+           nil
+           template
+           {:target page}))))))
+
 (defn run!
   []
   (let [chan (state/get-events-chan)]

src/main/frontend/handler/page.cljs

@@ -728,31 +728,24 @@
                 (and (= "local" repo) (not (mobile-util/is-native-platform?))))
         (let [title (date/today)
               today-page (util/page-name-sanity-lc title)
-              template (state/get-default-journal-template)
               format (state/get-preferred-format repo)
               file-name (date/journal-title->default title)
               path (str (config/get-journals-directory) "/" file-name "."
                         (config/get-file-extension format))
               file-path (str "/" path)
-              repo-dir (config/get-repo-dir repo)]
+              repo-dir (config/get-repo-dir repo)
+              template (state/get-default-journal-template)]
           (p/let [file-exists? (fs/file-exists? repo-dir file-path)
                   file-content (when file-exists?
                                  (fs/read-file repo-dir file-path))]
             (when (and (db/page-empty? repo today-page)
-                       (not (model/journal-day-exists? repo
-                                                       (date/journal-title->int (date/today))))
                        (or (not file-exists?)
                            (and file-exists? (string/blank? file-content))))
               (create! title {:redirect? false
                               :split-namespace? false
                               :create-first-block? (not template)
                               :journal? true})
-              (when template
-                (let [page (db/pull [:block/name today-page])]
-                  (editor-handler/insert-template!
-                   nil
-                   template
-                   {:target page})))
+              (state/pub-event! [:journal/insert-template today-page])
               (ui-handler/re-render-root!))))))))
 
 (defn open-today-in-sidebar

src/main/frontend/handler/repo.cljs

@@ -212,6 +212,7 @@
     (ui-handler/re-render-root! re-render-opts))
   (state/pub-event! [:graph/added repo-url opts])
   (state/reset-parsing-state!)
+  (state/set-loading-files! repo-url false)
   (async/offer! graph-added-chan true))
 
 (defn- parse-files-and-create-default-files-inner!
@@ -262,15 +263,9 @@
       (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts))
     (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths db-encrypted? re-render? re-render-opts opts)))
 
-(defn- update-parsing-state!
-  [repo-url]
-  (state/set-loading-files! repo-url false))
-
 (defn parse-files-and-load-to-db!
   [repo-url files {:keys [delete-files delete-blocks re-render? re-render-opts _refresh?] :as opts
                    :or {re-render? true}}]
-  (update-parsing-state! repo-url)
-
   (let [file-paths (map :file/path files)
         metadata-file (config/get-metadata-path)
         metadata-content (some #(when (= (:file/path %) metadata-file)

src/main/frontend/state.cljs

@@ -182,7 +182,9 @@
      :graph/parsing-state                   {}
 
      ;; copied blocks
-     :copy/blocks                           {:copy/content nil :copy/block-ids nil}
+     :copy/blocks                           {:copy/content nil
+                                             :copy/block-ids nil
+                                             :copy/graph nil}
 
      :copy/export-block-text-indent-style   (or (storage/get :copy/export-block-text-indent-style)
                                                 "dashes")
@@ -1426,13 +1428,15 @@
 
 (defn set-copied-blocks
   [content ids]
-  (set-state! :copy/blocks {:copy/content content
+  (set-state! :copy/blocks {:copy/graph (get-current-repo)
+                            :copy/content content
                             :copy/block-ids ids
                             :copy/full-blocks nil}))
 
 (defn set-copied-full-blocks
   [content blocks]
-  (set-state! :copy/blocks {:copy/content content
+  (set-state! :copy/blocks {:copy/graph (get-current-repo)
+                            :copy/content content
                             :copy/full-blocks blocks}))
 
 (defn set-copied-full-blocks!

src/main/frontend/ui.cljs

@@ -25,6 +25,7 @@
             ["react-tippy" :as react-tippy]
             ["react-transition-group" :refer [CSSTransition TransitionGroup]]
             ["@logseq/react-tweet-embed" :as react-tweet-embed]
+            ["react-visibility-sensor" :as rvs]
             [rum.core :as rum]
             [frontend.db-mixins :as db-mixins]
             [frontend.mobile.util :as mobile-util]
@@ -37,6 +38,7 @@
 (def resize-consumer (r/adapt-class (gobj/get Resize "ResizeConsumer")))
 (def Tippy (r/adapt-class (gobj/get react-tippy "Tooltip")))
 (def ReactTweetEmbed (r/adapt-class react-tweet-embed))
+(def visibility-sensor (r/adapt-class (gobj/get rvs "default")))
 
 (defn reset-ios-whole-page-offset!
   []
@@ -682,7 +684,7 @@
                         :margin-left -30}}
              (not title-trigger?)
              (assoc :on-mouse-down on-mouse-down))
-           [:span {:class (if @control? "control-show cursor-pointer" "control-hide")}
+           [:span {:class (if (or @control? @collapsed?) "control-show cursor-pointer" "control-hide")}
             (rotating-arrow @collapsed?)]])
         (if (fn? header)
           (header @collapsed?)
@@ -908,3 +910,24 @@
     [:span.text-sm.font-medium
      label-right]]
    (progress-bar width)])
+
+(rum/defc lazy-visible-inner
+  [visible? content-fn loading-label]
+  [:div.lazy-visibility
+   (if visible?
+     (when (fn? content-fn) (content-fn))
+     (when loading-label [:span.text-sm.font-medium
+                          loading-label]))])
+
+(rum/defcs lazy-visible <
+  (rum/local false ::visible?)
+  [state loading-label content-fn sensor-opts]
+  (let [*visible? (::visible? state)]
+    (visibility-sensor
+     (merge
+      {:on-change #(reset! *visible? %)
+       :partialVisibility true
+       :offset {:top -300
+                :bottom -300}}
+      sensor-opts)
+     (lazy-visible-inner @*visible? content-fn loading-label))))

yarn.lock

@@ -6624,7 +6624,7 @@ prompts@^2.3.2:
     kleur "^3.0.3"
     sisteransi "^1.0.5"
 
[email protected], prop-types@^15.5.4, prop-types@^15.6.0, prop-types@^15.6.2:
[email protected], prop-types@^15.5.4, prop-types@^15.6.0, prop-types@^15.6.2, prop-types@^15.7.2:
   version "15.8.1"
   resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5"
   integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==
@@ -6888,6 +6888,13 @@ [email protected]:
     loose-envify "^1.4.0"
     prop-types "^15.6.2"
 
+react-visibility-sensor@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/react-visibility-sensor/-/react-visibility-sensor-5.1.1.tgz#5238380960d3a0b2be0b7faddff38541e337f5a9"
+  integrity sha512-cTUHqIK+zDYpeK19rzW6zF9YfT4486TIgizZW53wEZ+/GPBbK7cNS0EHyJVyHYacwFEvvHLEKfgJndbemWhB/w==
+  dependencies:
+    prop-types "^15.7.2"
+
 [email protected]:
   version "17.0.2"
   resolved "https://registry.yarnpkg.com/react/-/react-17.0.2.tgz#d0b5cc516d29eb3eee383f75b62864cfb6800037"