Merge branch 'feat/db' into refactor/pipeline-worker

.carve/ignore

@@ -96,3 +96,5 @@ frontend.worker.rtc.op-mem-layer/_sync-loop
 frontend.db-worker/init
 ;; For defonce
 frontend.persist-db.browser/_do_not_reload_worker
+;; WIP fn, remove when it's ready
+frontend.db.rtc.asset-sync/<loop-for-assets-sync

deps/common/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v4"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v5"
   },
   "scripts": {
     "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"

deps/common/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v4":
-  version "1.2.173-feat-db-v4"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/ee701fab68a0d34b8638b2eed5037460c9766172"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v5":
+  version "1.2.173-feat-db-v5"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/090ad77480ad8065aa051e625c248bc77e07efa5"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/db/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v4"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v5"
   },
   "dependencies": {
     "better-sqlite3": "8.0.1"

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -227,9 +227,11 @@
   (vec
    (concat
     [:map]
-    [[:block/parent :int]
+    [[:block/content :string]
+     [:block/parent :int]
      ;; These blocks only associate with pages of type "whiteboard"
-     [:block/page :int]]
+     [:block/page :int]
+     [:block/path-refs {:optional true} [:set :int]]]
     page-or-block-attrs)))
 
 (def closed-value-block

deps/db/src/logseq/db/frontend/property/type.cljs

@@ -21,17 +21,14 @@
   "Valid schema :type for closed values"
   #{:default :number :url :date :page})
 
-(def closed-value-property-position-types
-  "Valid schema :type for closed values"
-  #{:default :number :url})
-
 (assert (set/subset? closed-value-property-types (set user-built-in-property-types))
         "All closed value types are valid property types")
 
 (def ^:private user-built-in-allowed-schema-attributes
   "Map of types to their set of allowed :schema attributes"
   (merge-with into
-              (zipmap closed-value-property-types (repeat #{:values :position}))
+              (zipmap closed-value-property-types (repeat #{:values}))
+              (zipmap #{:default :number :url} (repeat #{:position}))
               {:number #{:cardinality}
                :date #{:cardinality}
                :url #{:cardinality}

deps/db/src/logseq/db/sqlite/common_db.cljs

@@ -12,7 +12,7 @@
        vec))
 
 (defn restore-initial-data
-  "Given initial sqlite data, returns a datascript connection"
+  "Given initial sqlite data and schema, returns a datascript connection"
   [datoms schema]
   (d/conn-from-datoms datoms schema))
 

deps/db/src/logseq/db/sqlite/db.cljs

@@ -1,7 +1,6 @@
 (ns ^:node-only logseq.db.sqlite.db
   "Sqlite fns for db graphs"
-  (:require ["path" :as node-path]
-            ["better-sqlite3" :as sqlite3]
+  (:require ["better-sqlite3" :as sqlite3]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
             ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
             #_:clj-kondo/ignore

deps/db/test/logseq/db/sqlite/db_test.cljs

@@ -4,6 +4,7 @@
             ["path" :as node-path]
             [datascript.core :as d]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
+            [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.db :as sqlite-db]))
 
 (use-fixtures
@@ -32,7 +33,7 @@
           _ (d/transact! conn* blocks)
           ;; Simulate getting data from sqlite and restoring it for frontend
           conn (-> (sqlite-common-db/get-initial-data @conn*)
-                   sqlite-common-db/restore-initial-data)]
+                   (sqlite-common-db/restore-initial-data db-schema/schema-for-db-based-graph))]
       (is (= blocks
              (->> @conn
                   (d/q '[:find (pull ?b [:block/uuid :file/path :file/content]) :where [?b :file/content]])
@@ -61,7 +62,7 @@
           _ (d/transact! conn* blocks)
           ;; Simulate getting data from sqlite and restoring it for frontend
           conn (-> (sqlite-common-db/get-initial-data @conn*)
-                   sqlite-common-db/restore-initial-data)]
+                   (sqlite-common-db/restore-initial-data db-schema/schema-for-db-based-graph))]
       (is (= blocks
              (->> (d/q '[:find (pull ?b [*])
                          :where [?b :block/created-at]]

deps/db/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v4":
-  version "1.2.173-feat-db-v4"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/ee701fab68a0d34b8638b2eed5037460c9766172"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v5":
+  version "1.2.173-feat-db-v5"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/090ad77480ad8065aa051e625c248bc77e07efa5"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/graph-parser/.carve/ignore

@@ -25,8 +25,6 @@ logseq.graph-parser.util/unquote-string
 ;; API
 logseq.graph-parser.util.page-ref/page-ref-re
 ;; API
-logseq.graph-parser.whiteboard/page-block->tldr-page
-;; API
 logseq.graph-parser/get-blocks-to-delete
 ;; API
 logseq.graph-parser.util.db/resolve-input
@@ -42,3 +40,5 @@ logseq.graph-parser.schema.mldoc/block-ast-coll-schema
 logseq.graph-parser.config/img-formats
 ;; API
 logseq.graph-parser.config/text-formats
+;; API
+logseq.graph-parser/import-file-to-db-graph

deps/graph-parser/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v4",
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v5",
     "better-sqlite3": "8.0.1"
   },
   "dependencies": {

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -1,15 +1,16 @@
 (ns logseq.graph-parser
   "Main ns used by logseq app to parse graph from source files and then save to
   the given database connection"
-  (:require [datascript.core :as d]
+  (:require [clojure.set :as set]
+            [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.db.frontend.schema :as db-schema]
+            [logseq.graph-parser.date-time-util :as date-time-util]
             [logseq.graph-parser.extract :as extract]
             [logseq.common.util :as common-util]
-            [logseq.graph-parser.date-time-util :as date-time-util]
             [logseq.common.config :as common-config]
-            [logseq.db.frontend.schema :as db-schema]
             [logseq.db :as ldb]
-            [clojure.string :as string]
-            [clojure.set :as set]))
+            [logseq.db.sqlite.util :as sqlite-util]))
 
 (defn- retract-blocks-tx
   [blocks retain-uuids]
@@ -131,6 +132,144 @@ Options available:
      {:tx result
       :ast ast})))
 
+(defn- get-pid
+  "Get a property's id (name or uuid) given its name. For db graphs"
+  [db property-name]
+  (:block/uuid (d/entity db [:block/name (common-util/page-name-sanity-lc (name property-name))])))
+
+(defn- update-block-with-invalid-tags
+  [block]
+  (if (seq (:block/tags block))
+    (update block :block/tags
+            (fn [tags]
+              (mapv #(-> %
+                         sqlite-util/block-with-timestamps
+                         (merge {:block/journal? false
+                                 :block/format :markdown
+                                 :block/uuid (d/squuid)}))
+                    tags)))
+    block))
+
+(defn- update-imported-block
+  [conn block]
+  (prn ::block block)
+  (let [remove-keys (fn [m pred] (into {} (remove (comp pred key) m)))]
+    (-> block
+        ((fn [block']
+           (cond
+             (seq (:block/macros block'))
+             (update block' :block/macros
+                     (fn [macros]
+                       (mapv (fn [m]
+                               (-> m
+                                   (update :block/properties
+                                           (fn [props]
+                                             (update-keys #(get-pid @conn %) props)))
+                                   (assoc :block/uuid (d/squuid))))
+                             macros)))
+
+             (:block/pre-block? block')
+             block'
+
+             :else
+             (update-in block' [:block/properties]
+                        (fn [props]
+                          (-> props
+                              (update-keys (fn [k]
+                                             (if-let [new-key (get-pid @conn k)]
+                                               new-key
+                                               k)))
+                              (remove-keys keyword?)))))))
+        update-block-with-invalid-tags
+        ((fn [block']
+           (if (seq (:block/refs block'))
+             (update block' :block/refs
+                     (fn [refs]
+                       (mapv #(assoc % :block/format :markdown) refs)))
+             block')))
+        sqlite-util/block-with-timestamps
+        ;; FIXME: Remove when properties are supported
+        (assoc :block/properties {})
+        ;; TODO: org-mode content needs to be handled
+        (assoc :block/format :markdown)
+        ;; TODO: pre-block? can be removed once page properties are imported
+        (dissoc :block/pre-block? :block/properties-text-values :block/properties-order
+                :block/invalid-properties))))
+
+(defn import-file-to-db-graph
+  "Parse file and save parsed data to the given db graph."
+  [conn file content {:keys [delete-blocks-fn extract-options skip-db-transact?]
+                      :or {delete-blocks-fn (constantly [])
+                           skip-db-transact? false}
+                      :as options}]
+  (let [format (common-util/get-format file)
+        {:keys [tx ast]}
+        (let [extract-options' (merge {:block-pattern (common-config/get-block-pattern format)
+                                       :date-formatter "MMM do, yyyy"
+                                       :uri-encoded? false
+                                       :db-graph-mode? true
+                                       :filename-format :legacy}
+                                      extract-options
+                                      {:db @conn})
+              {:keys [pages blocks ast refs]
+               :or   {pages []
+                      blocks []
+                      ast []}}
+              (cond (contains? common-config/mldoc-support-formats format)
+                    (extract/extract file content extract-options')
+
+                    (common-config/whiteboard? file)
+                    (extract/extract-whiteboard-edn file content extract-options')
+
+                    :else nil)
+              ;; remove file path relative
+              pages (map #(dissoc % :block/file :block/properties) pages)
+              block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
+              delete-blocks (delete-blocks-fn @conn (first pages) file block-ids)
+              block-refs-ids (->> (mapcat :block/refs blocks)
+                                  (filter (fn [ref] (and (vector? ref)
+                                                         (= :block/uuid (first ref)))))
+                                  (map (fn [ref] {:block/uuid (second ref)}))
+                                  (seq))
+                 ;; To prevent "unique constraint" on datascript
+              block-ids (set/union (set block-ids) (set block-refs-ids))
+              pages (map #(-> (merge {:block/journal? false} %)
+                              ;; Fix pages missing :block/original-name. Shouldn't happen
+                              ((fn [m]
+                                 (if-not (:block/original-name m)
+                                   (assoc m :block/original-name (:block/name m))
+                                   m)))
+                              sqlite-util/block-with-timestamps
+                              ;; TODO: org-mode content needs to be handled
+                              (assoc :block/format :markdown)
+                              (dissoc :block/properties-text-values :block/properties-order :block/invalid-properties
+                                      :block/whiteboard?)
+                              update-block-with-invalid-tags
+                              ;; FIXME: Remove when properties are supported
+                              (assoc :block/properties {}))
+                         (extract/with-ref-pages pages blocks))
+
+              ;; post-handling
+              whiteboard-pages (->> pages
+                                    (filter #(= "whiteboard" (:block/type %)))
+                                    (map (fn [page-block]
+                                           (-> page-block
+                                               (assoc :block/journal? false
+                                                      :block/format :markdown
+                                                      ;; fixme: missing properties
+                                                      :block/properties {(get-pid @conn :ls-type) :whiteboard-page})))))
+              blocks (map #(update-imported-block conn %) blocks)
+              pages-index (map #(select-keys % [:block/name]) pages)]
+
+          {:tx (concat refs whiteboard-pages pages-index delete-blocks pages block-ids blocks)
+           :ast ast})
+        tx' (common-util/fast-remove-nils tx)
+        result (if skip-db-transact?
+                 tx'
+                 (d/transact! conn tx' (select-keys options [:new-graph? :from-disk?])))]
+    {:tx result
+     :ast ast}))
+
 (defn filter-files
   "Filters files in preparation for parsing. Only includes files that are
   supported by parser"

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -652,8 +652,8 @@
     `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids.
     `format`: content's format, it could be either :markdown or :org-mode.
     `options`: Options supported are :user-config, :block-pattern,
-               :extract-macros, :date-formatter, :page-name and :db"
-  [blocks content with-id? format {:keys [user-config] :as options}]
+               :extract-macros, :date-formatter, :page-name, :db-graph-mode? and :db"
+  [blocks content with-id? format {:keys [user-config db-graph-mode?] :as options}]
   {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]}
   (let [encoded-content (utf8/encode content)
         [blocks body pre-block-properties]
@@ -664,11 +664,13 @@
                body []]
           (if (seq blocks)
             (let [[block pos-meta] (first blocks)
-                  ;; fix start_pos
-                  pos-meta (assoc pos-meta :end_pos
-                                  (if (seq headings)
-                                    (get-in (last headings) [:meta :start_pos])
-                                    nil))]
+                  ;; in db-graph-mode, property part is not included in block/content
+                  pos-meta (if db-graph-mode?
+                             pos-meta
+                             (assoc pos-meta :end_pos
+                                    (if (seq headings)
+                                      (get-in (last headings) [:meta :start_pos])
+                                      nil)))]
               (cond
                 (paragraph-timestamp-block? block)
                 (let [timestamps (extract-timestamps block)

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -75,7 +75,7 @@
   ([dir options]
    (let [config (read-config dir)
          files (or (:files options) (build-graph-files dir config))
-         conn (or (:conn options) (ldb/start-conn {:file-based? true}))
+         conn (or (:conn options) (ldb/start-conn))
          _ (when-not (:files options) (println "Parsing" (count files) "files..."))
          asts (parse-files conn files (merge options {:config config}))]
      {:conn conn

deps/graph-parser/src/logseq/graph_parser/whiteboard.cljs

@@ -8,9 +8,6 @@
 (defn block->shape [block]
   (get-in block [:block/properties :logseq.tldraw.shape]))
 
-(defn page-block->tldr-page [block]
-  (get-in block [:block/properties :logseq.tldraw.page]))
-
 (defn shape-block? [block]
   (= :whiteboard-shape (get-in block [:block/properties :ls-type])))
 
@@ -75,6 +72,8 @@
                     (str "whiteboard " (:type shape)))})
 
 (defn with-whiteboard-block-props
+  "Builds additional block attributes for a whiteboard block. Expects :block/properties
+   to be in file graph format"
   [block page-name]
   (let [shape? (shape-block? block)
         shape (block->shape block)
@@ -96,5 +95,7 @@
                :block/page {:block/name page-name}
                :block/parent {:block/name page-name}
                :block/properties properties}
-        additional-props (with-whiteboard-block-props block page-name)]
+        additional-props (with-whiteboard-block-props
+                           (assoc block :block/properties {:ls-type :whiteboard-shape :logseq.tldraw.shape shape})
+                           page-name)]
     (merge block additional-props)))

deps/graph-parser/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v4":
-  version "1.2.173-feat-db-v4"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/ee701fab68a0d34b8638b2eed5037460c9766172"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v5":
+  version "1.2.173-feat-db-v5"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/090ad77480ad8065aa051e625c248bc77e07efa5"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/outliner/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v4"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v5"
   },
   "dependencies": {
     "better-sqlite3": "8.0.1"

deps/outliner/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v4":
-  version "1.2.173-feat-db-v4"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/ee701fab68a0d34b8638b2eed5037460c9766172"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v5":
+  version "1.2.173-feat-db-v5"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/090ad77480ad8065aa051e625c248bc77e07efa5"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/publishing/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v4",
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v5",
     "mldoc": "^1.5.1"
   },
   "dependencies": {

deps/publishing/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v4":
-  version "1.2.173-feat-db-v4"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/ee701fab68a0d34b8638b2eed5037460c9766172"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v5":
+  version "1.2.173-feat-db-v5"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/090ad77480ad8065aa051e625c248bc77e07efa5"
   dependencies:
     import-meta-resolve "^2.1.0"
 

resources/package.json

@@ -13,6 +13,7 @@
     "electron:make": "electron-forge make",
     "electron:make-macos-arm64": "electron-forge make --platform=darwin --arch=arm64",
     "electron:publish:github": "electron-forge publish",
+    "rebuild:all": "electron-rebuild -v 27.1.3 -f",
     "postinstall": "install-app-deps"
   },
   "config": {
@@ -41,7 +42,8 @@
     "abort-controller": "3.0.0",
     "fastify": "latest",
     "@fastify/cors": "8.2.0",
-    "command-exists": "1.2.9"
+    "command-exists": "1.2.9",
+    "better-sqlite3": "8.0.1"
   },
   "devDependencies": {
     "@electron-forge/cli": "^6.0.4",

scripts/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v4"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v5"
   },
   "dependencies": {
     "better-sqlite3": "8.0.1",

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -40,7 +40,7 @@
 
 (def file-graph-paths
   "Paths _only_ for file graphs"
-  ["src/main/frontend/handler/file_based" "src/main/frontend/handler/file_sync.cljs"
+  ["src/main/frontend/handler/file_based" "src/main/frontend/handler/file_sync.cljs" "src/main/frontend/db/file_based"
    "src/main/frontend/fs"
    "src/main/frontend/components/file_sync.cljs"
    "src/main/frontend/util/fs.cljs"])

scripts/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v4":
-  version "1.2.173-feat-db-v4"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/ee701fab68a0d34b8638b2eed5037460c9766172"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v5":
+  version "1.2.173-feat-db-v5"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/090ad77480ad8065aa051e625c248bc77e07efa5"
   dependencies:
     import-meta-resolve "^2.1.0"
 

src/main/frontend/components/db_based/page.cljs

@@ -5,11 +5,15 @@
             [frontend.components.class :as class-component]
             [frontend.components.property :as property-component]
             [frontend.components.property.value :as pv]
+            [frontend.components.icon :as icon-component]
             [frontend.config :as config]
             [frontend.db :as db]
             [frontend.handler.db-based.property :as db-property-handler]
+            [frontend.handler.property.util :as pu]
+            [frontend.handler.db-based.property.util :as db-pu]
             [frontend.ui :as ui]
             [frontend.util :as util]
+            [frontend.state :as state]
             [rum.core :as rum]))
 
 (rum/defc page-properties < rum/reactive
@@ -82,6 +86,29 @@
                                                   :page-configure? false
                                                   :class-schema? false})])]))])))
 
+(rum/defc icon-row < rum/reactive
+  [page]
+  [:div.grid.grid-cols-5.gap-1.items-center.leading-8
+   [:label.col-span-2 "Icon:"]
+   (let [icon-value (pu/get-property page :icon)]
+     [:div.col-span-3.flex.flex-row.items-center.gap-2
+      (icon-component/icon-picker icon-value
+                                  {:disabled? config/publishing?
+                                   :on-chosen (fn [_e icon]
+                                                (let [icon-property-id (db-pu/get-built-in-property-uuid :icon)]
+                                                  (db-property-handler/update-property!
+                                                   (state/get-current-repo)
+                                                   (:block/uuid page)
+                                                   {:properties {icon-property-id icon}})))})
+      (when (and icon-value (not config/publishing?))
+        [:a.fade-link.flex {:on-click (fn [_e]
+                                        (db-property-handler/remove-block-property!
+                                         (state/get-current-repo)
+                                         (:block/uuid page)
+                                         (db-pu/get-built-in-property-uuid :icon)))
+                            :title "Delete this icon"}
+        (ui/icon "X")])])])
+
 (rum/defcs page-configure-inner <
   (rum/local false ::show-page-properties?)
   {:will-unmount (fn [state]
@@ -101,9 +128,10 @@
       (cond
         (not class-or-property?)
         (when (and (not class?)
-                   (not property?)
-                   (not (db-property-handler/block-has-viewable-properties? page)))
-          (page-properties page page-opts))
+                   (not property?))
+          [:<>
+           (icon-row page)
+           (page-properties page page-opts)])
 
         @*show-page-properties?
         (page-properties page page-opts)
@@ -112,6 +140,8 @@
         [:<>
          (when class?
            (class-component/configure page))
+         (when class?
+           (icon-row page))
          (when class?
            (page-properties page page-opts))
          (when (and property? (not class?))

src/main/frontend/components/editor.cljs

@@ -332,6 +332,24 @@
             :item-render (fn [property] property)
             :class       "black"}))))))
 
+(rum/defc property-value-search-aux
+  [id property q]
+  (let [[values set-values!] (rum/use-state nil)]
+    (rum/use-effect!
+     (fn []
+       (p/let [result (editor-handler/get-matched-property-values property q)]
+         (set-values! result)))
+     [property q])
+    (ui/auto-complete
+         values
+         {:on-chosen (editor-handler/property-value-on-chosen-handler id q)
+          :on-enter (fn [_state]
+                      ((editor-handler/property-value-on-chosen-handler id q) nil))
+          :empty-placeholder [:div.px-4.py-2.text-sm (str "Create a new property value: " q)]
+          :header [:div.px-4.py-2.text-sm.font-medium "Matched property values: "]
+          :item-render (fn [property-value] property-value)
+          :class       "black"})))
+
 (rum/defc property-value-search < rum/reactive
   [id]
   (let [property (:property (state/get-editor-action-data))
@@ -346,18 +364,8 @@
                (when (>= current-pos (+ start-idx 2))
                  (subs edit-content (+ start-idx 2) current-pos))
                "")
-            q (string/triml q)
-            matched-values (editor-handler/get-matched-property-values property q)
-            non-exist-handler (fn [_state]
-                                ((editor-handler/property-value-on-chosen-handler id q) nil))]
-        (ui/auto-complete
-         matched-values
-         {:on-chosen (editor-handler/property-value-on-chosen-handler id q)
-          :on-enter non-exist-handler
-          :empty-placeholder [:div.px-4.py-2.text-sm (str "Create a new property value: " q)]
-          :header [:div.px-4.py-2.text-sm.font-medium "Matched property values: "]
-          :item-render (fn [property-value] property-value)
-          :class       "black"})))))
+            q (string/triml q)]
+        (property-value-search-aux id property q)))))
 
 (rum/defc code-block-mode-keyup-listener
   [_q _edit-content last-pos current-pos]

src/main/frontend/components/icon.cljs

@@ -11,6 +11,7 @@
             [frontend.util :as util]
             [goog.object :as gobj]
             [goog.functions :refer [debounce]]
+            [frontend.config :as config]
             [frontend.handler.property.util :as pu]))
 
 (defn icon
@@ -168,3 +169,22 @@
 
         (:name @*hover)]
        [:div {:style {:padding-bottom 32}}])]))
+
+(rum/defc icon-picker
+  [icon-value {:keys [disabled? on-chosen]}]
+  (ui/dropdown
+   (fn [{:keys [toggle-fn]}]
+     [:button.flex {:on-click #(when-not disabled? (toggle-fn))}
+      (if icon-value
+        (icon icon-value)
+        [:span.bullet-container.cursor [:span.bullet]])])
+   (if config/publishing?
+     (constantly [])
+     (fn [{:keys [toggle-fn]}]
+       [:div.p-4
+        (icon-search
+         {:on-chosen (fn [e icon-value]
+                       (on-chosen e icon-value)
+                       (toggle-fn))})]))
+   {:modal-class (util/hiccup->class
+                  "origin-top-right.absolute.left-0.rounded-md.shadow-lg")}))

src/main/frontend/components/imports.cljs

@@ -1,21 +1,33 @@
 (ns frontend.components.imports
   "Import data into Logseq."
-  (:require [frontend.state :as state]
-            [rum.core :as rum]
-            [frontend.ui :as ui]
-            [frontend.context.i18n :refer [t]]
-            [frontend.components.svg :as svg]
+  (:require [cljs.core.async.interop :refer [p->c]]
+            [clojure.core.async :as async]
+            [clojure.edn :as edn]
+            [clojure.string :as string]
+            [frontend.components.onboarding.setups :as setups]
             [frontend.components.repo :as repo]
+            [frontend.components.svg :as svg]
+            [frontend.config :as config]
+            [frontend.context.i18n :refer [t]]
+            [frontend.db :as db]
+            [frontend.fs :as fs]
+            [frontend.handler.db-based.editor :as db-editor-handler]
+            [frontend.handler.import :as import-handler]
+            [frontend.handler.notification :as notification]
             [frontend.handler.route :as route-handler]
             [frontend.handler.ui :as ui-handler]
-            [frontend.handler.notification :as notification]
-            [frontend.handler.import :as import-handler]
-            [clojure.string :as string]
-            [goog.object :as gobj]
-            [frontend.components.onboarding.setups :as setups]
+            [frontend.state :as state]
+            [frontend.ui :as ui]
+            [frontend.util :as util]
             [frontend.util.fs :as fs-util]
-            [frontend.util.text :as text-util]
-            [frontend.util :as util]))
+            [goog.functions :refer [debounce]]
+            [goog.object :as gobj]
+            [logseq.common.path :as path]
+            [logseq.graph-parser :as graph-parser]
+            [medley.core :as medley]
+            [promesa.core :as p]
+            [rum.core :as rum]
+            [frontend.handler.repo :as repo-handler]))
 
 ;; Can't name this component as `frontend.components.import` since shadow-cljs
 ;; will complain about it.
@@ -57,15 +69,12 @@
         json? (string/ends-with? file-name ".json")]
     (cond
       sqlite?
-      (let [graph-name (string/trim graph-name)
-            all-graphs (->> (state/get-repos)
-                            (map #(text-util/get-graph-name-from-path (:url %)))
-                            set)]
+      (let [graph-name (string/trim graph-name)]
         (cond
           (string/blank? graph-name)
           (notification/show! "Empty graph name." :error)
 
-          (contains? all-graphs graph-name)
+          (repo-handler/graph-already-exists? graph-name)
           (notification/show! "Please specify another name as another graph with this name already exists!" :error)
 
           :else
@@ -127,7 +136,7 @@
   (rum/local "" ::input)
   [state sqlite-input-e opts]
   (let [*input (::input state)
-        on-submit #(if (fs-util/include-reserved-chars? @*input)
+        on-submit #(if (repo/invalid-graph-name? @*input)
                      (repo/invalid-graph-name-warning)
                      (lsq-import-handler sqlite-input-e (assoc opts :graph-name @*input)))]
     [:div.container
@@ -141,77 +150,220 @@
        :on-change (fn [e]
                     (reset! *input (util/evalue e)))
        :on-key-press (fn [e]
-                        (when (= "Enter" (util/ekey e))
-                          (on-submit)))}]
+                       (when (= "Enter" (util/ekey e))
+                         (on-submit)))}]
 
      [:div.mt-5.sm:mt-4.flex
       (ui/button "Submit"
-        {:on-click on-submit})]]))
-
-(rum/defc importer < rum/reactive
-  [{:keys [query-params]}]
-  (if (state/sub :graph/importing)
-    (let [{:keys [total current-idx current-page]} (state/sub :graph/importing-state)
-          left-label [:div.flex.flex-row.font-bold
-                      (t :importing)
-                      [:div.hidden.md:flex.flex-row
-                       [:span.mr-1 ": "]
-                       [:div.text-ellipsis-wrapper {:style {:max-width 300}}
-                        current-page]]]
-          width (js/Math.round (* (.toFixed (/ current-idx total) 2) 100))
-          process (when (and total current-idx)
-                    (str current-idx "/" total))]
-      (ui/progress-bar-with-label width left-label process))
-    (setups/setups-container
-     :importer
-     [:article.flex.flex-col.items-center.importer.py-16.px-8
-      [:section.c.text-center
-       [:h1 (t :on-boarding/importing-title)]
-       [:h2 (t :on-boarding/importing-desc)]]
-      [:section.d.md:flex.flex-col
-       [:label.action-input.flex.items-center.mx-2.my-2
-        [:span.as-flex-center [:i (svg/logo 28)]]
-        [:span.flex.flex-col
-         [[:strong "SQLite"]
-          [:small (t :on-boarding/importing-sqlite-desc)]]]
-        [:input.absolute.hidden
-         {:id        "import-sqlite-db"
-          :type      "file"
-          :on-change (fn [e]
-                       (state/set-modal!
-                        #(set-graph-name-dialog e {:sqlite? true})))}]]
-
-       [:label.action-input.flex.items-center.mx-2.my-2
-        [:span.as-flex-center [:i (svg/logo 28)]]
-        [:span.flex.flex-col
-         [[:strong "EDN / JSON"]
-          [:small (t :on-boarding/importing-lsq-desc)]]]
-        [:input.absolute.hidden
-         {:id        "import-lsq"
-          :type      "file"
-          :on-change lsq-import-handler}]]
-
-       [:label.action-input.flex.items-center.mx-2.my-2
-        [:span.as-flex-center [:i (svg/roam-research 28)]]
-        [:div.flex.flex-col
-         [[:strong "RoamResearch"]
-          [:small (t :on-boarding/importing-roam-desc)]]]
-        [:input.absolute.hidden
-         {:id        "import-roam"
-          :type      "file"
-          :on-change roam-import-handler}]]
-
-       [:label.action-input.flex.items-center.mx-2.my-2
-        [:span.as-flex-center.ml-1 (ui/icon "sitemap" {:size 26})]
-        [:span.flex.flex-col
-         [[:strong "OPML"]
-          [:small (t :on-boarding/importing-opml-desc)]]]
-
-        [:input.absolute.hidden
-         {:id        "import-opml"
-          :type      "file"
-          :on-change opml-import-handler}]]]
-
-      (when (= "picker" (:from query-params))
-        [:section.e
-         [:a.button {:on-click #(route-handler/redirect-to-home!)} "Skip"]])])))
+                 {:on-click on-submit})]]))
+
+
+(defn- import-from-doc-files!
+  [db-conn doc-files]
+  (let [imported-chan (async/promise-chan)]
+    (try
+      (let [docs-chan (async/to-chan! (medley/indexed doc-files))]
+        (state/set-state! [:graph/importing-state :total] (count doc-files))
+        (async/go-loop []
+          (if-let [[i ^js file] (async/<! docs-chan)]
+            (do
+              (state/set-state! [:graph/importing-state :current-idx] (inc i))
+              (state/set-state! [:graph/importing-state :current-page] (.-rpath file))
+              (async/<! (async/timeout 10))
+              (async/<! (p->c (-> (.text file)
+                                  (p/then (fn [content]
+                                            (prn :import- (.-rpath file))
+                                            {:file/path (.-rpath file)
+                                             :file/content content}))
+                                  (p/then (fn [file]
+                                            (graph-parser/import-file-to-db-graph db-conn (:file/path file) (:file/content file) {})
+                                            file)))))
+              (recur))
+            (async/offer! imported-chan true))))
+      (catch :default e
+        (notification/show! (str "Error happens when importing:\n" e) :error)
+        (async/offer! imported-chan true)))))
+
+(defn- import-from-asset-files!
+  [asset-files]
+  (let [ch (async/to-chan! asset-files)
+        repo (state/get-current-repo)
+        repo-dir (config/get-repo-dir repo)]
+    (prn :in-files asset-files)
+    (async/go-loop []
+      (if-let [^js file (async/<! ch)]
+        (do
+          (async/<! (p->c (-> (.arrayBuffer file)
+                              (p/then (fn [buffer]
+                                        (let [content (js/Uint8Array. buffer)
+                                              parent-dir (path/path-join repo-dir (path/dirname (.-rpath file)))]
+                                          (p/do!
+                                           (fs/mkdir-if-not-exists parent-dir)
+                                           (fs/write-file! repo repo-dir (.-rpath file) content nil))))))))
+          (recur))
+        true))))
+
+(defn- import-config-file!
+  [config-file]
+  (-> (when config-file
+        (.text config-file))
+      (p/then (fn [content]
+                (when content
+                  (p/do!
+                   (db-editor-handler/save-file! "logseq/config.edn" content))
+                  (edn/read-string content))))))
+
+
+(rum/defc confirm-graph-name-dialog
+  [initial-name on-graph-name-confirmed]
+  (let [[input set-input!] (rum/use-state initial-name)
+        on-submit #(do (on-graph-name-confirmed input)
+                       (state/close-modal!))]
+    [:div.container
+     [:div.sm:flex.sm:items-start
+      [:div.mt-3.text-center.sm:mt-0.sm:text-left
+       [:h3#modal-headline.leading-6.font-medium
+        "Imported new graph name:"]]]
+
+     [:input.form-input.block.w-full.sm:text-sm.sm:leading-5.my-2.mb-4
+      {:auto-focus true
+       :default-value input
+       :on-change (fn [e]
+                    (set-input! (util/evalue e)))
+       :on-key-press (fn [e]
+                       (when (= "Enter" (util/ekey e))
+                         (on-submit)))}]
+
+     [:div.mt-5.sm:mt-4.flex
+      (ui/button "Confirm"
+                 {:on-click on-submit})]]))
+
+(defn graph-folder-to-db-import-handler
+  "Import from a graph folder as a DB-based graph.
+
+- Page name, journal name creation"
+  [ev _opts]
+  (let [^js file-objs (array-seq (.-files (.-target ev)))
+        original-graph-name (string/replace (.-webkitRelativePath (first file-objs)) #"/.*" "")
+        import-graph-fn (fn [graph-name]
+                          (let [_ (doseq [^js file file-objs]
+                                    (set! (.-rpath file) (path/trim-dir-prefix original-graph-name (.-webkitRelativePath file))))
+                                asset-files (filter (fn [^js f]
+                                                      (string/starts-with? (.-rpath f) "assets/"))
+                                                    file-objs)
+                                file-objs (remove (fn [^js f] (fs-util/ignored-path? original-graph-name (.-webkitRelativePath f))) file-objs)
+                                                                                     ;; TODO handle, logseq/config.edn, logseq/custom.css, custom.js are ignored
+                                doc-files (filter (fn [^js f]
+                                                    (contains? #{"md" "org" "markdown" "edn"} (path/file-ext (.-rpath f))))
+                                                  file-objs)
+                                config-file (first (filter (fn [^js f]
+                                                             (= (.-rpath f) "logseq/config.edn"))
+                                                           file-objs))]
+                            (state/set-state! :graph/importing :folder)
+                            (state/set-state! [:graph/importing-state :current-page] (str graph-name " Assets"))
+                            (async/go
+                              (async/<! (p->c (repo-handler/new-db! graph-name {:file-graph-import? true})))
+                              (let [repo (state/get-current-repo)
+                                    db-conn (db/get-db repo false)]
+                                (async/<! (p->c (import-config-file! config-file)))
+                                (async/<! (import-from-asset-files! asset-files))
+                                (async/<! (import-from-doc-files! db-conn doc-files))
+                                (state/set-state! :graph/importing nil)
+                                (finished-cb)))))]
+    (state/set-modal!
+     #(confirm-graph-name-dialog original-graph-name
+                                 (fn [graph-name]
+                                   (cond
+                                     (repo/invalid-graph-name? graph-name)
+                                     (repo/invalid-graph-name-warning)
+
+                                     (string/blank? graph-name)
+                                     (notification/show! "Empty graph name." :error)
+
+                                     (repo-handler/graph-already-exists? graph-name)
+                                     (notification/show! "Please specify another name as another graph with this name already exists!" :error)
+
+                                     :else
+                                     (import-graph-fn graph-name)))))))
+
+
+  (rum/defc importer < rum/reactive
+    [{:keys [query-params]}]
+    (if (state/sub :graph/importing)
+      (let [{:keys [total current-idx current-page]} (state/sub :graph/importing-state)
+            left-label [:div.flex.flex-row.font-bold
+                        (t :importing)
+                        [:div.hidden.md:flex.flex-row
+                         [:span.mr-1 ": "]
+                         [:div.text-ellipsis-wrapper {:style {:max-width 300}}
+                          current-page]]]
+            width (js/Math.round (* (.toFixed (/ current-idx total) 2) 100))
+            process (when (and total current-idx)
+                      (str current-idx "/" total))]
+        (ui/progress-bar-with-label width left-label process))
+      (setups/setups-container
+       :importer
+       [:article.flex.flex-col.items-center.importer.py-16.px-8
+        [:section.c.text-center
+         [:h1 (t :on-boarding/importing-title)]
+         [:h2 (t :on-boarding/importing-desc)]]
+        [:section.d.md:flex.flex-col
+         [:label.action-input.flex.items-center.mx-2.my-2
+          [:span.as-flex-center [:i (svg/logo 28)]]
+          [:span.flex.flex-col
+           [[:strong "SQLite"]
+            [:small (t :on-boarding/importing-sqlite-desc)]]]
+          [:input.absolute.hidden
+           {:id        "import-sqlite-db"
+            :type      "file"
+            :on-change (fn [e]
+                         (state/set-modal!
+                          #(set-graph-name-dialog e {:sqlite? true})))}]]
+
+         [:label.action-input.flex.items-center.mx-2.my-2
+          [:span.as-flex-center [:i (svg/logo 28)]]
+          [:span.flex.flex-col
+           [[:strong "Graph Folder"]
+            [:small  "Import from a graph folder as a DB-based graph"]]]
+          [:input.absolute.hidden
+           {:id        "import-graph-folder"
+            :type      "file"
+            :webkitdirectory "true"
+            :on-change (debounce (fn [e]
+                                   (graph-folder-to-db-import-handler e {}))
+                                 1000)}]]
+
+         [:label.action-input.flex.items-center.mx-2.my-2
+          [:span.as-flex-center [:i (svg/logo 28)]]
+          [:span.flex.flex-col
+           [[:strong "EDN / JSON"]
+            [:small (t :on-boarding/importing-lsq-desc)]]]
+          [:input.absolute.hidden
+           {:id        "import-lsq"
+            :type      "file"
+            :on-change lsq-import-handler}]]
+
+         [:label.action-input.flex.items-center.mx-2.my-2
+          [:span.as-flex-center [:i (svg/roam-research 28)]]
+          [:div.flex.flex-col
+           [[:strong "RoamResearch"]
+            [:small (t :on-boarding/importing-roam-desc)]]]
+          [:input.absolute.hidden
+           {:id        "import-roam"
+            :type      "file"
+            :on-change roam-import-handler}]]
+
+         [:label.action-input.flex.items-center.mx-2.my-2
+          [:span.as-flex-center.ml-1 (ui/icon "sitemap" {:size 26})]
+          [:span.flex.flex-col
+           [[:strong "OPML"]
+            [:small (t :on-boarding/importing-opml-desc)]]]
+
+          [:input.absolute.hidden
+           {:id        "import-opml"
+            :type      "file"
+            :on-change opml-import-handler}]]]
+
+        (when (= "picker" (:from query-params))
+          [:section.e
+           [:a.button {:on-click #(route-handler/redirect-to-home!)} "Skip"]])])))

src/main/frontend/components/page.cljs

@@ -9,7 +9,7 @@
             [frontend.components.query :as query]
             [frontend.components.reference :as reference]
             [frontend.components.scheduled-deadlines :as scheduled]
-            [frontend.components.property :as property-component]
+            [frontend.components.icon :as icon-component]
             [frontend.components.property.value :as pv]
             [frontend.components.db-based.page :as db-page]
             [frontend.handler.property.util :as pu]
@@ -360,14 +360,14 @@
            (when icon
              [:div.page-icon {:on-mouse-down util/stop-propagation}
               (if (and (map? icon) db-based?)
-                (property-component/icon icon {:on-chosen (fn [_e icon]
-                                                            (let [icon-property-id (db-pu/get-built-in-property-uuid :icon)]
-                                                              (db-property-handler/update-property!
-                                                               repo
-                                                               (:block/uuid page)
-                                                               {:properties {icon-property-id icon}})))})
+                (icon-component/icon-picker icon
+                                            {:on-chosen (fn [_e icon]
+                                                          (let [icon-property-id (db-pu/get-built-in-property-uuid :icon)]
+                                                            (db-property-handler/update-property!
+                                                             repo
+                                                             (:block/uuid page)
+                                                             {:properties {icon-property-id icon}})))})
                 icon)])
-
            [:div.flex.flex-1.flex-row.flex-wrap.items-center.gap-4
             [:h1.page-title.flex-1.cursor-pointer.gap-1
              {:class (when-not whiteboard-page? "title")
@@ -533,10 +533,10 @@
                 (page-blocks-collapse-control title *control-show? *all-collapsed?)])
              (let [original-name (:block/original-name (db/entity [:block/name (util/page-name-sanity-lc page-name)]))]
                (when (and (not whiteboard?) original-name)
-                (page-title page-name {:journal? journal?
-                                       :fmt-journal? fmt-journal?
-                                       :built-in-property? built-in-property?
-                                       :preview? preview?})))
+                 (page-title page-name {:journal? journal?
+                                        :fmt-journal? fmt-journal?
+                                        :built-in-property? built-in-property?
+                                        :preview? preview?})))
              (when (not config/publishing?)
                (when config/lsp-enabled?
                  [:div.flex.flex-row
@@ -718,20 +718,20 @@
                                (set-setting! :excluded-pages? value)))
                            true)]]
               (when (config/db-based-graph? (state/get-current-repo))
-               [:div.flex.flex-col.mb-2
-                [:p "Created before"]
-                (when created-at-filter
-                  [:div (.toDateString (js/Date. (+ created-at-filter (get-in graph [:all-pages :created-at-min]))))])
-                (ui/tippy {:html [:div.pr-3 (str (js/Date. (+ created-at-filter (get-in graph [:all-pages :created-at-min]))))]}
+                [:div.flex.flex-col.mb-2
+                 [:p "Created before"]
+                 (when created-at-filter
+                   [:div (.toDateString (js/Date. (+ created-at-filter (get-in graph [:all-pages :created-at-min]))))])
+                 (ui/tippy {:html [:div.pr-3 (str (js/Date. (+ created-at-filter (get-in graph [:all-pages :created-at-min]))))]}
                           ;; Slider keeps track off the range from min created-at to max created-at
                           ;; because there were bugs with setting min and max directly
-                          (ui/slider created-at-filter
-                                     {:min 0
-                                      :max (- (get-in graph [:all-pages :created-at-max])
-                                              (get-in graph [:all-pages :created-at-min]))
-                                      :on-change #(do
-                                                    (reset! *created-at-filter (int %))
-                                                    (set-setting! :created-at-filter (int %)))}))])
+                           (ui/slider created-at-filter
+                                      {:min 0
+                                       :max (- (get-in graph [:all-pages :created-at-max])
+                                               (get-in graph [:all-pages :created-at-min]))
+                                       :on-change #(do
+                                                     (reset! *created-at-filter (int %))
+                                                     (set-setting! :created-at-filter (int %)))}))])
               (when (seq focus-nodes)
                 [:div.flex.flex-col.mb-2
                  [:p {:title "N hops from selected nodes"}
@@ -861,27 +861,25 @@
 
 (rum/defc page-graph-inner < rum/reactive
   [_page graph dark?]
-   (let [ show-journals-in-page-graph? (rum/react *show-journals-in-page-graph?) ]
-  [:div.sidebar-item.flex-col
-             [:div.flex.items-center.justify-between.mb-0
-              [:span (t :right-side-bar/show-journals)]
-              [:div.mt-1
-               (ui/toggle show-journals-in-page-graph? ;my-val;
-                           (fn []
-                             (let [value (not show-journals-in-page-graph?)]
-                               (reset! *show-journals-in-page-graph? value)
-                               ))
-                          true)]
-              ]
-
-   (graph/graph-2d {:nodes (:nodes graph)
-                    :links (:links graph)
-                    :width 600
-                    :height 600
-                    :dark? dark?
-                    :register-handlers-fn
-                    (fn [graph]
-                      (graph-register-handlers graph (atom nil) (atom nil) dark?))})]))
+  (let [show-journals-in-page-graph? (rum/react *show-journals-in-page-graph?)]
+    [:div.sidebar-item.flex-col
+     [:div.flex.items-center.justify-between.mb-0
+      [:span (t :right-side-bar/show-journals)]
+      [:div.mt-1
+       (ui/toggle show-journals-in-page-graph? ;my-val;
+                  (fn []
+                    (let [value (not show-journals-in-page-graph?)]
+                      (reset! *show-journals-in-page-graph? value)))
+                  true)]]
+
+     (graph/graph-2d {:nodes (:nodes graph)
+                      :links (:links graph)
+                      :width 600
+                      :height 600
+                      :dark? dark?
+                      :register-handlers-fn
+                      (fn [graph]
+                        (graph-register-handlers graph (atom nil) (atom nil) dark?))})]))
 
 (rum/defc page-graph < db-mixins/query rum/reactive
   []
@@ -928,8 +926,8 @@
   [:th
    {:class [(name key)]}
    [:a.fade-link {:on-click (fn []
-                    (reset! by-item key)
-                    (swap! desc? not))}
+                              (reset! by-item key)
+                              (swap! desc? not))}
     [:span.flex.items-center
      [:span.mr-1 title]
      (when (= @by-item key)
@@ -1050,10 +1048,10 @@
 
         *indeterminate (rum/derived-atom
                         [*checks] ::indeterminate
-                        (fn [checks]
-                          (when-let [checks (vals checks)]
-                            (if (every? true? checks)
-                              1 (if (some true? checks) -1 0)))))
+                         (fn [checks]
+                           (when-let [checks (vals checks)]
+                             (if (every? true? checks)
+                               1 (if (some true? checks) -1 0)))))
 
         mobile? (util/mobile?)
         total-items (count @*results-all)

src/main/frontend/components/property.cljs

@@ -32,8 +32,6 @@
             [frontend.components.property.util :as components-pu]
             [promesa.core :as p]))
 
-(def icon closed-value/icon)
-
 (defn- create-class-if-not-exists!
   [value]
   (when (string? value)
@@ -141,8 +139,7 @@
         class? (contains? (:block/type block) "class")
         property-type (get-in property [:block/schema :type])
         save-property-fn (fn [] (components-pu/update-property! property @*property-name @*property-schema))
-        enable-closed-values? (contains? db-property-type/closed-value-property-types (or property-type :default))
-        enable-position? (contains? db-property-type/closed-value-property-position-types (or property-type :default))]
+        enable-closed-values? (contains? db-property-type/closed-value-property-types (or property-type :default))]
     [:div.property-configure.flex.flex-1.flex-col
      {:on-mouse-down #(state/set-state! :editor/mouse-down-from-property-configure? true)
       :on-mouse-up #(state/set-state! :editor/mouse-down-from-property-configure? nil)}
@@ -161,15 +158,23 @@
       [:div.grid.grid-cols-4.gap-1.items-center.leading-8
        [:label.col-span-1 "Icon:"]
        (let [icon-value (pu/get-property property :icon)]
-         [:div.col-span-3
-          (closed-value/icon icon-value
-                             {:disabled? disabled?
-                              :on-chosen (fn [_e icon]
-                                           (let [icon-property-id (db-pu/get-built-in-property-uuid :icon)]
-                                             (db-property-handler/update-property!
-                                              (state/get-current-repo)
-                                              (:block/uuid property)
-                                              {:properties {icon-property-id icon}})))})])]
+         [:div.col-span-3.flex.flex-row.items-center.gap-2
+          (icon-component/icon-picker icon-value
+                                      {:disabled? disabled?
+                                       :on-chosen (fn [_e icon]
+                                                    (let [icon-property-id (db-pu/get-built-in-property-uuid :icon)]
+                                                      (db-property-handler/update-property!
+                                                       (state/get-current-repo)
+                                                       (:block/uuid property)
+                                                       {:properties {icon-property-id icon}})))})
+          (when (and icon-value (not disabled?))
+            [:a.fade-link.flex {:on-click (fn [_e]
+                                            (db-property-handler/remove-block-property!
+                                             (state/get-current-repo)
+                                             (:block/uuid property)
+                                             (db-pu/get-built-in-property-uuid :icon)))
+                                :title "Delete this icon"}
+             (ui/icon "X")])])]
 
       [:div.grid.grid-cols-4.gap-1.items-center.leading-8
        [:label.col-span-1 "Schema type:"]
@@ -252,7 +257,7 @@
           (closed-value/choices property *property-name *property-schema opts)]])
 
       (when (and enable-closed-values?
-                 enable-position?
+                 (db-property-type/property-type-allows-schema-attribute? (:type @*property-schema) :position)
                  (seq (:values @*property-schema)))
         (let [position (:position @*property-schema)
               choices (map

src/main/frontend/components/property/closed_value.cljs

@@ -24,25 +24,6 @@
     (when (seq tx-data) (db/transact! tx-data))
     block-id))
 
-(rum/defc icon
-  [icon {:keys [disabled? on-chosen]}]
-  (ui/dropdown
-   (fn [{:keys [toggle-fn]}]
-     [:button.flex {:on-click #(when-not disabled? (toggle-fn))}
-      (if icon
-        (icon-component/icon icon)
-        [:span.bullet-container.cursor [:span.bullet]])])
-   (if config/publishing?
-     (constantly [])
-     (fn [{:keys [toggle-fn]}]
-       [:div.p-4
-        (icon-component/icon-search
-         {:on-chosen (fn [e icon]
-                       (on-chosen e icon)
-                       (toggle-fn))})]))
-   {:modal-class (util/hiccup->class
-                  "origin-top-right.absolute.left-0.rounded-md.shadow-lg")}))
-
 (rum/defc item-value
   [type *value]
   (case type
@@ -91,16 +72,23 @@
       (item-value property-type *value)]
      [:div.grid.grid-cols-5.gap-1.items-center.leading-8
       [:label.col-span-2 "Icon:"]
-      [:div.col-span-3
-       (icon (rum/react *icon)
-             {:on-chosen (fn [_e icon]
-                           (reset! *icon icon))})]]
-     [:div.grid.grid-cols-5.gap-1.items-start.leading-8
-      [:label.col-span-2 "Description:"]
-      [:div.col-span-3
-       (ui/ls-textarea
-        {:on-change #(reset! *description (util/evalue %))
-         :default-value @*description})]]
+      [:div.col-span-3.flex.flex-row.items-center.gap-2
+       (icon-component/icon-picker (rum/react *icon)
+                                   {:on-chosen (fn [_e icon]
+                                                 (reset! *icon icon))})
+       (when (rum/react *icon)
+        [:a.fade-link.flex {:on-click (fn [_e]
+                                        (reset! *icon nil))
+                            :title "Delete this icon"}
+        (ui/icon "X")])]]
+     ;; Disable description for types that can't edit them
+     (when-not (#{:page :date} property-type)
+       [:div.grid.grid-cols-5.gap-1.items-start.leading-8
+        [:label.col-span-2 "Description:"]
+        [:div.col-span-3
+         (ui/ls-textarea
+          {:on-change #(reset! *description (util/evalue %))
+           :default-value @*description})]])
      [:div
       (ui/button
        "Save"
@@ -117,9 +105,9 @@
      {:on-mouse-over #(reset! *hover? true)
       :on-mouse-out #(reset! *hover? false)}
      [:div.flex.flex-row.items-center.gap-2
-      (icon (pu/get-property item :icon)
-            {:on-chosen (fn [_e icon]
-                          (update-icon icon))})
+      (icon-component/icon-picker (pu/get-property item :icon)
+                                  {:on-chosen (fn [_e icon]
+                                                (update-icon icon))})
       (if (and page? (:page-cp parent-opts))
         ((:page-cp parent-opts) {} item)
         [:a {:on-click toggle-fn}

src/main/frontend/components/repo.cljs

@@ -128,8 +128,8 @@
                     (mobile-util/native-platform?))
             [:div.mr-8
              (ui/button
-               (t :open-a-directory)
-               :on-click #(state/pub-event! [:graph/setup-a-repo]))])]]
+              (t :open-a-directory)
+              :on-click #(state/pub-event! [:graph/setup-a-repo]))])]]
 
         (when (and (file-sync/enable-sync?) login?)
           [:div
@@ -283,14 +283,19 @@
      [:li "+ (plus)"]]]
    :warning false))
 
+(defn invalid-graph-name?
+  "Returns boolean indicating if DB graph name is invalid. Must be kept in sync with invalid-graph-name-warning"
+  [graph-name]
+  (or (fs-util/include-reserved-chars? graph-name)
+      (string/includes? graph-name "+")))
+
 (rum/defcs new-db-graph <
   (rum/local "" ::graph-name)
   [state]
   (let [*graph-name (::graph-name state)
         new-db-f (fn []
                    (when-not (string/blank? @*graph-name)
-                     (if (or (fs-util/include-reserved-chars? @*graph-name)
-                             (string/includes? @*graph-name "+"))
+                     (if (invalid-graph-name? @*graph-name)
                        (invalid-graph-name-warning)
                        (do
                          (repo-handler/new-db! @*graph-name)

src/main/frontend/db/async/util.cljs

@@ -6,7 +6,7 @@
 
 (defn <q
   [graph & inputs]
-  (assert (not-any? fn? inputs) "Async query inptus can't include fns because fn can't be serialized")
+  (assert (not-any? fn? inputs) "Async query inputs can't include fns because fn can't be serialized")
   (when-let [sqlite @db-browser/*worker]
     (p/let [result (.q sqlite graph (pr-str inputs))]
       (bean/->clj result))))

src/main/frontend/db/rtc/asset_sync.cljs

@@ -0,0 +1,98 @@
+(ns frontend.db.rtc.asset-sync
+  "Fns for syncing assets"
+  {:clj-kondo/ignore true}              ;; TODO: remove when this ns is ready
+  (:require [malli.core :as m]
+            [malli.util :as mu]
+            [cljs.core.async :as async :refer [<! >! chan go go-loop]]
+            [frontend.db.rtc.const :as rtc-const]
+            [frontend.db.rtc.op-mem-layer :as op-mem-layer]
+            [frontend.handler.user :as user]
+            [frontend.db.rtc.ws :as ws]))
+
+(def state-schema
+  [:map {:closed true}
+   [:*graph-uuid :any]
+   [:*repo :any]
+   [:*assets-update-state :any]
+   [:data-from-ws-pub :any]
+   [:*auto-push-assets-update-ops? :any]
+   [:toggle-auto-push-assets-update-ops-chan :any]])
+
+(def state-validator
+  (let [validator (m/validator state-schema)]
+    (fn [data]
+      (if (validator data)
+        true
+        (prn (mu/explain-data state-schema data))))))
+
+
+(defn- <push-data-from-ws-handler
+  [repo push-data-from-ws]
+  (prn ::push-data-from-ws :push-data-from-ws)
+  (go nil)
+  ;; TODO
+  )
+
+(defn- <client-op-update-handler
+  [state]
+  {:pre [(some? @(:*graph-uuid state))
+         (some? @(:*repo state))]}
+  (go nil
+    ;; TODO
+    ))
+
+
+(defn- make-push-assets-update-ops-timeout-ch
+  [repo never-timeout?]
+  (if never-timeout?
+    (chan)
+    (go
+      (<! (async/timeout 2000))
+      ;; TODO: get-unpushed-assets-update-count
+      (pos? (op-mem-layer/get-unpushed-block-update-count repo)))))
+
+(defn <loop-for-assets-sync
+  [state graph-uuid repo]
+  {:pre [(state-validator state)]}
+  (go
+    (reset! (:*repo state) repo)
+    (reset! (:*graph-uuid state) graph-uuid)
+    (let [{:keys [data-from-ws-pub]} state
+          *auto-push-assets-update-ops? (:*auto-push-assets-update-ops? state)
+          toggle-auto-push-assets-update-ops-ch (:toggle-auto-push-assets-update-ops-chan state)
+          push-data-from-ws-ch (chan (async/sliding-buffer 100) (map rtc-const/data-from-ws-coercer))
+          stop-assets-sync-loop-chan (chan)]
+      (async/sub data-from-ws-pub "push-assets-updates" push-data-from-ws-ch)
+      (<! (go-loop [push-assets-update-ops-ch
+                    (make-push-assets-update-ops-timeout-ch repo (not @*auto-push-assets-update-ops?))]
+            (let [{:keys [continue push-data-from-ws client-assets-update stop]}
+                  (async/alt!
+                    toggle-auto-push-assets-update-ops-ch {:continue true}
+                    push-assets-update-ops-ch ([v] (if (and @*auto-push-assets-update-ops? (true? v))
+                                                     {:client-assets-update true}
+                                                     {:continue true}))
+                    push-data-from-ws-ch ([v] {:push-data-from-ws v})
+                    stop-assets-sync-loop-chan {:stop true}
+                    :priority true)]
+              (cond
+                continue
+                (recur (make-push-assets-update-ops-timeout-ch repo (not @*auto-push-assets-update-ops?)))
+
+                push-data-from-ws
+                (let [r (<push-data-from-ws-handler repo push-data-from-ws)]
+                  (prn ::<push-data-from-ws-handler r)
+                  (recur (make-push-assets-update-ops-timeout-ch repo (not @*auto-push-assets-update-ops?))))
+
+                client-assets-update
+                (let [maybe-exp (<! (user/<wrap-ensure-id&access-token
+                                     (<! (<client-op-update-handler state))))]
+                  (if (= :expired-token (:anom (ex-data maybe-exp)))
+                    (prn ::<loop-for-assets-sync "quitting loop" maybe-exp)
+                    (recur (make-push-assets-update-ops-timeout-ch repo (not @*auto-push-assets-update-ops?)))))
+
+                stop
+                ;; (ws/stop @(:*ws state)) ;; use same ws with <rtc-loop
+                (reset! (:*assets-update-state state) :closed)
+
+                :else nil))))
+      (async/unsub data-from-ws-pub "push-assets-update" push-data-from-ws-ch))))

src/main/frontend/db/rtc/util.cljs

@@ -0,0 +1,2 @@
+(ns frontend.db.rtc.util
+  "Some common util fns for rtc")

src/main/frontend/handler/import.cljs

@@ -144,6 +144,7 @@
             page-block (db/entity [:block/name page-name])]
         ;; Missing support for per block format (or deprecated?)
         (try (if whiteboard?
+               ;; only works for file graph :block/properties
                (let [blocks (->> children
                                  (map (partial medley/map-keys (fn [k] (keyword "block" k))))
                                  (map gp-whiteboard/migrate-shape-block)

src/main/frontend/handler/repo.cljs

@@ -354,7 +354,7 @@
                             (notification/show! (str "Removed graph "
                                                      (pr-str (text-util/get-graph-name-from-path url))
                                                      ". Redirecting to graph "
-                                                     (pr-str (text-util/get-graph-name-from-path url)))
+                                                     (pr-str (text-util/get-graph-name-from-path graph)))
                                                 :success)
                             (state/pub-event! [:graph/switch graph {:persist? false}]))
                           (notification/show! (str "Removed graph " (pr-str (text-util/get-graph-name-from-path url))) :success))))]
@@ -511,18 +511,24 @@
   (when (util/electron?)
     (ipc/ipc "graphReady" graph)))
 
-(defn- create-db [full-graph-name]
+(defn graph-already-exists?
+  "Checks to see if given db graph name already exists"
+  [graph-name]
+  (let [full-graph-name (string/lower-case (str config/db-version-prefix graph-name))]
+    (some #(= (string/lower-case (:url %)) full-graph-name) (state/get-repos))))
+
+(defn- create-db [full-graph-name {:keys [file-graph-import?]}]
   (->
    (p/let [_ (persist-db/<new full-graph-name)
            _ (start-repo-db-if-not-exists! full-graph-name)
            _ (state/add-repo! {:url full-graph-name})
-           _ (route-handler/redirect-to-home!)
+           _ (when-not file-graph-import? (route-handler/redirect-to-home!))
            initial-data (sqlite-create-graph/build-db-initial-data config/config-default-content)
            _ (db/transact! full-graph-name initial-data)
            _ (repo-config-handler/set-repo-config-state! full-graph-name config/config-default-content)
           ;; TODO: handle global graph
            _ (state/pub-event! [:init/commands])
-           _ (state/pub-event! [:page/create (date/today) {:redirect? false}])]
+           _ (when-not file-graph-import? (state/pub-event! [:page/create (date/today) {:redirect? false}]))]
      (js/setTimeout ui-handler/re-render-root! 100)
      (prn "New db created: " full-graph-name))
    (p/catch (fn [error]
@@ -531,11 +537,11 @@
 
 (defn new-db!
   "Handler for creating a new database graph"
-  [graph]
-  (let [full-graph-name (str config/db-version-prefix graph)
-        graph-already-exists? (some #(= (:url %) full-graph-name) (state/get-repos))]
-    (if graph-already-exists?
-      (state/pub-event! [:notification/show
-                         {:content (str "The graph '" graph "' already exists. Please try again with another name.")
-                          :status :error}])
-      (create-db full-graph-name))))
+  ([graph] (new-db! graph {}))
+  ([graph opts]
+   (let [full-graph-name (str config/db-version-prefix graph)]
+     (if (graph-already-exists? graph)
+       (state/pub-event! [:notification/show
+                          {:content (str "The graph '" graph "' already exists. Please try again with another name.")
+                           :status :error}])
+       (create-db full-graph-name opts)))))

src/main/frontend/search.cljs

@@ -96,8 +96,8 @@
   ([property q limit]
    (when-let [repo (state/get-current-repo)]
      (when q
-      (let [q (fuzzy/clean-str q)
-            result (db-async/<get-property-values repo (keyword property))]
+      (p/let [q (fuzzy/clean-str q)
+              result (db-async/<get-property-values repo (keyword property))]
         (when (seq result)
           (if (string/blank? q)
             result

src/main/frontend/worker/rtc/const.cljs

@@ -165,6 +165,12 @@
       [:req-id :string]
       [:action :string]
       [:graph-uuid :string]
+      [:block-uuids [:sequential :uuid]]]]
+    ["query-blocks"
+     [:map
+      [:req-id :string]
+      [:action :string]
+      [:graph-uuid :uuid]
       [:block-uuids [:sequential :uuid]]]]]))
 (def data-to-ws-encoder (m/encoder data-to-ws-schema mt/string-transformer))
 (def data-to-ws-coercer (m/coercer data-to-ws-schema mt/string-transformer))

src/main/frontend/worker/rtc/core.cljs

@@ -747,6 +747,7 @@
 (defonce *state (atom nil))
 
 (defn <loop-for-rtc
+  ":loop-started-ch used to notify that rtc-loop started"
   [state graph-uuid repo conn date-formatter & {:keys [loop-started-ch token]}]
   {:pre [(state-validator state)
          (some? graph-uuid)
@@ -848,6 +849,13 @@
               (p/resolve! d (bean/->js versions)))))))
     d))
 
+;; (defn- <query-page-blocks
+;;   [state page-block-uuid]
+;;   (go
+;;     (when (some-> state :*graph-uuid deref)
+;;       (<! (ws/<send&receive state {:action "query-blocks" :graph-uuid @(:*graph-uuid state)
+;;                                    :block-uuids [page-block-uuid]})))))
+
 
 (defn init-state
   [ws data-from-ws-chan repo token]

src/main/frontend/worker/rtc/op_mem_layer.cljs

@@ -58,7 +58,20 @@
      [:op :string]
      [:value [:map
               [:block-uuid :uuid]
+              [:epoch :int]]]]]
+   ["update-asset"
+    [:catn
+     [:op :string]
+     [:value [:map
+              [:asset-uuid :uuid]
+              [:epoch :int]]]]]
+   ["remove-asset"
+    [:catn
+     [:op :string]
+     [:value [:map
+              [:asset-uuid :uuid]
               [:epoch :int]]]]]])
+
 (def ops-schema [:sequential op-schema])
 
 (def ops-from-store-schema [:sequential [:catn
@@ -77,7 +90,10 @@
    [:local-tx {:optional true} :int]
    [:block-uuid->ops [:map-of :uuid
                       [:map-of [:enum :move :remove :update :update-page :remove-page] :any]]]
-   [:epoch->block-uuid-sorted-map [:map-of :int :uuid]]])
+   [:asset-uuid->ops [:map-of :uuid
+                      [:map-of [:enum :update-asset :remove-asset] :any]]]
+   [:epoch->block-uuid-sorted-map [:map-of :int :uuid]]
+   [:epoch->asset-uuid-sorted-map [:map-of :int :uuid]]])
 
 (def ops-store-schema
   [:map-of :string                      ; repo-name
@@ -138,22 +154,32 @@
            seq
            (apply min)))
 
-(defn add-ops-to-block-uuid->ops
-  [ops block-uuid->ops epoch->block-uuid-sorted-map]
+(defn- asset-uuid->min-epoch
+  [asset-uuid->ops asset-uuid]
+  (block-uuid->min-epoch asset-uuid->ops asset-uuid))
+
+(defn ^:large-vars/cleanup-todo add-ops-aux
+  [ops block-uuid->ops epoch->block-uuid-sorted-map asset-uuid->ops epoch->asset-uuid-sorted-map]
   (loop [block-uuid->ops block-uuid->ops
          epoch->block-uuid-sorted-map epoch->block-uuid-sorted-map
+         asset-uuid->ops asset-uuid->ops
+         epoch->asset-uuid-sorted-map epoch->asset-uuid-sorted-map
          [op & others] ops]
     (if-not op
       {:block-uuid->ops block-uuid->ops
-       :epoch->block-uuid-sorted-map epoch->block-uuid-sorted-map}
+       :asset-uuid->ops asset-uuid->ops
+       :epoch->block-uuid-sorted-map epoch->block-uuid-sorted-map
+       :epoch->asset-uuid-sorted-map epoch->asset-uuid-sorted-map}
       (let [[op-type value] op
-            {:keys [block-uuid epoch]} value
-            exist-ops (block-uuid->ops block-uuid)]
+            {:keys [block-uuid asset-uuid epoch]} value
+            exist-ops (some-> block-uuid block-uuid->ops)
+            exist-asset-ops (some-> asset-uuid asset-uuid->ops)]
         (case op-type
           "move"
           (let [already-removed? (some-> (get exist-ops :remove) second :epoch (> epoch))]
             (if already-removed?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map others)
+              (recur block-uuid->ops epoch->block-uuid-sorted-map
+                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
               (let [block-uuid->ops* (-> block-uuid->ops
                                          (assoc-in [block-uuid :move] op)
                                          (update block-uuid dissoc :remove))
@@ -165,11 +191,12 @@
                        (-> epoch->block-uuid-sorted-map
                            (dissoc origin-min-epoch)
                            (assoc min-epoch block-uuid))
-                       others))))
+                       asset-uuid->ops epoch->asset-uuid-sorted-map others))))
           "update"
           (let [already-removed? (some-> (get exist-ops :remove) second :epoch (> epoch))]
             (if already-removed?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map others)
+              (recur block-uuid->ops epoch->block-uuid-sorted-map
+                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
               (let [origin-update-op (get-in block-uuid->ops [block-uuid :update])
                     op* (if origin-update-op (merge-update-ops origin-update-op op) op)
                     block-uuid->ops* (-> block-uuid->ops
@@ -181,11 +208,12 @@
                        (-> epoch->block-uuid-sorted-map
                            (dissoc origin-min-epoch)
                            (assoc min-epoch block-uuid))
-                       others))))
+                       asset-uuid->ops epoch->asset-uuid-sorted-map others))))
           "remove"
           (let [add-after-remove? (some-> (get exist-ops :move) second :epoch (> epoch))]
             (if add-after-remove?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map others)
+              (recur block-uuid->ops epoch->block-uuid-sorted-map
+                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
               (let [block-uuid->ops* (assoc block-uuid->ops block-uuid {:remove op})
                     origin-min-epoch (block-uuid->min-epoch block-uuid->ops block-uuid)
                     min-epoch (block-uuid->min-epoch block-uuid->ops* block-uuid)]
@@ -193,11 +221,12 @@
                        (-> epoch->block-uuid-sorted-map
                            (dissoc origin-min-epoch)
                            (assoc min-epoch block-uuid))
-                       others))))
+                       asset-uuid->ops epoch->asset-uuid-sorted-map others))))
           "update-page"
           (let [already-removed? (some-> (get exist-ops :remove-page) second :epoch (> epoch))]
             (if already-removed?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map others)
+              (recur block-uuid->ops epoch->block-uuid-sorted-map
+                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
               (let [block-uuid->ops* (-> block-uuid->ops
                                          (assoc-in [block-uuid :update-page] op)
                                          (update block-uuid dissoc :remove-page))
@@ -207,11 +236,12 @@
                        (-> epoch->block-uuid-sorted-map
                            (dissoc origin-min-epoch)
                            (assoc min-epoch block-uuid))
-                       others))))
+                       asset-uuid->ops epoch->asset-uuid-sorted-map others))))
           "remove-page"
           (let [add-after-remove? (some-> (get exist-ops :update-page) second :epoch (> epoch))]
             (if add-after-remove?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map others)
+              (recur block-uuid->ops epoch->block-uuid-sorted-map
+                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
               (let [block-uuid->ops* (assoc block-uuid->ops block-uuid {:remove-page op})
                     origin-min-epoch (block-uuid->min-epoch block-uuid->ops block-uuid)
                     min-epoch (block-uuid->min-epoch block-uuid->ops* block-uuid)]
@@ -219,11 +249,42 @@
                        (-> epoch->block-uuid-sorted-map
                            (dissoc origin-min-epoch)
                            (assoc min-epoch block-uuid))
-                       others)))))))))
+                       asset-uuid->ops epoch->asset-uuid-sorted-map others))))
+          "update-asset"
+          (let [already-removed? (some-> (get exist-asset-ops :remove-asset) second :epoch (> epoch))]
+            (if already-removed?
+              (recur block-uuid->ops epoch->block-uuid-sorted-map
+                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
+              (let [asset-uuid->ops* (assoc asset-uuid->ops asset-uuid {:update-asset op})
+                    origin-min-epoch (asset-uuid->min-epoch asset-uuid->ops asset-uuid)
+                    min-epoch (asset-uuid->min-epoch asset-uuid->ops* asset-uuid)]
+                (recur block-uuid->ops epoch->block-uuid-sorted-map
+                       asset-uuid->ops*
+                       (-> epoch->asset-uuid-sorted-map
+                           (dissoc origin-min-epoch)
+                           (assoc min-epoch asset-uuid))
+                       others))))
+          "remove-asset"
+          (let [add-after-remove? (some-> (get exist-asset-ops :update-asset) second :epoch (> epoch))]
+            (if add-after-remove?
+              (recur block-uuid->ops epoch->block-uuid-sorted-map
+                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
+              (let [asset-uuid->ops* (assoc asset-uuid->ops asset-uuid {:remove-asset op})
+                    origin-min-epoch (asset-uuid->min-epoch asset-uuid->ops asset-uuid)
+                    min-epoch (asset-uuid->min-epoch asset-uuid->ops* asset-uuid)]
+                (recur block-uuid->ops epoch->block-uuid-sorted-map
+                       asset-uuid->ops*
+                       (-> epoch->asset-uuid-sorted-map
+                           (dissoc origin-min-epoch)
+                           (assoc min-epoch asset-uuid))
+                       others))))
+          )))))
 
 
 (def empty-ops-store-value {:current-branch {:block-uuid->ops {}
-                                             :epoch->block-uuid-sorted-map (sorted-map-by <)}})
+                                             :epoch->block-uuid-sorted-map (sorted-map-by <)
+                                             :asset-uuid->ops {}
+                                             :epoch->asset-uuid-sorted-map (sorted-map-by <)}})
 
 (defn init-empty-ops-store!
   [repo]
@@ -239,14 +300,19 @@
   (let [ops (ops-coercer ops)
         {{old-branch-block-uuid->ops :block-uuid->ops
           old-epoch->block-uuid-sorted-map :epoch->block-uuid-sorted-map
+          old-branch-asset-uuid->ops :asset-uuid->ops
+          old-epoch->asset-uuid-sorted-map :epoch->asset-uuid-sorted-map
           :as old-branch} :old-branch
-         {:keys [block-uuid->ops epoch->block-uuid-sorted-map]} :current-branch}
+         {:keys [block-uuid->ops epoch->block-uuid-sorted-map
+                 asset-uuid->ops epoch->asset-uuid-sorted-map]} :current-branch}
         (get @*ops-store repo)
         {:keys [block-uuid->ops epoch->block-uuid-sorted-map]}
-        (add-ops-to-block-uuid->ops ops block-uuid->ops epoch->block-uuid-sorted-map)
+        (add-ops-aux ops block-uuid->ops epoch->block-uuid-sorted-map
+                                    asset-uuid->ops epoch->asset-uuid-sorted-map)
         {old-branch-block-uuid->ops :block-uuid->ops old-epoch->block-uuid-sorted-map :epoch->block-uuid-sorted-map}
         (when old-branch
-          (add-ops-to-block-uuid->ops ops old-branch-block-uuid->ops old-epoch->block-uuid-sorted-map))]
+          (add-ops-aux ops old-branch-block-uuid->ops old-epoch->block-uuid-sorted-map
+                                      old-branch-asset-uuid->ops old-epoch->asset-uuid-sorted-map))]
     (swap! *ops-store update repo
            (fn [{:keys [current-branch old-branch]}]
              (cond-> {:current-branch
@@ -259,6 +325,37 @@
                              :block-uuid->ops old-branch-block-uuid->ops
                              :epoch->block-uuid-sorted-map old-epoch->block-uuid-sorted-map)))))))
 
+(comment
+  (defn add-asset-ops!
+   [repo ops]
+   (assert (contains? (@*ops-store repo) :current-branch) (@*ops-store repo))
+   (let [ops (ops-coercer ops)
+         {{old-branch-block-uuid->ops :block-uuid->ops
+           old-epoch->block-uuid-sorted-map :epoch->block-uuid-sorted-map
+           old-branch-asset-uuid->ops :asset-uuid->ops
+           old-epoch->asset-uuid-sorted-map :epoch->asset-uuid-sorted-map
+           :as old-branch} :old-branch
+          {:keys [block-uuid->ops epoch->block-uuid-sorted-map
+                  asset-uuid->ops epoch->asset-uuid-sorted-map]} :current-branch}
+         (get @*ops-store repo)
+         {:keys [asset-uuid->ops epoch->asset-uuid-sorted-map]}
+         (add-ops-aux ops block-uuid->ops epoch->block-uuid-sorted-map
+                      asset-uuid->ops epoch->asset-uuid-sorted-map)
+         {old-branch-asset-uuid->ops :asset-uuid->ops old-epoch->asset-uuid-sorted-map :epoch->asset-uuid-sorted-map}
+         (when old-branch
+           (add-ops-aux ops old-branch-block-uuid->ops old-epoch->block-uuid-sorted-map
+                        old-branch-asset-uuid->ops old-epoch->asset-uuid-sorted-map))]
+     (swap! *ops-store update repo
+            (fn [{:keys [current-branch old-branch]}]
+              (cond-> {:current-branch
+                       (assoc current-branch
+                              :asset-uuid->ops asset-uuid->ops
+                              :epoch->asset-uuid-sorted-map epoch->asset-uuid-sorted-map)}
+                old-branch
+                (assoc :old-branch
+                       (assoc old-branch
+                              :asset-uuid->ops old-branch-asset-uuid->ops
+                              :epoch->asset-uuid-sorted-map old-epoch->asset-uuid-sorted-map))))))))
 
 (defn update-local-tx!
   [repo t]
@@ -360,6 +457,18 @@
              :block-uuid->ops (dissoc block-uuid->ops block-uuid)
              :epoch->block-uuid-sorted-map (dissoc epoch->block-uuid-sorted-map min-epoch)))))
 
+(comment
+  (defn remove-asset-ops!
+   [repo asset-uuid]
+   {:pre [(uuid? asset-uuid)]}
+   (let [repo-ops-store (get @*ops-store repo)
+         {:keys [epoch->asset-uuid-sorted-map asset-uuid->ops]} (:current-branch repo-ops-store)]
+     (assert (contains? repo-ops-store :current-branch) repo)
+     (let [min-epoch (asset-uuid->min-epoch asset-uuid->ops asset-uuid)]
+       (swap! *ops-store update-in [repo :current-branch] assoc
+              :asset-uuid->ops (dissoc asset-uuid->ops asset-uuid)
+              :epoch->asset-uuid-sorted-map (dissoc epoch->asset-uuid-sorted-map min-epoch))))))
+
 
 (defn <init-load-from-indexeddb!
   [repo]
@@ -373,10 +482,12 @@
                      (sort-by first <)
                      ops-from-store-coercer
                      (map second))
-            {:keys [block-uuid->ops epoch->block-uuid-sorted-map]}
-            (add-ops-to-block-uuid->ops ops {} (sorted-map-by <))
+            {:keys [block-uuid->ops epoch->block-uuid-sorted-map asset-uuid->ops epoch->asset-uuid-sorted-map]}
+            (add-ops-aux ops {} (sorted-map-by <) {} (sorted-map-by <))
             r (cond-> {:block-uuid->ops block-uuid->ops
-                       :epoch->block-uuid-sorted-map epoch->block-uuid-sorted-map}
+                       :epoch->block-uuid-sorted-map epoch->block-uuid-sorted-map
+                       :asset-uuid->ops asset-uuid->ops
+                       :epoch->asset-uuid-sorted-map epoch->asset-uuid-sorted-map}
                 graph-uuid (assoc :graph-uuid graph-uuid)
                 local-tx (assoc :local-tx local-tx))]
         (assert (ops-validator ops) ops)

src/main/logseq/api.cljs

@@ -139,7 +139,7 @@
 (def ^:export get_current_graph_templates
   (fn []
     (when-let [repo (state/get-current-repo)]
-      (let [templates (db-async/<get-all-templates repo)]
+      (p/let [templates (db-async/<get-all-templates repo)]
         (some-> templates
                 (update-vals db/pull)
                 (sdk-utils/normalize-keyword-for-json)

src/test/frontend/worker/rtc/op_mem_layer_test.cljs

@@ -17,7 +17,7 @@
     (let [ops [["move" {:block-uuid "f4abd682-fb9e-4f1a-84bf-5fe11fe7844b" :epoch 1}]
                ["move" {:block-uuid "8e6d8355-ded7-4500-afaa-6f721f3b0dc6" :epoch 2}]]
           {:keys [block-uuid->ops epoch->block-uuid-sorted-map]}
-          (op-layer/add-ops-to-block-uuid->ops (op-layer/ops-coercer ops) {} (sorted-map-by <))]
+          (op-layer/add-ops-aux (op-layer/ops-coercer ops) {} (sorted-map-by <) {} (sorted-map-by <))]
       (is (= [{#uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b"
                {:move ["move" {:block-uuid #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b", :epoch 1}]},
                #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"
@@ -34,7 +34,7 @@
                ["update" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 3
                           :updated-attrs {:type {:add #{"type1"}}}}]]
           {:keys [block-uuid->ops epoch->block-uuid-sorted-map]}
-          (op-layer/add-ops-to-block-uuid->ops (op-layer/ops-coercer ops) {} (sorted-map-by <))]
+          (op-layer/add-ops-aux (op-layer/ops-coercer ops) {} (sorted-map-by <) {} (sorted-map-by <))]
       (is (= [{#uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02"
                {:move
                 ["move" {:block-uuid #uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02", :epoch 1}],
@@ -52,7 +52,7 @@
                ["update" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 4
                           :updated-attrs {:content nil :link nil}}]]
           {:keys [block-uuid->ops]}
-          (op-layer/add-ops-to-block-uuid->ops (op-layer/ops-coercer ops) {} (sorted-map-by <))]
+          (op-layer/add-ops-aux (op-layer/ops-coercer ops) {} (sorted-map-by <) {} (sorted-map-by <))]
       (is (= ["update"
               {:block-uuid #uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02"
                :updated-attrs {:content nil :link nil}
@@ -61,14 +61,26 @@
   (testing "case4: update-page then remove-page"
     (let [ops1 [["update-page" {:block-uuid #uuid "65564abe-1e79-4ae8-af60-215826cefea9" :epoch 1}]]
           ops2 [["remove-page" {:block-uuid #uuid "65564abe-1e79-4ae8-af60-215826cefea9" :epoch 2}]]
-          {:keys [block-uuid->ops epoch->block-uuid-sorted-map]}
-          (op-layer/add-ops-to-block-uuid->ops (op-layer/ops-coercer ops1) {} (sorted-map-by <))
+          {:keys [block-uuid->ops epoch->block-uuid-sorted-map asset-uuid->ops epoch->asset-uuid-sorted-map]}
+          (op-layer/add-ops-aux (op-layer/ops-coercer ops1) {} (sorted-map-by <) {} (sorted-map-by <))
           {block-uuid->ops2 :block-uuid->ops}
-          (op-layer/add-ops-to-block-uuid->ops (op-layer/ops-coercer ops2) block-uuid->ops epoch->block-uuid-sorted-map)]
+          (op-layer/add-ops-aux (op-layer/ops-coercer ops2)
+                                               block-uuid->ops epoch->block-uuid-sorted-map
+                                               asset-uuid->ops epoch->asset-uuid-sorted-map)]
       (is (= {#uuid "65564abe-1e79-4ae8-af60-215826cefea9"
               {:remove-page ["remove-page" {:block-uuid #uuid "65564abe-1e79-4ae8-af60-215826cefea9", :epoch 2}]}}
              block-uuid->ops2)))))
 
+(deftest add-ops-to-asset-uuid->ops-test
+  (let [[uuid1 uuid2] (repeatedly random-uuid)
+        ops1 [["update-asset" {:asset-uuid uuid1 :epoch 1}]
+              ["update-asset" {:asset-uuid uuid2 :epoch 2}]]
+        {:keys [asset-uuid->ops]}
+        (op-layer/add-ops-aux (op-layer/ops-coercer ops1) {} (sorted-map-by <) {} (sorted-map-by <))]
+    (is (= {uuid1 {:update-asset ["update-asset" {:asset-uuid uuid1 :epoch 1}]}
+            uuid2 {:update-asset ["update-asset" {:asset-uuid uuid2 :epoch 2}]}}
+           asset-uuid->ops))))
+
 
 (deftest process-test
   (let [repo (make-db-graph-repo-name "process-test")
@@ -197,6 +209,8 @@
                           :epoch 4}]}},
                       :epoch->block-uuid-sorted-map
                       {1 #uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02"}
+                      :asset-uuid->ops {}
+                      :epoch->asset-uuid-sorted-map {}
                       :local-tx 1}}
                     repo-ops-store1))
              (is (= repo-ops-store1 repo-ops-store2)))))