Merge branch 'feat/db' into enhance/plugin-web

.lsp/config.edn

@@ -1,4 +1,13 @@
 {:source-aliases #{:cljs}
  :source-paths-ignore-regex ["src/resources" "target.*"]
  :paths-ignore-regex ["src/resources"]
- :clean {:ns-inner-blocks-indentation :same-line}}
+ :clean {:ns-inner-blocks-indentation :same-line}
+ :additional-snippets [{:name "profile"
+                        :detail "Insert profile-fn"
+                        :snippet
+                        "
+(comment
+(require '[logseq.common.profile :as c.p])
+(do (vreset! c.p/*key->call-count {})
+    (vreset! c.p/*key->time-sum {}))
+(c.p/profile-fn! $1) )"}]}

deps/common/.carve/ignore

@@ -2,3 +2,8 @@
 logseq.common.graph/get-files
 ;; API fn
 logseq.common.graph/read-directories
+
+;; Profile utils
+logseq.common.profile/profile-fn!
+logseq.common.profile/*key->call-count
+logseq.common.profile/*key->time-sum

deps/common/src/logseq/common/profile.clj

@@ -0,0 +1,18 @@
+(ns logseq.common.profile
+  "Utils for profiling")
+
+(defmacro profile-fn!
+  [f & {:keys [print-on-call? gen-k-fn]
+        :or {print-on-call? true}}]
+  `(let [origin-f# ~f
+         gen-k-fn# (or ~gen-k-fn (constantly (keyword ~f)))]
+     (set! ~f (fn [& args#]
+                (let [start# (cljs.core/system-time)
+                      r# (apply origin-f# args#)
+                      end# (cljs.core/system-time)
+                      k# (gen-k-fn# r#)]
+                  (vswap! *key->call-count update k# inc)
+                  (vswap! *key->time-sum update k# #(+ % (- end# start#)))
+                  (when ~print-on-call?
+                    (println "call-count:" (get @*key->call-count k#) "time-sum(ms):" (get @*key->time-sum k#)))
+                  r#)))))

deps/common/src/logseq/common/profile.cljs

@@ -0,0 +1,11 @@
+(ns logseq.common.profile
+  "Utils for profiling"
+  (:require-macros [logseq.common.profile]))
+
+(def *key->call-count
+  "key -> count"
+  (volatile! {}))
+
+(def *key->time-sum
+  "docstring"
+  (volatile! {}))

deps/db/src/logseq/db/frontend/entity_plus.cljc

@@ -7,36 +7,42 @@
                                         :unresolved-symbol {:level :off}}}})
   (:require #?(:org.babashka/nbb [datascript.db])
             [cljs.core]
+            [clojure.data :as data]
             [datascript.core :as d]
             [datascript.impl.entity :as entity :refer [Entity]]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]))
 
-(def immutable-db-idents
-  "These db-ident entities are immutable,
-  it means `(db/entity :block/title)` always return same result"
-  #{:block/created-at :block/updated-at
-    :block/uuid :block/title :block/tags :block/name :block/format
-    :block/schema :block/path-refs :block/refs :block/tx-id :block/type
-    :block/page :block/parent :block/order :block/journal-day :block/closed-value-property
-    :block/link :block/marker :block/warning :block/collapsed? :block/deadline :block/scheduled :block/level
-    :block/pre-block? :block/heading-level
+(def nil-db-ident-entities
+  "No such entities with these :db/ident, but `(d/entity <db> <ident>)` has been called somewhere."
+  #{:block/tx-id :block/warning :block/pre-block? :block/uuid :block/scheduled
+    :block/deadline :block/journal-day :block/format :block/level :block/heading-level
+    :block/type :block/name :block/marker :block/_refs
 
-    :block/_refs :logseq.property/_query
+    :block.temp/ast-title :block.temp/top? :block.temp/bottom? :block.temp/search?
+    :block.temp/fully-loaded? :block.temp/ast-body
 
-    :block.temp/search? :block.temp/ast-title :block.temp/ast-body
-    :block.temp/fully-loaded? :block.temp/top? :block.temp/bottom?
+    :db/valueType :db/cardinality :db/ident :db/index
 
-    :db/cardinality :db/ident :db/index :db/valueType
+    :logseq.property/_query})
 
-    :logseq.kv/db-type
+(def immutable-db-ident-entities
+  "These db-ident entities are immutable,
+  it means `(db/entity :block/title)` always return same result"
+  #{:block/link :block/updated-at :block/refs :block/closed-value-property
+    :block/created-at :block/collapsed? :block/schema :block/tags :block/title
+    :block/path-refs :block/parent :block/order :block/page
 
-    :logseq.property.node/display-type :logseq.property/icon
-    :logseq.property.asset/type :logseq.property.asset/checksum
     :logseq.property/created-from-property
+    :logseq.property/icon
+    :logseq.property.asset/type
+    :logseq.property.asset/checksum
+    :logseq.property.node/display-type
+
+    :logseq.kv/db-type})
 
-    :logseq.class/Query :logseq.class/Journal :logseq.class/Cards :logseq.class/Task})
+(assert (empty? (last (data/diff immutable-db-ident-entities nil-db-ident-entities))))
 
 (def ^:private lookup-entity @#'entity/lookup-entity)
 
@@ -55,14 +61,15 @@
 
 (defn entity-memoized
   [db eid]
-  (if (and @*reset-cache-background-task-running?
-           (qualified-keyword? eid)
-           (contains? immutable-db-idents eid))
-    (if-let [e (find @*seen-immutable-entities eid)]
-      (val e)
-      (let [r (d/entity db eid)]
-        (vswap! *seen-immutable-entities assoc eid r)
-        r))
+  (if (qualified-keyword? eid)
+    (when-not (contains? nil-db-ident-entities eid) ;fast return nil
+      (if (and @*reset-cache-background-task-running?
+               (contains? immutable-db-ident-entities eid)) ;return cache entity if possible which isn't nil
+        (or (get @*seen-immutable-entities eid)
+            (let [r (d/entity db eid)]
+              (when r (vswap! *seen-immutable-entities assoc eid r))
+              r))
+        (d/entity db eid)))
     (d/entity db eid)))
 
 (defn db-based-graph?

deps/db/src/logseq/db/frontend/entity_util.cljs

@@ -7,10 +7,16 @@
 
 (defn- has-tag?
   [entity tag-ident]
-  (let [tags (:block/tags entity)]
-    (some (fn [t] (or (= (:db/ident t) tag-ident)
-                      (= t tag-ident)))
-          (if (coll? tags) tags [tags]))))
+  (some (fn [t]
+          (or (keyword-identical? (:db/ident t) tag-ident)
+              (keyword-identical? t tag-ident)))
+        (:block/tags entity)))
+
+(comment
+  (require '[logseq.common.profile :as c.p])
+  (do (vreset! c.p/*key->call-count {})
+      (vreset! c.p/*key->time-sum {}))
+  (c.p/profile-fn! has-tag? :print-on-call? false))
 
 (defn internal-page?
   [entity]
@@ -18,8 +24,8 @@
 
 (defn class?
   [entity]
-  (or (= (:db/ident entity) :logseq.class/Tag)
-      (has-tag? entity :logseq.class/Tag)))
+  (or (has-tag? entity :logseq.class/Tag)
+      (keyword-identical? (:db/ident entity) :logseq.class/Tag)))
 
 (defn property?
   [entity]
@@ -32,7 +38,7 @@
    ;; db based graph
    (has-tag? entity :logseq.class/Whiteboard)
    ;; file based graph
-   (= "whiteboard" (:block/type entity))))
+   (identical? "whiteboard" (:block/type entity))))
 
 (defn closed-value?
   [entity]
@@ -45,7 +51,7 @@
    ;; db based graph
    (has-tag? entity :logseq.class/Journal)
    ;; file based graph
-   (= "journal" (:block/type entity))))
+   (identical? "journal" (:block/type entity))))
 
 (defn page?
   [entity]

gulpfile.js

@@ -6,7 +6,7 @@ const path = require('path')
 const gulp = require('gulp')
 const del = require('del')
 const ip = require('ip')
-const replace = require('gulp-replace');
+const replace = require('gulp-replace')
 
 const outputPath = path.join(__dirname, 'static')
 const resourcesPath = path.join(__dirname, 'resources')
@@ -19,26 +19,27 @@ const css = {
   watchCSS () {
     return cp.spawn(`yarn css:watch`, {
       shell: true,
-      stdio: 'inherit'
+      stdio: 'inherit',
     })
   },
 
   buildCSS (...params) {
     return gulp.series(
       () => exec(`yarn css:build`, {}),
-      css._optimizeCSSForRelease
+      css._optimizeCSSForRelease,
     )(...params)
   },
 
   _optimizeCSSForRelease () {
-    return gulp.src(path.join(outputPath, 'css', 'style.css'))
-      .pipe(gulp.dest(path.join(outputPath, 'css')))
-  }
+    return gulp.src(path.join(outputPath, 'css', 'style.css')).
+      pipe(gulp.dest(path.join(outputPath, 'css')))
+  },
 }
 
 const common = {
   clean () {
-    return del(['./static/**/*', '!./static/yarn.lock', '!./static/node_modules'])
+    return del(
+      ['./static/**/*', '!./static/yarn.lock', '!./static/node_modules'])
   },
 
   syncResourceFile () {
@@ -50,7 +51,7 @@ const common = {
     return gulp.series(
       () => gulp.src([
         './node_modules/@excalidraw/excalidraw/dist/excalidraw-assets/**',
-        '!**/*/i18n-*.js'
+        '!**/*/i18n-*.js',
       ]).pipe(gulp.dest(path.join(outputPath, 'js', 'excalidraw-assets'))),
       () => gulp.src([
         'node_modules/katex/dist/katex.min.js',
@@ -70,11 +71,14 @@ const common = {
         'node_modules/react/umd/react.development.js',
         'node_modules/react-dom/umd/react-dom.production.min.js',
         'node_modules/react-dom/umd/react-dom.development.js',
-        'node_modules/prop-types/prop-types.min.js'
+        'node_modules/prop-types/prop-types.min.js',
       ]).pipe(gulp.dest(path.join(outputPath, 'js'))),
       () => gulp.src([
-        'node_modules/@tabler/icons-react/dist/umd/tabler-icons-react.min.js'
-      ]).pipe(replace('"@tabler/icons-react"]={},a.react,', '"tablerIcons"]={},a.React,')).pipe(gulp.dest(path.join(outputPath, 'js'))),
+        'node_modules/@tabler/icons-react/dist/umd/tabler-icons-react.min.js',
+      ]).
+        pipe(replace('"@tabler/icons-react"]={},a.react,',
+          '"tablerIcons"]={},a.React,')).
+        pipe(gulp.dest(path.join(outputPath, 'js'))),
       () => gulp.src([
         'node_modules/@glidejs/glide/dist/glide.min.js',
         'node_modules/@glidejs/glide/dist/css/glide.core.min.css',
@@ -83,7 +87,7 @@ const common = {
       () => gulp.src([
         'node_modules/pdfjs-dist/build/pdf.js',
         'node_modules/pdfjs-dist/build/pdf.worker.js',
-        'node_modules/pdfjs-dist/web/pdf_viewer.js'
+        'node_modules/pdfjs-dist/web/pdf_viewer.js',
       ]).pipe(gulp.dest(path.join(outputPath, 'js', 'pdfjs'))),
       () => gulp.src([
         'node_modules/pdfjs-dist/cmaps/*.*',
@@ -92,37 +96,38 @@ const common = {
         'node_modules/inter-ui/inter.css',
         'node_modules/reveal.js/dist/theme/fonts/source-sans-pro/**',
       ]).pipe(gulp.dest(path.join(outputPath, 'css'))),
-      () => gulp.src('node_modules/inter-ui/Inter (web)/*.*')
-        .pipe(gulp.dest(path.join(outputPath, 'css', 'Inter (web)'))),
+      () => gulp.src('node_modules/inter-ui/Inter (web)/*.*').
+        pipe(gulp.dest(path.join(outputPath, 'css', 'Inter (web)'))),
       () => gulp.src([
         'node_modules/@tabler/icons-webfont/fonts/**',
-        'node_modules/katex/dist/fonts/*.woff2'
+        'node_modules/katex/dist/fonts/*.woff2',
       ]).pipe(gulp.dest(path.join(outputPath, 'css', 'fonts'))),
     )(...params)
   },
 
   keepSyncResourceFile () {
-    return gulp.watch(resourceFilePath, { ignoreInitial: true }, common.syncResourceFile)
+    return gulp.watch(resourceFilePath, { ignoreInitial: true },
+      common.syncResourceFile)
   },
 
   syncAllStatic () {
     return gulp.src([
       outputFilePath,
-      '!' + path.join(outputPath, 'node_modules/**')
+      '!' + path.join(outputPath, 'node_modules/**'),
     ]).pipe(gulp.dest(publicStaticPath))
   },
 
   syncJS_CSSinRt () {
     return gulp.src([
       path.join(outputPath, 'js/**'),
-      path.join(outputPath, 'css/**')
+      path.join(outputPath, 'css/**'),
     ], { base: outputPath }).pipe(gulp.dest(publicStaticPath))
   },
 
   keepSyncStaticInRt () {
     return gulp.watch([
       path.join(outputPath, 'js/**'),
-      path.join(outputPath, 'css/**')
+      path.join(outputPath, 'css/**'),
     ], { ignoreInitial: true }, common.syncJS_CSSinRt)
   },
 
@@ -136,7 +141,8 @@ const common = {
       try {
         await fetch(LOGSEQ_APP_SERVER_URL)
       } catch (e) {
-        return cb(new Error(`/* ❌ Please check if the service is ON. (${LOGSEQ_APP_SERVER_URL}) ❌ */`))
+        return cb(new Error(
+          `/* ❌ Please check if the service is ON. (${LOGSEQ_APP_SERVER_URL}) ❌ */`))
       }
     }
 
@@ -147,49 +153,63 @@ const common = {
     cp.execSync(`npx cap sync ${mode}`, {
       stdio: 'inherit',
       env: Object.assign(process.env, {
-        LOGSEQ_APP_SERVER_URL
-      })
+        LOGSEQ_APP_SERVER_URL,
+      }),
     })
 
     cp.execSync(`rm -rf ios/App/App/public/static/out`, {
-      stdio: 'inherit'
+      stdio: 'inherit',
     })
 
-
     cp.execSync(`npx cap run ${mode} --external`, {
       stdio: 'inherit',
       env: Object.assign(process.env, {
-        LOGSEQ_APP_SERVER_URL
-      })
+        LOGSEQ_APP_SERVER_URL,
+      }),
     })
 
     cb()
-  }
+  },
+
+  switchReactDevelopmentMode (cb) {
+    const reactFrom = path.join(outputPath, 'js', 'react.development.js')
+    const reactTo = path.join(outputPath, 'js', 'react.production.min.js')
+    cp.execSync(`mv ${reactFrom} ${reactTo}`, { stdio: 'inherit' })
+
+    const reactDomFrom = path.join(outputPath, 'js', 'react-dom.development.js')
+    const reactDomTo = path.join(outputPath, 'js',
+      'react-dom.production.min.js')
+    cp.execSync(`mv ${reactDomFrom} ${reactDomTo}`, { stdio: 'inherit' })
+
+    cb()
+  },
 }
 
 exports.electron = () => {
   if (!fs.existsSync(path.join(outputPath, 'node_modules'))) {
     cp.execSync('yarn', {
       cwd: outputPath,
-      stdio: 'inherit'
+      stdio: 'inherit',
     })
   }
 
   cp.execSync('yarn electron:dev', {
     cwd: outputPath,
-    stdio: 'inherit'
+    stdio: 'inherit',
   })
 }
 
 exports.electronMaker = async () => {
   cp.execSync('yarn cljs:release-electron', {
-    stdio: 'inherit'
+    stdio: 'inherit',
   })
 
   const pkgPath = path.join(outputPath, 'package.json')
   const pkg = require(pkgPath)
-  const version = fs.readFileSync(path.join(__dirname, 'src/main/frontend/version.cljs'))
-    .toString().match(/[0-9.]{3,}/)[0]
+  const version = fs.readFileSync(
+    path.join(__dirname, 'src/main/frontend/version.cljs')).
+    toString().
+    match(/[0-9.]{3,}/)[0]
 
   if (!version) {
     throw new Error('release version error in src/**/*/version.cljs')
@@ -201,18 +221,21 @@ exports.electronMaker = async () => {
   if (!fs.existsSync(path.join(outputPath, 'node_modules'))) {
     cp.execSync('yarn', {
       cwd: outputPath,
-      stdio: 'inherit'
+      stdio: 'inherit',
     })
   }
 
   cp.execSync('yarn electron:make', {
     cwd: outputPath,
-    stdio: 'inherit'
+    stdio: 'inherit',
   })
 }
 
 exports.cap = common.runCapWithLocalDevServerEntry
 exports.clean = common.clean
-exports.watch = gulp.series(common.syncResourceFile, common.syncAssetFiles, common.syncAllStatic,
+exports.watch = gulp.series(common.syncResourceFile,
+  common.syncAssetFiles, common.syncAllStatic,
+  common.switchReactDevelopmentMode,
   gulp.parallel(common.keepSyncResourceFile, css.watchCSS))
-exports.build = gulp.series(common.clean, common.syncResourceFile, common.syncAssetFiles, css.buildCSS)
+exports.build = gulp.series(common.clean, common.syncResourceFile,
+  common.syncAssetFiles, css.buildCSS)

src/main/frontend/components/container.cljs

@@ -1056,7 +1056,7 @@
       [:a#download.hidden]
       [:a#download-as-edn-v2.hidden]
       [:a#download-as-json-v2.hidden]
-      [:a#download-as-json-debug.hidden]
+      [:a#download-as-transit-debug.hidden]
       [:a#download-as-sqlite-db.hidden]
       [:a#download-as-roam-json.hidden]
       [:a#download-as-html.hidden]

src/main/frontend/components/export.cljs

@@ -94,9 +94,9 @@
             (t :export-zip)]])
         (when db-based?
           [:div
-           [:a.font-medium {:on-click #(export/export-repo-as-debug-json! current-repo)}
-            "Export debug JSON"]
-           [:p.text-sm.opacity-70 "Any sensitive data will be removed in the exported json file, you can send it to us for debugging."]])
+           [:a.font-medium {:on-click #(export/export-repo-as-debug-transit! current-repo)}
+            "Export debug transit file"]
+           [:p.text-sm.opacity-70 "Any sensitive data will be removed in the exported transit file, you can send it to us for debugging."]])
 
         (when (util/electron?)
           [:div

src/main/frontend/components/imports.cljs

@@ -69,7 +69,7 @@
                           :error))))
 
 (defn- lsq-import-handler
-  [e & {:keys [sqlite? graph-name]}]
+  [e & {:keys [sqlite? debug-transit? graph-name]}]
   (let [file      (first (array-seq (.-files (.-target e))))
         file-name (some-> (gobj/get file "name")
                           (string/lower-case))
@@ -98,6 +98,31 @@
                                        (js/console.error e)))
             (.readAsArrayBuffer reader file))))
 
+      debug-transit?
+      (let [graph-name (string/trim graph-name)]
+        (cond
+          (string/blank? graph-name)
+          (notification/show! "Empty graph name." :error)
+
+          (repo-handler/graph-already-exists? graph-name)
+          (notification/show! "Please specify another name as another graph with this name already exists!" :error)
+
+          :else
+          (do
+            (state/set-state! :graph/importing :logseq)
+            (let [reader (js/FileReader.)
+                  import-f import-handler/import-from-debug-transit!]
+              (set! (.-onload reader)
+                    (fn [e]
+                      (let [text (.. e -target -result)]
+                        (import-f
+                         graph-name
+                         text
+                         #(do
+                            (state/set-state! :graph/importing nil)
+                            (finished-cb))))))
+              (.readAsText reader file)))))
+
       (or edn? json?)
       (do
         (state/set-state! :graph/importing :logseq)
@@ -142,11 +167,11 @@
 (rum/defcs set-graph-name-dialog
   < rum/reactive
   (rum/local "" ::input)
-  [state sqlite-input-e opts]
+  [state input-e opts]
   (let [*input (::input state)
         on-submit #(if (repo/invalid-graph-name? @*input)
                      (repo/invalid-graph-name-warning)
-                     (lsq-import-handler sqlite-input-e (assoc opts :graph-name @*input)))]
+                     (lsq-import-handler input-e (assoc opts :graph-name @*input)))]
     [:div.container
      [:div.sm:flex.sm:items-start
       [:div.mt-3.text-center.sm:mt-0.sm:text-left
@@ -184,74 +209,74 @@
                             ;; (js/console.log "[form] submit: " e (js->clj e))
                             (on-submit-fn (js->clj e :keywordize-keys true))
                             (shui/dialog-close!)))
-        [convert-all-tags-input set-convert-all-tags-input!] (rum/use-state true)]
+         [convert-all-tags-input set-convert-all-tags-input!] (rum/use-state true)]
 
      (shui/form-provider form-ctx
-       [:form
-        {:on-submit on-submit-valid}
-
-        (shui/form-field {:name "graph-name"}
-                         (fn [field error]
-                           (shui/form-item
-                            (shui/form-label "New graph name")
-                            (shui/form-control
-                             (shui/input (merge {:placeholder "Graph name"} field)))
-                            (when error
-                              (shui/form-description
-                               [:b.text-red-800 (:message error)])))))
-
-        (shui/form-field {:name "convert-all-tags?"}
-                         (fn [field]
-                           (shui/form-item
-                            {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
-                            (shui/form-label "Import all tags")
-                            (shui/form-control
-                             (shui/checkbox {:checked (:value field)
-                                             :on-checked-change (fn [e]
-                                                                  ((:onChange field) e)
-                                                                  (set-convert-all-tags-input! (not convert-all-tags-input)))})))))
-
-        (shui/form-field {:name "tag-classes"}
-                         (fn [field _error]
-                           (shui/form-item
-                            {:class "pt-3"}
-                            (shui/form-label "Import specific tags")
-                            (shui/form-control
-                             (shui/input (merge field
-                                                {:placeholder "tag 1, tag 2" :disabled convert-all-tags-input})))
-                            (shui/form-description "Tags are case insensitive"))))
-
-        (shui/form-field {:name "remove-inline-tags?"}
-                         (fn [field]
-                           (shui/form-item
-                            {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
-                            (shui/form-label "Remove inline tags")
-                            (shui/form-description "Default behavior for DB graphs")
-                            (shui/form-control
-                             (shui/checkbox {:checked (:value field)
-                                             :on-checked-change (:onChange field)})))))
-
-        (shui/form-field {:name "property-classes"}
-                         (fn [field _error]
-                           (shui/form-item
-                            {:class "pt-3"}
-                            (shui/form-label "Import additional tags from property values")
-                            (shui/form-control
-                             (shui/input (merge {:placeholder "e.g. type"} field)))
-                            (shui/form-description
-                             "Properties are case insensitive and separated by commas"))))
-
-        (shui/form-field {:name "property-parent-classes"}
-                         (fn [field _error]
-                           (shui/form-item
-                            {:class "pt-3"}
-                            (shui/form-label "Import tag parents from property values")
-                            (shui/form-control
-                             (shui/input (merge {:placeholder "e.g. parent"} field)))
-                            (shui/form-description
-                             "Properties are case insensitive and separated by commas"))))
-
-        (shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))])
+                         [:form
+                          {:on-submit on-submit-valid}
+
+                          (shui/form-field {:name "graph-name"}
+                                           (fn [field error]
+                                             (shui/form-item
+                                              (shui/form-label "New graph name")
+                                              (shui/form-control
+                                               (shui/input (merge {:placeholder "Graph name"} field)))
+                                              (when error
+                                                (shui/form-description
+                                                 [:b.text-red-800 (:message error)])))))
+
+                          (shui/form-field {:name "convert-all-tags?"}
+                                           (fn [field]
+                                             (shui/form-item
+                                              {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
+                                              (shui/form-label "Import all tags")
+                                              (shui/form-control
+                                               (shui/checkbox {:checked (:value field)
+                                                               :on-checked-change (fn [e]
+                                                                                    ((:onChange field) e)
+                                                                                    (set-convert-all-tags-input! (not convert-all-tags-input)))})))))
+
+                          (shui/form-field {:name "tag-classes"}
+                                           (fn [field _error]
+                                             (shui/form-item
+                                              {:class "pt-3"}
+                                              (shui/form-label "Import specific tags")
+                                              (shui/form-control
+                                               (shui/input (merge field
+                                                                  {:placeholder "tag 1, tag 2" :disabled convert-all-tags-input})))
+                                              (shui/form-description "Tags are case insensitive"))))
+
+                          (shui/form-field {:name "remove-inline-tags?"}
+                                           (fn [field]
+                                             (shui/form-item
+                                              {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
+                                              (shui/form-label "Remove inline tags")
+                                              (shui/form-description "Default behavior for DB graphs")
+                                              (shui/form-control
+                                               (shui/checkbox {:checked (:value field)
+                                                               :on-checked-change (:onChange field)})))))
+
+                          (shui/form-field {:name "property-classes"}
+                                           (fn [field _error]
+                                             (shui/form-item
+                                              {:class "pt-3"}
+                                              (shui/form-label "Import additional tags from property values")
+                                              (shui/form-control
+                                               (shui/input (merge {:placeholder "e.g. type"} field)))
+                                              (shui/form-description
+                                               "Properties are case insensitive and separated by commas"))))
+
+                          (shui/form-field {:name "property-parent-classes"}
+                                           (fn [field _error]
+                                             (shui/form-item
+                                              {:class "pt-3"}
+                                              (shui/form-label "Import tag parents from property values")
+                                              (shui/form-control
+                                               (shui/input (merge {:placeholder "e.g. parent"} field)))
+                                              (shui/form-description
+                                               "Properties are case insensitive and separated by commas"))))
+
+                          (shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))])
 
 (defn- counts-from-entities
   [entities]
@@ -416,14 +441,14 @@
 (rum/defc import-indicator
   [importing?]
   (rum/use-effect!
-    (fn []
-      (when (and importing? (not (shui-dialog/get-modal :import-indicator)))
-        (shui/dialog-open! indicator-progress
-          {:id :import-indicator
-           :content-props
-           {:onPointerDownOutside #(.preventDefault %)
-            :onOpenAutoFocus #(.preventDefault %)}})))
-    [importing?])
+   (fn []
+     (when (and importing? (not (shui-dialog/get-modal :import-indicator)))
+       (shui/dialog-open! indicator-progress
+                          {:id :import-indicator
+                           :content-props
+                           {:onPointerDownOutside #(.preventDefault %)
+                            :onOpenAutoFocus #(.preventDefault %)}})))
+   [importing?])
   [:<>])
 
 (rum/defc importer < rum/reactive
@@ -468,6 +493,21 @@
                                       (import-file-to-db-handler e {}))
                                     1000)}]])
 
+          (when (or (util/electron?) util/web-platform?)
+            [:label.action-input.flex.items-center.mx-2.my-2
+             [:span.as-flex-center [:i (svg/logo 28)]]
+             [:span.flex.flex-col
+              [[:strong "Debug Transit"]
+               [:small "Import debug transit file into a new DB graph"]]]
+             ;; Test form style changes
+             #_[:a.button {:on-click #(import-file-to-db-handler nil {:import-graph-fn js/alert})} "Open"]
+             [:input.absolute.hidden
+              {:id "import-debug-transit"
+               :type "file"
+               :on-change (fn [e]
+                            (shui/dialog-open!
+                             #(set-graph-name-dialog e {:debug-transit? true})))}]])
+
           (when (and (util/electron?) support-file-based?)
             [:label.action-input.flex.items-center.mx-2.my-2
              [:span.as-flex-center [:i (svg/logo 28)]]

src/main/frontend/db/restore.cljs

@@ -10,10 +10,10 @@
 
 (defn restore-graph!
   "Restore db from SQLite"
-  [repo]
+  [repo & {:as opts}]
   (state/set-state! :graph/loading? true)
   (p/let [start-time (t/now)
-          data (persist-db/<fetch-init-data repo)
+          data (persist-db/<fetch-init-data repo opts)
           _ (assert (some? data) "No data found when reloading db")
           {:keys [schema initial-data]} (dt/read-transit-str data)
           conn (try

src/main/frontend/handler/export.cljs

@@ -190,16 +190,13 @@
         (.setAttribute anchor "download" filename)
         (.click anchor)))))
 
-(defn export-repo-as-debug-json!
+(defn export-repo-as-debug-transit!
   [repo]
   (p/let [result (export-common-handler/<get-debug-datoms repo)
-          json-str (-> result
-                       bean/->js
-                       js/JSON.stringify)
-          filename (file-name (str repo "-debug-datoms") :json)
-          data-str (str "data:text/json;charset=utf-8,"
-                        (js/encodeURIComponent json-str))]
-    (when-let [anchor (gdom/getElement "download-as-json-debug")]
+          filename (file-name (str repo "-debug-datoms") :transit)
+          data-str (str "data:text/transit;charset=utf-8,"
+                        (js/encodeURIComponent result))]
+    (when-let [anchor (gdom/getElement "download-as-transit-debug")]
       (.setAttribute anchor "href" data-str)
       (.setAttribute anchor "download" filename)
       (.click anchor))))

src/main/frontend/handler/export/common.cljs

@@ -191,8 +191,7 @@
 (defn <get-debug-datoms
   [repo]
   (when-let [^object worker @db-browser/*worker]
-    (p/let [result (.get-debug-datoms worker repo)]
-      (ldb/read-transit-str result))))
+    (.get-debug-datoms worker repo)))
 
 (defn <get-all-page->content
   [repo]

src/main/frontend/handler/import.cljs

@@ -27,7 +27,8 @@
             [frontend.persist-db :as persist-db]
             [promesa.core :as p]
             [frontend.db.async :as db-async]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.db :as ldb]))
 
 (defn index-files!
   "Create file structure, then parse into DB (client only)"
@@ -233,7 +234,7 @@
      (p/do!
       (persist-db/<import-db graph buffer)
       (state/add-repo! {:url graph})
-      (repo-handler/restore-and-setup-repo! graph)
+      (repo-handler/restore-and-setup-repo! graph {:import-type "sqlite"})
       (state/set-current-repo! graph)
       (persist-db/<export-db graph {})
       (db/transact! graph (sqlite-util/import-tx :sqlite-db))
@@ -291,3 +292,15 @@
     (async/go
       (async/<! (import-from-tree! clj-data tree-vec-translate-json))
       (finished-ok-handler nil)))) ;; it was designed to accept a list of imported page names but now deprecated
+
+(defn import-from-debug-transit!
+  [bare-graph-name raw finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)
+        datoms (ldb/read-transit-str raw)]
+    (p/do!
+     (persist-db/<new graph {:import-type "debug-transit"
+                             :datoms datoms})
+     (state/add-repo! {:url graph})
+     (repo-handler/restore-and-setup-repo! graph {:import-type "debug-transit"})
+     (state/set-current-repo! graph)
+     (finished-ok-handler nil))))

src/main/frontend/handler/repo.cljs

@@ -66,10 +66,10 @@
 (defn restore-and-setup-repo!
   "Restore the db of a graph from the persisted data, and setup. Create a new
   conn, or replace the conn in state with a new one."
-  [repo]
+  [repo & {:as opts}]
   (p/do!
    (state/set-db-restoring! true)
-   (db-restore/restore-graph! repo)
+   (db-restore/restore-graph! repo opts)
    (repo-config-handler/restore-repo-config! repo)
    (when (config/global-config-enabled?)
      (global-config-handler/restore-global-config!))

src/main/frontend/handler/worker.cljs

@@ -42,6 +42,9 @@
 (defmethod handle :export-current-db [_]
   (state/pub-event! [:db/export-sqlite]))
 
+(defmethod handle :capture-error [_ _worker data]
+  (state/pub-event! [:capture-error data]))
+
 (defmethod handle :default [_ _worker data]
   (prn :debug "Worker data not handled: " data))
 

src/main/frontend/persist_db/browser.cljs

@@ -190,13 +190,13 @@
     (when-let [^js sqlite @*worker]
       (.releaseAccessHandles sqlite repo)))
 
-  (<fetch-initial-data [_this repo _opts]
+  (<fetch-initial-data [_this repo opts]
     (when-let [^js sqlite @*worker]
       (-> (p/let [db-exists? (.dbExists sqlite repo)
                   disk-db-data (when-not db-exists? (ipc/ipc :db-get repo))
                   _ (when disk-db-data
                       (.importDb sqlite repo disk-db-data))
-                  _ (.createOrOpenDB sqlite repo (ldb/write-transit-str {}))]
+                  _ (.createOrOpenDB sqlite repo (ldb/write-transit-str opts))]
             (.getInitialData sqlite repo))
           (p/catch sqlite-error-handler))))
 

src/main/frontend/worker/db/migrate.cljs

@@ -16,7 +16,8 @@
             [logseq.common.uuid :as common-uuid]
             [clojure.string :as string]
             [logseq.db.frontend.content :as db-content]
-            [logseq.common.util.page-ref :as page-ref]))
+            [logseq.common.util.page-ref :as page-ref]
+            [datascript.impl.entity :as de]))
 
 ;; TODO: fixes/rollback
 ;; Frontend migrations
@@ -553,6 +554,23 @@
   (when (< db-schema/version max-schema-version)
     (js/console.warn (str "Current db schema-version is " db-schema/version ", max available schema-version is " max-schema-version))))
 
+(defn- ensure-built-in-class-exists!
+  [conn]
+  (let [classes' [:logseq.class/Property :logseq.class/Tag :logseq.class/Page :logseq.class/Journal :logseq.class/Whiteboard]
+        new-classes (->> (select-keys db-class/built-in-classes classes')
+                         ;; class already exists, this should never happen
+                         (remove (fn [[k _]] (d/entity @conn k)))
+                         (into {})
+                         (#(sqlite-create-graph/build-initial-classes* % {}))
+                         (map (fn [b] (assoc b :logseq.property/built-in? true))))
+        new-class-idents (keep (fn [class]
+                                 (when-let [db-ident (:db/ident class)]
+                                   {:db/ident db-ident})) new-classes)
+        tx-data (concat new-class-idents new-classes)]
+    (when (seq tx-data)
+      (d/transact! conn tx-data {:fix-db? true
+                                 :db-migrate? true}))))
+
 (defn- upgrade-version!
   [conn search-db db-based? version {:keys [properties classes fix]}]
   (let [db @conn
@@ -584,6 +602,73 @@
     (ldb/transact! conn tx-data' {:db-migrate? true})
     (println "DB schema migrated to" version)))
 
+(defn fix-path-refs!
+  [conn]
+  (let [data (keep
+              (fn [d]
+                (when (not (de/entity? (d/entity @conn (:v d))))
+                  [:db/retract (:e d) (:a d) (:v d)]))
+              (d/datoms @conn :avet :block/path-refs))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
+(defn fix-missing-title!
+  [conn]
+  (let [data (->>
+              (mapcat
+               (fn [d]
+                 (let [entity (d/entity @conn (:e d))]
+                   [(when-not (:block/title entity)
+                      [:db/add (:e d) :block/title (:v d)])
+                    (when-not (:block/uuid entity)
+                      [:db/add (:e d) :block/uuid (d/squuid)])
+                    (when-not (:block/format entity)
+                      [:db/add (:e d) :block/format :markdown])]))
+               (d/datoms @conn :avet :block/name))
+              (remove nil?))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
+(defn fix-block-timestamps!
+  [conn]
+  (let [data (map
+              (fn [d]
+                (let [entity (d/entity @conn (:e d))]
+                  (when (or (nil? (:block/created-at entity))
+                            (nil? (:block/updated-at entity)))
+                    (-> (select-keys entity [:db/id :block/created-at :block/updated-at])
+                        sqlite-util/block-with-timestamps))))
+              (d/datoms @conn :avet :block/uuid))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
+(defn fix-properties!
+  [conn]
+  (let [schema (:schema @conn)
+        wrong-properties (filter (fn [[k v]]
+                                   (and (int? k) (not (qualified-ident? v)))) schema)
+        data (map (fn [[k _v]]
+                    [:db/retract k :db/valueType]) wrong-properties)]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true})
+      (d/reset-schema! conn (apply dissoc schema (keys wrong-properties))))))
+
+(defn fix-missing-page-tag!
+  [conn]
+  (let [data (keep
+              (fn [d]
+                (let [entity (d/entity @conn (:e d))]
+                  (when-not (:block/tags entity)
+                    [:db/add (:e d) :block/tags :logseq.class/Page])))
+              (d/datoms @conn :avet :block/name))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
 (defn migrate
   "Migrate 'frontend' datascript schema and data. To add a new migration,
   add an entry to schema-version->updates and bump db-schema/version"
@@ -606,14 +691,28 @@
                                 (when (and (< version-in-db v) (<= v db-schema/version))
                                   [v updates]))
                               schema-version->updates)]
+            (fix-path-refs! conn)
+            (fix-missing-title! conn)
+            (fix-properties! conn)
+            (fix-block-timestamps! conn)
             (println "DB schema migrated from" version-in-db)
             (doseq [[v m] updates]
-              (upgrade-version! conn search-db db-based? v m)))
+              (upgrade-version! conn search-db db-based? v m))
+            (fix-missing-page-tag! conn))
           (catch :default e
             (prn :error (str "DB migration failed to migrate to " db-schema/version " from " version-in-db ":"))
             (js/console.error e)
             (throw e)))))))
 
+(defn fix-db!
+  [conn]
+  (ensure-built-in-class-exists! conn)
+  (fix-path-refs! conn)
+  (fix-missing-title! conn)
+  (fix-properties! conn)
+  (fix-block-timestamps! conn)
+  (fix-missing-page-tag! conn))
+
 ;; Backend migrations
 ;; ==================
 

src/main/frontend/worker/db_worker.cljs

@@ -37,7 +37,8 @@
             [logseq.outliner.op :as outliner-op]
             [me.tonsky.persistent-sorted-set :as set :refer [BTSet]]
             [promesa.core :as p]
-            [shadow.cljs.modern :refer [defclass]]))
+            [shadow.cljs.modern :refer [defclass]]
+            [clojure.set]))
 
 (defonce *sqlite worker-state/*sqlite)
 (defonce *sqlite-conns worker-state/*sqlite-conns)
@@ -108,7 +109,7 @@
 
 (defn- rebuild-db-from-datoms!
   "Persistent-sorted-set has been broken, used addresses can't be found"
-  [datascript-conn sqlite-db]
+  [datascript-conn sqlite-db import-type]
   (let [datoms (get-all-datoms-from-sqlite-db sqlite-db)
         db (d/init-db [] db-schema/schema-for-db-based-graph
                       {:storage (storage/storage @datascript-conn)})
@@ -117,10 +118,12 @@
                              [:db/add (:e d) (:a d) (:v d) (:t d)]) datoms))]
     (prn :debug :rebuild-db-from-datoms :datoms-count (count datoms))
     ;; export db first
-    (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false])
-    (worker-util/post-message :export-current-db [])
+    (when-not import-type
+      (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false])
+      (worker-util/post-message :export-current-db []))
     (.exec sqlite-db #js {:sql "delete from kvs"})
-    (d/reset-conn! datascript-conn db)))
+    (d/reset-conn! datascript-conn db)
+    (db-migrate/fix-db! datascript-conn)))
 
 (comment
   (defn- gc-kvs-table!
@@ -137,7 +140,7 @@
                              [addr (bean/->clj (js/JSON.parse addresses))])))
           used-addresses (set (concat (mapcat second result)
                                       [0 1 (:eavt schema) (:avet schema) (:aevt schema)]))
-          unused-addresses (set/difference (set (map first result)) used-addresses)]
+          unused-addresses (clojure.set/difference (set (map first result)) used-addresses)]
       (when unused-addresses
         (prn :debug :db-gc :unused-addresses unused-addresses)
         (.transaction db (fn [tx]
@@ -145,6 +148,27 @@
                              (.exec tx #js {:sql "Delete from kvs where addr = ?"
                                             :bind #js [addr]}))))))))
 
+(defn- find-missing-addresses
+  [^Object db]
+  (let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0"
+                                       :rowMode "array"})
+                        bean/->clj
+                        ffirst
+                        sqlite-util/transit-read)
+        result (->> (.exec db #js {:sql "select addr, addresses from kvs"
+                                   :rowMode "array"})
+                    bean/->clj
+                    (map (fn [[addr addresses]]
+                           [addr (bean/->clj (js/JSON.parse addresses))])))
+        used-addresses (set (concat (mapcat second result)
+                                    [0 1 (:eavt schema) (:avet schema) (:aevt schema)]))
+        missing-addresses (clojure.set/difference used-addresses (set (map first result)))]
+    (when (seq missing-addresses)
+      (worker-util/post-message :capture-error
+                                {:error "db-missing-addresses"
+                                 :payload {:missing-addresses missing-addresses}})
+      (prn :error :missing-addresses missing-addresses))))
+
 (defn upsert-addr-content!
   "Upsert addr+data-seq. Update sqlite-cli/upsert-addr-content! when making changes"
   [repo data delete-addrs & {:keys [client-ops-db?] :or {client-ops-db? false}}]
@@ -287,7 +311,7 @@
   (.exec db "PRAGMA journal_mode=WAL"))
 
 (defn- create-or-open-db!
-  [repo {:keys [config import-type]}]
+  [repo {:keys [config import-type datoms]}]
   (when-not (worker-state/get-sqlite-conn repo)
     (p/let [[db search-db client-ops-db :as dbs] (get-dbs repo)
             storage (new-sqlite-storage repo {})
@@ -305,12 +329,19 @@
       (search/create-tables-and-triggers! search-db)
       (let [schema (sqlite-util/get-schema repo)
             conn (sqlite-common-db/get-storage-conn storage schema)
-            client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn client-ops-storage client-op/schema-in-db))
-            initial-data-exists? (and (d/entity @conn :logseq.class/Root)
-                                      (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type))))]
+            _ (when datoms
+                (let [data (map (fn [datom]
+                                  [:db/add (:e datom) (:a datom) (:v datom)]) datoms)]
+                  (d/transact! conn data {:initial-db? true})))
+            client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn
+                                                     client-ops-storage
+                                                     client-op/schema-in-db))
+            initial-data-exists? (when (nil? datoms)
+                                   (and (d/entity @conn :logseq.class/Root)
+                                        (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type)))))]
         (swap! *datascript-conns assoc repo conn)
         (swap! *client-ops-conns assoc repo client-ops-conn)
-        (when (and db-based? (not initial-data-exists?))
+        (when (and db-based? (not initial-data-exists?) (not datoms))
           (let [config (or config "")
                 initial-data (sqlite-create-graph/build-db-initial-data config
                                                                         (when import-type {:import-type import-type}))]
@@ -322,12 +353,14 @@
               (ldb/transact! conn (sqlite-create-graph/build-initial-views)))
             (catch :default _e)))
 
+        (find-missing-addresses db)
         ;; (gc-kvs-table! db)
+
         (try
           (db-migrate/migrate conn search-db)
           (catch :default _e
             (when db-based?
-              (rebuild-db-from-datoms! conn db))))
+              (rebuild-db-from-datoms! conn db import-type))))
 
         (db-listener/listen-db-changes! repo (get @*datascript-conns repo))))))
 
@@ -729,7 +762,8 @@
   (get-debug-datoms
    [this repo]
    (when-let [db (worker-state/get-sqlite-conn repo)]
-     (ldb/write-transit-str (worker-export/get-debug-datoms db))))
+     (let [conn (worker-state/get-datascript-conn repo)]
+       (ldb/write-transit-str (worker-export/get-debug-datoms conn db)))))
 
   (get-all-pages
    [this repo]

src/main/frontend/worker/export.cljs

@@ -53,7 +53,7 @@
                  (common-file/block->content repo db (:block/uuid e) {} {})])))))
 
 (defn get-debug-datoms
-  [^Object db]
+  [conn ^Object db]
   (some->> (.exec db #js {:sql "select content from kvs"
                           :rowMode "array"})
            bean/->clj
@@ -61,26 +61,10 @@
                      (let [result (sqlite-util/transit-read (first result))]
                        (when (map? result)
                          (:keys result)))))
-           (group-by first)
-           (mapcat (fn [[_id col]]
-                     (let [ident (some (fn [[_e a v _t]]
-                                         (when (= a :db/ident)
-                                           v)) col)
-                           journal (some (fn [[_e a v _t]]
-                                           (when (= a :block/journal-day)
-                                             v)) col)]
-                       (map
-                        (fn [[e a v t]]
-                          (cond
-                            (and (contains? #{:block/title :block/name} a)
-                                 (not (or ident journal)))
-                            [e a (str "debug " e) t]
-
-                            (= a :block/uuid)
-                            [e a (str v) t]
-
-                            :else
-                            [e a v t]))
-                        col))))
-           (distinct)
-           (sort-by first)))
+           (map (fn [[e a v t]]
+                  (if (and (contains? #{:block/title :block/name} a)
+                           (let [entity (d/entity @conn e)]
+                             (and (not (:db/ident entity))
+                                  (not (ldb/journal? entity)))))
+                    (d/datom e a (str "debug " e) t)
+                    (d/datom e a v t))))))