Merge branch 'master' into feat/db

.clj-kondo/config.edn

@@ -15,6 +15,8 @@
   :aliased-namespace-symbol {:level :warning}
   ;; Disable until it doesn't trigger false positives on rum/defcontext
   :earmuffed-var-not-dynamic {:level :off}
+  ;; Disable until we decide to use conj! as recommended in docs
+  :unused-value {:level :off}
   :unresolved-symbol {:exclude [goog.DEBUG
                                 goog.string.unescapeEntities
                                 ;; TODO:lint: Fix when fixing all type hints
@@ -39,6 +41,7 @@
              electron.utils utils
              "/electron/utils" js-utils
              frontend.commands commands
+             frontend.components.block.macros block-macros
              frontend.components.query query
              frontend.components.query.result query-result
              frontend.config config

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -7,6 +7,10 @@ body:
         Thank you very much for opening a bug report with Logseq.
 
         If you have a feature idea or need help, please go to [our Forum](https://discuss.logseq.com/) or [our Discord](https://discord.com/invite/KpN4eHY).
+
+        Please make sure to provide a descriptive, deterministic, and reproducible report. It saves time for both the developers and users who are looking for solutions. Providing as much information as possible, including screenshots and logs, is highly appreciated. This will help us to better understand the issue and respond more effectively.
+
+        Please DO NOT use this template to ask questions. There are other appropriate channels to ask questions. This template is strictly for reporting bugs.
   - type: checkboxes
     id: confirm-search
     attributes:

.github/workflows/build-desktop-release.yml

@@ -122,7 +122,7 @@ jobs:
         run: |
           sed -i 's/defonce version ".*"/defonce version "${{ steps.ref.outputs.version }}"/g' src/main/frontend/version.cljs
 
-      - name: Set Build Environment Variables (only when workflow_dispath)
+      - name: Set Build Environment Variables (only when workflow_dispatch)
         if: ${{ github.event_name == 'workflow_dispatch' }}
         # if scheduled, use default settings
         run: |
@@ -204,8 +204,19 @@ jobs:
         env:
           PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: true
 
+      - name: Install Fluxbox
+        run: sudo apt-get update && sudo apt-get install -y fluxbox
+
+      # Emulate a virtual framebuffer on machines with no display hardware
+      - name: Run XVFB
+        run: Xvfb :1 -screen 0 1024x768x24 >/dev/null 2>&1 &
+
+      # Start a lightweight window manager to simulate window actions (maximize,restore etc)
+      - name: Start Fluxbox
+        run:  DISPLAY=:1.0 fluxbox >/dev/null 2>&1 &
+
       - name: Run Playwright test
-        run: xvfb-run -- npx playwright test --reporter github --shard=${{ matrix.shard }}/3
+        run: DISPLAY=:1.0 npx playwright test --reporter github --shard=${{ matrix.shard }}/3
         env:
           LOGSEQ_CI: true
           DEBUG: "pw:api"

.github/workflows/build.yml

@@ -182,15 +182,26 @@ jobs:
       - name: Ensure static yarn.lock is up to date
         run: git diff --exit-code static/yarn.lock
 
+      - name: Install Fluxbox
+        run: sudo apt-get update && sudo apt-get install -y fluxbox
+
+      # Emulate a virtual framebuffer on machines with no display hardware
+      - name: Run XVFB
+        run: Xvfb :1 -screen 0 1024x768x24 >/dev/null 2>&1 &
+
+      # Start a lightweight window manager to simulate window actions (maximize,restore etc)
+      - name: Start Fluxbox
+        run:  DISPLAY=:1.0 fluxbox >/dev/null 2>&1 &
+
       - name: Run Playwright test - 1/2
-        run: xvfb-run -- npx playwright test --reporter github --shard=1/2
+        run: DISPLAY=:1.0 npx playwright test --reporter github --shard=1/2
         env:
           LOGSEQ_CI: true
           DEBUG: "pw:api"
           RELEASE: true # skip dev only test
 
       - name: Run Playwright test - 2/2
-        run: xvfb-run -- npx playwright test --reporter github --shard=2/2
+        run: DISPLAY=:1.0 npx playwright test --reporter github --shard=2/2
         env:
           LOGSEQ_CI: true
           DEBUG: "pw:api"

.github/workflows/e2e.yml

@@ -137,8 +137,19 @@ jobs:
       - name: Ensure static yarn.lock is up to date
         run: git diff --exit-code static/yarn.lock
 
+      - name: Install Fluxbox
+        run: sudo apt-get update && sudo apt-get install -y fluxbox
+
+      # Emulate a virtual framebuffer on machines with no display hardware
+      - name: Run XVFB
+        run: Xvfb :1 -screen 0 1024x768x24 >/dev/null 2>&1 &
+
+      # Start a lightweight window manager to simulate window actions (maximize,restore etc)
+      - name: Start Fluxbox
+        run:  DISPLAY=:1.0 fluxbox >/dev/null 2>&1 &
+
       - name: Run Playwright test
-        run: xvfb-run -- npx playwright test --reporter github --shard=${{ matrix.shard }}/3
+        run: DISPLAY=:1.0 npx playwright test --reporter github --shard=${{ matrix.shard }}/3
         env:
           LOGSEQ_CI: true
           DEBUG: "pw:api"

CONTRIBUTING.md

@@ -133,6 +133,7 @@ When submitting a Pull Request (PR) or expecting a subsequent review, please fol
    * Unrelated refactoring or heavy refactoring
    * Code or doc formatting changes including whitespace changes
    * Dependency updates e.g. in package.json
+   * Changes that contain multiple unverified resources. This is risky for our users and is a lot of work to verify. A change with one resource that can be verified is acceptable.
 
 ### PR Additional Links
 

android/app/build.gradle

@@ -6,8 +6,8 @@ android {
         applicationId "com.logseq.app"
         minSdkVersion rootProject.ext.minSdkVersion
         targetSdkVersion rootProject.ext.targetSdkVersion
-        versionCode 61
-        versionName "0.9.8"
+        versionCode 62
+        versionName "0.9.9"
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
         aaptOptions {
              // Files and dirs to omit from the packaged assets dir, modified to accommodate modern web apps.

bb.edn

@@ -5,13 +5,13 @@
   logseq/bb-tasks
   #_{:local/root "../bb-tasks"}
   {:git/url "https://github.com/logseq/bb-tasks"
-   :git/sha "4295d5df0458cc06a09c5d506510ee49b785407d"}
+   :git/sha "70d3edeb287f5cec7192e642549a401f7d6d4263"}
   logseq/graph-parser
   {:local/root "deps/graph-parser"}
   org.clj-commons/digest
   {:mvn/version "1.4.100"}}
  :pods
- {clj-kondo/clj-kondo {:version "2022.10.05"}
+ {clj-kondo/clj-kondo {:version "2023.05.26"}
   org.babashka/fswatcher {:version "0.0.3"}
   org.babashka/go-sqlite3 {:version "0.1.0"}}
  :tasks
@@ -39,8 +39,8 @@
   {:depends [dev:build-publishing]
    :doc "Build publishing spa app given graph and output dirs"
    :task (apply shell {:dir "scripts"}
-           "yarn -s nbb-logseq -cp src -m logseq.tasks.dev.publishing"
-           (into ["static"] *command-line-args*))}
+                "yarn -s nbb-logseq -cp src -m logseq.tasks.dev.publishing"
+                (into ["static"] *command-line-args*))}
 
   dev:npx-cap-run-ios
   logseq.tasks.dev.mobile/npx-cap-run-ios
@@ -113,9 +113,6 @@
   lang:missing
   logseq.tasks.lang/list-missing
 
-  lang:duplicates
-  logseq.tasks.lang/list-duplicates
-
   lang:validate-translations
   logseq.tasks.lang/validate-translations
 

deps.edn

@@ -1,4 +1,4 @@
-{:paths ["src/main" "src/electron" "templates" "src/resources"]
+{:paths ["src/main" "src/electron" "src/resources"]
  :deps
  {org.clojure/clojure                   {:mvn/version "1.11.1"}
   rum/rum                               {:mvn/version "0.12.9"}
@@ -55,5 +55,5 @@
                    :main-opts ["-m" "cljs-test-runner.main" "-d" "src/bench" "-n" "frontend.benchmark-test-runner"]}
 
            ;; Use :replace-deps for tools. See https://github.com/clj-kondo/clj-kondo/issues/1536#issuecomment-1013006889
-           :clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2022.12.08"}}
+           :clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.05.26"}}
                        :main-opts  ["-m" "clj-kondo.main"]}}}

deps/common/bb.edn

@@ -3,10 +3,10 @@
  {logseq/bb-tasks
   #_{:local/root "../../../bb-tasks"}
   {:git/url "https://github.com/logseq/bb-tasks"
-   :git/sha "0d49051909bfa0c6b414e86606d82b4ea54f382c"}}
+   :git/sha "70d3edeb287f5cec7192e642549a401f7d6d4263"}}
 
  :pods
- {clj-kondo/clj-kondo {:version "2023.03.17"}}
+ {clj-kondo/clj-kondo {:version "2023.05.26"}}
 
  :tasks
  {test:load-all-namespaces-with-nbb
@@ -23,4 +23,4 @@
 
  :tasks/config
  {:large-vars
-  {:max-lines-count 45}}}
+  {:max-lines-count 45}}}

deps/common/deps.edn

@@ -4,5 +4,5 @@
                        org.clojure/clojurescript {:mvn/version "1.11.54"}}
          :main-opts   ["-m" "cljs-test-runner.main"]}
   :clj-kondo
-  {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2022.12.08"}}
+  {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.05.26"}}
    :main-opts  ["-m" "clj-kondo.main"]}}}

deps/db/bb.edn

@@ -4,10 +4,10 @@
  {logseq/bb-tasks
   #_{:local/root "../../../bb-tasks"}
   {:git/url "https://github.com/logseq/bb-tasks"
-   :git/sha "1815db538241082a01e95601e23e4290dd64d0c0"}}
+   :git/sha "70d3edeb287f5cec7192e642549a401f7d6d4263"}}
 
  :pods
- {clj-kondo/clj-kondo {:version "2022.10.05"}}
+ {clj-kondo/clj-kondo {:version "2023.05.26"}}
 
  :tasks
  {test:load-all-namespaces-with-nbb

deps/db/deps.edn

@@ -3,5 +3,5 @@
  {datascript/datascript {:mvn/version "1.3.8"}}
  :aliases
  {:clj-kondo
-  {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2022.12.08"}}
+  {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.05.26"}}
    :main-opts  ["-m" "clj-kondo.main"]}}}

deps/graph-parser/bb.edn

@@ -3,10 +3,10 @@
  {logseq/bb-tasks
   #_{:local/root "../../../bb-tasks"}
   {:git/url "https://github.com/logseq/bb-tasks"
-   :git/sha "1815db538241082a01e95601e23e4290dd64d0c0"}}
+   :git/sha "70d3edeb287f5cec7192e642549a401f7d6d4263"}}
 
  :pods
- {clj-kondo/clj-kondo {:version "2022.10.05"}}
+ {clj-kondo/clj-kondo {:version "2023.05.26"}}
 
  :tasks
  {test:load-all-namespaces-with-nbb

deps/graph-parser/deps.edn

@@ -20,5 +20,5 @@
                        org.clojure/clojurescript {:mvn/version "1.11.54"}}
          :main-opts   ["-m" "cljs-test-runner.main"]}
 
-  :clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2022.12.08"}}
+  :clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.05.26"}}
               :main-opts    ["-m" "clj-kondo.main"]}}}

deps/graph-parser/src/logseq/graph_parser/mldoc.cljc

@@ -75,13 +75,26 @@
           js/JSON.stringify))))
 
 (defn remove-indentation-spaces
+  "Remove the indentation spaces from the content. Only for markdown.
+   level - ast level + 1 (2 for the first level, 3 for the second level, etc., as the non-first line of multi-line block has 2 more space
+           Ex.
+              - level 1 multiline block first line
+                level 1 multiline block second line
+              \t- level 2 multiline block first line
+              \t  level 2 multiline block second line
+   remove-first-line? - apply the indentation removal to the first line or not"
   [s level remove-first-line?]
   (let [lines (string/split-lines s)
         [f & r] lines
         body (map (fn [line]
+                    ;; Check if the indentation area only contains white spaces
+                    ;; Level = ast level + 1, 1-based indentation level
+                    ;; For markdown in Logseq, the indentation area for the non-first line of multi-line block is (ast level - 1) * "\t" + 2 * "(space)"
                     (if (string/blank? (gp-util/safe-subs line 0 level))
+                      ;; If valid, then remove the indentation area spaces. Keep the rest of the line (might contain leading spaces)
                       (gp-util/safe-subs line level)
-                      line))
+                      ;; Otherwise, trim these invalid spaces
+                      (string/triml line)))
                (if remove-first-line? lines r))
         content (if remove-first-line? body (cons f body))]
     (string/join "\n" content)))

deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs

@@ -119,6 +119,24 @@ body"
       (is ["@tag" "tag1" "tag2"] (sort (:filetags props)))
       (is ["@tag" "tag1" "tag2" "tag3"] (sort (:tags props))))))
 
+(deftest remove-indentation-spaces
+  (testing "Remove indentations for every line"
+    (is (=  "block 1.1\n  line 1\n    line 2\nline 3\nline 4"
+            (let [s "block 1.1
+    line 1
+      line 2
+ line 3
+line 4"]
+              (gp-mldoc/remove-indentation-spaces s 2 false)))) 
+    (is (=  "\t- block 1.1\n  line 1\n    line 2\nline 3\nline 4"
+            (let [s "\t- block 1.1
+\t    line 1
+\t      line 2
+\t line 3
+\tline 4"]
+              (gp-mldoc/remove-indentation-spaces s 3 false))))))
+
+
 (deftest ^:integration test->edn
   (let [graph-dir "test/docs-0.9.2"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.9.2")

deps/publishing/bb.edn

@@ -3,10 +3,10 @@
  {logseq/bb-tasks
   #_{:local/root "../../../bb-tasks"}
   {:git/url "https://github.com/logseq/bb-tasks"
-   :git/sha "0d49051909bfa0c6b414e86606d82b4ea54f382c"}}
+   :git/sha "70d3edeb287f5cec7192e642549a401f7d6d4263"}}
 
  :pods
- {clj-kondo/clj-kondo {:version "2023.03.17"}}
+ {clj-kondo/clj-kondo {:version "2023.05.26"}}
 
  :tasks
  {test:load-all-namespaces-with-nbb

deps/publishing/deps.edn

@@ -3,5 +3,5 @@
  {logseq/db {:local/root "../db"}}
 
  :aliases
- {:clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.03.17"}}
+ {:clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.05.26"}}
               :main-opts    ["-m" "clj-kondo.main"]}}}

deps/publishing/src/logseq/publishing/db.cljs

@@ -2,6 +2,8 @@
   "Provides db fns and associated util fns for publishing"
   (:require [datascript.core :as d]
             [logseq.db.schema :as db-schema]
+            [logseq.db.rules :as rules]
+            [clojure.set :as set]
             [clojure.string :as string]))
 
 (defn ^:api get-area-block-asset-url
@@ -92,6 +94,20 @@
      flatten
      distinct)))
 
+(defn- get-aliases-for-page-ids
+  [db page-ids]
+  (->> (d/q '[:find ?e
+              :in $ ?pages %
+              :where
+              [?page :block/name]
+              [(contains? ?pages ?page)]
+              (alias ?page ?e)]
+            db
+            (set page-ids)
+            (:alias rules/rules))
+       (map first)
+       set))
+
 (defn clean-export!
   "Prepares a database assuming all pages are public unless a page has a 'public:: false'"
   [db]
@@ -113,7 +129,8 @@
   "Prepares a database assuming all pages are private unless a page has a 'public:: true'"
   [db]
   (when-let [public-pages* (seq (get-public-pages db))]
-    (let [public-pages (set public-pages*)
+    (let [public-pages (set/union (set public-pages*)
+                                  (get-aliases-for-page-ids db public-pages*))
           exported-namespace? #(contains? #{"block" "recent"} %)
           filtered-db (d/filter db
                                 (fn [db datom]

deps/publishing/test/logseq/publishing/db_test.cljs

@@ -36,7 +36,7 @@
 (deftest filter-only-public-pages-and-blocks
   (let [conn (ldb/start-conn)
         _ (graph-parser/parse-file conn "page1.md" "- b11\n- b12\n- ![awesome.png](../assets/awesome_1648822509908_0.png)")
-        _ (graph-parser/parse-file conn "page2.md" "public:: true\n- b21\n- ![thumb-on-fire.PNG](../assets/thumb-on-fire_1648822523866_0.PNG)")
+        _ (graph-parser/parse-file conn "page2.md" "alias:: page2-alias\npublic:: true\n- b21\n- ![thumb-on-fire.PNG](../assets/thumb-on-fire_1648822523866_0.PNG)")
         _ (graph-parser/parse-file conn "page3.md" "public:: true\n- b31")
         [filtered-db assets] (publish-db/filter-only-public-pages-and-blocks @conn)
         exported-pages (->> (d/q '[:find (pull ?b [*])
@@ -56,6 +56,8 @@
         "Contains all pages that have been marked public")
     (is (not (contains? exported-pages "page1"))
         "Doesn't contain private page")
+    (is (seq (d/entity filtered-db [:block/name "page2-alias"]))
+          "Alias of public page is exported")
     (is (= #{"page2" "page3"} exported-block-pages)
         "Only exports blocks from public pages")
     (is (= ["thumb-on-fire_1648822523866_0.PNG"] assets)