dev: e2e testing workflow of running multiple times

.github/workflows/e2e.yml

@@ -0,0 +1,111 @@
+name: E2E Test Repeat
+
+# Running E2E test multiple times to confirm test stability.
+# E2E test could be randomly failed due to the batch update mechanism of React.
+# Robust E2E test could help improving dev experience.
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - 'e2e-tests/**'
+  pull_request:
+    branches: [master]
+    paths:
+      - 'e2e-tests/**'
+
+env:
+  CLOJURE_VERSION: '1.10.1.727'
+  # setup-java@v2 dropped support for legacy Java version syntax.
+  # This is the same as 1.8.
+  JAVA_VERSION: '8'
+  # This is the latest node version we can run.
+  NODE_VERSION: '16'
+  BABASHKA_VERSION: '0.8.1'
+
+jobs:
+
+  e2e-test-repeat:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        repeat: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] # Test 10 times for E2E robustness
+
+    steps:
+      - name: Repeat message
+        run: echo running E2E test with repeat id ${{ matrix.repeat }}
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+          cache-dependency-path: |
+            yarn.lock
+            static/yarn.lock
+
+      - name: Set up Java
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: ${{ env.JAVA_VERSION }}
+
+      - name: Set up Clojure
+        uses: DeLaGuardo/setup-clojure@master
+        with:
+          cli: ${{ env.CLOJURE_VERSION }}
+
+      - name: Clojure cache
+        uses: actions/cache@v2
+        id: clojure-deps
+        with:
+          path: |
+            ~/.m2/repository
+            ~/.gitlibs
+          key: ${{ runner.os }}-clojure-deps-${{ hashFiles('deps.edn') }}
+          restore-keys: ${{ runner.os }}-clojure-deps-
+
+      - name: Fetch Clojure deps
+        if: steps.clojure-deps.outputs.cache-hit != 'true'
+        run: clojure -A:cljs -P
+
+      - name: Shadow-cljs cache
+        uses: actions/cache@v2
+        with:
+          path: .shadow-cljs
+          # ensure update cache every time
+          key: ${{ runner.os }}-shadow-cljs-${{ github.sha }}
+          # will match most recent upload
+          restore-keys: |
+            ${{ runner.os }}-shadow-cljs-
+
+      - name: Fetch yarn deps
+        run: yarn install
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: true
+
+      # NOTE: require the app to be build in debug mode(compile instead of build).
+      - name: Prepare E2E test build
+        run: |
+          yarn gulp:build && clojure -M:cljs compile app publishing electron
+          (cd static && yarn install && yarn rebuild:better-sqlite3)
+
+      # Exits with 0 if yarn.lock is up to date or 1 if we forgot to update it
+      - name: Ensure static yarn.lock is up to date
+        run: git diff --exit-code static/yarn.lock
+
+      - name: Run Playwright test
+        run: xvfb-run -- yarn e2e-test
+        env:
+          CI: true
+          DEBUG: "pw:api"
+
+      # - name: Save test artifacts
+      #   if: ${{ failure() }}
+      #   uses: actions/upload-artifact@v2
+      #   with:
+      #     name: e2e-test-report
+      #     path: artifacts.zip

e2e-tests/page-search.spec.ts

@@ -65,7 +65,6 @@ async function alias_test(page: Page, page_name: string, search_kws: string[]) {
   await page.keyboard.press(hotkeyOpenLink)
 
   await lastBlock(page)
-  await page.waitForTimeout(500)
 
   // build target Page with alias
   // the target page will contains the content in
@@ -76,8 +75,9 @@ async function alias_test(page: Page, page_name: string, search_kws: string[]) {
   await page.press('textarea >> nth=0', 'Enter') // Enter for finishing selection
   await page.press('textarea >> nth=0', 'Enter') // double Enter for exit property editing
   await page.press('textarea >> nth=0', 'Enter') // double Enter for exit property editing
-  await page.waitForTimeout(500)
+  await lastBlock(page)
   await page.type('textarea >> nth=0', alias_test_content_1)
+  await lastBlock(page)
   await page.keyboard.press(hotkeyBack)
 
   await page.waitForTimeout(100) // await navigation
@@ -111,6 +111,7 @@ async function alias_test(page: Page, page_name: string, search_kws: string[]) {
   await page.keyboard.press(hotkeyBack)
 
   // clicking opening test
+  await newBlock(page)
   await page.waitForSelector('.page-blocks-inner .ls-block .page-ref >> nth=-1')
   await page.click('.page-blocks-inner .ls-block .page-ref >> nth=-1')
   await lastBlock(page)

e2e-tests/utils.ts

@@ -117,7 +117,7 @@ export async function newInnerBlock(page: Page): Promise<Locator> {
 
 export async function newBlock(page: Page): Promise<Locator> {
   let blockNumber = await page.locator('.page-blocks-inner .ls-block').count()
-  const prev = await lastBlock(page)
+  await lastBlock(page)
   await page.press('textarea >> nth=0', 'Enter')
   await page.waitForSelector(`.page-blocks-inner .ls-block >> nth=${blockNumber} >> textarea`, { state: 'visible' })
   return page.locator('textarea >> nth=0')