Enable lerna caching for build scripts (#6767)

* Enable lerna caching for build scripts

* Fix script definitions

* Lint

* simplify cacheable tasks

* Update to lerna 7

* update config

* cleanup

* Stay on lerna 6 for now

* Revert "cleanup"

This reverts commit c061fc07812e7ae231ff8eeedff8ed75f89e4a35.

* lint

* Skip cache when building prod

* Cache labextension builds

* Update `targetDefaults`

* Update dev command

* Fix dependencies

* Add missing dependency

* fix dep order

* fix script deps

* Update scripts

* Update scripts dependencies

* Add back scripts

* lint

* cache more

* Mention task caching in contributing guide

.gitpod.yml

@@ -24,7 +24,7 @@ tasks:
       source /workspace/bin/activate-env.sh
       micromamba install -n base -y -c conda-forge python=3.11 nodejs=18
       source /workspace/bin/activate-env.sh
-      python -m pip install -e ".[dev,test]" && jlpm && jlpm run build && jlpm develop
+      python -m pip install -e ".[dev,test]" && jlpm run build && jlpm develop
       gp sync-done setup
     command: |
       gp sync-done setup

CONTRIBUTING.md

@@ -94,6 +94,17 @@ Running the end to end tests in headful mode will trigger something like the fol
 
 ![playwight-headed-demo](https://user-images.githubusercontent.com/591645/141274633-ca9f9c2f-eef6-430e-9228-a35827f8133d.gif)
 
+## Tasks caching
+
+The repository is configured to use the Lerna caching system (via `nx`) for some of the development scripts.
+
+This helps speed up rebuilds when running `jlpm run build` multiple times to avoid rebuilding packages that have not changed on disk.
+
+To learn more about Lerna caching:
+
+- https://lerna.js.org/docs/features/cache-tasks
+- https://nx.dev/core-features/cache-task-results
+
 ### Updating reference snapshots
 
 Often a PR might make changes to the user interface, which can cause the visual regression tests to fail.

app/package.json

@@ -169,7 +169,10 @@
     "@jupyterlab/tooltip-extension": "^4.0.3",
     "@jupyterlab/translation-extension": "^4.0.3",
     "@jupyterlab/ui-components-extension": "^4.0.3",
-    "@jupyterlab/vega5-extension": "^4.0.3"
+    "@jupyterlab/vega5-extension": "^4.0.3",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "yjs": "^13.5.40"
   },
   "devDependencies": {
     "@jupyterlab/builder": "^4.0.3",

lerna.json

@@ -1,5 +1,5 @@
 {
-  "npmClient": "jlpm",
+  "$schema": "node_modules/lerna/schemas/lerna-schema.json",
   "version": "independent",
   "useWorkspaces": true
 }

nx.json

@@ -0,0 +1,35 @@
+{
+  "$schema": "./node_modules/nx/schemas/nx-schema.json",
+  "tasksRunnerOptions": {
+    "default": {
+      "runner": "nx/tasks-runners/default",
+      "options": {
+        "cacheableOperations": [
+          "build",
+          "build:prod",
+          "build:lib",
+          "build:labextension:dev",
+          "build:labextension"
+        ]
+      }
+    }
+  },
+  "namedInputs": {
+    "default": ["{projectRoot}/**/*"]
+  },
+  "targetDefaults": {
+    "build:lib": {
+      "dependsOn": ["^build:lib"],
+      "inputs": ["default", "^default"],
+      "outputs": ["{projectRoot}/lib"]
+    },
+    "build": {
+      "dependsOn": ["^build"],
+      "inputs": ["default", "^default"]
+    },
+    "build:prod": {
+      "dependsOn": ["^build:prod"],
+      "inputs": ["default", "^default"]
+    }
+  }
+}

package.json

@@ -19,9 +19,11 @@
       "packages/*"
     ]
   },
+  "nx": {},
   "scripts": {
     "build": "lerna run build",
-    "build:prod": "lerna run build:prod",
+    "build:lib": "lerna run build:lib",
+    "build:prod": "lerna run build:prod --skip-nx-cache",
     "build:test": "lerna run build:test",
     "build:utils": "cd buildutils && npm run build",
     "clean": "lerna run clean",
@@ -50,10 +52,15 @@
     "eslint-plugin-jest": "^27.2.1",
     "eslint-plugin-prettier": "^4.2.1",
     "eslint-plugin-react": "^7.32.2",
-    "lerna": "^6.5.1",
+    "lerna": "^6.6.2",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.8.5",
     "rimraf": "^3.0.2",
     "typescript": "~5.0.2"
+  },
+  "resolutions": {
+    "@types/react": "^18.0.26",
+    "react": "^18.2.0",
+    "yjs": "^13.5.40"
   }
 }

packages/docmanager-extension/package.json

@@ -38,6 +38,7 @@
     "watch": "tsc -b --watch"
   },
   "dependencies": {
+    "@jupyter-notebook/application": "^7.0.0",
     "@jupyterlab/application": "^4.0.3",
     "@jupyterlab/coreutils": "^6.0.3",
     "@jupyterlab/docmanager": "^4.0.3",

packages/docmanager-extension/tsconfig.json

@@ -4,5 +4,10 @@
     "outDir": "lib",
     "rootDir": "src"
   },
-  "include": ["src/**/*"]
+  "include": ["src/**/*"],
+  "references": [
+    {
+      "path": "../application"
+    }
+  ]
 }

packages/lab-extension/package.json

@@ -30,11 +30,11 @@
     "style/index.js"
   ],
   "scripts": {
-    "build": "jlpm run build:lib && jlpm run build:labextension:dev",
+    "build": "jlpm run build:labextension:dev",
     "build:labextension": "jupyter labextension build .",
     "build:labextension:dev": "jupyter labextension build --development True .",
-    "build:lib": "tsc",
-    "build:prod": "jlpm run build:lib && jlpm run build:labextension",
+    "build:lib": "tsc -b",
+    "build:prod": "jlpm run build:labextension",
     "clean": "jlpm run clean:lib && jlpm run clean:labextension",
     "clean:labextension": "rimraf ../../notebook/labextension",
     "clean:lib": "rimraf lib tsconfig.tsbuildinfo",
@@ -42,6 +42,39 @@
     "watch:labextension": "jupyter labextension watch .",
     "watch:src": "tsc -w"
   },
+  "nx": {
+    "targets": {
+      "build:labextension:dev": {
+        "dependsOn": [
+          "^build:lib",
+          "build:lib"
+        ],
+        "outputs": [
+          "{workspaceRoot}/notebook/labextension",
+          "{workspaceRoot}/notebook/labextension/build_log.json"
+        ]
+      },
+      "build:labextension": {
+        "dependsOn": [
+          "^build:lib",
+          "build:lib"
+        ],
+        "outputs": [
+          "{workspaceRoot}/notebook/labextension"
+        ]
+      },
+      "build": {
+        "dependsOn": [
+          "build:labextension:dev"
+        ]
+      },
+      "build:prod": {
+        "dependsOn": [
+          "build:labextension"
+        ]
+      }
+    }
+  },
   "dependencies": {
     "@jupyter-notebook/application": "^7.0.0",
     "@jupyterlab/application": "^4.0.3",