Merge remote-tracking branch 'upstream/develop' into update-mongodb

# Conflicts:
#	.pnp.cjs
#	yarn.lock

.github/workflows/test-pulls.yml

@@ -16,41 +16,10 @@ jobs:
       - name: yarn install
         run: yarn install
       - name: Build packages
-        run: yarn build --ignore='@lowdefy/blocks-*'
+        run: yarn build
 
-      # format tests don't pass on node 12 since icu is missing and tests don't work with locales
       - name: Test packages
-        run: yarn test --ignore='@lowdefy/format' --ignore='@lowdefy/blocks-*'
-
-      - name: Upload coverage to codecov
-        run: bash <(curl -s https://codecov.io/bash)
-
-      - name: Upload coverage to codeclimate
-        uses: paambaati/codeclimate-action@v2.7.5
-        env:
-          CC_TEST_REPORTER_ID: ${{secrets.CC_TEST_REPORTER_ID}}
-        with:
-          coverageLocations: |
-            ${{github.workspace}}/packages/**/coverage/lcov.info:lcov
-        continue-on-error: true
-
-  test-blocks:
-    name: Test blocks
-    runs-on: ubuntu-latest
-    steps:
-      - name: Setup Node
-        uses: actions/setup-node@v2
-        with:
-          node-version: '12.x'
-      - uses: actions/checkout@v2
-      - name: yarn install
-        run: yarn install
-      - name: Build packages
-        run: yarn build --ignore='@lowdefy/api' --ignore='@lowdefy/build' --ignore='@lowdefy/cli' --ignore='@lowdefy/client' --ignore='@lowdefy/engine' --ignore='@lowdefy/layout' --ignore='@lowdefy/node-utils' --ignore='@lowdefy/operators' --ignore='@lowdefy/server*'
-
-      # format tests don't pass on node 12 since icu is missing and tests don't work with locales
-      - name: Test packages
-        run: yarn test --scope='@lowdefy/blocks-*'
+        run: yarn test --ignore='@lowdefy/engine' --ignore='@lowdefy/blocks-*' --ignore='@lowdefy/plugin-aws'
 
       - name: Upload coverage to codecov
         run: bash <(curl -s https://codecov.io/bash)
@@ -71,7 +40,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v2
         with:
-          node-version: '12.x'
+          node-version: '14.x'
       - uses: actions/checkout@v2
       - name: Check yarn cache integrity
         run: yarn install --immutable --immutable-cache --check-cache

.gitignore

@@ -18,6 +18,7 @@ packages/server/build/**
 packages/server-dev/build/**
 !packages/docs/lowdefy.yaml
 !packages/docs/howto/**/lowdefy.yaml
+packages/cli/server/**
 
 .DS_Store
 

package.json

@@ -32,14 +32,28 @@
   "scripts": {
     "build": "lerna run build",
     "build:dev": "NODE_ENV=development lerna run build",
+    "build:plugins": "yarn build:blocks && yarn build:connections && yarn build:operators",
+    "build:blocks": "lerna run --scope '@lowdefy/blocks-*' build",
+    "build:connections": "lerna run --scope '@lowdefy/connection-*' build",
+    "build:operators": "lerna run --scope '@lowdefy/operators-*' build",
     "clean": "lerna run clean",
     "lerna:version": "lerna version --no-git-tag-version",
     "lerna:publish": "lerna publish from-git",
     "postversion": "yarn install",
     "prettier": "prettier --config .prettierrc --write **/*.js",
-    "start:server-dev": "yarn workspace @lowdefy/server-dev start --package-manager yarn --config-directory ../../app",
-    "start": "yarn workspace @lowdefy/server build:lowdefy --config-directory ../../app && yarn && yarn workspace @lowdefy/server build:next && yarn workspace @lowdefy/server start",
-    "start:dev": "yarn workspace @lowdefy/server build:lowdefy --config-directory ../../app && yarn && yarn workspace @lowdefy/server dev",
+    "start": "yarn start:server:app",
+    "start:cli:build:app": "yarn workspace lowdefy start build --config-directory ../../app --server-directory ../server --output-directory ../",
+    "start:cli:build:docs": "yarn workspace lowdefy start build --config-directory ../docs --server-directory ../server --output-directory ../",
+    "start:cli:dev:app": "yarn workspace lowdefy start dev --config-directory ../../app --dev-directory ../server-dev",
+    "start:cli:dev:docs": "yarn workspace lowdefy start dev --config-directory ../docs --dev-directory ../server-dev",
+    "start:cli:start:app": "yarn workspace lowdefy start start --config-directory ../../app --server-directory ../server --output-directory ../",
+    "start:cli:start:docs": "yarn workspace lowdefy start start --config-directory ../docs --server-directory ../server --output-directory ../",
+    "start:server-dev:app": "yarn workspace @lowdefy/server-dev start --package-manager yarn --config-directory ../../app",
+    "start:server-dev:docs": "yarn workspace @lowdefy/server-dev start --package-manager yarn --config-directory ../docs",
+    "start:server:app": "yarn workspace @lowdefy/server build:lowdefy --config-directory ../../app && yarn && yarn workspace @lowdefy/server build:next && yarn workspace @lowdefy/server start",
+    "start:server:docs": "yarn workspace @lowdefy/server build:lowdefy --config-directory ../docs && yarn && yarn workspace @lowdefy/server build:next && yarn workspace @lowdefy/server start",
+    "start:server:next-dev:app": "yarn workspace @lowdefy/server build:lowdefy --config-directory ../../app && yarn && yarn workspace @lowdefy/server dev",
+    "start:server:next-dev:docs": "yarn workspace @lowdefy/server build:lowdefy --config-directory ../docs && yarn && yarn workspace @lowdefy/server dev",
     "test": "lerna run test"
   },
   "devDependencies": {

packages/api/package.json

@@ -38,7 +38,7 @@
     "clean": "rm -rf dist",
     "prepare": "yarn build",
     "swc": "swc src --out-dir dist --config-file ../../.swcrc --delete-dir-on-start",
-    "test": "jest --coverage"
+    "test": "yarn node --experimental-vm-modules $(yarn bin jest)"
   },
   "dependencies": {
     "@lowdefy/ajv": "4.0.0-alpha.6",
@@ -48,10 +48,12 @@
     "@lowdefy/operators": "4.0.0-alpha.6"
   },
   "devDependencies": {
+    "@jest/globals": "27.5.1",
+    "@lowdefy/operators-js": "4.0.0-alpha.6",
     "@swc/cli": "0.1.55",
     "@swc/core": "1.2.135",
     "@swc/jest": "0.2.17",
-    "jest": "27.4.7"
+    "jest": "27.5.1"
   },
   "publishConfig": {
     "access": "public"

packages/api/src/context/readConfigFile.test.js

@@ -0,0 +1,35 @@
+/*
+  Copyright 2020-2021 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+import { jest } from '@jest/globals';
+
+import { getFileExtension } from '@lowdefy/node-utils';
+
+jest.unstable_mockModule('@lowdefy/node-utils', () => {
+  return {
+    getFileExtension,
+    readFile: jest.fn(),
+  };
+});
+
+test('readConfigFile', async () => {
+  const nodeUtils = await import('@lowdefy/node-utils');
+
+  nodeUtils.readFile.mockImplementation(() => Promise.resolve('config value'));
+  const createReadConfigFile = (await import('./readConfigFile.js')).default;
+  const readConfigFile = createReadConfigFile({ buildDirectory: '/build' });
+  const res = await readConfigFile('file');
+  expect(res).toEqual('config value');
+});

packages/api/src/routes/page/getPageConfig.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import getPageConfig from './getPageConfig.js';
 import testContext from '../../test/testContext.js';
 

packages/api/src/routes/request/callRequest.test.js

@@ -13,12 +13,16 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 */
+import { jest } from '@jest/globals';
+import { operatorsServer } from '@lowdefy/operators-js';
 
 import callRequest from './callRequest.js';
 import testContext from '../../test/testContext.js';
 
 import { ConfigurationError, RequestError } from '../../context/errors.js';
 
+const { _date, _payload, _secret, _user } = operatorsServer;
+
 console.error = () => {};
 
 const mockReadConfigFile = jest.fn();
@@ -26,7 +30,14 @@ const mockTestRequest = jest.fn();
 const mockTestRequestCheckRead = jest.fn();
 const mockTestRequestCheckWrite = jest.fn();
 
-mockTestRequest.schema = {};
+mockTestRequest.schema = {
+  type: 'object',
+  properties: {
+    schemaPropString: {
+      type: 'string',
+    },
+  },
+};
 mockTestRequestCheckRead.schema = {};
 mockTestRequestCheckWrite.schema = {};
 
@@ -61,15 +72,31 @@ const connections = {
   },
 };
 
+const operators = {
+  _date,
+  _payload,
+  _secret,
+  _user,
+  _error: () => {
+    throw new Error('Test error.');
+  },
+};
+
 const secrets = {
   CONNECTION: 'connectionSecret',
   REQUEST: 'requestSecret',
 };
 
-const context = testContext({ connections, readConfigFile: mockReadConfigFile, secrets });
+const context = testContext({
+  connections,
+  readConfigFile: mockReadConfigFile,
+  operators,
+  secrets,
+});
 const authenticatedContext = testContext({
   connections,
   readConfigFile: mockReadConfigFile,
+  operators,
   secrets,
   user: { sub: 'sub' },
 });
@@ -530,7 +557,7 @@ test('request properties operator error', async () => {
         connectionId: 'testConnection',
         auth: { public: true },
         properties: {
-          willError: { _get: null },
+          willError: { _error: null },
         },
       },
     })
@@ -538,9 +565,7 @@ test('request properties operator error', async () => {
   mockTestRequest.mockImplementation(defaultResolverImp);
 
   await expect(callRequest(context, defaultParams)).rejects.toThrow(RequestError);
-  await expect(callRequest(context, defaultParams)).rejects.toThrow(
-    'Error: Operator Error: _get takes an object as params. Received: null at requestId.'
-  );
+  await expect(callRequest(context, defaultParams)).rejects.toThrow('Error: Test error.');
 });
 
 test('connection properties operator error', async () => {
@@ -552,7 +577,7 @@ test('connection properties operator error', async () => {
         connectionId: 'testConnection',
         auth: { public: true },
         properties: {
-          willError: { _get: null },
+          willError: { _error: null },
         },
       },
     })
@@ -560,9 +585,7 @@ test('connection properties operator error', async () => {
   mockTestRequest.mockImplementation(defaultResolverImp);
 
   await expect(callRequest(context, defaultParams)).rejects.toThrow(RequestError);
-  await expect(callRequest(context, defaultParams)).rejects.toThrow(
-    'Error: Operator Error: _get takes an object as params. Received: null at testConnection.'
-  );
+  await expect(callRequest(context, defaultParams)).rejects.toThrow('Error: Test error.');
 });
 
 test('request resolver throws  error', async () => {

packages/api/src/routes/rootConfig/getHomeAndMenus.test.js

@@ -13,11 +13,12 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 */
+import { jest } from '@jest/globals';
+
 import testContext from '../../test/testContext.js';
 
 const mockGetMenu = jest.fn();
-jest.mock('./menus/getMenus.js', () => ({
-  __esModule: true,
+jest.unstable_mockModule('./menus/getMenus.js', () => ({
   default: mockGetMenu,
 }));
 

packages/api/src/routes/rootConfig/getLowdefyGlobal.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import getLowdefyGlobal from './getLowdefyGlobal.js';
 import testContext from '../../test/testContext.js';
 

packages/api/src/routes/rootConfig/getRootConfig.test.js

@@ -13,12 +13,12 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 */
+import { jest } from '@jest/globals';
 
 import testContext from '../../test/testContext.js';
 
 const mockGetMenu = jest.fn();
-jest.mock('./menus/getMenus.js', () => ({
-  __esModule: true,
+jest.unstable_mockModule('./menus/getMenus.js', () => ({
   default: mockGetMenu,
 }));
 

packages/api/src/routes/rootConfig/menus/getMenus.test.js

@@ -13,6 +13,7 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 */
+import { jest } from '@jest/globals';
 
 import getMenus from './getMenus.js';
 import testContext from '../../../test/testContext.js';

packages/build/package.json

@@ -42,7 +42,7 @@
     "prepare": "yarn build",
     "start": "node dist/scripts/run.js",
     "swc": "swc src --out-dir dist --config-file ../../.swcrc --delete-dir-on-start",
-    "test": "jest --coverage"
+    "test": "yarn node --experimental-vm-modules $(yarn bin jest)"
   },
   "dependencies": {
     "@lowdefy/ajv": "4.0.0-alpha.6",
@@ -50,12 +50,14 @@
     "@lowdefy/node-utils": "4.0.0-alpha.6",
     "@lowdefy/nunjucks": "4.0.0-alpha.6",
     "ajv": "8.9.0",
-    "js-yaml": "4.1.0",
     "json5": "2.2.0",
     "uuid": "8.3.2",
+    "yaml": "2.0.0-10",
     "yargs": "17.3.1"
   },
   "devDependencies": {
+    "@jest/globals": "27.5.1",
+    "@lowdefy/actions-core": "4.0.0-alpha.6",
     "@lowdefy/blocks-antd": "4.0.0-alpha.6",
     "@lowdefy/blocks-basic": "4.0.0-alpha.6",
     "@lowdefy/blocks-color-selectors": "4.0.0-alpha.6",
@@ -80,7 +82,7 @@
     "@swc/cli": "0.1.55",
     "@swc/core": "1.2.135",
     "@swc/jest": "0.2.17",
-    "jest": "27.4.7"
+    "jest": "27.5.1"
   },
   "publishConfig": {
     "access": "public"

packages/build/src/build/addDefaultPages/addDefaultPages.js

@@ -29,7 +29,12 @@ async function addDefaultPages({ components }) {
     throw new Error('lowdefy.pages is not an array.');
   }
 
-  const pageIds = components.pages.map((page) => page.id);
+  const pageIds = components.pages.map((page, index) => {
+    if (!type.isObject(page)) {
+      throw new Error(`pages[${index}] is not an object. Received ${JSON.stringify(page)}`);
+    }
+    return page.id;
+  });
   // deep copy to avoid mutating defaultConfig
   const filteredDefaultPages = defaultPages.filter(
     (defaultPage) => !pageIds.includes(defaultPage.id)

packages/build/src/build/addDefaultPages/addDefaultPages.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import addDefaultPages from './addDefaultPages.js';
 import testContext from '../../test/testContext.js';
 
@@ -202,6 +204,15 @@ test('addDefaultPages, pages not an array', async () => {
   );
 });
 
+test('addDefaultPages, with a page not an object', async () => {
+  const components = {
+    pages: [null],
+  };
+  await expect(addDefaultPages({ components, context })).rejects.toThrow(
+    'pages[0] is not an object. Received null'
+  );
+});
+
 test('addDefaultPages, pages are copied', async () => {
   const components1 = {};
   const res1 = await addDefaultPages({ components: components1, context });

packages/build/src/build/buildMenu.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import buildMenu from './buildMenu.js';
 import testContext from '../test/testContext.js';
 

packages/build/src/build/buildPages/buildBlock/buildEvents.js

@@ -64,7 +64,10 @@ function buildEvents(block, pageContext) {
           }". Received ${JSON.stringify(block.events[key].try)}`
         );
       }
-      if (!type.isArray(block.events[key].catch) && !type.isNone(block.events[key].catch)) {
+      if (type.isNone(block.events[key].catch)) {
+        block.events[key].catch = [];
+      }
+      if (!type.isArray(block.events[key].catch)) {
         throw new Error(
           `Catch events must be an array of actions at "${
             block.blockId

packages/build/src/build/buildPages/buildBlock/buildEvents.test.js

@@ -13,6 +13,9 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 */
+
+import { jest } from '@jest/globals';
+
 import { get } from '@lowdefy/helpers';
 import buildPages from '../buildPages.js';
 import testContext from '../../../test/testContext.js';
@@ -117,6 +120,42 @@ test('block events actions as try catch arrays', async () => {
   ]);
 });
 
+test('block events actions as try array and catch not defined.', async () => {
+  const components = {
+    pages: [
+      {
+        id: 'page_1',
+        type: 'Container',
+        auth,
+        blocks: [
+          {
+            id: 'block_1',
+            type: 'Input',
+            events: {
+              onClick: {
+                try: [
+                  {
+                    id: 'action_1',
+                    type: 'Reset',
+                  },
+                ],
+              },
+            },
+          },
+        ],
+      },
+    ],
+  };
+  const res = await buildPages({ components, context });
+  expect(get(res, 'pages.0.areas.content.blocks.0.events.onClick.try')).toEqual([
+    {
+      id: 'action_1',
+      type: 'Reset',
+    },
+  ]);
+  expect(get(res, 'pages.0.areas.content.blocks.0.events.onClick.catch')).toEqual([]);
+});
+
 test('block events actions try not an array', async () => {
   const components = {
     pages: [

packages/build/src/build/buildPages/buildBlock/countBlockOperators.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import buildPages from '../buildPages.js';
 import testContext from '../../../test/testContext.js';
 

packages/build/src/build/buildPages/buildPages.test.js

@@ -13,6 +13,9 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 */
+
+import { jest } from '@jest/globals';
+
 import buildPages from './buildPages.js';
 import testContext from '../../test/testContext.js';
 

packages/build/src/build/buildPages/buildRequests.test.js

@@ -13,6 +13,9 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 */
+
+import { jest } from '@jest/globals';
+
 import buildPages from './buildPages.js';
 import testContext from '../../test/testContext.js';
 

packages/build/src/build/buildRefs/buildRefs.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import buildRefs from './buildRefs.js';
 import testContext from '../../test/testContext.js';
 

packages/build/src/build/buildRefs/getUserJavascriptFunction.js

@@ -16,8 +16,12 @@
 import path from 'path';
 
 async function getUserJavascriptFunction({ context, filePath }) {
-  const module = await import(path.resolve(context.directories.config, filePath));
-  return module.default;
+  try {
+    return (await import(path.resolve(context.directories.config, filePath))).default;
+  } catch (error) {
+    context.logger.error(`Error importing ${filePath}.`);
+    throw Error(error);
+  }
 }
 
 export default getUserJavascriptFunction;

packages/build/src/build/buildRefs/parseRefContent.js

@@ -17,7 +17,7 @@
 import { type } from '@lowdefy/helpers';
 import { getFileExtension, getFileSubExtension } from '@lowdefy/node-utils';
 import JSON5 from 'json5';
-import YAML from 'js-yaml';
+import YAML from 'yaml';
 
 import parseNunjucks from './parseNunjucks.js';
 
@@ -31,7 +31,7 @@ function parseRefContent({ content, refDef }) {
     }
 
     if (ext === 'yaml' || ext === 'yml') {
-      return YAML.load(content);
+      return YAML.parse(content);
     }
     if (ext === 'json') {
       return JSON5.parse(content);

packages/build/src/build/buildRefs/runTransformer.js

@@ -22,7 +22,13 @@ async function runTransformer({ context, parsedFile, refDef }) {
       context,
       filePath: refDef.transformer,
     });
-    return transformerFn(parsedFile, refDef.vars);
+    try {
+      return transformerFn(parsedFile, refDef.vars);
+    } catch (error) {
+      throw Error(
+        `Error calling transformer "${refDef.transformer}" from "${refDef.path}": ${error.message}`
+      );
+    }
   }
   return parsedFile;
 }

packages/build/src/build/buildTypes.js

@@ -53,12 +53,12 @@ function buildTypes({ components, context }) {
     },
   };
 
-  // buildTypeClass({
-  //   counter: typeCounters.actions,
-  //   definitions: context.types.actions,
-  //   store: components.types.actions,
-  //   typeClass: 'Action',
-  // });
+  buildTypeClass(context, {
+    counter: typeCounters.actions,
+    definitions: context.types.actions,
+    store: components.types.actions,
+    typeClass: 'Action',
+  });
 
   buildTypeClass(context, {
     counter: typeCounters.blocks,

packages/build/src/build/cleanBuildDirectory.test.js

@@ -14,7 +14,9 @@
   limitations under the License.
 */
 
-jest.mock('@lowdefy/node-utils', () => {
+import { jest } from '@jest/globals';
+
+jest.unstable_mockModule('@lowdefy/node-utils', () => {
   return {
     cleanDirectory: jest.fn(),
   };

packages/build/src/build/testSchema.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import testSchema from './testSchema.js';
 import testContext from '../test/testContext.js';
 

packages/build/src/build/validateConfig.js

@@ -39,6 +39,11 @@ async function validateConfig({ components }) {
   if (type.isNone(components.config.theme)) {
     components.config.theme = {};
   }
+  if (type.isString(components.config.basePath)) {
+    if (components.config.basePath[0] !== '/') {
+      throw Error('Base path must start with "/".');
+    }
+  }
   validate({
     schema: lowdefySchema.definitions.authConfig,
     data: components.config.auth,

packages/build/src/build/validateConfig.test.js

@@ -155,3 +155,31 @@ test('validateConfig config error when protected or public are false.', async ()
     'Public pages can not be set to false.'
   );
 });
+
+test('validateConfig config error when basePath does not start with "/".', async () => {
+  let components = {
+    config: {
+      basePath: '/base',
+    },
+  };
+  const result = await validateConfig({ components, context });
+  expect(result).toEqual({
+    config: {
+      auth: {
+        pages: {
+          roles: {},
+        },
+      },
+      basePath: '/base',
+      theme: {},
+    },
+  });
+  components = {
+    config: {
+      basePath: 'base',
+    },
+  };
+  await expect(validateConfig({ components, context })).rejects.toThrow(
+    'Base path must start with "/".'
+  );
+});

packages/build/src/build/writeApp.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import writeApp from './writeApp.js';
 import testContext from '../test/testContext.js';
 

packages/build/src/build/writeConfig.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import writeConfig from './writeConfig.js';
 import testContext from '../test/testContext.js';
 

packages/build/src/build/writeConnections.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import writeConnections from './writeConnections.js';
 import testContext from '../test/testContext.js';
 

packages/build/src/build/writeGlobal.test.js

@@ -14,6 +14,8 @@
   limitations under the License.
 */
 
+import { jest } from '@jest/globals';
+
 import writeGlobal from './writeGlobal.js';
 import testContext from '../test/testContext.js';