feat(build): add buildRefs function

This commit is contained in:
Sam Tolmay 2020-10-20 12:03:20 +02:00
parent 46586f05f2
commit 3ab2819944
2 changed files with 824 additions and 0 deletions

View File

@ -0,0 +1,168 @@
/*
Copyright 2020 Lowdefy, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { get, type } from '@lowdefy/helpers';
import { nunjucksFunction } from '@lowdefy/nunjucks';
import JSON5 from 'json5';
import YAML from 'js-yaml';
import { v1 as uuid } from 'uuid';
import getFileExtension, { getFileSubExtension } from '../utils/files/getFileExtension';
function getRefPath(refDefinition) {
if (type.isObject(refDefinition) && refDefinition.path) {
return refDefinition.path;
}
if (type.isString(refDefinition)) {
return refDefinition;
}
return null;
}
function getRefVars(refDefinition) {
if (type.isObject(refDefinition) && refDefinition.vars) {
return refDefinition.vars;
}
return {};
}
function makeRefDefinition(refDefinition) {
return {
// uuid is overkill but it is already in bundle
id: uuid(),
path: getRefPath(refDefinition),
vars: getRefVars(refDefinition),
original: refDefinition,
};
}
function getRefsFromFile(fileContent) {
const foundRefs = [];
const reviver = (key, value) => {
if (type.isObject(value)) {
if (!type.isUndefined(value._ref)) {
const def = makeRefDefinition(value._ref);
foundRefs.push(def);
return {
_ref: def,
};
}
}
return value;
};
const fileContentBuiltRefs = JSON.stringify(JSON.parse(fileContent, reviver));
return { foundRefs, fileContentBuiltRefs };
}
function parseNunjucks(fileContent, vars, path) {
const template = nunjucksFunction(JSON.parse(fileContent));
const templated = template(vars);
const subExt = getFileSubExtension(path);
if (subExt === 'yaml' || subExt === 'yml') {
return JSON.stringify(YAML.safeLoad(templated));
}
if (subExt === 'json') {
return JSON.stringify(JSON5.parse(templated));
}
return JSON.stringify(templated);
}
function refReviver(key, value) {
if (type.isObject(value)) {
if (!type.isUndefined(value._ref)) {
return this.parsedFiles[value._ref.id];
}
if (value._var) {
if (type.isObject(value._var) && type.isString(value._var.name)) {
return JSON.parse(
JSON.stringify(get(this.vars, value._var.name, { default: value._var.default || null }))
);
}
return JSON.parse(JSON.stringify(get(this.vars, value._var, { default: null })));
}
}
return value;
}
class RefBuilder {
constructor({ context }) {
this.rootPath = 'lowdefy.yaml';
this.configLoader = context.configLoader;
this.refContent = {};
this.MAX_RECURSION_DEPTH = context.MAX_RECURSION_DEPTH || 20;
}
async getFileContent(path) {
const file = await this.configLoader.load(path);
if (!file) {
throw new Error(`File "${path}" not found.`);
}
return file;
}
async build() {
return this.recursiveParseFile({
path: this.rootPath,
vars: {},
count: 0,
});
}
async recursiveParseFile({ path, count, vars }) {
if (count > this.MAX_RECURSION_DEPTH) {
throw new Error(
`Maximum recursion depth of references exceeded. Only ${this.MAX_RECURSION_DEPTH} consecutive references are allowed`
);
}
let fileContent = await this.getFileContent(path);
if (getFileExtension(path) === 'njk') {
fileContent = parseNunjucks(fileContent, vars, path);
}
const { foundRefs, fileContentBuiltRefs } = getRefsFromFile(fileContent);
const parsedFiles = {};
// Since we can have references in the variables of a reference, we need to first parse
// the deeper nodes, so we can use those parsed files in references higher in the tree.
// To do this, since foundRefs is an array of ref definitions that are in order of the
// deepest nodes first we for loop over over foundRefs one by one, awaiting each result.
// eslint-disable-next-line no-restricted-syntax
for (const refDef of foundRefs.values()) {
if (refDef.path === null) {
throw new Error(
`Invalid _ref definition ${JSON.stringify(refDef.original)} at file ${path}`
);
}
// eslint-disable-next-line no-await-in-loop
parsedFiles[refDef.id] = await this.recursiveParseFile({
path: refDef.path,
// Parse vars before passing down to parse new file
vars: JSON.parse(JSON.stringify(refDef.vars), refReviver.bind({ parsedFiles, vars })),
count: count + 1,
});
}
return JSON.parse(fileContentBuiltRefs, refReviver.bind({ parsedFiles, vars }));
}
}
async function buildRefs({ context }) {
const builder = new RefBuilder({ context });
const components = await builder.build();
await context.logger.info('Built References');
return components;
}
export default buildRefs;

View File

@ -0,0 +1,656 @@
/*
Copyright 2020 Lowdefy, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import buildRefs from './buildRefs';
const configLoaderMockImplementation = (files) => {
const mockImp = (key) => {
const file = files.find((file) => file.path === key);
if (!file) return null;
return file.content;
};
return mockImp;
};
const mockConfigLoader = jest.fn();
const context = {
logger: { log: () => {}, info: () => {} },
configLoader: {
load: mockConfigLoader,
},
};
test('buildRefs file not found', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"doesNotExist": { "_ref": "doesNotExist" }
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
await expect(buildRefs({ context })).rejects.toThrow('File "doesNotExist" not found.');
});
test('buildRefs', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"jsonFile": { "_ref": "jsonFile.json" },
"twoLevels": { "_ref": "twoLevels.json"},
"vars": {
"_ref": {
"path": "vars.json",
"vars": {
"var_1": "var_1_value"
}
}
}
}`,
},
{
path: 'jsonFile.json',
content: `{
"path": "jsonFile.json"
}`,
},
{
path: 'twoLevels.json',
content: `{
"path": "twoLevels.json",
"jsonFile": { "_ref": "jsonFile.json" }
}`,
},
{
path: 'vars.json',
content: `{
"path": "vars.json",
"var1": {
"_var": "var_1"
}
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
jsonFile: { path: 'jsonFile.json' },
twoLevels: { path: 'twoLevels.json', jsonFile: { path: 'jsonFile.json' } },
vars: { path: 'vars.json', var1: 'var_1_value' },
});
});
test('buildRefs max recursion depth', async () => {
const ctx = {
...context,
MAX_RECURSION_DEPTH: 3,
};
const files = [
{
path: 'lowdefy.yaml',
content: `{
"1": { "_ref": "maxRecursion1.json" }
}`,
},
{
path: 'maxRecursion1.json',
content: `{
"2": { "_ref": "maxRecursion2.json" }
}`,
},
{
path: 'maxRecursion2.json',
content: `{
"3": { "_ref": "maxRecursion3.json" }
}`,
},
{
path: 'maxRecursion3.json',
content: `{
"4": { "_ref": "maxRecursion4.json" }
}`,
},
{
path: 'maxRecursion4.json',
content: `{
"4": "done"
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
await expect(buildRefs({ context: ctx })).rejects.toThrow();
await expect(buildRefs({ context: ctx })).rejects.toThrow(
'Maximum recursion depth of references exceeded. Only 3 consecutive references are allowed'
);
});
test('load refs to text files', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"text": { "_ref": "text.txt" },
"html": { "_ref": "html.html" },
"md": { "_ref": "markdown.md" }
}`,
},
{
path: 'text.txt',
content: JSON.stringify(`Some multiline
text.
Hello.`),
},
{
path: 'html.html',
content: JSON.stringify(`<h1>Heading</h1>
<p>Hello there</p>`),
},
{
path: 'markdown.md',
content: JSON.stringify(`### Title
Hello there`),
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toMatchInlineSnapshot(`
Object {
"html": "<h1>Heading</h1>
<p>Hello there</p>",
"md": "### Title
Hello there",
"text": "Some multiline
text.
Hello.",
}
`);
});
test('buildRefs should copy vars', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"ref": {
"_ref": {
"path": "file.yaml",
"vars": {
"var1": {
"key": "value"
}
}
}
}
}`,
},
{
path: 'file.yaml',
content: `{
"ref1": {
"_var": "var1"
},
"ref2": {
"_var": "var1"
}
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
ref: {
ref1: { key: 'value' },
ref2: { key: 'value' },
},
});
res.ref.ref1.key = 'newValue';
expect(res).toEqual({
ref: {
ref1: { key: 'newValue' },
ref2: { key: 'value' },
},
});
});
test('buildRefs null ref definition', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"invalid": { "_ref": null }
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
expect(buildRefs({ context })).rejects.toMatchInlineSnapshot(
`[Error: Invalid _ref definition null at file lowdefy.yaml]`
);
});
test('buildRefs invalid ref definition', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"invalid": { "_ref": 1 }
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
expect(buildRefs({ context })).rejects.toMatchInlineSnapshot(
`[Error: Invalid _ref definition 1 at file lowdefy.yaml]`
);
});
test('buildRefs nunjucks text file', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"templated": {
"_ref": {
"path": "template.njk",
"vars": {
"var_1": "There"
}
}
}
}`,
},
{
path: 'template.njk',
content: JSON.stringify('Hello {{ var_1 }}'),
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
templated: 'Hello There',
});
});
test('buildRefs nunjucks json file', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"templated": {
"_ref": {
"path": "template.json.njk",
"vars": {
"key": "key1"
}
}
}
}`,
},
{
path: 'template.json.njk',
content: JSON.stringify('{ "{{ key }}": true }'),
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
templated: { key1: true },
});
});
test('buildRefs nunjucks yaml file', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"templated": {
"_ref": {
"path": "template.yaml.njk",
"vars": {
"values": ["value1", "value2"]
}
}
}
}`,
},
{
path: 'template.yaml.njk',
content: JSON.stringify(`list:
{% for value in values %}
- key: {{ value }}
{% endfor %}
`),
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
templated: { list: [{ key: 'value1' }, { key: 'value2' }] },
});
});
test('buildRefs nunjucks yml file', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"templated": {
"_ref": {
"path": "template.yml.njk",
"vars": {
"values": ["value1", "value2"]
}
}
}
}`,
},
{
path: 'template.yml.njk',
content: JSON.stringify(`list:
{% for value in values %}
- key: {{ value }}
{% endfor %}
`),
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
templated: { list: [{ key: 'value1' }, { key: 'value2' }] },
});
});
test('buildRefs pass vars two levels', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"ref1": {
"_ref": {
"path": "file1.yaml",
"vars": {
"var1": "Hello"
}
}
}
}`,
},
{
path: 'file1.yaml',
content: `{
"ref2": {
"_ref": {
"path": "file2.yaml",
"vars": {
"var2": {
"_var": "var1"
}
}
}
}
}`,
},
{
path: 'file2.yaml',
content: `{
"value": {
"_var": "var2"
}
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
ref1: {
ref2: {
value: 'Hello',
},
},
});
});
test('buildRefs use a ref in a var', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"ref1": {
"_ref": {
"path": "file1.yaml",
"vars": {
"file2": {
"_ref": "file2.md"
}
}
}
}
}`,
},
{
path: 'file1.yaml',
content: `{
"content": {
"_var": "file2"
}
}`,
},
{
path: 'file2.md',
content: `"Hello"`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
ref1: {
content: 'Hello',
},
});
});
test('buildRefs use a ref in var, with a var from parent as a var', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"ref1": {
"_ref": {
"path": "file1.yaml",
"vars": {
"parent1": 1,
"parent2": 2
}
}
}
}`,
},
{
path: 'file1.yaml',
content: `{
"ref2": {
"_ref": {
"path": "file2.yaml",
"vars": {
"var": {
"_var": "parent1"
},
"ref": {
"_ref": {
"path": "file3.yaml",
"vars": {
"var": {
"_var": "parent2"
},
"const": 3
}
}
}
}
}
}
}`,
},
{
path: 'file2.yaml',
content: `{
"value": {
"_var": "var"
},
"ref": {
"_var": "ref"
}
}`,
},
{
path: 'file3.yaml',
content: `{
"value": {
"_var": "var"
},
"const": {
"_var": "const"
}
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
ref1: {
ref2: {
value: 1,
ref: {
value: 2,
const: 3,
},
},
},
});
});
test('buildRefs var specified by name', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"ref": {
"_ref": {
"path": "file.yaml",
"vars": {
"var1": "value"
}
}
}
}`,
},
{
path: 'file.yaml',
content: `{
"key": {
"_var": {
"name": "var1"
}
}
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
ref: {
key: 'value',
},
});
});
test('buildRefs var with default value, but value specified', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"ref": {
"_ref": {
"path": "file.yaml",
"vars": {
"var1": "value"
}
}
}
}`,
},
{
path: 'file.yaml',
content: `{
"key": {
"_var": {
"name": "var1",
"default": "default"
}
}
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
ref: {
key: 'value',
},
});
});
test('buildRefs var uses default value if value not specified', async () => {
const files = [
{
path: 'lowdefy.yaml',
content: `{
"ref": {
"_ref": {
"path": "file.yaml",
"vars": {
"var2": "value"
}
}
}
}`,
},
{
path: 'file.yaml',
content: `{
"key": {
"_var": {
"name": "var1",
"default": "default"
}
}
}`,
},
];
mockConfigLoader.mockImplementation(configLoaderMockImplementation(files));
const res = await buildRefs({ context });
expect(res).toEqual({
ref: {
key: 'default',
},
});
});