wasm gc: support source maps

This commit is contained in:
Alexey Andreev 2024-10-14 20:24:34 +02:00
parent 2f678ccb6c
commit d68018d2d3
8 changed files with 448 additions and 39 deletions

View File

@ -22,9 +22,11 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.teavm.backend.wasm.debug.CompositeDebugLines;
import org.teavm.backend.wasm.debug.DebugLines;
import org.teavm.backend.wasm.debug.ExternalDebugFile;
import org.teavm.backend.wasm.debug.GCDebugInfoBuilder;
import org.teavm.backend.wasm.debug.sourcemap.SourceMapBuilder;
import org.teavm.backend.wasm.gc.TeaVMWasmGCHost;
import org.teavm.backend.wasm.gc.WasmGCClassConsumer;
import org.teavm.backend.wasm.gc.WasmGCClassConsumerContext;
@ -40,6 +42,7 @@ import org.teavm.backend.wasm.generators.gc.WasmGCCustomGenerators;
import org.teavm.backend.wasm.intrinsics.gc.WasmGCIntrinsic;
import org.teavm.backend.wasm.intrinsics.gc.WasmGCIntrinsicFactory;
import org.teavm.backend.wasm.intrinsics.gc.WasmGCIntrinsics;
import org.teavm.backend.wasm.model.WasmCustomSection;
import org.teavm.backend.wasm.model.WasmFunction;
import org.teavm.backend.wasm.model.WasmModule;
import org.teavm.backend.wasm.model.WasmTag;
@ -77,6 +80,8 @@ public class WasmGCTarget implements TeaVMTarget, TeaVMWasmGCHost {
private boolean strict;
private boolean obfuscated;
private boolean debugInfo;
private SourceMapBuilder sourceMapBuilder;
private String sourceMapLocation;
private WasmDebugInfoLocation debugLocation = WasmDebugInfoLocation.EXTERNAL;
private WasmDebugInfoLevel debugLevel = WasmDebugInfoLevel.FULL;
private List<WasmGCIntrinsicFactory> intrinsicFactories = new ArrayList<>();
@ -107,6 +112,14 @@ public class WasmGCTarget implements TeaVMTarget, TeaVMWasmGCHost {
this.debugLocation = debugLocation;
}
public void setSourceMapBuilder(SourceMapBuilder sourceMapBuilder) {
this.sourceMapBuilder = sourceMapBuilder;
}
public void setSourceMapLocation(String sourceMapLocation) {
this.sourceMapLocation = sourceMapLocation;
}
@Override
public void addIntrinsicFactory(WasmGCIntrinsicFactory intrinsicFactory) {
intrinsicFactories.add(intrinsicFactory);
@ -337,7 +350,22 @@ public class WasmGCTarget implements TeaVMTarget, TeaVMWasmGCHost {
var binaryWriter = new WasmBinaryWriter();
DebugLines debugLines = null;
if (debugInfo) {
debugLines = debugInfoBuilder.lines();
if (sourceMapBuilder != null) {
debugLines = new CompositeDebugLines(debugInfoBuilder.lines(), sourceMapBuilder);
} else {
debugLines = debugInfoBuilder.lines();
}
} else if (sourceMapBuilder != null) {
debugLines = sourceMapBuilder;
}
if (!outputName.endsWith(".wasm")) {
outputName += ".wasm";
}
if (sourceMapBuilder != null && sourceMapLocation != null) {
var sourceMapBinding = new WasmBinaryWriter();
sourceMapBinding.writeAsciiString(sourceMapLocation);
var sourceMapSection = new WasmCustomSection("sourceMappingURL", sourceMapBinding.getData());
module.add(sourceMapSection);
}
var binaryRenderer = new WasmBinaryRenderer(binaryWriter, WasmBinaryVersion.V_0x1, obfuscated,
null, null, debugLines, null, WasmBinaryStatsCollector.EMPTY);
@ -349,9 +377,6 @@ public class WasmGCTarget implements TeaVMTarget, TeaVMWasmGCHost {
binaryRenderer.render(module);
}
var data = binaryWriter.getData();
if (!outputName.endsWith(".wasm")) {
outputName += ".wasm";
}
try (var output = buildTarget.createResource(outputName)) {
output.write(data);
}

View File

@ -0,0 +1,59 @@
/*
* Copyright 2024 Alexey Andreev.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.teavm.backend.wasm.debug;
import org.teavm.backend.wasm.debug.sourcemap.SourceMapBuilder;
import org.teavm.model.MethodReference;
public class CompositeDebugLines implements DebugLines {
private DebugLines debugLinesBuilder;
private SourceMapBuilder sourceMapBuilder;
public CompositeDebugLines(DebugLines debugLinesBuilder, SourceMapBuilder sourceMapBuilder) {
this.debugLinesBuilder = debugLinesBuilder;
this.sourceMapBuilder = sourceMapBuilder;
}
@Override
public void advance(int ptr) {
debugLinesBuilder.advance(ptr);
sourceMapBuilder.advance(ptr);
}
@Override
public void location(String file, int line) {
debugLinesBuilder.location(file, line);
sourceMapBuilder.location(file, line);
}
@Override
public void emptyLocation() {
debugLinesBuilder.emptyLocation();
sourceMapBuilder.emptyLocation();
}
@Override
public void start(MethodReference methodReference) {
debugLinesBuilder.start(methodReference);
sourceMapBuilder.start(methodReference);
}
@Override
public void end() {
debugLinesBuilder.end();
sourceMapBuilder.end();
}
}

View File

@ -0,0 +1,205 @@
/*
* Copyright 2024 Alexey Andreev.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.teavm.backend.wasm.debug.sourcemap;
import com.carrotsearch.hppc.ObjectIntHashMap;
import com.carrotsearch.hppc.ObjectIntMap;
import java.io.IOException;
import java.io.Writer;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.List;
import java.util.Objects;
import org.teavm.backend.wasm.debug.DebugLines;
import org.teavm.common.JsonUtil;
import org.teavm.debugging.information.SourceFileResolver;
import org.teavm.model.MethodReference;
public class SourceMapBuilder implements DebugLines {
private static final String BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
private List<String> fileNames = new ArrayList<>();
private ObjectIntMap<String> fileNameIndexes = new ObjectIntHashMap<>();
private int ptr;
private StringBuilder mappings = new StringBuilder();
private String currentFile;
private int currentLine;
private String lastWrittenFile;
private int lastWrittenLine;
private boolean pendingLocation;
private Deque<InlineState> inlineStack = new ArrayDeque<>();
private List<SourceFileResolver> sourceFileResolvers = new ArrayList<>();
private String lastFileInMappings;
private int lastLineInMappings;
private int lastPtrInMappings;
public void addSourceResolver(SourceFileResolver sourceFileResolver) {
sourceFileResolvers.add(sourceFileResolver);
}
public void writeSourceMap(Writer output) throws IOException {
output.write("{\"version\":3");
var files = resolveFiles();
if (!files.isEmpty()) {
var commonPrefix = files.get(0);
var commonPrefixLength = commonPrefix.length();
for (var i = 1; i < files.size(); ++i) {
var file = files.get(i);
commonPrefixLength = Math.min(file.length(), commonPrefixLength);
for (var j = 0; j < commonPrefixLength; ++j) {
if (commonPrefix.charAt(j) != file.charAt(j)) {
commonPrefixLength = j;
break;
}
}
if (commonPrefixLength == 0) {
break;
}
}
if (commonPrefixLength > 0) {
for (var i = 0; i < files.size(); ++i) {
files.set(i, files.get(i).substring(commonPrefixLength));
}
output.write(",\"sourceRoot\":\"");
JsonUtil.writeEscapedString(output, commonPrefix.substring(0, commonPrefixLength));
output.write("\"");
}
}
output.write(",\"sources\":[");
for (int i = 0; i < files.size(); ++i) {
if (i > 0) {
output.write(',');
}
output.write("\"");
var name = files.get(i);
JsonUtil.writeEscapedString(output, name);
output.write("\"");
}
output.write("]");
output.write(",\"names\":[]");
output.write(",\"mappings\":\"");
output.write(mappings.toString());
output.write("\"}");
}
private List<String> resolveFiles() throws IOException {
var result = new ArrayList<String>();
for (var file : fileNames) {
var resolvedFile = file;
for (var resolver : sourceFileResolvers) {
var candidate = resolver.resolveFile(file);
if (candidate != null) {
resolvedFile = candidate;
break;
}
}
result.add(resolvedFile);
}
return result;
}
@Override
public void advance(int ptr) {
if (ptr != this.ptr) {
if (pendingLocation) {
pendingLocation = false;
if (!Objects.equals(currentFile, lastWrittenFile) || currentLine != lastWrittenLine) {
lastWrittenFile = currentFile;
lastWrittenLine = currentLine;
writeMapping(currentFile, currentLine, this.ptr);
}
}
this.ptr = ptr;
}
}
@Override
public void location(String file, int line) {
currentFile = file;
currentLine = line;
pendingLocation = true;
}
@Override
public void emptyLocation() {
currentLine = -1;
currentFile = null;
pendingLocation = false;
}
@Override
public void start(MethodReference methodReference) {
inlineStack.push(new InlineState(currentFile, currentLine));
}
@Override
public void end() {
var state = inlineStack.pop();
location(state.file, state.line);
}
private void writeMapping(String file, int line, int ptr) {
if (mappings.length() > 0) {
mappings.append(",");
}
writeVLQ(ptr - lastPtrInMappings);
if (file != null && line > 0) {
--line;
var lastFileIndex = fileNameIndexes.get(lastFileInMappings);
var fileIndex = fileNameIndexes.getOrDefault(file, -1);
if (fileIndex < 0) {
fileIndex = fileNames.size();
fileNames.add(file);
fileNameIndexes.put(file, fileIndex);
}
writeVLQ(fileIndex - lastFileIndex);
writeVLQ(line - lastLineInMappings);
writeVLQ(0);
lastLineInMappings = line;
lastFileInMappings = file;
}
lastPtrInMappings = ptr;
}
private void writeVLQ(int number) {
if (number < 0) {
number = ((-number) << 1) | 1;
} else {
number = number << 1;
}
do {
int digit = number & 0x1F;
int next = number >>> 5;
if (next != 0) {
digit |= 0x20;
}
mappings.append(BASE64_CHARS.charAt(digit));
number = next;
} while (number != 0);
}
private static class InlineState {
String file;
int line;
InlineState(String file, int line) {
this.file = file;
this.line = line;
}
}
}

View File

@ -43,6 +43,7 @@ import org.teavm.backend.wasm.WasmDebugInfoLocation;
import org.teavm.backend.wasm.WasmGCTarget;
import org.teavm.backend.wasm.WasmRuntimeType;
import org.teavm.backend.wasm.WasmTarget;
import org.teavm.backend.wasm.debug.sourcemap.SourceMapBuilder;
import org.teavm.backend.wasm.render.WasmBinaryVersion;
import org.teavm.cache.AlwaysStaleCacheStatus;
import org.teavm.cache.CacheStatus;
@ -65,6 +66,7 @@ import org.teavm.model.PreOptimizingClassHolderSource;
import org.teavm.model.ReferenceCache;
import org.teavm.model.transformation.AssertionRemoval;
import org.teavm.parsing.ClasspathClassHolderSource;
import org.teavm.tooling.sources.DefaultSourceFileResolver;
import org.teavm.tooling.sources.SourceFileProvider;
import org.teavm.vm.BuildTarget;
import org.teavm.vm.DirectoryBuildTarget;
@ -121,6 +123,7 @@ public class TeaVMTool {
private boolean heapDump;
private boolean shortFileNames;
private boolean assertionsRemoved;
private SourceMapBuilder wasmSourceMapWriter;
public File getTargetDirectory() {
return targetDirectory;
@ -403,6 +406,10 @@ public class TeaVMTool {
target.setDebugInfo(debugInformationGenerated);
target.setDebugInfoLevel(debugInformationGenerated ? WasmDebugInfoLevel.FULL : wasmDebugInfoLevel);
target.setDebugInfoLocation(wasmDebugInfoLocation);
if (sourceMapsFileGenerated) {
target.setSourceMapBuilder(wasmSourceMapWriter);
target.setSourceMapLocation(getResolvedTargetFileName() + ".map");
}
return target;
}
@ -527,6 +534,8 @@ public class TeaVMTool {
Writer writer = new OutputStreamWriter(output, StandardCharsets.UTF_8)) {
additionalJavaScriptOutput(writer);
}
} else if (targetType == TeaVMTargetType.WEBASSEMBLY_GC) {
additionalWasmGCOutput();
}
if (incremental) {
@ -591,42 +600,39 @@ public class TeaVMTool {
}
}
private void additionalWasmGCOutput() throws IOException {
if (sourceMapsFileGenerated) {
var targetDir = new File(targetDirectory, "src");
var resolver = new DefaultSourceFileResolver(targetDir, sourceFileProviders);
resolver.setSourceFilePolicy(sourceFilePolicy);
resolver.open();
if (sourceFilePolicy != TeaVMSourceFilePolicy.DO_NOTHING) {
wasmSourceMapWriter.addSourceResolver(resolver);
}
var file = new File(targetDirectory, getResolvedTargetFileName() + ".map");
try (var out = new FileOutputStream(file);
var writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
wasmSourceMapWriter.writeSourceMap(writer);
}
resolver.close();
}
}
private void writeSourceMaps(Writer out, DebugInformation debugInfo) throws IOException {
var sourceMapWriter = new SourceMapsWriter(out);
for (var provider : sourceFileProviders) {
provider.open();
}
var targetDir = new File(targetDirectory, "src");
var resolver = new DefaultSourceFileResolver(targetDir, sourceFileProviders);
resolver.setSourceFilePolicy(sourceFilePolicy);
resolver.open();
if (sourceFilePolicy != TeaVMSourceFilePolicy.DO_NOTHING) {
sourceMapWriter.addSourceResolver(fileName -> {
for (var provider : sourceFileProviders) {
var sourceFile = provider.getSourceFile(fileName);
if (sourceFile != null) {
if (sourceFilePolicy == TeaVMSourceFilePolicy.COPY || sourceFile.getFile() == null) {
var outputFile = new File(targetDir, fileName);
outputFile.getParentFile().mkdirs();
try (var input = sourceFile.open();
var output = new FileOutputStream(outputFile)) {
input.transferTo(output);
}
if (sourceFilePolicy == TeaVMSourceFilePolicy.LINK_LOCAL_FILES) {
return "file://" + outputFile.getCanonicalPath();
}
} else {
return "file://" + sourceFile.getFile().getCanonicalPath();
}
break;
}
}
return null;
});
sourceMapWriter.addSourceResolver(resolver);
}
sourceMapWriter.write(getResolvedTargetFileName(), "src", debugInfo);
for (var provider : sourceFileProviders) {
provider.close();
}
resolver.close();
}
private void printStats() {

View File

@ -0,0 +1,74 @@
/*
* Copyright 2024 Alexey Andreev.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.teavm.tooling.sources;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import org.teavm.debugging.information.SourceFileResolver;
import org.teavm.tooling.TeaVMSourceFilePolicy;
public class DefaultSourceFileResolver implements SourceFileResolver {
private File targetDir;
private List<SourceFileProvider> sourceFileProviders;
private TeaVMSourceFilePolicy sourceFilePolicy = TeaVMSourceFilePolicy.DO_NOTHING;
public DefaultSourceFileResolver(File targetDir, List<SourceFileProvider> sourceFileProviders) {
this.targetDir = targetDir;
this.sourceFileProviders = sourceFileProviders;
}
public void setSourceFilePolicy(TeaVMSourceFilePolicy sourceFilePolicy) {
this.sourceFilePolicy = sourceFilePolicy;
}
public void open() throws IOException {
for (var provider : sourceFileProviders) {
provider.open();
}
}
@Override
public String resolveFile(String file) throws IOException {
for (var provider : sourceFileProviders) {
var sourceFile = provider.getSourceFile(file);
if (sourceFile != null) {
if (sourceFilePolicy == TeaVMSourceFilePolicy.COPY || sourceFile.getFile() == null) {
var outputFile = new File(targetDir, file);
outputFile.getParentFile().mkdirs();
try (var input = sourceFile.open();
var output = new FileOutputStream(outputFile)) {
input.transferTo(output);
}
if (sourceFilePolicy == TeaVMSourceFilePolicy.LINK_LOCAL_FILES) {
return "file://" + outputFile.getCanonicalPath();
}
} else {
return "file://" + sourceFile.getFile().getCanonicalPath();
}
break;
}
}
return null;
}
public void close() throws IOException {
for (var provider : sourceFileProviders) {
provider.close();
}
}
}

View File

@ -56,7 +56,7 @@ abstract class BaseWebAssemblyPlatformSupport extends TestPlatformSupport<WasmTa
if (sourceDirs != null) {
var dirs = new ArrayList<File>();
for (var tokenizer = new StringTokenizer(sourceDirs, Character.toString(File.pathSeparatorChar));
tokenizer.hasMoreTokens();) {
tokenizer.hasMoreTokens();) {
var dir = new File(tokenizer.nextToken());
if (dir.isDirectory()) {
dirs.add(dir);

View File

@ -124,16 +124,18 @@ abstract class TestPlatformSupport<T extends TeaVMTarget> {
}
protected final File getOutputFile(File path, String baseName, String suffix, String extension) {
return new File(path, getOutputSimpleNameFile(baseName, suffix, extension));
}
protected final String getOutputSimpleNameFile(String baseName, String suffix, String extension) {
StringBuilder simpleName = new StringBuilder();
simpleName.append(baseName);
if (!suffix.isEmpty()) {
simpleName.append('-').append(suffix);
}
File outputFile;
simpleName.append(extension);
outputFile = new File(path, simpleName.toString());
return outputFile;
return simpleName.toString();
}
private String buildErrorMessage(TeaVM vm) {

View File

@ -15,6 +15,7 @@
*/
package org.teavm.junit;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.teavm.junit.PropertyNames.OPTIMIZED;
import static org.teavm.junit.PropertyNames.SOURCE_DIRS;
import static org.teavm.junit.PropertyNames.WASM_GC_ENABLED;
@ -35,20 +36,39 @@ import java.util.function.Supplier;
import org.teavm.backend.wasm.WasmDebugInfoLevel;
import org.teavm.backend.wasm.WasmDebugInfoLocation;
import org.teavm.backend.wasm.WasmGCTarget;
import org.teavm.backend.wasm.debug.sourcemap.SourceMapBuilder;
import org.teavm.backend.wasm.disasm.Disassembler;
import org.teavm.backend.wasm.disasm.DisassemblyHTMLWriter;
import org.teavm.browserrunner.BrowserRunner;
import org.teavm.model.ClassHolderSource;
import org.teavm.model.MethodReference;
import org.teavm.model.ReferenceCache;
import org.teavm.tooling.TeaVMSourceFilePolicy;
import org.teavm.tooling.sources.DefaultSourceFileResolver;
import org.teavm.tooling.sources.DirectorySourceFileProvider;
import org.teavm.tooling.sources.JarSourceFileProvider;
import org.teavm.tooling.sources.SourceFileProvider;
import org.teavm.vm.TeaVM;
class WebAssemblyGCPlatformSupport extends TestPlatformSupport<WasmGCTarget> {
private boolean disassembly;
private List<SourceFileProvider> sourceFileProviders = new ArrayList<>();
WebAssemblyGCPlatformSupport(ClassHolderSource classSource, ReferenceCache referenceCache, boolean disassembly) {
super(classSource, referenceCache);
this.disassembly = disassembly;
var sourceDirs = System.getProperty(SOURCE_DIRS);
if (sourceDirs != null) {
for (var tokenizer = new StringTokenizer(sourceDirs, Character.toString(File.pathSeparatorChar));
tokenizer.hasMoreTokens();) {
var file = new File(tokenizer.nextToken());
if (file.isDirectory()) {
sourceFileProviders.add(new DirectorySourceFileProvider(file));
} else if (file.isFile() && file.getName().endsWith(".jar")) {
sourceFileProviders.add(new JarSourceFileProvider(file));
}
}
}
}
@Override
@ -67,6 +87,8 @@ class WebAssemblyGCPlatformSupport extends TestPlatformSupport<WasmGCTarget> {
@Override
CompileResult compile(Consumer<TeaVM> additionalProcessing, String baseName,
TeaVMTestConfiguration<WasmGCTarget> configuration, File path, AnnotatedElement element) {
var sourceMapBuilder = new SourceMapBuilder();
var sourceMapFile = getOutputFile(path, baseName, configuration.getSuffix(), ".wasm.map");
Supplier<WasmGCTarget> targetSupplier = () -> {
var target = new WasmGCTarget();
target.setObfuscated(false);
@ -74,6 +96,8 @@ class WebAssemblyGCPlatformSupport extends TestPlatformSupport<WasmGCTarget> {
target.setDebugInfo(true);
target.setDebugInfoLevel(WasmDebugInfoLevel.DEOBFUSCATION);
target.setDebugInfoLocation(WasmDebugInfoLocation.EMBEDDED);
target.setSourceMapBuilder(sourceMapBuilder);
target.setSourceMapLocation(getOutputSimpleNameFile(baseName, configuration.getSuffix(), ".wasm.map"));
var sourceDirs = System.getProperty(SOURCE_DIRS);
if (sourceDirs != null) {
var dirs = new ArrayList<File>();
@ -87,8 +111,22 @@ class WebAssemblyGCPlatformSupport extends TestPlatformSupport<WasmGCTarget> {
}
return target;
};
CompilePostProcessor postBuild = (vm, file) -> {
var resolver = new DefaultSourceFileResolver(new File(path, "src"), sourceFileProviders);
resolver.setSourceFilePolicy(TeaVMSourceFilePolicy.LINK_LOCAL_FILES);
sourceMapBuilder.addSourceResolver(resolver);
try {
resolver.open();
try (var sourceMapOut = new OutputStreamWriter(new FileOutputStream(sourceMapFile), UTF_8)) {
sourceMapBuilder.writeSourceMap(sourceMapOut);
}
resolver.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
};
return compile(configuration, targetSupplier, TestWasmGCEntryPoint.class.getName(), path,
".wasm", null, additionalProcessing, baseName);
".wasm", postBuild, additionalProcessing, baseName);
}
@Override