Skip to content

Commit

Permalink
[WASM] Implement modular string literals.
Browse files Browse the repository at this point in the history
The bundler now needs to instantiate a frontend to be able to construct AST to synthesize methods. In contrast to the exports generator that can run concurrently with transpilation (since it only needs the class jars), to synthesize getters the bundler relies on transpilation to produce summaries.

PiperOrigin-RevId: 590001386
  • Loading branch information
rluble authored and copybara-github committed Dec 12, 2023
1 parent c4c1052 commit 070e29d
Show file tree
Hide file tree
Showing 17 changed files with 339 additions and 196 deletions.
4 changes: 3 additions & 1 deletion build_defs/internal_do_not_use/j2wasm_application.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -115,14 +115,16 @@ def _impl_j2wasm_application(ctx):
)

all_modules = module_outputs.to_list() + [exports_module_output]
jre_jars = ctx.attr._jre[J2wasmInfo]._private_.java_info.compile_jars.to_list()

# Bundle the module outputs.
bundler_args = ctx.actions.args()
bundler_args.add_all(all_modules, expand_directories = False)
bundler_args.add_joined("-classpath", jre_jars, join_with = ctx.configuration.host_path_separator)
bundler_args.add("-output", ctx.outputs.bundle)
ctx.actions.run(
progress_message = "Bundling modules for Wasm %s" % ctx.label,
inputs = all_modules,
inputs = all_modules + jre_jars,
outputs = [ctx.outputs.bundle],
executable = ctx.executable._bundler,
arguments = [bundler_args],
Expand Down
5 changes: 5 additions & 0 deletions transpiler/java/com/google/j2cl/transpiler/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,14 @@ java_library(
"//third_party:args4j",
"//third_party:auto_value",
"//third_party:guava",
# TODO(b/294284380): Make this independent of the frontend.
"//third_party:jdt-core",
"//transpiler/java/com/google/j2cl/common",
"//transpiler/java/com/google/j2cl/common/bazel",
"//transpiler/java/com/google/j2cl/transpiler/ast",
"//transpiler/java/com/google/j2cl/transpiler/backend/wasm",
"//transpiler/java/com/google/j2cl/transpiler/backend/wasm:summary_java_proto",
"//transpiler/java/com/google/j2cl/transpiler/frontend/jdt",
],
)

Expand Down
124 changes: 123 additions & 1 deletion transpiler/java/com/google/j2cl/transpiler/BazelJ2wasmBundler.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,37 @@
import static java.lang.String.format;
import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Streams;
import com.google.common.io.Files;
import com.google.j2cl.common.Problems;
import com.google.j2cl.common.Problems.FatalError;
import com.google.j2cl.common.SourcePosition;
import com.google.j2cl.common.bazel.BazelWorker;
import com.google.j2cl.common.bazel.FileCache;
import com.google.j2cl.transpiler.ast.CompilationUnit;
import com.google.j2cl.transpiler.ast.Library;
import com.google.j2cl.transpiler.ast.Method;
import com.google.j2cl.transpiler.ast.MethodCall;
import com.google.j2cl.transpiler.ast.MethodDescriptor;
import com.google.j2cl.transpiler.ast.ReturnStatement;
import com.google.j2cl.transpiler.ast.StringLiteral;
import com.google.j2cl.transpiler.ast.StringLiteralGettersCreator;
import com.google.j2cl.transpiler.ast.TypeDeclaration;
import com.google.j2cl.transpiler.ast.TypeDeclaration.Kind;
import com.google.j2cl.transpiler.ast.TypeDescriptors;
import com.google.j2cl.transpiler.backend.wasm.Summary;
import com.google.j2cl.transpiler.backend.wasm.TypeHierarchyInfo;
import com.google.j2cl.transpiler.backend.wasm.WasmGeneratorStage;
import com.google.j2cl.transpiler.frontend.jdt.JdtEnvironment;
import com.google.j2cl.transpiler.frontend.jdt.JdtParser;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashMap;
Expand Down Expand Up @@ -66,6 +83,13 @@ final class BazelJ2wasmBundler extends BazelWorker {
usage = "Directory or zip into which to place the bundled output.")
Path output;

@Option(
name = "-classpath",
required = true,
metaVar = "<path>",
usage = "Specifies where to find all the class files for the application.")
String classPath;

@Override
protected void run(Problems problems) {
createBundle(problems);
Expand All @@ -75,6 +99,30 @@ private void createBundle(Problems problems) {
var typeGraph = new TypeGraph();
var classes = typeGraph.build(getSummaries());

// Create an environment to initialize the well known type descriptors to be able to synthesize
// code.
// TODO(b/294284380): consider removing JDT and manually synthesizing required types.
var classPathEntries = Splitter.on(File.pathSeparatorChar).splitToList(this.classPath);
new JdtEnvironment(
new JdtParser(classPathEntries, problems), TypeDescriptors.getWellKnownTypeNames());

// Synthesize globals and methods for string literals.
var stringLiteralsCompilationUnit = synthesizeStringLiteralGetters();

var library =
Library.newBuilder()
.setCompilationUnits(ImmutableList.of(stringLiteralsCompilationUnit))
.build();

var generatorStage = new WasmGeneratorStage(library, problems);

Stream<String> literalGetterMethods =
stringLiteralsCompilationUnit.getTypes().stream()
.flatMap(t -> t.getMethods().stream())
.map(m -> generatorStage.emitToString(g -> g.renderMethod(m)));

String literalGlobals = generatorStage.emitToString(g -> g.emitGlobals(library));

ImmutableList<String> moduleContents =
Streams.concat(
Stream.of("(rec"),
Expand All @@ -85,12 +133,86 @@ private void createBundle(Problems problems) {
getModuleParts("data"),
getModuleParts("globals"),
classes.stream().map(TypeGraph.Type::getItableInitialization),
getModuleParts("functions"))
Stream.of(literalGlobals),
getModuleParts("functions"),
literalGetterMethods)
.collect(toImmutableList());

writeToFile(output.toString(), moduleContents, problems);
}

private CompilationUnit synthesizeStringLiteralGetters() {

var compilationUnit = CompilationUnit.createSynthetic("wasm.stringliterals");

var stringLiteralHolder =
new com.google.j2cl.transpiler.ast.Type(
SourcePosition.NONE,
TypeDeclaration.newBuilder()
.setClassComponents(
ImmutableList.of("wasm", "stringLiteral", "StringLiteralHolder"))
.setKind(Kind.CLASS)
.build());

var stringLiteralGetterCreator = new StringLiteralGettersCreator();
Map<TypeDeclaration, com.google.j2cl.transpiler.ast.Type> typesByDeclaration =
new LinkedHashMap<>();
getSummaries()
.flatMap(s -> s.getStringLiteralList().stream())
.forEach(
s -> {
// Get descriptor for the getter and synthesize the method logic if it is the
// first time it was found.
MethodDescriptor m =
stringLiteralGetterCreator.getOrCreateLiteralMethod(
stringLiteralHolder,
new StringLiteral(s.getLiteral()),
/* synthesizeMethod= */ true);

// Synthesize the forwarding logic.
String qualifiedBinaryName = s.getEnclosingTypeName();
TypeDeclaration typeDeclaration =
TypeDeclaration.newBuilder()
.setClassComponents(Arrays.asList(qualifiedBinaryName.split("\\.")))
.setKind(Kind.CLASS)
.build();
var type =
typesByDeclaration.computeIfAbsent(
typeDeclaration,
t -> {
var newType =
new com.google.j2cl.transpiler.ast.Type(SourcePosition.NONE, t);
compilationUnit.addType(newType);
return newType;
});

Method forwarderMethod =
synthesizeForwardingMethod(m, typeDeclaration, s.getMethodName());
type.addMember(forwarderMethod);
});
return compilationUnit;
}

private static Method synthesizeForwardingMethod(
MethodDescriptor literalGetter, TypeDeclaration fromType, String forwardingMethodName) {
MethodDescriptor forwarderDescriptor =
MethodDescriptor.newBuilder()
.setEnclosingTypeDescriptor(fromType.toUnparameterizedTypeDescriptor())
.setName(forwardingMethodName)
.setStatic(true)
.setReturnTypeDescriptor(TypeDescriptors.get().javaLangString)
.build();
return Method.newBuilder()
.setMethodDescriptor(forwarderDescriptor)
.setStatements(
ReturnStatement.newBuilder()
.setExpression(MethodCall.Builder.from(literalGetter).build())
.setSourcePosition(SourcePosition.NONE)
.build())
.setSourcePosition(SourcePosition.NONE)
.build();
}

/** Represents the inheritance structure of the whole application. */
private static class TypeGraph {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ protected void run(Problems problems) {
.flatMap(t -> t.getDeclaredMethodDescriptors().stream())
.collect(toImmutableList()));

WasmGeneratorStage.generateMethods(exportedMethods, out, problems);
WasmGeneratorStage.generateWasmExportMethods(exportedMethods, out, problems);
problems.abortIfHasErrors();
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/*
* Copyright 2021 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.google.j2cl.transpiler.ast;

import com.google.common.base.Ascii;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.Multiset;
import com.google.j2cl.transpiler.ast.MethodDescriptor.MethodOrigin;
import java.util.HashMap;
import java.util.Map;

/** Implements lazy initialization of String literals. */
public class StringLiteralGettersCreator {

private final Map<String, MethodDescriptor> literalMethodByString = new HashMap<>();

public Map<String, MethodDescriptor> getLiteralMethodByString() {
return literalMethodByString;
}

/**
* Returns the descriptor for the getter of {@code stringLiteral}, creating it if it did not
* exist.
*/
public MethodDescriptor getOrCreateLiteralMethod(
Type type, StringLiteral stringLiteral, boolean synthesizeMethod) {
String value = stringLiteral.getValue();
if (literalMethodByString.containsKey(value)) {
return literalMethodByString.get(value);
}

String snippet = createSnippet(type, value);
MethodDescriptor getLiteralMethod =
getLazyStringLiteralGettterMethodDescriptor(
type.getTypeDescriptor(), "$getString_" + snippet);

if (synthesizeMethod) {
type.synthesizeLazilyInitializedField(
"$string_" + snippet, synthesizeStringCreation(stringLiteral), getLiteralMethod);
}

literalMethodByString.put(value, getLiteralMethod);

return getLiteralMethod;
}

/** Returns the descriptor for the getter of the string literal. */
private static MethodDescriptor getLazyStringLiteralGettterMethodDescriptor(
DeclaredTypeDescriptor enclosingTypeDescriptor, String name) {
return MethodDescriptor.newBuilder()
.setName(name)
.setReturnTypeDescriptor(TypeDescriptors.get().javaLangString)
.setEnclosingTypeDescriptor(enclosingTypeDescriptor)
.setOrigin(MethodOrigin.SYNTHETIC_STRING_LITERAL_GETTER)
.setStatic(true)
.setSynthetic(true)
.setSideEffectFree(true)
.build();
}

private final Map<Type, Multiset<String>> snippetsByType = new HashMap<>();

private String createSnippet(Type type, String value) {
// Take the first few characters of the string and remove invalid identifier characters.
String prefix = Ascii.truncate(value, 15, "...").replaceAll("[^A-Za-z0-9.]", "_");

var typeSnippets = snippetsByType.computeIfAbsent(type, t -> HashMultiset.create());
int occurrences = typeSnippets.count(prefix);
typeSnippets.add(prefix);
if (occurrences > 0) {
return String.format("|%s|_%d", prefix, occurrences);
} else {
return String.format("|%s|", prefix);
}
}

/**
* Converts the StringLiteral into a call to the runtime to initialize create a String from a char
* array.
*/
private static Expression synthesizeStringCreation(StringLiteral stringLiteral) {
ArrayTypeDescriptor charArrayDescriptor =
ArrayTypeDescriptor.newBuilder().setComponentTypeDescriptor(PrimitiveTypes.CHAR).build();
MethodDescriptor fromInternalArray =
TypeDescriptors.get()
.javaLangString
.getMethodDescriptor("fromInternalArray", charArrayDescriptor);
if (fromInternalArray != null) {
// TODO(b/272381112): Remove after non-stringref experiment.
// This is the non-stringref j.l.String.
return MethodCall.Builder.from(fromInternalArray)
.setArguments(new ArrayLiteral(charArrayDescriptor, stringLiteral.toCharLiterals()))
.build();
}
return RuntimeMethods.createStringFromJsStringMethodCall(stringLiteral);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,6 @@ public ImmutableList<Supplier<NormalizationPass>> getPassFactories(BackendOption
NormalizeInstantiationThroughFactoryMethods::new,
ImplementStaticInitializationViaConditionChecks::new,
ImplementClassMetadataViaGetters::new,
ImplementStringCompileTimeConstants::new,
NormalizeArrayCreationsWasm::new,
InsertCastOnArrayAccess::new,
options.getWasmRemoveAssertStatement()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,13 @@

import com.google.j2cl.common.Problems;
import com.google.j2cl.common.Problems.FatalError;
import com.google.j2cl.transpiler.ast.AbstractRewriter;
import com.google.j2cl.transpiler.ast.DeclaredTypeDescriptor;
import com.google.j2cl.transpiler.ast.Expression;
import com.google.j2cl.transpiler.ast.Library;
import com.google.j2cl.transpiler.ast.MethodCall;
import com.google.j2cl.transpiler.ast.StringLiteral;
import com.google.j2cl.transpiler.ast.StringLiteralGettersCreator;
import com.google.j2cl.transpiler.ast.Type;
import com.google.protobuf.util.JsonFormat;
import java.io.IOException;
Expand All @@ -44,6 +49,11 @@ public final class SummaryBuilder {
SummaryBuilder(Library library, WasmGenerationEnvironment environment, Problems problems) {
this.environment = environment;

summaryTypeHierarchy(library);
summarizeStringLiterals(library);
}

private void summaryTypeHierarchy(Library library) {
library
.streamTypes()
.sorted(Comparator.comparing(t -> t.getDeclaration().getClassHierarchyDepth()))
Expand Down Expand Up @@ -79,6 +89,34 @@ private int getDeclaredTypeId(DeclaredTypeDescriptor typeDescriptor) {
return declaredTypes.computeIfAbsent(typeName, x -> declaredTypes.size() + 1);
}

private void summarizeStringLiterals(Library library) {
// Replace stringliterals with the name of the literal getter method that will be synthesized
// by the bundler.
var stringLiteralGetterCreator = new StringLiteralGettersCreator();
library.accept(
new AbstractRewriter() {
@Override
public Expression rewriteStringLiteral(StringLiteral stringLiteral) {
return MethodCall.Builder.from(
stringLiteralGetterCreator.getOrCreateLiteralMethod(
getCurrentType(), stringLiteral, /* synthesizeMethod= */ false))
.build();
}
});

stringLiteralGetterCreator
.getLiteralMethodByString()
.forEach(
(s, m) ->
summary.addStringLiteral(
StringLiteralInfo.newBuilder()
.setLiteral(s)
.setEnclosingTypeName(
m.getEnclosingTypeDescriptor().getQualifiedBinaryName())
.setMethodName(m.getName())
.build()));
}

private Summary build() {
summary.clearTypeReferenceMap();
String[] typeReferenceMap = new String[types.size()];
Expand Down
Loading

0 comments on commit 070e29d

Please sign in to comment.