diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/CreateExecutableTask.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/CreateExecutableTask.java index 63598406879c..ffe2901a2d71 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/CreateExecutableTask.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/CreateExecutableTask.java @@ -146,13 +146,11 @@ public void execute(Project project) { if (project.buildOptions().getTargetPath() != null) { this.out.println("\t" + relativePathToExecutableString); + } else if (relativePathToExecutableString.contains("..") + || relativePathToExecutableString.contains("." + File.separator)) { + this.out.println("\t" + executablePathString); } else { - if (relativePathToExecutableString.contains("..") - || relativePathToExecutableString.contains("." + File.separator)) { - this.out.println("\t" + executablePathString); - } else { - this.out.println("\t" + relativePathToExecutableString); - } + this.out.println("\t" + relativePathToExecutableString); } } diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/TestUtils.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/TestUtils.java index ef2afb050ca4..49847a11db06 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/TestUtils.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/TestUtils.java @@ -480,11 +480,10 @@ private static String getFunctionToMockClassName(String id) { public static String getClassPath(JBallerinaBackend jBallerinaBackend, Package currentPackage) { JarResolver jarResolver = jBallerinaBackend.jarResolver(); - List dependencies = getTestDependencyPaths(currentPackage, jarResolver); - List jarList = getModuleJarPaths(jBallerinaBackend, currentPackage); - dependencies.removeAll(jarList); - dependencies.removeAll(jarResolver.optimizedJarLibraryPaths); + List dependencies = getTestDependencyPaths(currentPackage, jarResolver).stream() + .filter(each -> !jarList.contains(each) && !jarResolver.optimizedJarLibraryPaths.contains(each)) + .toList(); StringJoiner classPath = joinClassPaths(dependencies); return classPath.toString(); diff --git a/compiler/ballerina-lang/src/main/java/io/ballerina/projects/JBallerinaBackend.java b/compiler/ballerina-lang/src/main/java/io/ballerina/projects/JBallerinaBackend.java index d0655f485a51..640d77aa1c1c 100644 --- a/compiler/ballerina-lang/src/main/java/io/ballerina/projects/JBallerinaBackend.java +++ b/compiler/ballerina-lang/src/main/java/io/ballerina/projects/JBallerinaBackend.java @@ -131,10 +131,10 @@ public class JBallerinaBackend extends CompilerBackend { private final UsedBIRNodeAnalyzer usedBIRNodeAnalyzer; private final CodeGenOptimizationReportEmitter codeGenOptimizationReportEmitter; private final Map optimizedJarStreams; - protected final Set unusedCompilerLevelPackageIds; - protected final Set unusedProjectLevelPackageIds; - protected final Set unusedModuleIds; - protected final Map> pkgWiseUsedNativeClassPaths; + private final Set unusedCompilerLevelPackageIds; + final Set unusedProjectLevelPackageIds; + final Set unusedModuleIds; + final Map> pkgWiseUsedNativeClassPaths; public static JBallerinaBackend from(PackageCompilation packageCompilation, JvmTarget jdkVersion) { return from(packageCompilation, jdkVersion, true); @@ -231,8 +231,7 @@ private void performCodeGen(boolean shrink) { ModuleContext.shrinkDocuments(moduleContext); } // Codegen happens later when --optimize flag is active. We cannot clean the BlangPkgs until then. - if (!project.buildOptions().optimizeCodegen() && - project.kind() == ProjectKind.BALA_PROJECT) { + if (!project.buildOptions().optimizeCodegen() && project.kind() == ProjectKind.BALA_PROJECT) { moduleContext.cleanBLangPackage(); } } @@ -303,6 +302,7 @@ private void markCommonDependencies(BLangPackage bLangPackage) { collectDependencies(bLangPackage.symbol, buildPkgDependencies); collectDependencies(bLangPackage.getTestablePkg().symbol, testablePkgDependencies); + // FIXME: this filter should be applied only if env variable is not set? buildPkgDependencies.stream() .filter(testablePkgDependencies::contains).filter(pkgSymbol -> !isWhiteListedModule(pkgSymbol.pkgID)) .forEach(pkgSymbol -> { diff --git a/compiler/ballerina-lang/src/main/java/io/ballerina/projects/JarResolver.java b/compiler/ballerina-lang/src/main/java/io/ballerina/projects/JarResolver.java index d00274aea53e..0f10eb2b1dc4 100644 --- a/compiler/ballerina-lang/src/main/java/io/ballerina/projects/JarResolver.java +++ b/compiler/ballerina-lang/src/main/java/io/ballerina/projects/JarResolver.java @@ -20,6 +20,7 @@ import io.ballerina.projects.internal.DefaultDiagnosticResult; import io.ballerina.projects.internal.PackageDiagnostic; import io.ballerina.projects.internal.ProjectDiagnosticErrorCode; +import io.ballerina.projects.util.CodegenOptimizationUtils; import io.ballerina.projects.util.ProjectConstants; import io.ballerina.projects.util.ProjectUtils; import io.ballerina.tools.diagnostics.Diagnostic; @@ -48,10 +49,6 @@ import java.util.zip.ZipEntry; import static io.ballerina.identifier.Utils.encodeNonFunctionIdentifier; -import static io.ballerina.projects.util.CodegenOptimizationConstants.BALLERINAI_OBSERVE; -import static io.ballerina.projects.util.CodegenOptimizationConstants.BALLERINAX; -import static io.ballerina.projects.util.CodegenOptimizationConstants.BALLERINA_OBSERVE; -import static io.ballerina.projects.util.CodegenOptimizationConstants.DOT_DRIVER; import static io.ballerina.projects.util.ProjectConstants.ANON_ORG; import static io.ballerina.projects.util.ProjectConstants.DOT; @@ -264,7 +261,7 @@ private static boolean hasEmptyIdOrVersion(JarLibrary entry) { private boolean isUsedDependency(JarLibrary otherJarDependency, Set usedNativeClassPaths) { String pkgName = otherJarDependency.packageName().orElseThrow(); - if (isWhiteListedPkg(pkgName)) { + if (CodegenOptimizationUtils.isWhiteListedModule(pkgName)) { return true; } if (usedNativeClassPaths.isEmpty()) { @@ -283,16 +280,6 @@ private boolean isUsedDependency(JarLibrary otherJarDependency, Set used } } - private boolean isWhiteListedPkg(String pkgName) { - return pkgName.equals(BALLERINA_OBSERVE) || pkgName.equals(BALLERINAI_OBSERVE) || isDriverPkg(pkgName); - } - - // Driver pkgs are pkgs such as "ballerinax/mysql.driver". - // These pkgs contain only native jars without any source code. - private boolean isDriverPkg(String pkgName) { - return pkgName.startsWith(BALLERINAX) && pkgName.endsWith(DOT_DRIVER); - } - private void reportDiagnostic(JarLibrary existingEntry, JarLibrary newEntry) { // Report diagnostic only for non ballerina dependencies if (!existingEntry.packageName().orElseThrow().startsWith(ProjectConstants.BALLERINA_ORG) diff --git a/compiler/ballerina-lang/src/main/java/io/ballerina/projects/UsedBIRNodeAnalyzer.java b/compiler/ballerina-lang/src/main/java/io/ballerina/projects/UsedBIRNodeAnalyzer.java index f52243bbcd83..51989e5ee6d1 100644 --- a/compiler/ballerina-lang/src/main/java/io/ballerina/projects/UsedBIRNodeAnalyzer.java +++ b/compiler/ballerina-lang/src/main/java/io/ballerina/projects/UsedBIRNodeAnalyzer.java @@ -17,6 +17,7 @@ */ package io.ballerina.projects; +import io.ballerina.projects.util.CodegenOptimizationUtils; import org.ballerinalang.model.elements.PackageID; import org.ballerinalang.model.types.TypeKind; import org.wso2.ballerinalang.compiler.PackageCache; @@ -34,11 +35,9 @@ import org.wso2.ballerinalang.compiler.util.CompilerContext; import org.wso2.ballerinalang.util.Flags; -import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; @@ -46,14 +45,14 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; -import static io.ballerina.projects.util.CodegenOptimizationConstants.BALLERINA_JBALLERINA_JAVA; import static io.ballerina.projects.util.CodegenOptimizationConstants.CLASS; import static io.ballerina.projects.util.CodegenOptimizationConstants.DOT_CLASS; import static io.ballerina.projects.util.CodegenOptimizationConstants.EXTERNAL_DEPENDENCY_ANNOT; -import static io.ballerina.projects.util.CodegenOptimizationConstants.INTEROP_DEPENDENCIES_PROPERTIES_FILE; +import static io.ballerina.projects.util.CodegenOptimizationUtils.isJBallerinaModule; +import static io.ballerina.projects.util.CodegenOptimizationUtils.isPackageWithWhiteListedFiles; +import static io.ballerina.projects.util.CodegenOptimizationUtils.isWhiteListedFile; import static org.wso2.ballerinalang.compiler.util.Constants.RECORD_DELIMITER; /** @@ -66,22 +65,21 @@ public final class UsedBIRNodeAnalyzer extends BIRVisitor { private static final CompilerContext.Key USED_BIR_NODE_ANALYZER_KEY = new CompilerContext.Key<>(); private static final Set USED_FUNCTION_NAMES = - new HashSet<>(Arrays.asList("main", ".", ".", ".", "__execute__")); + Set.of("main", ".", ".", ".", "__execute__"); private static final Map> INTEROP_DEPENDENCIES = new HashMap<>(); - private static final Set ANALYZED_INSTRUCTION_KINDS = new HashSet<>( - Arrays.asList(InstructionKind.NEW_TYPEDESC, InstructionKind.NEW_INSTANCE, InstructionKind.TYPE_CAST, + private static final Set ANALYZED_INSTRUCTION_KINDS = + Set.of(InstructionKind.NEW_TYPEDESC, InstructionKind.NEW_INSTANCE, InstructionKind.TYPE_CAST, InstructionKind.FP_LOAD, InstructionKind.TYPE_TEST, InstructionKind.RECORD_DEFAULT_FP_LOAD, InstructionKind.NEW_TABLE, InstructionKind.NEW_ARRAY, InstructionKind.MOVE, - InstructionKind.NEW_ERROR)); + InstructionKind.NEW_ERROR); private static final Set ANALYZED_TERMINATOR_KINDS = - new HashSet<>(Arrays.asList(InstructionKind.CALL, InstructionKind.FP_CALL)); + Set.of(InstructionKind.CALL, InstructionKind.FP_CALL); private static final String EXTERNAL_METHOD_ANNOTATION_TAG = "Method"; private static final String EXECUTE_TEST_REGISTRAR = "executeTestRegistrar"; // To check whether a given FP is "USED" or not, we have to keep track of the existing FPs of a given scope. // Variable Declarations holding an FPs are needed to be tracked to achieve that. private Map localFpHolders = new HashMap<>(); - // pkgWiseInvocationData is used for debugging purposes - public final Map pkgWiseInvocationData = new LinkedHashMap<>(); + final Map pkgWiseInvocationData = new LinkedHashMap<>(); UsedBIRNodeAnalyzer.InvocationData currentInvocationData; boolean isTestablePkgAnalysis = false; PackageID currentPkgID; @@ -412,22 +410,11 @@ public InvocationData getInvocationData(PackageID pkgId) { } private void initInteropDependencies() { - Properties prop = new Properties(); - try { - InputStream stream = getClass().getClassLoader().getResourceAsStream(INTEROP_DEPENDENCIES_PROPERTIES_FILE); - prop.load(stream); - for (Map.Entry entry : prop.entrySet()) { - Set usedRecordNames = new HashSet<>(Arrays.asList(entry.getValue().toString().split(","))); - INTEROP_DEPENDENCIES.putIfAbsent(entry.getKey().toString(), usedRecordNames); - } - - } catch (IOException e) { - throw new RuntimeException("Failed to load interop-dependencies : ", e); - } + INTEROP_DEPENDENCIES.putAll(CodegenOptimizationUtils.getHardcodedDependencies()); } - private HashSet getVarDeclarations(BIRNonTerminator instruction) { - HashSet rhsVars = new HashSet<>(); + private Set getVarDeclarations(BIRNonTerminator instruction) { + Set rhsVars = new HashSet<>(); for (BIROperand rhsOperand : instruction.getRhsOperands()) { rhsVars.add(rhsOperand.variableDcl); } @@ -469,7 +456,7 @@ private void registerUsedNativeClassPaths(BIRNode.BIRFunction birFunction) { if (annot.annotTagRef.value.equals(EXTERNAL_METHOD_ANNOTATION_TAG)) { BIRNode.ConstValue constValue = ((BIRNode.BIRConstAnnotationAttachment) annot).annotValue; String filePath = - ((HashMap) constValue.value).get(CLASS).value.toString(); + ((Map) constValue.value).get(CLASS).value.toString(); currentInvocationData.usedNativeClassPaths.add(filePath.replace(".", "/") + DOT_CLASS); } }); @@ -499,32 +486,28 @@ private void addDependentFunctionAndVisit(BIRNode.BIRDocumentableNode parentNode // Child is from another package childInvocationData.moduleIsUsed = true; childInvocationData.startPointNodes.add(childFunction); - currentInvocationData.childPackages.add(childFunction.getPackageID()); } public static class InvocationData { // Following Sets are used to whitelist the bal types called through ValueCreator in native dependencies - private static final Set WHITELSITED_FILE_NAMES = - new HashSet<>(Arrays.asList("types.bal", "error.bal", "stream_types.bal")); - private static final Set PKGS_WITH_WHITELSITED_FILES = new HashSet<>( - Arrays.asList("ballerinax/mysql", "ballerina/sql", "ballerinax/persist.sql")); + private static final Set WHITELSITED_FILE_NAMES = Set.of("types.bal", "error.bal", "stream_types.bal"); + private static final Set PKGS_WITH_WHITELSITED_FILES = + Set.of("ballerinax/mysql", "ballerina/sql", "ballerinax/persist.sql"); private static final String BALLERINA_TEST_PKG_NAME = "ballerina/test:0.0.0"; private static final String MOCK_FUNCTION_PREFIX = "$MOCK_"; - protected final Set usedFunctions = new LinkedHashSet<>(); - protected final Set unusedFunctions = new LinkedHashSet<>(); - protected final Set usedTypeDefs = new LinkedHashSet<>(); - protected final Set unusedTypeDefs = new LinkedHashSet<>(); - protected final Set startPointNodes = new LinkedHashSet<>(); + final Set usedFunctions = new LinkedHashSet<>(); + final Set unusedFunctions = new LinkedHashSet<>(); + final Set usedTypeDefs = new LinkedHashSet<>(); + final Set unusedTypeDefs = new LinkedHashSet<>(); + final Set startPointNodes = new LinkedHashSet<>(); private final Map functionPool = new HashMap<>(); private final Map functionPointerDataPool = new IdentityHashMap<>(); private final Map globalVarFPDataPool = new HashMap<>(); private final Map> recordDefTypeWiseFPDataPool = new HashMap<>(); - private final Set childPackages = new HashSet<>(); protected Set usedNativeClassPaths = new HashSet<>(); - protected boolean moduleIsUsed = false; - private Set interopDependencies = new HashSet<>(); + boolean moduleIsUsed = false; private InvocationData testablePkgInvocationData; private static boolean isResourceFunction(BIRNode.BIRFunction birFunction) { @@ -557,9 +540,10 @@ private static boolean isExternalDependencyBIRNode(BIRNode.BIRTypeDefinition bir return containsExternalDependencyAnnot(birTypeDefinition.annotAttachments); } - private static boolean containsExternalDependencyAnnot(List annotAttachments) { + private static boolean containsExternalDependencyAnnot( + Iterable annotAttachments) { for (BIRNode.BIRAnnotationAttachment annotAttachment : annotAttachments) { - if (annotAttachment.annotPkgId.getPackageNameWithOrg().equals(BALLERINA_JBALLERINA_JAVA) && + if (isJBallerinaModule(annotAttachment.annotPkgId) && annotAttachment.annotTagRef.toString().equals(EXTERNAL_DEPENDENCY_ANNOT)) { return true; } @@ -567,7 +551,7 @@ private static boolean containsExternalDependencyAnnot(List { unusedTypeDefs.add(typeDef); typeDef.markSelfAsUnused(); @@ -588,10 +572,8 @@ protected void registerNodes(UsedTypeDefAnalyzer typeDefAnalyzer, BIRNode.BIRPac } private void whiteListExternalDependencyTypedefs(String packageName, BIRNode.BIRTypeDefinition typeDef) { - if (PKGS_WITH_WHITELSITED_FILES.contains(packageName)) { - if (isInWhiteListedBalFile(typeDef)) { - this.startPointNodes.add(typeDef); - } + if (isPackageWithWhiteListedFiles(packageName) && isInWhiteListedBalFile(typeDef)) { + this.startPointNodes.add(typeDef); } if (isExternalDependencyBIRNode(typeDef)) { this.startPointNodes.add(typeDef); @@ -599,7 +581,7 @@ private void whiteListExternalDependencyTypedefs(String packageName, BIRNode.BIR } private boolean isInWhiteListedBalFile(BIRNode.BIRTypeDefinition typedef) { - return typedef.pos != null && WHITELSITED_FILE_NAMES.contains(typedef.pos.lineRange().fileName()); + return typedef.pos != null && isWhiteListedFile(typedef.pos.lineRange().fileName()); } private void initializeFunction(BIRNode.BIRFunction birFunction) { @@ -621,7 +603,7 @@ private void initializeAttachedFunction(BIRNode.BIRTypeDefinition parentTypeDef, functionPool.putIfAbsent(parentTypeDef.internalName.value + "." + attachedFunc.name.value, attachedFunc); } - protected void addToUsedPool(BIRNode.BIRFunction birFunction) { + private void addToUsedPool(BIRNode.BIRFunction birFunction) { if (this.unusedFunctions.remove(birFunction)) { this.usedFunctions.add(birFunction); } else if (this.testablePkgInvocationData != null) { @@ -630,7 +612,7 @@ protected void addToUsedPool(BIRNode.BIRFunction birFunction) { } } - protected void addToUsedPool(BIRNode.BIRTypeDefinition birTypeDef) { + void addToUsedPool(BIRNode.BIRTypeDefinition birTypeDef) { if (this.unusedTypeDefs.remove(birTypeDef)) { this.usedTypeDefs.add(birTypeDef); } else if (this.testablePkgInvocationData != null) { @@ -643,29 +625,29 @@ private FunctionPointerData getFPData(BIRNode.BIRVariableDcl variableDcl) { return functionPointerDataPool.get(variableDcl); } - protected HashSet getFpData(BType bType) { + Set getFpData(BType bType) { return this.recordDefTypeWiseFPDataPool.get(bType); } - protected Collection getFpDataPool() { + Collection getFpDataPool() { return this.functionPointerDataPool.values(); } } - protected static class FunctionPointerData { + static final class FunctionPointerData { // Can be deleted from the BirFunctions of the enclosing module - protected final BIRNode.BIRFunction lambdaFunction; + final BIRNode.BIRFunction lambdaFunction; // Can be deleted from the localVar or global var list - protected final BIRNode.BIRVariableDcl lambdaPointerVar; + final BIRNode.BIRVariableDcl lambdaPointerVar; // Can be deleted from the instruction array of either the parent function or init<> function private final BIRNonTerminator.FPLoad fpLoadInstruction; private final List instructionArray; // Holds a recordDefaultFPLoad instruction if the respective fpLoadInstruction has one private BIRNonTerminator.RecordDefaultFPLoad recordDefaultFPLoad; - protected FunctionPointerData(BIRNonTerminator.FPLoad fpLoadInstruction, - List instructionArray, BIRNode.BIRFunction lambdaFunction) { + private FunctionPointerData(BIRNonTerminator.FPLoad fpLoadInstruction, + List instructionArray, BIRNode.BIRFunction lambdaFunction) { this.fpLoadInstruction = fpLoadInstruction; this.instructionArray = instructionArray; this.lambdaPointerVar = fpLoadInstruction.lhsOp.variableDcl; @@ -677,7 +659,7 @@ protected FunctionPointerData(BIRNonTerminator.FPLoad fpLoadInstruction, // We need to delete those if the lambda function is not used. // If the FP_LOAD instruction is not removed its references will be used for CodeGen. // We need to prevent this from happening to fully clean the UNUSED functions. - protected void deleteIfUnused() { + void deleteIfUnused() { // if the parent record is used, instructions should not be deleted if (recordDefaultFPLoad != null && recordDefaultFPLoad.enclosedType.isUsed) { return; diff --git a/compiler/ballerina-lang/src/main/java/io/ballerina/projects/UsedTypeDefAnalyzer.java b/compiler/ballerina-lang/src/main/java/io/ballerina/projects/UsedTypeDefAnalyzer.java index 223723dae428..8355ca8a86c2 100644 --- a/compiler/ballerina-lang/src/main/java/io/ballerina/projects/UsedTypeDefAnalyzer.java +++ b/compiler/ballerina-lang/src/main/java/io/ballerina/projects/UsedTypeDefAnalyzer.java @@ -24,6 +24,7 @@ import org.wso2.ballerinalang.compiler.util.CompilerContext; import org.wso2.ballerinalang.compiler.util.TypeTags; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -40,8 +41,8 @@ public final class UsedTypeDefAnalyzer extends SimpleBTypeAnalyzer(); private final Map globalTypeDefPool = new HashMap<>(); private final Set visitedTypes = new HashSet<>(); - private PackageCache pkgCache; - private UsedBIRNodeAnalyzer usedBIRNodeAnalyzer; + private final PackageCache pkgCache; + private final UsedBIRNodeAnalyzer usedBIRNodeAnalyzer; private UsedTypeDefAnalyzer(CompilerContext context) { context.put(BIR_TYPE_DEF_ANALYZER_KEY, this); @@ -91,7 +92,7 @@ public void analyzeType(BType bType, AnalyzerData data) { } } - protected void populateTypeDefPool(BIRNode.BIRPackage birPackage, Set interopDependencies) { + void populateTypeDefPool(BIRNode.BIRPackage birPackage, Collection interopDependencies) { birPackage.typeDefs.forEach(typeDef -> { // In case there are more than one reference types referencing to the same type if (typeDef.referenceType != null && typeDef.type.tag == TypeTags.TYPEREFDESC) { @@ -99,20 +100,20 @@ protected void populateTypeDefPool(BIRNode.BIRPackage birPackage, Set in } else { globalTypeDefPool.putIfAbsent(typeDef.type, typeDef); } - if (interopDependencies != null && interopDependencies.contains(typeDef.internalName.toString())) { + if (interopDependencies.contains(typeDef.internalName.toString())) { usedBIRNodeAnalyzer.getInvocationData(typeDef.getPackageID()).startPointNodes.add(typeDef); } }); } - protected void analyzeTypeDef(BIRNode.BIRTypeDefinition typeDef) { + void analyzeTypeDef(BIRNode.BIRTypeDefinition typeDef) { final AnalyzerData data = new AnalyzerData(); data.currentParentNode = typeDef; typeDef.referencedTypes.forEach(refType -> visitType(refType, data)); visitType(typeDef.type, data); } - protected void analyzeTypeDefWithinScope(BType bType, BIRNode.BIRDocumentableNode parentNode) { + void analyzeTypeDefWithinScope(BType bType, BIRNode.BIRDocumentableNode parentNode) { final AnalyzerData data = new AnalyzerData(); data.currentParentNode = parentNode; visitType(bType, data); @@ -170,7 +171,7 @@ private void addDependency(BType bType, AnalyzerData data) { }); } - public boolean isTestablePkgImportedModuleDependency(BType bType) { + private boolean isTestablePkgImportedModuleDependency(BType bType) { if (bType.tsymbol == null || bType.tsymbol.pkgID == null) { return false; } @@ -179,7 +180,8 @@ public boolean isTestablePkgImportedModuleDependency(BType bType) { !bType.tsymbol.pkgID.equals(usedBIRNodeAnalyzer.currentPkgID); } - public static class AnalyzerData { + public static final class AnalyzerData { + BIRNode.BIRDocumentableNode currentParentNode; boolean shouldAnalyzeChildren = true; } diff --git a/compiler/ballerina-lang/src/main/java/io/ballerina/projects/util/CodegenOptimizationUtils.java b/compiler/ballerina-lang/src/main/java/io/ballerina/projects/util/CodegenOptimizationUtils.java index 64e7af682460..36558cb0e5a3 100644 --- a/compiler/ballerina-lang/src/main/java/io/ballerina/projects/util/CodegenOptimizationUtils.java +++ b/compiler/ballerina-lang/src/main/java/io/ballerina/projects/util/CodegenOptimizationUtils.java @@ -20,11 +20,20 @@ import org.ballerinalang.model.elements.PackageID; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + import static io.ballerina.projects.util.CodegenOptimizationConstants.BALLERINAX; import static io.ballerina.projects.util.CodegenOptimizationConstants.BALLERINA_JBALLERINA_JAVA; import static io.ballerina.projects.util.CodegenOptimizationConstants.BALLERINA_LANG; import static io.ballerina.projects.util.CodegenOptimizationConstants.BALLERINA_OBSERVE; import static io.ballerina.projects.util.CodegenOptimizationConstants.DOT_DRIVER; +import static io.ballerina.projects.util.CodegenOptimizationConstants.INTEROP_DEPENDENCIES_PROPERTIES_FILE; /** * Common utils for codegen optimization implementation. @@ -33,28 +42,107 @@ */ public final class CodegenOptimizationUtils { + private static final Set WHITELSITED_FILE_NAMES = Set.of("types.bal", "error.bal", "stream_types.bal"); + private static final Set PKGS_WITH_WHITELSITED_FILES = + Set.of("ballerinax/mysql", "ballerina/sql", "ballerinax/persist.sql"); + + // These directories are whitelisted due to usages of "sun.misc.Unsafe" class + // TODO Find a way to whitelist only the necessary class files + private static final Set WHITELISTED_DIRECTORIES = Set.of("io/netty/util"); + + /** + * These classes are used by GraalVM when building the native-image. Since they are not connected to the root class, + * they will be removed by the NativeDependencyOptimizer if they are not whitelisted. + */ + private static final Set GRAALVM_FEATURE_CLASSES = + Set.of("io/ballerina/stdlib/crypto/svm/BouncyCastleFeature"); + public static final String CLASS = ".class"; + private static final String SERVICE_PROVIDER_DIRECTORY = "META-INF/services/"; + public static final String MODULE_INFO = "module-info"; + private static final String NATIVE_IMAGE_DIRECTORY = "META-INF/native-image"; + private static final String REFLECT_CONFIG_JSON = "reflect-config.json"; + private static final String JNI_CONFIG_JSON = "jni-config.json"; + private CodegenOptimizationUtils() { } - public static boolean isObserveModule(PackageID packageID) { - return packageID.getPackageNameWithOrg().equals(BALLERINA_OBSERVE); + public static boolean isObserveModule(String packageName) { + return packageName.equals(BALLERINA_OBSERVE); + } + + public static boolean isDriverModule(String packageName) { + return packageName.startsWith(BALLERINAX) && packageName.endsWith(DOT_DRIVER); } - public static boolean isDriverModule(PackageID packageID) { - return packageID.getPackageNameWithOrg().startsWith(BALLERINAX) && - packageID.getPackageNameWithOrg().endsWith(DOT_DRIVER); + public static boolean isLangLibModule(String packageName) { + return packageName.startsWith(BALLERINA_LANG); } - public static boolean isLangLibModule(PackageID packageID) { - return packageID.getPackageNameWithOrg().startsWith(BALLERINA_LANG); + public static boolean isJBallerinaModule(PackageID packageName) { + return isJBallerinaModule(packageName.getPackageNameWithOrg()); } - public static boolean isJBallerinaModule(PackageID packageID) { - return packageID.getPackageNameWithOrg().equals(BALLERINA_JBALLERINA_JAVA); + public static boolean isJBallerinaModule(String packageName) { + return packageName.equals(BALLERINA_JBALLERINA_JAVA); } public static boolean isWhiteListedModule(PackageID packageID) { - return isObserveModule(packageID) || isDriverModule(packageID) || isJBallerinaModule(packageID) - || isLangLibModule(packageID); + String packageName = packageID.getPackageNameWithOrg(); + return isWhiteListedModule(packageName); + } + + public static boolean isWhiteListedModule(String packageName) { + return isObserveModule(packageName) || isDriverModule(packageName) || isJBallerinaModule(packageName) || + isLangLibModule(packageName); + } + + public static boolean isPackageWithWhiteListedFiles(String packageName) { + return PKGS_WITH_WHITELSITED_FILES.contains(packageName); + } + + public static boolean isWhiteListedFile(String fileName) { + return WHITELSITED_FILE_NAMES.contains(fileName); + } + + public static boolean isWhiteListedDirectory(String path) { + return WHITELISTED_DIRECTORIES.stream().anyMatch(path::startsWith); + } + + public static Set getWhiteListedClasses() { + return GRAALVM_FEATURE_CLASSES; + } + + public static boolean isWhiteListedEntryName(String entryName) { + if (entryName.equals(MODULE_INFO + CLASS)) { + return true; + } + return isWhiteListedDirectory(entryName); + } + + public static boolean isServiceProvider(String entryName) { + return entryName.startsWith(SERVICE_PROVIDER_DIRECTORY); + } + + public static boolean isReflectionConfig(String entryName) { + return (entryName.endsWith(REFLECT_CONFIG_JSON) || entryName.endsWith(JNI_CONFIG_JSON)) && + entryName.startsWith(NATIVE_IMAGE_DIRECTORY); + } + + public static Map> getHardcodedDependencies() { + Properties prop = new Properties(); + try { + InputStream stream = CodegenOptimizationUtils.class.getClassLoader() + .getResourceAsStream(INTEROP_DEPENDENCIES_PROPERTIES_FILE); + prop.load(stream); + Pattern pattern = Pattern.compile(","); + return prop.entrySet().stream() + .collect(Collectors.toMap( + e -> e.getKey().toString(), + e -> pattern.splitAsStream(e.getValue().toString()) + .collect(Collectors.toUnmodifiableSet()) + )); + } catch (IOException e) { + throw new RuntimeException("Failed to load interop-dependencies : ", e); + } } } diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/JvmPackageGen.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/JvmPackageGen.java index 1221af7529e6..fb7a2b3c7290 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/JvmPackageGen.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/JvmPackageGen.java @@ -708,7 +708,6 @@ CompiledJarFile generate(BIRPackage module) { } serviceEPAvailable |= listenerDeclarationFound(pkgSymbol); } - String moduleInitClass = JvmCodeGenUtil.getModuleLevelClassName(module.packageID, MODULE_INIT_CLASS_NAME); String typesClass = getModuleLevelClassName(module.packageID, MODULE_TYPES_CLASS_NAME); Map jvmClassMapping = generateClassNameLinking(module, moduleInitClass, true); @@ -744,7 +743,6 @@ CompiledJarFile generate(BIRPackage module) { configMethodGen.generateConfigMapper(immediateImports, module, moduleInitClass, jvmConstantsGen, typeHashVisitor, jarEntries, symbolTable); - // generate the shutdown listener class. new ShutDownListenerGen().generateShutdownSignalListener(moduleInitClass, jarEntries ); diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/bytecodeoptimizer/NativeDependencyOptimizationReport.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/bytecodeoptimizer/NativeDependencyOptimizationReport.java index 6fbdb6b5b416..d5f770bcc442 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/bytecodeoptimizer/NativeDependencyOptimizationReport.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/bytecodeoptimizer/NativeDependencyOptimizationReport.java @@ -31,6 +31,6 @@ * @since 2201.10.0 */ record NativeDependencyOptimizationReport(Set startPointClasses, Set usedExternalClasses, - Set usedClasses, Set unusedClasses) { + Set usedClasses, Set unusedClasses) { } diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/bytecodeoptimizer/NativeDependencyOptimizer.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/bytecodeoptimizer/NativeDependencyOptimizer.java index f639991ed1a2..c1c9ada43e3a 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/bytecodeoptimizer/NativeDependencyOptimizer.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/bytecodeoptimizer/NativeDependencyOptimizer.java @@ -20,6 +20,7 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; +import io.ballerina.projects.util.CodegenOptimizationUtils; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveEntryPredicate; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; @@ -32,14 +33,17 @@ import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.StandardCharsets; +import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.Stack; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static io.ballerina.projects.util.CodegenOptimizationUtils.getWhiteListedClasses; /** * Optimizes a given JAR on class file level. @@ -48,37 +52,11 @@ */ public final class NativeDependencyOptimizer { - private static final String CLASS = ".class"; - private static final String SERVICE_PROVIDER_DIRECTORY = "META-INF/services/"; - private static final String NATIVE_IMAGE_DIRECTORY = "META-INF/native-image"; - private static final String REFLECT_CONFIG_JSON = "reflect-config.json"; - private static final String JNI_CONFIG_JSON = "jni-config.json"; - private static final String MODULE_INFO = "module-info"; - - // These directories are whitelisted due to usages of "sun.misc.Unsafe" class - // TODO Find a way to whitelist only the necessary class files - private static final Set WHITELISTED_DIRECTORIES = new HashSet<>(List.of("io/netty/util")); - - /** - * These classes are used by GraalVM when building the native-image. Since they are not connected to the root class, - * they will be removed by the NativeDependencyOptimizer if they are not whitelisted. - */ - private static final Set GRAALVM_FEATURE_CLASSES = - new HashSet<>(List.of("io/ballerina/stdlib/crypto/svm/BouncyCastleFeature")); - - /** - * key = implementation class name, value = interface class name. - * Since one interface can be implemented by more than one child class, it is possible to have duplicate values. - *

- * TODO modify the service provider files and delete the lines containing the UNUSED implementations of interfaces - */ - private static final Map implementationWiseAllServiceProviders = new HashMap<>(); - /** * key = used interface, value = used implementation. */ private static final Map> interfaceWiseAllServiceProviders = new HashMap<>(); - private static final Set usedSpInterfaces = new LinkedHashSet<>(); + private static final Collection usedSpInterfaces = new LinkedHashSet<>(); private static final Gson gson = new Gson(); private final Set startPointClasses; private final Stack usedClassesStack; @@ -90,45 +68,21 @@ public final class NativeDependencyOptimizer { public NativeDependencyOptimizer(Set startPointClasses, ZipFile originalJarFile, ZipArchiveOutputStream optimizedJarStream) { - this.startPointClasses = startPointClasses; this.originalJarFile = originalJarFile; this.optimizedJarStream = optimizedJarStream; this.usedClassesStack = new Stack<>(); } - private static boolean isWhiteListedEntryName(String entryName) { - if (entryName.equals(MODULE_INFO + CLASS)) { - return true; - } - - for (String whiteListedDirectory : WHITELISTED_DIRECTORIES) { - if (entryName.startsWith(whiteListedDirectory)) { - return true; - } - } - return false; - } - - private static LinkedHashSet getServiceProviderImplementations(ZipFile originalJarFile, - ZipArchiveEntry entry) throws IOException { + private static Iterable getServiceProviderImplementations(ZipFile originalJarFile, + ZipArchiveEntry entry) throws IOException { String allImplString = IOUtils.toString(originalJarFile.getInputStream(entry), StandardCharsets.UTF_8); - String[] serviceImplClassesArr = allImplString.split("\n"); - LinkedHashSet serviceProviderDependencies = new LinkedHashSet<>(); - - for (String serviceClass : serviceImplClassesArr) { - // Skipping the licensing comments - if (serviceClass.startsWith("#") || serviceClass.isBlank()) { - continue; - } - serviceProviderDependencies.add(getServiceProviderClassName(serviceClass)); - } - - return serviceProviderDependencies; - } - - private static boolean isServiceProvider(String entryName) { - return entryName.startsWith(SERVICE_PROVIDER_DIRECTORY); + Pattern pattern = Pattern.compile("\n"); + return pattern.splitAsStream(allImplString) + // Skipping the licensing comments + .filter(serviceClass -> !serviceClass.startsWith("#") && !serviceClass.isBlank()) + .map(NativeDependencyOptimizer::getServiceProviderClassName) + .collect(Collectors.toCollection(LinkedHashSet::new)); } private static String getServiceProviderClassName(String providerFileName) { @@ -144,12 +98,12 @@ public void analyzeUsedClasses() throws IOException { while (!usedClassesStack.empty()) { String usedClassName = usedClassesStack.pop(); - if (visitedClasses.contains(usedClassName + CLASS)) { + if (visitedClasses.contains(usedClassName + CodegenOptimizationUtils.CLASS)) { continue; } - visitedClasses.add(usedClassName + CLASS); - ZipArchiveEntry usedJarEntry = originalJarFile.getEntry(usedClassName + CLASS); + visitedClasses.add(usedClassName + CodegenOptimizationUtils.CLASS); + ZipArchiveEntry usedJarEntry = originalJarFile.getEntry(usedClassName + CodegenOptimizationUtils.CLASS); if (usedJarEntry == null) { externalClasses.add(usedClassName); @@ -175,34 +129,28 @@ public void analyzeWhiteListedClasses() throws IOException { while (jarEntries.hasMoreElements()) { ZipArchiveEntry currentEntry = jarEntries.nextElement(); - if (isServiceProvider(currentEntry.getName())) { + if (CodegenOptimizationUtils.isServiceProvider(currentEntry.getName())) { analyzeServiceProviders(currentEntry); } - if (isReflectionConfig(currentEntry.getName())) { + if (CodegenOptimizationUtils.isReflectionConfig(currentEntry.getName())) { try (BufferedReader reader = new BufferedReader( new InputStreamReader(originalJarFile.getInputStream(currentEntry), StandardCharsets.UTF_8))) { whitelistReflectionClasses(reader); } } } - startPointClasses.addAll(GRAALVM_FEATURE_CLASSES); + startPointClasses.addAll(getWhiteListedClasses()); } private void analyzeServiceProviders(ZipArchiveEntry currentEntry) throws IOException { String spInterfaceClassName = getServiceProviderClassName(currentEntry.getName()); for (String spImplementationClassName : getServiceProviderImplementations(originalJarFile, currentEntry)) { - implementationWiseAllServiceProviders.put(spImplementationClassName, spInterfaceClassName); interfaceWiseAllServiceProviders.putIfAbsent(spInterfaceClassName, new LinkedHashSet<>()); interfaceWiseAllServiceProviders.get(spInterfaceClassName).add(spImplementationClassName); } } - private boolean isReflectionConfig(String entryName) { - return (entryName.endsWith(REFLECT_CONFIG_JSON) || entryName.endsWith(JNI_CONFIG_JSON)) && - entryName.startsWith(NATIVE_IMAGE_DIRECTORY); - } - private void whitelistReflectionClasses(Reader reader) { JsonElement jsonElement = gson.fromJson(reader, JsonElement.class); @@ -222,13 +170,13 @@ public void copyUsedEntries() throws IOException { if (entry.isDirectory()) { return true; } - if (!entryName.endsWith(CLASS)) { - if (isServiceProvider(entryName)) { + if (!entryName.endsWith(CodegenOptimizationUtils.CLASS)) { + if (CodegenOptimizationUtils.isServiceProvider(entryName)) { return usedSpInterfaces.contains(getServiceProviderClassName(entryName)); } return true; } - return visitedClasses.contains(entryName) || isWhiteListedEntryName(entryName); + return visitedClasses.contains(entryName) || CodegenOptimizationUtils.isWhiteListedEntryName(entryName); }; originalJarFile.copyRawEntries(optimizedJarStream, usedClassPredicate); @@ -245,8 +193,9 @@ private LinkedHashSet getUnusedClasses() { while (entries.hasMoreElements()) { ZipArchiveEntry entry = entries.nextElement(); String className = entry.getName(); - if (!entry.isDirectory() && className.endsWith(CLASS) && !visitedClasses.contains(className) && - !className.equals(MODULE_INFO + CLASS)) { + if (!entry.isDirectory() && className.endsWith(CodegenOptimizationUtils.CLASS) && + !visitedClasses.contains(className) && + !className.equals(CodegenOptimizationUtils.MODULE_INFO + CodegenOptimizationUtils.CLASS)) { unusedClasses.add(className); } } @@ -256,9 +205,9 @@ private LinkedHashSet getUnusedClasses() { /* This function can be used when modularizing the native dependency optimization. */ - public boolean jarContainsStartPoints() { + private boolean jarContainsStartPoints() { for (String startPoint : startPointClasses) { - ZipArchiveEntry jarEntry = originalJarFile.getEntry(startPoint + CLASS); + ZipArchiveEntry jarEntry = originalJarFile.getEntry(startPoint + CodegenOptimizationUtils.CLASS); if (jarEntry != null) { return true; } diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/model/JBIRFunction.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/model/JBIRFunction.java index 7309b629bfee..e18e695759c6 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/model/JBIRFunction.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/model/JBIRFunction.java @@ -26,7 +26,6 @@ */ public class JBIRFunction extends BIRNode.BIRFunction { - public JBIRFunction(BIRFunction birFunction) { super(birFunction.pos, birFunction.name, birFunction.originalName, birFunction.flags, birFunction.origin, birFunction.type, birFunction.requiredParams, birFunction.receiver, birFunction.restParam,