Add operation retries to prevent unnecessary failures when multiple S…

…cala CLI instances are run on the same project in parallel

modules/build/src/main/scala/scala/build/Bloop.scala

@@ -12,9 +12,10 @@ import java.io.{File, IOException}
 import scala.annotation.tailrec
 import scala.build.EitherCps.{either, value}
 import scala.build.errors.{BuildException, ModuleFormatError}
-import scala.build.internal.CsLoggerUtil._
+import scala.build.internal.CsLoggerUtil.*
+import scala.concurrent.ExecutionException
 import scala.concurrent.duration.FiniteDuration
-import scala.jdk.CollectionConverters._
+import scala.jdk.CollectionConverters.*
 
 object Bloop {
 
@@ -35,32 +36,37 @@ object Bloop {
     logger: Logger,
     buildTargetsTimeout: FiniteDuration
   ): Either[Throwable, Boolean] =
-    try {
-      logger.debug("Listing BSP build targets")
-      val results = buildServer.workspaceBuildTargets()
-        .get(buildTargetsTimeout.length, buildTargetsTimeout.unit)
-      val buildTargetOpt = results.getTargets.asScala.find(_.getDisplayName == projectName)
+    try retry()(logger) {
+        logger.debug("Listing BSP build targets")
+        val results = buildServer.workspaceBuildTargets()
+          .get(buildTargetsTimeout.length, buildTargetsTimeout.unit)
+        val buildTargetOpt = results.getTargets.asScala.find(_.getDisplayName == projectName)
 
-      val buildTarget = buildTargetOpt.getOrElse {
-        throw new Exception(
-          s"Expected to find project '$projectName' in build targets (only got ${results.getTargets
-              .asScala.map("'" + _.getDisplayName + "'").mkString(", ")})"
-        )
-      }
+        val buildTarget = buildTargetOpt.getOrElse {
+          throw new Exception(
+            s"Expected to find project '$projectName' in build targets (only got ${results.getTargets
+                .asScala.map("'" + _.getDisplayName + "'").mkString(", ")})"
+          )
+        }
 
-      logger.debug(s"Compiling $projectName with Bloop")
-      val compileRes = buildServer.buildTargetCompile(
-        new bsp4j.CompileParams(List(buildTarget.getId).asJava)
-      ).get()
+        logger.debug(s"Compiling $projectName with Bloop")
+        val compileRes = buildServer.buildTargetCompile(
+          new bsp4j.CompileParams(List(buildTarget.getId).asJava)
+        ).get()
 
-      val success = compileRes.getStatusCode == bsp4j.StatusCode.OK
-      logger.debug(if (success) "Compilation succeeded" else "Compilation failed")
-      Right(success)
-    }
+        val success = compileRes.getStatusCode == bsp4j.StatusCode.OK
+        logger.debug(if (success) "Compilation succeeded" else "Compilation failed")
+        Right(success)
+      }
     catch {
       case ex @ BrokenPipeInCauses(e) =>
         logger.debug(s"Caught $ex while exchanging with Bloop server, assuming Bloop server exited")
         Left(ex)
+      case ex: ExecutionException =>
+        logger.debug(
+          s"Caught $ex while exchanging with Bloop server, you may consider restarting the build server"
+        )
+        Left(ex)
     }
 
   def bloopClassPath(

modules/build/src/main/scala/scala/build/Build.scala

@@ -50,16 +50,17 @@ object Build {
     output: os.Path,
     diagnostics: Option[Seq[(Either[String, os.Path], bsp4j.Diagnostic)]],
     generatedSources: Seq[GeneratedSource],
-    isPartial: Boolean
+    isPartial: Boolean,
+    logger: Logger
   ) extends Build {
     def success: Boolean                  = true
     def successfulOpt: Some[this.type]    = Some(this)
     def outputOpt: Some[os.Path]          = Some(output)
     def dependencyClassPath: Seq[os.Path] = sources.resourceDirs ++ artifacts.classPath
     def fullClassPath: Seq[os.Path]       = Seq(output) ++ dependencyClassPath
-    private lazy val mainClassesFoundInProject: Seq[String] = MainClass.find(output).sorted
+    private lazy val mainClassesFoundInProject: Seq[String] = MainClass.find(output, logger).sorted
     private lazy val mainClassesFoundOnExtraClasspath: Seq[String] =
-      options.classPathOptions.extraClassPath.flatMap(MainClass.find).sorted
+      options.classPathOptions.extraClassPath.flatMap(MainClass.find(_, logger)).sorted
     private lazy val mainClassesFoundInUserExtraDependencies: Seq[String] =
       artifacts.jarsForUserExtraDependencies.flatMap(MainClass.findInDependency).sorted
     def foundMainClasses(): Seq[String] = {
@@ -1184,7 +1185,8 @@ object Build {
         classesDir0,
         buildClient.diagnostics,
         generatedSources,
-        partial
+        partial,
+        logger
       )
     else
       Failed(

modules/build/src/main/scala/scala/build/compiler/BloopCompilerMaker.scala

@@ -3,11 +3,11 @@ package scala.build.compiler
 import bloop.rifle.{BloopRifleConfig, BloopServer, BloopThreads}
 import ch.epfl.scala.bsp4j.BuildClient
 
-import scala.build.Logger
 import scala.build.errors.{BuildException, FetchingDependenciesError, Severity}
 import scala.build.internal.Constants
 import scala.build.internal.util.WarningMessages
 import scala.build.options.BuildOptions
+import scala.build.{Logger, retry}
 import scala.concurrent.duration.DurationInt
 import scala.util.Try
 
@@ -37,16 +37,25 @@ final class BloopCompilerMaker(
       case Right(config) =>
         val createBuildServer =
           () =>
-            BloopServer.buildServer(
-              config,
-              "scala-cli",
-              Constants.version,
-              workspace.toNIO,
-              classesDir.toNIO,
-              buildClient,
-              threads,
-              logger.bloopRifleLogger
-            )
+            // retrying here in case a number of Scala CLI processes are started at the same time
+            // and they all try to connect to the server / spawn a new server
+            // otherwise, users may run into one of:
+            //   - libdaemonjvm.client.ConnectError$ZombieFound
+            //   - Caught java.lang.RuntimeException: Fatal error, could not spawn Bloop: not running
+            //   - java.lang.RuntimeException: Bloop BSP connection in (...) was unexpectedly closed or bloop didn't start.
+            // if a sufficiently large number of processes was started, this may happen anyway, of course
+            retry(if offline then 1 else 3)(logger) {
+              BloopServer.buildServer(
+                config,
+                "scala-cli",
+                Constants.version,
+                workspace.toNIO,
+                classesDir.toNIO,
+                buildClient,
+                threads,
+                logger.bloopRifleLogger
+              )
+            }
 
         val res = Try(new BloopCompiler(createBuildServer, 20.seconds, strictBloopJsonCheck))
           .toEither

modules/build/src/main/scala/scala/build/internal/MainClass.scala

@@ -4,11 +4,13 @@ import org.objectweb.asm
 import org.objectweb.asm.ClassReader
 
 import java.io.{ByteArrayInputStream, InputStream}
+import java.nio.file.NoSuchFileException
 import java.util.jar.{Attributes, JarFile, JarInputStream, Manifest}
 import java.util.zip.ZipEntry
 
 import scala.build.input.Element
 import scala.build.internal.zip.WrappedZipInputStream
+import scala.build.{Logger, retry}
 
 object MainClass {
 
@@ -44,29 +46,54 @@ object MainClass {
       if (foundMainClass) nameOpt else None
   }
 
-  def findInClass(path: os.Path): Iterator[String] =
-    findInClass(os.read.inputStream(path))
-  def findInClass(is: InputStream): Iterator[String] =
+  private def findInClass(path: os.Path, logger: Logger): Iterator[String] =
     try {
-      val reader  = new ClassReader(is)
-      val checker = new MainMethodChecker
-      reader.accept(checker, 0)
-      checker.mainClassOpt.iterator
+      val is = retry()(logger)(os.read.inputStream(path))
+      findInClass(is, logger)
+    }
+    catch {
+      case e: NoSuchFileException =>
+        e.getStackTrace.foreach(ste => logger.debug(ste.toString))
+        logger.log(s"Class file $path not found: $e")
+        logger.log("Are you trying to run too many builds at once? Trying to recover...")
+        Iterator.empty
+    }
+  private def findInClass(is: InputStream, logger: Logger): Iterator[String] =
+    try retry()(logger) {
+        val reader  = new ClassReader(is)
+        val checker = new MainMethodChecker
+        reader.accept(checker, 0)
+        checker.mainClassOpt.iterator
+      }
+    catch {
+      case e: ArrayIndexOutOfBoundsException =>
+        e.getStackTrace.foreach(ste => logger.debug(ste.toString))
+        logger.log(s"Class input stream could not be created: $e")
+        logger.log("Are you trying to run too many builds at once? Trying to recover...")
+        Iterator.empty
     }
     finally is.close()
 
-  def findInJar(path: os.Path): Iterator[String] = {
-    val content        = os.read.bytes(path)
-    val jarInputStream = WrappedZipInputStream.create(new ByteArrayInputStream(content))
-    jarInputStream.entries().flatMap(ent =>
-      if !ent.isDirectory && ent.getName.endsWith(".class") then {
-        val content     = jarInputStream.readAllBytes()
-        val inputStream = new ByteArrayInputStream(content)
-        findInClass(inputStream)
+  private def findInJar(path: os.Path, logger: Logger): Iterator[String] =
+    try retry()(logger) {
+        val content        = os.read.bytes(path)
+        val jarInputStream = WrappedZipInputStream.create(new ByteArrayInputStream(content))
+        jarInputStream.entries().flatMap(ent =>
+          if !ent.isDirectory && ent.getName.endsWith(".class") then {
+            val content     = jarInputStream.readAllBytes()
+            val inputStream = new ByteArrayInputStream(content)
+            findInClass(inputStream, logger)
+          }
+          else Iterator.empty
+        )
       }
-      else Iterator.empty
-    )
-  }
+    catch {
+      case e: NoSuchFileException =>
+        logger.debugStackTrace(e)
+        logger.log(s"JAR file $path not found: $e, trying to recover...")
+        logger.log("Are you trying to run too many builds at once? Trying to recover...")
+        Iterator.empty
+    }
 
   def findInDependency(jar: os.Path): Option[String] =
     jar match {
@@ -79,19 +106,19 @@ object MainClass {
       case _ => None
     }
 
-  def find(output: os.Path): Seq[String] =
+  def find(output: os.Path, logger: Logger): Seq[String] =
     output match {
       case o if os.isFile(o) && o.last.endsWith(".class") =>
-        findInClass(o).toVector
+        findInClass(o, logger).toVector
       case o if os.isFile(o) && o.last.endsWith(".jar") =>
-        findInJar(o).toVector
+        findInJar(o, logger).toVector
       case o if os.isDir(o) =>
         os.walk(o)
           .iterator
           .filter(os.isFile)
           .flatMap {
             case classFilePath if classFilePath.last.endsWith(".class") =>
-              findInClass(classFilePath)
+              findInClass(classFilePath, logger)
             case _ => Iterator.empty
           }
           .toVector

modules/build/src/main/scala/scala/build/package.scala

@@ -0,0 +1,33 @@
+package scala
+
+import scala.annotation.tailrec
+import scala.concurrent.duration.{DurationInt, FiniteDuration}
+import scala.util.Random
+
+package object build {
+  def retry[T](
+    maxAttempts: Int = 3,
+    waitDuration: FiniteDuration = 1.seconds,
+    variableWaitDelayInMs: Int = 500
+  )(logger: Logger)(
+    run: => T
+  ): T = {
+    @tailrec
+    def helper(count: Int): T =
+      try run
+      catch {
+        case t: Throwable =>
+          if count >= maxAttempts then throw t
+          else
+            logger.debugStackTrace(t)
+            val variableDelay       = Random.between(0, variableWaitDelayInMs + 1).milliseconds
+            val currentWaitDuration = waitDuration + variableDelay
+            logger.log(s"Caught $t, trying again in $currentWaitDuration…")
+            Thread.sleep(currentWaitDuration.toMillis)
+            helper(count + 1)
+      }
+
+    helper(1)
+  }
+
+}

modules/build/src/main/scala/scala/build/postprocessing/AsmPositionUpdater.scala

@@ -2,7 +2,10 @@ package scala.build.postprocessing
 
 import org.objectweb.asm
 
-import scala.build.{Logger, Os}
+import java.io
+import java.nio.file.{FileAlreadyExistsException, NoSuchFileException}
+
+import scala.build.{Logger, Os, retry}
 
 object AsmPositionUpdater {
 
@@ -53,20 +56,34 @@ object AsmPositionUpdater {
       .filter(os.isFile(_))
       .filter(_.last.endsWith(".class"))
       .foreach { path =>
-        val is = os.read.inputStream(path)
-        val updateByteCodeOpt =
-          try {
-            val reader  = new asm.ClassReader(is)
-            val writer  = new asm.ClassWriter(reader, 0)
-            val checker = new LineNumberTableClassVisitor(mappings, writer)
-            reader.accept(checker, 0)
-            if (checker.mappedStuff) Some(writer.toByteArray)
-            else None
+        try retry()(logger) {
+            val is = os.read.inputStream(path)
+            val updateByteCodeOpt =
+              try retry()(logger) {
+                  val reader  = new asm.ClassReader(is)
+                  val writer  = new asm.ClassWriter(reader, 0)
+                  val checker = new LineNumberTableClassVisitor(mappings, writer)
+                  reader.accept(checker, 0)
+                  if checker.mappedStuff then Some(writer.toByteArray) else None
+                }
+              catch {
+                case e: ArrayIndexOutOfBoundsException =>
+                  e.getStackTrace.foreach(ste => logger.debug(ste.toString))
+                  logger.log(s"Error while processing ${path.relativeTo(Os.pwd)}: $e.")
+                  logger.log("Are you trying to run too many builds at once? Trying to recover...")
+                  None
+              }
+              finally is.close()
+            for (b <- updateByteCodeOpt) {
+              logger.debug(s"Overwriting ${path.relativeTo(Os.pwd)}")
+              os.write.over(path, b)
+            }
           }
-          finally is.close()
-        for (b <- updateByteCodeOpt) {
-          logger.debug(s"Overwriting ${path.relativeTo(Os.pwd)}")
-          os.write.over(path, b)
+        catch {
+          case e: (NoSuchFileException | FileAlreadyExistsException | ArrayIndexOutOfBoundsException) =>
+            logger.debugStackTrace(e)
+            logger.log(s"Error while processing ${path.relativeTo(Os.pwd)}: $e")
+            logger.log("Are you trying to run too many builds at once? Trying to recover...")
         }
       }
   }