From 5064bd8dcdf08311a917eeba7809b458a2bba2c5 Mon Sep 17 00:00:00 2001 From: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> Date: Wed, 8 Jun 2022 22:24:57 +0200 Subject: [PATCH 1/3] Add option to pass lib dependencies to coursier This can be used to add custom url handlers to coursier, so scala-steward could for example reach s3 buckets, google cloud storage or artifactregistry. See https://get-coursier.io/docs/extra.html#extra-protocols and https://github.com/coursier/coursier/issues/1987 --- docs/help.md | 3 + .../scalasteward/core/application/Cli.scala | 10 ++- .../core/application/Config.scala | 3 +- .../core/application/Context.scala | 6 +- .../core/coursier/CoursierAlg.scala | 78 ++++++++++++------- .../CoursierDependenciesFetchAlg.scala | 44 +++++++++++ 6 files changed, 110 insertions(+), 34 deletions(-) create mode 100644 modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierDependenciesFetchAlg.scala diff --git a/docs/help.md b/docs/help.md index e810cc3f22..8360b20a0a 100644 --- a/docs/help.md +++ b/docs/help.md @@ -6,6 +6,7 @@ All command line arguments for the `scala-steward` application. Usage: scala-steward --workspace --repos-file [--repos-file ]... [--git-author-name ] --git-author-email [--git-author-signing-key ] --git-ask-pass [--sign-commits] [--signoff] [--forge-type ] [--forge-api-host ] --forge-login [--do-not-fork] [--add-labels] [--ignore-opts-files] [--env-var ]... [--process-timeout ] [--whitelist ]... [--read-only ]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size ] [--repo-config ]... [--disable-default-repo-config] [--scalafix-migrations ]... [--disable-default-scalafix-migrations] [--artifact-migrations ]... [--disable-default-artifact-migrations] [--cache-ttl ] [--bitbucket-use-default-reviewers] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--gitlab-required-reviewers ] [--gitlab-remove-source-branch] [--azure-repos-organization ] [--github-app-id --github-app-key-file ] [--url-checker-test-url ]... [--default-maven-repo ] [--refresh-backoff-period ] [--exit-code-success-if-any-repo-succeeds] scala-steward validate-repo-config +Usage: scala-steward --workspace --repos-file [--git-author-name ] --git-author-email [--git-author-signing-key ] --git-ask-pass [--sign-commits] [--vcs-type ] [--vcs-api-host ] --vcs-login [--do-not-fork] [--ignore-opts-files] [--env-var ]... [--process-timeout ] [--whitelist ]... [--read-only ]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size ] [--repo-config ]... [--disable-default-repo-config] [--scalafix-migrations ]... [--disable-default-scalafix-migrations] [--artifact-migrations ]... [--disable-default-artifact-migrations] [--cache-ttl ] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--github-app-id --github-app-key-file ] [--url-checker-test-url ] [--default-maven-repo ] [--refresh-backoff-period ] [--coursier-dependencies ]... @@ -92,6 +93,8 @@ Options and flags: Period of time a failed build won't be triggered again; default: 0days --exit-code-success-if-any-repo-succeeds Whether the Scala Steward process should exit with success (exit code 0) if any repo succeeds; default: false + --coursier-dependencies + Additional coursier dependencies in the format ::, i.e. additional URL handlers. (can be used multiple times) Subcommands: validate-repo-config diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala index 8b0da910a3..cef80db259 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala @@ -301,6 +301,13 @@ object Cli { private val azureReposCfg: Opts[AzureReposCfg] = azureReposOrganization.map(AzureReposCfg.apply) + private val coursierDependencies: Opts[List[String]] = + options[String]( + "coursier-dependencies", + "Additional coursier dependencies in the format ::, " + + "i.e. additional URL handlers. (can be used multiple times)" + ).orEmpty + private val refreshBackoffPeriod: Opts[FiniteDuration] = { val default = 0.days val help = "Period of time a failed build won't be triggered again" + @@ -349,7 +356,8 @@ object Cli { urlCheckerTestUrls, defaultMavenRepo, refreshBackoffPeriod, - exitCodePolicy + exitCodePolicy, + coursierDependencies ).mapN(Config.apply).map(Usage.Regular.apply) private val validateRepoConfig: Opts[Usage] = diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala index 95a080885c..a24a2f313d 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala @@ -64,7 +64,8 @@ final case class Config( urlCheckerTestUrls: Nel[Uri], defaultResolver: Resolver, refreshBackoffPeriod: FiniteDuration, - exitCodePolicy: ExitCodePolicy + exitCodePolicy: ExitCodePolicy, + coursierDependencies: List[String] ) { def forgeSpecificCfg: ForgeSpecificCfg = forgeCfg.tpe match { diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala index 09ed1ac883..333d27ee4a 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala @@ -30,7 +30,7 @@ import org.scalasteward.core.buildtool.mill.MillAlg import org.scalasteward.core.buildtool.sbt.SbtAlg import org.scalasteward.core.buildtool.scalacli.ScalaCliAlg import org.scalasteward.core.client.ClientConfiguration -import org.scalasteward.core.coursier.{CoursierAlg, VersionsCache} +import org.scalasteward.core.coursier.{CoursierAlg, CoursierDependenciesFetchAlg, VersionsCache} import org.scalasteward.core.data.Repo import org.scalasteward.core.edit.EditAlg import org.scalasteward.core.edit.hooks.HookExecutor @@ -172,7 +172,9 @@ object Context { implicit val scalafixCli: ScalafixCli[F] = new ScalafixCli[F] implicit val scalafmtAlg: ScalafmtAlg[F] = new ScalafmtAlg[F](config.defaultResolver) implicit val selfCheckAlg: SelfCheckAlg[F] = new SelfCheckAlg[F](config) - implicit val coursierAlg: CoursierAlg[F] = CoursierAlg.create[F] + implicit val coursierDependenciesFetchAlg: CoursierDependenciesFetchAlg[F] = + CoursierDependenciesFetchAlg.create[F] + implicit val coursierAlg: CoursierAlg[F] = CoursierAlg.create[F](config) implicit val versionsCache: VersionsCache[F] = new VersionsCache[F](config.cacheTtl, versionsStore) implicit val updateAlg: UpdateAlg[F] = new UpdateAlg[F] diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala index 58142abac8..ce4343ee2d 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala @@ -22,6 +22,9 @@ import cats.implicits.* import coursier.cache.{CachePolicy, FileCache} import coursier.core.{Authentication, Project} import coursier.{Fetch, Module, ModuleName, Organization} +import coursier.{Fetch, Info, Module, ModuleName, Organization} +import org.http4s.Uri +import org.scalasteward.core.application.Config import org.scalasteward.core.data.Resolver.Credentials import org.scalasteward.core.data.{Dependency, Resolver, Version} import org.scalasteward.core.util.uri @@ -37,16 +40,29 @@ trait CoursierAlg[F[_]] { } object CoursierAlg { - def create[F[_]](implicit + def create[F[_]](config: Config)(implicit logger: Logger[F], parallel: Parallel[F], - F: Async[F] + F: Async[F], + fetchAlg: CoursierDependenciesFetchAlg[F] ): CoursierAlg[F] = { - val fetch: Fetch[F] = - Fetch[F](FileCache[F]()) + val fetch: F[Fetch[F]] = fetchAlg + .classLoader(config.coursierDependencies) + .map { loader => + Fetch[F]( + FileCache[F]().withClassLoaders(loader :: Nil) + ) + } - val cacheNoTtl: FileCache[F] = - FileCache[F]().withTtl(None).withCachePolicies(List(CachePolicy.Update)) + val cacheNoTtl: F[FileCache[F]] = + fetchAlg + .classLoader(config.coursierDependencies) + .map { loader => + FileCache[F]() + .withTtl(None) + .withCachePolicies(List(CachePolicy.Update)) + .withClassLoaders(loader :: Nil) + } new CoursierAlg[F] { override def getMetadata( @@ -64,38 +80,40 @@ object CoursierAlg { repositories: List[coursier.Repository], acc: DependencyMetadata ): F[DependencyMetadata] = { - val fetchArtifacts = fetch - .withArtifactTypes(Set(coursier.Type.pom, coursier.Type.ivy)) - .withDependencies(List(dependency)) - .withRepositories(repositories) - - fetchArtifacts.ioResult.attempt.flatMap { - case Left(throwable) => - logger.debug(throwable)(s"Failed to fetch artifacts of $dependency").as(acc) - case Right(result) => - val maybeProject = result.resolution.projectCache - .get(dependency.moduleVersion) - .map { case (_, project) => project } - - maybeProject.fold(F.pure(acc)) { project => - val metadata = acc.enrichWith(metadataFrom(project)) - val recurse = Option.when(metadata.repoUrl.isEmpty)(()) + val fetchArtifacts = fetch.map( + _.withArtifactTypes(Set(coursier.Type.pom, coursier.Type.ivy)) + .withDependencies(List(dependency)) + .withRepositories(repositories) + ) + fetchArtifacts.flatMap { + _.ioResult.attempt.flatMap { + case Left(throwable) => + logger.debug(throwable)(s"Failed to fetch artifacts of $dependency").as(acc) + case Right(result) => + val maybeProject = result.resolution.projectCache + .get(dependency.moduleVersion) + .map { case (_, project) => project } + maybeProject.fold(F.pure(acc)) { project => + val metadata = acc.enrichWith(metadataFrom(project)) + val recurse = Option.when(metadata.repoUrl.isEmpty)(()) (recurse >> parentOf(project)).fold(F.pure(metadata)) { parent => - getMetadataImpl(parent, repositories, metadata) + getMetadataImpl(parent, repositories, metadata) + } } - } + } } } override def getVersions(dependency: Dependency, resolver: Resolver): F[List[Version]] = convertResolver(resolver).flatMap { repository => val module = toCoursierModule(dependency) - repository.versions(module, cacheNoTtl.fetch).run.flatMap { - case Left(message) => - logger.debug(message) >> F.raiseError[List[Version]](new Throwable(message)) - case Right((versions, _)) => - F.pure(versions.available.map(Version.apply).sorted) - } + cacheNoTtl.flatMap { cacheNoTtl => + repository.versions(module, cacheNoTtl.fetch).run.flatMap { + case Left(message) => + logger.debug(message) >> F.raiseError[List[Version]](new Throwable(message)) + case Right((versions, _)) => F.pure(versions.available.map(Version.apply).sorted) + } + } } private def convertResolver(resolver: Resolver): F[coursier.Repository] = diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierDependenciesFetchAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierDependenciesFetchAlg.scala new file mode 100644 index 0000000000..c2e51d70c4 --- /dev/null +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierDependenciesFetchAlg.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2018-2022 Scala Steward contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.scalasteward.core.coursier + +import cats.Parallel +import cats.effect._ +import cats.implicits._ +import coursier._ +import coursier.cache.FileCache + +import java.net.URLClassLoader + +/** + * An interface to fetch dependencies for the coursier alg via coursier. + */ +trait CoursierDependenciesFetchAlg[F[_]] { + def classLoader(dependencies: Seq[String]): F[ClassLoader] +} + +object CoursierDependenciesFetchAlg { + def create[F[_]](implicit parallel: Parallel[F], F: Sync[F]): CoursierDependenciesFetchAlg[F] = + (dependencies: Seq[String]) => + Fetch[F](FileCache[F]()) + .withDependencies(dependencies.map { dep => + val Array(org, name, version) = dep.split(':') + Dependency(Module(Organization(org), ModuleName(name)), version) + }) + .io + .map(result => new URLClassLoader(result.map(_.toURI.toURL).toArray)) +} From 266c8e53cbd024585face95d4599570d0b6a6f15 Mon Sep 17 00:00:00 2001 From: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> Date: Mon, 18 Jul 2022 11:14:08 +0200 Subject: [PATCH 2/3] Merge main --- docs/help.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/help.md b/docs/help.md index 8360b20a0a..a4e913e0cf 100644 --- a/docs/help.md +++ b/docs/help.md @@ -95,6 +95,8 @@ Options and flags: Whether the Scala Steward process should exit with success (exit code 0) if any repo succeeds; default: false --coursier-dependencies Additional coursier dependencies in the format ::, i.e. additional URL handlers. (can be used multiple times) + --coursier-dependencies + Additional coursier dependencies in the format ::, i.e. additional URL handlers. (can be used multiple times) Subcommands: validate-repo-config From 9bf37edcf1b8c8a2d5cef28b7720563dd7959f2e Mon Sep 17 00:00:00 2001 From: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:20:16 +0100 Subject: [PATCH 3/3] Coursier alg migration --- .../scalasteward/core/coursier/CoursierAlg.scala | 2 -- .../coursier/CoursierDependenciesFetchAlg.scala | 14 ++++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala index ce4343ee2d..880ff5c221 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala @@ -22,8 +22,6 @@ import cats.implicits.* import coursier.cache.{CachePolicy, FileCache} import coursier.core.{Authentication, Project} import coursier.{Fetch, Module, ModuleName, Organization} -import coursier.{Fetch, Info, Module, ModuleName, Organization} -import org.http4s.Uri import org.scalasteward.core.application.Config import org.scalasteward.core.data.Resolver.Credentials import org.scalasteward.core.data.{Dependency, Resolver, Version} diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierDependenciesFetchAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierDependenciesFetchAlg.scala index c2e51d70c4..3b04d5dca4 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierDependenciesFetchAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierDependenciesFetchAlg.scala @@ -32,13 +32,15 @@ trait CoursierDependenciesFetchAlg[F[_]] { } object CoursierDependenciesFetchAlg { - def create[F[_]](implicit parallel: Parallel[F], F: Sync[F]): CoursierDependenciesFetchAlg[F] = - (dependencies: Seq[String]) => + def create[F[_]](implicit parallel: Parallel[F], F: Async[F]): CoursierDependenciesFetchAlg[F] = + (dependencies: Seq[String]) => { Fetch[F](FileCache[F]()) - .withDependencies(dependencies.map { dep => - val Array(org, name, version) = dep.split(':') - Dependency(Module(Organization(org), ModuleName(name)), version) - }) + .withDependencies( + dependencies.map { dep => + val Array(org, name, version) = dep.split(':') + Dependency(Module(Organization(org), ModuleName(name)), version) + }) .io .map(result => new URLClassLoader(result.map(_.toURI.toURL).toArray)) + } }