Merge branch 'main' into entitlements/security_policy

.buildkite/pipelines/intake.template.yml

@@ -75,7 +75,7 @@ steps:
             ES_VERSION:
               - "9.0.0"
             ES_COMMIT:
-              - "b2cc9d9b8f00ee621f93ddca07ea9c671aab1578" # update to match last commit before lucene bump
+              - "10352e57d85505984582616e1e38530d3ec6ca59" # update to match last commit before lucene bump maintained from combat-lucene-10-0-0 branch
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/intake.yml

@@ -76,7 +76,7 @@ steps:
             ES_VERSION:
               - "9.0.0"
             ES_COMMIT:
-              - "b2cc9d9b8f00ee621f93ddca07ea9c671aab1578" # update to match last commit before lucene bump
+              - "10352e57d85505984582616e1e38530d3ec6ca59" # update to match last commit before lucene bump maintained from combat-lucene-10-0-0 branch
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=296742a352f0b20ec14b143fb684965ad66086c7810b7b255dee216670716175
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.12.1-all.zip
+distributionSha256Sum=fba8464465835e74f7270bbf43d6d8a8d7709ab0a43ce1aa3323f73e9aa0c612
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle

@@ -34,7 +34,7 @@ subprojects {
 }
 
 def calculateBranchVersion() {
-    File f = rootProject.file(".git/refs/heads/origin")
+    File f = layout.settingsDirectory.file(".git/refs/heads/origin").asFile
     def branchName = f.list()?.first().trim()
     return branchName + ".1"
 

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -133,7 +133,7 @@ develocity {
         }
       } else {
         tag 'LOCAL'
-        if (providers.systemProperty('idea.active').present) {
+        if (System.getProperty('idea.active') == 'true') {
           tag 'IDEA'
         }
       }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -39,7 +39,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(JarHellPrecommitPlugin.class);
         project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
         project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
-        boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).isCi();
+        boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
         // Clear default dependencies added by public PluginBuildPlugin as we add our
         // own project dependencies for internal builds
         // TODO remove once we removed default dependencies from PluginBuildPlugin

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:c66fdafe581a6ab1668a962015de4ce4666a60ed601d24f019f03bb4aaab8eeb",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:15a4191ff8ec8305dcba449365e8a1586c9cda8e016ae838d960b9009c6a5cac",
         "-wolfi",
         "apk"
     ),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -161,7 +161,7 @@ public void configureCompile(Project project) {
             compileTask.getConventionMapping().map("sourceCompatibility", () -> java.getSourceCompatibility().toString());
             compileTask.getConventionMapping().map("targetCompatibility", () -> java.getTargetCompatibility().toString());
             compileOptions.getRelease().set(releaseVersionProviderFromCompileTask(project, compileTask));
-            compileOptions.setIncremental(buildParams.isCi() == false);
+            compileOptions.setIncremental(buildParams.getCi() == false);
         });
         // also apply release flag to groovy, which is used in build-tools
         project.getTasks().withType(GroovyCompile.class).configureEach(compileTask -> {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -73,7 +73,7 @@ public void apply(Project project) {
         project.getPlugins().apply(JvmToolchainsPlugin.class);
         toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
         var buildParams = loadBuildParams(project).get();
-        Boolean isCi = buildParams.isCi();
+        Boolean isCi = buildParams.getCi();
         buildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> {
             configureBwcProject(
                 project.project(unreleasedVersion.gradleProjectPath()),
@@ -365,7 +365,7 @@ static void createBuildBwcTask(
             } else {
                 c.getOutputs().files(expectedOutputFile);
             }
-            c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> buildParams.isCi() == false);
+            c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> buildParams.getCi() == false);
             c.getArgs().add("-p");
             c.getArgs().add(projectPath);
             c.getArgs().add(assembleTaskName);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java

@@ -30,7 +30,7 @@ public void apply(Project project) {
         TestClustersPlugin testClustersPlugin = project.getPlugins().apply(TestClustersPlugin.class);
         testClustersPlugin.setRuntimeJava(buildParams.getRuntimeJavaHome());
         testClustersPlugin.setIsReleasedVersion(
-            version -> (version.equals(VersionProperties.getElasticsearchVersion()) && buildParams.isSnapshotBuild() == false)
+            version -> (version.equals(VersionProperties.getElasticsearchVersion()) && buildParams.getSnapshotBuild() == false)
                 || buildParams.getBwcVersions().unreleasedInfo(version) == null
         );
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java

@@ -40,7 +40,7 @@ public void apply(Project project) {
             .getSharedServices()
             .registerIfAbsent(DOCKER_SUPPORT_SERVICE_NAME, DockerSupportService.class, spec -> spec.parameters(params -> {
                 params.setExclusionsFile(new File(project.getRootDir(), DOCKER_ON_LINUX_EXCLUSIONS_FILE));
-                params.getIsCI().set(buildParams.isCi());
+                params.getIsCI().set(buildParams.getCi());
             }));
 
         // Ensure that if we are trying to run any DockerBuildTask tasks, we assert an available Docker installation exists

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java

@@ -54,17 +54,17 @@ public interface BuildParameterExtension {
 
     String getTestSeed();
 
-    Boolean isCi();
+    Boolean getCi();
 
     Integer getDefaultParallel();
 
-    Boolean isSnapshotBuild();
+    Boolean getSnapshotBuild();
 
     BwcVersions getBwcVersions();
 
     Provider<BwcVersions> getBwcVersionsProvider();
 
     Random getRandom();
 
-    Boolean isGraalVmRuntime();
+    Boolean getGraalVmRuntime();
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java

@@ -42,7 +42,7 @@ public abstract class DefaultBuildParameterExtension implements BuildParameterEx
     private final String testSeed;
     private final Boolean isCi;
     private final Integer defaultParallel;
-    private final Boolean isSnapshotBuild;
+    private final Boolean snapshotBuild;
 
     // not final for testing
     private Provider<BwcVersions> bwcVersions;
@@ -81,7 +81,7 @@ public DefaultBuildParameterExtension(
         this.testSeed = testSeed;
         this.isCi = isCi;
         this.defaultParallel = defaultParallel;
-        this.isSnapshotBuild = isSnapshotBuild;
+        this.snapshotBuild = isSnapshotBuild;
         this.bwcVersions = cache(providers, bwcVersions);
         this.gitOrigin = gitOrigin;
     }
@@ -183,7 +183,7 @@ public String getTestSeed() {
     }
 
     @Override
-    public Boolean isCi() {
+    public Boolean getCi() {
         return isCi;
     }
 
@@ -193,8 +193,8 @@ public Integer getDefaultParallel() {
     }
 
     @Override
-    public Boolean isSnapshotBuild() {
-        return isSnapshotBuild;
+    public Boolean getSnapshotBuild() {
+        return snapshotBuild;
     }
 
     @Override
@@ -208,7 +208,7 @@ public Random getRandom() {
     }
 
     @Override
-    public Boolean isGraalVmRuntime() {
+    public Boolean getGraalVmRuntime() {
         return runtimeJavaDetails.get().toLowerCase().contains("graalvm");
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java

@@ -51,7 +51,7 @@ public void apply(Project project) {
                 }
 
                 // Don't fail when all tests are ignored when running in CI
-                filter.setFailOnNoMatchingTests(buildParams.isCi() == false);
+                filter.setFailOnNoMatchingTests(buildParams.getCi() == false);
             });
         });
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java

@@ -33,7 +33,7 @@ public void apply(Project project) {
         var buildParams = loadBuildParams(project).get();
         NamedDomainObjectContainer<TestFixtureDeployment> fixtures = project.container(TestFixtureDeployment.class);
         project.getExtensions().add("dockerFixtures", fixtures);
-        registerDeployTaskPerFixture(project, fixtures, buildParams.isCi());
+        registerDeployTaskPerFixture(project, fixtures, buildParams.getCi());
         project.getTasks().register(DEPLOY_FIXTURE_TASK_NAME, task -> task.dependsOn(project.getTasks().withType(DockerBuildTask.class)));
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java

@@ -131,7 +131,7 @@ public void apply(Project project) {
 
         tasks.withType(ComposeUp.class).named("composeUp").configure(t -> {
             // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions
-            if (buildParams.isCi()) {
+            if (buildParams.getCi()) {
                 t.usesService(dockerComposeThrottle);
                 t.usesService(dockerSupport);
             }

build.gradle

@@ -229,6 +229,22 @@ tasks.register("verifyVersions") {
   }
 }
 
+def generateUpgradeCompatibilityFile = tasks.register("generateUpgradeCompatibilityFile") {
+  def outputFile = project.layout.buildDirectory.file("rolling-upgrade-compatible-${VersionProperties.elasticsearch}.json")
+  def rollingUpgradeCompatibleVersions = buildParams.bwcVersions.wireCompatible - VersionProperties.elasticsearchVersion
+  inputs.property("rollingUpgradeCompatibleVersions", rollingUpgradeCompatibleVersions)
+  outputs.file(outputFile)
+  doLast {
+    def versionsString = rollingUpgradeCompatibleVersions.collect { "\"${it.toString()}\"" }.join(', ')
+    outputFile.get().asFile.write("""{"rolling_upgrade_compatible_versions" : [${versionsString}]}""")
+  }
+}
+
+def upgradeCompatibilityZip = tasks.register("upgradeCompatibilityZip", Zip) {
+  archiveFile.set(project.layout.buildDirectory.file("distributions/rolling-upgrade-compatible-${VersionProperties.elasticsearch}.zip"))
+  from(generateUpgradeCompatibilityFile)
+}
+
 // TODO: This flag existed as a mechanism to disable bwc tests during a backport. It is no
 // longer used for that purpose, but instead a way to run only functional tests. We should
 // rework the functionalTests task to be more explicit about which tasks it wants to run
@@ -483,6 +499,7 @@ tasks.register("buildReleaseArtifacts").configure {
   }
     .collect { GradleUtils.findByName(it.tasks, 'assemble') }
     .findAll { it != null }
+  dependsOn upgradeCompatibilityZip
 }
 
 tasks.register("spotlessApply").configure {

client/rest/build.gradle

@@ -37,7 +37,7 @@ base {
 
 // LLRC is licenses under Apache 2.0
 projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
-licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
+licenseFile.set(layout.getSettingsDirectory().file('licenses/APACHE-LICENSE-2.0.txt').asFile)
 
 dependencies {
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"

client/sniffer/build.gradle

@@ -33,7 +33,7 @@ base {
 
 // rest client sniffer is licenses under Apache 2.0
 projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
-licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
+licenseFile.set(layout.getSettingsDirectory().file('licenses/APACHE-LICENSE-2.0.txt').asFile)
 
 dependencies {
   api project(":client:rest")

client/test/build.gradle

@@ -19,7 +19,7 @@ group = "${group}.client.test"
 
 // rest client sniffer is licenses under Apache 2.0
 projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
-licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
+licenseFile.set(layout.getSettingsDirectory().file('licenses/APACHE-LICENSE-2.0.txt').asFile)
 
 dependencies {
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"

distribution/archives/build.gradle

@@ -57,13 +57,13 @@ CopySpec archiveFiles(String distributionType, String os, String architecture, b
           pluginsDir.getParent()
         }
       }
-      from(rootProject.projectDir) {
+      from(layout.settingsDirectory.asFile) {
         filePermissions {
           unix(0644)
         }
         include 'README.asciidoc'
       }
-      from(rootProject.file('licenses')) {
+      from(layout.settingsDirectory.file('licenses').asFile) {
         include isTestDistro ? 'AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt' : 'ELASTIC-LICENSE-2.0.txt'
         filePermissions {
           unix(0644)

distribution/build.gradle

@@ -377,7 +377,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
             exclude "**/platform/${excludePlatform}/**"
           }
         }
-        if (buildParams.isSnapshotBuild()) {
+        if (buildParams.getSnapshotBuild()) {
           from(buildExternalTestModulesTaskProvider)
         }
         if (project.path.startsWith(':distribution:packages')) {
@@ -518,9 +518,9 @@ subprojects {
 
     String licenseText
     if (isTestDistro) {
-      licenseText = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').getText('UTF-8')
+      licenseText = layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile.getText('UTF-8')
     } else {
-      licenseText = rootProject.file('licenses/ELASTIC-LICENSE-2.0.txt').getText('UTF-8')
+      licenseText = layout.settingsDirectory.file('licenses/ELASTIC-LICENSE-2.0.txt').asFile.getText('UTF-8')
     }
     // license text needs to be indented with a single space
     licenseText = ' ' + licenseText.replace('\n', '\n ')

distribution/docker/build.gradle

@@ -119,7 +119,7 @@ ext.expansions = { Architecture architecture, DockerBase base ->
   // the image. When developing the Docker images, it's very tedious to completely rebuild
   // an image for every single change. Therefore, outside of CI, we fix the
   // build time to midnight so that the Docker build cache is usable.
-  def buildDate = buildParams.isCi() ? buildParams.buildDate : buildParams.buildDate.truncatedTo(ChronoUnit.DAYS).toString()
+  def buildDate = buildParams.ci ? buildParams.buildDate : buildParams.buildDate.truncatedTo(ChronoUnit.DAYS).toString()
 
   return [
     'arch'               : architecture.classifier,
@@ -389,7 +389,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
 
       dockerContext.fileProvider(transformTask.map { Sync task -> task.getDestinationDir() })
 
-      noCache = buildParams.isCi()
+      noCache = buildParams.ci
       tags = generateTags(base, architecture)
       platforms.add(architecture.dockerPlatform)
 
@@ -484,7 +484,7 @@ void addBuildEssDockerImageTask(Architecture architecture) {
 
       dockerContext.fileProvider(buildContextTask.map { it.getDestinationDir() })
 
-      noCache = buildParams.isCi()
+      noCache = buildParams.ci
       baseImages = []
       tags = generateTags(dockerBase, architecture)
       platforms.add(architecture.dockerPlatform)

distribution/packages/build.gradle

@@ -43,7 +43,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.10.1"
+  alias(buildLibs.plugins.ospackage)
 }
 
 ['deb', 'rpm'].each { type ->
@@ -174,7 +174,7 @@ def commonPackageConfig(String type, String architecture) {
     } else {
       assert type == 'rpm'
       into('/usr/share/elasticsearch') {
-        from(rootProject.file('licenses')) {
+        from(layout.settingsDirectory.file('licenses').asFile) {
           include 'ELASTIC-LICENSE-2.0.txt'
           rename { 'LICENSE.txt' }
         }
@@ -300,7 +300,7 @@ ospackage {
   url = 'https://www.elastic.co/'
 
   // signing setup
-  if (project.hasProperty('signing.password') && buildParams.isSnapshotBuild() == false) {
+  if (project.hasProperty('signing.password') && buildParams.snapshotBuild == false) {
     signingKeyId = project.hasProperty('signing.keyId') ? project.property('signing.keyId') : 'D88E42B4'
     signingKeyPassphrase = project.property('signing.password')
     signingKeyRingFile = project.hasProperty('signing.secretKeyRingFile') ?

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java

@@ -109,6 +109,7 @@ private List<String> getJvmArgs() {
             esHome.resolve("lib").toString(),
             // Special circumstances require some modules (not depended on by the main server module) to be explicitly added:
             "--add-modules=jdk.net", // needed to reflectively set extended socket options
+            "--add-modules=jdk.management.agent", // needed by external debug tools to grab thread and heap dumps
             // we control the module path, which may have additional modules not required by server
             "--add-modules=ALL-MODULE-PATH",
             "-m",

docs/README.md

@@ -17,7 +17,7 @@ Docs live in **three places**:
 
 1. **Reference content** lives in this repo. This covers low-level stuff like settings and configuration information that is tightly coupled to code.
 - 👩🏽‍💻 **Engineers** own the bulk of this content.
-2.  **API reference docs** live in the [Elasticsearch specification](https://github.com/elastic/elasticsearch-specification)
+2.  **API reference docs** live in the [Elasticsearch specification](https://github.com/elastic/elasticsearch-specification/blob/main/README.md#how-to-generate-the-openapi-representation)
     - This is where you need to update API docs published in the [new API docs system](https://www.elastic.co/docs/api/doc/elasticsearch/v8/)
 - 👩🏽‍💻 **Engineers** own this content.
 3. **Narrative, overview, and conceptual content** mostly lives in the [`docs-content`](https://github.com/elastic/docs-content/) repo. 
@@ -106,4 +106,4 @@ This allows slightly more expressive testing of the snippets. Since that syntax
 % TEST[s/\n$/\nstartyaml\n  - compare_analyzers: {index: thai_example, first: thai, second: rebuilt_thai}\nendyaml\n/]
 ```
 
-Any place you can use json you can use elements like `$body.path.to.thing` which is replaced on the fly with the contents of the thing at `path.to.thing` in the last response.
+Any place you can use json you can use elements like `$body.path.to.thing` which is replaced on the fly with the contents of the thing at `path.to.thing` in the last response.

docs/build.gradle

@@ -38,7 +38,7 @@ ext.docsFileTree = fileTree(projectDir) {
 
 tasks.named("yamlRestTest") {
   enabled = false
-  if (buildParams.isSnapshotBuild() == false) {
+  if (buildParams.snapshotBuild == false) {
     // LOOKUP is not available in snapshots
     systemProperty 'tests.rest.blacklist', [
       "reference/esql/processing-commands/lookup/esql-lookup-example"
@@ -84,7 +84,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'indices.lifecycle.history_index_enabled', 'false'
     keystorePassword 'keystore-password'
-    if (buildParams.isSnapshotBuild() == false) {
+    if (buildParams.snapshotBuild == false) {
       requiresFeature 'es.failure_store_feature_flag_enabled', new Version(8, 12, 0)
     }
   }