Merge branch 'main' into tsc-stats-bug

Signed-off-by: Peter Alfonsi <[email protected]>

.github/workflows/detect-breaking-change.yml

@@ -17,7 +17,7 @@ jobs:
       with:
         cache-disabled: true
         arguments: japicmp
-        gradle-version: 8.7
+        gradle-version: 8.11
         build-root-directory: server
     - if: failure()
       run: cat server/build/reports/java-compatibility/report.txt

.github/workflows/gradle-check.yml

@@ -115,7 +115,7 @@ jobs:
 
       - name: Upload Coverage Report
         if: success()
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./codeCoverage.xml

CHANGELOG.md

@@ -20,12 +20,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489))
 - Support prefix list for remote repository attributes([#16271](https://github.com/opensearch-project/OpenSearch/pull/16271))
 - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)).
+- Add stats for remote publication failure and move download failure stats to remote methods([#16682](https://github.com/opensearch-project/OpenSearch/pull/16682/))
 
 ### Dependencies
 - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))
 - Bump `google-auth-library-oauth2-http` from 1.7.0 to 1.29.0 in /plugins/repository-gcs ([#16520](https://github.com/opensearch-project/OpenSearch/pull/16520))
 - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521))
-- Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548))
+- Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241105-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548), [#16613](https://github.com/opensearch-project/OpenSearch/pull/16613))
 - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550))
 - Bump `org.apache.xmlbeans:xmlbeans` from 5.2.1 to 5.2.2 ([#16612](https://github.com/opensearch-project/OpenSearch/pull/16612))
@@ -35,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `mockito` from 5.14.1 to 5.14.2, `objenesis` from 3.2 to 3.3 and `bytebuddy` from 1.15.4 to 1.15.10 ([#16655](https://github.com/opensearch-project/OpenSearch/pull/16655))
 - Bump `Netty` from 4.1.114.Final to 4.1.115.Final ([#16661](https://github.com/opensearch-project/OpenSearch/pull/16661))
 - Bump `org.xerial.snappy:snappy-java` from 1.1.10.6 to 1.1.10.7 ([#16665](https://github.com/opensearch-project/OpenSearch/pull/16665))
+- Bump `codecov/codecov-action` from 4 to 5 ([#16667](https://github.com/opensearch-project/OpenSearch/pull/16667))
 
 ### Changed
 
@@ -51,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483))
 - Fix permissions error on scripted query against remote snapshot ([#16544](https://github.com/opensearch-project/OpenSearch/pull/16544))
 - Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628))
+- Fix stale cluster state custom file deletion ([#16670](https://github.com/opensearch-project/OpenSearch/pull/16670))
 - [Tiered Caching] Fix bug in cache stats API ([#16560](https://github.com/opensearch-project/OpenSearch/pull/16560))
 
 ### Security

build.gradle

@@ -162,14 +162,20 @@ Map<String, String> buildMetadataMap = buildMetadataValue.tokenize(';').collectE
   return [key, value]
 }
 
-    /**
-     * Using 'git' command line (if available), tries to fetch the commit date of the current revision
-     * @return commit date of the current revision or 0 if it is not available
-     */
+// See please https://docs.gradle.org/8.11/userguide/service_injection.html#execoperations
+interface InjectedExecOps {
+  @Inject ExecOperations getExecOps()
+}
+
+/**
+ * Using 'git' command line (if available), tries to fetch the commit date of the current revision
+ * @return commit date of the current revision or 0 if it is not available
+ */
 long gitRevisionDate = {
+  def execOps = project.objects.newInstance(InjectedExecOps)
   // Try to get last commit date as Unix timestamp
   try (ByteArrayOutputStream stdout = new ByteArrayOutputStream()) {
-     ExecResult result = project.exec(spec -> {
+     ExecResult result = execOps.execOps.exec(spec -> {
         spec.setIgnoreExitValue(true);
         spec.setStandardOutput(stdout);
         spec.commandLine("git", "log", "-1", "--format=%ct");
@@ -362,7 +368,7 @@ allprojects {
         if ((dep instanceof ProjectDependency) == false) {
           return
         }
-        Project upstreamProject = dep.dependencyProject
+        Project upstreamProject = project.project(dep.path)
         if (upstreamProject == null) {
           return
         }
@@ -438,7 +444,7 @@ gradle.projectsEvaluated {
 
     configurations.matching { it.canBeResolved }.all { Configuration configuration ->
       dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
-        Project upstreamProject = dep.dependencyProject
+        Project upstreamProject = project.project(dep.path)
         if (upstreamProject != null) {
           if (project.path == upstreamProject.path) {
             // TODO: distribution integ tests depend on themselves (!), fix that

buildSrc/src/main/groovy/org/opensearch/gradle/test/TestWithDependenciesPlugin.groovy

@@ -56,9 +56,12 @@ class TestWithDependenciesPlugin implements Plugin<Project> {
 
         project.configurations.testImplementation.dependencies.all { Dependency dep ->
             // this closure is run every time a compile dependency is added
-            if (dep instanceof ProjectDependency && dep.dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) {
-                project.gradle.projectsEvaluated {
-                    addPluginResources(project, dep.dependencyProject)
+            if (dep instanceof ProjectDependency) {
+                Project dependencyProject = project.project(((ProjectDependency)dep).path)
+                if (dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) {
+                    project.gradle.projectsEvaluated {
+                        addPluginResources(project, dependencyProject)
+                    }
                 }
             }
         }

buildSrc/src/main/java/org/opensearch/gradle/LoggedExec.java

@@ -71,6 +71,11 @@ public class LoggedExec extends Exec implements FileSystemOperationsAware {
     private Consumer<Logger> outputLogger;
     private FileSystemOperations fileSystemOperations;
 
+    interface InjectedExecOps {
+        @Inject
+        ExecOperations getExecOps();
+    }
+
     @Inject
     public LoggedExec(FileSystemOperations fileSystemOperations) {
         this.fileSystemOperations = fileSystemOperations;
@@ -133,15 +138,17 @@ public void setSpoolOutput(boolean spoolOutput) {
     }
 
     public static ExecResult exec(Project project, Action<ExecSpec> action) {
-        return genericExec(project::exec, action);
+        final InjectedExecOps execOps = project.getObjects().newInstance(InjectedExecOps.class);
+        return exec(execOps.getExecOps(), action);
     }
 
     public static ExecResult exec(ExecOperations execOperations, Action<ExecSpec> action) {
         return genericExec(execOperations::exec, action);
     }
 
     public static ExecResult javaexec(Project project, Action<JavaExecSpec> action) {
-        return genericExec(project::javaexec, action);
+        final InjectedExecOps execOps = project.getObjects().newInstance(InjectedExecOps.class);
+        return genericExec(execOps.getExecOps()::javaexec, action);
     }
 
     /** Returns JVM arguments suitable for a short-lived forked task */

buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java

@@ -121,7 +121,7 @@ public String call() throws Exception {
                         Node dependencyNode = dependenciesNode.appendNode("dependency");
                         dependencyNode.appendNode("groupId", dependency.getGroup());
                         ProjectDependency projectDependency = (ProjectDependency) dependency;
-                        String artifactId = getArchivesBaseName(projectDependency.getDependencyProject());
+                        String artifactId = getArchivesBaseName(project.project(projectDependency.getPath()));
                         dependencyNode.appendNode("artifactId", artifactId);
                         dependencyNode.appendNode("version", dependency.getVersion());
                         dependencyNode.appendNode("scope", "compile");

buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -60,8 +60,11 @@
 import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.TaskAction;
+import org.gradle.process.ExecOperations;
 import org.gradle.process.ExecResult;
 
+import javax.inject.Inject;
+
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
@@ -108,6 +111,11 @@ public class ThirdPartyAuditTask extends DefaultTask {
 
     public boolean jarHellEnabled = true;
 
+    interface InjectedExecOps {
+        @Inject
+        ExecOperations getExecOps();
+    }
+
     @Input
     public Property<JavaVersion> getTargetCompatibility() {
         return targetCompatibility;
@@ -357,7 +365,8 @@ private String formatClassList(Set<String> classList) {
 
     private String runForbiddenAPIsCli() throws IOException {
         ByteArrayOutputStream errorOut = new ByteArrayOutputStream();
-        ExecResult result = getProject().javaexec(spec -> {
+        InjectedExecOps execOps = getProject().getObjects().newInstance(InjectedExecOps.class);
+        ExecResult result = execOps.getExecOps().javaexec(spec -> {
             if (javaHome != null) {
                 spec.setExecutable(javaHome + "/bin/java");
             }
@@ -391,7 +400,8 @@ private String runForbiddenAPIsCli() throws IOException {
 
     private Set<String> runJdkJarHellCheck() throws IOException {
         ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
-        ExecResult execResult = getProject().javaexec(spec -> {
+        InjectedExecOps execOps = getProject().getObjects().newInstance(InjectedExecOps.class);
+        ExecResult execResult = execOps.getExecOps().javaexec(spec -> {
             spec.classpath(
                 jdkJarHellClasspath,
                 getRuntimeConfiguration(),

gradle/missing-javadoc.gradle

@@ -8,6 +8,7 @@
 
 
 import javax.annotation.Nullable
+import javax.inject.Inject
 import org.gradle.api.tasks.PathSensitive;
 import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.internal.jvm.Jvm
@@ -227,6 +228,11 @@ class MissingJavadocTask extends DefaultTask {
   @PathSensitive(PathSensitivity.RELATIVE)
   def taskResources
 
+  // See please https://docs.gradle.org/8.11/userguide/service_injection.html#execoperations
+  interface InjectedExecOps {
+    @Inject ExecOperations getExecOps()
+  }
+
   /** Utility method to recursively collect all tasks with same name like this one that we depend on */
   private Set findRenderTasksInDependencies() {
     Set found = []
@@ -317,11 +323,12 @@ class MissingJavadocTask extends DefaultTask {
       }
     }()
 
+    def execOps = project.objects.newInstance(InjectedExecOps)
     def outputFile = project.file("${getTemporaryDir()}/javadoc-output.txt")
     def result
 
     outputFile.withOutputStream { output ->
-      result = project.exec {
+      result = execOps.execOps.exec {
         executable javadocCmd
 
         // we want to capture both stdout and stderr to the same

gradle/wrapper/gradle-wrapper.properties

@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.11-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6
+distributionSha256Sum=73d2d553933194d8eefed0a291acbe45392ca3572ba13834cbbf373da375276d

libs/build.gradle

@@ -42,7 +42,7 @@ subprojects {
   project.afterEvaluate {
     configurations.all { Configuration conf ->
       dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
-        Project depProject = dep.dependencyProject
+        Project depProject = project.project(dep.path)
         if (depProject != null
           && (false == depProject.path.equals(':libs:opensearch-core') &&
               false == depProject.path.equals(':libs:opensearch-common'))

modules/lang-painless/build.gradle

@@ -115,11 +115,17 @@ testClusters {
   }
 }
 
+interface InjectedExecOps {
+  @Inject ExecOperations getExecOps()
+}
+
+
 tasks.register("generateContextDoc", DefaultTestClustersTask) {
   dependsOn sourceSets.doc.runtimeClasspath
   useCluster testClusters.generateContextCluster
   doFirst {
-    project.javaexec {
+    def execOps = project.objects.newInstance(InjectedExecOps)
+    execOps.execOps.javaexec {
       mainClass = 'org.opensearch.painless.ContextDocGenerator'
       classpath = sourceSets.doc.runtimeClasspath
       systemProperty "cluster.uri", "${-> testClusters.generateContextCluster.singleNode().getAllHttpSocketURI().get(0)}"

plugins/discovery-gce/build.gradle

@@ -18,7 +18,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api "com.google.apis:google-api-services-compute:v1-rev20241021-2.0.0"
+  api "com.google.apis:google-api-services-compute:v1-rev20241105-2.0.0"
   api "com.google.api-client:google-api-client:1.35.2"
   api "com.google.oauth-client:google-oauth-client:1.36.0"
   api "com.google.http-client:google-http-client:${versions.google_http_client}"

plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241105-2.0.0.jar.sha1

@@ -0,0 +1 @@
+9a82357e628b39e189c575094b384b442bff4f8e

server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java

@@ -90,7 +90,7 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING;
-import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING;
+import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING;
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
 import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram;
 import static org.opensearch.search.aggregations.AggregationBuilders.dateRange;
@@ -582,21 +582,33 @@ public void testCanCache() throws Exception {
         assertThat(r4.getHits().getTotalHits().value, equalTo(7L));
         assertCacheState(client, index, 0, 4);
 
-        // If size > 0 we should cache if this is enabled via cluster setting
+        // Update max cacheable size for request cache from default value of 0
         ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        int maxCacheableSize = 5;
         updateSettingsRequest.persistentSettings(
-            Settings.builder().put(INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING.getKey(), true)
+            Settings.builder().put(INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING.getKey(), maxCacheableSize)
         );
         assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
 
+        // Sizes <= the cluster setting value should be cached
         final SearchResponse r7 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
-            .setSize(1)
+            .setSize(maxCacheableSize)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-22").lte("2016-03-26"))
             .get();
         OpenSearchAssertions.assertAllSuccessful(r7);
         assertThat(r7.getHits().getTotalHits().value, equalTo(5L));
         assertCacheState(client, index, 0, 6);
+
+        // Sizes > the cluster setting value should not be cached
+        final SearchResponse r8 = client.prepareSearch(index)
+            .setSearchType(SearchType.QUERY_THEN_FETCH)
+            .setSize(maxCacheableSize + 1)
+            .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-22").lte("2016-03-26"))
+            .get();
+        OpenSearchAssertions.assertAllSuccessful(r8);
+        assertThat(r8.getHits().getTotalHits().value, equalTo(5L));
+        assertCacheState(client, index, 0, 6);
     }
 
     public void testCacheWithFilteredAlias() throws InterruptedException {

server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java

@@ -298,9 +298,9 @@ PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest
             }
         } catch (Exception e) {
             if (applyFullState) {
-                remoteClusterStateService.fullDownloadFailed();
+                remoteClusterStateService.fullIncomingPublicationFailed();
             } else {
-                remoteClusterStateService.diffDownloadFailed();
+                remoteClusterStateService.diffIncomingPublicationFailed();
             }
             throw e;
         }

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

@@ -519,7 +519,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 IndicesRequestCache.INDICES_CACHE_QUERY_EXPIRE,
                 IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING,
                 IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING,
-                IndicesRequestCache.INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING,
+                IndicesRequestCache.INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING,
                 HunspellService.HUNSPELL_LAZY_LOAD,
                 HunspellService.HUNSPELL_IGNORE_CASE,
                 HunspellService.HUNSPELL_DICTIONARY_OPTIONS,

server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java

@@ -306,7 +306,7 @@ void deleteClusterMetadata(
                     staleEphemeralAttributePaths.add(clusterMetadataManifest.getHashesOfConsistentSettings().getUploadedFilename());
                 }
                 if (clusterMetadataManifest.getClusterStateCustomMap() != null) {
-                    clusterMetadataManifest.getCustomMetadataMap()
+                    clusterMetadataManifest.getClusterStateCustomMap()
                         .values()
                         .stream()
                         .filter(u -> !filesToKeep.contains(u.getUploadedFilename()))